You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by om...@apache.org on 2017/03/28 22:31:29 UTC
[01/50] [abbrv] hive git commit: HIVE-16023: Wrong estimation for
number of rows generated by IN expression (Jesus Camacho Rodriguez,
reviewed by Ashutosh Chauhan) [Forced Update!]
Repository: hive
Updated Branches:
refs/heads/branch-2.2 ab3af5b02 -> f31f7495e (forced update)
HIVE-16023: Wrong estimation for number of rows generated by IN expression (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
Change-Id: I81ab162ba9df684d1c0c8b9f195779144b3d465a
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ceea54fd
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ceea54fd
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ceea54fd
Branch: refs/heads/branch-2.2
Commit: ceea54fdf9bf646f1f782071f042ab24350e2540
Parents: 9e7daaf
Author: Jesus Camacho Rodriguez <jc...@apache.org>
Authored: Mon Feb 27 15:22:23 2017 +0000
Committer: Owen O'Malley <om...@apache.org>
Committed: Tue Mar 28 15:27:49 2017 -0700
----------------------------------------------------------------------
.../stats/annotation/StatsRulesProcFactory.java | 46 +++++++++++++-------
.../clientpositive/remove_exprs_stats.q.out | 18 ++++----
2 files changed, 39 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/ceea54fd/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
index 1000dc9..faf26c5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
@@ -287,8 +287,13 @@ public class StatsRulesProcFactory {
long newNumRows = 0;
Statistics andStats = null;
- if (stats.getNumRows() <= 1 || stats.getDataSize() <= 0)
+ if (stats.getNumRows() <= 1 || stats.getDataSize() <= 0) {
+ if (isDebugEnabled) {
+ LOG.debug("Estimating row count for " + pred + " Original num rows: " + stats.getNumRows() +
+ " Original data size: " + stats.getDataSize() + " New num rows: 1");
+ }
return 1;
+ }
if (pred instanceof ExprNodeGenericFuncDesc) {
ExprNodeGenericFuncDesc genFunc = (ExprNodeGenericFuncDesc) pred;
@@ -345,23 +350,31 @@ public class StatsRulesProcFactory {
if (colType.equalsIgnoreCase(serdeConstants.BOOLEAN_TYPE_NAME)) {
ColStatistics cs = stats.getColumnStatisticsFromColName(colName);
if (cs != null) {
- return cs.getNumTrues();
+ newNumRows = cs.getNumTrues();
+ } else {
+ // default
+ newNumRows = stats.getNumRows() / 2;
}
+ } else {
+ // if not boolean column return half the number of rows
+ newNumRows = stats.getNumRows() / 2;
}
-
- // if not boolean column return half the number of rows
- return stats.getNumRows() / 2;
} else if (pred instanceof ExprNodeConstantDesc) {
// special case for handling false constants
ExprNodeConstantDesc encd = (ExprNodeConstantDesc) pred;
if (Boolean.FALSE.equals(encd.getValue())) {
- return 0;
+ newNumRows = 0;
} else {
- return stats.getNumRows();
+ newNumRows = stats.getNumRows();
}
}
+ if (isDebugEnabled) {
+ LOG.debug("Estimating row count for " + pred + " Original num rows: " + stats.getNumRows() +
+ " New num rows: " + newNumRows);
+ }
+
return newNumRows;
}
@@ -443,15 +456,16 @@ public class StatsRulesProcFactory {
}
// 3. Calculate IN selectivity
- float factor = 1;
+ double factor = 1d;
for (int i = 0; i < columnStats.size(); i++) {
long dvs = columnStats.get(i) == null ? 0 : columnStats.get(i).getCountDistint();
- // ( num of distinct vals for col / num of rows ) * num of distinct vals for col in IN clause
- float columnFactor = dvs == 0 ? 0.5f : ((float)dvs / numRows) * values.get(i).size();
- factor *= columnFactor;
+ // (num of distinct vals for col in IN clause / num of distinct vals for col )
+ double columnFactor = dvs == 0 ? 0.5d : ((double) values.get(i).size() / dvs);
+ // max can be 1, even when ndv is larger in IN clause than in column stats
+ factor *= columnFactor > 1d ? 1d : columnFactor;
}
float inFactor = HiveConf.getFloatVar(aspCtx.getConf(), HiveConf.ConfVars.HIVE_STATS_IN_CLAUSE_FACTOR);
- return Math.round( (double)numRows * factor * inFactor);
+ return Math.round( (double) numRows * factor * inFactor);
}
private long evaluateBetweenExpr(Statistics stats, ExprNodeDesc pred, AnnotateStatsProcCtx aspCtx,
@@ -1790,11 +1804,11 @@ public class StatsRulesProcFactory {
Map<Integer, Long> rowCountParents) {
if (newNumRows < 0) {
- LOG.info("STATS-" + jop.toString() + ": Overflow in number of rows."
+ LOG.debug("STATS-" + jop.toString() + ": Overflow in number of rows. "
+ newNumRows + " rows will be set to Long.MAX_VALUE");
}
if (newNumRows == 0) {
- LOG.info("STATS-" + jop.toString() + ": Equals 0 in number of rows."
+ LOG.debug("STATS-" + jop.toString() + ": Equals 0 in number of rows. "
+ newNumRows + " rows will be set to 1");
newNumRows = 1;
}
@@ -2214,12 +2228,12 @@ public class StatsRulesProcFactory {
boolean updateNDV) {
if (newNumRows < 0) {
- LOG.info("STATS-" + op.toString() + ": Overflow in number of rows."
+ LOG.debug("STATS-" + op.toString() + ": Overflow in number of rows. "
+ newNumRows + " rows will be set to Long.MAX_VALUE");
newNumRows = StatsUtils.getMaxIfOverflow(newNumRows);
}
if (newNumRows == 0) {
- LOG.info("STATS-" + op.toString() + ": Equals 0 in number of rows."
+ LOG.debug("STATS-" + op.toString() + ": Equals 0 in number of rows. "
+ newNumRows + " rows will be set to 1");
newNumRows = 1;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/ceea54fd/ql/src/test/results/clientpositive/remove_exprs_stats.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/remove_exprs_stats.q.out b/ql/src/test/results/clientpositive/remove_exprs_stats.q.out
index 15e2e29..b1ba942 100644
--- a/ql/src/test/results/clientpositive/remove_exprs_stats.q.out
+++ b/ql/src/test/results/clientpositive/remove_exprs_stats.q.out
@@ -449,14 +449,14 @@ STAGE PLANS:
Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: (locid) IN (5) (type: boolean)
- Statistics: Num rows: 5 Data size: 498 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: int)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 5 Data size: 498 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 5 Data size: 498 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -485,14 +485,14 @@ STAGE PLANS:
Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: (locid) IN (5, 2, 3) (type: boolean)
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 5 Data size: 498 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: int)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 5 Data size: 498 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 5 Data size: 498 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -521,14 +521,14 @@ STAGE PLANS:
Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: (locid) IN (1, 6) (type: boolean)
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 3 Data size: 306 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: int)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 3 Data size: 306 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 3 Data size: 306 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
[05/50] [abbrv] hive git commit: HIVE-14959: Fix DISTINCT with
windowing when CBO is enabled/disabled (Jesus Camacho Rodriguez,
reviewed by Ashutosh Chauhan)
Posted by om...@apache.org.
HIVE-14959: Fix DISTINCT with windowing when CBO is enabled/disabled (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
(cherry-picked 36e810fa6fd1b353ee1d9907927aa472ae53dd48)
Change-Id: I2862a55926c8f2b0ea6c62894c2ce4a5511b45ff
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/9e7daaf4
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/9e7daaf4
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/9e7daaf4
Branch: refs/heads/branch-2.2
Commit: 9e7daaf4f2b888b3fd637dc8f99cde76ad97c507
Parents: 43ab7ef
Author: Jesus Camacho Rodriguez <jc...@apache.org>
Authored: Thu Oct 13 22:04:37 2016 +0100
Committer: Owen O'Malley <om...@apache.org>
Committed: Tue Mar 28 15:27:49 2017 -0700
----------------------------------------------------------------------
.../llap/cbo_rp_windowing_2.q.out | 2247 +++++++++++++++++
.../results/clientpositive/llap/windowing.q.out | 2348 ++++++++++++++++++
.../test/results/clientpositive/windowing.q.out | 105 +-
3 files changed, 4698 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
[31/50] [abbrv] hive git commit: HIVE-15084: Flaky test:
TestMiniTezCliDriver:explainanalyze_1, 2, 3, 4, 5 (Pengcheng Xiong)
Posted by om...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/9f5bea3b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java.orig
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java.orig b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java.orig
deleted file mode 100644
index 699bb11..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java.orig
+++ /dev/null
@@ -1,13038 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.parse;
-
-import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.HIVESTATSDBCLASS;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.Serializable;
-import java.security.AccessControlException;
-import java.util.ArrayDeque;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Deque;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedHashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Queue;
-import java.util.Set;
-import java.util.TreeSet;
-import java.util.UUID;
-import java.util.regex.Pattern;
-import java.util.regex.PatternSyntaxException;
-
-import org.antlr.runtime.ClassicToken;
-import org.antlr.runtime.CommonToken;
-import org.antlr.runtime.Token;
-import org.antlr.runtime.tree.Tree;
-import org.antlr.runtime.tree.TreeWizard;
-import org.antlr.runtime.tree.TreeWizard.ContextVisitor;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsAction;
-import org.apache.hadoop.hive.common.BlobStorageUtils;
-import org.apache.hadoop.hive.ql.plan.AlterTableDesc;
-import org.apache.hadoop.hive.common.FileUtils;
-import org.apache.hadoop.hive.common.ObjectPair;
-import org.apache.hadoop.hive.common.StatsSetupConst;
-import org.apache.hadoop.hive.common.StatsSetupConst.StatDB;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.conf.HiveConf.StrictChecks;
-import org.apache.hadoop.hive.metastore.MetaStoreUtils;
-import org.apache.hadoop.hive.metastore.TableType;
-import org.apache.hadoop.hive.metastore.Warehouse;
-import org.apache.hadoop.hive.metastore.api.Database;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.Order;
-import org.apache.hadoop.hive.metastore.api.SQLForeignKey;
-import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey;
-import org.apache.hadoop.hive.ql.CompilationOpContext;
-import org.apache.hadoop.hive.ql.ErrorMsg;
-import org.apache.hadoop.hive.ql.QueryProperties;
-import org.apache.hadoop.hive.ql.QueryState;
-import org.apache.hadoop.hive.ql.exec.AbstractMapJoinOperator;
-import org.apache.hadoop.hive.ql.exec.ArchiveUtils;
-import org.apache.hadoop.hive.ql.exec.ColumnInfo;
-import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluatorFactory;
-import org.apache.hadoop.hive.ql.exec.FetchTask;
-import org.apache.hadoop.hive.ql.exec.FileSinkOperator;
-import org.apache.hadoop.hive.ql.exec.FilterOperator;
-import org.apache.hadoop.hive.ql.exec.FunctionInfo;
-import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
-import org.apache.hadoop.hive.ql.exec.GroupByOperator;
-import org.apache.hadoop.hive.ql.exec.JoinOperator;
-import org.apache.hadoop.hive.ql.exec.Operator;
-import org.apache.hadoop.hive.ql.exec.OperatorFactory;
-import org.apache.hadoop.hive.ql.exec.RecordReader;
-import org.apache.hadoop.hive.ql.exec.RecordWriter;
-import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator;
-import org.apache.hadoop.hive.ql.exec.RowSchema;
-import org.apache.hadoop.hive.ql.exec.SMBMapJoinOperator;
-import org.apache.hadoop.hive.ql.exec.SelectOperator;
-import org.apache.hadoop.hive.ql.exec.TableScanOperator;
-import org.apache.hadoop.hive.ql.exec.Task;
-import org.apache.hadoop.hive.ql.exec.TaskFactory;
-import org.apache.hadoop.hive.ql.exec.UnionOperator;
-import org.apache.hadoop.hive.ql.exec.Utilities;
-import org.apache.hadoop.hive.ql.hooks.ReadEntity;
-import org.apache.hadoop.hive.ql.hooks.WriteEntity;
-import org.apache.hadoop.hive.ql.io.AcidOutputFormat;
-import org.apache.hadoop.hive.ql.io.AcidUtils;
-import org.apache.hadoop.hive.ql.io.AcidUtils.Operation;
-import org.apache.hadoop.hive.ql.io.CombineHiveInputFormat;
-import org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat;
-import org.apache.hadoop.hive.ql.io.HiveOutputFormat;
-import org.apache.hadoop.hive.ql.io.NullRowsInputFormat;
-import org.apache.hadoop.hive.ql.io.RCFileInputFormat;
-import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat;
-import org.apache.hadoop.hive.ql.lib.DefaultGraphWalker;
-import org.apache.hadoop.hive.ql.lib.Dispatcher;
-import org.apache.hadoop.hive.ql.lib.GraphWalker;
-import org.apache.hadoop.hive.ql.lib.Node;
-import org.apache.hadoop.hive.ql.metadata.DummyPartition;
-import org.apache.hadoop.hive.ql.metadata.Hive;
-import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.ql.metadata.HiveUtils;
-import org.apache.hadoop.hive.ql.metadata.InvalidTableException;
-import org.apache.hadoop.hive.ql.metadata.Partition;
-import org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient;
-import org.apache.hadoop.hive.ql.metadata.Table;
-import org.apache.hadoop.hive.ql.metadata.VirtualColumn;
-import org.apache.hadoop.hive.ql.optimizer.Optimizer;
-import org.apache.hadoop.hive.ql.optimizer.Transform;
-import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException;
-import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException.UnsupportedFeature;
-import org.apache.hadoop.hive.ql.optimizer.calcite.translator.HiveOpConverterPostProc;
-import org.apache.hadoop.hive.ql.optimizer.lineage.Generator;
-import org.apache.hadoop.hive.ql.optimizer.unionproc.UnionProcContext;
-import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.TableSpec.SpecType;
-import org.apache.hadoop.hive.ql.parse.CalcitePlanner.ASTSearcher;
-import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.OrderExpression;
-import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.OrderSpec;
-import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.PTFInputSpec;
-import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.PTFQueryInputSpec;
-import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.PTFQueryInputType;
-import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.PartitionExpression;
-import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.PartitionSpec;
-import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.PartitionedTableFunctionSpec;
-import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.PartitioningSpec;
-import org.apache.hadoop.hive.ql.parse.QBSubQuery.SubQueryType;
-import org.apache.hadoop.hive.ql.parse.SubQueryUtils.ISubQueryJoinInfo;
-import org.apache.hadoop.hive.ql.parse.WindowingSpec.BoundarySpec;
-import org.apache.hadoop.hive.ql.parse.WindowingSpec.CurrentRowSpec;
-import org.apache.hadoop.hive.ql.parse.WindowingSpec.Direction;
-import org.apache.hadoop.hive.ql.parse.WindowingSpec.RangeBoundarySpec;
-import org.apache.hadoop.hive.ql.parse.WindowingSpec.ValueBoundarySpec;
-import org.apache.hadoop.hive.ql.parse.WindowingSpec.WindowExpressionSpec;
-import org.apache.hadoop.hive.ql.parse.WindowingSpec.WindowFrameSpec;
-import org.apache.hadoop.hive.ql.parse.WindowingSpec.WindowFunctionSpec;
-import org.apache.hadoop.hive.ql.parse.WindowingSpec.WindowSpec;
-import org.apache.hadoop.hive.ql.plan.AggregationDesc;
-import org.apache.hadoop.hive.ql.plan.AlterTableDesc.AlterTableTypes;
-import org.apache.hadoop.hive.ql.plan.CreateTableDesc;
-import org.apache.hadoop.hive.ql.plan.CreateTableLikeDesc;
-import org.apache.hadoop.hive.ql.plan.CreateViewDesc;
-import org.apache.hadoop.hive.ql.plan.DDLWork;
-import org.apache.hadoop.hive.ql.plan.DynamicPartitionCtx;
-import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
-import org.apache.hadoop.hive.ql.plan.ExprNodeColumnListDesc;
-import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
-import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
-import org.apache.hadoop.hive.ql.plan.ExprNodeDescUtils;
-import org.apache.hadoop.hive.ql.plan.ExprNodeFieldDesc;
-import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
-import org.apache.hadoop.hive.ql.plan.FileSinkDesc;
-import org.apache.hadoop.hive.ql.plan.FilterDesc;
-import org.apache.hadoop.hive.ql.plan.FilterDesc.SampleDesc;
-import org.apache.hadoop.hive.ql.plan.ForwardDesc;
-import org.apache.hadoop.hive.ql.plan.GroupByDesc;
-import org.apache.hadoop.hive.ql.plan.HiveOperation;
-import org.apache.hadoop.hive.ql.plan.JoinCondDesc;
-import org.apache.hadoop.hive.ql.plan.JoinDesc;
-import org.apache.hadoop.hive.ql.plan.LateralViewForwardDesc;
-import org.apache.hadoop.hive.ql.plan.LateralViewJoinDesc;
-import org.apache.hadoop.hive.ql.plan.LimitDesc;
-import org.apache.hadoop.hive.ql.plan.ListBucketingCtx;
-import org.apache.hadoop.hive.ql.plan.LoadFileDesc;
-import org.apache.hadoop.hive.ql.plan.LoadTableDesc;
-import org.apache.hadoop.hive.ql.plan.MapJoinDesc;
-import org.apache.hadoop.hive.ql.plan.OperatorDesc;
-import org.apache.hadoop.hive.ql.plan.PTFDesc;
-import org.apache.hadoop.hive.ql.plan.PlanUtils;
-import org.apache.hadoop.hive.ql.plan.ReduceSinkDesc;
-import org.apache.hadoop.hive.ql.plan.ScriptDesc;
-import org.apache.hadoop.hive.ql.plan.SelectDesc;
-import org.apache.hadoop.hive.ql.plan.TableDesc;
-import org.apache.hadoop.hive.ql.plan.TableScanDesc;
-import org.apache.hadoop.hive.ql.plan.UDTFDesc;
-import org.apache.hadoop.hive.ql.plan.UnionDesc;
-import org.apache.hadoop.hive.ql.plan.ptf.OrderExpressionDef;
-import org.apache.hadoop.hive.ql.plan.ptf.PTFExpressionDef;
-import org.apache.hadoop.hive.ql.plan.ptf.PartitionedTableFunctionDef;
-import org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrivilegeObject;
-import org.apache.hadoop.hive.ql.session.SessionState;
-import org.apache.hadoop.hive.ql.session.SessionState.ResourceType;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.Mode;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDFHash;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPOr;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDTF;
-import org.apache.hadoop.hive.ql.util.ResourceDownloader;
-import org.apache.hadoop.hive.serde.serdeConstants;
-import org.apache.hadoop.hive.serde2.Deserializer;
-import org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe;
-import org.apache.hadoop.hive.serde2.NoOpFetchFormatter;
-import org.apache.hadoop.hive.serde2.NullStructSerDe;
-import org.apache.hadoop.hive.serde2.SerDeException;
-import org.apache.hadoop.hive.serde2.SerDeUtils;
-import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe;
-import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
-import org.apache.hadoop.hive.serde2.objectinspector.StandardStructObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.StructField;
-import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
-import org.apache.hadoop.hive.serde2.thrift.ThriftJDBCBinarySerDe;
-import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
-import org.apache.hadoop.hive.shims.HadoopShims;
-import org.apache.hadoop.hive.shims.Utils;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.InputFormat;
-import org.apache.hadoop.mapred.OutputFormat;
-import org.apache.hadoop.security.UserGroupInformation;
-
-import com.google.common.collect.Sets;
-
-/**
- * Implementation of the semantic analyzer. It generates the query plan.
- * There are other specific semantic analyzers for some hive operations such as
- * DDLSemanticAnalyzer for ddl operations.
- */
-
-public class SemanticAnalyzer extends BaseSemanticAnalyzer {
-
- public static final String DUMMY_DATABASE = "_dummy_database";
- public static final String DUMMY_TABLE = "_dummy_table";
- public static final String SUBQUERY_TAG_1 = "-subquery1";
- public static final String SUBQUERY_TAG_2 = "-subquery2";
-
- // Max characters when auto generating the column name with func name
- private static final int AUTOGEN_COLALIAS_PRFX_MAXLENGTH = 20;
-
- private static final String VALUES_TMP_TABLE_NAME_PREFIX = "Values__Tmp__Table__";
-
- static final String MATERIALIZATION_MARKER = "$MATERIALIZATION";
-
- private HashMap<TableScanOperator, ExprNodeDesc> opToPartPruner;
- private HashMap<TableScanOperator, PrunedPartitionList> opToPartList;
- protected HashMap<String, TableScanOperator> topOps;
- protected LinkedHashMap<Operator<? extends OperatorDesc>, OpParseContext> opParseCtx;
- private List<LoadTableDesc> loadTableWork;
- private List<LoadFileDesc> loadFileWork;
- private List<ColumnStatsAutoGatherContext> columnStatsAutoGatherContexts;
- private final Map<JoinOperator, QBJoinTree> joinContext;
- private final Map<SMBMapJoinOperator, QBJoinTree> smbMapJoinContext;
- private final HashMap<TableScanOperator, Table> topToTable;
- private final Map<FileSinkOperator, Table> fsopToTable;
- private final List<ReduceSinkOperator> reduceSinkOperatorsAddedByEnforceBucketingSorting;
- private final HashMap<TableScanOperator, Map<String, String>> topToTableProps;
- private QB qb;
- private ASTNode ast;
- private int destTableId;
- private UnionProcContext uCtx;
- List<AbstractMapJoinOperator<? extends MapJoinDesc>> listMapJoinOpsNoReducer;
- private HashMap<TableScanOperator, SampleDesc> opToSamplePruner;
- private final Map<TableScanOperator, Map<String, ExprNodeDesc>> opToPartToSkewedPruner;
- private Map<SelectOperator, Table> viewProjectToTableSchema;
- /**
- * a map for the split sampling, from alias to an instance of SplitSample
- * that describes percentage and number.
- */
- private final HashMap<String, SplitSample> nameToSplitSample;
- Map<GroupByOperator, Set<String>> groupOpToInputTables;
- Map<String, PrunedPartitionList> prunedPartitions;
- protected List<FieldSchema> resultSchema;
- private CreateViewDesc createVwDesc;
- private ArrayList<String> viewsExpanded;
- private ASTNode viewSelect;
- protected final UnparseTranslator unparseTranslator;
- private final GlobalLimitCtx globalLimitCtx;
-
- // prefix for column names auto generated by hive
- private final String autogenColAliasPrfxLbl;
- private final boolean autogenColAliasPrfxIncludeFuncName;
-
- // Keep track of view alias to read entity corresponding to the view
- // For eg: for a query like 'select * from V3', where V3 -> V2, V2 -> V1, V1 -> T
- // keeps track of aliases for V3, V3:V2, V3:V2:V1.
- // This is used when T is added as an input for the query, the parents of T is
- // derived from the alias V3:V2:V1:T
- private final Map<String, ReadEntity> viewAliasToInput;
-
- //need merge isDirect flag to input even if the newInput does not have a parent
- private boolean mergeIsDirect;
-
- // flag for no scan during analyze ... compute statistics
- protected boolean noscan;
-
- //flag for partial scan during analyze ... compute statistics
- protected boolean partialscan;
-
- protected volatile boolean disableJoinMerge = false;
-
- /*
- * Capture the CTE definitions in a Query.
- */
- final Map<String, CTEClause> aliasToCTEs;
-
- /*
- * Used to check recursive CTE invocations. Similar to viewsExpanded
- */
- ArrayList<String> ctesExpanded;
-
- /*
- * Whether root tasks after materialized CTE linkage have been resolved
- */
- boolean rootTasksResolved;
-
- protected TableMask tableMask;
-
- CreateTableDesc tableDesc;
-
- /** Not thread-safe. */
- final ASTSearcher astSearcher = new ASTSearcher();
-
- protected AnalyzeRewriteContext analyzeRewrite;
-
- // A mapping from a tableName to a table object in metastore.
- Map<String, Table> tabNameToTabObject;
-
- // The tokens we should ignore when we are trying to do table masking.
- private final Set<Integer> ignoredTokens = Sets.newHashSet(HiveParser.TOK_GROUPBY,
- HiveParser.TOK_ORDERBY, HiveParser.TOK_WINDOWSPEC, HiveParser.TOK_CLUSTERBY,
- HiveParser.TOK_DISTRIBUTEBY, HiveParser.TOK_SORTBY);
-
- static class Phase1Ctx {
- String dest;
- int nextNum;
- }
-
- public SemanticAnalyzer(QueryState queryState) throws SemanticException {
- super(queryState);
- opToPartPruner = new HashMap<TableScanOperator, ExprNodeDesc>();
- opToPartList = new HashMap<TableScanOperator, PrunedPartitionList>();
- opToSamplePruner = new HashMap<TableScanOperator, SampleDesc>();
- nameToSplitSample = new HashMap<String, SplitSample>();
- // Must be deterministic order maps - see HIVE-8707
- topOps = new LinkedHashMap<String, TableScanOperator>();
- loadTableWork = new ArrayList<LoadTableDesc>();
- loadFileWork = new ArrayList<LoadFileDesc>();
- columnStatsAutoGatherContexts = new ArrayList<ColumnStatsAutoGatherContext>();
- opParseCtx = new LinkedHashMap<Operator<? extends OperatorDesc>, OpParseContext>();
- joinContext = new HashMap<JoinOperator, QBJoinTree>();
- smbMapJoinContext = new HashMap<SMBMapJoinOperator, QBJoinTree>();
- // Must be deterministic order map for consistent q-test output across Java versions
- topToTable = new LinkedHashMap<TableScanOperator, Table>();
- fsopToTable = new HashMap<FileSinkOperator, Table>();
- reduceSinkOperatorsAddedByEnforceBucketingSorting = new ArrayList<ReduceSinkOperator>();
- topToTableProps = new HashMap<TableScanOperator, Map<String, String>>();
- destTableId = 1;
- uCtx = null;
- listMapJoinOpsNoReducer = new ArrayList<AbstractMapJoinOperator<? extends MapJoinDesc>>();
- groupOpToInputTables = new HashMap<GroupByOperator, Set<String>>();
- prunedPartitions = new HashMap<String, PrunedPartitionList>();
- tabNameToTabObject = new HashMap<String, Table>();
- unparseTranslator = new UnparseTranslator(conf);
- autogenColAliasPrfxLbl = HiveConf.getVar(conf,
- HiveConf.ConfVars.HIVE_AUTOGEN_COLUMNALIAS_PREFIX_LABEL);
- autogenColAliasPrfxIncludeFuncName = HiveConf.getBoolVar(conf,
- HiveConf.ConfVars.HIVE_AUTOGEN_COLUMNALIAS_PREFIX_INCLUDEFUNCNAME);
- queryProperties = new QueryProperties();
- opToPartToSkewedPruner = new HashMap<TableScanOperator, Map<String, ExprNodeDesc>>();
- aliasToCTEs = new HashMap<String, CTEClause>();
- globalLimitCtx = new GlobalLimitCtx();
- viewAliasToInput = new HashMap<String, ReadEntity>();
- mergeIsDirect = true;
- noscan = partialscan = false;
- tabNameToTabObject = new HashMap<>();
- }
-
- @Override
- protected void reset(boolean clearPartsCache) {
- super.reset(true);
- if(clearPartsCache) {
- prunedPartitions.clear();
-
- //When init(true) combine with genResolvedParseTree, it will generate Resolved Parse tree from syntax tree
- //ReadEntity created under these conditions should be all relevant to the syntax tree even the ones without parents
- //set mergeIsDirect to true here.
- mergeIsDirect = true;
- } else {
- mergeIsDirect = false;
- }
- tabNameToTabObject.clear();
- loadTableWork.clear();
- loadFileWork.clear();
- columnStatsAutoGatherContexts.clear();
- topOps.clear();
- destTableId = 1;
- idToTableNameMap.clear();
- qb = null;
- ast = null;
- uCtx = null;
- joinContext.clear();
- smbMapJoinContext.clear();
- opParseCtx.clear();
- groupOpToInputTables.clear();
- disableJoinMerge = false;
- aliasToCTEs.clear();
- topToTable.clear();
- opToPartPruner.clear();
- opToPartList.clear();
- opToPartToSkewedPruner.clear();
- opToSamplePruner.clear();
- nameToSplitSample.clear();
- fsopToTable.clear();
- resultSchema = null;
- createVwDesc = null;
- viewsExpanded = null;
- viewSelect = null;
- ctesExpanded = null;
- globalLimitCtx.disableOpt();
- viewAliasToInput.clear();
- reduceSinkOperatorsAddedByEnforceBucketingSorting.clear();
- topToTableProps.clear();
- listMapJoinOpsNoReducer.clear();
- unparseTranslator.clear();
- queryProperties.clear();
- outputs.clear();
- }
-
- public void initParseCtx(ParseContext pctx) {
- opToPartPruner = pctx.getOpToPartPruner();
- opToPartList = pctx.getOpToPartList();
- opToSamplePruner = pctx.getOpToSamplePruner();
- topOps = pctx.getTopOps();
- loadTableWork = pctx.getLoadTableWork();
- loadFileWork = pctx.getLoadFileWork();
- ctx = pctx.getContext();
- destTableId = pctx.getDestTableId();
- idToTableNameMap = pctx.getIdToTableNameMap();
- uCtx = pctx.getUCtx();
- listMapJoinOpsNoReducer = pctx.getListMapJoinOpsNoReducer();
- prunedPartitions = pctx.getPrunedPartitions();
- tabNameToTabObject = pctx.getTabNameToTabObject();
- fetchTask = pctx.getFetchTask();
- setLineageInfo(pctx.getLineageInfo());
- }
-
- public ParseContext getParseContext() {
- // Make sure the basic query properties are initialized
- copyInfoToQueryProperties(queryProperties);
- return new ParseContext(queryState, opToPartPruner, opToPartList, topOps,
- new HashSet<JoinOperator>(joinContext.keySet()),
- new HashSet<SMBMapJoinOperator>(smbMapJoinContext.keySet()),
- loadTableWork, loadFileWork, columnStatsAutoGatherContexts, ctx, idToTableNameMap, destTableId, uCtx,
- listMapJoinOpsNoReducer, prunedPartitions, tabNameToTabObject,
- opToSamplePruner, globalLimitCtx, nameToSplitSample, inputs, rootTasks,
- opToPartToSkewedPruner, viewAliasToInput, reduceSinkOperatorsAddedByEnforceBucketingSorting,
- analyzeRewrite, tableDesc, queryProperties, viewProjectToTableSchema, acidFileSinks);
- }
-
- public CompilationOpContext getOpContext() {
- return ctx.getOpContext();
- }
-
- public void doPhase1QBExpr(ASTNode ast, QBExpr qbexpr, String id, String alias)
- throws SemanticException {
- doPhase1QBExpr(ast, qbexpr, id, alias, false);
- }
- @SuppressWarnings("nls")
- public void doPhase1QBExpr(ASTNode ast, QBExpr qbexpr, String id, String alias, boolean insideView)
- throws SemanticException {
-
- assert (ast.getToken() != null);
- switch (ast.getToken().getType()) {
- case HiveParser.TOK_QUERY: {
- QB qb = new QB(id, alias, true);
- qb.setInsideView(insideView);
- Phase1Ctx ctx_1 = initPhase1Ctx();
- doPhase1(ast, qb, ctx_1, null);
-
- qbexpr.setOpcode(QBExpr.Opcode.NULLOP);
- qbexpr.setQB(qb);
- }
- break;
- case HiveParser.TOK_UNIONALL: {
- qbexpr.setOpcode(QBExpr.Opcode.UNION);
- // query 1
- assert (ast.getChild(0) != null);
- QBExpr qbexpr1 = new QBExpr(alias + SUBQUERY_TAG_1);
- doPhase1QBExpr((ASTNode) ast.getChild(0), qbexpr1, id + SUBQUERY_TAG_1,
- alias + SUBQUERY_TAG_1, insideView);
- qbexpr.setQBExpr1(qbexpr1);
-
- // query 2
- assert (ast.getChild(1) != null);
- QBExpr qbexpr2 = new QBExpr(alias + SUBQUERY_TAG_2);
- doPhase1QBExpr((ASTNode) ast.getChild(1), qbexpr2, id + SUBQUERY_TAG_2,
- alias + SUBQUERY_TAG_2, insideView);
- qbexpr.setQBExpr2(qbexpr2);
- }
- break;
- }
- }
-
- private LinkedHashMap<String, ASTNode> doPhase1GetAggregationsFromSelect(
- ASTNode selExpr, QB qb, String dest) throws SemanticException {
-
- // Iterate over the selects search for aggregation Trees.
- // Use String as keys to eliminate duplicate trees.
- LinkedHashMap<String, ASTNode> aggregationTrees = new LinkedHashMap<String, ASTNode>();
- List<ASTNode> wdwFns = new ArrayList<ASTNode>();
- for (int i = 0; i < selExpr.getChildCount(); ++i) {
- ASTNode function = (ASTNode) selExpr.getChild(i);
- if (function.getType() == HiveParser.TOK_SELEXPR ||
- function.getType() == HiveParser.TOK_SUBQUERY_EXPR) {
- function = (ASTNode)function.getChild(0);
- }
- doPhase1GetAllAggregations(function, aggregationTrees, wdwFns);
- }
-
- // window based aggregations are handled differently
- for (ASTNode wdwFn : wdwFns) {
- WindowingSpec spec = qb.getWindowingSpec(dest);
- if(spec == null) {
- queryProperties.setHasWindowing(true);
- spec = new WindowingSpec();
- qb.addDestToWindowingSpec(dest, spec);
- }
- HashMap<String, ASTNode> wExprsInDest = qb.getParseInfo().getWindowingExprsForClause(dest);
- int wColIdx = spec.getWindowExpressions() == null ? 0 : spec.getWindowExpressions().size();
- WindowFunctionSpec wFnSpec = processWindowFunction(wdwFn,
- (ASTNode)wdwFn.getChild(wdwFn.getChildCount()-1));
- // If this is a duplicate invocation of a function; don't add to WindowingSpec.
- if ( wExprsInDest != null &&
- wExprsInDest.containsKey(wFnSpec.getExpression().toStringTree())) {
- continue;
- }
- wFnSpec.setAlias(wFnSpec.getName() + "_window_" + wColIdx);
- spec.addWindowFunction(wFnSpec);
- qb.getParseInfo().addWindowingExprToClause(dest, wFnSpec.getExpression());
- }
-
- return aggregationTrees;
- }
-
- private void doPhase1GetColumnAliasesFromSelect(
- ASTNode selectExpr, QBParseInfo qbp) {
- for (int i = 0; i < selectExpr.getChildCount(); ++i) {
- ASTNode selExpr = (ASTNode) selectExpr.getChild(i);
- if ((selExpr.getToken().getType() == HiveParser.TOK_SELEXPR)
- && (selExpr.getChildCount() == 2)) {
- String columnAlias = unescapeIdentifier(selExpr.getChild(1).getText());
- qbp.setExprToColumnAlias((ASTNode) selExpr.getChild(0), columnAlias);
- }
- }
- }
-
- /**
- * DFS-scan the expressionTree to find all aggregation subtrees and put them
- * in aggregations.
- *
- * @param expressionTree
- * @param aggregations
- * the key to the HashTable is the toStringTree() representation of
- * the aggregation subtree.
- * @throws SemanticException
- */
- private void doPhase1GetAllAggregations(ASTNode expressionTree,
- HashMap<String, ASTNode> aggregations, List<ASTNode> wdwFns) throws SemanticException {
- int exprTokenType = expressionTree.getToken().getType();
- if (exprTokenType == HiveParser.TOK_FUNCTION
- || exprTokenType == HiveParser.TOK_FUNCTIONDI
- || exprTokenType == HiveParser.TOK_FUNCTIONSTAR) {
- assert (expressionTree.getChildCount() != 0);
- if (expressionTree.getChild(expressionTree.getChildCount()-1).getType()
- == HiveParser.TOK_WINDOWSPEC) {
- // If it is a windowing spec, we include it in the list
- // Further, we will examine its children AST nodes to check whether
- // there are aggregation functions within
- wdwFns.add(expressionTree);
- doPhase1GetAllAggregations((ASTNode) expressionTree.getChild(expressionTree.getChildCount()-1),
- aggregations, wdwFns);
- return;
- }
- if (expressionTree.getChild(0).getType() == HiveParser.Identifier) {
- String functionName = unescapeIdentifier(expressionTree.getChild(0)
- .getText());
- // Validate the function name
- if (FunctionRegistry.getFunctionInfo(functionName) == null) {
- throw new SemanticException(ErrorMsg.INVALID_FUNCTION.getMsg(functionName));
- }
- if(FunctionRegistry.impliesOrder(functionName)) {
- throw new SemanticException(ErrorMsg.MISSING_OVER_CLAUSE.getMsg(functionName));
- }
- if (FunctionRegistry.getGenericUDAFResolver(functionName) != null) {
- if(containsLeadLagUDF(expressionTree)) {
- throw new SemanticException(ErrorMsg.MISSING_OVER_CLAUSE.getMsg(functionName));
- }
- aggregations.put(expressionTree.toStringTree(), expressionTree);
- FunctionInfo fi = FunctionRegistry.getFunctionInfo(functionName);
- if (!fi.isNative()) {
- unparseTranslator.addIdentifierTranslation((ASTNode) expressionTree
- .getChild(0));
- }
- return;
- }
- }
- }
- for (int i = 0; i < expressionTree.getChildCount(); i++) {
- doPhase1GetAllAggregations((ASTNode) expressionTree.getChild(i),
- aggregations, wdwFns);
- }
- }
-
- private List<ASTNode> doPhase1GetDistinctFuncExprs(
- HashMap<String, ASTNode> aggregationTrees) throws SemanticException {
- List<ASTNode> exprs = new ArrayList<ASTNode>();
- for (Map.Entry<String, ASTNode> entry : aggregationTrees.entrySet()) {
- ASTNode value = entry.getValue();
- assert (value != null);
- if (value.getToken().getType() == HiveParser.TOK_FUNCTIONDI) {
- exprs.add(value);
- }
- }
- return exprs;
- }
-
- public static String generateErrorMessage(ASTNode ast, String message) {
- StringBuilder sb = new StringBuilder();
- if (ast == null) {
- sb.append(message).append(". Cannot tell the position of null AST.");
- return sb.toString();
- }
- sb.append(ast.getLine());
- sb.append(":");
- sb.append(ast.getCharPositionInLine());
- sb.append(" ");
- sb.append(message);
- sb.append(". Error encountered near token '");
- sb.append(ErrorMsg.getText(ast));
- sb.append("'");
- return sb.toString();
- }
-
- ASTNode getAST() {
- return this.ast;
- }
-
- protected void setAST(ASTNode newAST) {
- this.ast = newAST;
- }
-
- /**
- * Goes though the tabref tree and finds the alias for the table. Once found,
- * it records the table name-> alias association in aliasToTabs. It also makes
- * an association from the alias to the table AST in parse info.
- *
- * @return the alias of the table
- */
- private String processTable(QB qb, ASTNode tabref) throws SemanticException {
- // For each table reference get the table name
- // and the alias (if alias is not present, the table name
- // is used as an alias)
- int aliasIndex = 0;
- int propsIndex = -1;
- int tsampleIndex = -1;
- int ssampleIndex = -1;
- for (int index = 1; index < tabref.getChildCount(); index++) {
- ASTNode ct = (ASTNode) tabref.getChild(index);
- if (ct.getToken().getType() == HiveParser.TOK_TABLEBUCKETSAMPLE) {
- tsampleIndex = index;
- } else if (ct.getToken().getType() == HiveParser.TOK_TABLESPLITSAMPLE) {
- ssampleIndex = index;
- } else if (ct.getToken().getType() == HiveParser.TOK_TABLEPROPERTIES) {
- propsIndex = index;
- } else {
- aliasIndex = index;
- }
- }
-
- ASTNode tableTree = (ASTNode) (tabref.getChild(0));
-
- String tabIdName = getUnescapedName(tableTree).toLowerCase();
-
- String alias;
- if (aliasIndex != 0) {
- alias = unescapeIdentifier(tabref.getChild(aliasIndex).getText());
- }
- else {
- alias = getUnescapedUnqualifiedTableName(tableTree);
- }
-
- if (propsIndex >= 0) {
- Tree propsAST = tabref.getChild(propsIndex);
- Map<String, String> props = DDLSemanticAnalyzer.getProps((ASTNode) propsAST.getChild(0));
- // We get the information from Calcite.
- if ("TRUE".equals(props.get("insideView"))) {
- qb.getAliasInsideView().add(alias.toLowerCase());
- }
- qb.setTabProps(alias, props);
- }
-
- // If the alias is already there then we have a conflict
- if (qb.exists(alias)) {
- throw new SemanticException(ErrorMsg.AMBIGUOUS_TABLE_ALIAS.getMsg(tabref
- .getChild(aliasIndex)));
- }
- if (tsampleIndex >= 0) {
- ASTNode sampleClause = (ASTNode) tabref.getChild(tsampleIndex);
- ArrayList<ASTNode> sampleCols = new ArrayList<ASTNode>();
- if (sampleClause.getChildCount() > 2) {
- for (int i = 2; i < sampleClause.getChildCount(); i++) {
- sampleCols.add((ASTNode) sampleClause.getChild(i));
- }
- }
- // TODO: For now only support sampling on up to two columns
- // Need to change it to list of columns
- if (sampleCols.size() > 2) {
- throw new SemanticException(generateErrorMessage(
- (ASTNode) tabref.getChild(0),
- ErrorMsg.SAMPLE_RESTRICTION.getMsg()));
- }
- TableSample tabSample = new TableSample(
- unescapeIdentifier(sampleClause.getChild(0).getText()),
- unescapeIdentifier(sampleClause.getChild(1).getText()),
- sampleCols);
- qb.getParseInfo().setTabSample(alias, tabSample);
- if (unparseTranslator.isEnabled()) {
- for (ASTNode sampleCol : sampleCols) {
- unparseTranslator.addIdentifierTranslation((ASTNode) sampleCol
- .getChild(0));
- }
- }
- } else if (ssampleIndex >= 0) {
- ASTNode sampleClause = (ASTNode) tabref.getChild(ssampleIndex);
-
- Tree type = sampleClause.getChild(0);
- Tree numerator = sampleClause.getChild(1);
- String value = unescapeIdentifier(numerator.getText());
-
-
- SplitSample sample;
- if (type.getType() == HiveParser.TOK_PERCENT) {
- assertCombineInputFormat(numerator, "Percentage");
- Double percent = Double.valueOf(value).doubleValue();
- if (percent < 0 || percent > 100) {
- throw new SemanticException(generateErrorMessage((ASTNode) numerator,
- "Sampling percentage should be between 0 and 100"));
- }
- int seedNum = conf.getIntVar(ConfVars.HIVESAMPLERANDOMNUM);
- sample = new SplitSample(percent, seedNum);
- } else if (type.getType() == HiveParser.TOK_ROWCOUNT) {
- sample = new SplitSample(Integer.parseInt(value));
- } else {
- assert type.getType() == HiveParser.TOK_LENGTH;
- assertCombineInputFormat(numerator, "Total Length");
- long length = Integer.parseInt(value.substring(0, value.length() - 1));
- char last = value.charAt(value.length() - 1);
- if (last == 'k' || last == 'K') {
- length <<= 10;
- } else if (last == 'm' || last == 'M') {
- length <<= 20;
- } else if (last == 'g' || last == 'G') {
- length <<= 30;
- }
- int seedNum = conf.getIntVar(ConfVars.HIVESAMPLERANDOMNUM);
- sample = new SplitSample(length, seedNum);
- }
- String alias_id = getAliasId(alias, qb);
- nameToSplitSample.put(alias_id, sample);
- }
- // Insert this map into the stats
- qb.setTabAlias(alias, tabIdName);
- if (qb.isInsideView()) {
- qb.getAliasInsideView().add(alias.toLowerCase());
- }
- qb.addAlias(alias);
-
- qb.getParseInfo().setSrcForAlias(alias, tableTree);
-
- // if alias to CTE contains the alias, we do not do the translation because
- // cte is actually a subquery.
- if (!this.aliasToCTEs.containsKey(alias)) {
- unparseTranslator.addTableNameTranslation(tableTree, SessionState.get().getCurrentDatabase());
- if (aliasIndex != 0) {
- unparseTranslator.addIdentifierTranslation((ASTNode) tabref.getChild(aliasIndex));
- }
- }
-
- return alias;
- }
-
- Map<String, SplitSample> getNameToSplitSampleMap() {
- return this.nameToSplitSample;
- }
-
- /**
- * Convert a string to Text format and write its bytes in the same way TextOutputFormat would do.
- * This is needed to properly encode non-ascii characters.
- */
- private static void writeAsText(String text, FSDataOutputStream out) throws IOException {
- Text to = new Text(text);
- out.write(to.getBytes(), 0, to.getLength());
- }
-
- /**
- * Generate a temp table out of a value clause
- * See also {@link #preProcessForInsert(ASTNode, QB)}
- */
- private ASTNode genValuesTempTable(ASTNode originalFrom, QB qb) throws SemanticException {
- Path dataDir = null;
- if(!qb.getEncryptedTargetTablePaths().isEmpty()) {
- //currently only Insert into T values(...) is supported thus only 1 values clause
- //and only 1 target table are possible. If/when support for
- //select ... from values(...) is added an insert statement may have multiple
- //encrypted target tables.
- dataDir = ctx.getMRTmpPath(qb.getEncryptedTargetTablePaths().get(0).toUri());
- }
- // Pick a name for the table
- SessionState ss = SessionState.get();
- String tableName = VALUES_TMP_TABLE_NAME_PREFIX + ss.getNextValuesTempTableSuffix();
-
- // Step 1, parse the values clause we were handed
- List<? extends Node> fromChildren = originalFrom.getChildren();
- // First child should be the virtual table ref
- ASTNode virtualTableRef = (ASTNode)fromChildren.get(0);
- assert virtualTableRef.getToken().getType() == HiveParser.TOK_VIRTUAL_TABREF :
- "Expected first child of TOK_VIRTUAL_TABLE to be TOK_VIRTUAL_TABREF but was " +
- virtualTableRef.getName();
-
- List<? extends Node> virtualTableRefChildren = virtualTableRef.getChildren();
- // First child of this should be the table name. If it's anonymous,
- // then we don't have a table name.
- ASTNode tabName = (ASTNode)virtualTableRefChildren.get(0);
- if (tabName.getToken().getType() != HiveParser.TOK_ANONYMOUS) {
- // TODO, if you want to make select ... from (values(...) as foo(...) work,
- // you need to parse this list of columns names and build it into the table
- throw new SemanticException(ErrorMsg.VALUES_TABLE_CONSTRUCTOR_NOT_SUPPORTED.getMsg());
- }
-
- // The second child of the TOK_VIRTUAL_TABLE should be TOK_VALUES_TABLE
- ASTNode valuesTable = (ASTNode)fromChildren.get(1);
- assert valuesTable.getToken().getType() == HiveParser.TOK_VALUES_TABLE :
- "Expected second child of TOK_VIRTUAL_TABLE to be TOK_VALUE_TABLE but was " +
- valuesTable.getName();
- // Each of the children of TOK_VALUES_TABLE will be a TOK_VALUE_ROW
- List<? extends Node> valuesTableChildren = valuesTable.getChildren();
-
- // Now that we're going to start reading through the rows, open a file to write the rows too
- // If we leave this method before creating the temporary table we need to be sure to clean up
- // this file.
- Path tablePath = null;
- FileSystem fs = null;
- FSDataOutputStream out = null;
- try {
- if(dataDir == null) {
- tablePath = Warehouse.getDnsPath(new Path(ss.getTempTableSpace(), tableName), conf);
- }
- else {
- //if target table of insert is encrypted, make sure temporary table data is stored
- //similarly encrypted
- tablePath = Warehouse.getDnsPath(new Path(dataDir, tableName), conf);
- }
- fs = tablePath.getFileSystem(conf);
- fs.mkdirs(tablePath);
- Path dataFile = new Path(tablePath, "data_file");
- out = fs.create(dataFile);
- List<FieldSchema> fields = new ArrayList<FieldSchema>();
-
- boolean firstRow = true;
- for (Node n : valuesTableChildren) {
- ASTNode valuesRow = (ASTNode) n;
- assert valuesRow.getToken().getType() == HiveParser.TOK_VALUE_ROW :
- "Expected child of TOK_VALUE_TABLE to be TOK_VALUE_ROW but was " + valuesRow.getName();
- // Each of the children of this should be a literal
- List<? extends Node> valuesRowChildren = valuesRow.getChildren();
- boolean isFirst = true;
- int nextColNum = 1;
- for (Node n1 : valuesRowChildren) {
- ASTNode value = (ASTNode) n1;
- if (firstRow) {
- fields.add(new FieldSchema("tmp_values_col" + nextColNum++, "string", ""));
- }
- if (isFirst) isFirst = false;
- else writeAsText("\u0001", out);
- writeAsText(unparseExprForValuesClause(value), out);
- }
- writeAsText("\n", out);
- firstRow = false;
- }
-
- // Step 2, create a temp table, using the created file as the data
- StorageFormat format = new StorageFormat(conf);
- format.processStorageFormat("TextFile");
- Table table = db.newTable(tableName);
- table.setSerializationLib(format.getSerde());
- table.setFields(fields);
- table.setDataLocation(tablePath);
- table.getTTable().setTemporary(true);
- table.setStoredAsSubDirectories(false);
- table.setInputFormatClass(format.getInputFormat());
- table.setOutputFormatClass(format.getOutputFormat());
- db.createTable(table, false);
- } catch (Exception e) {
- String errMsg = ErrorMsg.INSERT_CANNOT_CREATE_TEMP_FILE.getMsg() + e.getMessage();
- LOG.error(errMsg);
- // Try to delete the file
- if (fs != null && tablePath != null) {
- try {
- fs.delete(tablePath, false);
- } catch (IOException swallowIt) {}
- }
- throw new SemanticException(errMsg, e);
- } finally {
- IOUtils.closeStream(out);
- }
-
- // Step 3, return a new subtree with a from clause built around that temp table
- // The form of the tree is TOK_TABREF->TOK_TABNAME->identifier(tablename)
- Token t = new ClassicToken(HiveParser.TOK_TABREF);
- ASTNode tabRef = new ASTNode(t);
- t = new ClassicToken(HiveParser.TOK_TABNAME);
- ASTNode tabNameNode = new ASTNode(t);
- tabRef.addChild(tabNameNode);
- t = new ClassicToken(HiveParser.Identifier, tableName);
- ASTNode identifier = new ASTNode(t);
- tabNameNode.addChild(identifier);
- return tabRef;
- }
-
- // Take an expression in the values clause and turn it back into a string. This is far from
- // comprehensive. At the moment it only supports:
- // * literals (all types)
- // * unary negatives
- // * true/false
- private String unparseExprForValuesClause(ASTNode expr) throws SemanticException {
- switch (expr.getToken().getType()) {
- case HiveParser.Number:
- return expr.getText();
-
- case HiveParser.StringLiteral:
- return BaseSemanticAnalyzer.unescapeSQLString(expr.getText());
-
- case HiveParser.KW_FALSE:
- // UDFToBoolean casts any non-empty string to true, so set this to false
- return "";
-
- case HiveParser.KW_TRUE:
- return "TRUE";
-
- case HiveParser.MINUS:
- return "-" + unparseExprForValuesClause((ASTNode)expr.getChildren().get(0));
-
- case HiveParser.TOK_NULL:
- // Hive's text input will translate this as a null
- return "\\N";
-
- default:
- throw new SemanticException("Expression of type " + expr.getText() +
- " not supported in insert/values");
- }
-
- }
-
- private void assertCombineInputFormat(Tree numerator, String message) throws SemanticException {
- String inputFormat = conf.getVar(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE).equals("tez") ?
- HiveConf.getVar(conf, HiveConf.ConfVars.HIVETEZINPUTFORMAT):
- HiveConf.getVar(conf, HiveConf.ConfVars.HIVEINPUTFORMAT);
- if (!inputFormat.equals(CombineHiveInputFormat.class.getName())) {
- throw new SemanticException(generateErrorMessage((ASTNode) numerator,
- message + " sampling is not supported in " + inputFormat));
- }
- }
-
- private String processSubQuery(QB qb, ASTNode subq) throws SemanticException {
-
- // This is a subquery and must have an alias
- if (subq.getChildCount() != 2) {
- throw new SemanticException(ErrorMsg.NO_SUBQUERY_ALIAS.getMsg(subq));
- }
- ASTNode subqref = (ASTNode) subq.getChild(0);
- String alias = unescapeIdentifier(subq.getChild(1).getText());
-
- // Recursively do the first phase of semantic analysis for the subquery
- QBExpr qbexpr = new QBExpr(alias);
-
- doPhase1QBExpr(subqref, qbexpr, qb.getId(), alias);
-
- // If the alias is already there then we have a conflict
- if (qb.exists(alias)) {
- throw new SemanticException(ErrorMsg.AMBIGUOUS_TABLE_ALIAS.getMsg(subq
- .getChild(1)));
- }
- // Insert this map into the stats
- qb.setSubqAlias(alias, qbexpr);
- qb.addAlias(alias);
-
- unparseTranslator.addIdentifierTranslation((ASTNode) subq.getChild(1));
-
- return alias;
- }
-
- /*
- * Phase1: hold onto any CTE definitions in aliasToCTE.
- * CTE definitions are global to the Query.
- */
- private void processCTE(QB qb, ASTNode ctes) throws SemanticException {
-
- int numCTEs = ctes.getChildCount();
-
- for(int i=0; i <numCTEs; i++) {
- ASTNode cte = (ASTNode) ctes.getChild(i);
- ASTNode cteQry = (ASTNode) cte.getChild(0);
- String alias = unescapeIdentifier(cte.getChild(1).getText());
-
- String qName = qb.getId() == null ? "" : qb.getId() + ":";
- qName += alias.toLowerCase();
-
- if ( aliasToCTEs.containsKey(qName)) {
- throw new SemanticException(ErrorMsg.AMBIGUOUS_TABLE_ALIAS.getMsg(cte.getChild(1)));
- }
- aliasToCTEs.put(qName, new CTEClause(qName, cteQry));
- }
- }
-
- /*
- * We allow CTE definitions in views. So we can end up with a hierarchy of CTE definitions:
- * - at the top level of a query statement
- * - where a view is referenced.
- * - views may refer to other views.
- *
- * The scoping rules we use are: to search for a CTE from the current QB outwards. In order to
- * disambiguate between CTES are different levels we qualify(prefix) them with the id of the QB
- * they appear in when adding them to the <code>aliasToCTEs</code> map.
- *
- */
- private CTEClause findCTEFromName(QB qb, String cteName) {
- StringBuilder qId = new StringBuilder();
- if (qb.getId() != null) {
- qId.append(qb.getId());
- }
-
- while (qId.length() > 0) {
- String nm = qId + ":" + cteName;
- CTEClause cte = aliasToCTEs.get(nm);
- if (cte != null) {
- return cte;
- }
- int lastIndex = qId.lastIndexOf(":");
- lastIndex = lastIndex < 0 ? 0 : lastIndex;
- qId.setLength(lastIndex);
- }
- return aliasToCTEs.get(cteName);
- }
-
- /*
- * If a CTE is referenced in a QueryBlock:
- * - add it as a SubQuery for now.
- * - SQ.alias is the alias used in QB. (if no alias is specified,
- * it used the CTE name. Works just like table references)
- * - Adding SQ done by:
- * - copying AST of CTE
- * - setting ASTOrigin on cloned AST.
- * - trigger phase 1 on new QBExpr.
- * - update QB data structs: remove this as a table reference, move it to a SQ invocation.
- */
- private void addCTEAsSubQuery(QB qb, String cteName, String cteAlias)
- throws SemanticException {
- cteAlias = cteAlias == null ? cteName : cteAlias;
- CTEClause cte = findCTEFromName(qb, cteName);
- ASTNode cteQryNode = cte.cteNode;
- QBExpr cteQBExpr = new QBExpr(cteAlias);
- doPhase1QBExpr(cteQryNode, cteQBExpr, qb.getId(), cteAlias);
- qb.rewriteCTEToSubq(cteAlias, cteName, cteQBExpr);
- }
-
- private final CTEClause rootClause = new CTEClause(null, null);
-
- @Override
- public List<Task<? extends Serializable>> getAllRootTasks() {
- if (!rootTasksResolved) {
- rootTasks = toRealRootTasks(rootClause.asExecutionOrder());
- rootTasksResolved = true;
- }
- return rootTasks;
- }
-
- @Override
- public HashSet<ReadEntity> getAllInputs() {
- HashSet<ReadEntity> readEntities = new HashSet<ReadEntity>(getInputs());
- for (CTEClause cte : rootClause.asExecutionOrder()) {
- if (cte.source != null) {
- readEntities.addAll(cte.source.getInputs());
- }
- }
- return readEntities;
- }
-
- @Override
- public HashSet<WriteEntity> getAllOutputs() {
- HashSet<WriteEntity> writeEntities = new HashSet<WriteEntity>(getOutputs());
- for (CTEClause cte : rootClause.asExecutionOrder()) {
- if (cte.source != null) {
- writeEntities.addAll(cte.source.getOutputs());
- }
- }
- return writeEntities;
- }
-
- class CTEClause {
- CTEClause(String alias, ASTNode cteNode) {
- this.alias = alias;
- this.cteNode = cteNode;
- }
- String alias;
- ASTNode cteNode;
- boolean materialize;
- int reference;
- QBExpr qbExpr;
- List<CTEClause> parents = new ArrayList<CTEClause>();
-
- // materialized
- Table table;
- SemanticAnalyzer source;
-
- List<Task<? extends Serializable>> getTasks() {
- return source == null ? null : source.rootTasks;
- }
-
- List<CTEClause> asExecutionOrder() {
- List<CTEClause> execution = new ArrayList<CTEClause>();
- asExecutionOrder(new HashSet<CTEClause>(), execution);
- return execution;
- }
-
- void asExecutionOrder(Set<CTEClause> visited, List<CTEClause> execution) {
- for (CTEClause parent : parents) {
- if (visited.add(parent)) {
- parent.asExecutionOrder(visited, execution);
- }
- }
- execution.add(this);
- }
-
- @Override
- public String toString() {
- return alias == null ? "<root>" : alias;
- }
- }
-
- private List<Task<? extends Serializable>> toRealRootTasks(List<CTEClause> execution) {
- List<Task<? extends Serializable>> cteRoots = new ArrayList<>();
- List<Task<? extends Serializable>> cteLeafs = new ArrayList<>();
- List<Task<? extends Serializable>> curTopRoots = null;
- List<Task<? extends Serializable>> curBottomLeafs = null;
- for (int i = 0; i < execution.size(); i++) {
- CTEClause current = execution.get(i);
- if (current.parents.isEmpty() && curTopRoots != null) {
- cteRoots.addAll(curTopRoots);
- cteLeafs.addAll(curBottomLeafs);
- curTopRoots = curBottomLeafs = null;
- }
- List<Task<? extends Serializable>> curTasks = current.getTasks();
- if (curTasks == null) {
- continue;
- }
- if (curTopRoots == null) {
- curTopRoots = curTasks;
- }
- if (curBottomLeafs != null) {
- for (Task<?> topLeafTask : curBottomLeafs) {
- for (Task<?> currentRootTask : curTasks) {
- topLeafTask.addDependentTask(currentRootTask);
- }
- }
- }
- curBottomLeafs = Task.findLeafs(curTasks);
- }
- if (curTopRoots != null) {
- cteRoots.addAll(curTopRoots);
- cteLeafs.addAll(curBottomLeafs);
- }
-
- if (cteRoots.isEmpty()) {
- return rootTasks;
- }
- for (Task<?> cteLeafTask : cteLeafs) {
- for (Task<?> mainRootTask : rootTasks) {
- cteLeafTask.addDependentTask(mainRootTask);
- }
- }
- return cteRoots;
- }
-
- Table materializeCTE(String cteName, CTEClause cte) throws HiveException {
-
- ASTNode createTable = new ASTNode(new ClassicToken(HiveParser.TOK_CREATETABLE));
-
- ASTNode tableName = new ASTNode(new ClassicToken(HiveParser.TOK_TABNAME));
- tableName.addChild(new ASTNode(new ClassicToken(HiveParser.Identifier, cteName)));
-
- ASTNode temporary = new ASTNode(new ClassicToken(HiveParser.KW_TEMPORARY, MATERIALIZATION_MARKER));
-
- createTable.addChild(tableName);
- createTable.addChild(temporary);
- createTable.addChild(cte.cteNode);
-
- SemanticAnalyzer analyzer = new SemanticAnalyzer(queryState);
- analyzer.initCtx(ctx);
- analyzer.init(false);
-
- // should share cte contexts
- analyzer.aliasToCTEs.putAll(aliasToCTEs);
-
- HiveOperation operation = queryState.getHiveOperation();
- try {
- analyzer.analyzeInternal(createTable);
- } finally {
- queryState.setCommandType(operation);
- }
-
- Table table = analyzer.tableDesc.toTable(conf);
- Path location = table.getDataLocation();
- try {
- location.getFileSystem(conf).mkdirs(location);
- } catch (IOException e) {
- throw new HiveException(e);
- }
- table.setMaterializedTable(true);
-
- LOG.info(cteName + " will be materialized into " + location);
- cte.table = table;
- cte.source = analyzer;
-
- ctx.addMaterializedTable(cteName, table);
-
- return table;
- }
-
-
- static boolean isJoinToken(ASTNode node) {
- if ((node.getToken().getType() == HiveParser.TOK_JOIN)
- || (node.getToken().getType() == HiveParser.TOK_CROSSJOIN)
- || isOuterJoinToken(node)
- || (node.getToken().getType() == HiveParser.TOK_LEFTSEMIJOIN)
- || (node.getToken().getType() == HiveParser.TOK_UNIQUEJOIN)) {
- return true;
- }
-
- return false;
- }
-
- static private boolean isOuterJoinToken(ASTNode node) {
- return (node.getToken().getType() == HiveParser.TOK_LEFTOUTERJOIN)
- || (node.getToken().getType() == HiveParser.TOK_RIGHTOUTERJOIN)
- || (node.getToken().getType() == HiveParser.TOK_FULLOUTERJOIN);
- }
-
- /**
- * Given the AST with TOK_JOIN as the root, get all the aliases for the tables
- * or subqueries in the join.
- *
- * @param qb
- * @param join
- * @throws SemanticException
- */
- @SuppressWarnings("nls")
- private void processJoin(QB qb, ASTNode join) throws SemanticException {
- int numChildren = join.getChildCount();
- if ((numChildren != 2) && (numChildren != 3)
- && join.getToken().getType() != HiveParser.TOK_UNIQUEJOIN) {
- throw new SemanticException(generateErrorMessage(join,
- "Join with multiple children"));
- }
-
- queryProperties.incrementJoinCount(isOuterJoinToken(join));
- for (int num = 0; num < numChildren; num++) {
- ASTNode child = (ASTNode) join.getChild(num);
- if (child.getToken().getType() == HiveParser.TOK_TABREF) {
- processTable(qb, child);
- } else if (child.getToken().getType() == HiveParser.TOK_SUBQUERY) {
- processSubQuery(qb, child);
- } else if (child.getToken().getType() == HiveParser.TOK_PTBLFUNCTION) {
- queryProperties.setHasPTF(true);
- processPTF(qb, child);
- PTFInvocationSpec ptfInvocationSpec = qb.getPTFInvocationSpec(child);
- String inputAlias = ptfInvocationSpec == null ? null :
- ptfInvocationSpec.getFunction().getAlias();;
- if ( inputAlias == null ) {
- throw new SemanticException(generateErrorMessage(child,
- "PTF invocation in a Join must have an alias"));
- }
-
- } else if (child.getToken().getType() == HiveParser.TOK_LATERAL_VIEW ||
- child.getToken().getType() == HiveParser.TOK_LATERAL_VIEW_OUTER) {
- // SELECT * FROM src1 LATERAL VIEW udtf() AS myTable JOIN src2 ...
- // is not supported. Instead, the lateral view must be in a subquery
- // SELECT * FROM (SELECT * FROM src1 LATERAL VIEW udtf() AS myTable) a
- // JOIN src2 ...
- throw new SemanticException(ErrorMsg.LATERAL_VIEW_WITH_JOIN
- .getMsg(join));
- } else if (isJoinToken(child)) {
- processJoin(qb, child);
- }
- }
- }
-
- /**
- * Given the AST with TOK_LATERAL_VIEW as the root, get the alias for the
- * table or subquery in the lateral view and also make a mapping from the
- * alias to all the lateral view AST's.
- *
- * @param qb
- * @param lateralView
- * @return the alias for the table/subquery
- * @throws SemanticException
- */
-
- private String processLateralView(QB qb, ASTNode lateralView)
- throws SemanticException {
- int numChildren = lateralView.getChildCount();
-
- assert (numChildren == 2);
- ASTNode next = (ASTNode) lateralView.getChild(1);
-
- String alias = null;
-
- switch (next.getToken().getType()) {
- case HiveParser.TOK_TABREF:
- alias = processTable(qb, next);
- break;
- case HiveParser.TOK_SUBQUERY:
- alias = processSubQuery(qb, next);
- break;
- case HiveParser.TOK_LATERAL_VIEW:
- case HiveParser.TOK_LATERAL_VIEW_OUTER:
- alias = processLateralView(qb, next);
- break;
- default:
- throw new SemanticException(ErrorMsg.LATERAL_VIEW_INVALID_CHILD
- .getMsg(lateralView));
- }
- alias = alias.toLowerCase();
- qb.getParseInfo().addLateralViewForAlias(alias, lateralView);
- qb.addAlias(alias);
- return alias;
- }
-
- /**
- * Phase 1: (including, but not limited to):
- *
- * 1. Gets all the aliases for all the tables / subqueries and makes the
- * appropriate mapping in aliasToTabs, aliasToSubq 2. Gets the location of the
- * destination and names the clause "inclause" + i 3. Creates a map from a
- * string representation of an aggregation tree to the actual aggregation AST
- * 4. Creates a mapping from the clause name to the select expression AST in
- * destToSelExpr 5. Creates a mapping from a table alias to the lateral view
- * AST's in aliasToLateralViews
- *
- * @param ast
- * @param qb
- * @param ctx_1
- * @throws SemanticException
- */
- @SuppressWarnings({"fallthrough", "nls"})
- public boolean doPhase1(ASTNode ast, QB qb, Phase1Ctx ctx_1, PlannerContext plannerCtx)
- throws SemanticException {
-
- boolean phase1Result = true;
- QBParseInfo qbp = qb.getParseInfo();
- boolean skipRecursion = false;
-
- if (ast.getToken() != null) {
- skipRecursion = true;
- switch (ast.getToken().getType()) {
- case HiveParser.TOK_SELECTDI:
- qb.countSelDi();
- // fall through
- case HiveParser.TOK_SELECT:
- qb.countSel();
- qbp.setSelExprForClause(ctx_1.dest, ast);
-
- int posn = 0;
- if (((ASTNode) ast.getChild(0)).getToken().getType() == HiveParser.TOK_HINTLIST) {
- qbp.setHints((ASTNode) ast.getChild(0));
- posn++;
- }
-
- if ((ast.getChild(posn).getChild(0).getType() == HiveParser.TOK_TRANSFORM))
- queryProperties.setUsesScript(true);
-
- LinkedHashMap<String, ASTNode> aggregations = doPhase1GetAggregationsFromSelect(ast,
- qb, ctx_1.dest);
- doPhase1GetColumnAliasesFromSelect(ast, qbp);
- qbp.setAggregationExprsForClause(ctx_1.dest, aggregations);
- qbp.setDistinctFuncExprsForClause(ctx_1.dest,
- doPhase1GetDistinctFuncExprs(aggregations));
- break;
-
- case HiveParser.TOK_WHERE:
- qbp.setWhrExprForClause(ctx_1.dest, ast);
- if (!SubQueryUtils.findSubQueries((ASTNode) ast.getChild(0)).isEmpty())
- queryProperties.setFilterWithSubQuery(true);
- break;
-
- case HiveParser.TOK_INSERT_INTO:
- String currentDatabase = SessionState.get().getCurrentDatabase();
- String tab_name = getUnescapedName((ASTNode) ast.getChild(0).getChild(0), currentDatabase);
- qbp.addInsertIntoTable(tab_name, ast);
-
- case HiveParser.TOK_DESTINATION:
- ctx_1.dest = "insclause-" + ctx_1.nextNum;
- ctx_1.nextNum++;
- boolean isTmpFileDest = false;
- if (ast.getChildCount() > 0 && ast.getChild(0) instanceof ASTNode) {
- ASTNode ch = (ASTNode) ast.getChild(0);
- if (ch.getToken().getType() == HiveParser.TOK_DIR && ch.getChildCount() > 0
- && ch.getChild(0) instanceof ASTNode) {
- ch = (ASTNode) ch.getChild(0);
- isTmpFileDest = ch.getToken().getType() == HiveParser.TOK_TMP_FILE;
- } else {
- if (ast.getToken().getType() == HiveParser.TOK_DESTINATION
- && ast.getChild(0).getType() == HiveParser.TOK_TAB) {
- String fullTableName = getUnescapedName((ASTNode) ast.getChild(0).getChild(0),
- SessionState.get().getCurrentDatabase());
- qbp.getInsertOverwriteTables().put(fullTableName, ast);
- }
- }
- }
-
- // is there a insert in the subquery
- if (qbp.getIsSubQ() && !isTmpFileDest) {
- throw new SemanticException(ErrorMsg.NO_INSERT_INSUBQUERY.getMsg(ast));
- }
-
- if (plannerCtx != null) {
- plannerCtx.setInsertToken(ast, isTmpFileDest);
- }
-
- qbp.setDestForClause(ctx_1.dest, (ASTNode) ast.getChild(0));
- handleInsertStatementSpecPhase1(ast, qbp, ctx_1);
- if (qbp.getClauseNamesForDest().size() > 1) {
- queryProperties.setMultiDestQuery(true);
- }
- break;
-
- case HiveParser.TOK_FROM:
- int child_count = ast.getChildCount();
- if (child_count != 1) {
- throw new SemanticException(generateErrorMessage(ast,
- "Multiple Children " + child_count));
- }
-
- // Check if this is a subquery / lateral view
- ASTNode frm = (ASTNode) ast.getChild(0);
- if (frm.getToken().getType() == HiveParser.TOK_TABREF) {
- processTable(qb, frm);
- } else if (frm.getToken().getType() == HiveParser.TOK_VIRTUAL_TABLE) {
- // Create a temp table with the passed values in it then rewrite this portion of the
- // tree to be from that table.
- ASTNode newFrom = genValuesTempTable(frm, qb);
- ast.setChild(0, newFrom);
- processTable(qb, newFrom);
- } else if (frm.getToken().getType() == HiveParser.TOK_SUBQUERY) {
- processSubQuery(qb, frm);
- } else if (frm.getToken().getType() == HiveParser.TOK_LATERAL_VIEW ||
- frm.getToken().getType() == HiveParser.TOK_LATERAL_VIEW_OUTER) {
- queryProperties.setHasLateralViews(true);
- processLateralView(qb, frm);
- } else if (isJoinToken(frm)) {
- processJoin(qb, frm);
- qbp.setJoinExpr(frm);
- }else if(frm.getToken().getType() == HiveParser.TOK_PTBLFUNCTION){
- queryProperties.setHasPTF(true);
- processPTF(qb, frm);
- }
- break;
-
- case HiveParser.TOK_CLUSTERBY:
- // Get the clusterby aliases - these are aliased to the entries in the
- // select list
- queryProperties.setHasClusterBy(true);
- qbp.setClusterByExprForClause(ctx_1.dest, ast);
- break;
-
- case HiveParser.TOK_DISTRIBUTEBY:
- // Get the distribute by aliases - these are aliased to the entries in
- // the
- // select list
- queryProperties.setHasDistributeBy(true);
- qbp.setDistributeByExprForClause(ctx_1.dest, ast);
- if (qbp.getClusterByForClause(ctx_1.dest) != null) {
- throw new SemanticException(generateErrorMessage(ast,
- ErrorMsg.CLUSTERBY_DISTRIBUTEBY_CONFLICT.getMsg()));
- } else if (qbp.getOrderByForClause(ctx_1.dest) != null) {
- throw new SemanticException(generateErrorMessage(ast,
- ErrorMsg.ORDERBY_DISTRIBUTEBY_CONFLICT.getMsg()));
- }
- break;
-
- case HiveParser.TOK_SORTBY:
- // Get the sort by aliases - these are aliased to the entries in the
- // select list
- queryProperties.setHasSortBy(true);
- qbp.setSortByExprForClause(ctx_1.dest, ast);
- if (qbp.getClusterByForClause(ctx_1.dest) != null) {
- throw new SemanticException(generateErrorMessage(ast,
- ErrorMsg.CLUSTERBY_SORTBY_CONFLICT.getMsg()));
- } else if (qbp.getOrderByForClause(ctx_1.dest) != null) {
- throw new SemanticException(generateErrorMessage(ast,
- ErrorMsg.ORDERBY_SORTBY_CONFLICT.getMsg()));
- }
-
- break;
-
- case HiveParser.TOK_ORDERBY:
- // Get the order by aliases - these are aliased to the entries in the
- // select list
- queryProperties.setHasOrderBy(true);
- qbp.setOrderByExprForClause(ctx_1.dest, ast);
- if (qbp.getClusterByForClause(ctx_1.dest) != null) {
- throw new SemanticException(generateErrorMessage(ast,
- ErrorMsg.CLUSTERBY_ORDERBY_CONFLICT.getMsg()));
- }
- break;
-
- case HiveParser.TOK_GROUPBY:
- case HiveParser.TOK_ROLLUP_GROUPBY:
- case HiveParser.TOK_CUBE_GROUPBY:
- case HiveParser.TOK_GROUPING_SETS:
- // Get the groupby aliases - these are aliased to the entries in the
- // select list
- queryProperties.setHasGroupBy(true);
- if (qbp.getJoinExpr() != null) {
- queryProperties.setHasJoinFollowedByGroupBy(true);
- }
- if (qbp.getSelForClause(ctx_1.dest).getToken().getType() == HiveParser.TOK_SELECTDI) {
- throw new SemanticException(generateErrorMessage(ast,
- ErrorMsg.SELECT_DISTINCT_WITH_GROUPBY.getMsg()));
- }
- qbp.setGroupByExprForClause(ctx_1.dest, ast);
- skipRecursion = true;
-
- // Rollup and Cubes are syntactic sugar on top of grouping sets
- if (ast.getToken().getType() == HiveParser.TOK_ROLLUP_GROUPBY) {
- qbp.getDestRollups().add(ctx_1.dest);
- } else if (ast.getToken().getType() == HiveParser.TOK_CUBE_GROUPBY) {
- qbp.getDestCubes().add(ctx_1.dest);
- } else if (ast.getToken().getType() == HiveParser.TOK_GROUPING_SETS) {
- qbp.getDestGroupingSets().add(ctx_1.dest);
- }
- break;
-
- case HiveParser.TOK_HAVING:
- qbp.setHavingExprForClause(ctx_1.dest, ast);
- qbp.addAggregationExprsForClause(ctx_1.dest,
- doPhase1GetAggregationsFromSelect(ast, qb, ctx_1.dest));
- break;
-
- case HiveParser.KW_WINDOW:
- if (!qb.hasWindowingSpec(ctx_1.dest) ) {
- throw new SemanticException(generateErrorMessage(ast,
- "Query has no Cluster/Distribute By; but has a Window definition"));
- }
- handleQueryWindowClauses(qb, ctx_1, ast);
- break;
-
- case HiveParser.TOK_LIMIT:
- if (ast.getChildCount() == 2) {
- qbp.setDestLimit(ctx_1.dest,
- new Integer(ast.getChild(0).getText()),
- new Integer(ast.getChild(1).getText()));
- } else {
- qbp.setDestLimit(ctx_1.dest, new Integer(0),
- new Integer(ast.getChild(0).getText()));
- }
- break;
-
- case HiveParser.TOK_ANALYZE:
- // Case of analyze command
-
- String table_name = getUnescapedName((ASTNode) ast.getChild(0).getChild(0)).toLowerCase();
-
-
- qb.setTabAlias(table_name, table_name);
- qb.addAlias(table_name);
- qb.getParseInfo().setIsAnalyzeCommand(true);
- qb.getParseInfo().setNoScanAnalyzeCommand(this.noscan);
- qb.getParseInfo().setPartialScanAnalyzeCommand(this.partialscan);
- // Allow analyze the whole table and dynamic partitions
- HiveConf.setVar(conf, HiveConf.ConfVars.DYNAMICPARTITIONINGMODE, "nonstrict");
- HiveConf.setVar(conf, HiveConf.ConfVars.HIVEMAPREDMODE, "nonstrict");
-
- break;
-
- case HiveParser.TOK_UNIONALL:
- if (!qbp.getIsSubQ()) {
- // this shouldn't happen. The parser should have converted the union to be
- // contained in a subquery. Just in case, we keep the error as a fallback.
- throw new SemanticException(generateErrorMessage(ast,
- ErrorMsg.UNION_NOTIN_SUBQ.getMsg()));
- }
- skipRecursion = false;
- break;
-
- case HiveParser.TOK_INSERT:
- ASTNode destination = (ASTNode) ast.getChild(0);
- Tree tab = destination.getChild(0);
-
- // Proceed if AST contains partition & If Not Exists
- if (destination.getChildCount() == 2 &&
- tab.getChildCount() == 2 &&
- destination.getChild(1).getType() == HiveParser.TOK_IFNOTEXISTS) {
- String tableName = tab.getChild(0).getChild(0).getText();
-
- Tree partitions = tab.getChild(1);
- int childCount = partitions.getChildCount();
- HashMap<String, String> partition = new HashMap<String, String>();
- for (int i = 0; i < childCount; i++) {
- String partitionName = partitions.getChild(i).getChild(0).getText();
- Tree pvalue = partitions.getChild(i).getChild(1);
- if (pvalue == null) {
- break;
- }
- String partitionVal = stripQuotes(pvalue.getText());
- partition.put(partitionName, partitionVal);
- }
- // if it is a dynamic partition throw the exception
- if (childCount != partition.size()) {
- throw new SemanticException(ErrorMsg.INSERT_INTO_DYNAMICPARTITION_IFNOTEXISTS
- .getMsg(partition.toString()));
- }
- Table table = null;
- try {
- table = this.getTableObjectByName(tableName);
- } catch (HiveException ex) {
- throw new SemanticException(ex);
- }
- try {
- Partition parMetaData = db.getPartition(table, partition, false);
- // Check partition exists if it exists skip the overwrite
- if (parMetaData != null) {
- phase1Result = false;
- skipRecursion = true;
- LOG.info("Partition already exists so insert into overwrite " +
- "skipped for partition : " + parMetaData.toString());
- break;
- }
- } catch (HiveException e) {
- LOG.info("Error while getting metadata : ", e);
- }
- validatePartSpec(table, partition, (ASTNode)tab, conf, false);
- }
- skipRecursion = false;
- break;
- case HiveParser.TOK_LATERAL_VIEW:
- case HiveParser.TOK_LATERAL_VIEW_OUTER:
- // todo: nested LV
- assert ast.getChildCount() == 1;
- qb.getParseInfo().getDestToLateralView().put(ctx_1.dest, ast);
- break;
- case HiveParser.TOK_CTE:
- processCTE(qb, ast);
- break;
- default:
- skipRecursion = false;
- break;
- }
- }
-
- if (!skipRecursion) {
- // Iterate over the rest of the children
- int child_count = ast.getChildCount();
- for (int child_pos = 0; child_pos < child_count && phase1Result; ++child_pos) {
- // Recurse
- phase1Result = phase1Result && doPhase1(
- (ASTNode)ast.getChild(child_pos), qb, ctx_1, plannerCtx);
- }
- }
- return phase1Result;
- }
-
- /**
- * This is phase1 of supporting specifying schema in insert statement
- * insert into foo(z,y) select a,b from bar;
- * @see #handleInsertStatementSpec(java.util.List, String, RowResolver, RowResolver, QB, ASTNode)
- * @throws SemanticException
- */
- private void handleInsertStatementSpecPhase1(ASTNode ast, QBParseInfo qbp, Phase1Ctx ctx_1) throws SemanticException {
- ASTNode tabColName = (ASTNode)ast.getChild(1);
- if(ast.getType() == HiveParser.TOK_INSERT_INTO && tabColName != null && tabColName.getType() == HiveParser.TOK_TABCOLNAME) {
- //we have "insert into foo(a,b)..."; parser will enforce that 1+ columns are listed if TOK_TABCOLNAME is present
- List<String> targetColNames = new ArrayList<String>();
- for(Node col : tabColName.getChildren()) {
- assert ((ASTNode)col).getType() == HiveParser.Identifier :
- "expected token " + HiveParser.Identifier + " found " + ((ASTNode)col).getType();
- targetColNames.add(((ASTNode)col).getText());
- }
- String fullTableName = getUnescapedName((ASTNode) ast.getChild(0).getChild(0),
- SessionState.get().getCurrentDatabase());
- qbp.setDestSchemaForClause(ctx_1.dest, targetColNames);
- Set<String> targetColumns = new HashSet<String>();
- targetColumns.addAll(targetColNames);
- if(targetColNames.size() != targetColumns.size()) {
- throw new SemanticException(generateErrorMessage(tabColName,
- "Duplicate column name detected in " + fullTableName + " table schema specification"));
- }
- Table targetTable = null;
- try {
- targetTable = db.getTable(fullTableName, false);
- }
- catch (HiveException ex) {
- LOG.error("Error processing HiveParser.TOK_DESTINATION: " + ex.getMessage(), ex);
- throw new SemanticException(ex);
- }
- if(targetTable == null) {
- throw new SemanticException(generateErrorMessage(ast,
- "Unable to access metadata for table " + fullTableName));
- }
- for(FieldSchema f : targetTable.getCols()) {
- //parser only allows foo(a,b), not foo(foo.a, foo.b)
- targetColumns.remove(f.getName());
- }
- if(!targetColumns.isEmpty()) {//here we need to see if remaining columns are dynamic partition columns
- /* We just checked the user specified schema columns among regular table column and found some which are not
- 'regular'. Now check is they are dynamic partition columns
- For dynamic partitioning,
- Given "create table multipart(a int, b int) partitioned by (c int, d int);"
- for "insert into multipart partition(c='1',d)(d,a) values(2,3);" we expect parse tree to look like this
- (TOK_INSERT_INTO
- (TOK_TAB
- (TOK_TABNAME multipart)
- (TOK_PARTSPEC
- (TOK_PARTVAL c '1')
- (TOK_PARTVAL d)
- )
- )
- (TOK_TABCOLNAME d a)
- )*/
- List<String> dynamicPartitionColumns = new ArrayList<String>();
- if(ast.getChild(0) != null && ast.getChild(0).getType() == HiveParser.TOK_TAB) {
- ASTNode tokTab = (ASTNode)ast.getChild(0);
- ASTNode tokPartSpec = (ASTNode)tokTab.getFirstChildWithType(HiveParser.TOK_PARTSPEC);
- if(tokPartSpec != null) {
- for(Node n : tokPartSpec.getChildren()) {
- ASTNode tokPartVal = null;
- if(n instanceof ASTNode) {
- tokPartVal = (ASTNode)n;
- }
- if(tokPartVal != null && tokPartVal.getType() == HiveParser.TOK_PARTVAL && tokPartVal.getChildCount() == 1) {
- assert tokPartVal.getChild(0).getType() == HiveParser.Identifier :
- "Expected column name; found tokType=" + tokPartVal.getType();
- dynamicPartitionColumns.add(tokPartVal.getChild(0).getText());
- }
- }
- }
- }
- for(String colName : dynamicPartitionColumns) {
- targetColumns.remove(colName);
- }
- if(!targetColumns.isEmpty()) {
- //Found some columns in user specified schema which are neither regular not dynamic partition columns
- throw new SemanticException(generateErrorMessage(tabColName,
- "'" + (targetColumns.size() == 1 ? targetColumns.iterator().next() : targetColumns) +
- "' in insert schema specification " + (targetColumns.size() == 1 ? "is" : "are") +
- " not found among regular columns of " +
- fullTableName + " nor dynamic partition columns."));
- }
- }
- }
- }
-
- public void getMaterializationMetadata(QB qb) throws SemanticException {
- try {
- gatherCTEReferences(qb, rootClause);
- int threshold = HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVE_CTE_MATERIALIZE_THRESHOLD);
- for (CTEClause cte : Sets.newHashSet(aliasToCTEs.values())) {
- if (threshold >= 0 && cte.reference >= threshold) {
- cte.materialize = true;
- }
- }
- } catch (HiveException e) {
- // Has to use full name to make sure it does not conflict with
- // org.apache.commons.lang.StringUtils
- LOG.error(org.apache.hadoop.util.StringUtils.stringifyException(e));
- if (e instanceof SemanticException) {
- throw (SemanticException)e;
- }
- throw new SemanticException(e.getMessage(), e);
- }
- }
-
- private void gatherCTEReferences(QBExpr qbexpr, CTEClause parent) throws HiveException {
- if (qbexpr.getOpcode() == QBExpr.Opcode.NULLOP) {
- gatherCTEReferences(qbexpr.getQB(), parent);
- } else {
- gatherCTEReferences(qbexpr.getQBExpr1(), parent);
- gatherCTEReferences(qbexpr.getQBExpr2(), parent);
- }
- }
-
- // TODO: check view references, too
- private void gatherCTEReferences(QB qb, CTEClause current) throws HiveException {
- for (String alias : qb.getTabAliases()) {
- String tabName = qb.getTabNameForAlias(alias);
- String cteName = tabName.toLowerCase();
-
- CTEClause cte = findCTEFromName(qb, cteName);
- if (cte != null) {
- if (ctesExpanded.contains(cteName)) {
- throw new SemanticException("Recursive cte " + cteName +
- " detected (cycle: " + StringUtils.join(ctesExpanded, " -> ") +
- " -> " + cteName + ").");
- }
- cte.reference++;
- current.parents.add(cte);
- if (cte.qbExpr != null) {
- continue;
- }
- cte.qbExpr = new QBExpr(cteName);
- doPhase1QBExpr(cte.cteNode, cte.qbExpr, qb.getId(), cteName);
-
- ctesExpanded.add(cteName);
- gatherCTEReferences(cte.qbExpr, cte);
- ctesExpanded.remove(ctesExpanded.size() - 1);
- }
- }
- for (String alias : qb.getSubqAliases()) {
- gatherCTEReferences(qb.getSubqForAlias(alias), current);
- }
- }
-
- public void getMetaData(QB qb) throws SemanticException {
- getMetaData(qb, false);
- }
-
- public void getMetaData(QB qb, boolean enableMaterialization) throws SemanticException {
- try {
- if (enableMaterialization) {
- getMaterializationMetadata(qb);
- }
- getMetaData(qb, null);
- } catch (HiveException e) {
- // Has to use full name to make sure it does not conflict with
- // org.apache.commons.lang.StringUtils
- LOG.error(org.apache.hadoop.util.StringUtils.stringifyException(e));
- if (e instanceof SemanticException) {
- throw (SemanticException)e;
- }
- throw new SemanticException(e.getMessage(), e);
- }
- }
-
- private void getMetaData(QBExpr qbexpr, ReadEntity parentInput)
- throws HiveException {
- if (qbexpr.getOpcode() == QBExpr.Opcode.NULLOP) {
- getMetaData(qbexpr.getQB(), parentInput);
- } else {
- getMetaData(qbexpr.getQBExpr1(), parentInput);
- getMetaData(qbexpr.getQBExpr2(), parentInput);
- }
- }
-
- @SuppressWarnings("nls")
- private void getMetaData(QB qb, ReadEntity parentInput)
- throws HiveException {
- LOG.info("Get metadata for source tables");
-
- // Go over the tables and populate the related structures.
- // We have to materialize the table alias list since we might
- // modify it in the middle for view rewrite.
- List<String> tabAliases = new ArrayList<String>(qb.getTabAliases());
-
- // Keep track of view alias to view name and read entity
- // For eg: for a query like 'select * from V3', where V3 -> V2, V2 -> V1, V1 -> T
- // keeps track of full view name and read entity corresponding to alias V3, V3:V2, V3:V2:V1.
- // This is needed for tracking the dependencies for inputs, along with their parents.
- Map<String, ObjectPair<String, ReadEntity>> aliasToViewInfo =
- new HashMap<String, ObjectPair<String, ReadEntity>>();
-
- /*
- * used to capture view to SQ conversions. This is used to check for
- * recursive CTE invocations.
- */
- Map<String, String> sqAliasToCTEName = new HashMap<String, String>();
-
- for (String alias : tabAliases) {
- String tabName = qb.getTabNameForAlias(alias);
- String cteName = tabName.toLowerCase();
-
- Table tab = db.getTable(tabName, false);
- if (tab == null ||
- tab.getDbName().equals(SessionState.get().getCurrentDatabase())) {
- Table materializedTab = ctx.getMaterializedTable(cteName);
- if (materializedTab == null) {
- // we first look for this alias from CTE, and then from catalog.
- CTEClause cte = findCTEFromName(qb, cteName);
- if (cte != null) {
- if (!cte.materialize) {
- addCTEAsSubQuery(qb, cteName, alias);
- sqAliasToCTEName.put(alias, cteName);
- continue;
- }
- tab = materializeCTE(cteName, cte);
- }
- } else {
- tab = materializedTab;
- }
- }
-
- if (tab == null) {
- ASTNode src = qb.getParseInfo().getSrcForAlias(alias);
- if (null != src) {
- throw new SemanticException(ErrorMsg.INVALID_TABLE.getMsg(src));
- } else {
- throw new SemanticException(ErrorMsg.INVALID_TABLE.getMsg(alias));
- }
- }
-
- // Disallow INSERT INTO on bucketized tables
- boolean isAcid = AcidUtils.isAcidTable(tab);
- boolean isTableWrittenTo = qb.getParseInfo().isInsertIntoTable(tab.getDbName(), tab.getTableName());
- if (isTableWrittenTo &&
- tab.getNumBuckets() > 0 && !isAcid) {
- throw new SemanticException(ErrorMsg.INSERT_INTO_BUCKETIZED_TABLE.
- getMsg("Table: " + tabName));
- }
- // Disallow update and delete on non-acid tables
- if ((updating() || deleting()) && !isAcid && isTableWrittenTo) {
- //isTableWrittenTo: delete from acidTbl where a in (select id from nonAcidTable)
- //so only assert this if we are actually writing to this table
- // Whether we are using an acid compliant transaction manager has already been caught in
- // UpdateDeleteSemanticAnalyzer, so if we are updating or deleting and getting nonAcid
- // here, it means the table itself doesn't support it.
- throw new SemanticException(ErrorMsg.ACID_OP_ON_NONACID_TABLE, tabName);
- }
-
- if (tab.isView()) {
- if (qb.getParseInfo().isAnalyzeCommand()) {
- throw new SemanticException(ErrorMsg.ANALYZE_VIEW.getMsg());
- }
- String fullViewName = tab.getDbName() + "." + tab.getTableName();
- // Prevent view cycles
- if (viewsExpanded.contains(fullViewName)) {
- throw new SemanticException("Recursive view " + fullViewName +
- " detected (cycle: " + StringUtils.join(viewsExpanded, " -> ") +
- " -> " + fullViewName + ").");
- }
- replaceViewReferenceWithDefinition(qb, tab, tabName, alias);
- // This is the last time we'll see the Table objects for views, so add it to the inputs
- // now. isInsideView will tell if this view is embedded in another view.
- ReadEntity viewInput = new ReadEntity(tab, parentInput, !qb.isInsideView());
- viewInput = PlanUtils.addInput(inputs, viewInput);
- aliasToViewInfo.put(alias, new ObjectPair<String, ReadEntity>(fullViewName, viewInput));
- String aliasId = getAliasId(alias, qb);
- if (aliasId != null) {
- aliasId = aliasId.replace(SemanticAnalyzer.SUBQUERY_TAG_1, "")
- .replace(SemanticAnalyzer.SUBQUERY_TAG_2, "");
- }
- viewAliasToInput.put(aliasId, viewInput);
- continue;
- }
-
- if (!InputFormat.class.isAssignableFrom(tab.getInputFormatClass())) {
- throw new SemanticException(generateErrorMessage(
- qb.getParseInfo().getSrcForAlias(alias),
- ErrorMsg.INVALID_INPUT_FORMAT_TYPE.getMsg()));
- }
-
- qb.getMetaData().setSrcForAlias(alias, tab);
-
- if (qb.getParseInfo().isAnalyzeCommand()) {
- // allow partial partition specification for nonscan since noscan is fast.
- TableSpec ts = new TableSpec(db, conf, (ASTNode) ast.getChild(0), true, this.noscan);
- if (ts.specType == SpecType.DYNAMIC_PARTITION) { // dynamic partitions
- try {
- ts.partitions = db.getPartitionsByNames(ts.tableHandle, ts.partSpec);
- } catch (HiveException e) {
- throw new SemanticException(generateErrorMessage(
- qb.getParseInfo().getSrcForAlias(alias),
- "Cannot get partitions for " + ts.partSpec), e);
- }
- }
- // validate partial scan command
- QBParseInfo qbpi = qb.getParseInfo();
- if (qbpi.isPartialScanAnalyzeCommand()) {
- Class<? extends InputFormat> inputFormatClass = null;
- switch (ts.specType) {
- case TABLE_ONLY:
- case DYNAMIC_PARTITION:
- inputFormatClass = ts.tableHandle.getInputFormatClass();
- break;
- case STATIC_PARTITION:
- inputFormatClass = ts.partHandle.getInputFormatClass();
- break;
- default:
- assert false;
- }
- // throw a HiveException for formats other than rcfile or orcfile.
- if (!(inputFormatClass.equals(RCFileInputFormat.class) || inputFormatClass
- .equals(OrcInputFormat.class))) {
- throw new SemanticException(ErrorMsg.ANALYZE_TABLE_PARTIALSCAN_NON_RCFILE.getMsg());
- }
- }
-
- tab.setTableSpec(ts);
- qb.getParseInfo().addTableSpec(alias, ts);
- }
-
- ReadEntity parentViewInfo = PlanUtils.getParentViewInfo(getAliasId(alias, qb), viewAliasToInput);
- PlanUtils.addInput(inputs,
- new ReadEntity(tab, parentViewInfo, parentViewInfo == null),mergeIsDirect);
- }
-
- LOG.info("Get metadata for subqueries");
- // Go over the subqueries and getMetaData for these
- for (String alias : qb.getSubqAliases()) {
- boolean wasView = aliasToViewInfo.containsKey(alias);
- boolean wasCTE = sqAliasToCTEName.containsKey(alias);
- ReadEntity newParentInput = null;
- if (wasView) {
- viewsExpanded.add(aliasToViewInfo.get(alias).getFirst());
- newParentInput = aliasToViewInfo.get(alias).getSecond();
- } else if (wasCTE) {
- ctesExpanded.add(sqAliasToCTEName.get(alias));
- }
- QBExpr qbexpr = qb.getSubqForAlias(alias);
- getMetaData(qbexpr, newParentInput);
- if (wasView) {
- viewsExpanded.remove(viewsExpanded.size() - 1);
- } else if (wasCTE) {
- ctesExpanded.remove(ctesExpanded.size() - 1);
- }
- }
-
- RowFormatParams rowFormatParams = new RowFormatParams();
- StorageFormat storageFormat = new StorageFormat(conf);
-
- LOG.info("Get metadata for destination tables");
- // Go over all the destination structures and populate the related
- // metadata
- QBParseInfo qbp = qb.getParseInfo();
-
- for (String name : qbp.getClauseNamesForDest()) {
- ASTNode ast = qbp.getDestForClause(name);
- switch (ast.getToken().getType()) {
- case HiveParser.TOK_TAB: {
- TableSpec ts = new TableSpec(db, conf, ast);
- if (ts.tableHandle.isView()) {
- throw new SemanticException(ErrorMsg.DML_AGAINST_VIEW.getMsg());
- }
-
- Class<?> outputFormatClass = ts.tableHandle.getOutputFormatClass();
- if (!ts.tableHandle.isNonNative() &&
- !HiveOutputFormat.class.isAssignableFrom(outputFormatClass)) {
- throw new SemanticException(ErrorMsg.INVALID_OUTPUT_FORMAT_TYPE
- .getMsg(ast, "The class is " + outputFormatClass.toString()));
- }
-
- // TableSpec ts is got from the query (user specified),
- // which means the user didn't specify partitions in their query,
- // but whether the table itself is partitioned is not know.
- if (ts.specType != SpecType.STATIC_PARTITION) {
- // This is a table or dynamic partition
- qb.getMetaData().setDestForAlias(name, ts.tableHandle);
- // has dynamic as well as static partitions
- if (ts.partSpec != null && ts.partSpec.size() > 0) {
- qb.getMetaData().setPartSpecForAlias(name, ts.partSpec);
- }
- } else {
- // This is a partition
- qb.getMetaData().setDestForAlias(name, ts.partHandle);
- }
- if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVESTATSAUTOGATHER)) {
- // Add the table spec for the destination table.
- qb.getParseInfo().addTableSpec(ts.tableName.toLowerCase(), ts);
- }
- break;
- }
-
- case HiveParser.TOK_DIR: {
- // This is a dfs file
- String fname = stripQuotes(ast.getChild(0).getText());
- if ((!qb.getParseInfo().getIsSubQ())
- && (((ASTNode) ast.getChild(0)).getToken().getType() == HiveParser.TOK_TMP_FILE)) {
-
- if (qb.isCTAS()) {
- qb.setIsQuery(false);
- ctx.setResDir(null);
- ctx.setResFile(null);
-
- // allocate
<TRUNCATED>
[35/50] [abbrv] hive git commit: HIVE-15955: make explain formatted
to include opId and etc (Pengcheng Xiong, reviewed by Ashutosh Chauhan)
Posted by om...@apache.org.
HIVE-15955: make explain formatted to include opId and etc (Pengcheng Xiong, reviewed by Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e27199b9
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e27199b9
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e27199b9
Branch: refs/heads/branch-2.2
Commit: e27199b9dde7ee489def5748ccc27b3344739fc1
Parents: 44e4998
Author: Pengcheng Xiong <px...@apache.org>
Authored: Wed Feb 22 20:10:19 2017 -0800
Committer: Owen O'Malley <om...@apache.org>
Committed: Tue Mar 28 15:27:54 2017 -0700
----------------------------------------------------------------------
.../apache/hadoop/hive/ql/exec/ExplainTask.java | 10 ++-
.../AnnotateReduceSinkOutputOperator.java | 73 ++++++++++++++++++++
.../hadoop/hive/ql/optimizer/Optimizer.java | 4 ++
.../hadoop/hive/ql/plan/ReduceSinkDesc.java | 13 ++++
.../clientpositive/explain_formatted_oid.q | 18 +++++
.../clientpositive/explain_formatted_oid.q.out | 38 ++++++++++
ql/src/test/results/clientpositive/input4.q.out | 2 +-
ql/src/test/results/clientpositive/join0.q.out | 2 +-
.../results/clientpositive/parallel_join0.q.out | 2 +-
.../test/results/clientpositive/plan_json.q.out | 2 +-
10 files changed, 159 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/e27199b9/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java
index fb39188..fe5eccf 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java
@@ -75,6 +75,7 @@ import org.slf4j.LoggerFactory;
public class ExplainTask extends Task<ExplainWork> implements Serializable {
private static final long serialVersionUID = 1L;
public static final String EXPL_COLUMN_NAME = "Explain";
+ public static final String OUTPUT_OPERATORS = "OutputOperators:";
private final Set<Operator<?>> visitedOps = new HashSet<Operator<?>>();
private boolean isLogical = false;
protected final Logger LOG;
@@ -598,10 +599,17 @@ public class ExplainTask extends Task<ExplainWork> implements Serializable {
String appender = isLogical ? " (" + operator.getOperatorId() + ")" : "";
JSONObject jsonOut = outputPlan(operator.getConf(), out, extended,
jsonOutput, jsonOutput ? 0 : indent, appender);
- if (this.work != null && this.work.isUserLevelExplain()) {
+ if (this.work != null && (this.work.isUserLevelExplain() || this.work.isFormatted())) {
if (jsonOut != null && jsonOut.length() > 0) {
((JSONObject) jsonOut.get(JSONObject.getNames(jsonOut)[0])).put("OperatorId:",
operator.getOperatorId());
+ if (!this.work.isUserLevelExplain() && this.work.isFormatted()
+ && operator instanceof ReduceSinkOperator) {
+ ((JSONObject) jsonOut.get(JSONObject.getNames(jsonOut)[0])).put(
+ OUTPUT_OPERATORS,
+ Arrays.toString(((ReduceSinkOperator) operator).getConf().getOutputOperators()
+ .toArray()));
+ }
}
}
if (jsonOutput) {
http://git-wip-us.apache.org/repos/asf/hive/blob/e27199b9/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AnnotateReduceSinkOutputOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AnnotateReduceSinkOutputOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AnnotateReduceSinkOutputOperator.java
new file mode 100644
index 0000000..0b61f4b
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AnnotateReduceSinkOutputOperator.java
@@ -0,0 +1,73 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.optimizer;
+
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Stack;
+
+import org.apache.hadoop.hive.ql.exec.Operator;
+import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator;
+import org.apache.hadoop.hive.ql.lib.DefaultGraphWalker;
+import org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher;
+import org.apache.hadoop.hive.ql.lib.GraphWalker;
+import org.apache.hadoop.hive.ql.lib.Node;
+import org.apache.hadoop.hive.ql.lib.NodeProcessor;
+import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
+import org.apache.hadoop.hive.ql.lib.Rule;
+import org.apache.hadoop.hive.ql.lib.RuleRegExp;
+import org.apache.hadoop.hive.ql.parse.ParseContext;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.plan.OperatorDesc;
+
+/**
+ * Implementation of AnnotateReduceSinkOutputOperator optimization step.
+ */
+public class AnnotateReduceSinkOutputOperator extends Transform {
+ @Override
+ public ParseContext transform(ParseContext pctx) throws SemanticException {
+
+ // 1. We apply the transformation
+ Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>();
+ opRules.put(new RuleRegExp("R1",
+ "(" + ReduceSinkOperator.getOperatorName() + "%)"), new ReduceSinkOutputOperatorAnnotator());
+ GraphWalker ogw = new DefaultGraphWalker(new DefaultRuleDispatcher(null, opRules, null));
+ ArrayList<Node> topNodes = new ArrayList<Node>();
+ topNodes.addAll(pctx.getTopOps().values());
+ ogw.startWalking(topNodes, null);
+ return pctx;
+ }
+
+ private static class ReduceSinkOutputOperatorAnnotator implements NodeProcessor {
+ @Override
+ public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx,
+ Object... nodeOutputs) throws SemanticException {
+ ReduceSinkOperator rs = (ReduceSinkOperator) nd;
+ List<Operator<? extends OperatorDesc>> children = rs.getChildOperators();
+ List<String> outputOperators = new ArrayList<>();
+ for (Operator<? extends OperatorDesc> operator : children) {
+ outputOperators.add(operator.getOperatorId());
+ }
+ rs.getConf().setOutputOperators(outputOperators);
+ return null;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/e27199b9/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java
index c6287e4..a3a19f4 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java
@@ -232,6 +232,10 @@ public class Optimizer {
transformations.add(new SimpleFetchAggregation());
}
+ if (pctx.getContext().getExplainConfig() != null
+ && pctx.getContext().getExplainConfig().isFormatted()) {
+ transformations.add(new AnnotateReduceSinkOutputOperator());
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/hive/blob/e27199b9/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java
index d7e404c..cd24894 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java
@@ -73,6 +73,12 @@ public class ReduceSinkDesc extends AbstractOperatorDesc {
private String outputName;
/**
+ * Holds the name of the output operators
+ * that this reduce sink is outputing to.
+ */
+ private List<String> outputOperators;
+
+ /**
* The partition columns (CLUSTER BY or DISTRIBUTE BY in Hive language).
* Partition columns decide the reducer that the current row goes to.
* Partition columns are not passed to reducer.
@@ -490,4 +496,11 @@ public class ReduceSinkDesc extends AbstractOperatorDesc {
this.hasOrderBy = hasOrderBy;
}
+ public List<String> getOutputOperators() {
+ return outputOperators;
+ }
+
+ public void setOutputOperators(List<String> outputOperators) {
+ this.outputOperators = outputOperators;
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/e27199b9/ql/src/test/queries/clientpositive/explain_formatted_oid.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/explain_formatted_oid.q b/ql/src/test/queries/clientpositive/explain_formatted_oid.q
new file mode 100644
index 0000000..932f119
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/explain_formatted_oid.q
@@ -0,0 +1,18 @@
+set hive.auto.convert.join=false;
+
+create table srcTable (key string, value string);
+
+explain formatted
+SELECT x.key, z.value, y.value
+FROM srcTable x JOIN srcTable y ON (x.key = y.key)
+JOIN srcTable z ON (x.value = z.value);
+
+explain formatted
+SELECT x.key, z.value, y.value
+FROM srcTable x JOIN srcTable y ON (x.key = y.key)
+JOIN (select * from srcTable union select * from srcTable)z ON (x.value = z.value)
+union
+SELECT x.key, z.value, y.value
+FROM srcTable x JOIN srcTable y ON (x.key = y.key)
+JOIN (select * from srcTable union select * from srcTable)z ON (x.value = z.value);
+
http://git-wip-us.apache.org/repos/asf/hive/blob/e27199b9/ql/src/test/results/clientpositive/explain_formatted_oid.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/explain_formatted_oid.q.out b/ql/src/test/results/clientpositive/explain_formatted_oid.q.out
new file mode 100644
index 0000000..46c2090
--- /dev/null
+++ b/ql/src/test/results/clientpositive/explain_formatted_oid.q.out
@@ -0,0 +1,38 @@
+PREHOOK: query: create table srcTable (key string, value string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@srcTable
+POSTHOOK: query: create table srcTable (key string, value string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@srcTable
+PREHOOK: query: explain formatted
+SELECT x.key, z.value, y.value
+FROM srcTable x JOIN srcTable y ON (x.key = y.key)
+JOIN srcTable z ON (x.value = z.value)
+PREHOOK: type: QUERY
+POSTHOOK: query: explain formatted
+SELECT x.key, z.value, y.value
+FROM srcTable x JOIN srcTable y ON (x.key = y.key)
+JOIN srcTable z ON (x.value = z.value)
+POSTHOOK: type: QUERY
+{"STAGE DEPENDENCIES":{"Stage-1":{"ROOT STAGE":"TRUE"},"Stage-2":{"DEPENDENT STAGES":"Stage-1"},"Stage-0":{"DEPENDENT STAGES":"Stage-2"}},"STAGE PLANS":{"Stage-1":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"x","Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"TS_0","children":{"Filter Operator":{"predicate:":"(key is not null and value is not null) (type: boolean)","Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"FIL_17","children":{"Select Operator":{"expressions:":"key (type: string), value (type: string)","outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"SEL_2","children":{"Reduce Output Operator":{"key expressions:":"_col0 (type: string)","sort order:":"+","Map-reduce partition columns:":"_col0 (type: string)","Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NON
E","value expressions:":"_col1 (type: string)","OperatorId:":"RS_9","OutputOperators:":"[JOIN_11]"}}}}}}}},{"TableScan":{"alias:":"y","Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"TS_3","children":{"Filter Operator":{"predicate:":"key is not null (type: boolean)","Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"FIL_18","children":{"Select Operator":{"expressions:":"key (type: string), value (type: string)","outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"SEL_5","children":{"Reduce Output Operator":{"key expressions:":"_col0 (type: string)","sort order:":"+","Map-reduce partition columns:":"_col0 (type: string)","Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","value expressions:":"_col1 (type: string)","OperatorId:":"RS_10","OutputOperators:":"[JOIN_11]"}}}}}}}}],"Reduce
Operator Tree:":{"Join Operator":{"condition map:":[{"":"Inner Join 0 to 1"}],"keys:":{"0":"_col0 (type: string)","1":"_col0 (type: string)"},"outputColumnNames:":["_col0","_col1","_col3"],"Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"JOIN_11","children":{"File Output Operator":{"compressed:":"false","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe"},"OperatorId:":"FS_20"}}}}}},"Stage-2":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"OperatorId:":"TS_21","children":{"Reduce Output Operator":{"key expressions:":"_col1 (type: string)","sort order:":"+","Map-reduce partition columns:":"_col1 (type: string)","Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","value expressions:":"_col0 (type: string), _col3 (type: string)","OperatorId
:":"RS_12","OutputOperators:":"[JOIN_14]"}}}},{"TableScan":{"alias:":"z","Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"TS_6","children":{"Filter Operator":{"predicate:":"value is not null (type: boolean)","Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"FIL_19","children":{"Select Operator":{"expressions:":"value (type: string)","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"SEL_8","children":{"Reduce Output Operator":{"key expressions:":"_col0 (type: string)","sort order:":"+","Map-reduce partition columns:":"_col0 (type: string)","Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"RS_13","OutputOperators:":"[JOIN_14]"}}}}}}}}],"Reduce Operator Tree:":{"Join Operator":{"condition map:":[{"":"Inner Join 0 to 1"}],"keys:":{"0":"_col1 (type: string)","1":"_col0 (type:
string)"},"outputColumnNames:":["_col0","_col3","_col4"],"Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"JOIN_14","children":{"Select Operator":{"expressions:":"_col0 (type: string), _col4 (type: string), _col3 (type: string)","outputColumnNames:":["_col0","_col1","_col2"],"Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"SEL_15","children":{"File Output Operator":{"compressed:":"false","Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_16"}}}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_22"}}}}}}
+PREHOOK: query: explain formatted
+SELECT x.key, z.value, y.value
+FROM srcTable x JOIN srcTable y ON (x.key = y.key)
+JOIN (select * from srcTable union select * from srcTable)z ON (x.value = z.value)
+union
+SELECT x.key, z.value, y.value
+FROM srcTable x JOIN srcTable y ON (x.key = y.key)
+JOIN (select * from srcTable union select * from srcTable)z ON (x.value = z.value)
+PREHOOK: type: QUERY
+POSTHOOK: query: explain formatted
+SELECT x.key, z.value, y.value
+FROM srcTable x JOIN srcTable y ON (x.key = y.key)
+JOIN (select * from srcTable union select * from srcTable)z ON (x.value = z.value)
+union
+SELECT x.key, z.value, y.value
+FROM srcTable x JOIN srcTable y ON (x.key = y.key)
+JOIN (select * from srcTable union select * from srcTable)z ON (x.value = z.value)
+POSTHOOK: type: QUERY
+{"STAGE DEPENDENCIES":{"Stage-1":{"ROOT STAGE":"TRUE"},"Stage-2":{"DEPENDENT STAGES":"Stage-1, Stage-5"},"Stage-3":{"DEPENDENT STAGES":"Stage-2, Stage-8"},"Stage-5":{"ROOT STAGE":"TRUE"},"Stage-7":{"ROOT STAGE":"TRUE"},"Stage-8":{"DEPENDENT STAGES":"Stage-7, Stage-10"},"Stage-10":{"ROOT STAGE":"TRUE"},"Stage-0":{"DEPENDENT STAGES":"Stage-3"}},"STAGE PLANS":{"Stage-1":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"x","Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"TS_0","children":{"Filter Operator":{"predicate:":"(key is not null and value is not null) (type: boolean)","Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"FIL_60","children":{"Select Operator":{"expressions:":"key (type: string), value (type: string)","outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"SEL_2","children":{"Reduce Outp
ut Operator":{"key expressions:":"_col0 (type: string)","sort order:":"+","Map-reduce partition columns:":"_col0 (type: string)","Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","value expressions:":"_col1 (type: string)","OperatorId:":"RS_19","OutputOperators:":"[JOIN_21]"}}}}}}}},{"TableScan":{"alias:":"y","Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"TS_3","children":{"Filter Operator":{"predicate:":"key is not null (type: boolean)","Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"FIL_61","children":{"Select Operator":{"expressions:":"key (type: string)","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"SEL_5","children":{"Reduce Output Operator":{"key expressions:":"_col0 (type: string)","sort order:":"+","Map-reduce partition columns:":"_col0 (type: string)","Statistics:":"Nu
m rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"RS_20","OutputOperators:":"[JOIN_21]"}}}}}}}}],"Reduce Operator Tree:":{"Join Operator":{"condition map:":[{"":"Inner Join 0 to 1"}],"keys:":{"0":"_col0 (type: string)","1":"_col0 (type: string)"},"outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"JOIN_21","children":{"File Output Operator":{"compressed:":"false","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe"},"OperatorId:":"FS_68"}}}}}},"Stage-2":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"OperatorId:":"TS_69","children":{"Reduce Output Operator":{"key expressions:":"_col1 (type: string)","sort order:":"+","Map-reduce partition columns:":"_col1 (type: string)","Statistics:":"Num rows: 1 Data
size: 0 Basic stats: PARTIAL Column stats: NONE","value expressions:":"_col0 (type: string)","OperatorId:":"RS_22","OutputOperators:":"[JOIN_24]"}}}},{"TableScan":{"OperatorId:":"TS_73","children":{"Reduce Output Operator":{"key expressions:":"_col1 (type: string)","sort order:":"+","Map-reduce partition columns:":"_col1 (type: string)","Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"RS_23","OutputOperators:":"[JOIN_24]"}}}}],"Reduce Operator Tree:":{"Join Operator":{"condition map:":[{"":"Inner Join 0 to 1"}],"keys:":{"0":"_col1 (type: string)","1":"_col1 (type: string)"},"outputColumnNames:":["_col0","_col4"],"Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"JOIN_24","children":{"Select Operator":{"expressions:":"_col0 (type: string), _col4 (type: string)","outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":
"SEL_25","children":{"File Output Operator":{"compressed:":"false","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe"},"OperatorId:":"FS_70"}}}}}}}},"Stage-3":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"OperatorId:":"TS_71","children":{"Union":{"Statistics:":"Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"UNION_52","children":{"Group By Operator":{"keys:":"_col0 (type: string), _col1 (type: string)","mode:":"hash","outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"GBY_55","children":{"Reduce Output Operator":{"key expressions:":"_col0 (type: string), _col1 (type: string)","sort order:":"++","Map-reduce partition columns:":"_col0 (type: string), _col1 (type: string)","Statistics:":"Num rows:
2 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"RS_56","OutputOperators:":"[GBY_57]"}}}}}}}},{"TableScan":{"OperatorId:":"TS_77","children":{"Union":{"Statistics:":"Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"UNION_52","children":{"Group By Operator":{"keys:":"_col0 (type: string), _col1 (type: string)","mode:":"hash","outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"GBY_55","children":{"Reduce Output Operator":{"key expressions:":"_col0 (type: string), _col1 (type: string)","sort order:":"++","Map-reduce partition columns:":"_col0 (type: string), _col1 (type: string)","Statistics:":"Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"RS_56","OutputOperators:":"[GBY_57]"}}}}}}}}],"Reduce Operator Tree:":{"Group By Operator":{"keys:":"KEY._col0 (type: string), KEY._col1 (type: string)","mode:":"mergepartial","out
putColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"GBY_57","children":{"File Output Operator":{"compressed:":"false","Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_59"}}}}}},"Stage-5":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"srctable","Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"TS_6","children":{"Filter Operator":{"predicate:":"value is not null (type: boolean)","Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"FIL_62","children":{"Select Operator":{"expressions:":"key (type: string), value (type: string)","outputColumnNames
:":["_col0","_col1"],"Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"SEL_8","children":{"Union":{"Statistics:":"Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"UNION_12","children":{"Group By Operator":{"keys:":"_col1 (type: string), _col0 (type: string)","mode:":"hash","outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"GBY_15","children":{"Reduce Output Operator":{"key expressions:":"_col0 (type: string), _col1 (type: string)","sort order:":"++","Map-reduce partition columns:":"_col0 (type: string), _col1 (type: string)","Statistics:":"Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"RS_16","OutputOperators:":"[GBY_17]"}}}}}}}}}}}},{"TableScan":{"alias:":"srctable","Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"TS_9","children":{"Filter Ope
rator":{"predicate:":"value is not null (type: boolean)","Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"FIL_63","children":{"Select Operator":{"expressions:":"key (type: string), value (type: string)","outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"SEL_11","children":{"Union":{"Statistics:":"Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"UNION_12","children":{"Group By Operator":{"keys:":"_col1 (type: string), _col0 (type: string)","mode:":"hash","outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"GBY_15","children":{"Reduce Output Operator":{"key expressions:":"_col0 (type: string), _col1 (type: string)","sort order:":"++","Map-reduce partition columns:":"_col0 (type: string), _col1 (type: string)","Statistics:":"Num rows: 2 Data siz
e: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"RS_16","OutputOperators:":"[GBY_17]"}}}}}}}}}}}}],"Reduce Operator Tree:":{"Group By Operator":{"keys:":"KEY._col0 (type: string), KEY._col1 (type: string)","mode:":"mergepartial","outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"GBY_17","children":{"Select Operator":{"expressions:":"_col0 (type: string)","outputColumnNames:":["_col1"],"Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"SEL_18","children":{"File Output Operator":{"compressed:":"false","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe"},"OperatorId:":"FS_72"}}}}}}}},"Stage-7":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"x","Statistics:":"Num rows:
1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"TS_26","children":{"Filter Operator":{"predicate:":"(key is not null and value is not null) (type: boolean)","Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"FIL_64","children":{"Select Operator":{"expressions:":"key (type: string), value (type: string)","outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"SEL_28","children":{"Reduce Output Operator":{"key expressions:":"_col0 (type: string)","sort order:":"+","Map-reduce partition columns:":"_col0 (type: string)","Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","value expressions:":"_col1 (type: string)","OperatorId:":"RS_45","OutputOperators:":"[JOIN_47]"}}}}}}}},{"TableScan":{"alias:":"y","Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"TS_29","children":
{"Filter Operator":{"predicate:":"key is not null (type: boolean)","Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"FIL_65","children":{"Select Operator":{"expressions:":"key (type: string)","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"SEL_31","children":{"Reduce Output Operator":{"key expressions:":"_col0 (type: string)","sort order:":"+","Map-reduce partition columns:":"_col0 (type: string)","Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"RS_46","OutputOperators:":"[JOIN_47]"}}}}}}}}],"Reduce Operator Tree:":{"Join Operator":{"condition map:":[{"":"Inner Join 0 to 1"}],"keys:":{"0":"_col0 (type: string)","1":"_col0 (type: string)"},"outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"JOIN_47","children":{"File Output Operator"
:{"compressed:":"false","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe"},"OperatorId:":"FS_74"}}}}}},"Stage-8":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"OperatorId:":"TS_75","children":{"Reduce Output Operator":{"key expressions:":"_col1 (type: string)","sort order:":"+","Map-reduce partition columns:":"_col1 (type: string)","Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","value expressions:":"_col0 (type: string)","OperatorId:":"RS_48","OutputOperators:":"[JOIN_50]"}}}},{"TableScan":{"OperatorId:":"TS_79","children":{"Reduce Output Operator":{"key expressions:":"_col1 (type: string)","sort order:":"+","Map-reduce partition columns:":"_col1 (type: string)","Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"RS_49","OutputOpera
tors:":"[JOIN_50]"}}}}],"Reduce Operator Tree:":{"Join Operator":{"condition map:":[{"":"Inner Join 0 to 1"}],"keys:":{"0":"_col1 (type: string)","1":"_col1 (type: string)"},"outputColumnNames:":["_col0","_col4"],"Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"JOIN_50","children":{"Select Operator":{"expressions:":"_col0 (type: string), _col4 (type: string)","outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"SEL_51","children":{"File Output Operator":{"compressed:":"false","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe"},"OperatorId:":"FS_76"}}}}}}}},"Stage-10":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"srctable","Statistics:":"Num rows: 1 Data size: 0 Basic stats:
PARTIAL Column stats: NONE","OperatorId:":"TS_32","children":{"Filter Operator":{"predicate:":"value is not null (type: boolean)","Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"FIL_66","children":{"Select Operator":{"expressions:":"key (type: string), value (type: string)","outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"SEL_34","children":{"Union":{"Statistics:":"Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"UNION_38","children":{"Group By Operator":{"keys:":"_col1 (type: string), _col0 (type: string)","mode:":"hash","outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"GBY_41","children":{"Reduce Output Operator":{"key expressions:":"_col0 (type: string), _col1 (type: string)","sort order:":"++","Map-reduce partition columns:":"_col0
(type: string), _col1 (type: string)","Statistics:":"Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"RS_42","OutputOperators:":"[GBY_43]"}}}}}}}}}}}},{"TableScan":{"alias:":"srctable","Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"TS_35","children":{"Filter Operator":{"predicate:":"value is not null (type: boolean)","Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"FIL_67","children":{"Select Operator":{"expressions:":"key (type: string), value (type: string)","outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"SEL_37","children":{"Union":{"Statistics:":"Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"UNION_38","children":{"Group By Operator":{"keys:":"_col1 (type: string), _col0 (type: string)","mode:":"hash","outputColumnNames:":["_col0","_
col1"],"Statistics:":"Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"GBY_41","children":{"Reduce Output Operator":{"key expressions:":"_col0 (type: string), _col1 (type: string)","sort order:":"++","Map-reduce partition columns:":"_col0 (type: string), _col1 (type: string)","Statistics:":"Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"RS_42","OutputOperators:":"[GBY_43]"}}}}}}}}}}}}],"Reduce Operator Tree:":{"Group By Operator":{"keys:":"KEY._col0 (type: string), KEY._col1 (type: string)","mode:":"mergepartial","outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"GBY_43","children":{"Select Operator":{"expressions:":"_col0 (type: string)","outputColumnNames:":["_col1"],"Statistics:":"Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE","OperatorId:":"SEL_44","children":{"File Output Operator":{"compressed:":"false","table:
":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe"},"OperatorId:":"FS_78"}}}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_80"}}}}}}
http://git-wip-us.apache.org/repos/asf/hive/blob/e27199b9/ql/src/test/results/clientpositive/input4.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/input4.q.out b/ql/src/test/results/clientpositive/input4.q.out
index 83912f6..6984318 100644
--- a/ql/src/test/results/clientpositive/input4.q.out
+++ b/ql/src/test/results/clientpositive/input4.q.out
@@ -44,7 +44,7 @@ PREHOOK: type: QUERY
POSTHOOK: query: EXPLAIN FORMATTED
SELECT Input4Alias.VALUE, Input4Alias.KEY FROM INPUT4 AS Input4Alias
POSTHOOK: type: QUERY
-{"STAGE DEPENDENCIES":{"Stage-0":{"ROOT STAGE":"TRUE"}},"STAGE PLANS":{"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"TableScan":{"alias:":"input4alias","Statistics:":"Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE","children":{"Select Operator":{"expressions:":"value (type: string), key (type: string)","outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE","children":{"ListSink":{}}}}}}}}}}
+{"STAGE DEPENDENCIES":{"Stage-0":{"ROOT STAGE":"TRUE"}},"STAGE PLANS":{"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"TableScan":{"alias:":"input4alias","Statistics:":"Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"value (type: string), key (type: string)","outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"SEL_1","children":{"ListSink":{"OperatorId:":"LIST_SINK_3"}}}}}}}}}}
PREHOOK: query: SELECT Input4Alias.VALUE, Input4Alias.KEY FROM INPUT4 AS Input4Alias
PREHOOK: type: QUERY
PREHOOK: Input: default@input4
http://git-wip-us.apache.org/repos/asf/hive/blob/e27199b9/ql/src/test/results/clientpositive/join0.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/join0.q.out b/ql/src/test/results/clientpositive/join0.q.out
index c02319e..b1e85a3 100644
--- a/ql/src/test/results/clientpositive/join0.q.out
+++ b/ql/src/test/results/clientpositive/join0.q.out
@@ -112,7 +112,7 @@ SELECT src1.key as k1, src1.value as v1,
(SELECT * FROM src WHERE src.key < 10) src2
SORT BY k1, v1, k2, v2
POSTHOOK: type: QUERY
-{"STAGE DEPENDENCIES":{"Stage-1":{"ROOT STAGE":"TRUE"},"Stage-2":{"DEPENDENT STAGES":"Stage-1"},"Stage-0":{"DEPENDENT STAGES":"Stage-2"}},"STAGE PLANS":{"Stage-1":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"src","Statistics:":"Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE","children":{"Filter Operator":{"predicate:":"(key < 10) (type: boolean)","Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","children":{"Select Operator":{"expressions:":"key (type: string), value (type: string)","outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","children":{"Reduce Output Operator":{"sort order:":"","Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","value expressions:":"_col0 (type: string), _col1 (type: string)"}}}}}}}},{"TableScan":{"alias:":"src","Statistics:":"Num rows: 500 Data size: 5312 Basic stats: COM
PLETE Column stats: NONE","children":{"Filter Operator":{"predicate:":"(key < 10) (type: boolean)","Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","children":{"Select Operator":{"expressions:":"key (type: string), value (type: string)","outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","children":{"Reduce Output Operator":{"sort order:":"","Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","value expressions:":"_col0 (type: string), _col1 (type: string)"}}}}}}}}],"Reduce Operator Tree:":{"Join Operator":{"condition map:":[{"":"Inner Join 0 to 1"}],"keys:":{},"outputColumnNames:":["_col0","_col1","_col2","_col3"],"Statistics:":"Num rows: 27556 Data size: 612872 Basic stats: COMPLETE Column stats: NONE","children":{"File Output Operator":{"compressed:":"false","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","
output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe"}}}}}}},"Stage-2":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"children":{"Reduce Output Operator":{"key expressions:":"_col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string)","sort order:":"++++","Statistics:":"Num rows: 27556 Data size: 612872 Basic stats: COMPLETE Column stats: NONE"}}}}],"Reduce Operator Tree:":{"Select Operator":{"expressions:":"KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string)","outputColumnNames:":["_col0","_col1","_col2","_col3"],"Statistics:":"Num rows: 27556 Data size: 612872 Basic stats: COMPLETE Column stats: NONE","children":{"File Output Operator":{"compressed:":"false","Statistics:":"Num rows: 27556 Data size: 612872 Basic stats: COMPLETE Column stats: NONE","table:":{"input format:":"
org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"}}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{}}}}}}
+{"STAGE DEPENDENCIES":{"Stage-1":{"ROOT STAGE":"TRUE"},"Stage-2":{"DEPENDENT STAGES":"Stage-1"},"Stage-0":{"DEPENDENT STAGES":"Stage-2"}},"STAGE PLANS":{"Stage-1":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"src","Statistics:":"Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"TS_0","children":{"Filter Operator":{"predicate:":"(key < 10) (type: boolean)","Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"FIL_13","children":{"Select Operator":{"expressions:":"key (type: string), value (type: string)","outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"SEL_2","children":{"Reduce Output Operator":{"sort order:":"","Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","value expressions:":"_col0 (type: string), _col1 (type: string)","OperatorId:":"RS_6","OutputOper
ators:":"[JOIN_8]"}}}}}}}},{"TableScan":{"alias:":"src","Statistics:":"Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"TS_3","children":{"Filter Operator":{"predicate:":"(key < 10) (type: boolean)","Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"FIL_14","children":{"Select Operator":{"expressions:":"key (type: string), value (type: string)","outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"SEL_5","children":{"Reduce Output Operator":{"sort order:":"","Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","value expressions:":"_col0 (type: string), _col1 (type: string)","OperatorId:":"RS_7","OutputOperators:":"[JOIN_8]"}}}}}}}}],"Reduce Operator Tree:":{"Join Operator":{"condition map:":[{"":"Inner Join 0 to 1"}],"keys:":{},"outputColumnNames:":["_col0","_col1","_col2","_
col3"],"Statistics:":"Num rows: 27556 Data size: 612872 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"JOIN_8","children":{"File Output Operator":{"compressed:":"false","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe"},"OperatorId:":"FS_15"}}}}}},"Stage-2":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"OperatorId:":"TS_16","children":{"Reduce Output Operator":{"key expressions:":"_col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string)","sort order:":"++++","Statistics:":"Num rows: 27556 Data size: 612872 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"RS_10","OutputOperators:":"[SEL_11]"}}}}],"Reduce Operator Tree:":{"Select Operator":{"expressions:":"KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), KEY.
reducesinkkey3 (type: string)","outputColumnNames:":["_col0","_col1","_col2","_col3"],"Statistics:":"Num rows: 27556 Data size: 612872 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"SEL_11","children":{"File Output Operator":{"compressed:":"false","Statistics:":"Num rows: 27556 Data size: 612872 Basic stats: COMPLETE Column stats: NONE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_12"}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_17"}}}}}}
Warning: Shuffle Join JOIN[8][tables = [src1, src2]] in Stage 'Stage-1:MAPRED' is a cross product
PREHOOK: query: SELECT src1.key as k1, src1.value as v1,
src2.key as k2, src2.value as v2 FROM
http://git-wip-us.apache.org/repos/asf/hive/blob/e27199b9/ql/src/test/results/clientpositive/parallel_join0.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/parallel_join0.q.out b/ql/src/test/results/clientpositive/parallel_join0.q.out
index c02319e..b1e85a3 100644
--- a/ql/src/test/results/clientpositive/parallel_join0.q.out
+++ b/ql/src/test/results/clientpositive/parallel_join0.q.out
@@ -112,7 +112,7 @@ SELECT src1.key as k1, src1.value as v1,
(SELECT * FROM src WHERE src.key < 10) src2
SORT BY k1, v1, k2, v2
POSTHOOK: type: QUERY
-{"STAGE DEPENDENCIES":{"Stage-1":{"ROOT STAGE":"TRUE"},"Stage-2":{"DEPENDENT STAGES":"Stage-1"},"Stage-0":{"DEPENDENT STAGES":"Stage-2"}},"STAGE PLANS":{"Stage-1":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"src","Statistics:":"Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE","children":{"Filter Operator":{"predicate:":"(key < 10) (type: boolean)","Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","children":{"Select Operator":{"expressions:":"key (type: string), value (type: string)","outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","children":{"Reduce Output Operator":{"sort order:":"","Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","value expressions:":"_col0 (type: string), _col1 (type: string)"}}}}}}}},{"TableScan":{"alias:":"src","Statistics:":"Num rows: 500 Data size: 5312 Basic stats: COM
PLETE Column stats: NONE","children":{"Filter Operator":{"predicate:":"(key < 10) (type: boolean)","Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","children":{"Select Operator":{"expressions:":"key (type: string), value (type: string)","outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","children":{"Reduce Output Operator":{"sort order:":"","Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","value expressions:":"_col0 (type: string), _col1 (type: string)"}}}}}}}}],"Reduce Operator Tree:":{"Join Operator":{"condition map:":[{"":"Inner Join 0 to 1"}],"keys:":{},"outputColumnNames:":["_col0","_col1","_col2","_col3"],"Statistics:":"Num rows: 27556 Data size: 612872 Basic stats: COMPLETE Column stats: NONE","children":{"File Output Operator":{"compressed:":"false","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","
output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe"}}}}}}},"Stage-2":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"children":{"Reduce Output Operator":{"key expressions:":"_col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string)","sort order:":"++++","Statistics:":"Num rows: 27556 Data size: 612872 Basic stats: COMPLETE Column stats: NONE"}}}}],"Reduce Operator Tree:":{"Select Operator":{"expressions:":"KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string)","outputColumnNames:":["_col0","_col1","_col2","_col3"],"Statistics:":"Num rows: 27556 Data size: 612872 Basic stats: COMPLETE Column stats: NONE","children":{"File Output Operator":{"compressed:":"false","Statistics:":"Num rows: 27556 Data size: 612872 Basic stats: COMPLETE Column stats: NONE","table:":{"input format:":"
org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"}}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{}}}}}}
+{"STAGE DEPENDENCIES":{"Stage-1":{"ROOT STAGE":"TRUE"},"Stage-2":{"DEPENDENT STAGES":"Stage-1"},"Stage-0":{"DEPENDENT STAGES":"Stage-2"}},"STAGE PLANS":{"Stage-1":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"src","Statistics:":"Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"TS_0","children":{"Filter Operator":{"predicate:":"(key < 10) (type: boolean)","Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"FIL_13","children":{"Select Operator":{"expressions:":"key (type: string), value (type: string)","outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"SEL_2","children":{"Reduce Output Operator":{"sort order:":"","Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","value expressions:":"_col0 (type: string), _col1 (type: string)","OperatorId:":"RS_6","OutputOper
ators:":"[JOIN_8]"}}}}}}}},{"TableScan":{"alias:":"src","Statistics:":"Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"TS_3","children":{"Filter Operator":{"predicate:":"(key < 10) (type: boolean)","Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"FIL_14","children":{"Select Operator":{"expressions:":"key (type: string), value (type: string)","outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"SEL_5","children":{"Reduce Output Operator":{"sort order:":"","Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","value expressions:":"_col0 (type: string), _col1 (type: string)","OperatorId:":"RS_7","OutputOperators:":"[JOIN_8]"}}}}}}}}],"Reduce Operator Tree:":{"Join Operator":{"condition map:":[{"":"Inner Join 0 to 1"}],"keys:":{},"outputColumnNames:":["_col0","_col1","_col2","_
col3"],"Statistics:":"Num rows: 27556 Data size: 612872 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"JOIN_8","children":{"File Output Operator":{"compressed:":"false","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe"},"OperatorId:":"FS_15"}}}}}},"Stage-2":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"OperatorId:":"TS_16","children":{"Reduce Output Operator":{"key expressions:":"_col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string)","sort order:":"++++","Statistics:":"Num rows: 27556 Data size: 612872 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"RS_10","OutputOperators:":"[SEL_11]"}}}}],"Reduce Operator Tree:":{"Select Operator":{"expressions:":"KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), KEY.
reducesinkkey3 (type: string)","outputColumnNames:":["_col0","_col1","_col2","_col3"],"Statistics:":"Num rows: 27556 Data size: 612872 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"SEL_11","children":{"File Output Operator":{"compressed:":"false","Statistics:":"Num rows: 27556 Data size: 612872 Basic stats: COMPLETE Column stats: NONE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_12"}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_17"}}}}}}
Warning: Shuffle Join JOIN[8][tables = [src1, src2]] in Stage 'Stage-1:MAPRED' is a cross product
PREHOOK: query: SELECT src1.key as k1, src1.value as v1,
src2.key as k2, src2.value as v2 FROM
http://git-wip-us.apache.org/repos/asf/hive/blob/e27199b9/ql/src/test/results/clientpositive/plan_json.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/plan_json.q.out b/ql/src/test/results/clientpositive/plan_json.q.out
index 6851891..1f665f5 100644
--- a/ql/src/test/results/clientpositive/plan_json.q.out
+++ b/ql/src/test/results/clientpositive/plan_json.q.out
@@ -2,4 +2,4 @@ PREHOOK: query: EXPLAIN FORMATTED SELECT count(1) FROM src
PREHOOK: type: QUERY
POSTHOOK: query: EXPLAIN FORMATTED SELECT count(1) FROM src
POSTHOOK: type: QUERY
-{"STAGE DEPENDENCIES":{"Stage-1":{"ROOT STAGE":"TRUE"},"Stage-0":{"DEPENDENT STAGES":"Stage-1"}},"STAGE PLANS":{"Stage-1":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"src","Statistics:":"Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE","children":{"Select Operator":{"Statistics:":"Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE","children":{"Group By Operator":{"aggregations:":["count(1)"],"mode:":"hash","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","children":{"Reduce Output Operator":{"sort order:":"","Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","value expressions:":"_col0 (type: bigint)"}}}}}}}}],"Reduce Operator Tree:":{"Group By Operator":{"aggregations:":["count(VALUE._col0)"],"mode:":"mergepartial","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column
stats: COMPLETE","children":{"File Output Operator":{"compressed:":"false","Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"}}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{}}}}}}
+{"STAGE DEPENDENCIES":{"Stage-1":{"ROOT STAGE":"TRUE"},"Stage-0":{"DEPENDENT STAGES":"Stage-1"}},"STAGE PLANS":{"Stage-1":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"src","Statistics:":"Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"TS_0","children":{"Select Operator":{"Statistics:":"Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"SEL_1","children":{"Group By Operator":{"aggregations:":["count(1)"],"mode:":"hash","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_3","children":{"Reduce Output Operator":{"sort order:":"","Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","value expressions:":"_col0 (type: bigint)","OperatorId:":"RS_4","OutputOperators:":"[GBY_5]"}}}}}}}}],"Reduce Operator Tree:":{"Group By Operator":{"aggregations:":["count(VALUE._col0)"],"mode:":
"mergepartial","outputColumnNames:":["_col0"],"Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","OperatorId:":"GBY_5","children":{"File Output Operator":{"compressed:":"false","Statistics:":"Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_7"}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_8"}}}}}}
[44/50] [abbrv] hive git commit: HIVE-16068: BloomFilter
expectedEntries not always using NDV when it's available during runtime
filtering (Jason Dere, reviewed by Gunther Hagleitner)
Posted by om...@apache.org.
HIVE-16068: BloomFilter expectedEntries not always using NDV when it's available during runtime filtering (Jason Dere, reviewed by Gunther Hagleitner)
Change-Id: I9a5f3960e1b7fc1416f510aefbeab6f28bbafffa
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/43906f9c
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/43906f9c
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/43906f9c
Branch: refs/heads/branch-2.2
Commit: 43906f9c3dbe2e63ffc744b69925b753a119e8be
Parents: 72ca985
Author: Jason Dere <jd...@hortonworks.com>
Authored: Fri Mar 3 11:53:02 2017 -0800
Committer: Owen O'Malley <om...@apache.org>
Committed: Tue Mar 28 15:27:57 2017 -0700
----------------------------------------------------------------------
.../hive/ql/udf/generic/GenericUDAFBloomFilter.java | 16 +++++++++++-----
.../results/clientpositive/llap/mergejoin.q.out | 4 ++--
.../clientpositive/tez/explainanalyze_3.q.out | 4 ++--
3 files changed, 15 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/43906f9c/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFBloomFilter.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFBloomFilter.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFBloomFilter.java
index b32e04a..788aace 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFBloomFilter.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFBloomFilter.java
@@ -20,10 +20,13 @@ package org.apache.hadoop.hive.ql.udf.generic;
import org.apache.hadoop.hive.common.type.HiveDecimal;
import org.apache.hadoop.hive.ql.exec.Operator;
+import org.apache.hadoop.hive.ql.exec.SelectOperator;
import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.plan.ColStatistics;
+import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeDescUtils;
import org.apache.hadoop.hive.ql.plan.Statistics;
import org.apache.hadoop.hive.ql.plan.Statistics.State;
import org.apache.hadoop.hive.serde2.io.DateWritable;
@@ -68,7 +71,7 @@ public class GenericUDAFBloomFilter implements GenericUDAFResolver2 {
*/
public static class GenericUDAFBloomFilterEvaluator extends GenericUDAFEvaluator {
// Source operator to get the number of entries
- private Operator<?> sourceOperator;
+ private SelectOperator sourceOperator;
private long maxEntries = 0;
// ObjectInspector for input data.
@@ -258,10 +261,13 @@ public class GenericUDAFBloomFilter implements GenericUDAFResolver2 {
switch (stats.getColumnStatsState()) {
case COMPLETE:
case PARTIAL:
- // There should only be column stats for one column, use if that is the case.
+ // There should only be column in sourceOperator
List<ColStatistics> colStats = stats.getColumnStats();
- if (colStats.size() == 1) {
- long ndv = colStats.get(0).getCountDistint();
+ ExprNodeColumnDesc colExpr = ExprNodeDescUtils.getColumnExpr(
+ sourceOperator.getConf().getColList().get(0));
+ if (colExpr != null
+ && stats.getColumnStatisticsFromColName(colExpr.getColumn()) != null) {
+ long ndv = stats.getColumnStatisticsFromColName(colExpr.getColumn()).getCountDistint();
if (ndv > 0) {
expectedEntries = ndv;
}
@@ -279,7 +285,7 @@ public class GenericUDAFBloomFilter implements GenericUDAFResolver2 {
return sourceOperator;
}
- public void setSourceOperator(Operator<?> sourceOperator) {
+ public void setSourceOperator(SelectOperator sourceOperator) {
this.sourceOperator = sourceOperator;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/43906f9c/ql/src/test/results/clientpositive/llap/mergejoin.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/mergejoin.q.out b/ql/src/test/results/clientpositive/llap/mergejoin.q.out
index 85f14a0..4e6e2b4 100644
--- a/ql/src/test/results/clientpositive/llap/mergejoin.q.out
+++ b/ql/src/test/results/clientpositive/llap/mergejoin.q.out
@@ -63,7 +63,7 @@ STAGE PLANS:
outputColumnNames: _col0
Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=25)
+ aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=14)
mode: hash
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
@@ -95,7 +95,7 @@ STAGE PLANS:
Execution mode: vectorized, llap
Reduce Operator Tree:
Group By Operator
- aggregations: min(VALUE._col0), max(VALUE._col1), bloom_filter(VALUE._col2, expectedEntries=25)
+ aggregations: min(VALUE._col0), max(VALUE._col1), bloom_filter(VALUE._col2, expectedEntries=14)
mode: final
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
http://git-wip-us.apache.org/repos/asf/hive/blob/43906f9c/ql/src/test/results/clientpositive/tez/explainanalyze_3.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/explainanalyze_3.q.out b/ql/src/test/results/clientpositive/tez/explainanalyze_3.q.out
index dfe163e..14fac5f 100644
--- a/ql/src/test/results/clientpositive/tez/explainanalyze_3.q.out
+++ b/ql/src/test/results/clientpositive/tez/explainanalyze_3.q.out
@@ -869,10 +869,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: b
- filterExpr: (key is not null and key BETWEEN DynamicValue(RS_6_a_key_min) AND DynamicValue(RS_6_a_key_max) and in_bloom_filter(key, DynamicValue(RS_6_a_key_bloom_filter))) (type: boolean)
+ filterExpr: (key is not null and (key BETWEEN DynamicValue(RS_6_a_key_min) AND DynamicValue(RS_6_a_key_max) and in_bloom_filter(key, DynamicValue(RS_6_a_key_bloom_filter)))) (type: boolean)
Statistics: Num rows: 500/500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (key is not null and key BETWEEN DynamicValue(RS_6_a_key_min) AND DynamicValue(RS_6_a_key_max) and in_bloom_filter(key, DynamicValue(RS_6_a_key_bloom_filter))) (type: boolean)
+ predicate: (key is not null and (key BETWEEN DynamicValue(RS_6_a_key_min) AND DynamicValue(RS_6_a_key_max) and in_bloom_filter(key, DynamicValue(RS_6_a_key_bloom_filter)))) (type: boolean)
Statistics: Num rows: 500/244 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: int), value (type: string)
[29/50] [abbrv] hive git commit: HIVE-15084: Flaky test:
TestMiniTezCliDriver:explainanalyze_1, 2, 3, 4, 5 (Pengcheng Xiong)
Posted by om...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/9f5bea3b/ql/src/test/results/clientpositive/llap/tez_union_multiinsert.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/tez_union_multiinsert.q.out b/ql/src/test/results/clientpositive/llap/tez_union_multiinsert.q.out
index 92042d2..7756426 100644
--- a/ql/src/test/results/clientpositive/llap/tez_union_multiinsert.q.out
+++ b/ql/src/test/results/clientpositive/llap/tez_union_multiinsert.q.out
@@ -103,12 +103,12 @@ STAGE PLANS:
keys: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 1 Data size: 364 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 464 Basic stats: COMPLETE Column stats: PARTIAL
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string)
sort order: +++
Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
- Statistics: Num rows: 1 Data size: 364 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 464 Basic stats: COMPLETE Column stats: PARTIAL
Execution mode: llap
LLAP IO: no inputs
Map 7
@@ -136,12 +136,12 @@ STAGE PLANS:
keys: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 1 Data size: 364 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 464 Basic stats: COMPLETE Column stats: PARTIAL
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string)
sort order: +++
Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
- Statistics: Num rows: 1 Data size: 364 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 464 Basic stats: COMPLETE Column stats: PARTIAL
Execution mode: llap
LLAP IO: no inputs
Reducer 2
@@ -155,7 +155,7 @@ STAGE PLANS:
Select Operator
expressions: 'tst1' (type: string), UDFToString(_col0) (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: count(DISTINCT substr(_col1, 5))
keys: _col0 (type: string), substr(_col1, 5) (type: string)
@@ -172,12 +172,12 @@ STAGE PLANS:
keys: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 1 Data size: 364 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 464 Basic stats: COMPLETE Column stats: PARTIAL
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string)
sort order: +++
Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
- Statistics: Num rows: 1 Data size: 364 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 464 Basic stats: COMPLETE Column stats: PARTIAL
Reducer 4
Execution mode: llap
Reduce Operator Tree:
@@ -203,10 +203,10 @@ STAGE PLANS:
keys: KEY._col0 (type: string), KEY._col1 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 180 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 280 Basic stats: COMPLETE Column stats: PARTIAL
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 180 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 280 Basic stats: COMPLETE Column stats: PARTIAL
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -984,12 +984,12 @@ STAGE PLANS:
keys: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 1 Data size: 364 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 464 Basic stats: COMPLETE Column stats: PARTIAL
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string)
sort order: +++
Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
- Statistics: Num rows: 1 Data size: 364 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 464 Basic stats: COMPLETE Column stats: PARTIAL
Execution mode: llap
LLAP IO: no inputs
Map 5
@@ -1035,12 +1035,12 @@ STAGE PLANS:
keys: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 1 Data size: 364 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 464 Basic stats: COMPLETE Column stats: PARTIAL
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string)
sort order: +++
Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
- Statistics: Num rows: 1 Data size: 364 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 464 Basic stats: COMPLETE Column stats: PARTIAL
Execution mode: llap
LLAP IO: no inputs
Reducer 3
@@ -1068,10 +1068,10 @@ STAGE PLANS:
keys: KEY._col0 (type: string), KEY._col1 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 180 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 280 Basic stats: COMPLETE Column stats: PARTIAL
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 180 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 280 Basic stats: COMPLETE Column stats: PARTIAL
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -1088,7 +1088,7 @@ STAGE PLANS:
Select Operator
expressions: 'tst1' (type: string), UDFToString(_col0) (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: count(DISTINCT substr(_col1, 5))
keys: _col0 (type: string), substr(_col1, 5) (type: string)
@@ -1105,12 +1105,12 @@ STAGE PLANS:
keys: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 1 Data size: 364 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 464 Basic stats: COMPLETE Column stats: PARTIAL
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string)
sort order: +++
Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
- Statistics: Num rows: 1 Data size: 364 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 464 Basic stats: COMPLETE Column stats: PARTIAL
Union 2
Vertex: Union 2
@@ -1879,12 +1879,12 @@ STAGE PLANS:
keys: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 1 Data size: 364 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 464 Basic stats: COMPLETE Column stats: PARTIAL
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string)
sort order: +++
Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
- Statistics: Num rows: 1 Data size: 364 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 464 Basic stats: COMPLETE Column stats: PARTIAL
Execution mode: llap
LLAP IO: no inputs
Map 5
@@ -1930,12 +1930,12 @@ STAGE PLANS:
keys: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 1 Data size: 364 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 464 Basic stats: COMPLETE Column stats: PARTIAL
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string)
sort order: +++
Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
- Statistics: Num rows: 1 Data size: 364 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 464 Basic stats: COMPLETE Column stats: PARTIAL
Execution mode: llap
LLAP IO: no inputs
Reducer 3
@@ -1963,10 +1963,10 @@ STAGE PLANS:
keys: KEY._col0 (type: string), KEY._col1 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 180 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 280 Basic stats: COMPLETE Column stats: PARTIAL
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 180 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 280 Basic stats: COMPLETE Column stats: PARTIAL
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -1983,7 +1983,7 @@ STAGE PLANS:
Select Operator
expressions: 'tst1' (type: string), UDFToString(_col0) (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: count(DISTINCT substr(_col1, 5))
keys: _col0 (type: string), substr(_col1, 5) (type: string)
@@ -2000,12 +2000,12 @@ STAGE PLANS:
keys: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 1 Data size: 364 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 464 Basic stats: COMPLETE Column stats: PARTIAL
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string)
sort order: +++
Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
- Statistics: Num rows: 1 Data size: 364 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 464 Basic stats: COMPLETE Column stats: PARTIAL
Union 2
Vertex: Union 2
@@ -2783,12 +2783,12 @@ STAGE PLANS:
keys: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 1 Data size: 364 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 464 Basic stats: COMPLETE Column stats: PARTIAL
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string)
sort order: +++
Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
- Statistics: Num rows: 1 Data size: 364 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 464 Basic stats: COMPLETE Column stats: PARTIAL
Execution mode: llap
LLAP IO: no inputs
Reducer 2
@@ -2802,7 +2802,7 @@ STAGE PLANS:
Select Operator
expressions: 'tst1' (type: string), UDFToString(_col0) (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: count(DISTINCT substr(_col1, 5))
keys: _col0 (type: string), substr(_col1, 5) (type: string)
@@ -2819,12 +2819,12 @@ STAGE PLANS:
keys: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 1 Data size: 364 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 464 Basic stats: COMPLETE Column stats: PARTIAL
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string)
sort order: +++
Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
- Statistics: Num rows: 1 Data size: 364 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 464 Basic stats: COMPLETE Column stats: PARTIAL
Reducer 4
Execution mode: llap
Reduce Operator Tree:
@@ -2850,10 +2850,10 @@ STAGE PLANS:
keys: KEY._col0 (type: string), KEY._col1 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 180 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 280 Basic stats: COMPLETE Column stats: PARTIAL
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 180 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 280 Basic stats: COMPLETE Column stats: PARTIAL
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -3618,12 +3618,12 @@ STAGE PLANS:
keys: _col0 (type: string), _col1 (type: string)
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string)
sort order: +++
Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL
Execution mode: llap
LLAP IO: no inputs
Reducer 2
@@ -3637,17 +3637,17 @@ STAGE PLANS:
Select Operator
expressions: 'tst1' (type: string), UDFToString(_col0) (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
keys: _col0 (type: string), _col1 (type: string)
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string)
sort order: +++
Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL
Reducer 4
Execution mode: llap
Reduce Operator Tree:
@@ -3655,7 +3655,7 @@ STAGE PLANS:
keys: KEY._col0 (type: string), KEY._col1 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL
Group By Operator
aggregations: count(DISTINCT substr(_col1, 5))
keys: _col0 (type: string), substr(_col1, 5) (type: string)
@@ -3672,14 +3672,14 @@ STAGE PLANS:
keys: _col0 (type: string), _col1 (type: string)
mode: complete
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 364 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 464 Basic stats: COMPLETE Column stats: PARTIAL
Select Operator
expressions: _col0 (type: string), _col1 (type: string), _col2 (type: bigint)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 356 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 456 Basic stats: COMPLETE Column stats: PARTIAL
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 356 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 456 Basic stats: COMPLETE Column stats: PARTIAL
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/9f5bea3b/ql/src/test/results/clientpositive/llap/vectorized_dynamic_semijoin_reduction.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vectorized_dynamic_semijoin_reduction.q.out b/ql/src/test/results/clientpositive/llap/vectorized_dynamic_semijoin_reduction.q.out
index 229640c..62a6779 100644
--- a/ql/src/test/results/clientpositive/llap/vectorized_dynamic_semijoin_reduction.q.out
+++ b/ql/src/test/results/clientpositive/llap/vectorized_dynamic_semijoin_reduction.q.out
@@ -225,10 +225,10 @@ STAGE PLANS:
aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=57)
mode: hash
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary)
Execution mode: vectorized, llap
LLAP IO: all inputs
@@ -273,10 +273,10 @@ STAGE PLANS:
aggregations: min(VALUE._col0), max(VALUE._col1), bloom_filter(VALUE._col2, expectedEntries=57)
mode: final
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary)
Stage: Stage-0
@@ -361,10 +361,10 @@ STAGE PLANS:
aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=57)
mode: hash
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary)
Execution mode: vectorized, llap
LLAP IO: all inputs
@@ -409,10 +409,10 @@ STAGE PLANS:
aggregations: min(VALUE._col0), max(VALUE._col1), bloom_filter(VALUE._col2, expectedEntries=57)
mode: final
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary)
Stage: Stage-0
@@ -682,10 +682,10 @@ STAGE PLANS:
aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=57)
mode: hash
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary)
Select Operator
expressions: _col1 (type: int)
@@ -743,10 +743,10 @@ STAGE PLANS:
aggregations: min(VALUE._col0), max(VALUE._col1), bloom_filter(VALUE._col2, expectedEntries=57)
mode: final
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary)
Reducer 6
Execution mode: vectorized, llap
http://git-wip-us.apache.org/repos/asf/hive/blob/9f5bea3b/ql/src/test/results/clientpositive/llap/vectorized_dynamic_semijoin_reduction2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vectorized_dynamic_semijoin_reduction2.q.out b/ql/src/test/results/clientpositive/llap/vectorized_dynamic_semijoin_reduction2.q.out
index 3051a45..d53ca00 100644
--- a/ql/src/test/results/clientpositive/llap/vectorized_dynamic_semijoin_reduction2.q.out
+++ b/ql/src/test/results/clientpositive/llap/vectorized_dynamic_semijoin_reduction2.q.out
@@ -831,10 +831,10 @@ STAGE PLANS:
aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=20)
mode: hash
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary)
Execution mode: vectorized, llap
LLAP IO: all inputs
@@ -879,10 +879,10 @@ STAGE PLANS:
aggregations: min(VALUE._col0), max(VALUE._col1), bloom_filter(VALUE._col2, expectedEntries=20)
mode: final
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary)
Stage: Stage-0
@@ -967,10 +967,10 @@ STAGE PLANS:
aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=20)
mode: hash
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 282 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 282 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: char(10)), _col1 (type: char(10)), _col2 (type: binary)
Execution mode: vectorized, llap
LLAP IO: all inputs
@@ -1015,10 +1015,10 @@ STAGE PLANS:
aggregations: min(VALUE._col0), max(VALUE._col1), bloom_filter(VALUE._col2, expectedEntries=20)
mode: final
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 282 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 282 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: char(10)), _col1 (type: char(10)), _col2 (type: binary)
Stage: Stage-0
@@ -1103,10 +1103,10 @@ STAGE PLANS:
aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=20)
mode: hash
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 282 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 282 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: varchar(10)), _col1 (type: varchar(10)), _col2 (type: binary)
Execution mode: vectorized, llap
LLAP IO: all inputs
@@ -1151,10 +1151,10 @@ STAGE PLANS:
aggregations: min(VALUE._col0), max(VALUE._col1), bloom_filter(VALUE._col2, expectedEntries=20)
mode: final
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 282 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 282 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: varchar(10)), _col1 (type: varchar(10)), _col2 (type: binary)
Stage: Stage-0
http://git-wip-us.apache.org/repos/asf/hive/blob/9f5bea3b/ql/src/test/results/clientpositive/load_dyn_part14.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/load_dyn_part14.q.out b/ql/src/test/results/clientpositive/load_dyn_part14.q.out
index 6c949ba..a6a5c63 100644
--- a/ql/src/test/results/clientpositive/load_dyn_part14.q.out
+++ b/ql/src/test/results/clientpositive/load_dyn_part14.q.out
@@ -79,7 +79,7 @@ STAGE PLANS:
Select Operator
expressions: 'k1' (type: string), null (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 2 Data size: 172 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 256 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
table:
@@ -92,10 +92,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
Union
- Statistics: Num rows: 6 Data size: 854 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 6 Data size: 938 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 6 Data size: 854 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 6 Data size: 938 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -103,10 +103,10 @@ STAGE PLANS:
name: default.nzhang_part14
TableScan
Union
- Statistics: Num rows: 6 Data size: 854 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 6 Data size: 938 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 6 Data size: 854 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 6 Data size: 938 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -114,10 +114,10 @@ STAGE PLANS:
name: default.nzhang_part14
TableScan
Union
- Statistics: Num rows: 6 Data size: 854 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 6 Data size: 938 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 6 Data size: 854 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 6 Data size: 938 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/9f5bea3b/ql/src/test/results/clientpositive/stats1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/stats1.q.out b/ql/src/test/results/clientpositive/stats1.q.out
index 6955f4d..2d5b4f8 100644
--- a/ql/src/test/results/clientpositive/stats1.q.out
+++ b/ql/src/test/results/clientpositive/stats1.q.out
@@ -53,7 +53,7 @@ STAGE PLANS:
Select Operator
expressions: 'tst1' (type: string), UDFToString(_col0) (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
table:
@@ -66,10 +66,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
Union
- Statistics: Num rows: 26 Data size: 363 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 26 Data size: 463 Basic stats: COMPLETE Column stats: PARTIAL
File Output Operator
compressed: false
- Statistics: Num rows: 26 Data size: 363 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 26 Data size: 463 Basic stats: COMPLETE Column stats: PARTIAL
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -83,10 +83,10 @@ STAGE PLANS:
outputColumnNames: _col0, _col1
Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE
Union
- Statistics: Num rows: 26 Data size: 363 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 26 Data size: 463 Basic stats: COMPLETE Column stats: PARTIAL
File Output Operator
compressed: false
- Statistics: Num rows: 26 Data size: 363 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 26 Data size: 463 Basic stats: COMPLETE Column stats: PARTIAL
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/9f5bea3b/ql/src/test/results/clientpositive/tez/explainanalyze_1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/explainanalyze_1.q.out b/ql/src/test/results/clientpositive/tez/explainanalyze_1.q.out
index 6fe287a..3928b1b 100644
--- a/ql/src/test/results/clientpositive/tez/explainanalyze_1.q.out
+++ b/ql/src/test/results/clientpositive/tez/explainanalyze_1.q.out
@@ -23,19 +23,19 @@ Stage-0
Union 2
<-Map 1 [CONTAINS]
File Output Operator [FS_7]
- Limit [LIM_6] (rows=10/20 width=178)
+ Limit [LIM_6] (rows=10/20 width=10)
Number of rows:10
- Select Operator [SEL_1] (rows=500/12 width=178)
+ Select Operator [SEL_1] (rows=500/12 width=10)
Output:["_col0","_col1"]
- TableScan [TS_0] (rows=500/13 width=178)
+ TableScan [TS_0] (rows=500/13 width=10)
Output:["key","value"]
<-Map 3 [CONTAINS]
File Output Operator [FS_7]
- Limit [LIM_6] (rows=10/20 width=178)
+ Limit [LIM_6] (rows=10/20 width=10)
Number of rows:10
- Select Operator [SEL_3] (rows=500/12 width=178)
+ Select Operator [SEL_3] (rows=500/12 width=10)
Output:["_col0","_col1"]
- TableScan [TS_2] (rows=500/13 width=178)
+ TableScan [TS_2] (rows=500/13 width=10)
Output:["key","value"]
PREHOOK: query: select key from src
@@ -87,10 +87,10 @@ Stage-3
Map 1
File Output Operator [FS_2]
table:{"name:":"default.t"}
- Select Operator [SEL_1] (rows=500/500 width=87)
+ Select Operator [SEL_1] (rows=500/500 width=10)
Output:["_col0"]
- TableScan [TS_0] (rows=500/500 width=87)
- default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key"]
+ TableScan [TS_0] (rows=500/500 width=10)
+ default@src,src,Tbl:COMPLETE,Col:NONE,Output:["key"]
Stage-0
Move Operator
Please refer to the previous Stage-1
@@ -131,10 +131,10 @@ Stage-3
Map 1
File Output Operator [FS_2]
table:{"name:":"default.t"}
- Select Operator [SEL_1] (rows=500/500 width=87)
+ Select Operator [SEL_1] (rows=500/500 width=10)
Output:["_col0"]
- TableScan [TS_0] (rows=500/500 width=87)
- default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key"]
+ TableScan [TS_0] (rows=500/500 width=10)
+ default@src,src,Tbl:COMPLETE,Col:NONE,Output:["key"]
PREHOOK: query: select key from src limit 10
PREHOOK: type: QUERY
@@ -239,18 +239,16 @@ Stage-0
limit:-1
Stage-1
Reducer 2
- File Output Operator [FS_6]
- Group By Operator [GBY_4] (rows=205/309 width=95)
- Output:["_col0","_col1"],aggregations:["count(VALUE._col0)"],keys:KEY._col0
+ File Output Operator [FS_5]
+ Group By Operator [GBY_3] (rows=250/309 width=10)
+ Output:["_col0","_col1"],aggregations:["count(KEY._col0)"],keys:KEY._col0
<-Map 1 [SIMPLE_EDGE]
- SHUFFLE [RS_3]
- PartitionCols:_col0
- Group By Operator [GBY_2] (rows=205/309 width=95)
- Output:["_col0","_col1"],aggregations:["count(key)"],keys:key
- Select Operator [SEL_1] (rows=500/500 width=87)
- Output:["key"]
- TableScan [TS_0] (rows=500/500 width=87)
- default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key"]
+ SHUFFLE [RS_2]
+ PartitionCols:key
+ Select Operator [SEL_1] (rows=500/500 width=10)
+ Output:["key"]
+ TableScan [TS_0] (rows=500/500 width=10)
+ default@src,src,Tbl:COMPLETE,Col:NONE,Output:["key"]
PREHOOK: query: select count(*) from (select key, count(key) from src group by key)subq
PREHOOK: type: QUERY
@@ -284,33 +282,31 @@ Stage-0
limit:-1
Stage-1
Reducer 3
- File Output Operator [FS_14]
- Group By Operator [GBY_12] (rows=1/1 width=8)
- Output:["_col0"],aggregations:["count(VALUE._col0)"]
+ File Output Operator [FS_13]
+ Group By Operator [GBY_11] (rows=1/1 width=8)
+ Output:["_col0"],aggregations:["count()"]
<-Reducer 2 [SIMPLE_EDGE]
- SHUFFLE [RS_11]
- Group By Operator [GBY_10] (rows=1/1 width=8)
- Output:["_col0"],aggregations:["count()"]
- Merge Join Operator [MERGEJOIN_19] (rows=382/0 width=8)
- Conds:RS_6._col0=RS_7._col0(Inner)
- <-Map 1 [SIMPLE_EDGE]
- SHUFFLE [RS_6]
- PartitionCols:_col0
- Select Operator [SEL_2] (rows=166/497 width=87)
- Output:["_col0"]
- Filter Operator [FIL_17] (rows=166/497 width=87)
- predicate:(UDFToDouble(key) > 0.0)
- TableScan [TS_0] (rows=500/500 width=87)
- default@src,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key"]
- <-Map 4 [SIMPLE_EDGE]
- SHUFFLE [RS_7]
- PartitionCols:_col0
- Select Operator [SEL_5] (rows=166/0 width=91)
- Output:["_col0"]
- Filter Operator [FIL_18] (rows=166/0 width=91)
- predicate:(UDFToDouble(value) > 0.0)
- TableScan [TS_3] (rows=500/500 width=91)
- default@src,b,Tbl:COMPLETE,Col:COMPLETE,Output:["value"]
+ SHUFFLE [RS_10]
+ Merge Join Operator [MERGEJOIN_18] (rows=182/0 width=10)
+ Conds:RS_6._col0=RS_7._col0(Inner)
+ <-Map 1 [SIMPLE_EDGE]
+ SHUFFLE [RS_6]
+ PartitionCols:_col0
+ Select Operator [SEL_2] (rows=166/497 width=10)
+ Output:["_col0"]
+ Filter Operator [FIL_16] (rows=166/497 width=10)
+ predicate:(UDFToDouble(key) > 0.0)
+ TableScan [TS_0] (rows=500/500 width=10)
+ default@src,a,Tbl:COMPLETE,Col:NONE,Output:["key"]
+ <-Map 4 [SIMPLE_EDGE]
+ SHUFFLE [RS_7]
+ PartitionCols:_col0
+ Select Operator [SEL_5] (rows=166/0 width=10)
+ Output:["_col0"]
+ Filter Operator [FIL_17] (rows=166/0 width=10)
+ predicate:(UDFToDouble(value) > 0.0)
+ TableScan [TS_3] (rows=500/500 width=10)
+ default@src,b,Tbl:COMPLETE,Col:NONE,Output:["value"]
PREHOOK: query: select count(*) from src a join src b on a.key = b.key where a.key > 0
PREHOOK: type: QUERY
@@ -335,33 +331,31 @@ Stage-0
limit:-1
Stage-1
Reducer 3
- File Output Operator [FS_14]
- Group By Operator [GBY_12] (rows=1/1 width=8)
- Output:["_col0"],aggregations:["count(VALUE._col0)"]
+ File Output Operator [FS_13]
+ Group By Operator [GBY_11] (rows=1/1 width=8)
+ Output:["_col0"],aggregations:["count()"]
<-Reducer 2 [SIMPLE_EDGE]
- SHUFFLE [RS_11]
- Group By Operator [GBY_10] (rows=1/1 width=8)
- Output:["_col0"],aggregations:["count()"]
- Merge Join Operator [MERGEJOIN_19] (rows=399/1019 width=8)
- Conds:RS_6._col0=RS_7._col0(Inner)
- <-Map 1 [SIMPLE_EDGE]
- SHUFFLE [RS_6]
- PartitionCols:_col0
- Select Operator [SEL_2] (rows=166/497 width=87)
- Output:["_col0"]
- Filter Operator [FIL_17] (rows=166/497 width=87)
- predicate:(UDFToDouble(key) > 0.0)
- TableScan [TS_0] (rows=500/500 width=87)
- default@src,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key"]
- <-Map 4 [SIMPLE_EDGE]
- SHUFFLE [RS_7]
- PartitionCols:_col0
- Select Operator [SEL_5] (rows=166/497 width=87)
- Output:["_col0"]
- Filter Operator [FIL_18] (rows=166/497 width=87)
- predicate:(UDFToDouble(key) > 0.0)
- TableScan [TS_3] (rows=500/500 width=87)
- default@src,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key"]
+ SHUFFLE [RS_10]
+ Merge Join Operator [MERGEJOIN_18] (rows=182/1019 width=10)
+ Conds:RS_6._col0=RS_7._col0(Inner)
+ <-Map 1 [SIMPLE_EDGE]
+ SHUFFLE [RS_6]
+ PartitionCols:_col0
+ Select Operator [SEL_2] (rows=166/497 width=10)
+ Output:["_col0"]
+ Filter Operator [FIL_16] (rows=166/497 width=10)
+ predicate:(UDFToDouble(key) > 0.0)
+ TableScan [TS_0] (rows=500/500 width=10)
+ default@src,a,Tbl:COMPLETE,Col:NONE,Output:["key"]
+ <-Map 4 [SIMPLE_EDGE]
+ SHUFFLE [RS_7]
+ PartitionCols:_col0
+ Select Operator [SEL_5] (rows=166/497 width=10)
+ Output:["_col0"]
+ Filter Operator [FIL_17] (rows=166/497 width=10)
+ predicate:(UDFToDouble(key) > 0.0)
+ TableScan [TS_3] (rows=500/500 width=10)
+ default@src,b,Tbl:COMPLETE,Col:NONE,Output:["key"]
PREHOOK: query: select count(*) from src a join src b on a.key = b.key where a.key > 0
PREHOOK: type: QUERY
@@ -397,15 +391,15 @@ Stage-0
Union 2
<-Map 1 [CONTAINS]
File Output Operator [FS_6]
- Select Operator [SEL_1] (rows=500/500 width=178)
+ Select Operator [SEL_1] (rows=500/500 width=10)
Output:["_col0","_col1"]
- TableScan [TS_0] (rows=500/500 width=178)
+ TableScan [TS_0] (rows=500/500 width=10)
Output:["key","value"]
<-Map 3 [CONTAINS]
File Output Operator [FS_6]
- Select Operator [SEL_3] (rows=500/500 width=178)
+ Select Operator [SEL_3] (rows=500/500 width=10)
Output:["_col0","_col1"]
- TableScan [TS_2] (rows=500/500 width=178)
+ TableScan [TS_2] (rows=500/500 width=10)
Output:["key","value"]
PREHOOK: query: select count(*) from (select * from src a union all select * from src b)subq
@@ -438,34 +432,31 @@ POSTHOOK: type: QUERY
Plan optimized by CBO.
Vertex dependency in root stage
-Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE)
+Map 1 <- Map 2 (BROADCAST_EDGE)
Stage-0
Fetch Operator
limit:-1
Stage-1
- Reducer 2
+ Map 1
File Output Operator [FS_10]
- Select Operator [SEL_9] (rows=1219/1028 width=178)
+ Select Operator [SEL_9] (rows=550/1028 width=10)
Output:["_col0","_col1"]
- Merge Join Operator [MERGEJOIN_15] (rows=1219/1028 width=178)
- Conds:RS_6._col0=RS_7._col0(Inner),Output:["_col0","_col2"]
- <-Map 1 [SIMPLE_EDGE]
- SHUFFLE [RS_6]
- PartitionCols:_col0
- Select Operator [SEL_2] (rows=500/500 width=87)
- Output:["_col0"]
- Filter Operator [FIL_13] (rows=500/500 width=87)
- predicate:key is not null
- TableScan [TS_0] (rows=500/500 width=87)
- default@src,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key"]
- <-Map 3 [SIMPLE_EDGE]
- SHUFFLE [RS_7]
+ Map Join Operator [MAPJOIN_15] (rows=550/1028 width=10)
+ Conds:SEL_2._col0=RS_7._col0(Inner),HybridGraceHashJoin:true,Output:["_col0","_col2"]
+ <-Map 2 [BROADCAST_EDGE]
+ BROADCAST [RS_7]
PartitionCols:_col0
- Select Operator [SEL_5] (rows=500/500 width=178)
+ Select Operator [SEL_5] (rows=500/500 width=10)
Output:["_col0","_col1"]
- Filter Operator [FIL_14] (rows=500/500 width=178)
+ Filter Operator [FIL_14] (rows=500/500 width=10)
predicate:key is not null
- TableScan [TS_3] (rows=500/500 width=178)
- default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
+ TableScan [TS_3] (rows=500/500 width=10)
+ default@src,y,Tbl:COMPLETE,Col:NONE,Output:["key","value"]
+ <-Select Operator [SEL_2] (rows=500/500 width=10)
+ Output:["_col0"]
+ Filter Operator [FIL_13] (rows=500/500 width=10)
+ predicate:key is not null
+ TableScan [TS_0] (rows=500/500 width=10)
+ default@src,x,Tbl:COMPLETE,Col:NONE,Output:["key"]
[50/50] [abbrv] hive git commit: HIVE-13815: Improve logic to infer
false predicates (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
Posted by om...@apache.org.
HIVE-13815: Improve logic to infer false predicates (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
Conflicts:
ql/src/test/results/clientpositive/lineage3.q.out
ql/src/test/results/clientpositive/spark/constprog_partitioner.q.out
Change-Id: I0727a4681836aedd422fde69b6e32d78069dfa68
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f31f7495
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f31f7495
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f31f7495
Branch: refs/heads/branch-2.2
Commit: f31f7495ee89b41dfcd64a7b903b377aa75f346f
Parents: 9f8e090
Author: Jesus Camacho Rodriguez <jc...@apache.org>
Authored: Thu Jul 21 20:44:45 2016 +0100
Committer: Owen O'Malley <om...@apache.org>
Committed: Tue Mar 28 15:27:59 2017 -0700
----------------------------------------------------------------------
.../test/results/positive/hbase_pushdown.q.out | 4 +-
.../rules/HiveReduceExpressionsRule.java | 120 +++++++++++++----
.../clientpositive/annotate_stats_filter.q.out | 4 +-
.../clientpositive/annotate_stats_part.q.out | 6 +-
.../results/clientpositive/cbo_rp_join1.q.out | 26 ++--
.../results/clientpositive/constprog2.q.out | 2 +-
.../clientpositive/constprog_partitioner.q.out | 2 +-
.../clientpositive/constprog_semijoin.q.out | 128 +++++++++----------
ql/src/test/results/clientpositive/cte_5.q.out | 40 +++---
.../clientpositive/filter_cond_pushdown.q.out | 60 ++++-----
ql/src/test/results/clientpositive/join42.q.out | 4 +-
.../results/clientpositive/llap/cte_5.q.out | 40 +++---
.../llap/dynamic_partition_pruning_2.q.out | 4 +-
.../llap/hybridgrace_hashjoin_1.q.out | 4 +-
.../clientpositive/llap/tez_self_join.q.out | 2 +-
.../test/results/clientpositive/mapjoin2.q.out | 3 +
.../test/results/clientpositive/mergejoin.q.out | 13 ++
.../clientpositive/ppd_outer_join5.q.out | 4 +-
.../results/clientpositive/ppd_udf_case.q.out | 4 +-
.../results/clientpositive/ppd_union_view.q.out | 2 +-
.../results/clientpositive/smb_mapjoin_25.q.out | 8 +-
.../spark/constprog_partitioner.q.out | 5 +
.../spark/constprog_semijoin.q.out | 128 +++++++++----------
.../clientpositive/spark/ppd_outer_join5.q.out | 4 +-
.../clientpositive/spark/smb_mapjoin_25.q.out | 8 +-
.../clientpositive/tez/constprog_semijoin.q.out | 88 ++++++-------
.../test/results/clientpositive/tez/cte_5.q.out | 39 +++---
.../tez/dynamic_partition_pruning_2.q.out | 4 +-
.../tez/hybridgrace_hashjoin_1.q.out | 4 +-
.../results/clientpositive/tez/mapjoin2.q.out | 3 +
.../results/clientpositive/tez/mergejoin.q.out | 15 ++-
.../clientpositive/tez/tez_self_join.q.out | 2 +-
32 files changed, 441 insertions(+), 339 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/f31f7495/hbase-handler/src/test/results/positive/hbase_pushdown.q.out
----------------------------------------------------------------------
diff --git a/hbase-handler/src/test/results/positive/hbase_pushdown.q.out b/hbase-handler/src/test/results/positive/hbase_pushdown.q.out
index b94d8ad..5fac123 100644
--- a/hbase-handler/src/test/results/positive/hbase_pushdown.q.out
+++ b/hbase-handler/src/test/results/positive/hbase_pushdown.q.out
@@ -202,10 +202,10 @@ STAGE PLANS:
alias: hbase_pushdown
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Filter Operator
- predicate: (((key = 80) and (key = 90)) and (value like '%90%')) (type: boolean)
+ predicate: false (type: boolean)
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
- expressions: 90 (type: int), value (type: string)
+ expressions: key (type: int), value (type: string)
outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
File Output Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/f31f7495/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsRule.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsRule.java
index 6efc731..54ec4be 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsRule.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsRule.java
@@ -22,6 +22,7 @@ import org.apache.calcite.plan.RelOptPredicateList;
import org.apache.calcite.plan.RelOptRuleCall;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.core.Filter;
+import org.apache.calcite.rel.core.Project;
import org.apache.calcite.rel.metadata.RelMetadataQuery;
import org.apache.calcite.rel.rules.ReduceExpressionsRule;
import org.apache.calcite.rex.RexCall;
@@ -63,7 +64,7 @@ public abstract class HiveReduceExpressionsRule extends ReduceExpressionsRule {
* {@link org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter}.
*/
public static final ReduceExpressionsRule FILTER_INSTANCE =
- new FilterReduceExpressionsRule(HiveFilter.class, HiveRelFactories.HIVE_BUILDER);
+ new FilterReduceExpressionsRule(HiveFilter.class, HiveRelFactories.HIVE_BUILDER);
/**
* Singleton rule that reduces constants inside a
@@ -79,18 +80,6 @@ public abstract class HiveReduceExpressionsRule extends ReduceExpressionsRule {
public static final ReduceExpressionsRule JOIN_INSTANCE =
new JoinReduceExpressionsRule(HiveJoin.class, HiveRelFactories.HIVE_BUILDER);
- //~ Constructors -----------------------------------------------------------
-
- /**
- * Creates a HiveReduceExpressionsRule.
- *
- * @param clazz class of rels to which this rule should apply
- */
- protected HiveReduceExpressionsRule(Class<? extends RelNode> clazz,
- RelBuilderFactory relBuilderFactory, String desc) {
- super(clazz, relBuilderFactory, desc);
- }
-
/**
* Rule that reduces constants inside a {@link org.apache.calcite.rel.core.Filter}.
* If the condition is a constant, the filter is removed (if TRUE) or replaced with
@@ -99,11 +88,12 @@ public abstract class HiveReduceExpressionsRule extends ReduceExpressionsRule {
public static class FilterReduceExpressionsRule extends ReduceExpressionsRule {
public FilterReduceExpressionsRule(Class<? extends Filter> filterClass,
- RelBuilderFactory relBuilderFactory) {
+ RelBuilderFactory relBuilderFactory) {
super(filterClass, relBuilderFactory, "ReduceExpressionsRule(Filter)");
}
- @Override public void onMatch(RelOptRuleCall call) {
+ @Override
+ public void onMatch(RelOptRuleCall call) {
final Filter filter = call.rel(0);
final List<RexNode> expList =
Lists.newArrayList(filter.getCondition());
@@ -141,7 +131,7 @@ public abstract class HiveReduceExpressionsRule extends ReduceExpressionsRule {
// e.g condition(null = null) is reduced to condition (null) with null type
// since this is a condition which will always be boolean type we cast it to
// boolean type
- if(newConditionExp.getType().getSqlTypeName() == SqlTypeName.NULL) {
+ if (newConditionExp.getType().getSqlTypeName() == SqlTypeName.NULL) {
newConditionExp = call.builder().cast(newConditionExp, SqlTypeName.BOOLEAN);
}
call.transformTo(call.builder().
@@ -198,15 +188,15 @@ public abstract class HiveReduceExpressionsRule extends ReduceExpressionsRule {
// it with an Empty.
boolean alwaysTrue;
switch (rexCall.getKind()) {
- case IS_NULL:
- case IS_UNKNOWN:
- alwaysTrue = false;
- break;
- case IS_NOT_NULL:
- alwaysTrue = true;
- break;
- default:
- return;
+ case IS_NULL:
+ case IS_UNKNOWN:
+ alwaysTrue = false;
+ break;
+ case IS_NOT_NULL:
+ alwaysTrue = true;
+ break;
+ default:
+ return;
}
if (reverse) {
alwaysTrue = !alwaysTrue;
@@ -225,6 +215,86 @@ public abstract class HiveReduceExpressionsRule extends ReduceExpressionsRule {
}
}
+
+ /**
+ * Rule that reduces constants inside a {@link org.apache.calcite.rel.core.Project}.
+ */
+ public static class ProjectReduceExpressionsRule extends HiveReduceExpressionsRule {
+
+ public ProjectReduceExpressionsRule(Class<? extends Project> projectClass,
+ RelBuilderFactory relBuilderFactory) {
+ super(projectClass, relBuilderFactory, "HiveReduceExpressionsRule(Project)");
+ }
+
+ @Override
+ public void onMatch(RelOptRuleCall call) {
+ Project project = call.rel(0);
+ final RelOptPredicateList predicates =
+ RelMetadataQuery.instance().getPulledUpPredicates(project.getInput());
+ final List<RexNode> expList =
+ Lists.newArrayList(project.getProjects());
+ if (reduceExpressions(project, expList, predicates)) {
+ RelNode newProject = call.builder().push(project.getInput())
+ .project(expList, project.getRowType().getFieldNames()).build();
+ call.transformTo(newProject);
+
+ // New plan is absolutely better than old plan.
+ call.getPlanner().setImportance(project, 0.0);
+ }
+ }
+ }
+
+ /**
+ * Rule that reduces constants inside a {@link org.apache.calcite.rel.core.HiveJoin}.
+ */
+ public static class JoinReduceExpressionsRule extends HiveReduceExpressionsRule {
+
+ public JoinReduceExpressionsRule(Class<? extends HiveJoin> joinClass,
+ RelBuilderFactory relBuilderFactory) {
+ super(joinClass, relBuilderFactory, "HiveReduceExpressionsRule(HiveJoin)");
+ }
+
+ @Override
+ public void onMatch(RelOptRuleCall call) {
+ final HiveJoin join = call.rel(0);
+ final List<RexNode> expList = Lists.newArrayList(join.getCondition());
+ final int fieldCount = join.getLeft().getRowType().getFieldCount();
+ RelMetadataQuery mq = RelMetadataQuery.instance();
+ final RelOptPredicateList leftPredicates =
+ mq.getPulledUpPredicates(join.getLeft());
+ final RelOptPredicateList rightPredicates =
+ mq.getPulledUpPredicates(join.getRight());
+ final RelOptPredicateList predicates =
+ leftPredicates.union(rightPredicates.shift(fieldCount));
+ if (!reduceExpressions(join, expList, predicates, true)) {
+ return;
+ }
+ call.transformTo(
+ join.copy(
+ join.getTraitSet(),
+ expList.get(0),
+ join.getLeft(),
+ join.getRight(),
+ join.getJoinType(),
+ join.isSemiJoinDone()));
+
+ // New plan is absolutely better than old plan.
+ call.getPlanner().setImportance(join, 0.0);
+ }
+ }
+
+ //~ Constructors -----------------------------------------------------------
+
+ /**
+ * Creates a HiveReduceExpressionsRule.
+ *
+ * @param clazz class of rels to which this rule should apply
+ */
+ protected HiveReduceExpressionsRule(Class<? extends RelNode> clazz,
+ RelBuilderFactory relBuilderFactory, String desc) {
+ super(clazz, relBuilderFactory, desc);
+ }
+
}
// End HiveReduceExpressionsRule.java
http://git-wip-us.apache.org/repos/asf/hive/blob/f31f7495/ql/src/test/results/clientpositive/annotate_stats_filter.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/annotate_stats_filter.q.out b/ql/src/test/results/clientpositive/annotate_stats_filter.q.out
index 66dd92b..e22c3ef 100644
--- a/ql/src/test/results/clientpositive/annotate_stats_filter.q.out
+++ b/ql/src/test/results/clientpositive/annotate_stats_filter.q.out
@@ -700,10 +700,10 @@ STAGE PLANS:
alias: loc_orc
Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
- predicate: ((year = 2001) and (state = 'OH') and (state = 'FL')) (type: boolean)
+ predicate: false (type: boolean)
Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
- expressions: 'FL' (type: string), locid (type: int), zip (type: bigint), 2001 (type: int)
+ expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: int)
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/f31f7495/ql/src/test/results/clientpositive/annotate_stats_part.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/annotate_stats_part.q.out b/ql/src/test/results/clientpositive/annotate_stats_part.q.out
index 4f1b62f..866d30a 100644
--- a/ql/src/test/results/clientpositive/annotate_stats_part.q.out
+++ b/ql/src/test/results/clientpositive/annotate_stats_part.q.out
@@ -261,12 +261,12 @@ STAGE PLANS:
alias: loc_orc
Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL
Filter Operator
- predicate: ((year = '2001') and (year = '__HIVE_DEFAULT_PARTITION__')) (type: boolean)
+ predicate: false (type: boolean)
Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL
Select Operator
- expressions: state (type: string), locid (type: int), zip (type: bigint), '__HIVE_DEFAULT_PARTITION__' (type: string)
+ expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 1 Data size: 110 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: PARTIAL
ListSink
PREHOOK: query: analyze table loc_orc partition(year='2001') compute statistics for columns state,locid
http://git-wip-us.apache.org/repos/asf/hive/blob/f31f7495/ql/src/test/results/clientpositive/cbo_rp_join1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/cbo_rp_join1.q.out b/ql/src/test/results/clientpositive/cbo_rp_join1.q.out
index 28e97a5..695c6d6 100644
--- a/ql/src/test/results/clientpositive/cbo_rp_join1.q.out
+++ b/ql/src/test/results/clientpositive/cbo_rp_join1.q.out
@@ -119,7 +119,7 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@myinput1
#### A masked pattern was here ####
NULL
-Warning: Shuffle Join JOIN[10][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product
+Warning: Shuffle Join JOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product
PREHOOK: query: EXPLAIN SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b on a.key = 40 AND a.value = 40 AND a.key = a.value AND b.key = 40
PREHOOK: type: QUERY
POSTHOOK: query: EXPLAIN SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b on a.key = 40 AND a.value = 40 AND a.key = a.value AND b.key = 40
@@ -141,7 +141,7 @@ STAGE PLANS:
outputColumnNames: _col0, _col1
Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((_col0 = _col1) and (_col1 = 40) and (_col0 = 40)) (type: boolean)
+ predicate: ((_col1 = 40) and (_col0 = 40)) (type: boolean)
Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
@@ -224,6 +224,7 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@myinput1
#### A masked pattern was here ####
NULL
+Warning: Shuffle Join JOIN[8][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product
PREHOOK: query: EXPLAIN SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b on a.key = 40 AND a.key = b.key AND b.key = 40
PREHOOK: type: QUERY
POSTHOOK: query: EXPLAIN SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b on a.key = 40 AND a.key = b.key AND b.key = 40
@@ -248,11 +249,9 @@ STAGE PLANS:
predicate: (_col0 = 40) (type: boolean)
Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
+ sort order:
Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: int)
+ value expressions: _col0 (type: int), _col1 (type: int)
TableScan
alias: b
Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE
@@ -264,24 +263,22 @@ STAGE PLANS:
predicate: (_col0 = 40) (type: boolean)
Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
+ sort order:
Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: int)
+ value expressions: _col0 (type: int), _col1 (type: int)
Reduce Operator Tree:
Join Operator
condition map:
Outer Join 0 to 1
keys:
- 0 _col0 (type: int)
- 1 _col0 (type: int)
+ 0
+ 1
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 23 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: hash(_col0,_col1,_col2,_col3) (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 23 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: sum(_col0)
mode: hash
@@ -322,6 +319,7 @@ STAGE PLANS:
Processor Tree:
ListSink
+Warning: Shuffle Join JOIN[8][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product
PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b on a.key = 40 AND a.key = b.key AND b.key = 40
PREHOOK: type: QUERY
PREHOOK: Input: default@myinput1
http://git-wip-us.apache.org/repos/asf/hive/blob/f31f7495/ql/src/test/results/clientpositive/constprog2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/constprog2.q.out b/ql/src/test/results/clientpositive/constprog2.q.out
index cbd98ba..7bfd0cf 100644
--- a/ql/src/test/results/clientpositive/constprog2.q.out
+++ b/ql/src/test/results/clientpositive/constprog2.q.out
@@ -30,7 +30,7 @@ STAGE PLANS:
alias: src2
Statistics: Num rows: 1000 Data size: 10603 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (key = 86) (type: boolean)
+ predicate: (86 = key) (type: boolean)
Statistics: Num rows: 500 Data size: 5301 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: value (type: string)
http://git-wip-us.apache.org/repos/asf/hive/blob/f31f7495/ql/src/test/results/clientpositive/constprog_partitioner.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/constprog_partitioner.q.out b/ql/src/test/results/clientpositive/constprog_partitioner.q.out
index 508a928..8c7f9d3 100644
--- a/ql/src/test/results/clientpositive/constprog_partitioner.q.out
+++ b/ql/src/test/results/clientpositive/constprog_partitioner.q.out
@@ -30,7 +30,7 @@ STAGE PLANS:
alias: src2
Statistics: Num rows: 1000 Data size: 10603 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (key = 100) (type: boolean)
+ predicate: (100 = key) (type: boolean)
Statistics: Num rows: 500 Data size: 5301 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: value (type: string)
http://git-wip-us.apache.org/repos/asf/hive/blob/f31f7495/ql/src/test/results/clientpositive/constprog_semijoin.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/constprog_semijoin.q.out b/ql/src/test/results/clientpositive/constprog_semijoin.q.out
index 1940987..040cfb4 100644
--- a/ql/src/test/results/clientpositive/constprog_semijoin.q.out
+++ b/ql/src/test/results/clientpositive/constprog_semijoin.q.out
@@ -421,26 +421,26 @@ STAGE PLANS:
alias: table1
Statistics: Num rows: 10 Data size: 200 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (((dimid = 100) = true) and (dimid <> 100)) (type: boolean)
- Statistics: Num rows: 5 Data size: 100 Basic stats: COMPLETE Column stats: NONE
+ predicate: false (type: boolean)
+ Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: id (type: int), val (type: string), val1 (type: string), dimid (type: int)
- outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 5 Data size: 100 Basic stats: COMPLETE Column stats: NONE
+ expressions: id (type: int), val (type: string), val1 (type: string)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col3 (type: int), true (type: boolean)
+ key expressions: 100 (type: int), true (type: boolean)
sort order: ++
- Map-reduce partition columns: _col3 (type: int), true (type: boolean)
- Statistics: Num rows: 5 Data size: 100 Basic stats: COMPLETE Column stats: NONE
+ Map-reduce partition columns: 100 (type: int), true (type: boolean)
+ Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string)
TableScan
alias: table3
Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (((id = 100) = true) and (id <> 100)) (type: boolean)
+ predicate: ((id = 100) and (id = 100) is not null) (type: boolean)
Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: id (type: int), true (type: boolean)
+ expressions: 100 (type: int), true (type: boolean)
outputColumnNames: _col0, _col1
Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
Group By Operator
@@ -458,13 +458,13 @@ STAGE PLANS:
condition map:
Left Semi Join 0 to 1
keys:
- 0 _col3 (type: int), true (type: boolean)
+ 0 100 (type: int), true (type: boolean)
1 _col0 (type: int), _col1 (type: boolean)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 5 Data size: 110 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 5 Data size: 110 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -502,50 +502,50 @@ STAGE PLANS:
alias: table1
Statistics: Num rows: 10 Data size: 200 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((dimid) IN (100, 200) and ((dimid = 100) = true)) (type: boolean)
- Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((dimid = 100) and (dimid = 100) is not null) (type: boolean)
+ Statistics: Num rows: 5 Data size: 100 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: id (type: int), val (type: string), val1 (type: string), dimid (type: int)
- outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: NONE
+ expressions: id (type: int), val (type: string), val1 (type: string)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 5 Data size: 100 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col3 (type: int), true (type: boolean)
+ key expressions: 100 (type: int), true (type: boolean)
sort order: ++
- Map-reduce partition columns: _col3 (type: int), true (type: boolean)
- Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: NONE
+ Map-reduce partition columns: 100 (type: int), true (type: boolean)
+ Statistics: Num rows: 5 Data size: 100 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string)
TableScan
alias: table3
Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((id) IN (100, 200) and ((id = 100) = true)) (type: boolean)
- Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((id = 100) and (id = 100) is not null) (type: boolean)
+ Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: id (type: int), true (type: boolean)
+ expressions: 100 (type: int), true (type: boolean)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: int), _col1 (type: boolean)
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int), _col1 (type: boolean)
sort order: ++
Map-reduce partition columns: _col0 (type: int), _col1 (type: boolean)
- Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
Reduce Operator Tree:
Join Operator
condition map:
Left Semi Join 0 to 1
keys:
- 0 _col3 (type: int), true (type: boolean)
+ 0 100 (type: int), true (type: boolean)
1 _col0 (type: int), _col1 (type: boolean)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 2 Data size: 44 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 5 Data size: 110 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 2 Data size: 44 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 5 Data size: 110 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -585,50 +585,50 @@ STAGE PLANS:
alias: table1
Statistics: Num rows: 10 Data size: 200 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (((dimid = 100) = true) and (dimid = 200)) (type: boolean)
- Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: NONE
+ predicate: false (type: boolean)
+ Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: id (type: int), val (type: string), val1 (type: string)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: 200 (type: int), true (type: boolean)
+ key expressions: 100 (type: int), true (type: boolean)
sort order: ++
- Map-reduce partition columns: 200 (type: int), true (type: boolean)
- Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: NONE
+ Map-reduce partition columns: 100 (type: int), true (type: boolean)
+ Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string)
TableScan
alias: table3
Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (((id = 100) = true) and (id = 200)) (type: boolean)
- Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((id = 100) and (id = 100) is not null) (type: boolean)
+ Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: 200 (type: int), true (type: boolean)
+ expressions: 100 (type: int), true (type: boolean)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: int), _col1 (type: boolean)
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int), _col1 (type: boolean)
sort order: ++
Map-reduce partition columns: _col0 (type: int), _col1 (type: boolean)
- Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
Reduce Operator Tree:
Join Operator
condition map:
Left Semi Join 0 to 1
keys:
- 0 200 (type: int), true (type: boolean)
+ 0 100 (type: int), true (type: boolean)
1 _col0 (type: int), _col1 (type: boolean)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 2 Data size: 44 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 2 Data size: 44 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -666,38 +666,38 @@ STAGE PLANS:
alias: table1
Statistics: Num rows: 10 Data size: 200 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (((dimid = 100) = true) and (dimid = 100)) (type: boolean)
- Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((dimid = 100) and (dimid = 100) is not null) (type: boolean)
+ Statistics: Num rows: 5 Data size: 100 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: id (type: int), val (type: string), val1 (type: string)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 5 Data size: 100 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: 100 (type: int), true (type: boolean)
sort order: ++
Map-reduce partition columns: 100 (type: int), true (type: boolean)
- Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 5 Data size: 100 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string)
TableScan
alias: table3
Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (((id = 100) = true) and (id = 100)) (type: boolean)
- Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((id = 100) and (id = 100) is not null) (type: boolean)
+ Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: 100 (type: int), true (type: boolean)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: int), _col1 (type: boolean)
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int), _col1 (type: boolean)
sort order: ++
Map-reduce partition columns: _col0 (type: int), _col1 (type: boolean)
- Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
Reduce Operator Tree:
Join Operator
condition map:
@@ -706,10 +706,10 @@ STAGE PLANS:
0 100 (type: int), true (type: boolean)
1 _col0 (type: int), _col1 (type: boolean)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 2 Data size: 44 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 5 Data size: 110 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 2 Data size: 44 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 5 Data size: 110 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -749,26 +749,26 @@ STAGE PLANS:
alias: table1
Statistics: Num rows: 10 Data size: 200 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (((dimid = 100) = true) and dimid is not null) (type: boolean)
+ predicate: ((dimid = 100) and (dimid = 100) is not null) (type: boolean)
Statistics: Num rows: 5 Data size: 100 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: id (type: int), val (type: string), val1 (type: string), dimid (type: int)
- outputColumnNames: _col0, _col1, _col2, _col3
+ expressions: id (type: int), val (type: string), val1 (type: string)
+ outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 5 Data size: 100 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col3 (type: int), true (type: boolean)
+ key expressions: 100 (type: int), true (type: boolean)
sort order: ++
- Map-reduce partition columns: _col3 (type: int), true (type: boolean)
+ Map-reduce partition columns: 100 (type: int), true (type: boolean)
Statistics: Num rows: 5 Data size: 100 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string)
TableScan
alias: table3
Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (((id = 100) = true) and id is not null) (type: boolean)
+ predicate: ((id = 100) and (id = 100) is not null) (type: boolean)
Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: id (type: int), true (type: boolean)
+ expressions: 100 (type: int), true (type: boolean)
outputColumnNames: _col0, _col1
Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
Group By Operator
@@ -786,7 +786,7 @@ STAGE PLANS:
condition map:
Left Semi Join 0 to 1
keys:
- 0 _col3 (type: int), true (type: boolean)
+ 0 100 (type: int), true (type: boolean)
1 _col0 (type: int), _col1 (type: boolean)
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 5 Data size: 110 Basic stats: COMPLETE Column stats: NONE
http://git-wip-us.apache.org/repos/asf/hive/blob/f31f7495/ql/src/test/results/clientpositive/cte_5.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/cte_5.q.out b/ql/src/test/results/clientpositive/cte_5.q.out
index e9d700d..6bed6b1 100644
--- a/ql/src/test/results/clientpositive/cte_5.q.out
+++ b/ql/src/test/results/clientpositive/cte_5.q.out
@@ -63,6 +63,7 @@ src_thrift
srcbucket
srcbucket2
srcpart
+Warning: Shuffle Join JOIN[8][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product
PREHOOK: query: explain
with q1 as (select * from src where key= '5')
select a.colnum
@@ -87,18 +88,13 @@ STAGE PLANS:
alias: a
Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: colnum is not null (type: boolean)
+ predicate: (UDFToDouble(colnum) = 5.0) (type: boolean)
Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: colnum (type: int)
- outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: UDFToDouble(_col0) (type: double)
- sort order: +
- Map-reduce partition columns: UDFToDouble(_col0) (type: double)
+ sort order:
Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: int)
TableScan
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
@@ -108,26 +104,27 @@ STAGE PLANS:
Select Operator
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: 5.0 (type: double)
- sort order: +
- Map-reduce partition columns: 5.0 (type: double)
+ sort order:
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 UDFToDouble(_col0) (type: double)
- 1 5.0 (type: double)
- outputColumnNames: _col0
- Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ 0
+ 1
+ Statistics: Num rows: 250 Data size: 3656 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: 5 (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 250 Data size: 3656 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 250 Data size: 3656 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
@@ -135,6 +132,7 @@ STAGE PLANS:
Processor Tree:
ListSink
+Warning: Shuffle Join JOIN[8][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product
PREHOOK: query: with q1 as (select * from src where key= '5')
select a.colnum
from mydb.q1 as a join q1 as b
http://git-wip-us.apache.org/repos/asf/hive/blob/f31f7495/ql/src/test/results/clientpositive/filter_cond_pushdown.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/filter_cond_pushdown.q.out b/ql/src/test/results/clientpositive/filter_cond_pushdown.q.out
index 8cdc017..8c6f0f1 100644
--- a/ql/src/test/results/clientpositive/filter_cond_pushdown.q.out
+++ b/ql/src/test/results/clientpositive/filter_cond_pushdown.q.out
@@ -283,50 +283,50 @@ STAGE PLANS:
alias: t2
Statistics: Num rows: 20 Data size: 262 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: key is not null (type: boolean)
- Statistics: Num rows: 20 Data size: 262 Basic stats: COMPLETE Column stats: NONE
+ predicate: (UDFToDouble(key) = 1.0) (type: boolean)
+ Statistics: Num rows: 10 Data size: 131 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), c_int (type: int), c_float (type: float)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 20 Data size: 262 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 10 Data size: 131 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col0 (type: string), UDFToDouble(_col0) (type: double)
- sort order: ++
- Map-reduce partition columns: _col0 (type: string), UDFToDouble(_col0) (type: double)
- Statistics: Num rows: 20 Data size: 262 Basic stats: COMPLETE Column stats: NONE
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 10 Data size: 131 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: int), _col2 (type: float)
TableScan
alias: t3
Statistics: Num rows: 20 Data size: 262 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((c_int = 1) and key is not null) (type: boolean)
- Statistics: Num rows: 10 Data size: 131 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((c_int = 1) and (UDFToDouble(key) = 1.0)) (type: boolean)
+ Statistics: Num rows: 5 Data size: 65 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), c_float (type: float)
- outputColumnNames: _col0, _col2
- Statistics: Num rows: 10 Data size: 131 Basic stats: COMPLETE Column stats: NONE
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 5 Data size: 65 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col0 (type: string), 1.0 (type: double)
- sort order: ++
- Map-reduce partition columns: _col0 (type: string), 1.0 (type: double)
- Statistics: Num rows: 10 Data size: 131 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col2 (type: float)
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 5 Data size: 65 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: float)
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 _col0 (type: string), UDFToDouble(_col0) (type: double)
- 1 _col0 (type: string), 1.0 (type: double)
- outputColumnNames: _col0, _col1, _col2, _col5
- Statistics: Num rows: 22 Data size: 288 Basic stats: COMPLETE Column stats: NONE
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col4
+ Statistics: Num rows: 11 Data size: 144 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (((_col2 + _col5) > 2.0) or ((_col1 + 1) > 2)) (type: boolean)
- Statistics: Num rows: 14 Data size: 183 Basic stats: COMPLETE Column stats: NONE
+ predicate: (((_col2 + _col4) > 2.0) or ((_col1 + 1) > 2)) (type: boolean)
+ Statistics: Num rows: 6 Data size: 78 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 14 Data size: 183 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6 Data size: 78 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
table:
@@ -341,23 +341,23 @@ STAGE PLANS:
alias: t1
Statistics: Num rows: 20 Data size: 262 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: key is not null (type: boolean)
- Statistics: Num rows: 20 Data size: 262 Basic stats: COMPLETE Column stats: NONE
+ predicate: (UDFToDouble(key) = 1.0) (type: boolean)
+ Statistics: Num rows: 10 Data size: 131 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 20 Data size: 262 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 10 Data size: 131 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 20 Data size: 262 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 10 Data size: 131 Basic stats: COMPLETE Column stats: NONE
TableScan
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 14 Data size: 183 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6 Data size: 78 Basic stats: COMPLETE Column stats: NONE
Reduce Operator Tree:
Join Operator
condition map:
@@ -366,10 +366,10 @@ STAGE PLANS:
0 _col0 (type: string)
1 _col0 (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 22 Data size: 288 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 11 Data size: 144 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 22 Data size: 288 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 11 Data size: 144 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/f31f7495/ql/src/test/results/clientpositive/join42.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/join42.q.out b/ql/src/test/results/clientpositive/join42.q.out
index 04a67ac..f50a921 100644
--- a/ql/src/test/results/clientpositive/join42.q.out
+++ b/ql/src/test/results/clientpositive/join42.q.out
@@ -142,7 +142,7 @@ STAGE PLANS:
alias: la
Statistics: Num rows: 1 Data size: 14 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((loan_id = 4436) and aid is not null and pi_id is not null) (type: boolean)
+ predicate: ((4436 = loan_id) and aid is not null and pi_id is not null) (type: boolean)
Statistics: Num rows: 1 Data size: 14 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: aid (type: int), pi_id (type: int)
@@ -156,7 +156,7 @@ STAGE PLANS:
alias: fr
Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (loan_id = 4436) (type: boolean)
+ predicate: (4436 = loan_id) (type: boolean)
Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
Select Operator
Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
http://git-wip-us.apache.org/repos/asf/hive/blob/f31f7495/ql/src/test/results/clientpositive/llap/cte_5.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/cte_5.q.out b/ql/src/test/results/clientpositive/llap/cte_5.q.out
index 9f9b718..b69f73d 100644
--- a/ql/src/test/results/clientpositive/llap/cte_5.q.out
+++ b/ql/src/test/results/clientpositive/llap/cte_5.q.out
@@ -63,6 +63,7 @@ src_thrift
srcbucket
srcbucket2
srcpart
+Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product
PREHOOK: query: explain
with q1 as (select * from src where key= '5')
select a.colnum
@@ -93,18 +94,13 @@ STAGE PLANS:
alias: a
Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: colnum is not null (type: boolean)
+ predicate: (UDFToDouble(colnum) = 5.0) (type: boolean)
Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: colnum (type: int)
- outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: UDFToDouble(_col0) (type: double)
- sort order: +
- Map-reduce partition columns: UDFToDouble(_col0) (type: double)
+ sort order:
Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: int)
Execution mode: llap
LLAP IO: no inputs
Map 3
@@ -118,9 +114,7 @@ STAGE PLANS:
Select Operator
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: 5.0 (type: double)
- sort order: +
- Map-reduce partition columns: 5.0 (type: double)
+ sort order:
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Execution mode: llap
LLAP IO: no inputs
@@ -131,17 +125,20 @@ STAGE PLANS:
condition map:
Inner Join 0 to 1
keys:
- 0 UDFToDouble(_col0) (type: double)
- 1 5.0 (type: double)
- outputColumnNames: _col0
- Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ 0
+ 1
+ Statistics: Num rows: 250 Data size: 3656 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: 5 (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 250 Data size: 3656 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 250 Data size: 3656 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
@@ -149,6 +146,7 @@ STAGE PLANS:
Processor Tree:
ListSink
+Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product
PREHOOK: query: with q1 as (select * from src where key= '5')
select a.colnum
from mydb.q1 as a join q1 as b
http://git-wip-us.apache.org/repos/asf/hive/blob/f31f7495/ql/src/test/results/clientpositive/llap/dynamic_partition_pruning_2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/dynamic_partition_pruning_2.q.out b/ql/src/test/results/clientpositive/llap/dynamic_partition_pruning_2.q.out
index 0a1b0be..c83e955 100644
--- a/ql/src/test/results/clientpositive/llap/dynamic_partition_pruning_2.q.out
+++ b/ql/src/test/results/clientpositive/llap/dynamic_partition_pruning_2.q.out
@@ -632,10 +632,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: d1
- filterExpr: (id = 1) (type: boolean)
+ filterExpr: (1 = id) (type: boolean)
Statistics: Num rows: 3 Data size: 15 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (id = 1) (type: boolean)
+ predicate: (1 = id) (type: boolean)
Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE
Select Operator
Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE
http://git-wip-us.apache.org/repos/asf/hive/blob/f31f7495/ql/src/test/results/clientpositive/llap/hybridgrace_hashjoin_1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/hybridgrace_hashjoin_1.q.out b/ql/src/test/results/clientpositive/llap/hybridgrace_hashjoin_1.q.out
index 6ece5ca..ab1980d 100644
--- a/ql/src/test/results/clientpositive/llap/hybridgrace_hashjoin_1.q.out
+++ b/ql/src/test/results/clientpositive/llap/hybridgrace_hashjoin_1.q.out
@@ -1317,7 +1317,7 @@ STAGE PLANS:
alias: r
Statistics: Num rows: 12288 Data size: 2165060 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (cint = 6981) (type: boolean)
+ predicate: (6981 = cint) (type: boolean)
Statistics: Num rows: 6144 Data size: 1082530 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cdecimal2 (type: decimal(23,14))
@@ -1515,7 +1515,7 @@ STAGE PLANS:
alias: r
Statistics: Num rows: 12288 Data size: 2165060 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (cint = 6981) (type: boolean)
+ predicate: (6981 = cint) (type: boolean)
Statistics: Num rows: 6144 Data size: 1082530 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cdecimal2 (type: decimal(23,14))
http://git-wip-us.apache.org/repos/asf/hive/blob/f31f7495/ql/src/test/results/clientpositive/llap/tez_self_join.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/tez_self_join.q.out b/ql/src/test/results/clientpositive/llap/tez_self_join.q.out
index d1e8e5c..0d63cc1 100644
--- a/ql/src/test/results/clientpositive/llap/tez_self_join.q.out
+++ b/ql/src/test/results/clientpositive/llap/tez_self_join.q.out
@@ -102,7 +102,7 @@ STAGE PLANS:
alias: self2
Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (id3 = 'ab') (type: boolean)
+ predicate: ('ab' = id3) (type: boolean)
Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE
Select Operator
Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE
http://git-wip-us.apache.org/repos/asf/hive/blob/f31f7495/ql/src/test/results/clientpositive/mapjoin2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/mapjoin2.q.out b/ql/src/test/results/clientpositive/mapjoin2.q.out
index 5ae1ac7..4b96ed3 100644
--- a/ql/src/test/results/clientpositive/mapjoin2.q.out
+++ b/ql/src/test/results/clientpositive/mapjoin2.q.out
@@ -26,6 +26,7 @@ POSTHOOK: Input: default@values__tmp__table__2
POSTHOOK: Output: default@tbl
POSTHOOK: Lineage: tbl.n EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
POSTHOOK: Lineage: tbl.t SIMPLE [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col2, type:string, comment:), ]
+Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Stage-3:MAPRED' is a cross product
PREHOOK: query: select a.n, a.t, isnull(b.n), isnull(b.t) from (select * from tbl where n = 1) a left outer join (select * from tbl where 1 = 2) b on a.n = b.n
PREHOOK: type: QUERY
PREHOOK: Input: default@tbl
@@ -35,6 +36,7 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@tbl
#### A masked pattern was here ####
1 one true true
+Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Stage-3:MAPRED' is a cross product
PREHOOK: query: select isnull(a.n), isnull(a.t), b.n, b.t from (select * from tbl where 2 = 1) a right outer join (select * from tbl where n = 2) b on a.n = b.n
PREHOOK: type: QUERY
PREHOOK: Input: default@tbl
@@ -44,6 +46,7 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@tbl
#### A masked pattern was here ####
true true 2 two
+Warning: Shuffle Join JOIN[8][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product
PREHOOK: query: select isnull(a.n), isnull(a.t), isnull(b.n), isnull(b.t) from (select * from tbl where n = 1) a full outer join (select * from tbl where n = 2) b on a.n = b.n
PREHOOK: type: QUERY
PREHOOK: Input: default@tbl
http://git-wip-us.apache.org/repos/asf/hive/blob/f31f7495/ql/src/test/results/clientpositive/mergejoin.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/mergejoin.q.out b/ql/src/test/results/clientpositive/mergejoin.q.out
index 5ce3e1a..33f8740 100644
--- a/ql/src/test/results/clientpositive/mergejoin.q.out
+++ b/ql/src/test/results/clientpositive/mergejoin.q.out
@@ -2655,6 +2655,7 @@ POSTHOOK: Input: default@tab_part
POSTHOOK: Input: default@tab_part@ds=2008-04-08
#### A masked pattern was here ####
480
+Warning: Shuffle Join JOIN[8][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product
PREHOOK: query: select * from (select * from tab where tab.key = 0)a full outer join (select * from tab_part where tab_part.key = 98)b on a.key = b.key
PREHOOK: type: QUERY
PREHOOK: Input: default@tab
@@ -2674,6 +2675,7 @@ POSTHOOK: Input: default@tab_part@ds=2008-04-08
0 val_0 2008-04-08 NULL NULL NULL
NULL NULL NULL 98 val_98 2008-04-08
NULL NULL NULL 98 val_98 2008-04-08
+Warning: Shuffle Join JOIN[8][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product
PREHOOK: query: select * from (select * from tab where tab.key = 0)a right outer join (select * from tab_part where tab_part.key = 98)b on a.key = b.key
PREHOOK: type: QUERY
PREHOOK: Input: default@tab
@@ -2688,13 +2690,16 @@ POSTHOOK: Input: default@tab_part@ds=2008-04-08
#### A masked pattern was here ####
NULL NULL NULL 98 val_98 2008-04-08
NULL NULL NULL 98 val_98 2008-04-08
+Warning: Shuffle Join JOIN[13][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-1:MAPRED' is a cross product
PREHOOK: query: select * from
(select * from tab where tab.key = 0)a
full outer join
(select * from tab_part where tab_part.key = 98)b join tab_part c on a.key = b.key and b.key = c.key
PREHOOK: type: QUERY
PREHOOK: Input: default@tab
+PREHOOK: Input: default@tab@ds=2008-04-08
PREHOOK: Input: default@tab_part
+PREHOOK: Input: default@tab_part@ds=2008-04-08
#### A masked pattern was here ####
POSTHOOK: query: select * from
(select * from tab where tab.key = 0)a
@@ -2702,9 +2707,12 @@ full outer join
(select * from tab_part where tab_part.key = 98)b join tab_part c on a.key = b.key and b.key = c.key
POSTHOOK: type: QUERY
POSTHOOK: Input: default@tab
+POSTHOOK: Input: default@tab@ds=2008-04-08
POSTHOOK: Input: default@tab_part
+POSTHOOK: Input: default@tab_part@ds=2008-04-08
#### A masked pattern was here ####
Warning: Shuffle Join JOIN[14][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product
+Warning: Shuffle Join JOIN[11][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product
PREHOOK: query: select * from
(select * from tab where tab.key = 0)a
full outer join
@@ -2727,6 +2735,7 @@ NULL NULL NULL 98 val_98 2008-04-08 98 val_98 2008-04-08
NULL NULL NULL 98 val_98 2008-04-08 98 val_98 2008-04-08
NULL NULL NULL 98 val_98 2008-04-08 98 val_98 2008-04-08
NULL NULL NULL 98 val_98 2008-04-08 98 val_98 2008-04-08
+Warning: Shuffle Join JOIN[13][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product
Warning: Shuffle Join JOIN[10][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product
PREHOOK: query: select * from
(select * from tab where tab.key = 0)a
@@ -3254,12 +3263,15 @@ NULL NULL NULL NULL NULL NULL 97 val_97 2008-04-08
NULL NULL NULL NULL NULL NULL 97 val_97 2008-04-08
NULL NULL NULL NULL NULL NULL 98 val_98 2008-04-08
NULL NULL NULL NULL NULL NULL 98 val_98 2008-04-08
+Warning: Shuffle Join JOIN[14][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product
+Warning: Shuffle Join JOIN[11][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product
PREHOOK: query: select * from
(select * from tab where tab.key = 0)a
join
(select * from tab_part where tab_part.key = 98)b on a.key = b.key full outer join tab_part c on b.key = c.key
PREHOOK: type: QUERY
PREHOOK: Input: default@tab
+PREHOOK: Input: default@tab@ds=2008-04-08
PREHOOK: Input: default@tab_part
PREHOOK: Input: default@tab_part@ds=2008-04-08
#### A masked pattern was here ####
@@ -3269,6 +3281,7 @@ join
(select * from tab_part where tab_part.key = 98)b on a.key = b.key full outer join tab_part c on b.key = c.key
POSTHOOK: type: QUERY
POSTHOOK: Input: default@tab
+POSTHOOK: Input: default@tab@ds=2008-04-08
POSTHOOK: Input: default@tab_part
POSTHOOK: Input: default@tab_part@ds=2008-04-08
#### A masked pattern was here ####
http://git-wip-us.apache.org/repos/asf/hive/blob/f31f7495/ql/src/test/results/clientpositive/ppd_outer_join5.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/ppd_outer_join5.q.out b/ql/src/test/results/clientpositive/ppd_outer_join5.q.out
index cbc0e89..ab23d71 100644
--- a/ql/src/test/results/clientpositive/ppd_outer_join5.q.out
+++ b/ql/src/test/results/clientpositive/ppd_outer_join5.q.out
@@ -212,7 +212,7 @@ STAGE PLANS:
alias: t3
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Filter Operator
- predicate: (id = 20) (type: boolean)
+ predicate: (20 = id) (type: boolean)
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: 20 (type: int), key (type: string), value (type: string)
@@ -320,7 +320,7 @@ STAGE PLANS:
alias: t3
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Filter Operator
- predicate: (id = 20) (type: boolean)
+ predicate: (20 = id) (type: boolean)
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: 20 (type: int), key (type: string), value (type: string)
http://git-wip-us.apache.org/repos/asf/hive/blob/f31f7495/ql/src/test/results/clientpositive/ppd_udf_case.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/ppd_udf_case.q.out b/ql/src/test/results/clientpositive/ppd_udf_case.q.out
index 500168a..7678d03 100644
--- a/ql/src/test/results/clientpositive/ppd_udf_case.q.out
+++ b/ql/src/test/results/clientpositive/ppd_udf_case.q.out
@@ -52,7 +52,7 @@ STAGE PLANS:
alias: b
Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((ds = '2008-04-08') and (key = '27')) (type: boolean)
+ predicate: ((ds = '2008-04-08') and ('27' = key)) (type: boolean)
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: value (type: string), hr (type: string)
@@ -196,7 +196,7 @@ STAGE PLANS:
alias: b
Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (key = '27') (type: boolean)
+ predicate: ('27' = key) (type: boolean)
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: value (type: string), hr (type: string)
http://git-wip-us.apache.org/repos/asf/hive/blob/f31f7495/ql/src/test/results/clientpositive/ppd_union_view.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/ppd_union_view.q.out b/ql/src/test/results/clientpositive/ppd_union_view.q.out
index 1994807..c267102 100644
--- a/ql/src/test/results/clientpositive/ppd_union_view.q.out
+++ b/ql/src/test/results/clientpositive/ppd_union_view.q.out
@@ -510,7 +510,7 @@ STAGE PLANS:
GatherStats: false
Filter Operator
isSamplingPred: false
- predicate: ((ds = '2011-10-15') and keymap is not null) (type: boolean)
+ predicate: (('2011-10-15' = ds) and keymap is not null) (type: boolean)
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: key (type: string), keymap (type: string)
http://git-wip-us.apache.org/repos/asf/hive/blob/f31f7495/ql/src/test/results/clientpositive/smb_mapjoin_25.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/smb_mapjoin_25.q.out b/ql/src/test/results/clientpositive/smb_mapjoin_25.q.out
index 02fb1db..f72c2a7 100644
--- a/ql/src/test/results/clientpositive/smb_mapjoin_25.q.out
+++ b/ql/src/test/results/clientpositive/smb_mapjoin_25.q.out
@@ -80,7 +80,7 @@ STAGE PLANS:
alias: b
Statistics: Num rows: 51 Data size: 206 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (key = 5) (type: boolean)
+ predicate: (5 = key) (type: boolean)
Statistics: Num rows: 25 Data size: 100 Basic stats: COMPLETE Column stats: NONE
Select Operator
Statistics: Num rows: 25 Data size: 100 Basic stats: COMPLETE Column stats: NONE
@@ -151,7 +151,7 @@ STAGE PLANS:
alias: d
Statistics: Num rows: 55 Data size: 222 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (key = 5) (type: boolean)
+ predicate: (5 = key) (type: boolean)
Statistics: Num rows: 27 Data size: 108 Basic stats: COMPLETE Column stats: NONE
Select Operator
Statistics: Num rows: 27 Data size: 108 Basic stats: COMPLETE Column stats: NONE
@@ -216,7 +216,7 @@ STAGE PLANS:
alias: b
Statistics: Num rows: 51 Data size: 206 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (key = 5) (type: boolean)
+ predicate: (5 = key) (type: boolean)
Statistics: Num rows: 25 Data size: 100 Basic stats: COMPLETE Column stats: NONE
Select Operator
Statistics: Num rows: 25 Data size: 100 Basic stats: COMPLETE Column stats: NONE
@@ -392,7 +392,7 @@ STAGE PLANS:
alias: d
Statistics: Num rows: 55 Data size: 222 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (key = 5) (type: boolean)
+ predicate: (5 = key) (type: boolean)
Statistics: Num rows: 27 Data size: 108 Basic stats: COMPLETE Column stats: NONE
Select Operator
Statistics: Num rows: 27 Data size: 108 Basic stats: COMPLETE Column stats: NONE
http://git-wip-us.apache.org/repos/asf/hive/blob/f31f7495/ql/src/test/results/clientpositive/spark/constprog_partitioner.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/constprog_partitioner.q.out b/ql/src/test/results/clientpositive/spark/constprog_partitioner.q.out
index 2c4db83..c29a699 100644
--- a/ql/src/test/results/clientpositive/spark/constprog_partitioner.q.out
+++ b/ql/src/test/results/clientpositive/spark/constprog_partitioner.q.out
@@ -38,8 +38,13 @@ STAGE PLANS:
alias: src2
Statistics: Num rows: 1000 Data size: 10603 Basic stats: COMPLETE Column stats: NONE
Filter Operator
+<<<<<<< HEAD
predicate: (UDFToDouble(key) = 100.0) (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+=======
+ predicate: (100 = key) (type: boolean)
+ Statistics: Num rows: 500 Data size: 5301 Basic stats: COMPLETE Column stats: NONE
+>>>>>>> 6897655... HIVE-13815: Improve logic to infer false predicates (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
Select Operator
expressions: value (type: string)
outputColumnNames: _col1
[10/50] [abbrv] hive git commit: HIVE-14367 : Estimated size for
constant nulls is 0 (Ashutosh Chauhan via Jesus Camacho Rodriguez)
Posted by om...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/tez/vector_coalesce.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/vector_coalesce.q.out b/ql/src/test/results/clientpositive/tez/vector_coalesce.q.out
index 5f8c4dd..929b1ad 100644
--- a/ql/src/test/results/clientpositive/tez/vector_coalesce.q.out
+++ b/ql/src/test/results/clientpositive/tez/vector_coalesce.q.out
@@ -26,18 +26,18 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: alltypesorc
- Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 1045942 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: cdouble is null (type: boolean)
- Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 3114 Data size: 265164 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: cstring1 (type: string), cint (type: int), cfloat (type: float), csmallint (type: smallint), COALESCE(null,cstring1,cint,cfloat,csmallint) (type: string)
outputColumnNames: _col1, _col2, _col3, _col4, _col5
- Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 3114 Data size: 819540 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col1 (type: string), _col2 (type: int), _col3 (type: float), _col4 (type: smallint), _col5 (type: string)
sort order: +++++
- Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 3114 Data size: 819540 Basic stats: COMPLETE Column stats: COMPLETE
TopN Hash Memory Usage: 0.1
Execution mode: vectorized
Reducer 2
@@ -46,13 +46,13 @@ STAGE PLANS:
Select Operator
expressions: null (type: double), KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: int), KEY.reducesinkkey2 (type: float), KEY.reducesinkkey3 (type: smallint), KEY.reducesinkkey4 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
- Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 3114 Data size: 246572 Basic stats: COMPLETE Column stats: COMPLETE
Limit
Number of rows: 10
- Statistics: Num rows: 10 Data size: 2150 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 10 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 10 Data size: 2150 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 10 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -118,18 +118,18 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: alltypesorc
- Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 146792 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: ctinyint is null (type: boolean)
- Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 3115 Data size: 37224 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: cdouble (type: double), cint (type: int), COALESCE(null,(cdouble + log2(cint)),0) (type: double)
outputColumnNames: _col1, _col2, _col3
- Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 3115 Data size: 52844 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col1 (type: double), _col2 (type: int), _col3 (type: double)
sort order: +++
- Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 3115 Data size: 52844 Basic stats: COMPLETE Column stats: COMPLETE
TopN Hash Memory Usage: 0.1
Execution mode: vectorized
Reducer 2
@@ -138,13 +138,13 @@ STAGE PLANS:
Select Operator
expressions: null (type: tinyint), KEY.reducesinkkey0 (type: double), KEY.reducesinkkey1 (type: int), KEY.reducesinkkey2 (type: double)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 3115 Data size: 27928 Basic stats: COMPLETE Column stats: COMPLETE
Limit
Number of rows: 10
- Statistics: Num rows: 10 Data size: 2150 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 10 Data size: 2150 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -210,15 +210,15 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: alltypesorc
- Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 110088 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: (cfloat is null and cbigint is null) (type: boolean)
- Statistics: Num rows: 3072 Data size: 660491 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 790 Data size: 7092 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
- Statistics: Num rows: 3072 Data size: 660491 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 790 Data size: 3172 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
sort order:
- Statistics: Num rows: 3072 Data size: 660491 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 790 Data size: 3172 Basic stats: COMPLETE Column stats: COMPLETE
TopN Hash Memory Usage: 0.1
Execution mode: vectorized
Reducer 2
@@ -227,13 +227,13 @@ STAGE PLANS:
Select Operator
expressions: null (type: float), null (type: bigint), 0.0 (type: float)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 3072 Data size: 660491 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 790 Data size: 3172 Basic stats: COMPLETE Column stats: COMPLETE
Limit
Number of rows: 10
- Statistics: Num rows: 10 Data size: 2150 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 10 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 10 Data size: 2150 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 10 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -299,18 +299,18 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: alltypesorc
- Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 983040 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: (ctimestamp1 is not null or ctimestamp2 is not null) (type: boolean)
- Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 983040 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), COALESCE(ctimestamp1,ctimestamp2) (type: timestamp)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 1474560 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: timestamp), _col1 (type: timestamp), _col2 (type: timestamp)
sort order: +++
- Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 1474560 Basic stats: COMPLETE Column stats: COMPLETE
TopN Hash Memory Usage: 0.1
Execution mode: vectorized
Reducer 2
@@ -319,13 +319,13 @@ STAGE PLANS:
Select Operator
expressions: KEY.reducesinkkey0 (type: timestamp), KEY.reducesinkkey1 (type: timestamp), KEY.reducesinkkey2 (type: timestamp)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 1474560 Basic stats: COMPLETE Column stats: COMPLETE
Limit
Number of rows: 10
- Statistics: Num rows: 10 Data size: 2150 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 10 Data size: 1200 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 10 Data size: 2150 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 10 Data size: 1200 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -391,15 +391,15 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: alltypesorc
- Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 110088 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: (cfloat is null and cbigint is null) (type: boolean)
- Statistics: Num rows: 3072 Data size: 660491 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 790 Data size: 7092 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
- Statistics: Num rows: 3072 Data size: 660491 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 790 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
sort order:
- Statistics: Num rows: 3072 Data size: 660491 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 790 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
TopN Hash Memory Usage: 0.1
Execution mode: vectorized
Reducer 2
@@ -408,13 +408,13 @@ STAGE PLANS:
Select Operator
expressions: null (type: float), null (type: bigint), null (type: float)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 3072 Data size: 660491 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 790 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
Limit
Number of rows: 10
- Statistics: Num rows: 10 Data size: 2150 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 10 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 10 Data size: 2150 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 10 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/tez/vector_decimal_precision.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/vector_decimal_precision.q.out b/ql/src/test/results/clientpositive/tez/vector_decimal_precision.q.out
index 84b788c..6f32c66 100644
--- a/ql/src/test/results/clientpositive/tez/vector_decimal_precision.q.out
+++ b/ql/src/test/results/clientpositive/tez/vector_decimal_precision.q.out
@@ -574,10 +574,10 @@ STAGE PLANS:
aggregations: avg(dec), sum(dec)
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 400 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 400 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: struct<count:bigint,sum:decimal(30,10),input:decimal(20,10)>), _col1 (type: decimal(30,10))
Execution mode: vectorized
Reducer 2
@@ -586,10 +586,10 @@ STAGE PLANS:
aggregations: avg(VALUE._col0), sum(VALUE._col1)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 224 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 400 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 224 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 400 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/tez/vector_decimal_udf.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/vector_decimal_udf.q.out b/ql/src/test/results/clientpositive/tez/vector_decimal_udf.q.out
index 296f7b5..4cebbd8 100644
--- a/ql/src/test/results/clientpositive/tez/vector_decimal_udf.q.out
+++ b/ql/src/test/results/clientpositive/tez/vector_decimal_udf.q.out
@@ -2468,10 +2468,10 @@ STAGE PLANS:
aggregations: histogram_numeric(_col0, 3)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 160 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 160 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: array<double>)
Reducer 2
Reduce Operator Tree:
@@ -2479,10 +2479,10 @@ STAGE PLANS:
aggregations: histogram_numeric(VALUE._col0)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 720 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 720 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/tez/vector_interval_arithmetic.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/vector_interval_arithmetic.q.out b/ql/src/test/results/clientpositive/tez/vector_interval_arithmetic.q.out
index 1204e65..b346697 100644
--- a/ql/src/test/results/clientpositive/tez/vector_interval_arithmetic.q.out
+++ b/ql/src/test/results/clientpositive/tez/vector_interval_arithmetic.q.out
@@ -508,11 +508,11 @@ STAGE PLANS:
alias: interval_arithmetic_1
Statistics: Num rows: 50 Data size: 4800 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
- Statistics: Num rows: 50 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 50 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: CAST( 5-5 AS INTERVAL YEAR TO MONTH) (type: interval_year_month)
sort order: +
- Statistics: Num rows: 50 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 50 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE
TopN Hash Memory Usage: 0.1
Execution mode: vectorized
Reducer 2
@@ -521,13 +521,13 @@ STAGE PLANS:
Select Operator
expressions: 5-5 (type: interval_year_month), -1-1 (type: interval_year_month)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 50 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 50 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE
Limit
Number of rows: 2
- Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -1034,13 +1034,13 @@ STAGE PLANS:
Select Operator
expressions: 109 20:30:40.246913578 (type: interval_day_time), 89 02:14:26.000000000 (type: interval_day_time)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 50 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 50 Data size: 1200 Basic stats: COMPLETE Column stats: COMPLETE
Limit
Number of rows: 2
- Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/tez/vector_null_projection.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/vector_null_projection.q.out b/ql/src/test/results/clientpositive/tez/vector_null_projection.q.out
index f9e3648..1a69359 100644
--- a/ql/src/test/results/clientpositive/tez/vector_null_projection.q.out
+++ b/ql/src/test/results/clientpositive/tez/vector_null_projection.q.out
@@ -55,10 +55,10 @@ STAGE PLANS:
Select Operator
expressions: null (type: void)
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -105,52 +105,52 @@ STAGE PLANS:
alias: a
Statistics: Num rows: 1 Data size: 87 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
- Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
keys: null (type: void)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: void)
sort order: +
Map-reduce partition columns: _col0 (type: void)
- Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
Map 4
Map Operator Tree:
TableScan
alias: b
Statistics: Num rows: 1 Data size: 87 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
- Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
keys: null (type: void)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: void)
sort order: +
Map-reduce partition columns: _col0 (type: void)
- Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
Reducer 3
Reduce Operator Tree:
Group By Operator
keys: KEY._col0 (type: void)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: null (type: void)
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/tez/vectorization_0.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/vectorization_0.q.out b/ql/src/test/results/clientpositive/tez/vectorization_0.q.out
index a21658f..36ea154 100644
--- a/ql/src/test/results/clientpositive/tez/vectorization_0.q.out
+++ b/ql/src/test/results/clientpositive/tez/vectorization_0.q.out
@@ -239,10 +239,10 @@ STAGE PLANS:
aggregations: avg(ctinyint), variance(ctinyint), var_pop(ctinyint), var_samp(ctinyint), std(ctinyint), stddev(ctinyint), stddev_pop(ctinyint), stddev_samp(ctinyint)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 636 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 636 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: struct<count:bigint,sum:double,input:tinyint>), _col1 (type: struct<count:bigint,sum:double,variance:double>), _col2 (type: struct<count:bigint,sum:double,variance:double>), _col3 (type: struct<count:bigint,sum:double,variance:double>), _col4 (type: struct<count:bigint,sum:double,variance:double>), _col5 (type: struct<count:bigint,sum:double,variance:double>), _col6 (type: struct<count:bigint,sum:double,variance:double>), _col7 (type: struct<count:bigint,sum:double,variance:double>)
Execution mode: vectorized
Reducer 2
@@ -251,11 +251,11 @@ STAGE PLANS:
aggregations: avg(VALUE._col0), variance(VALUE._col1), var_pop(VALUE._col2), var_samp(VALUE._col3), std(VALUE._col4), stddev(VALUE._col5), stddev_pop(VALUE._col6), stddev_samp(VALUE._col7)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
- Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 636 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: double)
sort order: +
- Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 636 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: double), _col2 (type: double), _col3 (type: double), _col4 (type: double), _col5 (type: double), _col6 (type: double), _col7 (type: double)
Reducer 3
Execution mode: vectorized
@@ -263,10 +263,10 @@ STAGE PLANS:
Select Operator
expressions: KEY.reducesinkkey0 (type: double), VALUE._col0 (type: double), VALUE._col1 (type: double), VALUE._col2 (type: double), VALUE._col3 (type: double), VALUE._col4 (type: double), VALUE._col5 (type: double), VALUE._col6 (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
- Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 636 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 636 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -548,10 +548,10 @@ STAGE PLANS:
aggregations: avg(cbigint), variance(cbigint), var_pop(cbigint), var_samp(cbigint), std(cbigint), stddev(cbigint), stddev_pop(cbigint), stddev_samp(cbigint)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 640 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 640 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: struct<count:bigint,sum:double,input:bigint>), _col1 (type: struct<count:bigint,sum:double,variance:double>), _col2 (type: struct<count:bigint,sum:double,variance:double>), _col3 (type: struct<count:bigint,sum:double,variance:double>), _col4 (type: struct<count:bigint,sum:double,variance:double>), _col5 (type: struct<count:bigint,sum:double,variance:double>), _col6 (type: struct<count:bigint,sum:double,variance:double>), _col7 (type: struct<count:bigint,sum:double,variance:double>)
Execution mode: vectorized
Reducer 2
@@ -560,11 +560,11 @@ STAGE PLANS:
aggregations: avg(VALUE._col0), variance(VALUE._col1), var_pop(VALUE._col2), var_samp(VALUE._col3), std(VALUE._col4), stddev(VALUE._col5), stddev_pop(VALUE._col6), stddev_samp(VALUE._col7)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
- Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 640 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: double)
sort order: +
- Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 640 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: double), _col2 (type: double), _col3 (type: double), _col4 (type: double), _col5 (type: double), _col6 (type: double), _col7 (type: double)
Reducer 3
Execution mode: vectorized
@@ -572,10 +572,10 @@ STAGE PLANS:
Select Operator
expressions: KEY.reducesinkkey0 (type: double), VALUE._col0 (type: double), VALUE._col1 (type: double), VALUE._col2 (type: double), VALUE._col3 (type: double), VALUE._col4 (type: double), VALUE._col5 (type: double), VALUE._col6 (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
- Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 640 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 640 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -857,10 +857,10 @@ STAGE PLANS:
aggregations: avg(cfloat), variance(cfloat), var_pop(cfloat), var_samp(cfloat), std(cfloat), stddev(cfloat), stddev_pop(cfloat), stddev_samp(cfloat)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 636 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 636 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: struct<count:bigint,sum:double,input:float>), _col1 (type: struct<count:bigint,sum:double,variance:double>), _col2 (type: struct<count:bigint,sum:double,variance:double>), _col3 (type: struct<count:bigint,sum:double,variance:double>), _col4 (type: struct<count:bigint,sum:double,variance:double>), _col5 (type: struct<count:bigint,sum:double,variance:double>), _col6 (type: struct<count:bigint,sum:double,variance:double>), _col7 (type: struct<count:bigint,sum:double,variance:double>)
Execution mode: vectorized
Reducer 2
@@ -869,11 +869,11 @@ STAGE PLANS:
aggregations: avg(VALUE._col0), variance(VALUE._col1), var_pop(VALUE._col2), var_samp(VALUE._col3), std(VALUE._col4), stddev(VALUE._col5), stddev_pop(VALUE._col6), stddev_samp(VALUE._col7)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
- Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 636 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: double)
sort order: +
- Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 636 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: double), _col2 (type: double), _col3 (type: double), _col4 (type: double), _col5 (type: double), _col6 (type: double), _col7 (type: double)
Reducer 3
Execution mode: vectorized
@@ -881,10 +881,10 @@ STAGE PLANS:
Select Operator
expressions: KEY.reducesinkkey0 (type: double), VALUE._col0 (type: double), VALUE._col1 (type: double), VALUE._col2 (type: double), VALUE._col3 (type: double), VALUE._col4 (type: double), VALUE._col5 (type: double), VALUE._col6 (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
- Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 636 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 636 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -1008,10 +1008,10 @@ STAGE PLANS:
aggregations: avg(cbigint), stddev_pop(cbigint), var_samp(cbigint), count(), sum(cfloat), min(ctinyint)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
- Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 260 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 260 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: struct<count:bigint,sum:double,input:bigint>), _col1 (type: struct<count:bigint,sum:double,variance:double>), _col2 (type: struct<count:bigint,sum:double,variance:double>), _col3 (type: bigint), _col4 (type: double), _col5 (type: tinyint)
Execution mode: vectorized
Reducer 2
@@ -1020,14 +1020,14 @@ STAGE PLANS:
aggregations: avg(VALUE._col0), stddev_pop(VALUE._col1), var_samp(VALUE._col2), count(VALUE._col3), sum(VALUE._col4), min(VALUE._col5)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
- Statistics: Num rows: 1 Data size: 44 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 260 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: double), (- _col0) (type: double), (-6432.0 + _col0) (type: double), _col1 (type: double), (- (-6432.0 + _col0)) (type: double), ((- (-6432.0 + _col0)) + (-6432.0 + _col0)) (type: double), _col2 (type: double), (- (-6432.0 + _col0)) (type: double), (-6432.0 + (- (-6432.0 + _col0))) (type: double), (- (-6432.0 + _col0)) (type: double), ((- (-6432.0 + _col0)) / (- (-6432.0 + _col0))) (type: double), _col3 (type: bigint), _col4 (type: double), (_col2 % _col1) (type: double), (- _col2) (type: double), ((- (-6432.0 + _col0)) * (- _col0)) (type: double), _col5 (type: tinyint), (- _col5) (type: tinyint)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17
- Statistics: Num rows: 1 Data size: 44 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 260 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 44 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 260 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/tez/vectorization_pushdown.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/vectorization_pushdown.q.out b/ql/src/test/results/clientpositive/tez/vectorization_pushdown.q.out
index 2f53a3c..725cc02 100644
--- a/ql/src/test/results/clientpositive/tez/vectorization_pushdown.q.out
+++ b/ql/src/test/results/clientpositive/tez/vectorization_pushdown.q.out
@@ -32,10 +32,10 @@ STAGE PLANS:
aggregations: avg(cbigint)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: struct<count:bigint,sum:double,input:bigint>)
Execution mode: vectorized
Reducer 2
@@ -44,10 +44,10 @@ STAGE PLANS:
aggregations: avg(VALUE._col0)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/tez/vectorization_short_regress.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/vectorization_short_regress.q.out b/ql/src/test/results/clientpositive/tez/vectorization_short_regress.q.out
index 8508939..edc1d5d 100644
--- a/ql/src/test/results/clientpositive/tez/vectorization_short_regress.q.out
+++ b/ql/src/test/results/clientpositive/tez/vectorization_short_regress.q.out
@@ -99,10 +99,10 @@ STAGE PLANS:
aggregations: avg(cint), sum(cdouble), stddev_pop(cint), stddev_samp(csmallint), var_samp(cint), avg(cfloat), stddev_samp(cint), min(ctinyint), count(csmallint)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
- Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 492 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 492 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: struct<count:bigint,sum:double,input:int>), _col1 (type: double), _col2 (type: struct<count:bigint,sum:double,variance:double>), _col3 (type: struct<count:bigint,sum:double,variance:double>), _col4 (type: struct<count:bigint,sum:double,variance:double>), _col5 (type: struct<count:bigint,sum:double,input:float>), _col6 (type: struct<count:bigint,sum:double,variance:double>), _col7 (type: tinyint), _col8 (type: bigint)
Execution mode: vectorized
Reducer 2
@@ -111,14 +111,14 @@ STAGE PLANS:
aggregations: avg(VALUE._col0), sum(VALUE._col1), stddev_pop(VALUE._col2), stddev_samp(VALUE._col3), var_samp(VALUE._col4), avg(VALUE._col5), stddev_samp(VALUE._col6), min(VALUE._col7), count(VALUE._col8)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
- Statistics: Num rows: 1 Data size: 68 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 492 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: double), (_col0 + -3728.0) (type: double), (- (_col0 + -3728.0)) (type: double), (- (- (_col0 + -3728.0))) (type: double), ((- (- (_col0 + -3728.0))) * (_col0 + -3728.0)) (type: double), _col1 (type: double), (- _col0) (type: double), _col2 (type: double), (((- (- (_col0 + -3728.0))) * (_col0 + -3728.0)) * (- (- (_col0 + -3728.0)))) (type: double), _col3 (type: double), (- _col2) (type: double), (_col2 - (- (- (_col0 + -3728.0)))) (type: double), ((_col2 - (- (- (_col0 + -3728.0)))) * _col2) (type: double), _col4 (type: double), _col5 (type: double), (10.175 - _col4) (type: double), (- (10.175 - _col4)) (type: double), ((- _col2) / -563.0) (type: double), _col6 (type: double), (- ((- _col2) / -563.0)) (type: double), (_col0 / _col1) (type: double), _col7 (type: tinyint), _col8 (type: bigint), (UDFToDouble(_col7) / ((- _col2) / -563.0)) (type: double), (- (_col0 / _col1)) (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24
- Statistics: Num rows: 1 Data size: 68 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 492 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 68 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 492 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -302,10 +302,10 @@ STAGE PLANS:
aggregations: max(cint), var_pop(cbigint), stddev_pop(csmallint), max(cdouble), avg(ctinyint), min(cint), min(cdouble), stddev_samp(csmallint), var_samp(cint)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
- Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 420 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 420 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: int), _col1 (type: struct<count:bigint,sum:double,variance:double>), _col2 (type: struct<count:bigint,sum:double,variance:double>), _col3 (type: double), _col4 (type: struct<count:bigint,sum:double,input:tinyint>), _col5 (type: int), _col6 (type: double), _col7 (type: struct<count:bigint,sum:double,variance:double>), _col8 (type: struct<count:bigint,sum:double,variance:double>)
Execution mode: vectorized
Reducer 2
@@ -314,14 +314,14 @@ STAGE PLANS:
aggregations: max(VALUE._col0), var_pop(VALUE._col1), stddev_pop(VALUE._col2), max(VALUE._col3), avg(VALUE._col4), min(VALUE._col5), min(VALUE._col6), stddev_samp(VALUE._col7), var_samp(VALUE._col8)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
- Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 420 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: int), (UDFToDouble(_col0) / -3728.0) (type: double), (_col0 * -3728) (type: int), _col1 (type: double), (- (_col0 * -3728)) (type: int), _col2 (type: double), (-563 % (_col0 * -3728)) (type: int), (_col1 / _col2) (type: double), (- _col2) (type: double), _col3 (type: double), _col4 (type: double), (_col2 - 10.175) (type: double), _col5 (type: int), (UDFToDouble((_col0 * -3728)) % (_col2 - 10.175)) (type: double), (- _col3) (type: double), _col6 (type: double), (_col3 % -26.28) (type: double), _col7 (type: double), (- (UDFToDouble(_col0) / -3728.0)) (type: double), ((- (_col0 * -3728)) % (-563 % (_col0 * -3728))) (type: int), ((UDFToDouble(_col0) / -3728.0) - _col4) (type: double), (- (_col0 * -3728)) (type: int), _col8 (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22
- Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 420 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 420 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -496,10 +496,10 @@ STAGE PLANS:
aggregations: var_pop(cbigint), count(), max(ctinyint), stddev_pop(csmallint), max(cint), stddev_samp(cdouble), count(ctinyint), avg(ctinyint)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
- Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 340 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 340 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: struct<count:bigint,sum:double,variance:double>), _col1 (type: bigint), _col2 (type: tinyint), _col3 (type: struct<count:bigint,sum:double,variance:double>), _col4 (type: int), _col5 (type: struct<count:bigint,sum:double,variance:double>), _col6 (type: bigint), _col7 (type: struct<count:bigint,sum:double,input:tinyint>)
Execution mode: vectorized
Reducer 2
@@ -508,14 +508,14 @@ STAGE PLANS:
aggregations: var_pop(VALUE._col0), count(VALUE._col1), max(VALUE._col2), stddev_pop(VALUE._col3), max(VALUE._col4), stddev_samp(VALUE._col5), count(VALUE._col6), avg(VALUE._col7)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
- Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 340 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: double), (- _col0) (type: double), (_col0 - (- _col0)) (type: double), _col1 (type: bigint), (UDFToDouble(_col1) % 79.553) (type: double), _col2 (type: tinyint), (UDFToDouble(_col1) - (- _col0)) (type: double), (- (- _col0)) (type: double), (-1.0 % (- _col0)) (type: double), _col1 (type: bigint), (- _col1) (type: bigint), _col3 (type: double), (- (- (- _col0))) (type: double), (762 * (- _col1)) (type: bigint), _col4 (type: int), (UDFToLong(_col2) + (762 * (- _col1))) (type: bigint), ((- _col0) + UDFToDouble(_col4)) (type: double), _col5 (type: double), ((- _col1) % _col1) (type: bigint), _col6 (type: bigint), _col7 (type: double), (-3728 % (UDFToLong(_col2) + (762 * (- _col1)))) (type: bigint)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21
- Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 340 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 340 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -669,10 +669,10 @@ STAGE PLANS:
aggregations: avg(ctinyint), max(cbigint), stddev_samp(cint), var_pop(cint), var_pop(cbigint), max(cfloat)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
- Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 328 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 328 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: struct<count:bigint,sum:double,input:tinyint>), _col1 (type: bigint), _col2 (type: struct<count:bigint,sum:double,variance:double>), _col3 (type: struct<count:bigint,sum:double,variance:double>), _col4 (type: struct<count:bigint,sum:double,variance:double>), _col5 (type: float)
Execution mode: vectorized
Reducer 2
@@ -681,14 +681,14 @@ STAGE PLANS:
aggregations: avg(VALUE._col0), max(VALUE._col1), stddev_samp(VALUE._col2), var_pop(VALUE._col3), var_pop(VALUE._col4), max(VALUE._col5)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
- Statistics: Num rows: 1 Data size: 44 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 328 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: double), (_col0 + 6981.0) (type: double), ((_col0 + 6981.0) + _col0) (type: double), _col1 (type: bigint), (((_col0 + 6981.0) + _col0) / _col0) (type: double), (- (_col0 + 6981.0)) (type: double), _col2 (type: double), (_col0 % (- (_col0 + 6981.0))) (type: double), _col3 (type: double), _col4 (type: double), (- _col1) (type: bigint), (UDFToDouble((- _col1)) / _col2) (type: double), _col5 (type: float), (_col4 * -26.28) (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13
- Statistics: Num rows: 1 Data size: 44 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 328 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 44 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 328 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/tez/vectorized_distinct_gby.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/vectorized_distinct_gby.q.out b/ql/src/test/results/clientpositive/tez/vectorized_distinct_gby.q.out
index 936e0df..9f5c65c 100644
--- a/ql/src/test/results/clientpositive/tez/vectorized_distinct_gby.q.out
+++ b/ql/src/test/results/clientpositive/tez/vectorized_distinct_gby.q.out
@@ -136,10 +136,10 @@ STAGE PLANS:
aggregations: sum(_col0), count(_col0), avg(_col0), std(_col0)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: struct<count:bigint,sum:double,input:int>), _col3 (type: struct<count:bigint,sum:double,variance:double>)
Reducer 3
Reduce Operator Tree:
@@ -147,10 +147,10 @@ STAGE PLANS:
aggregations: sum(VALUE._col0), count(VALUE._col1), avg(VALUE._col2), std(VALUE._col3)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 1 Data size: 32 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 32 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
[41/50] [abbrv] hive git commit: HIVE-16040 : union column expansion
should take aliases from the leftmost branch (Sergey Shelukhin,
reviewed by Ashutosh Chauhan)
Posted by om...@apache.org.
HIVE-16040 : union column expansion should take aliases from the leftmost branch (Sergey Shelukhin, reviewed by Ashutosh Chauhan)
Conflicts:
ql/src/test/results/clientpositive/perf/query14.q.out
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ba565411
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ba565411
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ba565411
Branch: refs/heads/branch-2.2
Commit: ba56541134c1b32d3e593030c8768dcdcf27575f
Parents: 4bcb077
Author: Sergey Shelukhin <se...@apache.org>
Authored: Wed Mar 1 19:21:27 2017 -0800
Committer: Owen O'Malley <om...@apache.org>
Committed: Tue Mar 28 15:27:55 2017 -0700
----------------------------------------------------------------------
.../apache/hadoop/hive/ql/parse/ParseUtils.java | 32 +-
.../queries/clientpositive/union_pos_alias.q | 20 +-
.../clientpositive/union_pos_alias.q.out | 306 +++++++++++++++++++
3 files changed, 353 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/ba565411/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java
index 0023649..845305e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java
@@ -51,6 +51,8 @@ import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo;
+import com.google.common.base.Preconditions;
+
/**
* Library of utility functions used in the parse code.
@@ -402,10 +404,9 @@ public final class ParseUtils {
}
}
}
- // We find the SELECT closest to the top. This assumes there's only one FROM or FROM-s
- // are all equivalent (union case). Also, this assumption could be false for an already
- // malformed query; we don't check for that here - it will fail later anyway.
- // TODO: Maybe we should find ALL the SELECT-s not nested in another from, and compare.
+ // Note: we assume that this isn't an already malformed query;
+ // we don't check for that here - it will fail later anyway.
+ // First, we find the SELECT closest to the top.
ASTNode select = searcher.simpleBreadthFirstSearchAny((ASTNode)fromNode,
HiveParser.TOK_SELECT, HiveParser.TOK_SELECTDI);
if (select == null) {
@@ -414,6 +415,29 @@ public final class ParseUtils {
setCols.token.setType(HiveParser.TOK_ALLCOLREF);
return;
}
+
+ // Then, find the leftmost logical sibling select, because that's what Hive uses for aliases.
+ while (true) {
+ CommonTree queryOfSelect = select.parent;
+ while (queryOfSelect != null && queryOfSelect.getType() != HiveParser.TOK_QUERY) {
+ queryOfSelect = queryOfSelect.parent;
+ }
+ // We should have some QUERY; and also its parent because by supposition we are in subq.
+ if (queryOfSelect == null || queryOfSelect.parent == null) {
+ LOG.debug("Replacing SETCOLREF with ALLCOLREF because we couldn't find the QUERY");
+ setCols.token.setType(HiveParser.TOK_ALLCOLREF);
+ return;
+ }
+ if (queryOfSelect.childIndex == 0) break; // We are the left-most child.
+ Tree moreToTheLeft = queryOfSelect.parent.getChild(0);
+ Preconditions.checkState(moreToTheLeft != queryOfSelect);
+ ASTNode newSelect = searcher.simpleBreadthFirstSearchAny((ASTNode)moreToTheLeft,
+ HiveParser.TOK_SELECT, HiveParser.TOK_SELECTDI);
+ Preconditions.checkState(newSelect != select);
+ select = newSelect;
+ // Repeat the procedure for the new select.
+ }
+
// Found the proper columns.
List<ASTNode> newChildren = new ArrayList<>(select.getChildCount());
HashSet<String> aliases = new HashSet<>();
http://git-wip-us.apache.org/repos/asf/hive/blob/ba565411/ql/src/test/queries/clientpositive/union_pos_alias.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/union_pos_alias.q b/ql/src/test/queries/clientpositive/union_pos_alias.q
index c4eca68..60fea8f 100644
--- a/ql/src/test/queries/clientpositive/union_pos_alias.q
+++ b/ql/src/test/queries/clientpositive/union_pos_alias.q
@@ -27,4 +27,22 @@ UNION ALL
select 'test', value from src_10 s3
order by 2, 1 desc;
-drop table src_10;
\ No newline at end of file
+drop table src_10;
+
+
+drop view v;
+create view v as select key as k from src intersect all select key as k1 from src;
+desc formatted v;
+
+set hive.mapred.mode=nonstrict;
+set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest;
+
+create table masking_test as select cast(key as int) as key, value from src;
+
+explain
+select * from masking_test union all select * from masking_test ;
+select * from masking_test union all select * from masking_test ;
+
+explain
+select key as k1, value as v1 from masking_test where key > 0 intersect all select key as k2, value as v2 from masking_test where key > 0;
+select key as k1, value as v1 from masking_test where key > 0 intersect all select key as k2, value as v2 from masking_test where key > 0;
http://git-wip-us.apache.org/repos/asf/hive/blob/ba565411/ql/src/test/results/clientpositive/union_pos_alias.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/union_pos_alias.q.out b/ql/src/test/results/clientpositive/union_pos_alias.q.out
index 8eddbd9..6d2f018 100644
--- a/ql/src/test/results/clientpositive/union_pos_alias.q.out
+++ b/ql/src/test/results/clientpositive/union_pos_alias.q.out
@@ -306,3 +306,309 @@ POSTHOOK: query: drop table src_10
POSTHOOK: type: DROPTABLE
POSTHOOK: Input: default@src_10
POSTHOOK: Output: default@src_10
+PREHOOK: query: drop view v
+PREHOOK: type: DROPVIEW
+POSTHOOK: query: drop view v
+POSTHOOK: type: DROPVIEW
+PREHOOK: query: create view v as select key as k from src intersect all select key as k1 from src
+PREHOOK: type: CREATEVIEW
+PREHOOK: Input: default@src
+PREHOOK: Output: database:default
+PREHOOK: Output: default@v
+POSTHOOK: query: create view v as select key as k from src intersect all select key as k1 from src
+POSTHOOK: type: CREATEVIEW
+POSTHOOK: Input: default@src
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@v
+POSTHOOK: Lineage: v.k SCRIPT [(src)src.null, (src)src.FieldSchema(name:key, type:string, comment:default), ]
+PREHOOK: query: desc formatted v
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@v
+POSTHOOK: query: desc formatted v
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@v
+# col_name data_type comment
+
+k string
+
+# Detailed Table Information
+Database: default
+#### A masked pattern was here ####
+Retention: 0
+Table Type: VIRTUAL_VIEW
+Table Parameters:
+#### A masked pattern was here ####
+
+# Storage Information
+SerDe Library: null
+InputFormat: org.apache.hadoop.mapred.TextInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+
+# View Information
+View Original Text: select key as k from src intersect all select key as k1 from src
+View Expanded Text: select `src`.`key` as `k` from `default`.`src` intersect all select `src`.`key` as `k1` from `default`.`src`
+PREHOOK: query: create table masking_test as select cast(key as int) as key, value from src
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@src
+PREHOOK: Output: database:default
+PREHOOK: Output: default@masking_test
+POSTHOOK: query: create table masking_test as select cast(key as int) as key, value from src
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@src
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@masking_test
+POSTHOOK: Lineage: masking_test.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: masking_test.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: explain
+select * from masking_test union all select * from masking_test
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select * from masking_test union all select * from masking_test
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: masking_test
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (((key % 2) = 0) and (key < 10)) (type: boolean)
+ Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: int), reverse(value) (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
+ Union
+ Statistics: Num rows: 166 Data size: 1762 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 166 Data size: 1762 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ TableScan
+ alias: masking_test
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (((key % 2) = 0) and (key < 10)) (type: boolean)
+ Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: int), reverse(value) (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
+ Union
+ Statistics: Num rows: 166 Data size: 1762 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 166 Data size: 1762 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select * from masking_test union all select * from masking_test
+PREHOOK: type: QUERY
+PREHOOK: Input: default@masking_test
+#### A masked pattern was here ####
+POSTHOOK: query: select * from masking_test union all select * from masking_test
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@masking_test
+#### A masked pattern was here ####
+0 0_lav
+0 0_lav
+4 4_lav
+4 4_lav
+8 8_lav
+8 8_lav
+0 0_lav
+0 0_lav
+0 0_lav
+0 0_lav
+2 2_lav
+2 2_lav
+PREHOOK: query: explain
+select key as k1, value as v1 from masking_test where key > 0 intersect all select key as k2, value as v2 from masking_test where key > 0
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select key as k1, value as v1 from masking_test where key > 0 intersect all select key as k2, value as v2 from masking_test where key > 0
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1, Stage-3
+ Stage-3 is a root stage
+ Stage-0 depends on stages: Stage-2
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: masking_test
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (((key % 2) = 0) and (key < 10) and (key > 0)) (type: boolean)
+ Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: int), reverse(value) (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count(1)
+ keys: _col0 (type: int), _col1 (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int), _col1 (type: string)
+ sort order: ++
+ Map-reduce partition columns: _col0 (type: int), _col1 (type: string)
+ Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col2 (type: bigint)
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ keys: KEY._col0 (type: int), KEY._col1 (type: string)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 13 Data size: 137 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-2
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Union
+ Statistics: Num rows: 26 Data size: 274 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: min(_col2), count(_col2)
+ keys: _col0 (type: int), _col1 (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 26 Data size: 274 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int), _col1 (type: string)
+ sort order: ++
+ Map-reduce partition columns: _col0 (type: int), _col1 (type: string)
+ Statistics: Num rows: 26 Data size: 274 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col2 (type: bigint), _col3 (type: bigint)
+ TableScan
+ Union
+ Statistics: Num rows: 26 Data size: 274 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: min(_col2), count(_col2)
+ keys: _col0 (type: int), _col1 (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 26 Data size: 274 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int), _col1 (type: string)
+ sort order: ++
+ Map-reduce partition columns: _col0 (type: int), _col1 (type: string)
+ Statistics: Num rows: 26 Data size: 274 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col2 (type: bigint), _col3 (type: bigint)
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: min(VALUE._col0), count(VALUE._col1)
+ keys: KEY._col0 (type: int), KEY._col1 (type: string)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 13 Data size: 137 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (_col3 = 2) (type: boolean)
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col2 (type: bigint), _col0 (type: int), _col1 (type: string)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ UDTF Operator
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ function name: UDTFReplicateRows
+ Select Operator
+ expressions: col1 (type: int), col2 (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-3
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: masking_test
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (((key % 2) = 0) and (key < 10) and (key > 0)) (type: boolean)
+ Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: int), reverse(value) (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count(1)
+ keys: _col0 (type: int), _col1 (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int), _col1 (type: string)
+ sort order: ++
+ Map-reduce partition columns: _col0 (type: int), _col1 (type: string)
+ Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col2 (type: bigint)
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ keys: KEY._col0 (type: int), KEY._col1 (type: string)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 13 Data size: 137 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select key as k1, value as v1 from masking_test where key > 0 intersect all select key as k2, value as v2 from masking_test where key > 0
+PREHOOK: type: QUERY
+PREHOOK: Input: default@masking_test
+#### A masked pattern was here ####
+POSTHOOK: query: select key as k1, value as v1 from masking_test where key > 0 intersect all select key as k2, value as v2 from masking_test where key > 0
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@masking_test
+#### A masked pattern was here ####
+2 2_lav
+4 4_lav
+8 8_lav
[02/50] [abbrv] hive git commit: HIVE-14959: Fix DISTINCT with
windowing when CBO is enabled/disabled (Jesus Camacho Rodriguez,
reviewed by Ashutosh Chauhan)
Posted by om...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/9e7daaf4/ql/src/test/results/clientpositive/windowing.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/windowing.q.out b/ql/src/test/results/clientpositive/windowing.q.out
index b9391d2..f5082dc 100644
--- a/ql/src/test/results/clientpositive/windowing.q.out
+++ b/ql/src/test/results/clientpositive/windowing.q.out
@@ -1805,6 +1805,106 @@ Manufacturer#5 almond antique medium spring khaki 6 6 37 85
Manufacturer#5 almond antique sky peru orange 2 2 39 108
Manufacturer#5 almond aquamarine dodger light gainsboro 46 46 85 77
Manufacturer#5 almond azure blanched chiffon midnight 23 23 108 71
+PREHOOK: query: explain
+select DISTINCT p_mfgr, p_name, p_size,
+sum(p_size) over w1 as s
+from part
+window w1 as (distribute by p_mfgr sort by p_name rows between 2 preceding and 2 following)
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select DISTINCT p_mfgr, p_name, p_size,
+sum(p_size) over w1 as s
+from part
+window w1 as (distribute by p_mfgr sort by p_name rows between 2 preceding and 2 following)
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-0 depends on stages: Stage-2
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: part
+ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: p_mfgr (type: string), p_name (type: string)
+ sort order: ++
+ Map-reduce partition columns: p_mfgr (type: string)
+ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE
+ value expressions: p_size (type: int)
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int)
+ outputColumnNames: _col1, _col2, _col5
+ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE
+ PTF Operator
+ Function definitions:
+ Input definition
+ input alias: ptf_0
+ output shape: _col1: string, _col2: string, _col5: int
+ type: WINDOWING
+ Windowing table definition
+ input alias: ptf_1
+ name: windowingtablefunction
+ order by: _col1 ASC NULLS FIRST
+ partition by: _col2
+ raw input shape:
+ window functions:
+ window function definition
+ alias: sum_window_0
+ arguments: _col5
+ name: sum
+ window function: GenericUDAFSumLong
+ window frame: PRECEDING(2)~FOLLOWING(2)
+ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), sum_window_0 (type: bigint)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ keys: _col0 (type: string), _col1 (type: string), _col2 (type: int), _col3 (type: bigint)
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-2
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Reduce Output Operator
+ key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: int), _col3 (type: bigint)
+ sort order: ++++
+ Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: int), _col3 (type: bigint)
+ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE
+ Reduce Operator Tree:
+ Group By Operator
+ keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: int), KEY._col3 (type: bigint)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
PREHOOK: query: select DISTINCT p_mfgr, p_name, p_size,
sum(p_size) over w1 as s
from part
@@ -1819,8 +1919,9 @@ window w1 as (distribute by p_mfgr sort by p_name rows between 2 preceding and 2
POSTHOOK: type: QUERY
POSTHOOK: Input: default@part
#### A masked pattern was here ####
-Manufacturer#1 almond antique burnished rose metallic 2 42
-Manufacturer#1 almond antique chartreuse lavender yellow 34 70
+Manufacturer#1 almond antique burnished rose metallic 2 38
+Manufacturer#1 almond antique burnished rose metallic 2 44
+Manufacturer#1 almond antique chartreuse lavender yellow 34 72
Manufacturer#1 almond antique salmon chartreuse burlywood 6 112
Manufacturer#1 almond aquamarine burnished black steel 28 110
Manufacturer#1 almond aquamarine pink moccasin thistle 42 76
[36/50] [abbrv] hive git commit: HIVE-16018: Add more information for
DynamicPartitionPruningOptimization (Pengcheng Xiong,
reviewed by Ashutosh Chauhan)
Posted by om...@apache.org.
HIVE-16018: Add more information for DynamicPartitionPruningOptimization (Pengcheng Xiong, reviewed by Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/127deb2b
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/127deb2b
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/127deb2b
Branch: refs/heads/branch-2.2
Commit: 127deb2b38183ad38f32dc1b2575e18877a12206
Parents: e27199b
Author: Pengcheng Xiong <px...@apache.org>
Authored: Sat Feb 25 12:52:20 2017 -0800
Committer: Owen O'Malley <om...@apache.org>
Committed: Tue Mar 28 15:27:54 2017 -0700
----------------------------------------------------------------------
.../DynamicPartitionPruningOptimization.java | 16 ++++++++++++++++
1 file changed, 16 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/127deb2b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java
index 8692c45..3085f5d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java
@@ -585,6 +585,14 @@ public class DynamicPartitionPruningOptimization implements NodeProcessor {
groupByDescFinal, new RowSchema(rsOp.getSchema()), rsOp);
groupByOpFinal.setColumnExprMap(new HashMap<String, ExprNodeDesc>());
+ // for explain purpose
+ if (parseContext.getContext().getExplainConfig() != null
+ && parseContext.getContext().getExplainConfig().isFormatted()) {
+ List<String> outputOperators = new ArrayList<>();
+ outputOperators.add(groupByOpFinal.getOperatorId());
+ rsOp.getConf().setOutputOperators(outputOperators);
+ }
+
// Create the final Reduce Sink Operator
ReduceSinkDesc rsDescFinal = PlanUtils.getReduceSinkDesc(
new ArrayList<ExprNodeDesc>(), rsValueCols, gbOutputNames, false,
@@ -596,6 +604,14 @@ public class DynamicPartitionPruningOptimization implements NodeProcessor {
LOG.debug("DynamicMinMaxPushdown: Saving RS to TS mapping: " + rsOpFinal + ": " + ts);
parseContext.getRsOpToTsOpMap().put(rsOpFinal, ts);
+ // for explain purpose
+ if (parseContext.getContext().getExplainConfig() != null
+ && parseContext.getContext().getExplainConfig().isFormatted()) {
+ List<String> outputOperators = new ArrayList<>();
+ outputOperators.add(ts.getOperatorId());
+ rsOpFinal.getConf().setOutputOperators(outputOperators);
+ }
+
// Save the info that is required at query time to resolve dynamic/runtime values.
RuntimeValuesInfo runtimeValuesInfo = new RuntimeValuesInfo();
TableDesc rsFinalTableDesc = PlanUtils.getReduceValueTableDesc(
[18/50] [abbrv] hive git commit: HIVE-14362: Support explain analyze
in Hive (Pengcheng Xiong, reviewed by Ashutosh Chauhan,
Gopal V and Gabor Szadovszky)
Posted by om...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/fd72ec4a/ql/src/test/results/clientpositive/tez/explainanalyze_2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/explainanalyze_2.q.out b/ql/src/test/results/clientpositive/tez/explainanalyze_2.q.out
new file mode 100644
index 0000000..37fc7d3
--- /dev/null
+++ b/ql/src/test/results/clientpositive/tez/explainanalyze_2.q.out
@@ -0,0 +1,4827 @@
+PREHOOK: query: -- SORT_QUERY_RESULTS
+
+CREATE TABLE dest_j1(key STRING, value STRING, val2 STRING) STORED AS TEXTFILE
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@dest_j1
+POSTHOOK: query: -- SORT_QUERY_RESULTS
+
+CREATE TABLE dest_j1(key STRING, value STRING, val2 STRING) STORED AS TEXTFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@dest_j1
+PREHOOK: query: CREATE TABLE ss(k1 STRING,v1 STRING,k2 STRING,v2 STRING,k3 STRING,v3 STRING) STORED AS TEXTFILE
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@ss
+POSTHOOK: query: CREATE TABLE ss(k1 STRING,v1 STRING,k2 STRING,v2 STRING,k3 STRING,v3 STRING) STORED AS TEXTFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@ss
+PREHOOK: query: CREATE TABLE sr(k1 STRING,v1 STRING,k2 STRING,v2 STRING,k3 STRING,v3 STRING) STORED AS TEXTFILE
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@sr
+POSTHOOK: query: CREATE TABLE sr(k1 STRING,v1 STRING,k2 STRING,v2 STRING,k3 STRING,v3 STRING) STORED AS TEXTFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@sr
+PREHOOK: query: CREATE TABLE cs(k1 STRING,v1 STRING,k2 STRING,v2 STRING,k3 STRING,v3 STRING) STORED AS TEXTFILE
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@cs
+POSTHOOK: query: CREATE TABLE cs(k1 STRING,v1 STRING,k2 STRING,v2 STRING,k3 STRING,v3 STRING) STORED AS TEXTFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@cs
+PREHOOK: query: INSERT OVERWRITE TABLE ss
+SELECT x.key,x.value,y.key,y.value,z.key,z.value
+FROM src1 x
+JOIN src y ON (x.key = y.key)
+JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Input: default@src1
+PREHOOK: Input: default@srcpart
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+PREHOOK: Output: default@ss
+POSTHOOK: query: INSERT OVERWRITE TABLE ss
+SELECT x.key,x.value,y.key,y.value,z.key,z.value
+FROM src1 x
+JOIN src y ON (x.key = y.key)
+JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Input: default@src1
+POSTHOOK: Input: default@srcpart
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+POSTHOOK: Output: default@ss
+POSTHOOK: Lineage: ss.k1 SIMPLE [(src1)x.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: ss.k2 SIMPLE [(src)y.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: ss.k3 SIMPLE [(srcpart)z.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: ss.v1 SIMPLE [(src1)x.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: ss.v2 SIMPLE [(src)y.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: ss.v3 SIMPLE [(srcpart)z.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: INSERT OVERWRITE TABLE sr
+SELECT x.key,x.value,y.key,y.value,z.key,z.value
+FROM src1 x
+JOIN src y ON (x.key = y.key)
+JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=12)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Input: default@src1
+PREHOOK: Input: default@srcpart
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+PREHOOK: Output: default@sr
+POSTHOOK: query: INSERT OVERWRITE TABLE sr
+SELECT x.key,x.value,y.key,y.value,z.key,z.value
+FROM src1 x
+JOIN src y ON (x.key = y.key)
+JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=12)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Input: default@src1
+POSTHOOK: Input: default@srcpart
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+POSTHOOK: Output: default@sr
+POSTHOOK: Lineage: sr.k1 SIMPLE [(src1)x.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: sr.k2 SIMPLE [(src)y.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: sr.k3 SIMPLE [(srcpart)z.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: sr.v1 SIMPLE [(src1)x.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: sr.v2 SIMPLE [(src)y.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: sr.v3 SIMPLE [(srcpart)z.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: INSERT OVERWRITE TABLE cs
+SELECT x.key,x.value,y.key,y.value,z.key,z.value
+FROM src1 x
+JOIN src y ON (x.key = y.key)
+JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08')
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Input: default@src1
+PREHOOK: Input: default@srcpart
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+PREHOOK: Output: default@cs
+POSTHOOK: query: INSERT OVERWRITE TABLE cs
+SELECT x.key,x.value,y.key,y.value,z.key,z.value
+FROM src1 x
+JOIN src y ON (x.key = y.key)
+JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Input: default@src1
+POSTHOOK: Input: default@srcpart
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+POSTHOOK: Output: default@cs
+POSTHOOK: Lineage: cs.k1 SIMPLE [(src1)x.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: cs.k2 SIMPLE [(src)y.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: cs.k3 SIMPLE [(srcpart)z.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: cs.v1 SIMPLE [(src1)x.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: cs.v2 SIMPLE [(src)y.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: cs.v3 SIMPLE [(srcpart)z.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: ANALYZE TABLE ss COMPUTE STATISTICS
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ss
+PREHOOK: Output: default@ss
+POSTHOOK: query: ANALYZE TABLE ss COMPUTE STATISTICS
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ss
+POSTHOOK: Output: default@ss
+PREHOOK: query: ANALYZE TABLE ss COMPUTE STATISTICS FOR COLUMNS k1,v1,k2,v2,k3,v3
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ss
+#### A masked pattern was here ####
+POSTHOOK: query: ANALYZE TABLE ss COMPUTE STATISTICS FOR COLUMNS k1,v1,k2,v2,k3,v3
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ss
+#### A masked pattern was here ####
+PREHOOK: query: ANALYZE TABLE sr COMPUTE STATISTICS
+PREHOOK: type: QUERY
+PREHOOK: Input: default@sr
+PREHOOK: Output: default@sr
+POSTHOOK: query: ANALYZE TABLE sr COMPUTE STATISTICS
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@sr
+POSTHOOK: Output: default@sr
+PREHOOK: query: ANALYZE TABLE sr COMPUTE STATISTICS FOR COLUMNS k1,v1,k2,v2,k3,v3
+PREHOOK: type: QUERY
+PREHOOK: Input: default@sr
+#### A masked pattern was here ####
+POSTHOOK: query: ANALYZE TABLE sr COMPUTE STATISTICS FOR COLUMNS k1,v1,k2,v2,k3,v3
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@sr
+#### A masked pattern was here ####
+PREHOOK: query: ANALYZE TABLE cs COMPUTE STATISTICS
+PREHOOK: type: QUERY
+PREHOOK: Input: default@cs
+PREHOOK: Output: default@cs
+POSTHOOK: query: ANALYZE TABLE cs COMPUTE STATISTICS
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@cs
+POSTHOOK: Output: default@cs
+PREHOOK: query: ANALYZE TABLE cs COMPUTE STATISTICS FOR COLUMNS k1,v1,k2,v2,k3,v3
+PREHOOK: type: QUERY
+PREHOOK: Input: default@cs
+#### A masked pattern was here ####
+POSTHOOK: query: ANALYZE TABLE cs COMPUTE STATISTICS FOR COLUMNS k1,v1,k2,v2,k3,v3
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@cs
+#### A masked pattern was here ####
+PREHOOK: query: SELECT x.key, z.value, y.value
+FROM src1 x JOIN src y ON (x.key = y.key)
+JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Input: default@src1
+PREHOOK: Input: default@srcpart
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT x.key, z.value, y.value
+FROM src1 x JOIN src y ON (x.key = y.key)
+JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Input: default@src1
+POSTHOOK: Input: default@srcpart
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+#### A masked pattern was here ####
+PREHOOK: query: explain analyze
+SELECT x.key, z.value, y.value
+FROM src1 x JOIN src y ON (x.key = y.key)
+JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11)
+PREHOOK: type: QUERY
+POSTHOOK: query: explain analyze
+SELECT x.key, z.value, y.value
+FROM src1 x JOIN src y ON (x.key = y.key)
+JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11)
+POSTHOOK: type: QUERY
+Plan optimized by CBO.
+
+Vertex dependency in root stage
+Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE)
+Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE)
+
+Stage-0
+ Fetch Operator
+ limit:-1
+ Stage-1
+ Reducer 3
+ File Output Operator [FS_16]
+ Select Operator [SEL_15] (rows=141/85 width=268)
+ Output:["_col0","_col1","_col2"]
+ Merge Join Operator [MERGEJOIN_26] (rows=141/85 width=268)
+ Conds:RS_12._col3=RS_13._col0(Inner),Output:["_col0","_col3","_col6"]
+ <-Map 5 [SIMPLE_EDGE]
+ SHUFFLE [RS_13]
+ PartitionCols:_col0
+ Select Operator [SEL_8] (rows=500/500 width=178)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_24] (rows=500/500 width=178)
+ predicate:key is not null
+ TableScan [TS_6] (rows=500/500 width=178)
+ default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
+ <-Reducer 2 [SIMPLE_EDGE]
+ SHUFFLE [RS_12]
+ PartitionCols:_col3
+ Merge Join Operator [MERGEJOIN_25] (rows=58/41 width=177)
+ Conds:RS_9._col0=RS_10._col1(Inner),Output:["_col0","_col3"]
+ <-Map 1 [SIMPLE_EDGE]
+ SHUFFLE [RS_9]
+ PartitionCols:_col0
+ Select Operator [SEL_2] (rows=500/500 width=369)
+ Output:["_col0"]
+ Filter Operator [FIL_22] (rows=500/500 width=91)
+ predicate:value is not null
+ TableScan [TS_0] (rows=500/500 width=91)
+ default@srcpart,z,Tbl:COMPLETE,Col:COMPLETE,Output:["value"]
+ <-Map 4 [SIMPLE_EDGE]
+ SHUFFLE [RS_10]
+ PartitionCols:_col1
+ Select Operator [SEL_5] (rows=25/25 width=175)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_23] (rows=25/25 width=175)
+ predicate:(key is not null and value is not null)
+ TableScan [TS_3] (rows=25/25 width=175)
+ default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
+
+PREHOOK: query: select
+ss.k1,sr.k2,cs.k3,count(ss.v1),count(sr.v2),count(cs.v3)
+FROM
+ss,sr,cs,src d1,src d2,src d3,src1,srcpart
+where
+ ss.k1 = d1.key
+and sr.k1 = d2.key
+and cs.k1 = d3.key
+and ss.k2 = sr.k2
+and ss.k3 = sr.k3
+and ss.v1 = src1.value
+and ss.v2 = srcpart.value
+and sr.v2 = cs.v2
+and sr.v3 = cs.v3
+and ss.v3='ssv3'
+and sr.v1='srv1'
+and src1.key = 'src1key'
+and srcpart.key = 'srcpartkey'
+and d1.value = 'd1value'
+and d2.value in ('2000Q1','2000Q2','2000Q3')
+and d3.value in ('2000Q1','2000Q2','2000Q3')
+group by
+ss.k1,sr.k2,cs.k3
+order by
+ss.k1,sr.k2,cs.k3
+limit 100
+PREHOOK: type: QUERY
+PREHOOK: Input: default@cs
+PREHOOK: Input: default@sr
+PREHOOK: Input: default@src
+PREHOOK: Input: default@src1
+PREHOOK: Input: default@srcpart
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
+PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
+PREHOOK: Input: default@ss
+#### A masked pattern was here ####
+POSTHOOK: query: select
+ss.k1,sr.k2,cs.k3,count(ss.v1),count(sr.v2),count(cs.v3)
+FROM
+ss,sr,cs,src d1,src d2,src d3,src1,srcpart
+where
+ ss.k1 = d1.key
+and sr.k1 = d2.key
+and cs.k1 = d3.key
+and ss.k2 = sr.k2
+and ss.k3 = sr.k3
+and ss.v1 = src1.value
+and ss.v2 = srcpart.value
+and sr.v2 = cs.v2
+and sr.v3 = cs.v3
+and ss.v3='ssv3'
+and sr.v1='srv1'
+and src1.key = 'src1key'
+and srcpart.key = 'srcpartkey'
+and d1.value = 'd1value'
+and d2.value in ('2000Q1','2000Q2','2000Q3')
+and d3.value in ('2000Q1','2000Q2','2000Q3')
+group by
+ss.k1,sr.k2,cs.k3
+order by
+ss.k1,sr.k2,cs.k3
+limit 100
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@cs
+POSTHOOK: Input: default@sr
+POSTHOOK: Input: default@src
+POSTHOOK: Input: default@src1
+POSTHOOK: Input: default@srcpart
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
+POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
+POSTHOOK: Input: default@ss
+#### A masked pattern was here ####
+PREHOOK: query: explain analyze
+select
+ss.k1,sr.k2,cs.k3,count(ss.v1),count(sr.v2),count(cs.v3)
+FROM
+ss,sr,cs,src d1,src d2,src d3,src1,srcpart
+where
+ ss.k1 = d1.key
+and sr.k1 = d2.key
+and cs.k1 = d3.key
+and ss.k2 = sr.k2
+and ss.k3 = sr.k3
+and ss.v1 = src1.value
+and ss.v2 = srcpart.value
+and sr.v2 = cs.v2
+and sr.v3 = cs.v3
+and ss.v3='ssv3'
+and sr.v1='srv1'
+and src1.key = 'src1key'
+and srcpart.key = 'srcpartkey'
+and d1.value = 'd1value'
+and d2.value in ('2000Q1','2000Q2','2000Q3')
+and d3.value in ('2000Q1','2000Q2','2000Q3')
+group by
+ss.k1,sr.k2,cs.k3
+order by
+ss.k1,sr.k2,cs.k3
+limit 100
+PREHOOK: type: QUERY
+POSTHOOK: query: explain analyze
+select
+ss.k1,sr.k2,cs.k3,count(ss.v1),count(sr.v2),count(cs.v3)
+FROM
+ss,sr,cs,src d1,src d2,src d3,src1,srcpart
+where
+ ss.k1 = d1.key
+and sr.k1 = d2.key
+and cs.k1 = d3.key
+and ss.k2 = sr.k2
+and ss.k3 = sr.k3
+and ss.v1 = src1.value
+and ss.v2 = srcpart.value
+and sr.v2 = cs.v2
+and sr.v3 = cs.v3
+and ss.v3='ssv3'
+and sr.v1='srv1'
+and src1.key = 'src1key'
+and srcpart.key = 'srcpartkey'
+and d1.value = 'd1value'
+and d2.value in ('2000Q1','2000Q2','2000Q3')
+and d3.value in ('2000Q1','2000Q2','2000Q3')
+group by
+ss.k1,sr.k2,cs.k3
+order by
+ss.k1,sr.k2,cs.k3
+limit 100
+POSTHOOK: type: QUERY
+Plan optimized by CBO.
+
+Vertex dependency in root stage
+Reducer 10 <- Map 14 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE)
+Reducer 11 <- Reducer 10 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE)
+Reducer 16 <- Map 15 (SIMPLE_EDGE), Map 17 (SIMPLE_EDGE)
+Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE)
+Reducer 3 <- Reducer 11 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE)
+Reducer 4 <- Reducer 3 (SIMPLE_EDGE)
+Reducer 5 <- Reducer 4 (SIMPLE_EDGE)
+Reducer 8 <- Map 12 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE)
+Reducer 9 <- Map 13 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE)
+
+Stage-0
+ Fetch Operator
+ limit:100
+ Stage-1
+ Reducer 5
+ File Output Operator [FS_55]
+ Limit [LIM_54] (rows=24/0 width=285)
+ Number of rows:100
+ Select Operator [SEL_53] (rows=24/0 width=285)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
+ <-Reducer 4 [SIMPLE_EDGE]
+ SHUFFLE [RS_52]
+ Group By Operator [GBY_50] (rows=24/0 width=285)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["count(VALUE._col0)","count(VALUE._col1)","count(VALUE._col2)"],keys:KEY._col0, KEY._col1, KEY._col2
+ <-Reducer 3 [SIMPLE_EDGE]
+ SHUFFLE [RS_49]
+ PartitionCols:_col0, _col1, _col2
+ Group By Operator [GBY_48] (rows=24/0 width=285)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["count(_col13)","count(_col21)","count(_col3)"],keys:_col12, _col20, _col2
+ Select Operator [SEL_47] (rows=650/0 width=534)
+ Output:["_col12","_col20","_col2","_col13","_col21","_col3"]
+ Merge Join Operator [MERGEJOIN_97] (rows=650/0 width=534)
+ Conds:RS_44._col1, _col3=RS_45._col15, _col17(Inner),Output:["_col2","_col3","_col12","_col13","_col20","_col21"]
+ <-Reducer 11 [SIMPLE_EDGE]
+ SHUFFLE [RS_45]
+ PartitionCols:_col15, _col17
+ Select Operator [SEL_40] (rows=190/0 width=447)
+ Output:["_col14","_col15","_col17","_col6","_col7"]
+ Merge Join Operator [MERGEJOIN_96] (rows=190/0 width=447)
+ Conds:RS_37._col6, _col4=RS_38._col4, _col2(Inner),Output:["_col2","_col3","_col14","_col15","_col17"]
+ <-Reducer 10 [SIMPLE_EDGE]
+ SHUFFLE [RS_37]
+ PartitionCols:_col6, _col4
+ Merge Join Operator [MERGEJOIN_94] (rows=40/0 width=352)
+ Conds:RS_34._col3=RS_35._col1(Inner),Output:["_col2","_col3","_col4","_col6"]
+ <-Map 14 [SIMPLE_EDGE]
+ SHUFFLE [RS_35]
+ PartitionCols:_col1
+ Select Operator [SEL_17] (rows=2/0 width=180)
+ Output:["_col1"]
+ Filter Operator [FIL_88] (rows=2/0 width=175)
+ predicate:((key = 'src1key') and value is not null)
+ TableScan [TS_15] (rows=25/25 width=175)
+ default@src1,src1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
+ <-Reducer 9 [SIMPLE_EDGE]
+ SHUFFLE [RS_34]
+ PartitionCols:_col3
+ Merge Join Operator [MERGEJOIN_93] (rows=40/0 width=352)
+ Conds:RS_31._col2=RS_32._col0(Inner),Output:["_col2","_col3","_col4","_col6"]
+ <-Map 13 [SIMPLE_EDGE]
+ SHUFFLE [RS_32]
+ PartitionCols:_col0
+ Select Operator [SEL_14] (rows=2/0 width=178)
+ Output:["_col0"]
+ Filter Operator [FIL_87] (rows=2/0 width=178)
+ predicate:((value = 'd1value') and key is not null)
+ TableScan [TS_12] (rows=500/500 width=178)
+ default@src,d1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
+ <-Reducer 8 [SIMPLE_EDGE]
+ SHUFFLE [RS_31]
+ PartitionCols:_col2
+ Merge Join Operator [MERGEJOIN_92] (rows=40/0 width=352)
+ Conds:RS_28._col1=RS_29._col3(Inner),Output:["_col2","_col3","_col4","_col6"]
+ <-Map 12 [SIMPLE_EDGE]
+ SHUFFLE [RS_29]
+ PartitionCols:_col3
+ Select Operator [SEL_11] (rows=8/0 width=531)
+ Output:["_col0","_col1","_col2","_col3","_col4"]
+ Filter Operator [FIL_86] (rows=8/0 width=534)
+ predicate:((v3 = 'ssv3') and k2 is not null and k3 is not null and k1 is not null and v1 is not null and v2 is not null)
+ TableScan [TS_9] (rows=85/85 width=534)
+ default@ss,ss,Tbl:COMPLETE,Col:COMPLETE,Output:["k1","v1","k2","v2","k3","v3"]
+ <-Map 7 [SIMPLE_EDGE]
+ SHUFFLE [RS_28]
+ PartitionCols:_col1
+ Select Operator [SEL_8] (rows=10/0 width=185)
+ Output:["_col1"]
+ Filter Operator [FIL_85] (rows=10/0 width=178)
+ predicate:((key = 'srcpartkey') and value is not null)
+ TableScan [TS_6] (rows=2000/2000 width=178)
+ default@srcpart,srcpart,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
+ <-Reducer 16 [SIMPLE_EDGE]
+ SHUFFLE [RS_38]
+ PartitionCols:_col4, _col2
+ Merge Join Operator [MERGEJOIN_95] (rows=19/0 width=356)
+ Conds:RS_24._col0=RS_25._col0(Inner),Output:["_col2","_col3","_col4","_col5"]
+ <-Map 15 [SIMPLE_EDGE]
+ SHUFFLE [RS_24]
+ PartitionCols:_col0
+ Select Operator [SEL_20] (rows=8/0 width=531)
+ Output:["_col0","_col2","_col3","_col4","_col5"]
+ Filter Operator [FIL_89] (rows=8/0 width=534)
+ predicate:((v1 = 'srv1') and k2 is not null and k3 is not null and v2 is not null and v3 is not null and k1 is not null)
+ TableScan [TS_18] (rows=85/85 width=534)
+ default@sr,sr,Tbl:COMPLETE,Col:COMPLETE,Output:["k1","v1","k2","v2","k3","v3"]
+ <-Map 17 [SIMPLE_EDGE]
+ SHUFFLE [RS_25]
+ PartitionCols:_col0
+ Select Operator [SEL_23] (rows=500/0 width=178)
+ Output:["_col0"]
+ Filter Operator [FIL_90] (rows=500/0 width=178)
+ predicate:((value) IN ('2000Q1', '2000Q2', '2000Q3') and key is not null)
+ TableScan [TS_21] (rows=500/500 width=178)
+ default@src,d2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
+ <-Reducer 2 [SIMPLE_EDGE]
+ SHUFFLE [RS_44]
+ PartitionCols:_col1, _col3
+ Merge Join Operator [MERGEJOIN_91] (rows=414/0 width=269)
+ Conds:RS_41._col0=RS_42._col0(Inner),Output:["_col1","_col2","_col3"]
+ <-Map 1 [SIMPLE_EDGE]
+ SHUFFLE [RS_41]
+ PartitionCols:_col0
+ Select Operator [SEL_2] (rows=170/170 width=356)
+ Output:["_col0","_col1","_col2","_col3"]
+ Filter Operator [FIL_83] (rows=170/170 width=356)
+ predicate:(v2 is not null and v3 is not null and k1 is not null)
+ TableScan [TS_0] (rows=170/170 width=356)
+ default@cs,cs,Tbl:COMPLETE,Col:COMPLETE,Output:["k1","v2","k3","v3"]
+ <-Map 6 [SIMPLE_EDGE]
+ SHUFFLE [RS_42]
+ PartitionCols:_col0
+ Select Operator [SEL_5] (rows=500/0 width=178)
+ Output:["_col0"]
+ Filter Operator [FIL_84] (rows=500/0 width=178)
+ predicate:((value) IN ('2000Q1', '2000Q2', '2000Q3') and key is not null)
+ TableScan [TS_3] (rows=500/500 width=178)
+ default@src,d3,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
+
+PREHOOK: query: SELECT x.key, z.value, y.value
+FROM src1 x JOIN src y ON (x.key = y.key)
+JOIN (select * from src1 union select * from src)z ON (x.value = z.value)
+union
+SELECT x.key, z.value, y.value
+FROM src1 x JOIN src y ON (x.key = y.key)
+JOIN (select * from src1 union select * from src)z ON (x.value = z.value)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Input: default@src1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT x.key, z.value, y.value
+FROM src1 x JOIN src y ON (x.key = y.key)
+JOIN (select * from src1 union select * from src)z ON (x.value = z.value)
+union
+SELECT x.key, z.value, y.value
+FROM src1 x JOIN src y ON (x.key = y.key)
+JOIN (select * from src1 union select * from src)z ON (x.value = z.value)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Input: default@src1
+#### A masked pattern was here ####
+PREHOOK: query: explain analyze
+SELECT x.key, z.value, y.value
+FROM src1 x JOIN src y ON (x.key = y.key)
+JOIN (select * from src1 union select * from src)z ON (x.value = z.value)
+union
+SELECT x.key, z.value, y.value
+FROM src1 x JOIN src y ON (x.key = y.key)
+JOIN (select * from src1 union select * from src)z ON (x.value = z.value)
+PREHOOK: type: QUERY
+POSTHOOK: query: explain analyze
+SELECT x.key, z.value, y.value
+FROM src1 x JOIN src y ON (x.key = y.key)
+JOIN (select * from src1 union select * from src)z ON (x.value = z.value)
+union
+SELECT x.key, z.value, y.value
+FROM src1 x JOIN src y ON (x.key = y.key)
+JOIN (select * from src1 union select * from src)z ON (x.value = z.value)
+POSTHOOK: type: QUERY
+Plan optimized by CBO.
+
+Vertex dependency in root stage
+Map 1 <- Union 2 (CONTAINS)
+Map 11 <- Union 12 (CONTAINS)
+Map 16 <- Union 12 (CONTAINS)
+Map 8 <- Union 2 (CONTAINS)
+Reducer 13 <- Union 12 (SIMPLE_EDGE)
+Reducer 14 <- Map 17 (SIMPLE_EDGE), Reducer 13 (SIMPLE_EDGE)
+Reducer 15 <- Map 18 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE), Union 6 (CONTAINS)
+Reducer 3 <- Union 2 (SIMPLE_EDGE)
+Reducer 4 <- Map 9 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE)
+Reducer 5 <- Map 10 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE), Union 6 (CONTAINS)
+Reducer 7 <- Union 6 (SIMPLE_EDGE)
+
+Stage-0
+ Fetch Operator
+ limit:-1
+ Stage-1
+ Reducer 7
+ File Output Operator [FS_59]
+ Group By Operator [GBY_57] (rows=28/15 width=177)
+ Output:["_col0","_col1"],keys:KEY._col0, KEY._col1
+ <-Union 6 [SIMPLE_EDGE]
+ <-Reducer 15 [CONTAINS]
+ Reduce Output Operator [RS_56]
+ PartitionCols:_col0, _col1
+ Group By Operator [GBY_55] (rows=28/30 width=177)
+ Output:["_col0","_col1"],keys:_col0, _col1
+ Select Operator [SEL_51] (rows=73/61 width=177)
+ Output:["_col0","_col1"]
+ Merge Join Operator [MERGEJOIN_85] (rows=73/61 width=177)
+ Conds:RS_48._col2=RS_49._col0(Inner),Output:["_col1","_col2"]
+ <-Map 18 [SIMPLE_EDGE]
+ SHUFFLE [RS_49]
+ PartitionCols:_col0
+ Select Operator [SEL_44] (rows=500/500 width=87)
+ Output:["_col0"]
+ Filter Operator [FIL_81] (rows=500/500 width=87)
+ predicate:key is not null
+ TableScan [TS_42] (rows=500/500 width=87)
+ default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key"]
+ <-Reducer 14 [SIMPLE_EDGE]
+ SHUFFLE [RS_48]
+ PartitionCols:_col2
+ Merge Join Operator [MERGEJOIN_84] (rows=30/52 width=177)
+ Conds:RS_45._col1=RS_46._col1(Inner),Output:["_col1","_col2"]
+ <-Map 17 [SIMPLE_EDGE]
+ SHUFFLE [RS_46]
+ PartitionCols:_col1
+ Select Operator [SEL_41] (rows=25/25 width=175)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_80] (rows=25/25 width=175)
+ predicate:(key is not null and value is not null)
+ TableScan [TS_39] (rows=25/25 width=175)
+ default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
+ <-Reducer 13 [SIMPLE_EDGE]
+ SHUFFLE [RS_45]
+ PartitionCols:_col1
+ Select Operator [SEL_38] (rows=262/319 width=178)
+ Output:["_col1"]
+ Group By Operator [GBY_37] (rows=262/319 width=178)
+ Output:["_col0","_col1"],keys:KEY._col0, KEY._col1
+ <-Union 12 [SIMPLE_EDGE]
+ <-Map 11 [CONTAINS]
+ Reduce Output Operator [RS_36]
+ PartitionCols:_col0, _col1
+ Group By Operator [GBY_35] (rows=262/331 width=178)
+ Output:["_col0","_col1"],keys:_col1, _col0
+ Select Operator [SEL_28] (rows=25/25 width=175)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_78] (rows=25/25 width=175)
+ predicate:value is not null
+ TableScan [TS_26] (rows=25/25 width=175)
+ Output:["key","value"]
+ <-Map 16 [CONTAINS]
+ Reduce Output Operator [RS_36]
+ PartitionCols:_col0, _col1
+ Group By Operator [GBY_35] (rows=262/331 width=178)
+ Output:["_col0","_col1"],keys:_col1, _col0
+ Select Operator [SEL_31] (rows=500/500 width=178)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_79] (rows=500/500 width=178)
+ predicate:value is not null
+ TableScan [TS_29] (rows=500/500 width=178)
+ Output:["key","value"]
+ <-Reducer 5 [CONTAINS]
+ Reduce Output Operator [RS_56]
+ PartitionCols:_col0, _col1
+ Group By Operator [GBY_55] (rows=28/30 width=177)
+ Output:["_col0","_col1"],keys:_col0, _col1
+ Select Operator [SEL_25] (rows=73/61 width=177)
+ Output:["_col0","_col1"]
+ Merge Join Operator [MERGEJOIN_83] (rows=73/61 width=177)
+ Conds:RS_22._col2=RS_23._col0(Inner),Output:["_col1","_col2"]
+ <-Map 10 [SIMPLE_EDGE]
+ SHUFFLE [RS_23]
+ PartitionCols:_col0
+ Select Operator [SEL_18] (rows=500/500 width=87)
+ Output:["_col0"]
+ Filter Operator [FIL_77] (rows=500/500 width=87)
+ predicate:key is not null
+ TableScan [TS_16] (rows=500/500 width=87)
+ default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key"]
+ <-Reducer 4 [SIMPLE_EDGE]
+ SHUFFLE [RS_22]
+ PartitionCols:_col2
+ Merge Join Operator [MERGEJOIN_82] (rows=30/52 width=177)
+ Conds:RS_19._col1=RS_20._col1(Inner),Output:["_col1","_col2"]
+ <-Map 9 [SIMPLE_EDGE]
+ SHUFFLE [RS_20]
+ PartitionCols:_col1
+ Select Operator [SEL_15] (rows=25/25 width=175)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_76] (rows=25/25 width=175)
+ predicate:(key is not null and value is not null)
+ TableScan [TS_13] (rows=25/25 width=175)
+ default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
+ <-Reducer 3 [SIMPLE_EDGE]
+ SHUFFLE [RS_19]
+ PartitionCols:_col1
+ Select Operator [SEL_12] (rows=262/319 width=178)
+ Output:["_col1"]
+ Group By Operator [GBY_11] (rows=262/319 width=178)
+ Output:["_col0","_col1"],keys:KEY._col0, KEY._col1
+ <-Union 2 [SIMPLE_EDGE]
+ <-Map 1 [CONTAINS]
+ Reduce Output Operator [RS_10]
+ PartitionCols:_col0, _col1
+ Group By Operator [GBY_9] (rows=262/331 width=178)
+ Output:["_col0","_col1"],keys:_col1, _col0
+ Select Operator [SEL_2] (rows=25/25 width=175)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_74] (rows=25/25 width=175)
+ predicate:value is not null
+ TableScan [TS_0] (rows=25/25 width=175)
+ Output:["key","value"]
+ <-Map 8 [CONTAINS]
+ Reduce Output Operator [RS_10]
+ PartitionCols:_col0, _col1
+ Group By Operator [GBY_9] (rows=262/331 width=178)
+ Output:["_col0","_col1"],keys:_col1, _col0
+ Select Operator [SEL_5] (rows=500/500 width=178)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_75] (rows=500/500 width=178)
+ predicate:value is not null
+ TableScan [TS_3] (rows=500/500 width=178)
+ Output:["key","value"]
+
+PREHOOK: query: SELECT x.key, y.value
+FROM src1 x JOIN src y ON (x.key = y.key)
+JOIN (select * from src1 union select * from src)z ON (x.value = z.value)
+union
+SELECT x.key, y.value
+FROM src1 x JOIN src y ON (x.key = y.key)
+JOIN (select key, value from src1 union select key, value from src union select key, value from src)z ON (x.value = z.value)
+union
+SELECT x.key, y.value
+FROM src1 x JOIN src y ON (x.key = y.key)
+JOIN (select key, value from src1 union select key, value from src union select key, value from src union select key, value from src)z ON (x.value = z.value)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Input: default@src1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT x.key, y.value
+FROM src1 x JOIN src y ON (x.key = y.key)
+JOIN (select * from src1 union select * from src)z ON (x.value = z.value)
+union
+SELECT x.key, y.value
+FROM src1 x JOIN src y ON (x.key = y.key)
+JOIN (select key, value from src1 union select key, value from src union select key, value from src)z ON (x.value = z.value)
+union
+SELECT x.key, y.value
+FROM src1 x JOIN src y ON (x.key = y.key)
+JOIN (select key, value from src1 union select key, value from src union select key, value from src union select key, value from src)z ON (x.value = z.value)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Input: default@src1
+#### A masked pattern was here ####
+PREHOOK: query: explain analyze
+SELECT x.key, y.value
+FROM src1 x JOIN src y ON (x.key = y.key)
+JOIN (select * from src1 union select * from src)z ON (x.value = z.value)
+union
+SELECT x.key, y.value
+FROM src1 x JOIN src y ON (x.key = y.key)
+JOIN (select key, value from src1 union select key, value from src union select key, value from src)z ON (x.value = z.value)
+union
+SELECT x.key, y.value
+FROM src1 x JOIN src y ON (x.key = y.key)
+JOIN (select key, value from src1 union select key, value from src union select key, value from src union select key, value from src)z ON (x.value = z.value)
+PREHOOK: type: QUERY
+POSTHOOK: query: explain analyze
+SELECT x.key, y.value
+FROM src1 x JOIN src y ON (x.key = y.key)
+JOIN (select * from src1 union select * from src)z ON (x.value = z.value)
+union
+SELECT x.key, y.value
+FROM src1 x JOIN src y ON (x.key = y.key)
+JOIN (select key, value from src1 union select key, value from src union select key, value from src)z ON (x.value = z.value)
+union
+SELECT x.key, y.value
+FROM src1 x JOIN src y ON (x.key = y.key)
+JOIN (select key, value from src1 union select key, value from src union select key, value from src union select key, value from src)z ON (x.value = z.value)
+POSTHOOK: type: QUERY
+Plan optimized by CBO.
+
+Vertex dependency in root stage
+Map 1 <- Union 2 (CONTAINS)
+Map 10 <- Union 2 (CONTAINS)
+Map 13 <- Union 14 (CONTAINS)
+Map 20 <- Union 14 (CONTAINS)
+Map 21 <- Union 16 (CONTAINS)
+Map 24 <- Union 25 (CONTAINS)
+Map 33 <- Union 25 (CONTAINS)
+Map 34 <- Union 27 (CONTAINS)
+Map 35 <- Union 29 (CONTAINS)
+Reducer 15 <- Union 14 (SIMPLE_EDGE), Union 16 (CONTAINS)
+Reducer 17 <- Union 16 (SIMPLE_EDGE)
+Reducer 18 <- Map 22 (SIMPLE_EDGE), Reducer 17 (SIMPLE_EDGE)
+Reducer 19 <- Map 23 (SIMPLE_EDGE), Reducer 18 (SIMPLE_EDGE), Union 6 (CONTAINS)
+Reducer 26 <- Union 25 (SIMPLE_EDGE), Union 27 (CONTAINS)
+Reducer 28 <- Union 27 (SIMPLE_EDGE), Union 29 (CONTAINS)
+Reducer 3 <- Union 2 (SIMPLE_EDGE)
+Reducer 30 <- Union 29 (SIMPLE_EDGE)
+Reducer 31 <- Map 36 (SIMPLE_EDGE), Reducer 30 (SIMPLE_EDGE)
+Reducer 32 <- Map 37 (SIMPLE_EDGE), Reducer 31 (SIMPLE_EDGE), Union 8 (CONTAINS)
+Reducer 4 <- Map 11 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE)
+Reducer 5 <- Map 12 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE), Union 6 (CONTAINS)
+Reducer 7 <- Union 6 (SIMPLE_EDGE), Union 8 (CONTAINS)
+Reducer 9 <- Union 8 (SIMPLE_EDGE)
+
+Stage-0
+ Fetch Operator
+ limit:-1
+ Stage-1
+ Reducer 9
+ File Output Operator [FS_122]
+ Group By Operator [GBY_120] (rows=107/15 width=177)
+ Output:["_col0","_col1"],keys:KEY._col0, KEY._col1
+ <-Union 8 [SIMPLE_EDGE]
+ <-Reducer 32 [CONTAINS]
+ Reduce Output Operator [RS_119]
+ PartitionCols:_col0, _col1
+ Group By Operator [GBY_118] (rows=107/30 width=177)
+ Output:["_col0","_col1"],keys:_col0, _col1
+ Select Operator [SEL_114] (rows=124/61 width=177)
+ Output:["_col0","_col1"]
+ Merge Join Operator [MERGEJOIN_170] (rows=124/61 width=177)
+ Conds:RS_111._col2=RS_112._col0(Inner),Output:["_col2","_col5"]
+ <-Map 37 [SIMPLE_EDGE]
+ SHUFFLE [RS_112]
+ PartitionCols:_col0
+ Select Operator [SEL_107] (rows=500/500 width=178)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_164] (rows=500/500 width=178)
+ predicate:key is not null
+ TableScan [TS_105] (rows=500/500 width=178)
+ default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
+ <-Reducer 31 [SIMPLE_EDGE]
+ SHUFFLE [RS_111]
+ PartitionCols:_col2
+ Merge Join Operator [MERGEJOIN_169] (rows=51/52 width=86)
+ Conds:RS_108._col1=RS_109._col1(Inner),Output:["_col2"]
+ <-Map 36 [SIMPLE_EDGE]
+ SHUFFLE [RS_109]
+ PartitionCols:_col1
+ Select Operator [SEL_104] (rows=25/25 width=175)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_163] (rows=25/25 width=175)
+ predicate:(key is not null and value is not null)
+ TableScan [TS_102] (rows=25/25 width=175)
+ default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
+ <-Reducer 30 [SIMPLE_EDGE]
+ SHUFFLE [RS_108]
+ PartitionCols:_col1
+ Select Operator [SEL_101] (rows=440/319 width=178)
+ Output:["_col1"]
+ Group By Operator [GBY_100] (rows=440/319 width=178)
+ Output:["_col0","_col1"],keys:KEY._col0, KEY._col1
+ <-Union 29 [SIMPLE_EDGE]
+ <-Map 35 [CONTAINS]
+ Reduce Output Operator [RS_99]
+ PartitionCols:_col0, _col1
+ Group By Operator [GBY_98] (rows=440/628 width=178)
+ Output:["_col0","_col1"],keys:_col1, _col0
+ Select Operator [SEL_94] (rows=500/500 width=178)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_162] (rows=500/500 width=178)
+ predicate:value is not null
+ TableScan [TS_92] (rows=500/500 width=178)
+ Output:["key","value"]
+ <-Reducer 28 [CONTAINS]
+ Reduce Output Operator [RS_99]
+ PartitionCols:_col0, _col1
+ Group By Operator [GBY_98] (rows=440/628 width=178)
+ Output:["_col0","_col1"],keys:_col1, _col0
+ Select Operator [SEL_91] (rows=381/319 width=178)
+ Output:["_col0","_col1"]
+ Group By Operator [GBY_90] (rows=381/319 width=178)
+ Output:["_col0","_col1"],keys:KEY._col0, KEY._col1
+ <-Union 27 [SIMPLE_EDGE]
+ <-Map 34 [CONTAINS]
+ Reduce Output Operator [RS_89]
+ PartitionCols:_col0, _col1
+ Group By Operator [GBY_88] (rows=381/628 width=178)
+ Output:["_col0","_col1"],keys:_col1, _col0
+ Select Operator [SEL_84] (rows=500/500 width=178)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_161] (rows=500/500 width=178)
+ predicate:value is not null
+ TableScan [TS_82] (rows=500/500 width=178)
+ Output:["key","value"]
+ <-Reducer 26 [CONTAINS]
+ Reduce Output Operator [RS_89]
+ PartitionCols:_col0, _col1
+ Group By Operator [GBY_88] (rows=381/628 width=178)
+ Output:["_col0","_col1"],keys:_col1, _col0
+ Select Operator [SEL_81] (rows=262/319 width=178)
+ Output:["_col0","_col1"]
+ Group By Operator [GBY_80] (rows=262/319 width=178)
+ Output:["_col0","_col1"],keys:KEY._col0, KEY._col1
+ <-Union 25 [SIMPLE_EDGE]
+ <-Map 24 [CONTAINS]
+ Reduce Output Operator [RS_79]
+ PartitionCols:_col0, _col1
+ Group By Operator [GBY_78] (rows=262/331 width=178)
+ Output:["_col0","_col1"],keys:_col1, _col0
+ Select Operator [SEL_71] (rows=25/25 width=175)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_159] (rows=25/25 width=175)
+ predicate:value is not null
+ TableScan [TS_69] (rows=25/25 width=175)
+ Output:["key","value"]
+ <-Map 33 [CONTAINS]
+ Reduce Output Operator [RS_79]
+ PartitionCols:_col0, _col1
+ Group By Operator [GBY_78] (rows=262/331 width=178)
+ Output:["_col0","_col1"],keys:_col1, _col0
+ Select Operator [SEL_74] (rows=500/500 width=178)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_160] (rows=500/500 width=178)
+ predicate:value is not null
+ TableScan [TS_72] (rows=500/500 width=178)
+ Output:["key","value"]
+ <-Reducer 7 [CONTAINS]
+ Reduce Output Operator [RS_119]
+ PartitionCols:_col0, _col1
+ Group By Operator [GBY_118] (rows=107/30 width=177)
+ Output:["_col0","_col1"],keys:_col0, _col1
+ Group By Operator [GBY_67] (rows=90/15 width=177)
+ Output:["_col0","_col1"],keys:KEY._col0, KEY._col1
+ <-Union 6 [SIMPLE_EDGE]
+ <-Reducer 19 [CONTAINS]
+ Reduce Output Operator [RS_66]
+ PartitionCols:_col0, _col1
+ Group By Operator [GBY_65] (rows=90/30 width=177)
+ Output:["_col0","_col1"],keys:_col0, _col1
+ Select Operator [SEL_61] (rows=107/61 width=177)
+ Output:["_col0","_col1"]
+ Merge Join Operator [MERGEJOIN_168] (rows=107/61 width=177)
+ Conds:RS_58._col2=RS_59._col0(Inner),Output:["_col2","_col5"]
+ <-Map 23 [SIMPLE_EDGE]
+ SHUFFLE [RS_59]
+ PartitionCols:_col0
+ Select Operator [SEL_54] (rows=500/500 width=178)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_158] (rows=500/500 width=178)
+ predicate:key is not null
+ TableScan [TS_52] (rows=500/500 width=178)
+ default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
+ <-Reducer 18 [SIMPLE_EDGE]
+ SHUFFLE [RS_58]
+ PartitionCols:_col2
+ Merge Join Operator [MERGEJOIN_167] (rows=44/52 width=86)
+ Conds:RS_55._col1=RS_56._col1(Inner),Output:["_col2"]
+ <-Map 22 [SIMPLE_EDGE]
+ SHUFFLE [RS_56]
+ PartitionCols:_col1
+ Select Operator [SEL_51] (rows=25/25 width=175)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_157] (rows=25/25 width=175)
+ predicate:(key is not null and value is not null)
+ TableScan [TS_49] (rows=25/25 width=175)
+ default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
+ <-Reducer 17 [SIMPLE_EDGE]
+ SHUFFLE [RS_55]
+ PartitionCols:_col1
+ Select Operator [SEL_48] (rows=381/319 width=178)
+ Output:["_col1"]
+ Group By Operator [GBY_47] (rows=381/319 width=178)
+ Output:["_col0","_col1"],keys:KEY._col0, KEY._col1
+ <-Union 16 [SIMPLE_EDGE]
+ <-Map 21 [CONTAINS]
+ Reduce Output Operator [RS_46]
+ PartitionCols:_col0, _col1
+ Group By Operator [GBY_45] (rows=381/628 width=178)
+ Output:["_col0","_col1"],keys:_col1, _col0
+ Select Operator [SEL_41] (rows=500/500 width=178)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_156] (rows=500/500 width=178)
+ predicate:value is not null
+ TableScan [TS_39] (rows=500/500 width=178)
+ Output:["key","value"]
+ <-Reducer 15 [CONTAINS]
+ Reduce Output Operator [RS_46]
+ PartitionCols:_col0, _col1
+ Group By Operator [GBY_45] (rows=381/628 width=178)
+ Output:["_col0","_col1"],keys:_col1, _col0
+ Select Operator [SEL_38] (rows=262/319 width=178)
+ Output:["_col0","_col1"]
+ Group By Operator [GBY_37] (rows=262/319 width=178)
+ Output:["_col0","_col1"],keys:KEY._col0, KEY._col1
+ <-Union 14 [SIMPLE_EDGE]
+ <-Map 13 [CONTAINS]
+ Reduce Output Operator [RS_36]
+ PartitionCols:_col0, _col1
+ Group By Operator [GBY_35] (rows=262/331 width=178)
+ Output:["_col0","_col1"],keys:_col1, _col0
+ Select Operator [SEL_28] (rows=25/25 width=175)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_154] (rows=25/25 width=175)
+ predicate:value is not null
+ TableScan [TS_26] (rows=25/25 width=175)
+ Output:["key","value"]
+ <-Map 20 [CONTAINS]
+ Reduce Output Operator [RS_36]
+ PartitionCols:_col0, _col1
+ Group By Operator [GBY_35] (rows=262/331 width=178)
+ Output:["_col0","_col1"],keys:_col1, _col0
+ Select Operator [SEL_31] (rows=500/500 width=178)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_155] (rows=500/500 width=178)
+ predicate:value is not null
+ TableScan [TS_29] (rows=500/500 width=178)
+ Output:["key","value"]
+ <-Reducer 5 [CONTAINS]
+ Reduce Output Operator [RS_66]
+ PartitionCols:_col0, _col1
+ Group By Operator [GBY_65] (rows=90/30 width=177)
+ Output:["_col0","_col1"],keys:_col0, _col1
+ Select Operator [SEL_25] (rows=73/61 width=177)
+ Output:["_col0","_col1"]
+ Merge Join Operator [MERGEJOIN_166] (rows=73/61 width=177)
+ Conds:RS_22._col2=RS_23._col0(Inner),Output:["_col2","_col5"]
+ <-Map 12 [SIMPLE_EDGE]
+ SHUFFLE [RS_23]
+ PartitionCols:_col0
+ Select Operator [SEL_18] (rows=500/500 width=178)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_153] (rows=500/500 width=178)
+ predicate:key is not null
+ TableScan [TS_16] (rows=500/500 width=178)
+ default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
+ <-Reducer 4 [SIMPLE_EDGE]
+ SHUFFLE [RS_22]
+ PartitionCols:_col2
+ Merge Join Operator [MERGEJOIN_165] (rows=30/52 width=86)
+ Conds:RS_19._col1=RS_20._col1(Inner),Output:["_col2"]
+ <-Map 11 [SIMPLE_EDGE]
+ SHUFFLE [RS_20]
+ PartitionCols:_col1
+ Select Operator [SEL_15] (rows=25/25 width=175)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_152] (rows=25/25 width=175)
+ predicate:(key is not null and value is not null)
+ TableScan [TS_13] (rows=25/25 width=175)
+ default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
+ <-Reducer 3 [SIMPLE_EDGE]
+ SHUFFLE [RS_19]
+ PartitionCols:_col1
+ Select Operator [SEL_12] (rows=262/319 width=178)
+ Output:["_col1"]
+ Group By Operator [GBY_11] (rows=262/319 width=178)
+ Output:["_col0","_col1"],keys:KEY._col0, KEY._col1
+ <-Union 2 [SIMPLE_EDGE]
+ <-Map 1 [CONTAINS]
+ Reduce Output Operator [RS_10]
+ PartitionCols:_col0, _col1
+ Group By Operator [GBY_9] (rows=262/331 width=178)
+ Output:["_col0","_col1"],keys:_col1, _col0
+ Select Operator [SEL_2] (rows=25/25 width=175)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_150] (rows=25/25 width=175)
+ predicate:value is not null
+ TableScan [TS_0] (rows=25/25 width=175)
+ Output:["key","value"]
+ <-Map 10 [CONTAINS]
+ Reduce Output Operator [RS_10]
+ PartitionCols:_col0, _col1
+ Group By Operator [GBY_9] (rows=262/331 width=178)
+ Output:["_col0","_col1"],keys:_col1, _col0
+ Select Operator [SEL_5] (rows=500/500 width=178)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_151] (rows=500/500 width=178)
+ predicate:value is not null
+ TableScan [TS_3] (rows=500/500 width=178)
+ Output:["key","value"]
+
+PREHOOK: query: SELECT x.key, z.value, y.value
+FROM src1 x JOIN src y ON (x.key = y.key)
+JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Input: default@src1
+PREHOOK: Input: default@srcpart
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT x.key, z.value, y.value
+FROM src1 x JOIN src y ON (x.key = y.key)
+JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Input: default@src1
+POSTHOOK: Input: default@srcpart
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+#### A masked pattern was here ####
+PREHOOK: query: explain analyze
+SELECT x.key, z.value, y.value
+FROM src1 x JOIN src y ON (x.key = y.key)
+JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11)
+PREHOOK: type: QUERY
+POSTHOOK: query: explain analyze
+SELECT x.key, z.value, y.value
+FROM src1 x JOIN src y ON (x.key = y.key)
+JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11)
+POSTHOOK: type: QUERY
+Plan optimized by CBO.
+
+Vertex dependency in root stage
+Map 1 <- Map 2 (BROADCAST_EDGE), Map 3 (BROADCAST_EDGE)
+
+Stage-0
+ Fetch Operator
+ limit:-1
+ Stage-1
+ Map 1
+ File Output Operator [FS_16]
+ Select Operator [SEL_15] (rows=605/85 width=10)
+ Output:["_col0","_col1","_col2"]
+ Map Join Operator [MAPJOIN_26] (rows=605/85 width=10)
+ Conds:MAPJOIN_25._col3=RS_13._col0(Inner),HybridGraceHashJoin:true,Output:["_col0","_col3","_col6"]
+ <-Map 3 [BROADCAST_EDGE]
+ BROADCAST [RS_13]
+ PartitionCols:_col0
+ Select Operator [SEL_8] (rows=500/500 width=10)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_24] (rows=500/500 width=10)
+ predicate:key is not null
+ TableScan [TS_6] (rows=500/500 width=10)
+ default@src,y,Tbl:COMPLETE,Col:NONE,Output:["key","value"]
+ <-Map Join Operator [MAPJOIN_25] (rows=550/41 width=10)
+ Conds:SEL_2._col0=RS_10._col1(Inner),HybridGraceHashJoin:true,Output:["_col0","_col3"]
+ <-Map 2 [BROADCAST_EDGE]
+ BROADCAST [RS_10]
+ PartitionCols:_col1
+ Select Operator [SEL_5] (rows=25/25 width=7)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_23] (rows=25/25 width=7)
+ predicate:(key is not null and value is not null)
+ TableScan [TS_3] (rows=25/25 width=7)
+ default@src1,x,Tbl:COMPLETE,Col:NONE,Output:["key","value"]
+ <-Select Operator [SEL_2] (rows=500/500 width=10)
+ Output:["_col0"]
+ Filter Operator [FIL_22] (rows=500/500 width=10)
+ predicate:value is not null
+ TableScan [TS_0] (rows=500/500 width=10)
+ default@srcpart,z,Tbl:COMPLETE,Col:NONE,Output:["value"]
+
+PREHOOK: query: select
+ss.k1,sr.k2,cs.k3,count(ss.v1),count(sr.v2),count(cs.v3)
+FROM
+ss,sr,cs,src d1,src d2,src d3,src1,srcpart
+where
+ ss.k1 = d1.key
+and sr.k1 = d2.key
+and cs.k1 = d3.key
+and ss.k2 = sr.k2
+and ss.k3 = sr.k3
+and ss.v1 = src1.value
+and ss.v2 = srcpart.value
+and sr.v2 = cs.v2
+and sr.v3 = cs.v3
+and ss.v3='ssv3'
+and sr.v1='srv1'
+and src1.key = 'src1key'
+and srcpart.key = 'srcpartkey'
+and d1.value = 'd1value'
+and d2.value in ('2000Q1','2000Q2','2000Q3')
+and d3.value in ('2000Q1','2000Q2','2000Q3')
+group by
+ss.k1,sr.k2,cs.k3
+order by
+ss.k1,sr.k2,cs.k3
+limit 100
+PREHOOK: type: QUERY
+PREHOOK: Input: default@cs
+PREHOOK: Input: default@sr
+PREHOOK: Input: default@src
+PREHOOK: Input: default@src1
+PREHOOK: Input: default@srcpart
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
+PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
+PREHOOK: Input: default@ss
+#### A masked pattern was here ####
+POSTHOOK: query: select
+ss.k1,sr.k2,cs.k3,count(ss.v1),count(sr.v2),count(cs.v3)
+FROM
+ss,sr,cs,src d1,src d2,src d3,src1,srcpart
+where
+ ss.k1 = d1.key
+and sr.k1 = d2.key
+and cs.k1 = d3.key
+and ss.k2 = sr.k2
+and ss.k3 = sr.k3
+and ss.v1 = src1.value
+and ss.v2 = srcpart.value
+and sr.v2 = cs.v2
+and sr.v3 = cs.v3
+and ss.v3='ssv3'
+and sr.v1='srv1'
+and src1.key = 'src1key'
+and srcpart.key = 'srcpartkey'
+and d1.value = 'd1value'
+and d2.value in ('2000Q1','2000Q2','2000Q3')
+and d3.value in ('2000Q1','2000Q2','2000Q3')
+group by
+ss.k1,sr.k2,cs.k3
+order by
+ss.k1,sr.k2,cs.k3
+limit 100
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@cs
+POSTHOOK: Input: default@sr
+POSTHOOK: Input: default@src
+POSTHOOK: Input: default@src1
+POSTHOOK: Input: default@srcpart
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
+POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
+POSTHOOK: Input: default@ss
+#### A masked pattern was here ####
+PREHOOK: query: explain analyze
+select
+ss.k1,sr.k2,cs.k3,count(ss.v1),count(sr.v2),count(cs.v3)
+FROM
+ss,sr,cs,src d1,src d2,src d3,src1,srcpart
+where
+ ss.k1 = d1.key
+and sr.k1 = d2.key
+and cs.k1 = d3.key
+and ss.k2 = sr.k2
+and ss.k3 = sr.k3
+and ss.v1 = src1.value
+and ss.v2 = srcpart.value
+and sr.v2 = cs.v2
+and sr.v3 = cs.v3
+and ss.v3='ssv3'
+and sr.v1='srv1'
+and src1.key = 'src1key'
+and srcpart.key = 'srcpartkey'
+and d1.value = 'd1value'
+and d2.value in ('2000Q1','2000Q2','2000Q3')
+and d3.value in ('2000Q1','2000Q2','2000Q3')
+group by
+ss.k1,sr.k2,cs.k3
+order by
+ss.k1,sr.k2,cs.k3
+limit 100
+PREHOOK: type: QUERY
+POSTHOOK: query: explain analyze
+select
+ss.k1,sr.k2,cs.k3,count(ss.v1),count(sr.v2),count(cs.v3)
+FROM
+ss,sr,cs,src d1,src d2,src d3,src1,srcpart
+where
+ ss.k1 = d1.key
+and sr.k1 = d2.key
+and cs.k1 = d3.key
+and ss.k2 = sr.k2
+and ss.k3 = sr.k3
+and ss.v1 = src1.value
+and ss.v2 = srcpart.value
+and sr.v2 = cs.v2
+and sr.v3 = cs.v3
+and ss.v3='ssv3'
+and sr.v1='srv1'
+and src1.key = 'src1key'
+and srcpart.key = 'srcpartkey'
+and d1.value = 'd1value'
+and d2.value in ('2000Q1','2000Q2','2000Q3')
+and d3.value in ('2000Q1','2000Q2','2000Q3')
+group by
+ss.k1,sr.k2,cs.k3
+order by
+ss.k1,sr.k2,cs.k3
+limit 100
+POSTHOOK: type: QUERY
+Plan optimized by CBO.
+
+Vertex dependency in root stage
+Map 10 <- Map 9 (BROADCAST_EDGE)
+Map 2 <- Map 1 (BROADCAST_EDGE)
+Map 3 <- Map 10 (BROADCAST_EDGE), Map 2 (BROADCAST_EDGE), Map 6 (BROADCAST_EDGE), Map 7 (BROADCAST_EDGE), Map 8 (BROADCAST_EDGE)
+Reducer 4 <- Map 3 (SIMPLE_EDGE)
+Reducer 5 <- Reducer 4 (SIMPLE_EDGE)
+
+Stage-0
+ Fetch Operator
+ limit:100
+ Stage-1
+ Reducer 5
+ File Output Operator [FS_55]
+ Limit [LIM_54] (rows=100/0 width=10)
+ Number of rows:100
+ Select Operator [SEL_53] (rows=805/0 width=10)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
+ <-Reducer 4 [SIMPLE_EDGE]
+ SHUFFLE [RS_52]
+ Group By Operator [GBY_50] (rows=805/0 width=10)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["count(VALUE._col0)","count(VALUE._col1)","count(VALUE._col2)"],keys:KEY._col0, KEY._col1, KEY._col2
+ <-Map 3 [SIMPLE_EDGE]
+ SHUFFLE [RS_49]
+ PartitionCols:_col0, _col1, _col2
+ Group By Operator [GBY_48] (rows=1610/0 width=10)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["count(_col13)","count(_col21)","count(_col3)"],keys:_col12, _col20, _col2
+ Select Operator [SEL_47] (rows=1610/0 width=10)
+ Output:["_col12","_col20","_col2","_col13","_col21","_col3"]
+ Map Join Operator [MAPJOIN_97] (rows=1610/0 width=10)
+ Conds:RS_44._col1, _col3=SEL_40._col15, _col17(Inner),HybridGraceHashJoin:true,Output:["_col2","_col3","_col12","_col13","_col20","_col21"]
+ <-Map 2 [BROADCAST_EDGE]
+ BROADCAST [RS_44]
+ PartitionCols:_col1, _col3
+ Map Join Operator [MAPJOIN_91] (rows=275/0 width=10)
+ Conds:RS_41._col0=SEL_5._col0(Inner),HybridGraceHashJoin:true,Output:["_col1","_col2","_col3"]
+ <-Map 1 [BROADCAST_EDGE]
+ BROADCAST [RS_41]
+ PartitionCols:_col0
+ Select Operator [SEL_2] (rows=170/170 width=34)
+ Output:["_col0","_col1","_col2","_col3"]
+ Filter Operator [FIL_83] (rows=170/170 width=34)
+ predicate:(v2 is not null and v3 is not null and k1 is not null)
+ TableScan [TS_0] (rows=170/170 width=34)
+ default@cs,cs,Tbl:COMPLETE,Col:NONE,Output:["k1","v2","k3","v3"]
+ <-Select Operator [SEL_5] (rows=250/0 width=10)
+ Output:["_col0"]
+ Filter Operator [FIL_84] (rows=250/0 width=10)
+ predicate:((value) IN ('2000Q1', '2000Q2', '2000Q3') and key is not null)
+ TableScan [TS_3] (rows=500/500 width=10)
+ default@src,d3,Tbl:COMPLETE,Col:NONE,Output:["key","value"]
+ <-Select Operator [SEL_40] (rows=1464/0 width=10)
+ Output:["_col14","_col15","_col17","_col6","_col7"]
+ Map Join Operator [MAPJOIN_96] (rows=1464/0 width=10)
+ Conds:MAPJOIN_94._col6, _col4=RS_38._col4, _col2(Inner),HybridGraceHashJoin:true,Output:["_col2","_col3","_col14","_col15","_col17"]
+ <-Map 10 [BROADCAST_EDGE]
+ BROADCAST [RS_38]
+ PartitionCols:_col4, _col2
+ Map Join Operator [MAPJOIN_95] (rows=275/0 width=10)
+ Conds:RS_24._col0=SEL_23._col0(Inner),HybridGraceHashJoin:true,Output:["_col2","_col3","_col4","_col5"]
+ <-Map 9 [BROADCAST_EDGE]
+ BROADCAST [RS_24]
+ PartitionCols:_col0
+ Select Operator [SEL_20] (rows=42/0 width=34)
+ Output:["_col0","_col2","_col3","_col4","_col5"]
+ Filter Operator [FIL_89] (rows=42/0 width=34)
+ predicate:((v1 = 'srv1') and k2 is not null and k3 is not null and v2 is not null and v3 is not null and k1 is not null)
+ TableScan [TS_18] (rows=85/85 width=34)
+ default@sr,sr,Tbl:COMPLETE,Col:NONE,Output:["k1","v1","k2","v2","k3","v3"]
+ <-Select Operator [SEL_23] (rows=250/0 width=10)
+ Output:["_col0"]
+ Filter Operator [FIL_90] (rows=250/0 width=10)
+ predicate:((value) IN ('2000Q1', '2000Q2', '2000Q3') and key is not null)
+ TableScan [TS_21] (rows=500/500 width=10)
+ default@src,d2,Tbl:COMPLETE,Col:NONE,Output:["key","value"]
+ <-Map Join Operator [MAPJOIN_94] (rows=1331/0 width=10)
+ Conds:MAPJOIN_93._col3=RS_35._col1(Inner),HybridGraceHashJoin:true,Output:["_col2","_col3","_col4","_col6"]
+ <-Map 8 [BROADCAST_EDGE]
+ BROADCAST [RS_35]
+ PartitionCols:_col1
+ Select Operator [SEL_17] (rows=12/0 width=7)
+ Output:["_col1"]
+ Filter Operator [FIL_88] (rows=12/0 width=7)
+ predicate:((key = 'src1key') and value is not null)
+ TableScan [TS_15] (rows=25/25 width=7)
+ default@src1,src1,Tbl:COMPLETE,Col:NONE,Output:["key","value"]
+ <-Map Join Operator [MAPJOIN_93] (rows=1210/0 width=10)
+ Conds:MAPJOIN_92._col2=RS_32._col0(Inner),HybridGraceHashJoin:true,Output:["_col2","_col3","_col4","_col6"]
+ <-Map 7 [BROADCAST_EDGE]
+ BROADCAST [RS_32]
+ PartitionCols:_col0
+ Select Operator [SEL_14] (rows=250/0 width=10)
+ Output:["_col0"]
+ Filter Operator [FIL_87] (rows=250/0 width=10)
+ predicate:((value = 'd1value') and key is not null)
+ TableScan [TS_12] (rows=500/500 width=10)
+ default@src,d1,Tbl:COMPLETE,Col:NONE,Output:["key","value"]
+ <-Map Join Operator [MAPJOIN_92] (rows=1100/0 width=10)
+ Conds:SEL_8._col1=RS_29._col3(Inner),HybridGraceHashJoin:true,Output:["_col2","_col3","_col4","_col6"]
+ <-Map 6 [BROADCAST_EDGE]
+ BROADCAST [RS_29]
+ PartitionCols:_col3
+ Select Operator [SEL_11] (rows=42/0 width=34)
+ Output:["_col0","_col1","_col2","_col3","_col4"]
+ Filter Operator [FIL_86] (rows=42/0 width=34)
+ predicate:((v3 = 'ssv3') and k2 is not null and k3 is not null and k1 is not null and v1 is not null and v2 is not null)
+ TableScan [TS_9] (rows=85/85 width=34)
+ default@ss,ss,Tbl:COMPLETE,Col:NONE,Output:["k1","v1","k2","v2","k3","v3"]
+ <-Select Operator [SEL_8] (rows=1000/0 width=10)
+ Output:["_col1"]
+ Filter Operator [FIL_85] (rows=1000/0 width=10)
+ predicate:((key = 'srcpartkey') and value is not null)
+ TableScan [TS_6] (rows=2000/2000 width=10)
+ default@srcpart,srcpart,Tbl:COMPLETE,Col:NONE,Output:["key","value"]
+
+PREHOOK: query: SELECT x.key, z.value, y.value
+FROM src1 x JOIN src y ON (x.key = y.key)
+JOIN (select * from src1 union select * from src)z ON (x.value = z.value)
+union
+SELECT x.key, z.value, y.value
+FROM src1 x JOIN src y ON (x.key = y.key)
+JOIN (select * from src1 union select * from src)z ON (x.value = z.value)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Input: default@src1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT x.key, z.value, y.value
+FROM src1 x JOIN src y ON (x.key = y.key)
+JOIN (select * from src1 union select * from src)z ON (x.value = z.value)
+union
+SELECT x.key, z.value, y.value
+FROM src1 x JOIN src y ON (x.key = y.key)
+JOIN (select * from src1 union select * from src)z ON (x.value = z.value)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Input: default@src1
+#### A masked pattern was here ####
+PREHOOK: query: explain analyze
+SELECT x.key, z.value, y.value
+FROM src1 x JOIN src y ON (x.key = y.key)
+JOIN (select * from src1 union select * from src)z ON (x.value = z.value)
+union
+SELECT x.key, z.value, y.value
+FROM src1 x JOIN src y ON (x.key = y.key)
+JOIN (select * from src1 union select * from src)z ON (x.value = z.value)
+PREHOOK: type: QUERY
+POSTHOOK: query: explain analyze
+SELECT x.key, z.value, y.value
+FROM src1 x JOIN src y ON (x.key = y.key)
+JOIN (select * from src1 union select * from src)z ON (x.value = z.value)
+union
+SELECT x.key, z.value, y.value
+FROM src1 x JOIN src y ON (x.key = y.key)
+JOIN (select * from src1 union select * from src)z ON (x.value = z.value)
+POSTHOOK: type: QUERY
+Plan optimized by CBO.
+
+Vertex dependency in root stage
+Map 1 <- Union 2 (CONTAINS)
+Map 12 <- Union 10 (CONTAINS)
+Map 6 <- Union 2 (CONTAINS)
+Map 9 <- Union 10 (CONTAINS)
+Reducer 11 <- Map 13 (BROADCAST_EDGE), Map 14 (BROADCAST_EDGE), Union 10 (SIMPLE_EDGE), Union 4 (CONTAINS)
+Reducer 3 <- Map 7 (BROADCAST_EDGE), Map 8 (BROADCAST_EDGE), Union 2 (SIMPLE_EDGE), Union 4 (CONTAINS)
+Reducer 5 <- Union 4 (SIMPLE_EDGE)
+
+Stage-0
+ Fetch Operator
+ limit:-1
+ Stage-1
+ Reducer 5
+ File Output Operator [FS_59]
+ Group By Operator [GBY_57] (rows=550/15 width=10)
+ Output:["_col0","_col1"],keys:KEY._col0, KEY._col1
+ <-Union 4 [SIMPLE_EDGE]
+ <-Reducer 11 [CONTAINS]
+ Reduce Output Operator [RS_56]
+ PartitionCols:_col0, _col1
+ Group By Operator [GBY_55] (rows=1100/30 width=10)
+ Output:["_col0","_col1"],keys:_col0, _col1
+ Select Operator [SEL_51] (rows=550/61 width=10)
+ Output:["_col0","_col1"]
+ Map Join Operator [MAPJOIN_85] (rows=550/61 width=10)
+ Conds:MAPJOIN_84._col2=RS_49._col0(Inner),HybridGraceHashJoin:true,Output:["_col1","_col2"]
+ <-Map 14 [BROADCAST_EDGE]
+ BROADCAST [RS_49]
+ PartitionCols:_col0
+ Select Operator [SEL_44] (rows=500/500 width=10)
+ Output:["_col0"]
+ Filter Operator [FIL_81] (rows=500/500 width=10)
+ predicate:key is not null
+ TableScan [TS_42] (rows=500/500 width=10)
+ default@src,y,Tbl:COMPLETE,Col:NONE,Output:["key"]
+ <-Map Join Operator [MAPJOIN_84] (rows=288/52 width=10)
+ Conds:SEL_38._col1=RS_46._col1(Inner),HybridGraceHashJoin:true,Output:["_col1","_col2"]
+ <-Map 13 [BROADCAST_EDGE]
+ BROADCAST [RS_46]
+ PartitionCols:_col1
+ Select Operator [SEL_41] (rows=25/25 width=7)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_80] (rows=25/25 width=7)
+ predicate:(key is not null and value is not null)
+ TableScan [TS_39] (rows=25/25 width=7)
+ default@src1,x,Tbl:COMPLETE,Col:NONE,Output:["key","value"]
+ <-Select Operator [SEL_38] (rows=262/319 width=10)
+ Output:["_col1"]
+ Group By Operator [GBY_37] (rows=262/319 width=10)
+ Output:["_col0","_col1"],keys:KEY._col0, KEY._col1
+ <-Union 10 [SIMPLE_EDGE]
+ <-Map 12 [CONTAINS]
+ Reduce Output Operator [RS_36]
+ PartitionCols:_col0, _col1
+ Group By Operator [GBY_35] (rows=525/331 width=10)
+ Output:["_col0","_col1"],keys:_col1, _col0
+ Select Operator [SEL_31] (rows=500/500 width=10)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_79] (rows=500/500 width=10)
+ predicate:value is not null
+ TableScan [TS_29] (rows=500/500 width=10)
+ Output:["key","value"]
+ <-Map 9 [CONTAINS]
+ Reduce Output Operator [RS_36]
+ PartitionCols:_col0, _col1
+ Group By Operator [GBY_35] (rows=525/331 width=10)
+ Output:["_col0","_col1"],keys:_col1, _col0
+ Select Operator [SEL_28] (rows=25/25 width=7)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_78] (rows=25/25 width=7)
+ predicate:value is not null
+ TableScan [TS_26] (rows=25/25 width=7)
+ Output:["key","value"]
+ <-Reducer 3 [CONTAINS]
+ Reduce Output Operator [RS_56]
+ PartitionCols:_col0, _col1
+ Group By Operator [GBY_55] (rows=1100/30 width=10)
+ Output:["_col0","_col1"],keys:_col0, _col1
+ Select Operator [SEL_25] (rows=550/61 width=10)
+ Output:["_col0","_col1"]
+ Map Join Operator [MAPJOIN_83] (rows=550/61 width=10)
+ Conds:MAPJOIN_82._col2=RS_23._col0(Inner),HybridGraceHashJoin:true,Output:["_col1","_col2"]
+ <-Map 8 [BROADCAST_EDGE]
+ BROADCAST [RS_23]
+ PartitionCols:_col0
+ Select Operator [SEL_18] (rows=500/500 width=10)
+ Output:["_col0"]
+ Filter Operator [FIL_77] (rows=500/500 width=10)
+ predicate:key is not null
+ TableScan [TS_16] (rows=500/500 width=10)
+ default@src,y,Tbl:COMPLETE,Col:NONE,Output:["key"]
+ <-Map Join Operator [MAPJOIN_82] (rows=288/52 width=10)
+ Conds:SEL_12._col1=RS_20._col1(Inner),HybridGraceHashJoin:true,Output:["_col1","_col2"]
+ <-Map 7 [BROADCAST_EDGE]
+ BROADCAST [RS_20]
+ PartitionCols:_col1
+ Select Operator [SEL_15] (rows=25/25 width=7)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_76] (rows=25/25 width=7)
+ predicate:(key is not null and value is not null)
+ TableScan [TS_13] (rows=25/25 width=7)
+ default@src1,x,Tbl:COMPLETE,Col:NONE,Output:["key","value"]
+ <-Select Operator [SEL_12] (rows=262/319 width=10)
+ Output:["_col1"]
+ Group By Operator [GBY_11] (rows=262/319 width=10)
+ Output:["_col0","_col1"],keys:KEY._col0, KEY._col1
+ <-Union 2 [SIMPLE_EDGE]
+ <-Map 1 [CONTAINS]
+ Reduce Output Operator [RS_10]
+ PartitionCols:_col0, _col1
+ Group By Operator [GBY_9] (rows=525/331 width=10)
+ Output:["_col0","_col1"],keys:_col1, _col0
+ Select Operator [SEL_2] (rows=25/25 width=7)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_74] (rows=25/25 width=7)
+ predicate:value is not null
+ TableScan [TS_0] (rows=25/25 width=7)
+ Output:["key","value"]
+ <-Map 6 [CONTAINS]
+ Reduce Output Operator [RS_10]
+ PartitionCols:_col0, _col1
+ Group By Operator [GBY_9] (rows=525/331 width=10)
+ Output:["_col0","_col1"],keys:_col1, _col0
+ Select Operator [SEL_5] (rows=500/500 width=10)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_75] (rows=500/500 width=10)
+ predicate:value is not null
+ TableScan [TS_3] (rows=500/500 width=10)
+ Output:["key","value"]
+
+PREHOOK: query: SELECT x.key, y.value
+FROM src1 x JOIN src y ON (x.key = y.key)
+JOIN (select * from src1 union select * from src)z ON (x.value = z.value)
+union
+SELECT x.key, y.value
+FROM src1 x JOIN src y ON (x.key = y.key)
+JOIN (select key, value from src1 union select key, value from src union select key, value from src)z ON (x.value = z.value)
+union
+SELECT x.key, y.value
+FROM src1 x JOIN src y ON (x.key = y.key)
+JOIN (select key, value from src1 union select key, value from src union select key, value from src union select key, value from src)z ON (x.value = z.value)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Input: default@src1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT x.key, y.value
+FROM src1 x JOIN src y ON (x.key = y.key)
+JOIN (select * from src1 union select * from src)z ON (x.value = z.value)
+union
+SELECT x.key, y.value
+FROM src1 x JOIN src y ON (x.key = y.key)
+JOIN (select key, value from src1 union select key, value from src union select key, value from src)z ON (x.value = z.value)
+union
+SELECT x.key, y.value
+FROM src1 x JOIN src y ON (x.key = y.key)
+JOIN (select key, value from src1 union select key, value from src union select key, value from src union select key, value from src)z ON (x.value = z.value)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Input: default@src1
+#### A masked pattern was here ####
+PREHOOK: query: explain analyze
+SELECT x.key, y.value
+FROM src1 x JOIN src y ON (x.key = y.key)
+JOIN (select * from src1 union select * from src)z ON (x.value = z.value)
+union
+SELECT x.key, y.value
+FROM src1 x JOIN src y ON (x.key = y.key)
+JOIN (select key, value from src1 union select key, value from src union select key, value from src)z ON (x.value = z.value)
+union
+SELECT x.key, y.value
+FROM src1 x JOIN src y ON (x.key = y.key)
+JOIN (select key, value from src1 union select key, value from src union select key, value from src union select key, value from src)z ON (x.value = z.value)
+PREHOOK: type: QUERY
+POSTHOOK: query: explain analyze
+SELECT x.key, y.value
+FROM src1 x JOIN src y ON (x.key = y.key)
+JOIN (select * from src1 union select * from src)z ON (x.value = z.value)
+union
+SELECT x.key, y.value
+FROM src1 x JOIN src y ON (x.key = y.key)
+JOIN (select key, value from src1 union select key, value from src union select key, value from src)z ON (x.value = z.value)
+union
+SELECT x.key, y.value
+FROM src1 x JOIN src y ON (x.key = y.key)
+JOIN (select key, value from src1 union select key, value from src union select key, value from src union select key, value from src)z ON (x.value = z.value)
+POSTHOOK: type: QUERY
+Plan optimized by CBO.
+
+Vertex dependency in root stage
+Map 1 <- Union 2 (CONTAINS)
+Map 11 <- Union 12 (CONTAINS)
+Map 16 <- Union 12 (CONTAINS)
+Map 17 <- Union 14 (CONTAINS)
+Map 20 <- Union 21 (CONTAINS)
+Map 27 <- Union 21 (CONTAINS)
+Map 28 <- Union 23 (CONTAINS)
+Map 29 <- Union 25 (CONTAINS)
+Map 8 <- Union 2 (CONTAINS)
+Reducer 13 <- Union 12 (SIMPLE_EDGE), Union 14 (CONTAINS)
+Reducer 15 <- Map 18 (BROADCAST_EDGE), Map 19 (BROADCAST_EDGE), Union 14 (SIMPLE_EDGE), Union 4 (CONTAINS)
+Reducer 22 <- Union 21 (SIMPLE_EDGE), Union 23 (CONTAINS)
+Reducer 24 <- Union 23 (SIMPLE_EDGE), Union 25 (CONTAINS)
+Reducer 26 <- Map 30 (BROADCAST_EDGE), Map 31 (BROADCAST_EDGE), Union 25 (SIMPLE_EDGE), Union 6 (CONTAINS)
+Reducer 3 <- Map 10 (BROADCAST_EDGE), Map 9 (BROADCAST_EDGE), Union 2 (SIMPLE_EDGE), Union 4 (CONTAINS)
+Reducer 5 <- Union 4 (SIMPLE_EDGE), Union 6 (CONTAINS)
+Reducer 7 <- Union 6 (SIMPLE_EDGE)
+
+Stage-0
+ Fetch Operator
+ limit:-1
+ Stage-1
+ Reducer 7
+ File Output Operator [FS_122]
+ Group By Operator [GBY_120] (rows=550/15 width=10)
+ Output:["_col0","_col1"],keys:KEY._col0, KEY._col1
+ <-Union 6 [SIMPLE_EDGE]
+ <-Reducer 26 [CONTAINS]
+ Reduce Output Operator [RS_119]
+ PartitionCols:_col0, _col1
+ Group By Operator [GBY_118] (rows=1100/30 width=10)
+ Output:["_col0","_col1"],keys:_col0, _col1
+ Select Operator [SEL_114] (rows=550/61 width=10)
+ Output:["_col0","_col1"]
+ Map Join Operator [MAPJOIN_170] (rows=550/61 width=10)
+ Conds:MAPJOIN_169._col2=RS_112._col0(Inner),HybridGraceHashJoin:true,Output:["_col2","_col5"]
+ <-Map 31 [BROADCAST_EDGE]
+ BROADCAST [RS_112]
+ PartitionCols:_col0
+ Select Operator [SEL_107] (rows=500/500 width=10)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_164] (rows=500/500 width=10)
+ predicate:key is not null
+ TableScan [TS_105] (rows=500/500 width=10)
+ default@src,y,Tbl:COMPLETE,Col:NONE,Output:["key","value"]
+ <-Map Join Operator [MAPJOIN_169] (rows=484/52 width=10)
+ Conds:SEL_101._col1=RS_109._col1(Inner),HybridGraceHashJoin:true,Output:["_col2"]
+ <-Map 30 [BROADCAST_EDGE]
+ BROADCAST [RS_109]
+ PartitionCols:_col1
+ Select Operator [SEL_104] (rows=25/25 width=7)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_163] (rows=25/25 width=7)
+ predicate:(key is not null and value is not null)
+ TableScan [TS_102] (rows=25/25 width=7)
+ default@src1,x,Tbl:COMPLETE,Col:NONE,Output:["key","value"]
+ <-Select Operator [SEL_101] (rows=440/319 width=10)
+ Output:["_col1"]
+ Group By Operator [GBY_100] (rows=440/319 width=10)
+ Output:["_col0","_col1"],ke
<TRUNCATED>
[47/50] [abbrv] hive git commit: HIVE-16065: Vectorization: Wrong
Key/Value information used by Vectorizer (Matt McCline,
reviewed by Jason Dere)
Posted by om...@apache.org.
HIVE-16065: Vectorization: Wrong Key/Value information used by Vectorizer (Matt McCline, reviewed by Jason Dere)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/154c58b5
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/154c58b5
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/154c58b5
Branch: refs/heads/branch-2.2
Commit: 154c58b56163130a07f152a9bfe37cc7d6f2e6b5
Parents: 2f3c949
Author: Matt McCline <mm...@hortonworks.com>
Authored: Thu Mar 2 18:40:24 2017 -0800
Committer: Owen O'Malley <om...@apache.org>
Committed: Tue Mar 28 15:27:58 2017 -0700
----------------------------------------------------------------------
.../hive/ql/exec/tez/ReduceRecordProcessor.java | 2 +-
.../hive/ql/exec/tez/ReduceRecordSource.java | 8 ++-
.../hive/ql/optimizer/physical/Vectorizer.java | 74 ++++++++++++++------
.../apache/hadoop/hive/ql/plan/BaseWork.java | 10 +++
.../apache/hadoop/hive/ql/plan/ReduceWork.java | 31 --------
5 files changed, 70 insertions(+), 55 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/154c58b5/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordProcessor.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordProcessor.java
index 2d06545..3fb9fb1 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordProcessor.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordProcessor.java
@@ -300,7 +300,7 @@ public class ReduceRecordProcessor extends RecordProcessor{
boolean vectorizedRecordSource = (tag == bigTablePosition) && redWork.getVectorMode();
sources[tag].init(jconf, redWork.getReducer(), vectorizedRecordSource, keyTableDesc,
valueTableDesc, reader, tag == bigTablePosition, (byte) tag,
- redWork.getVectorizedRowBatchCtx());
+ redWork.getVectorizedRowBatchCtx(), redWork.getVectorizedVertexNum());
ois[tag] = sources[tag].getObjectInspector();
}
http://git-wip-us.apache.org/repos/asf/hive/blob/154c58b5/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordSource.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordSource.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordSource.java
index 7e41b7a..342e1ed 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordSource.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordSource.java
@@ -120,11 +120,14 @@ public class ReduceRecordSource implements RecordSource {
private final GroupIterator groupIterator = new GroupIterator();
+ private long vectorizedVertexNum;
+
void init(JobConf jconf, Operator<?> reducer, boolean vectorized, TableDesc keyTableDesc,
TableDesc valueTableDesc, Reader reader, boolean handleGroupKey, byte tag,
- VectorizedRowBatchCtx batchContext)
+ VectorizedRowBatchCtx batchContext, long vectorizedVertexNum)
throws Exception {
+ this.vectorizedVertexNum = vectorizedVertexNum;
ObjectInspector keyObjectInspector;
this.reducer = reducer;
@@ -471,7 +474,8 @@ public class ReduceRecordSource implements RecordSource {
+ StringUtils.stringifyException(e2) + " ]";
}
throw new HiveException("Hive Runtime Error while processing vector batch (tag="
- + tag + ") " + rowString, e);
+ + tag + ") (vectorizedVertexNum " + vectorizedVertexNum + ") " +
+ rowString, e);
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/154c58b5/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
index 864b783..a30923f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
@@ -168,6 +168,7 @@ import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.hive.serde2.ColumnProjectionUtils;
import org.apache.hadoop.hive.serde2.Deserializer;
import org.apache.hadoop.hive.serde2.NullStructSerDe;
+import org.apache.hadoop.hive.serde2.SerDeUtils;
import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe;
import org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
@@ -182,6 +183,7 @@ import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
import org.apache.hadoop.mapred.SequenceFileInputFormat;
import org.apache.hadoop.mapred.TextInputFormat;
+import org.apache.hadoop.util.ReflectionUtils;
import com.google.common.base.Preconditions;
@@ -242,6 +244,8 @@ public class Vectorizer implements PhysicalPlanResolver {
HiveVectorAdaptorUsageMode hiveVectorAdaptorUsageMode;
+ private long vectorizedVertexNum = -1;
+
public Vectorizer() {
/*
@@ -486,6 +490,7 @@ public class Vectorizer implements PhysicalPlanResolver {
private void convertMapWork(MapWork mapWork, boolean isTez) throws SemanticException {
VectorTaskColumnInfo vectorTaskColumnInfo = new VectorTaskColumnInfo();
+ mapWork.setVectorizedVertexNum(++vectorizedVertexNum);
boolean ret = validateMapWork(mapWork, vectorTaskColumnInfo, isTez);
if (ret) {
vectorizeMapWork(mapWork, vectorTaskColumnInfo, isTez);
@@ -935,6 +940,7 @@ public class Vectorizer implements PhysicalPlanResolver {
private void convertReduceWork(ReduceWork reduceWork, boolean isTez) throws SemanticException {
VectorTaskColumnInfo vectorTaskColumnInfo = new VectorTaskColumnInfo();
+ reduceWork.setVectorizedVertexNum(++vectorizedVertexNum);
boolean ret = validateReduceWork(reduceWork, vectorTaskColumnInfo, isTez);
if (ret) {
vectorizeReduceWork(reduceWork, vectorTaskColumnInfo, isTez);
@@ -947,36 +953,56 @@ public class Vectorizer implements PhysicalPlanResolver {
ArrayList<String> reduceColumnNames = new ArrayList<String>();
ArrayList<TypeInfo> reduceTypeInfos = new ArrayList<TypeInfo>();
+ if (reduceWork.getNeedsTagging()) {
+ LOG.info("Tagging not supported");
+ return false;
+ }
+
try {
- // Check key ObjectInspector.
- ObjectInspector keyObjectInspector = reduceWork.getKeyObjectInspector();
- if (keyObjectInspector == null || !(keyObjectInspector instanceof StructObjectInspector)) {
- return false;
+ TableDesc keyTableDesc = reduceWork.getKeyDesc();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Using reduce tag " + reduceWork.getTag());
}
- StructObjectInspector keyStructObjectInspector = (StructObjectInspector)keyObjectInspector;
- List<? extends StructField> keyFields = keyStructObjectInspector.getAllStructFieldRefs();
-
- // Tez doesn't use tagging...
- if (reduceWork.getNeedsTagging()) {
+ TableDesc valueTableDesc = reduceWork.getTagToValueDesc().get(reduceWork.getTag());
+
+ Deserializer keyDeserializer =
+ ReflectionUtils.newInstance(
+ keyTableDesc.getDeserializerClass(), null);
+ SerDeUtils.initializeSerDe(keyDeserializer, null, keyTableDesc.getProperties(), null);
+ ObjectInspector keyObjectInspector = keyDeserializer.getObjectInspector();
+ if (keyObjectInspector == null) {
+ LOG.info("Key object inspector null");
return false;
}
-
- // Check value ObjectInspector.
- ObjectInspector valueObjectInspector = reduceWork.getValueObjectInspector();
- if (valueObjectInspector == null ||
- !(valueObjectInspector instanceof StructObjectInspector)) {
+ if (!(keyObjectInspector instanceof StructObjectInspector)) {
+ LOG.info("Key object inspector not StructObjectInspector");
return false;
}
- StructObjectInspector valueStructObjectInspector = (StructObjectInspector)valueObjectInspector;
- List<? extends StructField> valueFields = valueStructObjectInspector.getAllStructFieldRefs();
+ StructObjectInspector keyStructObjectInspector = (StructObjectInspector) keyObjectInspector;
+ List<? extends StructField> keyFields = keyStructObjectInspector.getAllStructFieldRefs();
for (StructField field: keyFields) {
reduceColumnNames.add(Utilities.ReduceField.KEY.toString() + "." + field.getFieldName());
reduceTypeInfos.add(TypeInfoUtils.getTypeInfoFromTypeString(field.getFieldObjectInspector().getTypeName()));
}
- for (StructField field: valueFields) {
- reduceColumnNames.add(Utilities.ReduceField.VALUE.toString() + "." + field.getFieldName());
- reduceTypeInfos.add(TypeInfoUtils.getTypeInfoFromTypeString(field.getFieldObjectInspector().getTypeName()));
+
+ Deserializer valueDeserializer =
+ ReflectionUtils.newInstance(
+ valueTableDesc.getDeserializerClass(), null);
+ SerDeUtils.initializeSerDe(valueDeserializer, null, valueTableDesc.getProperties(), null);
+ ObjectInspector valueObjectInspector = valueDeserializer.getObjectInspector();
+ if (valueObjectInspector != null) {
+ if (!(valueObjectInspector instanceof StructObjectInspector)) {
+ LOG.info("Value object inspector not StructObjectInspector");
+ return false;
+ }
+ StructObjectInspector valueStructObjectInspector = (StructObjectInspector) valueObjectInspector;
+ List<? extends StructField> valueFields = valueStructObjectInspector.getAllStructFieldRefs();
+
+ for (StructField field: valueFields) {
+ reduceColumnNames.add(Utilities.ReduceField.VALUE.toString() + "." + field.getFieldName());
+ reduceTypeInfos.add(TypeInfoUtils.getTypeInfoFromTypeString(field.getFieldObjectInspector().getTypeName()));
+ }
}
} catch (Exception e) {
throw new SemanticException(e);
@@ -1254,6 +1280,10 @@ public class Vectorizer implements PhysicalPlanResolver {
if (op instanceof TableScanOperator) {
if (taskVectorizationContext == null) {
taskVectorizationContext = getVectorizationContext(op.getName(), vectorTaskColumnInfo);
+ if (LOG.isInfoEnabled()) {
+ LOG.info("MapWorkVectorizationNodeProcessor process vectorizedVertexNum " + vectorizedVertexNum + " mapColumnNames " + vectorTaskColumnInfo.allColumnNames.toString());
+ LOG.info("MapWorkVectorizationNodeProcessor process vectorizedVertexNum " + vectorizedVertexNum + " mapTypeInfos " + vectorTaskColumnInfo.allTypeInfos.toString());
+ }
}
vContext = taskVectorizationContext;
} else {
@@ -1320,8 +1350,10 @@ public class Vectorizer implements PhysicalPlanResolver {
boolean saveRootVectorOp = false;
if (op.getParentOperators().size() == 0) {
- LOG.info("ReduceWorkVectorizationNodeProcessor process reduceColumnNames " + vectorTaskColumnInfo.allColumnNames.toString());
-
+ if (LOG.isInfoEnabled()) {
+ LOG.info("ReduceWorkVectorizationNodeProcessor process vectorizedVertexNum " + vectorizedVertexNum + " reduceColumnNames " + vectorTaskColumnInfo.allColumnNames.toString());
+ LOG.info("ReduceWorkVectorizationNodeProcessor process vectorizedVertexNum " + vectorizedVertexNum + " reduceTypeInfos " + vectorTaskColumnInfo.allTypeInfos.toString());
+ }
vContext = new VectorizationContext("__Reduce_Shuffle__", vectorTaskColumnInfo.allColumnNames, hiveConf);
taskVectorizationContext = vContext;
http://git-wip-us.apache.org/repos/asf/hive/blob/154c58b5/ql/src/java/org/apache/hadoop/hive/ql/plan/BaseWork.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/BaseWork.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/BaseWork.java
index 8c341fc..990da12 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/BaseWork.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/BaseWork.java
@@ -73,6 +73,8 @@ public abstract class BaseWork extends AbstractOperatorDesc {
protected boolean useVectorizedInputFileFormat;
+ protected long vectorizedVertexNum;
+
protected boolean llapMode = false;
protected boolean uberMode = false;
@@ -167,6 +169,14 @@ public abstract class BaseWork extends AbstractOperatorDesc {
return returnSet;
}
+ public void setVectorizedVertexNum(long vectorizedVertexNum) {
+ this.vectorizedVertexNum = vectorizedVertexNum;
+ }
+
+ public long getVectorizedVertexNum() {
+ return vectorizedVertexNum;
+ }
+
// -----------------------------------------------------------------------------------------------
/*
http://git-wip-us.apache.org/repos/asf/hive/blob/154c58b5/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceWork.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceWork.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceWork.java
index 72fc4ca..1910ebd 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceWork.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceWork.java
@@ -103,37 +103,6 @@ public class ReduceWork extends BaseWork {
return keyDesc;
}
- private ObjectInspector getObjectInspector(TableDesc desc) {
- ObjectInspector objectInspector;
- try {
- Deserializer deserializer = ReflectionUtil.newInstance(desc
- .getDeserializerClass(), null);
- SerDeUtils.initializeSerDe(deserializer, null, desc.getProperties(), null);
- objectInspector = deserializer.getObjectInspector();
- } catch (Exception e) {
- return null;
- }
- return objectInspector;
- }
-
- public ObjectInspector getKeyObjectInspector() {
- if (keyObjectInspector == null) {
- keyObjectInspector = getObjectInspector(keyDesc);
- }
- return keyObjectInspector;
- }
-
- // Only works when not tagging.
- public ObjectInspector getValueObjectInspector() {
- if (needsTagging) {
- return null;
- }
- if (valueObjectInspector == null) {
- valueObjectInspector = getObjectInspector(tagToValueDesc.get(tag));
- }
- return valueObjectInspector;
- }
-
public List<TableDesc> getTagToValueDesc() {
return tagToValueDesc;
}
[32/50] [abbrv] hive git commit: HIVE-15084: Flaky test:
TestMiniTezCliDriver:explainanalyze_1, 2, 3, 4, 5 (Pengcheng Xiong)
Posted by om...@apache.org.
HIVE-15084: Flaky test: TestMiniTezCliDriver:explainanalyze_1, 2, 3, 4, 5 (Pengcheng Xiong)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/9f5bea3b
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/9f5bea3b
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/9f5bea3b
Branch: refs/heads/branch-2.2
Commit: 9f5bea3bae315a540a9ad4b13b085efb1a609b1e
Parents: fd72ec4
Author: Pengcheng Xiong <px...@apache.org>
Authored: Tue Jan 3 14:02:49 2017 -0800
Committer: Owen O'Malley <om...@apache.org>
Committed: Tue Mar 28 15:27:52 2017 -0700
----------------------------------------------------------------------
.../resources/testconfiguration.properties.orig | 1373 --
.../org/apache/hadoop/hive/ql/Context.java.orig | 829 --
.../apache/hadoop/hive/ql/parse/ParseUtils.java | 2 +-
.../hive/ql/parse/SemanticAnalyzer.java.orig | 13038 -----------------
.../hadoop/hive/ql/parse/TezCompiler.java | 2 +-
.../apache/hadoop/hive/ql/stats/StatsUtils.java | 2 +-
.../queries/clientpositive/explainanalyze_1.q | 1 +
.../queries/clientpositive/explainanalyze_2.q | 322 +-
.../queries/clientpositive/explainanalyze_3.q | 10 +-
.../queries/clientpositive/explainanalyze_4.q | 2 +
.../queries/clientpositive/explainanalyze_5.q | 10 +-
.../annotate_stats_join_pkfk.q.out | 8 +-
.../clientpositive/dynamic_rdd_cache.q.out | 24 +-
.../llap/dynamic_semijoin_reduction.q.out | 72 +-
.../clientpositive/llap/subquery_multi.q.out | 40 +-
.../clientpositive/llap/subquery_scalar.q.out | 120 +-
.../llap/tez_union_multiinsert.q.out | 86 +-
.../vectorized_dynamic_semijoin_reduction.q.out | 24 +-
...vectorized_dynamic_semijoin_reduction2.q.out | 24 +-
.../clientpositive/load_dyn_part14.q.out | 14 +-
ql/src/test/results/clientpositive/stats1.q.out | 10 +-
.../clientpositive/tez/explainanalyze_1.q.out | 189 +-
.../clientpositive/tez/explainanalyze_2.q.out | 4820 +-----
.../clientpositive/tez/explainanalyze_4.q.out | 258 +-
.../clientpositive/tez/explainanalyze_5.q.out | 166 +-
.../clientpositive/tez/explainuser_2.q.out | 30 +-
.../results/clientpositive/tez/mergejoin.q.out | 40 +-
.../tez/tez_union_multiinsert.q.out | 86 +-
.../results/clientpositive/tez/union6.q.out | 6 +-
.../test/results/clientpositive/union17.q.out | 16 +-
.../test/results/clientpositive/union18.q.out | 18 +-
.../test/results/clientpositive/union19.q.out | 14 +-
.../test/results/clientpositive/union20.q.out | 24 +-
ql/src/test/results/clientpositive/union6.q.out | 10 +-
.../clientpositive/unionDistinct_1.q.out | 190 +-
35 files changed, 814 insertions(+), 21066 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/9f5bea3b/itests/src/test/resources/testconfiguration.properties.orig
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties.orig b/itests/src/test/resources/testconfiguration.properties.orig
deleted file mode 100644
index d5ee9ed..0000000
--- a/itests/src/test/resources/testconfiguration.properties.orig
+++ /dev/null
@@ -1,1373 +0,0 @@
-# NOTE: files should be listed in alphabetical order
-minimr.query.files=auto_sortmerge_join_16.q,\
- bucket4.q,\
- bucket5.q,\
- bucket6.q,\
- bucket_many.q,\
- bucket_num_reducers.q,\
- bucket_num_reducers2.q,\
- bucketizedhiveinputformat.q,\
- bucketmapjoin6.q,\
- bucketmapjoin7.q,\
- disable_merge_for_bucketing.q,\
- empty_dir_in_table.q,\
- exchgpartition2lel.q,\
- external_table_with_space_in_location_path.q,\
- file_with_header_footer.q,\
- groupby2.q,\
- import_exported_table.q,\
- index_bitmap3.q,\
- index_bitmap_auto.q,\
- infer_bucket_sort_bucketed_table.q,\
- infer_bucket_sort_dyn_part.q,\
- infer_bucket_sort_map_operators.q,\
- infer_bucket_sort_merge.q,\
- infer_bucket_sort_num_buckets.q,\
- infer_bucket_sort_reducers_power_two.q,\
- input16_cc.q,\
- insert_dir_distcp.q,\
- join1.q,\
- join_acid_non_acid.q,\
- leftsemijoin_mr.q,\
- list_bucket_dml_10.q,\
- load_fs2.q,\
- load_hdfs_file_with_space_in_the_name.q,\
- non_native_window_udf.q, \
- orc_merge_diff_fs.q,\
- parallel_orderby.q,\
- quotedid_smb.q,\
- reduce_deduplicate.q,\
- remote_script.q,\
- root_dir_external_table.q,\
- schemeAuthority.q,\
- schemeAuthority2.q,\
- scriptfile1.q,\
- scriptfile1_win.q,\
- skewjoin_onesideskew.q,\
- table_nonprintable.q,\
- temp_table_external.q,\
- truncate_column_buckets.q,\
- uber_reduce.q,\
- udf_using.q
-
-# These tests are disabled for minimr
-# ql_rewrite_gbtoidx.q,\
-# ql_rewrite_gbtoidx_cbo_1.q,\
-# ql_rewrite_gbtoidx_cbo_2.q,\
-# smb_mapjoin_8.q,\
-
-
-# Tests that are not enabled for CLI Driver
-disabled.query.files=ql_rewrite_gbtoidx.q,\
- ql_rewrite_gbtoidx_cbo_1.q,\
- ql_rewrite_gbtoidx_cbo_2.q,\
- rcfile_merge1.q,\
- smb_mapjoin_8.q
-
-# NOTE: Add tests to minitez only if it is very
-# specific to tez and cannot be added to minillap.
-minitez.query.files.shared=delete_orig_table.q,\
- orc_merge12.q,\
- orc_vectorization_ppd.q,\
- unionDistinct_2.q,\
- update_orig_table.q,\
- vector_join_part_col_char.q,\
- vector_non_string_partition.q,\
- vectorization_div0.q,\
- vectorization_limit.q
-
-# NOTE: Add tests to minitez only if it is very
-# specific to tez and cannot be added to minillap.
-minitez.query.files=explainuser_3.q,\
- hybridgrace_hashjoin_1.q,\
- hybridgrace_hashjoin_2.q,\
- partition_column_names_with_leading_and_trailing_spaces.q,\
- stats_filemetadata.q,\
- tez_union_with_udf.q
-
-minillap.shared.query.files=acid_globallimit.q,\
- alter_merge_2_orc.q,\
- alter_merge_orc.q,\
- alter_merge_stats_orc.q,\
- auto_join0.q,\
- auto_join1.q,\
- auto_join21.q,\
- auto_join29.q,\
- auto_join30.q,\
- auto_join_filters.q,\
- auto_join_nulls.q,\
- auto_sortmerge_join_1.q,\
- auto_sortmerge_join_10.q,\
- auto_sortmerge_join_11.q,\
- auto_sortmerge_join_12.q,\
- auto_sortmerge_join_13.q,\
- auto_sortmerge_join_14.q,\
- auto_sortmerge_join_15.q,\
- auto_sortmerge_join_16.q,\
- auto_sortmerge_join_2.q,\
- auto_sortmerge_join_3.q,\
- auto_sortmerge_join_4.q,\
- auto_sortmerge_join_5.q,\
- auto_sortmerge_join_6.q,\
- auto_sortmerge_join_7.q,\
- auto_sortmerge_join_8.q,\
- auto_sortmerge_join_9.q,\
- bucket2.q,\
- bucket3.q,\
- bucket4.q,\
- bucket_map_join_tez1.q,\
- bucket_map_join_tez2.q,\
- cbo_gby.q,\
- cbo_gby_empty.q,\
- cbo_join.q,\
- cbo_limit.q,\
- cbo_semijoin.q,\
- cbo_simple_select.q,\
- cbo_stats.q,\
- cbo_subq_exists.q,\
- cbo_subq_in.q,\
- cbo_subq_not_in.q,\
- cbo_udf_udaf.q,\
- cbo_union.q,\
- cbo_views.q,\
- cbo_windowing.q,\
- column_names_with_leading_and_trailing_spaces.q,\
- constprog_dpp.q,\
- constprog_semijoin.q,\
- correlationoptimizer1.q,\
- count.q,\
- create_merge_compressed.q,\
- cross_join.q,\
- cross_product_check_1.q,\
- cross_product_check_2.q,\
- ctas.q,\
- cte_1.q,\
- cte_2.q,\
- cte_3.q,\
- cte_4.q,\
- cte_5.q,\
- cte_mat_1.q,\
- cte_mat_2.q,\
- cte_mat_3.q,\
- cte_mat_4.q,\
- cte_mat_5.q,\
- custom_input_output_format.q,\
- deleteAnalyze.q,\
- delete_all_non_partitioned.q,\
- delete_all_partitioned.q,\
- delete_tmp_table.q,\
- delete_where_no_match.q,\
- delete_where_non_partitioned.q,\
- delete_where_partitioned.q,\
- delete_whole_partition.q,\
- disable_merge_for_bucketing.q,\
- dynamic_partition_pruning.q,\
- dynamic_partition_pruning_2.q,\
- dynpart_sort_opt_vectorization.q,\
- dynpart_sort_optimization.q,\
- dynpart_sort_optimization2.q,\
- empty_join.q,\
- enforce_order.q,\
- filter_join_breaktask.q,\
- filter_join_breaktask2.q,\
- groupby1.q,\
- groupby2.q,\
- groupby3.q,\
- having.q,\
- identity_project_remove_skip.q,\
- insert1.q,\
- insert_into1.q,\
- insert_into2.q,\
- insert_orig_table.q,\
- insert_update_delete.q,\
- insert_values_dynamic_partitioned.q,\
- insert_values_non_partitioned.q,\
- insert_values_orig_table.,\
- insert_values_partitioned.q,\
- insert_values_tmp_table.q,\
- join0.q,\
- join1.q,\
- join_nullsafe.q,\
- leftsemijoin.q,\
- limit_pushdown.q,\
- llap_nullscan.q,\
- llapdecider.q,\
- load_dyn_part1.q,\
- load_dyn_part2.q,\
- load_dyn_part3.q,\
- lvj_mapjoin.q,\
- mapjoin2.q,\
- mapjoin_decimal.q,\
- mapjoin_mapjoin.q,\
- mapreduce1.q,\
- mapreduce2.q,\
- merge1.q,\
- merge2.q,\
- mergejoin.q,\
- metadata_only_queries.q,\
- metadata_only_queries_with_filters.q,\
- metadataonly1.q,\
- mrr.q,\
- nonmr_fetch_threshold.q,\
- optimize_nullscan.q,\
- orc_analyze.q,\
- orc_merge1.q,\
- orc_merge10.q,\
- orc_merge11.q,\
- orc_merge2.q,\
- orc_merge3.q,\
- orc_merge4.q,\
- orc_merge5.q,\
- orc_merge6.q,\
- orc_merge7.q,\
- orc_merge8.q,\
- orc_merge9.q,\
- orc_merge_incompat1.q,\
- orc_merge_incompat2.q,\
- orc_merge_incompat3.q,\
- orc_ppd_basic.q,\
- orc_ppd_schema_evol_1a.q,\
- orc_ppd_schema_evol_1b.q,\
- orc_ppd_schema_evol_2a.q,\
- orc_ppd_schema_evol_2b.q,\
- orc_ppd_schema_evol_3a.q,\
- order_null.q,\
- parallel.q,\
- ptf.q,\
- ptf_matchpath.q,\
- ptf_streaming.q,\
- sample1.q,\
- script_env_var1.q,\
- script_env_var2.q,\
- script_pipe.q,\
- scriptfile1.q,\
- selectDistinctStar.q,\
- select_dummy_source.q,\
- skewjoin.q,\
- stats_noscan_1.q,\
- stats_only_null.q,\
- subquery_exists.q,\
- subquery_in.q,\
- temp_table.q,\
- tez_bmj_schema_evolution.q,\
- tez_dml.q,\
- tez_dynpart_hashjoin_1.q,\
- tez_dynpart_hashjoin_2.q,\
- tez_fsstat.q,\
- tez_insert_overwrite_local_directory_1.q,\
- tez_join.q,\
- tez_join_hash.q,\
- tez_join_result_complex.q,\
- tez_join_tests.q,\
- tez_joins_explain.q,\
- tez_multi_union.q,\
- tez_schema_evolution.q,\
- tez_self_join.q,\
- tez_smb_1.q,\
- tez_smb_main.q,\
- tez_union.q,\
- tez_union2.q,\
- tez_union_decimal.q,\
- tez_union_dynamic_partition.q,\
- tez_union_group_by.q,\
- tez_union_multiinsert.q,\
- tez_union_view.q,\
- tez_vector_dynpart_hashjoin_1.q,\
- tez_vector_dynpart_hashjoin_2.q,\
- transform1.q,\
- transform2.q,\
- transform_ppr1.q,\
- transform_ppr2.q,\
- union2.q,\
- union3.q,\
- union4.q,\
- union5.q,\
- union6.q,\
- union7.q,\
- union8.q,\
- union9.q,\
- unionDistinct_1.q,\
- union_fast_stats.q,\
- union_stats.q,\
- union_type_chk.q,\
- update_after_multiple_inserts.q,\
- update_all_non_partitioned.q,\
- update_all_partitioned.q,\
- update_all_types.q,\
- update_tmp_table.q,\
- update_two_cols.q,\
- update_where_no_match.q,\
- update_where_non_partitioned.q,\
- update_where_partitioned.q,\
- vector_acid3.q,\
- vector_aggregate_9.q,\
- vector_aggregate_without_gby.q,\
- vector_auto_smb_mapjoin_14.q,\
- vector_between_columns.q,\
- vector_between_in.q,\
- vector_binary_join_groupby.q,\
- vector_bround.q,\
- vector_bucket.q,\
- vector_cast_constant.q,\
- vector_char_2.q,\
- vector_char_4.q,\
- vector_char_cast.q,\
- vector_char_mapjoin1.q,\
- vector_char_simple.q,\
- vector_coalesce.q,\
- vector_coalesce_2.q,\
- vector_complex_all.q,\
- vector_complex_join.q,\
- vector_count.q,\
- vector_count_distinct.q,\
- vector_data_types.q,\
- vector_date_1.q,\
- vector_decimal_1.q,\
- vector_decimal_10_0.q,\
- vector_decimal_2.q,\
- vector_decimal_3.q,\
- vector_decimal_4.q,\
- vector_decimal_5.q,\
- vector_decimal_6.q,\
- vector_decimal_aggregate.q,\
- vector_decimal_cast.q,\
- vector_decimal_expressions.q,\
- vector_decimal_mapjoin.q,\
- vector_decimal_math_funcs.q,\
- vector_decimal_precision.q,\
- vector_decimal_round.q,\
- vector_decimal_round_2.q,\
- vector_decimal_trailing.q,\
- vector_decimal_udf.q,\
- vector_decimal_udf2.q,\
- vector_distinct_2.q,\
- vector_elt.q,\
- vector_groupby4.q,\
- vector_groupby6.q,\
- vector_groupby_3.q,\
- vector_groupby_mapjoin.q,\
- vector_groupby_reduce.q,\
- vector_grouping_sets.q,\
- vector_if_expr.q,\
- vector_include_no_sel.q,\
- vector_inner_join.q,\
- vector_interval_1.q,\
- vector_interval_2.q,\
- vector_interval_arithmetic.q,\
- vector_interval_mapjoin.q,\
- vector_join30.q,\
- vector_join_filters.q,\
- vector_join_nulls.q,\
- vector_left_outer_join.q,\
- vector_left_outer_join2.q,\
- vector_leftsemi_mapjoin.q,\
- vector_mapjoin_reduce.q,\
- vector_mr_diff_schema_alias.q,\
- vector_multi_insert.q,\
- vector_null_projection.q,\
- vector_nullsafe_join.q,\
- vector_nvl.q,\
- vector_orderby_5.q,\
- vector_outer_join0.q,\
- vector_outer_join1.q,\
- vector_outer_join2.q,\
- vector_outer_join3.q,\
- vector_outer_join4.q,\
- vector_outer_join5.q,\
- vector_outer_join6.q,\
- vector_partition_diff_num_cols.q,\
- vector_partitioned_date_time.q,\
- vector_reduce1.q,\
- vector_reduce2.q,\
- vector_reduce3.q,\
- vector_reduce_groupby_decimal.q,\
- vector_string_concat.q,\
- vector_struct_in.q,\
- vector_varchar_4.q,\
- vector_varchar_mapjoin1.q,\
- vector_varchar_simple.q,\
- vector_when_case_null.q,\
- vectorization_0.q,\
- vectorization_1.q,\
- vectorization_10.q,\
- vectorization_11.q,\
- vectorization_12.q,\
- vectorization_13.q,\
- vectorization_14.q,\
- vectorization_15.q,\
- vectorization_16.q,\
- vectorization_17.q,\
- vectorization_2.q,\
- vectorization_3.q,\
- vectorization_4.q,\
- vectorization_5.q,\
- vectorization_6.q,\
- vectorization_7.q,\
- vectorization_8.q,\
- vectorization_9.q,\
- vectorization_decimal_date.q,\
- vectorization_nested_udf.q,\
- vectorization_not.q,\
- vectorization_part.q,\
- vectorization_part_project.q,\
- vectorization_part_varchar.q,\
- vectorization_pushdown.q,\
- vectorization_short_regress.q,\
- vectorized_bucketmapjoin1.q,\
- vectorized_case.q,\
- vectorized_casts.q,\
- vectorized_context.q,\
- vectorized_date_funcs.q,\
- vectorized_distinct_gby.q,\
- vectorized_dynamic_partition_pruning.q,\
- vectorized_mapjoin.q,\
- vectorized_math_funcs.q,\
- vectorized_nested_mapjoin.q,\
- vectorized_parquet.q,\
- vectorized_parquet_types.q,\
- vectorized_ptf.q,\
- vectorized_rcfile_columnar.q,\
- vectorized_shufflejoin.q,\
- vectorized_string_funcs.q,\
- vectorized_timestamp.q,\
- vectorized_timestamp_funcs.q,\
- vectorized_timestamp_ints_casts.q
-
-minillap.query.files=acid_bucket_pruning.q,\
- acid_vectorization_missing_cols.q,\
- bucket_map_join_tez1.q,\
- bucket_map_join_tez2.q,\
- bucketpruning1.q,\
- constprog_dpp.q,\
- dynamic_partition_pruning.q,\
- dynamic_partition_pruning_2.q,\
- explainuser_1.q,\
- explainuser_2.q,\
- explainuser_4.q,\
- hybridgrace_hashjoin_1.q,\
- hybridgrace_hashjoin_2.q,\
- llap_nullscan.q,\
- llap_udf.q,\
- llapdecider.q,\
- lvj_mapjoin.q,\
- mapjoin_decimal.q,\
- mergejoin_3way.q,\
- mrr.q,\
- orc_llap.q,\
- orc_llap_counters.q,\
- orc_llap_counters1.q,\
- orc_llap_nonvector.q,\
- orc_merge_diff_fs.q,\
- orc_ppd_basic.q,\
- schema_evol_orc_acid_part.q,\
- schema_evol_orc_acid_part_update.q,\
- schema_evol_orc_acid_table.q,\
- schema_evol_orc_acid_table_update.q,\
- schema_evol_orc_acidvec_part.q,\
- schema_evol_orc_acidvec_part_update.q,\
- schema_evol_orc_acidvec_table.q,\
- schema_evol_orc_acidvec_table_update.q,\
- schema_evol_orc_nonvec_part.q,\
- schema_evol_orc_nonvec_part_all_complex.q,\
- schema_evol_orc_nonvec_part_all_primitive.q,\
- schema_evol_orc_nonvec_table.q,\
- schema_evol_orc_vec_part.q,\
- schema_evol_orc_vec_part_all_complex.q,\
- schema_evol_orc_vec_part_all_primitive.q,\
- schema_evol_orc_vec_table.q,\
- schema_evol_stats.q,\
- schema_evol_text_nonvec_part.q,\
- schema_evol_text_nonvec_part_all_complex.q,\
- schema_evol_text_nonvec_part_all_primitive.q,\
- schema_evol_text_nonvec_table.q,\
- schema_evol_text_vec_part.q,\
- schema_evol_text_vec_part_all_complex.q,\
- schema_evol_text_vec_part_all_primitive.q,\
- schema_evol_text_vec_table.q,\
- schema_evol_text_vecrow_part.q,\
- schema_evol_text_vecrow_part_all_complex.q,\
- schema_evol_text_vecrow_part_all_primitive.q,\
- schema_evol_text_vecrow_table.q,\
- smb_cache.q,\
- tez_aggr_part_stats.q,\
- tez_bmj_schema_evolution.q,\
- tez_dml.q,\
- tez_dynpart_hashjoin_1.q,\
- tez_dynpart_hashjoin_2.q,\
- tez_dynpart_hashjoin_3.q,\
- tez_fsstat.q,\
- tez_insert_overwrite_local_directory_1.q,\
- tez_join.q,\
- tez_join_result_complex.q,\
- tez_join_tests.q,\
- tez_joins_explain.q,\
- tez_multi_union.q,\
- tez_schema_evolution.q,\
- tez_self_join.q,\
- tez_smb_1.q,\
- tez_smb_empty.q,\
- tez_smb_main.q,\
- tez_union.q,\
- tez_union2.q,\
- tez_union_decimal.q,\
- tez_union_dynamic_partition.q,\
- tez_union_group_by.q,\
- tez_union_multiinsert.q,\
- tez_union_view.q,\
- tez_vector_dynpart_hashjoin_1.q,\
- tez_vector_dynpart_hashjoin_2.q,\
- vectorized_dynamic_partition_pruning.q,\
- windowing_gby.q
-
-encrypted.query.files=encryption_join_unencrypted_tbl.q,\
- encryption_insert_partition_static.q,\
- encryption_insert_partition_dynamic.q,\
- encryption_join_with_different_encryption_keys.q,\
- encryption_select_read_only_encrypted_tbl.q,\
- encryption_select_read_only_unencrypted_tbl.q,\
- encryption_load_data_to_encrypted_tables.q, \
- encryption_unencrypted_nonhdfs_external_tables.q \
- encryption_move_tbl.q \
- encryption_drop_table.q \
- encryption_insert_values.q \
- encryption_drop_view.q \
- encryption_drop_partition.q \
- encryption_with_trash.q \
- encryption_ctas.q
-
-beeline.positive.exclude=add_part_exist.q,\
- alter1.q,\
- alter2.q,\
- alter4.q,\
- alter5.q,\
- alter_rename_partition.q,\
- alter_rename_partition_authorization.q,\
- archive.q,\
- archive_corrupt.q,\
- archive_mr_1806.q,\
- archive_multi.q,\
- archive_multi_mr_1806.q,\
- authorization_1.q,\
- authorization_2.q,\
- authorization_4.q,\
- authorization_5.q,\
- authorization_6.q,\
- authorization_7.q,\
- ba_table1.q,\
- ba_table2.q,\
- ba_table3.q,\
- ba_table_udfs.q,\
- binary_table_bincolserde.q,\
- binary_table_colserde.q,\
- cluster.q,\
- columnarserde_create_shortcut.q,\
- combine2.q,\
- constant_prop.q,\
- create_nested_type.q,\
- create_or_replace_view.q,\
- create_struct_table.q,\
- create_union_table.q,\
- database.q,\
- database_location.q,\
- database_properties.q,\
- describe_database_json.q,\
- drop_database_removes_partition_dirs.q,\
- escape1.q,\
- escape2.q,\
- exim_00_nonpart_empty.q,\
- exim_01_nonpart.q,\
- exim_02_00_part_empty.q,\
- exim_02_part.q,\
- exim_03_nonpart_over_compat.q,\
- exim_04_all_part.q,\
- exim_04_evolved_parts.q,\
- exim_05_some_part.q,\
- exim_06_one_part.q,\
- exim_07_all_part_over_nonoverlap.q,\
- exim_08_nonpart_rename.q,\
- exim_09_part_spec_nonoverlap.q,\
- exim_10_external_managed.q,\
- exim_11_managed_external.q,\
- exim_12_external_location.q,\
- exim_13_managed_location.q,\
- exim_14_managed_location_over_existing.q,\
- exim_15_external_part.q,\
- exim_16_part_external.q,\
- exim_17_part_managed.q,\
- exim_18_part_external.q,\
- exim_19_00_part_external_location.q,\
- exim_19_part_external_location.q,\
- exim_20_part_managed_location.q,\
- exim_21_export_authsuccess.q,\
- exim_22_import_exist_authsuccess.q,\
- exim_23_import_part_authsuccess.q,\
- exim_24_import_nonexist_authsuccess.q,\
- global_limit.q,\
- groupby_complex_types.q,\
- groupby_complex_types_multi_single_reducer.q,\
- index_auth.q,\
- index_auto.q,\
- index_auto_empty.q,\
- index_bitmap.q,\
- index_bitmap1.q,\
- index_bitmap2.q,\
- index_bitmap3.q,\
- index_bitmap_auto.q,\
- index_bitmap_rc.q,\
- index_compact.q,\
- index_compact_1.q,\
- index_compact_2.q,\
- index_compact_3.q,\
- index_stale_partitioned.q,\
- init_file.q,\
- input16.q,\
- input16_cc.q,\
- input46.q,\
- input_columnarserde.q,\
- input_dynamicserde.q,\
- input_lazyserde.q,\
- input_testxpath3.q,\
- input_testxpath4.q,\
- insert2_overwrite_partitions.q,\
- insertexternal1.q,\
- join_thrift.q,\
- lateral_view.q,\
- load_binary_data.q,\
- load_exist_part_authsuccess.q,\
- load_nonpart_authsuccess.q,\
- load_part_authsuccess.q,\
- loadpart_err.q,\
- lock1.q,\
- lock2.q,\
- lock3.q,\
- lock4.q,\
- merge_dynamic_partition.q,\
- multi_insert.q,\
- multi_insert_move_tasks_share_dependencies.q,\
- null_column.q,\
- ppd_clusterby.q,\
- query_with_semi.q,\
- rename_column.q,\
- sample6.q,\
- sample_islocalmode_hook.q,\
- set_processor_namespaces.q,\
- show_tables.q,\
- source.q,\
- split_sample.q,\
- str_to_map.q,\
- transform1.q,\
- udaf_collect_set.q,\
- udaf_context_ngrams.q,\
- udaf_histogram_numeric.q,\
- udaf_ngrams.q,\
- udaf_percentile_approx.q,\
- udf_array.q,\
- udf_bitmap_and.q,\
- udf_bitmap_or.q,\
- udf_explode.q,\
- udf_format_number.q,\
- udf_map.q,\
- udf_map_keys.q,\
- udf_map_values.q,\
- udf_mask.q,\
- udf_mask_first_n.q,\
- udf_mask_hash.q,\
- udf_mask_last_n.q,\
- udf_mask_show_first_n.q,\
- udf_mask_show_last_n.q,\
- udf_max.q,\
- udf_min.q,\
- udf_named_struct.q,\
- udf_percentile.q,\
- udf_printf.q,\
- udf_sentences.q,\
- udf_sort_array.q,\
- udf_split.q,\
- udf_struct.q,\
- udf_substr.q,\
- udf_translate.q,\
- udf_union.q,\
- udf_xpath.q,\
- udtf_stack.q,\
- view.q,\
- virtual_column.q
-
-minimr.query.negative.files=cluster_tasklog_retrieval.q,\
- file_with_header_footer_negative.q,\
- local_mapred_error_cache.q,\
- mapreduce_stack_trace.q,\
- mapreduce_stack_trace_hadoop20.q,\
- mapreduce_stack_trace_turnoff.q,\
- mapreduce_stack_trace_turnoff_hadoop20.q,\
- minimr_broken_pipe.q,\
- table_nonprintable_negative.q,\
- udf_local_resource.q
-
-# tests are sorted use: perl -pe 's@\\\s*\n@ @g' testconfiguration.properties \
-# | awk -F= '/spark.query.files/{print $2}' | perl -pe 's@.q *, *@\n@g' \
-# | egrep -v '^ *$' | sort -V | uniq | perl -pe 's@\n@.q, \\\n@g' | perl -pe 's@^@ @g'
-spark.query.files=add_part_multiple.q, \
- alter_merge_orc.q, \
- alter_merge_stats_orc.q, \
- annotate_stats_join.q, \
- auto_join0.q, \
- auto_join1.q, \
- auto_join10.q, \
- auto_join11.q, \
- auto_join12.q, \
- auto_join13.q, \
- auto_join14.q, \
- auto_join15.q, \
- auto_join16.q, \
- auto_join17.q, \
- auto_join18.q, \
- auto_join18_multi_distinct.q, \
- auto_join19.q, \
- auto_join2.q, \
- auto_join20.q, \
- auto_join21.q, \
- auto_join22.q, \
- auto_join23.q, \
- auto_join24.q, \
- auto_join26.q, \
- auto_join27.q, \
- auto_join28.q, \
- auto_join29.q, \
- auto_join3.q, \
- auto_join30.q, \
- auto_join31.q, \
- auto_join4.q, \
- auto_join5.q, \
- auto_join6.q, \
- auto_join7.q, \
- auto_join8.q, \
- auto_join9.q, \
- auto_join_filters.q, \
- auto_join_nulls.q, \
- auto_join_reordering_values.q, \
- auto_join_stats.q, \
- auto_join_stats2.q, \
- auto_join_without_localtask.q, \
- auto_smb_mapjoin_14.q, \
- auto_sortmerge_join_1.q, \
- auto_sortmerge_join_10.q, \
- auto_sortmerge_join_12.q, \
- auto_sortmerge_join_13.q, \
- auto_sortmerge_join_14.q, \
- auto_sortmerge_join_15.q, \
- auto_sortmerge_join_16.q, \
- auto_sortmerge_join_3.q, \
- auto_sortmerge_join_4.q, \
- auto_sortmerge_join_5.q, \
- auto_sortmerge_join_6.q, \
- auto_sortmerge_join_7.q, \
- auto_sortmerge_join_8.q, \
- auto_sortmerge_join_9.q, \
- avro_compression_enabled_native.q, \
- avro_decimal_native.q, \
- avro_joins.q, \
- avro_joins_native.q, \
- bucket2.q, \
- bucket3.q, \
- bucket4.q, \
- bucket_map_join_1.q, \
- bucket_map_join_2.q, \
- bucket_map_join_spark1.q, \
- bucket_map_join_spark2.q, \
- bucket_map_join_spark3.q, \
- bucket_map_join_spark4.q, \
- bucket_map_join_tez1.q, \
- bucket_map_join_tez2.q, \
- bucketmapjoin1.q, \
- bucketmapjoin10.q, \
- bucketmapjoin11.q, \
- bucketmapjoin12.q, \
- bucketmapjoin13.q, \
- bucketmapjoin2.q, \
- bucketmapjoin3.q, \
- bucketmapjoin4.q, \
- bucketmapjoin5.q, \
- bucketmapjoin7.q, \
- bucketmapjoin8.q, \
- bucketmapjoin9.q, \
- bucketmapjoin_negative.q, \
- bucketmapjoin_negative2.q, \
- bucketmapjoin_negative3.q, \
- bucketsortoptimize_insert_2.q, \
- bucketsortoptimize_insert_4.q, \
- bucketsortoptimize_insert_6.q, \
- bucketsortoptimize_insert_7.q, \
- bucketsortoptimize_insert_8.q, \
- cbo_gby.q, \
- cbo_gby_empty.q, \
- cbo_limit.q, \
- cbo_semijoin.q, \
- cbo_simple_select.q, \
- cbo_stats.q, \
- cbo_subq_in.q, \
- cbo_subq_not_in.q, \
- cbo_udf_udaf.q, \
- cbo_union.q, \
- column_access_stats.q, \
- count.q, \
- create_merge_compressed.q, \
- cross_join.q, \
- cross_product_check_1.q, \
- cross_product_check_2.q, \
- ctas.q, \
- custom_input_output_format.q, \
- date_join1.q, \
- date_udf.q, \
- decimal_1_1.q, \
- decimal_join.q, \
- disable_merge_for_bucketing.q, \
- dynamic_rdd_cache.q, \
- enforce_order.q, \
- escape_clusterby1.q, \
- escape_distributeby1.q, \
- escape_orderby1.q, \
- escape_sortby1.q, \
- filter_join_breaktask.q, \
- filter_join_breaktask2.q, \
- groupby1.q, \
- groupby10.q, \
- groupby11.q, \
- groupby1_map.q, \
- groupby1_map_nomap.q, \
- groupby1_map_skew.q, \
- groupby1_noskew.q, \
- groupby2.q, \
- groupby2_map.q, \
- groupby2_map_multi_distinct.q, \
- groupby2_map_skew.q, \
- groupby2_noskew.q, \
- groupby2_noskew_multi_distinct.q, \
- groupby3.q, \
- groupby3_map.q, \
- groupby3_map_multi_distinct.q, \
- groupby3_map_skew.q, \
- groupby3_noskew.q, \
- groupby3_noskew_multi_distinct.q, \
- groupby4.q, \
- groupby4_map.q, \
- groupby4_map_skew.q, \
- groupby4_noskew.q, \
- groupby5.q, \
- groupby5_map.q, \
- groupby5_map_skew.q, \
- groupby5_noskew.q, \
- groupby6.q, \
- groupby6_map.q, \
- groupby6_map_skew.q, \
- groupby6_noskew.q, \
- groupby7.q, \
- groupby7_map.q, \
- groupby7_map_multi_single_reducer.q, \
- groupby7_map_skew.q, \
- groupby7_noskew.q, \
- groupby7_noskew_multi_single_reducer.q, \
- groupby8.q, \
- groupby8_map.q, \
- groupby8_map_skew.q, \
- groupby8_noskew.q, \
- groupby9.q, \
- groupby_bigdata.q, \
- groupby_complex_types.q, \
- groupby_complex_types_multi_single_reducer.q, \
- groupby_cube1.q, \
- groupby_grouping_id2.q, \
- groupby_map_ppr.q, \
- groupby_map_ppr_multi_distinct.q, \
- groupby_multi_insert_common_distinct.q, \
- groupby_multi_single_reducer.q, \
- groupby_multi_single_reducer2.q, \
- groupby_multi_single_reducer3.q, \
- groupby_position.q, \
- groupby_ppr.q, \
- groupby_ppr_multi_distinct.q, \
- groupby_resolution.q, \
- groupby_rollup1.q, \
- groupby_sort_1_23.q, \
- groupby_sort_skew_1.q, \
- groupby_sort_skew_1_23.q, \
- qroupby_limit_extrastep.q, \
- having.q, \
- identity_project_remove_skip.q, \
- index_auto_self_join.q, \
- innerjoin.q, \
- input12.q, \
- input13.q, \
- input14.q, \
- input17.q, \
- input18.q, \
- input1_limit.q, \
- input_part2.q, \
- insert_into1.q, \
- insert_into2.q, \
- insert_into3.q, \
- join0.q, \
- join1.q, \
- join10.q, \
- join11.q, \
- join12.q, \
- join13.q, \
- join14.q, \
- join15.q, \
- join16.q, \
- join17.q, \
- join18.q, \
- join18_multi_distinct.q, \
- join19.q, \
- join2.q, \
- join20.q, \
- join21.q, \
- join22.q, \
- join23.q, \
- join24.q, \
- join25.q, \
- join26.q, \
- join27.q, \
- join28.q, \
- join29.q, \
- join3.q, \
- join30.q, \
- join31.q, \
- join32.q, \
- join32_lessSize.q, \
- join33.q, \
- join34.q, \
- join35.q, \
- join36.q, \
- join37.q, \
- join38.q, \
- join39.q, \
- join4.q, \
- join41.q, \
- join5.q, \
- join6.q, \
- join7.q, \
- join8.q, \
- join9.q, \
- join_1to1.q, \
- join_alt_syntax.q, \
- join_array.q, \
- join_casesensitive.q, \
- join_cond_pushdown_1.q, \
- join_cond_pushdown_2.q, \
- join_cond_pushdown_3.q, \
- join_cond_pushdown_4.q, \
- join_cond_pushdown_unqual1.q, \
- join_cond_pushdown_unqual2.q, \
- join_cond_pushdown_unqual3.q, \
- join_cond_pushdown_unqual4.q, \
- join_filters_overlap.q, \
- join_hive_626.q, \
- join_literals.q, \
- join_map_ppr.q, \
- join_merge_multi_expressions.q, \
- join_merging.q, \
- join_nullsafe.q, \
- join_rc.q, \
- join_reorder.q, \
- join_reorder2.q, \
- join_reorder3.q, \
- join_reorder4.q, \
- join_star.q, \
- join_thrift.q, \
- join_vc.q, \
- join_view.q, \
- lateral_view_explode2.q, \
- leftsemijoin.q, \
- leftsemijoin_mr.q, \
- limit_partition_metadataonly.q, \
- limit_pushdown.q, \
- list_bucket_dml_2.q, \
- load_dyn_part1.q, \
- load_dyn_part10.q, \
- load_dyn_part11.q, \
- load_dyn_part12.q, \
- load_dyn_part13.q, \
- load_dyn_part14.q, \
- load_dyn_part15.q, \
- load_dyn_part2.q, \
- load_dyn_part3.q, \
- load_dyn_part4.q, \
- load_dyn_part5.q, \
- load_dyn_part6.q, \
- load_dyn_part7.q, \
- load_dyn_part8.q, \
- load_dyn_part9.q, \
- louter_join_ppr.q, \
- mapjoin1.q, \
- mapjoin_addjar.q, \
- mapjoin_decimal.q, \
- mapjoin_distinct.q, \
- mapjoin_filter_on_outerjoin.q, \
- mapjoin_mapjoin.q, \
- mapjoin_memcheck.q, \
- mapjoin_subquery.q, \
- mapjoin_subquery2.q, \
- mapjoin_test_outer.q, \
- mapreduce1.q, \
- mapreduce2.q, \
- merge1.q, \
- merge2.q, \
- mergejoins.q, \
- mergejoins_mixed.q, \
- metadata_only_queries.q, \
- metadata_only_queries_with_filters.q, \
- multi_insert.q, \
- multi_insert_gby.q, \
- multi_insert_gby2.q, \
- multi_insert_gby3.q, \
- multi_insert_lateral_view.q, \
- multi_insert_mixed.q, \
- multi_insert_move_tasks_share_dependencies.q, \
- multi_insert_with_join.q, \
- multi_join_union.q, \
- multi_join_union_src.q, \
- multigroupby_singlemr.q, \
- nullgroup.q, \
- nullgroup2.q, \
- nullgroup4.q, \
- nullgroup4_multi_distinct.q, \
- optimize_nullscan.q, \
- order.q, \
- order2.q, \
- outer_join_ppr.q, \
- parallel.q, \
- parallel_join0.q, \
- parallel_join1.q, \
- parquet_join.q, \
- pcr.q, \
- ppd_gby_join.q, \
- ppd_join.q, \
- ppd_join2.q, \
- ppd_join3.q, \
- ppd_join5.q, \
- ppd_join_filter.q, \
- ppd_multi_insert.q, \
- ppd_outer_join1.q, \
- ppd_outer_join2.q, \
- ppd_outer_join3.q, \
- ppd_outer_join4.q, \
- ppd_outer_join5.q, \
- ppd_transform.q, \
- ptf.q, \
- ptf_decimal.q, \
- ptf_general_queries.q, \
- ptf_matchpath.q, \
- ptf_rcfile.q, \
- ptf_register_tblfn.q, \
- ptf_seqfile.q, \
- ptf_streaming.q, \
- rcfile_bigdata.q, \
- reduce_deduplicate_exclude_join.q, \
- router_join_ppr.q, \
- runtime_skewjoin_mapjoin_spark.q, \
- sample1.q, \
- sample10.q, \
- sample2.q, \
- sample3.q, \
- sample4.q, \
- sample5.q, \
- sample6.q, \
- sample7.q, \
- sample8.q, \
- sample9.q, \
- script_env_var1.q, \
- script_env_var2.q, \
- script_pipe.q, \
- scriptfile1.q, \
- semijoin.q, \
- skewjoin.q, \
- skewjoin_noskew.q, \
- skewjoin_union_remove_1.q, \
- skewjoin_union_remove_2.q, \
- skewjoinopt1.q, \
- skewjoinopt10.q, \
- skewjoinopt11.q, \
- skewjoinopt12.q, \
- skewjoinopt13.q, \
- skewjoinopt14.q, \
- skewjoinopt15.q, \
- skewjoinopt16.q, \
- skewjoinopt17.q, \
- skewjoinopt18.q, \
- skewjoinopt19.q, \
- skewjoinopt2.q, \
- skewjoinopt20.q, \
- skewjoinopt3.q, \
- skewjoinopt4.q, \
- skewjoinopt5.q, \
- skewjoinopt6.q, \
- skewjoinopt7.q, \
- skewjoinopt8.q, \
- skewjoinopt9.q, \
- smb_mapjoin_1.q, \
- smb_mapjoin_10.q, \
- smb_mapjoin_11.q, \
- smb_mapjoin_12.q, \
- smb_mapjoin_13.q, \
- smb_mapjoin_14.q, \
- smb_mapjoin_15.q, \
- smb_mapjoin_16.q, \
- smb_mapjoin_17.q, \
- smb_mapjoin_18.q, \
- smb_mapjoin_19.q, \
- smb_mapjoin_2.q, \
- smb_mapjoin_20.q, \
- smb_mapjoin_21.q, \
- smb_mapjoin_22.q, \
- smb_mapjoin_25.q, \
- smb_mapjoin_3.q, \
- smb_mapjoin_4.q, \
- smb_mapjoin_5.q, \
- smb_mapjoin_6.q, \
- smb_mapjoin_7.q, \
- smb_mapjoin_8.q, \
- smb_mapjoin_9.q, \
- sort.q, \
- stats0.q, \
- stats1.q, \
- stats10.q, \
- stats12.q, \
- stats13.q, \
- stats14.q, \
- stats15.q, \
- stats16.q, \
- stats18.q, \
- stats2.q, \
- stats3.q, \
- stats5.q, \
- stats6.q, \
- stats7.q, \
- stats8.q, \
- stats9.q, \
- stats_noscan_1.q, \
- stats_noscan_2.q, \
- stats_only_null.q, \
- stats_partscan_1_23.q, \
- statsfs.q, \
- subquery_exists.q, \
- subquery_in.q, \
- subquery_multiinsert.q, \
- table_access_keys_stats.q, \
- temp_table.q, \
- temp_table_gb1.q, \
- temp_table_join1.q, \
- tez_join_tests.q, \
- tez_joins_explain.q, \
- timestamp_1.q, \
- timestamp_2.q, \
- timestamp_3.q, \
- timestamp_comparison.q, \
- timestamp_lazy.q, \
- timestamp_null.q, \
- timestamp_udf.q, \
- transform2.q, \
- transform_ppr1.q, \
- transform_ppr2.q, \
- udaf_collect_set.q, \
- udf_example_add.q, \
- udf_in_file.q, \
- udf_max.q, \
- udf_min.q, \
- udf_percentile.q, \
- union.q, \
- union10.q, \
- union11.q, \
- union12.q, \
- union13.q, \
- union14.q, \
- union15.q, \
- union16.q, \
- union17.q, \
- union18.q, \
- union19.q, \
- union2.q, \
- union20.q, \
- union21.q, \
- union22.q, \
- union23.q, \
- union24.q, \
- union25.q, \
- union26.q, \
- union27.q, \
- union28.q, \
- union29.q, \
- union3.q, \
- union30.q, \
- union31.q, \
- union32.q, \
- union33.q, \
- union34.q, \
- union4.q, \
- union5.q, \
- union6.q, \
- union7.q, \
- union8.q, \
- union9.q, \
- union_date.q, \
- union_date_trim.q, \
- union_lateralview.q, \
- union_null.q, \
- union_ppr.q, \
- union_remove_1.q, \
- union_remove_10.q, \
- union_remove_11.q, \
- union_remove_12.q, \
- union_remove_13.q, \
- union_remove_14.q, \
- union_remove_15.q, \
- union_remove_16.q, \
- union_remove_17.q, \
- union_remove_18.q, \
- union_remove_19.q, \
- union_remove_2.q, \
- union_remove_20.q, \
- union_remove_21.q, \
- union_remove_22.q, \
- union_remove_23.q, \
- union_remove_24.q, \
- union_remove_25.q, \
- union_remove_3.q, \
- union_remove_4.q, \
- union_remove_5.q, \
- union_remove_6.q, \
- union_remove_6_subq.q, \
- union_remove_7.q, \
- union_remove_8.q, \
- union_remove_9.q, \
- union_script.q, \
- union_top_level.q, \
- union_view.q, \
- uniquejoin.q, \
- varchar_join1.q, \
- vector_between_in.q, \
- vector_cast_constant.q, \
- vector_char_4.q, \
- vector_count_distinct.q, \
- vector_data_types.q, \
- vector_decimal_aggregate.q, \
- vector_decimal_mapjoin.q, \
- vector_distinct_2.q, \
- vector_elt.q, \
- vector_groupby_3.q, \
- vector_left_outer_join.q, \
- vector_mapjoin_reduce.q, \
- vector_orderby_5.q, \
- vector_string_concat.q, \
- vector_varchar_4.q, \
- vectorization_0.q, \
- vectorization_1.q, \
- vectorization_10.q, \
- vectorization_11.q, \
- vectorization_12.q, \
- vectorization_13.q, \
- vectorization_14.q, \
- vectorization_15.q, \
- vectorization_16.q, \
- vectorization_17.q, \
- vectorization_2.q, \
- vectorization_3.q, \
- vectorization_4.q, \
- vectorization_5.q, \
- vectorization_6.q, \
- vectorization_9.q, \
- vectorization_decimal_date.q, \
- vectorization_div0.q, \
- vectorization_nested_udf.q, \
- vectorization_not.q, \
- vectorization_part.q, \
- vectorization_part_project.q, \
- vectorization_pushdown.q, \
- vectorization_short_regress.q, \
- vectorized_case.q, \
- vectorized_mapjoin.q, \
- vectorized_math_funcs.q, \
- vectorized_nested_mapjoin.q, \
- vectorized_ptf.q, \
- vectorized_rcfile_columnar.q, \
- vectorized_shufflejoin.q, \
- vectorized_string_funcs.q, \
- vectorized_timestamp_funcs.q, \
- windowing.q
-
-# Unlike "spark.query.files" above, these tests only run
-# under Spark engine.
-spark.only.query.files=spark_dynamic_partition_pruning.q,\
- spark_dynamic_partition_pruning_2.q,\
- spark_vectorized_dynamic_partition_pruning.q
-
-miniSparkOnYarn.query.files=auto_sortmerge_join_16.q,\
- bucket4.q,\
- bucket5.q,\
- bucket6.q,\
- bucketizedhiveinputformat.q,\
- bucketmapjoin6.q,\
- bucketmapjoin7.q,\
- constprog_partitioner.q,\
- constprog_semijoin.q,\
- disable_merge_for_bucketing.q,\
- empty_dir_in_table.q,\
- external_table_with_space_in_location_path.q,\
- file_with_header_footer.q,\
- gen_udf_example_add10.q,\
- import_exported_table.q,\
- index_bitmap3.q,\
- index_bitmap_auto.q,\
- infer_bucket_sort_bucketed_table.q,\
- infer_bucket_sort_map_operators.q,\
- infer_bucket_sort_merge.q,\
- infer_bucket_sort_num_buckets.q,\
- infer_bucket_sort_reducers_power_two.q,\
- input16_cc.q,\
- insert_overwrite_directory2.q,\
- leftsemijoin_mr.q,\
- list_bucket_dml_10.q,\
- load_fs2.q,\
- load_hdfs_file_with_space_in_the_name.q,\
- orc_merge1.q,\
- orc_merge2.q,\
- orc_merge3.q,\
- orc_merge4.q,\
- orc_merge5.q,\
- orc_merge6.q,\
- orc_merge7.q,\
- orc_merge8.q,\
- orc_merge9.q,\
- orc_merge_diff_fs.q,\
- orc_merge_incompat1.q,\
- orc_merge_incompat2.q,\
- parallel_orderby.q,\
- quotedid_smb.q,\
- reduce_deduplicate.q,\
- remote_script.q,\
- root_dir_external_table.q,\
- schemeAuthority.q,\
- schemeAuthority2.q,\
- scriptfile1.q,\
- scriptfile1_win.q,\
- temp_table_external.q,\
- truncate_column_buckets.q,\
- uber_reduce.q,\
- vector_inner_join.q,\
- vector_outer_join0.q,\
- vector_outer_join1.q,\
- vector_outer_join2.q,\
- vector_outer_join3.q,\
- vector_outer_join4.q,\
- vector_outer_join5.q
-
-# These tests are removed from miniSparkOnYarn.query.files
-# ql_rewrite_gbtoidx.q,\
-# ql_rewrite_gbtoidx_cbo_1.q,\
-# smb_mapjoin_8.q,\
-
-
-spark.query.negative.files=groupby2_map_skew_multi_distinct.q,\
- groupby2_multi_distinct.q,\
- groupby3_map_skew_multi_distinct.q,\
- groupby3_multi_distinct.q,\
- groupby_grouping_sets7.q
http://git-wip-us.apache.org/repos/asf/hive/blob/9f5bea3b/ql/src/java/org/apache/hadoop/hive/ql/Context.java.orig
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Context.java.orig b/ql/src/java/org/apache/hadoop/hive/ql/Context.java.orig
deleted file mode 100644
index 4667f68..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/Context.java.orig
+++ /dev/null
@@ -1,829 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql;
-
-import java.io.DataInput;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.net.URI;
-import java.text.SimpleDateFormat;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import org.antlr.runtime.TokenRewriteStream;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.ContentSummary;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.hive.common.FileUtils;
-import org.apache.hadoop.hive.common.BlobStorageUtils;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.ql.exec.TaskRunner;
-import org.apache.hadoop.hive.ql.exec.Utilities;
-import org.apache.hadoop.hive.ql.hooks.WriteEntity;
-import org.apache.hadoop.hive.ql.io.AcidUtils;
-import org.apache.hadoop.hive.ql.lockmgr.DbTxnManager.Heartbeater;
-import org.apache.hadoop.hive.ql.lockmgr.HiveLock;
-import org.apache.hadoop.hive.ql.lockmgr.HiveLockObj;
-import org.apache.hadoop.hive.ql.lockmgr.HiveTxnManager;
-import org.apache.hadoop.hive.ql.lockmgr.LockException;
-import org.apache.hadoop.hive.ql.metadata.Table;
-import org.apache.hadoop.hive.ql.plan.LoadTableDesc;
-import org.apache.hadoop.hive.ql.session.SessionState;
-import org.apache.hadoop.hive.shims.ShimLoader;
-import org.apache.hadoop.util.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Context for Semantic Analyzers. Usage: not reusable - construct a new one for
- * each query should call clear() at end of use to remove temporary folders
- */
-public class Context {
- private boolean isHDFSCleanup;
- private Path resFile;
- private Path resDir;
- private FileSystem resFs;
- private static final Logger LOG = LoggerFactory.getLogger("hive.ql.Context");
- private Path[] resDirPaths;
- private int resDirFilesNum;
- boolean initialized;
- String originalTracker = null;
- private final CompilationOpContext opContext;
- private final Map<String, ContentSummary> pathToCS = new ConcurrentHashMap<String, ContentSummary>();
-
- // scratch path to use for all non-local (ie. hdfs) file system tmp folders
- private final Path nonLocalScratchPath;
-
- // scratch directory to use for local file system tmp folders
- private final String localScratchDir;
-
- // the permission to scratch directory (local and hdfs)
- private final String scratchDirPermission;
-
- // Keeps track of scratch directories created for different scheme/authority
- private final Map<String, Path> fsScratchDirs = new HashMap<String, Path>();
-
- private final Configuration conf;
- protected int pathid = 10000;
- protected boolean explain = false;
- protected String cboInfo;
- protected boolean cboSucceeded;
- protected boolean explainLogical = false;
- protected String cmd = "";
- // number of previous attempts
- protected int tryCount = 0;
- private TokenRewriteStream tokenRewriteStream;
-
- private String executionId;
-
- // List of Locks for this query
- protected List<HiveLock> hiveLocks;
-
- // Transaction manager for this query
- protected HiveTxnManager hiveTxnManager;
-
- // Used to track what type of acid operation (insert, update, or delete) we are doing. Useful
- // since we want to change where bucket columns are accessed in some operators and
- // optimizations when doing updates and deletes.
- private AcidUtils.Operation acidOperation = AcidUtils.Operation.NOT_ACID;
-
- private boolean needLockMgr;
-
- private AtomicInteger sequencer = new AtomicInteger();
-
- private final Map<String, Table> cteTables = new HashMap<String, Table>();
-
- // Keep track of the mapping from load table desc to the output and the lock
- private final Map<LoadTableDesc, WriteEntity> loadTableOutputMap =
- new HashMap<LoadTableDesc, WriteEntity>();
- private final Map<WriteEntity, List<HiveLockObj>> outputLockObjects =
- new HashMap<WriteEntity, List<HiveLockObj>>();
-
- private final String stagingDir;
-
- private Heartbeater heartbeater;
-
- private boolean skipTableMasking;
-
- public Context(Configuration conf) throws IOException {
- this(conf, generateExecutionId());
- }
-
- /**
- * Create a Context with a given executionId. ExecutionId, together with
- * user name and conf, will determine the temporary directory locations.
- */
- public Context(Configuration conf, String executionId) {
- this.conf = conf;
- this.executionId = executionId;
-
- // local & non-local tmp location is configurable. however it is the same across
- // all external file systems
- nonLocalScratchPath = new Path(SessionState.getHDFSSessionPath(conf), executionId);
- localScratchDir = new Path(SessionState.getLocalSessionPath(conf), executionId).toUri().getPath();
- scratchDirPermission = HiveConf.getVar(conf, HiveConf.ConfVars.SCRATCHDIRPERMISSION);
- stagingDir = HiveConf.getVar(conf, HiveConf.ConfVars.STAGINGDIR);
- opContext = new CompilationOpContext();
- }
-
-
- public Map<LoadTableDesc, WriteEntity> getLoadTableOutputMap() {
- return loadTableOutputMap;
- }
-
- public Map<WriteEntity, List<HiveLockObj>> getOutputLockObjects() {
- return outputLockObjects;
- }
-
- /**
- * Set the context on whether the current query is an explain query.
- * @param value true if the query is an explain query, false if not
- */
- public void setExplain(boolean value) {
- explain = value;
- }
-
- /**
- * Find whether the current query is an explain query
- * @return true if the query is an explain query, false if not
- */
- public boolean getExplain() {
- return explain;
- }
-
- /**
- * Find whether the current query is a logical explain query
- */
- public boolean getExplainLogical() {
- return explainLogical;
- }
-
- /**
- * Set the context on whether the current query is a logical
- * explain query.
- */
- public void setExplainLogical(boolean explainLogical) {
- this.explainLogical = explainLogical;
- }
-
- /**
- * Set the original query command.
- * @param cmd the original query command string
- */
- public void setCmd(String cmd) {
- this.cmd = cmd;
- }
-
- /**
- * Find the original query command.
- * @return the original query command string
- */
- public String getCmd () {
- return cmd;
- }
-
- /**
- * Gets a temporary staging directory related to a path.
- * If a path already contains a staging directory, then returns the current directory; otherwise
- * create the directory if needed.
- *
- * @param inputPath URI of the temporary directory
- * @param mkdir Create the directory if True.
- * @return A temporary path.
- */
- private Path getStagingDir(Path inputPath, boolean mkdir) {
- final URI inputPathUri = inputPath.toUri();
- final String inputPathName = inputPathUri.getPath();
- final String fileSystem = inputPathUri.getScheme() + ":" + inputPathUri.getAuthority();
- final FileSystem fs;
-
- try {
- fs = inputPath.getFileSystem(conf);
- } catch (IOException e) {
- throw new IllegalStateException("Error getting FileSystem for " + inputPath + ": "+ e, e);
- }
-
- String stagingPathName;
- if (inputPathName.indexOf(stagingDir) == -1) {
- stagingPathName = new Path(inputPathName, stagingDir).toString();
- } else {
- stagingPathName = inputPathName.substring(0, inputPathName.indexOf(stagingDir) + stagingDir.length());
- }
-
- final String key = fileSystem + "-" + stagingPathName + "-" + TaskRunner.getTaskRunnerID();
-
- Path dir = fsScratchDirs.get(key);
- if (dir == null) {
- // Append task specific info to stagingPathName, instead of creating a sub-directory.
- // This way we don't have to worry about deleting the stagingPathName separately at
- // end of query execution.
- dir = fs.makeQualified(new Path(stagingPathName + "_" + this.executionId + "-" + TaskRunner.getTaskRunnerID()));
-
- LOG.debug("Created staging dir = " + dir + " for path = " + inputPath);
-
- if (mkdir) {
- try {
- boolean inheritPerms = HiveConf.getBoolVar(conf,
- HiveConf.ConfVars.HIVE_WAREHOUSE_SUBDIR_INHERIT_PERMS);
- if (!FileUtils.mkdir(fs, dir, inheritPerms, conf)) {
- throw new IllegalStateException("Cannot create staging directory '" + dir.toString() + "'");
- }
-
- if (isHDFSCleanup) {
- fs.deleteOnExit(dir);
- }
- } catch (IOException e) {
- throw new RuntimeException("Cannot create staging directory '" + dir.toString() + "': " + e.getMessage(), e);
- }
- }
-
- fsScratchDirs.put(key, dir);
- }
-
- return dir;
- }
-
- /**
- * Get a tmp directory on specified URI
- *
- * @param scheme Scheme of the target FS
- * @param authority Authority of the target FS
- * @param mkdir create the directory if true
- * @param scratchDir path of tmp directory
- */
- private Path getScratchDir(String scheme, String authority,
- boolean mkdir, String scratchDir) {
-
- String fileSystem = scheme + ":" + authority;
- Path dir = fsScratchDirs.get(fileSystem + "-" + TaskRunner.getTaskRunnerID());
-
- if (dir == null) {
- Path dirPath = new Path(scheme, authority,
- scratchDir + "-" + TaskRunner.getTaskRunnerID());
- if (mkdir) {
- try {
- FileSystem fs = dirPath.getFileSystem(conf);
- dirPath = new Path(fs.makeQualified(dirPath).toString());
- FsPermission fsPermission = new FsPermission(scratchDirPermission);
-
- if (!fs.mkdirs(dirPath, fsPermission)) {
- throw new RuntimeException("Cannot make directory: "
- + dirPath.toString());
- }
- if (isHDFSCleanup) {
- fs.deleteOnExit(dirPath);
- }
- } catch (IOException e) {
- throw new RuntimeException (e);
- }
- }
- dir = dirPath;
- fsScratchDirs.put(fileSystem + "-" + TaskRunner.getTaskRunnerID(), dir);
-
- }
-
- return dir;
- }
-
-
- /**
- * Create a local scratch directory on demand and return it.
- */
- public Path getLocalScratchDir(boolean mkdir) {
- try {
- FileSystem fs = FileSystem.getLocal(conf);
- URI uri = fs.getUri();
- return getScratchDir(uri.getScheme(), uri.getAuthority(),
- mkdir, localScratchDir);
- } catch (IOException e) {
- throw new RuntimeException (e);
- }
- }
-
-
- /**
- * Create a map-reduce scratch directory on demand and return it.
- *
- */
- public Path getMRScratchDir() {
-
- // if we are executing entirely on the client side - then
- // just (re)use the local scratch directory
- if(isLocalOnlyExecutionMode()) {
- return getLocalScratchDir(!explain);
- }
-
- try {
- Path dir = FileUtils.makeQualified(nonLocalScratchPath, conf);
- URI uri = dir.toUri();
-
- Path newScratchDir = getScratchDir(uri.getScheme(), uri.getAuthority(),
- !explain, uri.getPath());
- LOG.info("New scratch dir is " + newScratchDir);
- return newScratchDir;
- } catch (IOException e) {
- throw new RuntimeException(e);
- } catch (IllegalArgumentException e) {
- throw new RuntimeException("Error while making MR scratch "
- + "directory - check filesystem config (" + e.getCause() + ")", e);
- }
- }
-
- /**
- * Create a temporary directory depending of the path specified.
- * - If path is an Object store filesystem, then use the default MR scratch directory (HDFS)
- * - If path is on HDFS, then create a staging directory inside the path
- *
- * @param path Path used to verify the Filesystem to use for temporary directory
- * @return A path to the new temporary directory
- */
- public Path getTempDirForPath(Path path) {
- boolean isLocal = isPathLocal(path);
- if ((BlobStorageUtils.isBlobStoragePath(conf, path) && !BlobStorageUtils.isBlobStorageAsScratchDir(conf))
- || isLocal) {
- // For better write performance, we use HDFS for temporary data when object store is used.
- // Note that the scratch directory configuration variable must use HDFS or any other non-blobstorage system
- // to take advantage of this performance.
- return getMRTmpPath();
- } else {
- return getExtTmpPathRelTo(path);
- }
- }
-
- /*
- * Checks if the path is for the local filesystem or not
- */
- private boolean isPathLocal(Path path) {
- boolean isLocal = false;
- if (path != null) {
- String scheme = path.toUri().getScheme();
- if (scheme != null) {
- isLocal = scheme.equals(Utilities.HADOOP_LOCAL_FS_SCHEME);
- }
- }
- return isLocal;
- }
-
- private Path getExternalScratchDir(URI extURI) {
- return getStagingDir(new Path(extURI.getScheme(), extURI.getAuthority(), extURI.getPath()), !explain);
- }
-
- /**
- * Remove any created scratch directories.
- */
- public void removeScratchDir() {
- for (Map.Entry<String, Path> entry : fsScratchDirs.entrySet()) {
- try {
- Path p = entry.getValue();
- FileSystem fs = p.getFileSystem(conf);
- fs.delete(p, true);
- fs.cancelDeleteOnExit(p);
- } catch (Exception e) {
- LOG.warn("Error Removing Scratch: "
- + StringUtils.stringifyException(e));
- }
- }
- fsScratchDirs.clear();
- }
-
- /**
- * Remove any created directories for CTEs.
- */
- public void removeMaterializedCTEs() {
- // clean CTE tables
- for (Table materializedTable : cteTables.values()) {
- Path location = materializedTable.getDataLocation();
- try {
- FileSystem fs = location.getFileSystem(conf);
- boolean status = fs.delete(location, true);
- LOG.info("Removed " + location + " for materialized "
- + materializedTable.getTableName() + ", status=" + status);
- } catch (IOException e) {
- // ignore
- LOG.warn("Error removing " + location + " for materialized " + materializedTable.getTableName() +
- ": " + StringUtils.stringifyException(e));
- }
- }
- cteTables.clear();
- }
-
- private String nextPathId() {
- return Integer.toString(pathid++);
- }
-
-
- private static final String MR_PREFIX = "-mr-";
- private static final String EXT_PREFIX = "-ext-";
- private static final String LOCAL_PREFIX = "-local-";
-
- /**
- * Check if path is for intermediate data
- * @return true if a uri is a temporary uri for map-reduce intermediate data,
- * false otherwise
- */
- public boolean isMRTmpFileURI(String uriStr) {
- return (uriStr.indexOf(executionId) != -1) &&
- (uriStr.indexOf(MR_PREFIX) != -1);
- }
-
- public Path getMRTmpPath(URI uri) {
- return new Path(getStagingDir(new Path(uri), !explain), MR_PREFIX + nextPathId());
- }
-
- /**
- * Get a path to store map-reduce intermediate data in.
- *
- * @return next available path for map-red intermediate data
- */
- public Path getMRTmpPath() {
- return new Path(getMRScratchDir(), MR_PREFIX +
- nextPathId());
- }
-
- /**
- * Get a tmp path on local host to store intermediate data.
- *
- * @return next available tmp path on local fs
- */
- public Path getLocalTmpPath() {
- return new Path(getLocalScratchDir(true), LOCAL_PREFIX + nextPathId());
- }
-
- /**
- * Get a path to store tmp data destined for external Path.
- *
- * @param path external Path to which the tmp data has to be eventually moved
- * @return next available tmp path on the file system corresponding extURI
- */
- public Path getExternalTmpPath(Path path) {
- URI extURI = path.toUri();
- if (extURI.getScheme().equals("viewfs")) {
- // if we are on viewfs we don't want to use /tmp as tmp dir since rename from /tmp/..
- // to final /user/hive/warehouse/ will fail later, so instead pick tmp dir
- // on same namespace as tbl dir.
- return getExtTmpPathRelTo(path.getParent());
- }
- return new Path(getExternalScratchDir(extURI), EXT_PREFIX +
- nextPathId());
- }
-
- /**
- * This is similar to getExternalTmpPath() with difference being this method returns temp path
- * within passed in uri, whereas getExternalTmpPath() ignores passed in path and returns temp
- * path within /tmp
- */
- public Path getExtTmpPathRelTo(Path path) {
- return new Path(getStagingDir(path, !explain), EXT_PREFIX + nextPathId());
- }
-
- /**
- * @return the resFile
- */
- public Path getResFile() {
- return resFile;
- }
-
- /**
- * @param resFile
- * the resFile to set
- */
- public void setResFile(Path resFile) {
- this.resFile = resFile;
- resDir = null;
- resDirPaths = null;
- resDirFilesNum = 0;
- }
-
- /**
- * @return the resDir
- */
- public Path getResDir() {
- return resDir;
- }
-
- /**
- * @param resDir
- * the resDir to set
- */
- public void setResDir(Path resDir) {
- this.resDir = resDir;
- resFile = null;
-
- resDirFilesNum = 0;
- resDirPaths = null;
- }
-
- public void clear() throws IOException {
- if (resDir != null) {
- try {
- FileSystem fs = resDir.getFileSystem(conf);
- fs.delete(resDir, true);
- } catch (IOException e) {
- LOG.info("Context clear error: " + StringUtils.stringifyException(e));
- }
- }
-
- if (resFile != null) {
- try {
- FileSystem fs = resFile.getFileSystem(conf);
- fs.delete(resFile, false);
- } catch (IOException e) {
- LOG.info("Context clear error: " + StringUtils.stringifyException(e));
- }
- }
- removeMaterializedCTEs();
- removeScratchDir();
- originalTracker = null;
- setNeedLockMgr(false);
- }
-
- public DataInput getStream() {
- try {
- if (!initialized) {
- initialized = true;
- if ((resFile == null) && (resDir == null)) {
- return null;
- }
-
- if (resFile != null) {
- return resFile.getFileSystem(conf).open(resFile);
- }
-
- resFs = resDir.getFileSystem(conf);
- FileStatus status = resFs.getFileStatus(resDir);
- assert status.isDir();
- FileStatus[] resDirFS = resFs.globStatus(new Path(resDir + "/*"), FileUtils.HIDDEN_FILES_PATH_FILTER);
- resDirPaths = new Path[resDirFS.length];
- int pos = 0;
- for (FileStatus resFS : resDirFS) {
- if (!resFS.isDir()) {
- resDirPaths[pos++] = resFS.getPath();
- }
- }
- if (pos == 0) {
- return null;
- }
-
- return resFs.open(resDirPaths[resDirFilesNum++]);
- } else {
- return getNextStream();
- }
- } catch (FileNotFoundException e) {
- LOG.info("getStream error: " + StringUtils.stringifyException(e));
- return null;
- } catch (IOException e) {
- LOG.info("getStream error: " + StringUtils.stringifyException(e));
- return null;
- }
- }
-
- private DataInput getNextStream() {
- try {
- if (resDir != null && resDirFilesNum < resDirPaths.length
- && (resDirPaths[resDirFilesNum] != null)) {
- return resFs.open(resDirPaths[resDirFilesNum++]);
- }
- } catch (FileNotFoundException e) {
- LOG.info("getNextStream error: " + StringUtils.stringifyException(e));
- return null;
- } catch (IOException e) {
- LOG.info("getNextStream error: " + StringUtils.stringifyException(e));
- return null;
- }
-
- return null;
- }
-
- public void resetStream() {
- if (initialized) {
- resDirFilesNum = 0;
- initialized = false;
- }
- }
-
- /**
- * Little abbreviation for StringUtils.
- */
- private static boolean strEquals(String str1, String str2) {
- return org.apache.commons.lang.StringUtils.equals(str1, str2);
- }
-
- /**
- * Set the token rewrite stream being used to parse the current top-level SQL
- * statement. Note that this should <b>not</b> be used for other parsing
- * activities; for example, when we encounter a reference to a view, we switch
- * to a new stream for parsing the stored view definition from the catalog,
- * but we don't clobber the top-level stream in the context.
- *
- * @param tokenRewriteStream
- * the stream being used
- */
- public void setTokenRewriteStream(TokenRewriteStream tokenRewriteStream) {
- assert (this.tokenRewriteStream == null);
- this.tokenRewriteStream = tokenRewriteStream;
- }
-
- /**
- * @return the token rewrite stream being used to parse the current top-level
- * SQL statement, or null if it isn't available (e.g. for parser
- * tests)
- */
- public TokenRewriteStream getTokenRewriteStream() {
- return tokenRewriteStream;
- }
-
- /**
- * Generate a unique executionId. An executionId, together with user name and
- * the configuration, will determine the temporary locations of all intermediate
- * files.
- *
- * In the future, users can use the executionId to resume a query.
- */
- public static String generateExecutionId() {
- Random rand = new Random();
- SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd_HH-mm-ss_SSS");
- String executionId = "hive_" + format.format(new Date()) + "_"
- + Math.abs(rand.nextLong());
- return executionId;
- }
-
- /**
- * Does Hive wants to run tasks entirely on the local machine
- * (where the query is being compiled)?
- *
- * Today this translates into running hadoop jobs locally
- */
- public boolean isLocalOnlyExecutionMode() {
- // Always allow spark to run in a cluster mode. Without this, depending on
- // user's local hadoop settings, true may be returned, which causes plan to be
- // stored in local path.
- if (HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_EXECUTION_ENGINE).equals("spark")) {
- return false;
- }
-
- return ShimLoader.getHadoopShims().isLocalMode(conf);
- }
-
- public List<HiveLock> getHiveLocks() {
- return hiveLocks;
- }
-
- public void setHiveLocks(List<HiveLock> hiveLocks) {
- this.hiveLocks = hiveLocks;
- }
-
- public HiveTxnManager getHiveTxnManager() {
- return hiveTxnManager;
- }
-
- public void setHiveTxnManager(HiveTxnManager txnMgr) {
- hiveTxnManager = txnMgr;
- }
-
- public void setOriginalTracker(String originalTracker) {
- this.originalTracker = originalTracker;
- }
-
- public void restoreOriginalTracker() {
- if (originalTracker != null) {
- ShimLoader.getHadoopShims().setJobLauncherRpcAddress(conf, originalTracker);
- originalTracker = null;
- }
- }
-
- public void addCS(String path, ContentSummary cs) {
- pathToCS.put(path, cs);
- }
-
- public ContentSummary getCS(Path path) {
- return getCS(path.toString());
- }
-
- public ContentSummary getCS(String path) {
- return pathToCS.get(path);
- }
-
- public Map<String, ContentSummary> getPathToCS() {
- return pathToCS;
- }
-
- public Configuration getConf() {
- return conf;
- }
-
- /**
- * @return the isHDFSCleanup
- */
- public boolean isHDFSCleanup() {
- return isHDFSCleanup;
- }
-
- /**
- * @param isHDFSCleanup the isHDFSCleanup to set
- */
- public void setHDFSCleanup(boolean isHDFSCleanup) {
- this.isHDFSCleanup = isHDFSCleanup;
- }
-
- public boolean isNeedLockMgr() {
- return needLockMgr;
- }
-
- public void setNeedLockMgr(boolean needLockMgr) {
- this.needLockMgr = needLockMgr;
- }
-
- public int getTryCount() {
- return tryCount;
- }
-
- public void setTryCount(int tryCount) {
- this.tryCount = tryCount;
- }
-
- public void setAcidOperation(AcidUtils.Operation op) {
- acidOperation = op;
- }
-
- public AcidUtils.Operation getAcidOperation() {
- return acidOperation;
- }
-
- public String getCboInfo() {
- return cboInfo;
- }
-
- public void setCboInfo(String cboInfo) {
- this.cboInfo = cboInfo;
- }
-
- public boolean isCboSucceeded() {
- return cboSucceeded;
- }
-
- public void setCboSucceeded(boolean cboSucceeded) {
- this.cboSucceeded = cboSucceeded;
- }
-
- public Table getMaterializedTable(String cteName) {
- return cteTables.get(cteName);
- }
-
- public void addMaterializedTable(String cteName, Table table) {
- cteTables.put(cteName, table);
- }
-
- public AtomicInteger getSequencer() {
- return sequencer;
- }
-
- public CompilationOpContext getOpContext() {
- return opContext;
- }
-
- public Heartbeater getHeartbeater() {
- return heartbeater;
- }
-
- public void setHeartbeater(Heartbeater heartbeater) {
- this.heartbeater = heartbeater;
- }
-
- public void checkHeartbeaterLockException() throws LockException {
- if (getHeartbeater() != null && getHeartbeater().getLockException() != null) {
- throw getHeartbeater().getLockException();
- }
- }
-
- public boolean isSkipTableMasking() {
- return skipTableMasking;
- }
-
- public void setSkipTableMasking(boolean skipTableMasking) {
- this.skipTableMasking = skipTableMasking;
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/9f5bea3b/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java
index 1efbab5..0023649 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java
@@ -106,7 +106,7 @@ public final class ParseUtils {
*
* @return node at which descent stopped
*/
- private static ASTNode findRootNonNullToken(ASTNode tree) {
+ public static ASTNode findRootNonNullToken(ASTNode tree) {
while ((tree.getToken() == null) && (tree.getChildCount() > 0)) {
tree = (ASTNode) tree.getChild(0);
}
[06/50] [abbrv] hive git commit: HIVE-15994: Grouping function error
when grouping sets are not specified (Jesus Camacho Rodriguez,
reviewed by Ashutosh Chauhan)
Posted by om...@apache.org.
HIVE-15994: Grouping function error when grouping sets are not specified (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
Change-Id: I15e187973cb6e682579829b3a21d250ab4e276e0
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f9eb0bd5
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f9eb0bd5
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f9eb0bd5
Branch: refs/heads/branch-2.2
Commit: f9eb0bd522c3a79cd34da4c76d8b46147d56ab72
Parents: ceea54f
Author: Jesus Camacho Rodriguez <jc...@apache.org>
Authored: Mon Feb 27 15:35:23 2017 +0000
Committer: Owen O'Malley <om...@apache.org>
Committed: Tue Mar 28 15:27:49 2017 -0700
----------------------------------------------------------------------
.../hadoop/hive/ql/parse/CalcitePlanner.java | 19 +-
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 47 ++--
.../groupby_grouping_sets_grouping.q | 29 +++
.../clientpositive/groupby_grouping_window.q | 13 ++
.../groupby_grouping_sets_grouping.q.out | 231 +++++++++++++++++++
.../groupby_grouping_window.q.out | 48 ++++
6 files changed, 360 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/f9eb0bd5/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
index 6d3fe22..51a517a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
@@ -3286,10 +3286,11 @@ public class CalcitePlanner extends SemanticAnalyzer {
final boolean cubeRollupGrpSetPresent = (!qbp.getDestRollups().isEmpty()
|| !qbp.getDestGroupingSets().isEmpty() || !qbp.getDestCubes().isEmpty());
for (WindowExpressionSpec wExprSpec : windowExpressions) {
- if (cubeRollupGrpSetPresent) {
+ if (!qbp.getDestToGroupBy().isEmpty()) {
// Special handling of grouping function
wExprSpec.setExpression(rewriteGroupingFunctionAST(
- getGroupByForClause(qbp, selClauseName), wExprSpec.getExpression()));
+ getGroupByForClause(qbp, selClauseName), wExprSpec.getExpression(),
+ !cubeRollupGrpSetPresent));
}
if (out_rwsch.getExpression(wExprSpec.getExpression()) == null) {
Pair<RexNode, TypeInfo> wtp = genWindowingProj(qb, wExprSpec, srcRel);
@@ -3577,9 +3578,10 @@ public class CalcitePlanner extends SemanticAnalyzer {
TypeCheckCtx tcCtx = new TypeCheckCtx(inputRR);
// We allow stateful functions in the SELECT list (but nowhere else)
tcCtx.setAllowStatefulFunctions(true);
- if (cubeRollupGrpSetPresent) {
+ if (!qbp.getDestToGroupBy().isEmpty()) {
// Special handling of grouping function
- expr = rewriteGroupingFunctionAST(getGroupByForClause(qbp, selClauseName), expr);
+ expr = rewriteGroupingFunctionAST(getGroupByForClause(qbp, selClauseName), expr,
+ !cubeRollupGrpSetPresent);
}
ExprNodeDesc exp = genExprNodeDesc(expr, inputRR, tcCtx);
String recommended = recommendName(exp, colAlias);
@@ -3960,11 +3962,12 @@ public class CalcitePlanner extends SemanticAnalyzer {
}
ASTNode targetNode = (ASTNode) havingClause.getChild(0);
validateNoHavingReferenceToAlias(qb, targetNode);
- final boolean cubeRollupGrpSetPresent = (!qbp.getDestRollups().isEmpty()
- || !qbp.getDestGroupingSets().isEmpty() || !qbp.getDestCubes().isEmpty());
- if (cubeRollupGrpSetPresent) {
+ if (!qbp.getDestToGroupBy().isEmpty()) {
+ final boolean cubeRollupGrpSetPresent = (!qbp.getDestRollups().isEmpty()
+ || !qbp.getDestGroupingSets().isEmpty() || !qbp.getDestCubes().isEmpty());
// Special handling of grouping function
- targetNode = rewriteGroupingFunctionAST(getGroupByForClause(qbp, destClauseName), targetNode);
+ targetNode = rewriteGroupingFunctionAST(getGroupByForClause(qbp, destClauseName), targetNode,
+ !cubeRollupGrpSetPresent);
}
gbFilter = genFilterRelNode(qb, targetNode, srcRel, aliasToRel, null, null, true);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/f9eb0bd5/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index 98d57b3..252a916 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -3050,16 +3050,15 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
}
ASTNode condn = (ASTNode) havingExpr.getChild(0);
- if (!isCBOExecuted()) {
+ if (!isCBOExecuted() && !qb.getParseInfo().getDestToGroupBy().isEmpty()) {
// If CBO did not optimize the query, we might need to replace grouping function
final String destClauseName = qb.getParseInfo().getClauseNames().iterator().next();
final boolean cubeRollupGrpSetPresent = (!qb.getParseInfo().getDestRollups().isEmpty()
|| !qb.getParseInfo().getDestGroupingSets().isEmpty()
|| !qb.getParseInfo().getDestCubes().isEmpty());
- if (cubeRollupGrpSetPresent) {
- // Special handling of grouping function
- condn = rewriteGroupingFunctionAST(getGroupByForClause(qb.getParseInfo(), destClauseName), condn);
- }
+ // Special handling of grouping function
+ condn = rewriteGroupingFunctionAST(getGroupByForClause(qb.getParseInfo(), destClauseName), condn,
+ !cubeRollupGrpSetPresent);
}
/*
@@ -3072,7 +3071,8 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
return output;
}
- protected static ASTNode rewriteGroupingFunctionAST(final List<ASTNode> grpByAstExprs, ASTNode targetNode) throws SemanticException {
+ protected static ASTNode rewriteGroupingFunctionAST(final List<ASTNode> grpByAstExprs, ASTNode targetNode,
+ final boolean noneSet) throws SemanticException {
final MutableBoolean visited = new MutableBoolean(false);
final MutableBoolean found = new MutableBoolean(false);
@@ -3094,10 +3094,19 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
for (int i = 0; i < grpByAstExprs.size(); i++) {
ASTNode grpByExpr = grpByAstExprs.get(i);
if (grpByExpr.toStringTree().equals(c.toStringTree())) {
- ASTNode child1 = (ASTNode) ParseDriver.adaptor.create(
- HiveParser.TOK_TABLE_OR_COL, "TOK_TABLE_OR_COL");
- ParseDriver.adaptor.addChild(child1, ParseDriver.adaptor.create(
- HiveParser.Identifier, VirtualColumn.GROUPINGID.getName()));
+ ASTNode child1;
+ if (noneSet) {
+ // Query does not contain CUBE, ROLLUP, or GROUPING SETS, and thus,
+ // grouping should return 0
+ child1 = (ASTNode) ParseDriver.adaptor.create(HiveParser.Number,
+ String.valueOf(0));
+ } else {
+ // We refer to grouping_id column
+ child1 = (ASTNode) ParseDriver.adaptor.create(
+ HiveParser.TOK_TABLE_OR_COL, "TOK_TABLE_OR_COL");
+ ParseDriver.adaptor.addChild(child1, ParseDriver.adaptor.create(
+ HiveParser.Identifier, VirtualColumn.GROUPINGID.getName()));
+ }
ASTNode child2 = (ASTNode) ParseDriver.adaptor.create(HiveParser.Number,
String.valueOf(IntMath.mod(-i-1, grpByAstExprs.size())));
root.setChild(1, child1);
@@ -4305,10 +4314,11 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
// We allow stateful functions in the SELECT list (but nowhere else)
tcCtx.setAllowStatefulFunctions(true);
tcCtx.setAllowDistinctFunctions(false);
- if (!isCBOExecuted() && cubeRollupGrpSetPresent) {
+ if (!isCBOExecuted() && !qb.getParseInfo().getDestToGroupBy().isEmpty()) {
// If CBO did not optimize the query, we might need to replace grouping function
// Special handling of grouping function
- expr = rewriteGroupingFunctionAST(getGroupByForClause(qb.getParseInfo(), dest), expr);
+ expr = rewriteGroupingFunctionAST(getGroupByForClause(qb.getParseInfo(), dest), expr,
+ !cubeRollupGrpSetPresent);
}
ExprNodeDesc exp = genExprNodeDesc(expr, inputRR, tcCtx);
String recommended = recommendName(exp, colAlias);
@@ -13060,18 +13070,17 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
Operator genWindowingPlan(QB qb, WindowingSpec wSpec, Operator input) throws SemanticException {
wSpec.validateAndMakeEffective();
- if (!isCBOExecuted()) {
+ if (!isCBOExecuted() && !qb.getParseInfo().getDestToGroupBy().isEmpty()) {
// If CBO did not optimize the query, we might need to replace grouping function
final String selClauseName = qb.getParseInfo().getClauseNames().iterator().next();
final boolean cubeRollupGrpSetPresent = (!qb.getParseInfo().getDestRollups().isEmpty()
|| !qb.getParseInfo().getDestGroupingSets().isEmpty()
|| !qb.getParseInfo().getDestCubes().isEmpty());
- if (cubeRollupGrpSetPresent) {
- for (WindowExpressionSpec wExprSpec : wSpec.getWindowExpressions()) {
- // Special handling of grouping function
- wExprSpec.setExpression(rewriteGroupingFunctionAST(
- getGroupByForClause(qb.getParseInfo(), selClauseName), wExprSpec.getExpression()));
- }
+ for (WindowExpressionSpec wExprSpec : wSpec.getWindowExpressions()) {
+ // Special handling of grouping function
+ wExprSpec.setExpression(rewriteGroupingFunctionAST(
+ getGroupByForClause(qb.getParseInfo(), selClauseName), wExprSpec.getExpression(),
+ !cubeRollupGrpSetPresent));
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/f9eb0bd5/ql/src/test/queries/clientpositive/groupby_grouping_sets_grouping.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/groupby_grouping_sets_grouping.q b/ql/src/test/queries/clientpositive/groupby_grouping_sets_grouping.q
index 1b753e1..7856097 100644
--- a/ql/src/test/queries/clientpositive/groupby_grouping_sets_grouping.q
+++ b/ql/src/test/queries/clientpositive/groupby_grouping_sets_grouping.q
@@ -87,3 +87,32 @@ from T1
group by cube(key, value)
having grouping(key) = 1 OR grouping(value) = 1
order by x desc, case when x = 1 then key end;
+
+explain
+select key, value, grouping(key), grouping(value)
+from T1
+group by key, value;
+
+select key, value, grouping(key), grouping(value)
+from T1
+group by key, value;
+
+explain
+select key, value, grouping(value)
+from T1
+group by key, value;
+
+select key, value, grouping(value)
+from T1
+group by key, value;
+
+explain
+select key, value
+from T1
+group by key, value
+having grouping(key) = 0;
+
+select key, value
+from T1
+group by key, value
+having grouping(key) = 0;
http://git-wip-us.apache.org/repos/asf/hive/blob/f9eb0bd5/ql/src/test/queries/clientpositive/groupby_grouping_window.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/groupby_grouping_window.q b/ql/src/test/queries/clientpositive/groupby_grouping_window.q
index b456074..7ba73f9 100644
--- a/ql/src/test/queries/clientpositive/groupby_grouping_window.q
+++ b/ql/src/test/queries/clientpositive/groupby_grouping_window.q
@@ -13,3 +13,16 @@ FROM t
GROUP BY category
GROUPING SETS ((), (category))
HAVING max(comments) > 0;
+
+SELECT grouping(category), lead(live) over(partition by grouping(category))
+FROM t
+GROUP BY category, live
+GROUPING SETS ((), (category));
+
+SELECT grouping(category), lead(live) over(partition by grouping(category))
+FROM t
+GROUP BY category, live;
+
+SELECT grouping(category), lag(live) over(partition by grouping(category))
+FROM t
+GROUP BY category, live;
http://git-wip-us.apache.org/repos/asf/hive/blob/f9eb0bd5/ql/src/test/results/clientpositive/groupby_grouping_sets_grouping.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/groupby_grouping_sets_grouping.q.out b/ql/src/test/results/clientpositive/groupby_grouping_sets_grouping.q.out
index 62f40cd..6917dba 100644
--- a/ql/src/test/results/clientpositive/groupby_grouping_sets_grouping.q.out
+++ b/ql/src/test/results/clientpositive/groupby_grouping_sets_grouping.q.out
@@ -765,3 +765,234 @@ NULL 2 1
2 NULL 1
3 NULL 1
4 NULL 1
+PREHOOK: query: explain
+select key, value, grouping(key), grouping(value)
+from T1
+group by key, value
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select key, value, grouping(key), grouping(value)
+from T1
+group by key, value
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: t1
+ Statistics: Num rows: 3 Data size: 30 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: int), value (type: int)
+ outputColumnNames: key, value
+ Statistics: Num rows: 3 Data size: 30 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ keys: key (type: int), value (type: int)
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 3 Data size: 30 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int), _col1 (type: int)
+ sort order: ++
+ Map-reduce partition columns: _col0 (type: int), _col1 (type: int)
+ Statistics: Num rows: 3 Data size: 30 Basic stats: COMPLETE Column stats: NONE
+ Reduce Operator Tree:
+ Group By Operator
+ keys: KEY._col0 (type: int), KEY._col1 (type: int)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: int), _col1 (type: int), 0 (type: tinyint), 0 (type: tinyint)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select key, value, grouping(key), grouping(value)
+from T1
+group by key, value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t1
+#### A masked pattern was here ####
+POSTHOOK: query: select key, value, grouping(key), grouping(value)
+from T1
+group by key, value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t1
+#### A masked pattern was here ####
+1 NULL 0 0
+1 1 0 0
+2 2 0 0
+3 NULL 0 0
+3 3 0 0
+4 5 0 0
+PREHOOK: query: explain
+select key, value, grouping(value)
+from T1
+group by key, value
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select key, value, grouping(value)
+from T1
+group by key, value
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: t1
+ Statistics: Num rows: 3 Data size: 30 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: int), value (type: int)
+ outputColumnNames: key, value
+ Statistics: Num rows: 3 Data size: 30 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ keys: key (type: int), value (type: int)
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 3 Data size: 30 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int), _col1 (type: int)
+ sort order: ++
+ Map-reduce partition columns: _col0 (type: int), _col1 (type: int)
+ Statistics: Num rows: 3 Data size: 30 Basic stats: COMPLETE Column stats: NONE
+ Reduce Operator Tree:
+ Group By Operator
+ keys: KEY._col0 (type: int), KEY._col1 (type: int)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: int), _col1 (type: int), 0 (type: tinyint)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select key, value, grouping(value)
+from T1
+group by key, value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t1
+#### A masked pattern was here ####
+POSTHOOK: query: select key, value, grouping(value)
+from T1
+group by key, value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t1
+#### A masked pattern was here ####
+1 NULL 0
+1 1 0
+2 2 0
+3 NULL 0
+3 3 0
+4 5 0
+PREHOOK: query: explain
+select key, value
+from T1
+group by key, value
+having grouping(key) = 0
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select key, value
+from T1
+group by key, value
+having grouping(key) = 0
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: t1
+ Statistics: Num rows: 3 Data size: 30 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: int), value (type: int)
+ outputColumnNames: key, value
+ Statistics: Num rows: 3 Data size: 30 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ keys: key (type: int), value (type: int)
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 3 Data size: 30 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int), _col1 (type: int)
+ sort order: ++
+ Map-reduce partition columns: _col0 (type: int), _col1 (type: int)
+ Statistics: Num rows: 3 Data size: 30 Basic stats: COMPLETE Column stats: NONE
+ Reduce Operator Tree:
+ Group By Operator
+ keys: KEY._col0 (type: int), KEY._col1 (type: int)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select key, value
+from T1
+group by key, value
+having grouping(key) = 0
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t1
+#### A masked pattern was here ####
+POSTHOOK: query: select key, value
+from T1
+group by key, value
+having grouping(key) = 0
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t1
+#### A masked pattern was here ####
+1 NULL
+1 1
+2 2
+3 NULL
+3 3
+4 5
http://git-wip-us.apache.org/repos/asf/hive/blob/f9eb0bd5/ql/src/test/results/clientpositive/groupby_grouping_window.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/groupby_grouping_window.q.out b/ql/src/test/results/clientpositive/groupby_grouping_window.q.out
index 251f4f7..202fad0 100644
--- a/ql/src/test/results/clientpositive/groupby_grouping_window.q.out
+++ b/ql/src/test/results/clientpositive/groupby_grouping_window.q.out
@@ -151,3 +151,51 @@ NULL 0 2 1
86 0 2 1
238 0 2 1
311 0 2 1
+PREHOOK: query: SELECT grouping(category), lead(live) over(partition by grouping(category))
+FROM t
+GROUP BY category, live
+GROUPING SETS ((), (category))
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT grouping(category), lead(live) over(partition by grouping(category))
+FROM t
+GROUP BY category, live
+GROUPING SETS ((), (category))
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t
+#### A masked pattern was here ####
+0 NULL
+0 NULL
+0 NULL
+0 NULL
+PREHOOK: query: SELECT grouping(category), lead(live) over(partition by grouping(category))
+FROM t
+GROUP BY category, live
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT grouping(category), lead(live) over(partition by grouping(category))
+FROM t
+GROUP BY category, live
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t
+#### A masked pattern was here ####
+0 0
+0 0
+0 NULL
+PREHOOK: query: SELECT grouping(category), lag(live) over(partition by grouping(category))
+FROM t
+GROUP BY category, live
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT grouping(category), lag(live) over(partition by grouping(category))
+FROM t
+GROUP BY category, live
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t
+#### A masked pattern was here ####
+0 NULL
+0 0
+0 0
[15/50] [abbrv] hive git commit: HIVE-14367 : Estimated size for
constant nulls is 0 (Ashutosh Chauhan via Jesus Camacho Rodriguez)
Posted by om...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/annotate_stats_limit.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/annotate_stats_limit.q.out b/ql/src/test/results/clientpositive/annotate_stats_limit.q.out
index c910d41..5bba194 100644
--- a/ql/src/test/results/clientpositive/annotate_stats_limit.q.out
+++ b/ql/src/test/results/clientpositive/annotate_stats_limit.q.out
@@ -74,11 +74,11 @@ STAGE PLANS:
Processor Tree:
TableScan
alias: loc_orc
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: int)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
ListSink
PREHOOK: query: explain select * from loc_orc limit 4
@@ -95,14 +95,14 @@ STAGE PLANS:
Processor Tree:
TableScan
alias: loc_orc
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: int)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
Limit
Number of rows: 4
- Statistics: Num rows: 4 Data size: 396 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 4 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE
ListSink
PREHOOK: query: explain select * from loc_orc limit 16
@@ -119,14 +119,14 @@ STAGE PLANS:
Processor Tree:
TableScan
alias: loc_orc
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: int)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
Limit
Number of rows: 16
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
ListSink
PREHOOK: query: explain select * from loc_orc limit 0
@@ -143,11 +143,11 @@ STAGE PLANS:
Processor Tree:
TableScan
alias: loc_orc
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: int)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
Limit
Number of rows: 0
Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/annotate_stats_select.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/annotate_stats_select.q.out b/ql/src/test/results/clientpositive/annotate_stats_select.q.out
index 8bbb94a..2f70bc3 100644
--- a/ql/src/test/results/clientpositive/annotate_stats_select.q.out
+++ b/ql/src/test/results/clientpositive/annotate_stats_select.q.out
@@ -262,7 +262,7 @@ STAGE PLANS:
Select Operator
expressions: null (type: void)
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
ListSink
PREHOOK: query: explain select 11 from alltypes_orc
@@ -472,7 +472,7 @@ STAGE PLANS:
Select Operator
expressions: null (type: date)
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE
ListSink
PREHOOK: query: explain select cast("58.174" as DECIMAL) from alltypes_orc
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/annotate_stats_union.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/annotate_stats_union.q.out b/ql/src/test/results/clientpositive/annotate_stats_union.q.out
index 5346d1b..059f261 100644
--- a/ql/src/test/results/clientpositive/annotate_stats_union.q.out
+++ b/ql/src/test/results/clientpositive/annotate_stats_union.q.out
@@ -146,11 +146,11 @@ STAGE PLANS:
Processor Tree:
TableScan
alias: loc_orc
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: int)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
ListSink
PREHOOK: query: explain select * from (select * from loc_orc union all select * from loc_orc) tmp
@@ -167,32 +167,32 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: loc_orc
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: int)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
Union
- Statistics: Num rows: 16 Data size: 1608 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 16 Data size: 1632 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 16 Data size: 1608 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 16 Data size: 1632 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
TableScan
alias: loc_orc
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: int)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
Union
- Statistics: Num rows: 16 Data size: 1608 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 16 Data size: 1632 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 16 Data size: 1608 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 16 Data size: 1632 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/autoColumnStats_4.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/autoColumnStats_4.q.out b/ql/src/test/results/clientpositive/autoColumnStats_4.q.out
index bf4e0bb..c7b9b4f 100644
--- a/ql/src/test/results/clientpositive/autoColumnStats_4.q.out
+++ b/ql/src/test/results/clientpositive/autoColumnStats_4.q.out
@@ -118,7 +118,7 @@ STAGE PLANS:
aggregations: compute_stats(a, 16), compute_stats(b, 16)
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 968 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
table:
@@ -152,17 +152,17 @@ STAGE PLANS:
TableScan
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 968 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,bitvector:string,numbitvectors:int>), _col1 (type: struct<columntype:string,maxlength:bigint,sumlength:bigint,count:bigint,countnulls:bigint,bitvector:string,numbitvectors:int>)
Reduce Operator Tree:
Group By Operator
aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 972 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 972 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/autoColumnStats_7.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/autoColumnStats_7.q.out b/ql/src/test/results/clientpositive/autoColumnStats_7.q.out
index df43c8d..82768d1 100644
--- a/ql/src/test/results/clientpositive/autoColumnStats_7.q.out
+++ b/ql/src/test/results/clientpositive/autoColumnStats_7.q.out
@@ -135,7 +135,7 @@ STAGE PLANS:
aggregations: compute_stats(VALUE._col0, 16), compute_stats(VALUE._col2, 16), compute_stats(VALUE._col3, 16)
mode: partial1
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 1460 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
table:
@@ -149,17 +149,17 @@ STAGE PLANS:
TableScan
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 1460 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: struct<columntype:string,maxlength:bigint,sumlength:bigint,count:bigint,countnulls:bigint,bitvector:string,numbitvectors:int>), _col1 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,bitvector:string,numbitvectors:int>), _col2 (type: struct<columntype:string,maxlength:bigint,sumlength:bigint,count:bigint,countnulls:bigint,bitvector:string,numbitvectors:int>)
Reduce Operator Tree:
Group By Operator
aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2)
mode: final
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 1464 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 1464 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/autoColumnStats_9.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/autoColumnStats_9.q.out b/ql/src/test/results/clientpositive/autoColumnStats_9.q.out
index 3c8165f..a90d180 100644
--- a/ql/src/test/results/clientpositive/autoColumnStats_9.q.out
+++ b/ql/src/test/results/clientpositive/autoColumnStats_9.q.out
@@ -89,7 +89,7 @@ STAGE PLANS:
aggregations: compute_stats(key, 16), compute_stats(value, 16)
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 968 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
table:
@@ -145,7 +145,7 @@ STAGE PLANS:
aggregations: compute_stats(key, 16), compute_stats(value, 16)
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 968 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
table:
@@ -181,17 +181,17 @@ STAGE PLANS:
TableScan
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 968 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,bitvector:string,numbitvectors:int>), _col1 (type: struct<columntype:string,maxlength:bigint,sumlength:bigint,count:bigint,countnulls:bigint,bitvector:string,numbitvectors:int>)
Reduce Operator Tree:
Group By Operator
aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 972 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 972 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/cbo_rp_annotate_stats_groupby.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/cbo_rp_annotate_stats_groupby.q.out b/ql/src/test/results/clientpositive/cbo_rp_annotate_stats_groupby.q.out
index e1d2b1c..f260f03 100644
--- a/ql/src/test/results/clientpositive/cbo_rp_annotate_stats_groupby.q.out
+++ b/ql/src/test/results/clientpositive/cbo_rp_annotate_stats_groupby.q.out
@@ -196,21 +196,21 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: loc_orc
- Statistics: Num rows: 8 Data size: 28 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: year (type: int)
outputColumnNames: year
- Statistics: Num rows: 8 Data size: 28 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
keys: year (type: int)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 8 Data size: 28 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 8 Data size: 28 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Operator Tree:
Group By Operator
keys: KEY._col0 (type: int)
@@ -665,11 +665,11 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: loc_orc
- Statistics: Num rows: 8 Data size: 28 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: year (type: int)
outputColumnNames: year
- Statistics: Num rows: 8 Data size: 28 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
keys: year (type: int)
mode: hash
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/cbo_rp_auto_join0.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/cbo_rp_auto_join0.q.out b/ql/src/test/results/clientpositive/cbo_rp_auto_join0.q.out
index adcd19d..942e447 100644
--- a/ql/src/test/results/clientpositive/cbo_rp_auto_join0.q.out
+++ b/ql/src/test/results/clientpositive/cbo_rp_auto_join0.q.out
@@ -38,14 +38,14 @@ STAGE PLANS:
a:cbo_t1:cbo_t3
TableScan
alias: cbo_t3
- Statistics: Num rows: 20 Data size: 3060 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 20 Data size: 3230 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: (key < 10) (type: boolean)
- Statistics: Num rows: 6 Data size: 850 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 6 Data size: 1020 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: key (type: string), value (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 6 Data size: 850 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 6 Data size: 1020 Basic stats: COMPLETE Column stats: COMPLETE
HashTable Sink Operator
keys:
0
@@ -56,14 +56,14 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: cbo_t3
- Statistics: Num rows: 20 Data size: 3060 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 20 Data size: 3230 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: (key < 10) (type: boolean)
- Statistics: Num rows: 6 Data size: 850 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 6 Data size: 1020 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: key (type: string), value (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 6 Data size: 850 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 6 Data size: 1020 Basic stats: COMPLETE Column stats: COMPLETE
Map Join Operator
condition map:
Inner Join 0 to 1
@@ -171,14 +171,14 @@ STAGE PLANS:
a:cbo_t1:cbo_t3
TableScan
alias: cbo_t3
- Statistics: Num rows: 20 Data size: 3060 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 20 Data size: 3230 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: (key < 10) (type: boolean)
- Statistics: Num rows: 6 Data size: 850 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 6 Data size: 1020 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: key (type: string), value (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 6 Data size: 850 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 6 Data size: 1020 Basic stats: COMPLETE Column stats: COMPLETE
HashTable Sink Operator
keys:
0
@@ -189,14 +189,14 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: cbo_t3
- Statistics: Num rows: 20 Data size: 3060 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 20 Data size: 3230 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: (key < 10) (type: boolean)
- Statistics: Num rows: 6 Data size: 850 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 6 Data size: 1020 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: key (type: string), value (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 6 Data size: 850 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 6 Data size: 1020 Basic stats: COMPLETE Column stats: COMPLETE
Map Join Operator
condition map:
Inner Join 0 to 1
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/cbo_rp_groupby3_noskew_multi_distinct.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/cbo_rp_groupby3_noskew_multi_distinct.q.out b/ql/src/test/results/clientpositive/cbo_rp_groupby3_noskew_multi_distinct.q.out
index 95233b0..c09764c 100644
--- a/ql/src/test/results/clientpositive/cbo_rp_groupby3_noskew_multi_distinct.q.out
+++ b/ql/src/test/results/clientpositive/cbo_rp_groupby3_noskew_multi_distinct.q.out
@@ -61,14 +61,14 @@ STAGE PLANS:
aggregations: sum(KEY._col0:0._col0), avg(KEY._col0:0._col0), avg(DISTINCT KEY._col0:0._col0), max(KEY._col0:0._col0), min(KEY._col0:0._col0), std(KEY._col0:0._col0), stddev_samp(KEY._col0:0._col0), variance(KEY._col0:0._col0), var_samp(KEY._col0:0._col0), sum(DISTINCT KEY._col0:1._col0), count(DISTINCT KEY._col0:2._col0)
mode: complete
outputColumnNames: $f0, $f1, $f2, $f3, $f4, $f5, $f6, $f7, $f8, $f9, $f10
- Statistics: Num rows: 1 Data size: 240 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 440 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: $f0 (type: double), $f1 (type: double), $f2 (type: double), UDFToDouble($f3) (type: double), UDFToDouble($f4) (type: double), $f5 (type: double), $f6 (type: double), $f7 (type: double), $f8 (type: double), $f9 (type: double), UDFToDouble($f10) (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10
- Statistics: Num rows: 1 Data size: 240 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 440 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 240 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 440 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/cbo_rp_join0.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/cbo_rp_join0.q.out b/ql/src/test/results/clientpositive/cbo_rp_join0.q.out
index be7c505..785cf86 100644
--- a/ql/src/test/results/clientpositive/cbo_rp_join0.q.out
+++ b/ql/src/test/results/clientpositive/cbo_rp_join0.q.out
@@ -16,48 +16,48 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: cbo_t1
- Statistics: Num rows: 20 Data size: 1602 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 20 Data size: 1691 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: key is not null (type: boolean)
- Statistics: Num rows: 18 Data size: 1424 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 18 Data size: 1513 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: key (type: string), c_int (type: int)
outputColumnNames: key, c_int
- Statistics: Num rows: 18 Data size: 1424 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 18 Data size: 1513 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: key (type: string)
sort order: +
Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 18 Data size: 1424 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 18 Data size: 1513 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: c_int (type: int)
TableScan
alias: cbo_t2:cbo_t2
- Statistics: Num rows: 20 Data size: 1602 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 20 Data size: 1691 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: key is not null (type: boolean)
- Statistics: Num rows: 18 Data size: 1424 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 18 Data size: 1513 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: key (type: string), c_int (type: int)
outputColumnNames: key, c_int
- Statistics: Num rows: 18 Data size: 1424 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 18 Data size: 1513 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: key (type: string)
sort order: +
Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 18 Data size: 1424 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 18 Data size: 1513 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: c_int (type: int)
TableScan
alias: cbo_t3:cbo_t3
- Statistics: Num rows: 20 Data size: 1530 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 20 Data size: 1615 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: key (type: string)
outputColumnNames: key
- Statistics: Num rows: 20 Data size: 1530 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 20 Data size: 1615 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: key (type: string)
sort order: +
Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 20 Data size: 1530 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 20 Data size: 1615 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Operator Tree:
Join Operator
condition map:
@@ -663,60 +663,60 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: cbo_t1
- Statistics: Num rows: 20 Data size: 1602 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 20 Data size: 1691 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: key is not null (type: boolean)
- Statistics: Num rows: 18 Data size: 1424 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 18 Data size: 1513 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: key (type: string), c_int (type: int)
outputColumnNames: key, c_int
- Statistics: Num rows: 18 Data size: 1424 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 18 Data size: 1513 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: key (type: string)
sort order: +
Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 18 Data size: 1424 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 18 Data size: 1513 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: c_int (type: int)
TableScan
alias: cbo_t2:cbo_t2
- Statistics: Num rows: 20 Data size: 1602 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 20 Data size: 1691 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: key is not null (type: boolean)
- Statistics: Num rows: 18 Data size: 1424 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 18 Data size: 1513 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: key (type: string), c_int (type: int)
outputColumnNames: key, c_int
- Statistics: Num rows: 18 Data size: 1424 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 18 Data size: 1513 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: key (type: string)
sort order: +
Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 18 Data size: 1424 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 18 Data size: 1513 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: c_int (type: int)
TableScan
alias: cbo_t3:cbo_t3
- Statistics: Num rows: 20 Data size: 1530 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 20 Data size: 1615 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: key (type: string)
outputColumnNames: key
- Statistics: Num rows: 20 Data size: 1530 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 20 Data size: 1615 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: key (type: string)
sort order: +
Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 20 Data size: 1530 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 20 Data size: 1615 Basic stats: COMPLETE Column stats: COMPLETE
TableScan
alias: cbo_t4:cbo_t1
- Statistics: Num rows: 20 Data size: 1602 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 20 Data size: 1691 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: key (type: string), c_int (type: int)
outputColumnNames: key, c_int
- Statistics: Num rows: 20 Data size: 1602 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 20 Data size: 1691 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: key (type: string)
sort order: +
Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 20 Data size: 1602 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 20 Data size: 1691 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: c_int (type: int)
Reduce Operator Tree:
Join Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/cbo_rp_udaf_percentile_approx_23.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/cbo_rp_udaf_percentile_approx_23.q.out b/ql/src/test/results/clientpositive/cbo_rp_udaf_percentile_approx_23.q.out
index 117ac90..12d9c10 100644
--- a/ql/src/test/results/clientpositive/cbo_rp_udaf_percentile_approx_23.q.out
+++ b/ql/src/test/results/clientpositive/cbo_rp_udaf_percentile_approx_23.q.out
@@ -519,20 +519,20 @@ STAGE PLANS:
aggregations: percentile_approx($f0, 0.5)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 160 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 160 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: array<double>)
Reduce Operator Tree:
Group By Operator
aggregations: percentile_approx(VALUE._col0)
mode: mergepartial
outputColumnNames: $f0
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 168 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 168 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -578,20 +578,20 @@ STAGE PLANS:
aggregations: percentile_approx($f0, 0.5)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 160 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 160 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: array<double>)
Reduce Operator Tree:
Group By Operator
aggregations: percentile_approx(VALUE._col0)
mode: mergepartial
outputColumnNames: $f0
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 168 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 168 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/constantfolding.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/constantfolding.q.out b/ql/src/test/results/clientpositive/constantfolding.q.out
index eba1439..76eb552 100644
--- a/ql/src/test/results/clientpositive/constantfolding.q.out
+++ b/ql/src/test/results/clientpositive/constantfolding.q.out
@@ -215,7 +215,7 @@ STAGE PLANS:
Select Operator
expressions: null (type: date)
outputColumnNames: _col0
- Statistics: Num rows: 500 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 500 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE
ListSink
PREHOOK: query: CREATE TABLE dest1(c1 STRING) STORED AS TEXTFILE
@@ -269,7 +269,7 @@ STAGE PLANS:
Select Operator
expressions: 1.098612288668 (type: double), null (type: double), null (type: double), 1.098612288668 (type: double), null (type: double), null (type: double), 1.584962500721 (type: double), null (type: double), null (type: double), 0.47712125472 (type: double), null (type: double), null (type: double), 1.584962500721 (type: double), null (type: double), null (type: double), null (type: double), -1.0 (type: double), 7.389056098931 (type: double), 8.0 (type: double), 8.0 (type: double), 0.125 (type: double), 8.0 (type: double), 2.0 (type: double), NaN (type: double), 1.0 (type: double), 1.0 (type: double), 8.0 (type: double), 8.0 (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26, _col27
- Statistics: Num rows: 1 Data size: 136 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 224 Basic stats: COMPLETE Column stats: COMPLETE
ListSink
PREHOOK: query: SELECT ROUND(LN(3.0),12), LN(0.0), LN(-1), ROUND(LOG(3.0),12), LOG(0.0),
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/create_genericudaf.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/create_genericudaf.q.out b/ql/src/test/results/clientpositive/create_genericudaf.q.out
index 618fe0e..dd2ce7f 100644
--- a/ql/src/test/results/clientpositive/create_genericudaf.q.out
+++ b/ql/src/test/results/clientpositive/create_genericudaf.q.out
@@ -47,20 +47,20 @@ STAGE PLANS:
aggregations: test_avg(1), test_avg(_col1)
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 332 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 332 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: struct<count:bigint,sum:double,input:int>), _col1 (type: struct<count:bigint,sum:double,input:string>)
Reduce Operator Tree:
Group By Operator
aggregations: test_avg(VALUE._col0), test_avg(VALUE._col1)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 332 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 332 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/decimal_precision.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/decimal_precision.q.out b/ql/src/test/results/clientpositive/decimal_precision.q.out
index d156b62..b7a6881 100644
--- a/ql/src/test/results/clientpositive/decimal_precision.q.out
+++ b/ql/src/test/results/clientpositive/decimal_precision.q.out
@@ -545,20 +545,20 @@ STAGE PLANS:
aggregations: avg(dec), sum(dec)
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 400 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 400 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: struct<count:bigint,sum:decimal(30,10),input:decimal(20,10)>), _col1 (type: decimal(30,10))
Reduce Operator Tree:
Group By Operator
aggregations: avg(VALUE._col0), sum(VALUE._col1)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 224 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 400 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 224 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 400 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/decimal_stats.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/decimal_stats.q.out b/ql/src/test/results/clientpositive/decimal_stats.q.out
index 0d47ace..cb02f76 100644
--- a/ql/src/test/results/clientpositive/decimal_stats.q.out
+++ b/ql/src/test/results/clientpositive/decimal_stats.q.out
@@ -63,28 +63,28 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: decimal_1
- Statistics: Num rows: 500 Data size: 112000 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 500 Data size: 112112 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: t (type: decimal(4,2)), u (type: decimal(5,0)), v (type: decimal(10,0))
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 500 Data size: 112000 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 500 Data size: 112112 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: decimal(4,2))
sort order: +
- Statistics: Num rows: 500 Data size: 112000 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 500 Data size: 112112 Basic stats: COMPLETE Column stats: COMPLETE
TopN Hash Memory Usage: 0.1
value expressions: _col1 (type: decimal(5,0)), _col2 (type: decimal(10,0))
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: decimal(4,2)), VALUE._col0 (type: decimal(5,0)), VALUE._col1 (type: decimal(10,0))
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 500 Data size: 112000 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 500 Data size: 112112 Basic stats: COMPLETE Column stats: COMPLETE
Limit
Number of rows: 100
- Statistics: Num rows: 100 Data size: 22400 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 100 Data size: 22512 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 100 Data size: 22400 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 100 Data size: 22512 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/decimal_udf.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/decimal_udf.q.out b/ql/src/test/results/clientpositive/decimal_udf.q.out
index a3feed3..6e72f77 100644
--- a/ql/src/test/results/clientpositive/decimal_udf.q.out
+++ b/ql/src/test/results/clientpositive/decimal_udf.q.out
@@ -2010,20 +2010,20 @@ STAGE PLANS:
aggregations: histogram_numeric(_col0, 3)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 160 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 160 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: array<double>)
Reduce Operator Tree:
Group By Operator
aggregations: histogram_numeric(VALUE._col0)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 720 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 720 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/fetch_aggregation.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/fetch_aggregation.q.out b/ql/src/test/results/clientpositive/fetch_aggregation.q.out
index 91f47f8..f20320f 100644
--- a/ql/src/test/results/clientpositive/fetch_aggregation.q.out
+++ b/ql/src/test/results/clientpositive/fetch_aggregation.q.out
@@ -23,7 +23,7 @@ STAGE PLANS:
aggregations: count(key), sum(key), avg(key), min(key), max(key), std(key), variance(key)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
- Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 800 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
table:
@@ -39,7 +39,7 @@ STAGE PLANS:
aggregations: count(_col0), sum(_col1), avg(_col2), min(_col3), max(_col4), std(_col5), variance(_col6)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
- Statistics: Num rows: 1 Data size: 208 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 800 Basic stats: COMPLETE Column stats: NONE
ListSink
PREHOOK: query: select count(key),sum(key),avg(key),min(key),max(key),std(key),variance(key) from src
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/fold_case.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/fold_case.q.out b/ql/src/test/results/clientpositive/fold_case.q.out
index dc4051c..1cf00aa 100644
--- a/ql/src/test/results/clientpositive/fold_case.q.out
+++ b/ql/src/test/results/clientpositive/fold_case.q.out
@@ -415,10 +415,10 @@ STAGE PLANS:
Select Operator
expressions: null (type: void)
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -503,7 +503,7 @@ STAGE PLANS:
Select Operator
expressions: null (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 500 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 500 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE
ListSink
PREHOOK: query: explain
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/groupby3.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/groupby3.q.out b/ql/src/test/results/clientpositive/groupby3.q.out
index 4247d28..2ebeae4 100644
--- a/ql/src/test/results/clientpositive/groupby3.q.out
+++ b/ql/src/test/results/clientpositive/groupby3.q.out
@@ -59,7 +59,7 @@ STAGE PLANS:
aggregations: sum(KEY._col0:0._col0), avg(KEY._col0:0._col0), avg(DISTINCT KEY._col0:0._col0), max(KEY._col0:0._col0), min(KEY._col0:0._col0), std(KEY._col0:0._col0), stddev_samp(KEY._col0:0._col0), variance(KEY._col0:0._col0), var_samp(KEY._col0:0._col0)
mode: partial1
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
- Statistics: Num rows: 1 Data size: 176 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 1208 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
table:
@@ -73,21 +73,21 @@ STAGE PLANS:
TableScan
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 176 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 1208 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: double), _col1 (type: struct<count:bigint,sum:double,input:string>), _col2 (type: struct<count:bigint,sum:double,input:string>), _col3 (type: string), _col4 (type: string), _col5 (type: struct<count:bigint,sum:double,variance:double>), _col6 (type: struct<count:bigint,sum:double,variance:double>), _col7 (type: struct<count:bigint,sum:double,variance:double>), _col8 (type: struct<count:bigint,sum:double,variance:double>)
Reduce Operator Tree:
Group By Operator
aggregations: sum(VALUE._col0), avg(VALUE._col1), avg(VALUE._col2), max(VALUE._col3), min(VALUE._col4), std(VALUE._col5), stddev_samp(VALUE._col6), variance(VALUE._col7), var_samp(VALUE._col8)
mode: final
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
- Statistics: Num rows: 1 Data size: 224 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 1208 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: double), _col1 (type: double), _col2 (type: double), UDFToDouble(_col3) (type: double), UDFToDouble(_col4) (type: double), _col5 (type: double), _col6 (type: double), _col7 (type: double), _col8 (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
- Statistics: Num rows: 1 Data size: 224 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 1208 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 224 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 1208 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/groupby3_map.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/groupby3_map.q.out b/ql/src/test/results/clientpositive/groupby3_map.q.out
index 405daa9..07c122e 100644
--- a/ql/src/test/results/clientpositive/groupby3_map.q.out
+++ b/ql/src/test/results/clientpositive/groupby3_map.q.out
@@ -64,14 +64,14 @@ STAGE PLANS:
aggregations: sum(VALUE._col0), avg(VALUE._col1), avg(DISTINCT KEY._col0:0._col0), max(VALUE._col3), min(VALUE._col4), std(VALUE._col5), stddev_samp(VALUE._col6), variance(VALUE._col7), var_samp(VALUE._col8)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
- Statistics: Num rows: 1 Data size: 224 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 1216 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: double), _col1 (type: double), _col2 (type: double), UDFToDouble(_col3) (type: double), UDFToDouble(_col4) (type: double), _col5 (type: double), _col6 (type: double), _col7 (type: double), _col8 (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
- Statistics: Num rows: 1 Data size: 224 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 1216 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 224 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 1216 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/groupby3_map_multi_distinct.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/groupby3_map_multi_distinct.q.out b/ql/src/test/results/clientpositive/groupby3_map_multi_distinct.q.out
index a1d24e8..a4501f7 100644
--- a/ql/src/test/results/clientpositive/groupby3_map_multi_distinct.q.out
+++ b/ql/src/test/results/clientpositive/groupby3_map_multi_distinct.q.out
@@ -68,14 +68,14 @@ STAGE PLANS:
aggregations: sum(VALUE._col0), avg(VALUE._col1), avg(DISTINCT KEY._col0:0._col0), max(VALUE._col3), min(VALUE._col4), std(VALUE._col5), stddev_samp(VALUE._col6), variance(VALUE._col7), var_samp(VALUE._col8), sum(DISTINCT KEY._col0:1._col0), count(DISTINCT KEY._col0:2._col0)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10
- Statistics: Num rows: 1 Data size: 248 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 1232 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: double), _col1 (type: double), _col2 (type: double), UDFToDouble(_col3) (type: double), UDFToDouble(_col4) (type: double), _col5 (type: double), _col6 (type: double), _col7 (type: double), _col8 (type: double), _col9 (type: double), UDFToDouble(_col10) (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10
- Statistics: Num rows: 1 Data size: 248 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 1232 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 248 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 1232 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/groupby3_map_skew.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/groupby3_map_skew.q.out b/ql/src/test/results/clientpositive/groupby3_map_skew.q.out
index 5679770..e02bdeb 100644
--- a/ql/src/test/results/clientpositive/groupby3_map_skew.q.out
+++ b/ql/src/test/results/clientpositive/groupby3_map_skew.q.out
@@ -66,7 +66,7 @@ STAGE PLANS:
aggregations: sum(VALUE._col0), avg(VALUE._col1), avg(DISTINCT KEY._col0:0._col0), max(VALUE._col3), min(VALUE._col4), std(VALUE._col5), stddev_samp(VALUE._col6), variance(VALUE._col7), var_samp(VALUE._col8)
mode: partials
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
- Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 1464 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
table:
@@ -80,21 +80,21 @@ STAGE PLANS:
TableScan
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 1464 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: double), _col1 (type: struct<count:bigint,sum:double,input:string>), _col2 (type: struct<count:bigint,sum:double,input:string>), _col3 (type: string), _col4 (type: string), _col5 (type: struct<count:bigint,sum:double,variance:double>), _col6 (type: struct<count:bigint,sum:double,variance:double>), _col7 (type: struct<count:bigint,sum:double,variance:double>), _col8 (type: struct<count:bigint,sum:double,variance:double>)
Reduce Operator Tree:
Group By Operator
aggregations: sum(VALUE._col0), avg(VALUE._col1), avg(VALUE._col2), max(VALUE._col3), min(VALUE._col4), std(VALUE._col5), stddev_samp(VALUE._col6), variance(VALUE._col7), var_samp(VALUE._col8)
mode: final
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
- Statistics: Num rows: 1 Data size: 224 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 1464 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: double), _col1 (type: double), _col2 (type: double), UDFToDouble(_col3) (type: double), UDFToDouble(_col4) (type: double), _col5 (type: double), _col6 (type: double), _col7 (type: double), _col8 (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
- Statistics: Num rows: 1 Data size: 224 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 1464 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 224 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 1464 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/groupby3_noskew.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/groupby3_noskew.q.out b/ql/src/test/results/clientpositive/groupby3_noskew.q.out
index e6c13cf..624fd2d 100644
--- a/ql/src/test/results/clientpositive/groupby3_noskew.q.out
+++ b/ql/src/test/results/clientpositive/groupby3_noskew.q.out
@@ -57,14 +57,14 @@ STAGE PLANS:
aggregations: sum(KEY._col0:0._col0), avg(KEY._col0:0._col0), avg(DISTINCT KEY._col0:0._col0), max(KEY._col0:0._col0), min(KEY._col0:0._col0), std(KEY._col0:0._col0), stddev_samp(KEY._col0:0._col0), variance(KEY._col0:0._col0), var_samp(KEY._col0:0._col0)
mode: complete
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
- Statistics: Num rows: 1 Data size: 224 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 424 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: double), _col1 (type: double), _col2 (type: double), UDFToDouble(_col3) (type: double), UDFToDouble(_col4) (type: double), _col5 (type: double), _col6 (type: double), _col7 (type: double), _col8 (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
- Statistics: Num rows: 1 Data size: 224 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 424 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 224 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 424 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/groupby3_noskew_multi_distinct.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/groupby3_noskew_multi_distinct.q.out b/ql/src/test/results/clientpositive/groupby3_noskew_multi_distinct.q.out
index f6be869..a1d403d 100644
--- a/ql/src/test/results/clientpositive/groupby3_noskew_multi_distinct.q.out
+++ b/ql/src/test/results/clientpositive/groupby3_noskew_multi_distinct.q.out
@@ -61,14 +61,14 @@ STAGE PLANS:
aggregations: sum(KEY._col0:0._col0), avg(KEY._col0:0._col0), avg(DISTINCT KEY._col0:0._col0), max(KEY._col0:0._col0), min(KEY._col0:0._col0), std(KEY._col0:0._col0), stddev_samp(KEY._col0:0._col0), variance(KEY._col0:0._col0), var_samp(KEY._col0:0._col0), sum(DISTINCT KEY._col0:1._col0), count(DISTINCT KEY._col0:2._col0)
mode: complete
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10
- Statistics: Num rows: 1 Data size: 240 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 440 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: double), _col1 (type: double), _col2 (type: double), UDFToDouble(_col3) (type: double), UDFToDouble(_col4) (type: double), _col5 (type: double), _col6 (type: double), _col7 (type: double), _col8 (type: double), _col9 (type: double), UDFToDouble(_col10) (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10
- Statistics: Num rows: 1 Data size: 240 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 440 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 240 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 440 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/interval_arithmetic.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/interval_arithmetic.q.out b/ql/src/test/results/clientpositive/interval_arithmetic.q.out
index 0f0ca50..eba97fa 100644
--- a/ql/src/test/results/clientpositive/interval_arithmetic.q.out
+++ b/ql/src/test/results/clientpositive/interval_arithmetic.q.out
@@ -286,10 +286,10 @@ STAGE PLANS:
Select Operator
expressions: 5-5 (type: interval_year_month), -1-1 (type: interval_year_month)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 12288 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 12288 Data size: 196608 Basic stats: COMPLETE Column stats: COMPLETE
Limit
Number of rows: 2
- Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE
ListSink
PREHOOK: query: select
@@ -582,10 +582,10 @@ STAGE PLANS:
Select Operator
expressions: 109 20:30:40.246913578 (type: interval_day_time), 89 02:14:26.000000000 (type: interval_day_time)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 12288 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 12288 Data size: 294912 Basic stats: COMPLETE Column stats: COMPLETE
Limit
Number of rows: 2
- Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
ListSink
PREHOOK: query: select
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/literal_decimal.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/literal_decimal.q.out b/ql/src/test/results/clientpositive/literal_decimal.q.out
index 0b6299b..6f686eb 100644
--- a/ql/src/test/results/clientpositive/literal_decimal.q.out
+++ b/ql/src/test/results/clientpositive/literal_decimal.q.out
@@ -16,10 +16,10 @@ STAGE PLANS:
Select Operator
expressions: -1 (type: int), 0 (type: int), 1 (type: int), 3.14 (type: decimal(3,2)), -3.14 (type: decimal(3,2)), 99999999999999999 (type: bigint), 99999999999999999.9999999999999 (type: decimal(30,13)), null (type: void)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
- Statistics: Num rows: 500 Data size: 178000 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 500 Data size: 178004 Basic stats: COMPLETE Column stats: COMPLETE
Limit
Number of rows: 1
- Statistics: Num rows: 1 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 360 Basic stats: COMPLETE Column stats: COMPLETE
ListSink
PREHOOK: query: SELECT -1BD, 0BD, 1BD, 3.14BD, -3.14BD, 99999999999999999BD, 99999999999999999.9999999999999BD, 1E99BD FROM src LIMIT 1
[49/50] [abbrv] hive git commit: HIVE-13815: Improve logic to infer
false predicates (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
Posted by om...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/f31f7495/ql/src/test/results/clientpositive/spark/constprog_semijoin.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/constprog_semijoin.q.out b/ql/src/test/results/clientpositive/spark/constprog_semijoin.q.out
index de829e2..c6a9b14 100644
--- a/ql/src/test/results/clientpositive/spark/constprog_semijoin.q.out
+++ b/ql/src/test/results/clientpositive/spark/constprog_semijoin.q.out
@@ -434,17 +434,17 @@ STAGE PLANS:
alias: table1
Statistics: Num rows: 10 Data size: 200 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (((dimid = 100) = true) and (dimid <> 100)) (type: boolean)
- Statistics: Num rows: 5 Data size: 100 Basic stats: COMPLETE Column stats: NONE
+ predicate: false (type: boolean)
+ Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: id (type: int), val (type: string), val1 (type: string), dimid (type: int)
- outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 5 Data size: 100 Basic stats: COMPLETE Column stats: NONE
+ expressions: id (type: int), val (type: string), val1 (type: string)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col3 (type: int), true (type: boolean)
+ key expressions: 100 (type: int), true (type: boolean)
sort order: ++
- Map-reduce partition columns: _col3 (type: int), true (type: boolean)
- Statistics: Num rows: 5 Data size: 100 Basic stats: COMPLETE Column stats: NONE
+ Map-reduce partition columns: 100 (type: int), true (type: boolean)
+ Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string)
Map 3
Map Operator Tree:
@@ -452,10 +452,10 @@ STAGE PLANS:
alias: table3
Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (((id = 100) = true) and (id <> 100)) (type: boolean)
+ predicate: ((id = 100) and (id = 100) is not null) (type: boolean)
Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: id (type: int), true (type: boolean)
+ expressions: 100 (type: int), true (type: boolean)
outputColumnNames: _col0, _col1
Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
Group By Operator
@@ -474,13 +474,13 @@ STAGE PLANS:
condition map:
Left Semi Join 0 to 1
keys:
- 0 _col3 (type: int), true (type: boolean)
+ 0 100 (type: int), true (type: boolean)
1 _col0 (type: int), _col1 (type: boolean)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 5 Data size: 110 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 5 Data size: 110 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -523,17 +523,17 @@ STAGE PLANS:
alias: table1
Statistics: Num rows: 10 Data size: 200 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((dimid) IN (100, 200) and ((dimid = 100) = true)) (type: boolean)
- Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((dimid = 100) and (dimid = 100) is not null) (type: boolean)
+ Statistics: Num rows: 5 Data size: 100 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: id (type: int), val (type: string), val1 (type: string), dimid (type: int)
- outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: NONE
+ expressions: id (type: int), val (type: string), val1 (type: string)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 5 Data size: 100 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col3 (type: int), true (type: boolean)
+ key expressions: 100 (type: int), true (type: boolean)
sort order: ++
- Map-reduce partition columns: _col3 (type: int), true (type: boolean)
- Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: NONE
+ Map-reduce partition columns: 100 (type: int), true (type: boolean)
+ Statistics: Num rows: 5 Data size: 100 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string)
Map 3
Map Operator Tree:
@@ -541,35 +541,35 @@ STAGE PLANS:
alias: table3
Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((id) IN (100, 200) and ((id = 100) = true)) (type: boolean)
- Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((id = 100) and (id = 100) is not null) (type: boolean)
+ Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: id (type: int), true (type: boolean)
+ expressions: 100 (type: int), true (type: boolean)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: int), _col1 (type: boolean)
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int), _col1 (type: boolean)
sort order: ++
Map-reduce partition columns: _col0 (type: int), _col1 (type: boolean)
- Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
Reducer 2
Reduce Operator Tree:
Join Operator
condition map:
Left Semi Join 0 to 1
keys:
- 0 _col3 (type: int), true (type: boolean)
+ 0 100 (type: int), true (type: boolean)
1 _col0 (type: int), _col1 (type: boolean)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 2 Data size: 44 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 5 Data size: 110 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 2 Data size: 44 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 5 Data size: 110 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -614,17 +614,17 @@ STAGE PLANS:
alias: table1
Statistics: Num rows: 10 Data size: 200 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (((dimid = 100) = true) and (dimid = 200)) (type: boolean)
- Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: NONE
+ predicate: false (type: boolean)
+ Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: id (type: int), val (type: string), val1 (type: string)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: 200 (type: int), true (type: boolean)
+ key expressions: 100 (type: int), true (type: boolean)
sort order: ++
- Map-reduce partition columns: 200 (type: int), true (type: boolean)
- Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: NONE
+ Map-reduce partition columns: 100 (type: int), true (type: boolean)
+ Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string)
Map 3
Map Operator Tree:
@@ -632,35 +632,35 @@ STAGE PLANS:
alias: table3
Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (((id = 100) = true) and (id = 200)) (type: boolean)
- Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((id = 100) and (id = 100) is not null) (type: boolean)
+ Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: 200 (type: int), true (type: boolean)
+ expressions: 100 (type: int), true (type: boolean)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: int), _col1 (type: boolean)
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int), _col1 (type: boolean)
sort order: ++
Map-reduce partition columns: _col0 (type: int), _col1 (type: boolean)
- Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
Reducer 2
Reduce Operator Tree:
Join Operator
condition map:
Left Semi Join 0 to 1
keys:
- 0 200 (type: int), true (type: boolean)
+ 0 100 (type: int), true (type: boolean)
1 _col0 (type: int), _col1 (type: boolean)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 2 Data size: 44 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 2 Data size: 44 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -703,17 +703,17 @@ STAGE PLANS:
alias: table1
Statistics: Num rows: 10 Data size: 200 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (((dimid = 100) = true) and (dimid = 100)) (type: boolean)
- Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((dimid = 100) and (dimid = 100) is not null) (type: boolean)
+ Statistics: Num rows: 5 Data size: 100 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: id (type: int), val (type: string), val1 (type: string)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 5 Data size: 100 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: 100 (type: int), true (type: boolean)
sort order: ++
Map-reduce partition columns: 100 (type: int), true (type: boolean)
- Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 5 Data size: 100 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string)
Map 3
Map Operator Tree:
@@ -721,22 +721,22 @@ STAGE PLANS:
alias: table3
Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (((id = 100) = true) and (id = 100)) (type: boolean)
- Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((id = 100) and (id = 100) is not null) (type: boolean)
+ Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: 100 (type: int), true (type: boolean)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: int), _col1 (type: boolean)
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int), _col1 (type: boolean)
sort order: ++
Map-reduce partition columns: _col0 (type: int), _col1 (type: boolean)
- Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
Reducer 2
Reduce Operator Tree:
Join Operator
@@ -746,10 +746,10 @@ STAGE PLANS:
0 100 (type: int), true (type: boolean)
1 _col0 (type: int), _col1 (type: boolean)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 2 Data size: 44 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 5 Data size: 110 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 2 Data size: 44 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 5 Data size: 110 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -794,16 +794,16 @@ STAGE PLANS:
alias: table1
Statistics: Num rows: 10 Data size: 200 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (((dimid = 100) = true) and dimid is not null) (type: boolean)
+ predicate: ((dimid = 100) and (dimid = 100) is not null) (type: boolean)
Statistics: Num rows: 5 Data size: 100 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: id (type: int), val (type: string), val1 (type: string), dimid (type: int)
- outputColumnNames: _col0, _col1, _col2, _col3
+ expressions: id (type: int), val (type: string), val1 (type: string)
+ outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 5 Data size: 100 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col3 (type: int), true (type: boolean)
+ key expressions: 100 (type: int), true (type: boolean)
sort order: ++
- Map-reduce partition columns: _col3 (type: int), true (type: boolean)
+ Map-reduce partition columns: 100 (type: int), true (type: boolean)
Statistics: Num rows: 5 Data size: 100 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string)
Map 3
@@ -812,10 +812,10 @@ STAGE PLANS:
alias: table3
Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (((id = 100) = true) and id is not null) (type: boolean)
+ predicate: ((id = 100) and (id = 100) is not null) (type: boolean)
Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: id (type: int), true (type: boolean)
+ expressions: 100 (type: int), true (type: boolean)
outputColumnNames: _col0, _col1
Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
Group By Operator
@@ -834,7 +834,7 @@ STAGE PLANS:
condition map:
Left Semi Join 0 to 1
keys:
- 0 _col3 (type: int), true (type: boolean)
+ 0 100 (type: int), true (type: boolean)
1 _col0 (type: int), _col1 (type: boolean)
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 5 Data size: 110 Basic stats: COMPLETE Column stats: NONE
http://git-wip-us.apache.org/repos/asf/hive/blob/f31f7495/ql/src/test/results/clientpositive/spark/ppd_outer_join5.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/ppd_outer_join5.q.out b/ql/src/test/results/clientpositive/spark/ppd_outer_join5.q.out
index ae266e5..8ad9ac5 100644
--- a/ql/src/test/results/clientpositive/spark/ppd_outer_join5.q.out
+++ b/ql/src/test/results/clientpositive/spark/ppd_outer_join5.q.out
@@ -197,7 +197,7 @@ STAGE PLANS:
alias: t3
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Filter Operator
- predicate: (id = 20) (type: boolean)
+ predicate: (20 = id) (type: boolean)
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: 20 (type: int), key (type: string), value (type: string)
@@ -305,7 +305,7 @@ STAGE PLANS:
alias: t3
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Filter Operator
- predicate: (id = 20) (type: boolean)
+ predicate: (20 = id) (type: boolean)
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: 20 (type: int), key (type: string), value (type: string)
http://git-wip-us.apache.org/repos/asf/hive/blob/f31f7495/ql/src/test/results/clientpositive/spark/smb_mapjoin_25.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/smb_mapjoin_25.q.out b/ql/src/test/results/clientpositive/spark/smb_mapjoin_25.q.out
index a7d6e21..74f3db9 100644
--- a/ql/src/test/results/clientpositive/spark/smb_mapjoin_25.q.out
+++ b/ql/src/test/results/clientpositive/spark/smb_mapjoin_25.q.out
@@ -87,7 +87,7 @@ STAGE PLANS:
alias: b
Statistics: Num rows: 51 Data size: 206 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (key = 5) (type: boolean)
+ predicate: (5 = key) (type: boolean)
Statistics: Num rows: 25 Data size: 100 Basic stats: COMPLETE Column stats: NONE
Select Operator
Statistics: Num rows: 25 Data size: 100 Basic stats: COMPLETE Column stats: NONE
@@ -113,7 +113,7 @@ STAGE PLANS:
alias: d
Statistics: Num rows: 55 Data size: 222 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (key = 5) (type: boolean)
+ predicate: (5 = key) (type: boolean)
Statistics: Num rows: 27 Data size: 108 Basic stats: COMPLETE Column stats: NONE
Select Operator
Statistics: Num rows: 27 Data size: 108 Basic stats: COMPLETE Column stats: NONE
@@ -197,7 +197,7 @@ STAGE PLANS:
alias: b
Statistics: Num rows: 51 Data size: 206 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (key = 5) (type: boolean)
+ predicate: (5 = key) (type: boolean)
Statistics: Num rows: 25 Data size: 100 Basic stats: COMPLETE Column stats: NONE
Select Operator
Statistics: Num rows: 25 Data size: 100 Basic stats: COMPLETE Column stats: NONE
@@ -264,7 +264,7 @@ STAGE PLANS:
alias: d
Statistics: Num rows: 55 Data size: 222 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (key = 5) (type: boolean)
+ predicate: (5 = key) (type: boolean)
Statistics: Num rows: 27 Data size: 108 Basic stats: COMPLETE Column stats: NONE
Select Operator
Statistics: Num rows: 27 Data size: 108 Basic stats: COMPLETE Column stats: NONE
http://git-wip-us.apache.org/repos/asf/hive/blob/f31f7495/ql/src/test/results/clientpositive/tez/constprog_semijoin.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/constprog_semijoin.q.out b/ql/src/test/results/clientpositive/tez/constprog_semijoin.q.out
index 0f3bd63..15a8351 100644
--- a/ql/src/test/results/clientpositive/tez/constprog_semijoin.q.out
+++ b/ql/src/test/results/clientpositive/tez/constprog_semijoin.q.out
@@ -261,17 +261,17 @@ Stage-0
Stage-1
Reducer 2
File Output Operator [FS_12]
- Merge Join Operator [MERGEJOIN_17] (rows=5 width=22)
- Conds:RS_8._col3, true=RS_9._col0, _col1(Left Semi),Output:["_col0","_col1","_col2"]
+ Merge Join Operator [MERGEJOIN_17] (rows=2 width=3)
+ Conds:RS_8.100, true=RS_9._col0, _col1(Left Semi),Output:["_col0","_col1","_col2"]
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_8]
- PartitionCols:_col3, true
- Select Operator [SEL_2] (rows=5 width=20)
- Output:["_col0","_col1","_col2","_col3"]
- Filter Operator [FIL_15] (rows=5 width=20)
- predicate:(((dimid = 100) = true) and (dimid <> 100))
+ PartitionCols:100, true
+ Select Operator [SEL_2] (rows=1 width=20)
+ Output:["_col0","_col1","_col2"]
+ Filter Operator [FIL_15] (rows=1 width=20)
+ predicate:false
TableScan [TS_0] (rows=10 width=20)
- default@table1,table1,Tbl:COMPLETE,Col:NONE,Output:["id","val","val1","dimid"]
+ default@table1,table1,Tbl:COMPLETE,Col:NONE,Output:["id","val","val1"]
<-Map 3 [SIMPLE_EDGE]
SHUFFLE [RS_9]
PartitionCols:_col0, _col1
@@ -280,7 +280,7 @@ Stage-0
Select Operator [SEL_5] (rows=2 width=3)
Output:["_col0","_col1"]
Filter Operator [FIL_16] (rows=2 width=3)
- predicate:(((id = 100) = true) and (id <> 100))
+ predicate:((id = 100) and (id = 100) is not null)
TableScan [TS_3] (rows=5 width=3)
default@table3,table3,Tbl:COMPLETE,Col:NONE,Output:["id"]
@@ -309,26 +309,26 @@ Stage-0
Stage-1
Reducer 2
File Output Operator [FS_12]
- Merge Join Operator [MERGEJOIN_17] (rows=2 width=22)
- Conds:RS_8._col3, true=RS_9._col0, _col1(Left Semi),Output:["_col0","_col1","_col2"]
+ Merge Join Operator [MERGEJOIN_17] (rows=5 width=22)
+ Conds:RS_8.100, true=RS_9._col0, _col1(Left Semi),Output:["_col0","_col1","_col2"]
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_8]
- PartitionCols:_col3, true
- Select Operator [SEL_2] (rows=2 width=20)
- Output:["_col0","_col1","_col2","_col3"]
- Filter Operator [FIL_15] (rows=2 width=20)
- predicate:((dimid) IN (100, 200) and ((dimid = 100) = true))
+ PartitionCols:100, true
+ Select Operator [SEL_2] (rows=5 width=20)
+ Output:["_col0","_col1","_col2"]
+ Filter Operator [FIL_15] (rows=5 width=20)
+ predicate:((dimid = 100) and (dimid = 100) is not null)
TableScan [TS_0] (rows=10 width=20)
default@table1,table1,Tbl:COMPLETE,Col:NONE,Output:["id","val","val1","dimid"]
<-Map 3 [SIMPLE_EDGE]
SHUFFLE [RS_9]
PartitionCols:_col0, _col1
- Group By Operator [GBY_7] (rows=1 width=3)
+ Group By Operator [GBY_7] (rows=2 width=3)
Output:["_col0","_col1"],keys:_col0, _col1
- Select Operator [SEL_5] (rows=1 width=3)
+ Select Operator [SEL_5] (rows=2 width=3)
Output:["_col0","_col1"]
- Filter Operator [FIL_16] (rows=1 width=3)
- predicate:((id) IN (100, 200) and ((id = 100) = true))
+ Filter Operator [FIL_16] (rows=2 width=3)
+ predicate:((id = 100) and (id = 100) is not null)
TableScan [TS_3] (rows=5 width=3)
default@table3,table3,Tbl:COMPLETE,Col:NONE,Output:["id"]
@@ -359,26 +359,26 @@ Stage-0
Stage-1
Reducer 2
File Output Operator [FS_12]
- Merge Join Operator [MERGEJOIN_17] (rows=2 width=22)
- Conds:RS_8.200, true=RS_9._col0, _col1(Left Semi),Output:["_col0","_col1","_col2"]
+ Merge Join Operator [MERGEJOIN_17] (rows=2 width=3)
+ Conds:RS_8.100, true=RS_9._col0, _col1(Left Semi),Output:["_col0","_col1","_col2"]
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_8]
- PartitionCols:200, true
- Select Operator [SEL_2] (rows=2 width=20)
+ PartitionCols:100, true
+ Select Operator [SEL_2] (rows=1 width=20)
Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_15] (rows=2 width=20)
- predicate:(((dimid = 100) = true) and (dimid = 200))
+ Filter Operator [FIL_15] (rows=1 width=20)
+ predicate:false
TableScan [TS_0] (rows=10 width=20)
- default@table1,table1,Tbl:COMPLETE,Col:NONE,Output:["id","val","val1","dimid"]
+ default@table1,table1,Tbl:COMPLETE,Col:NONE,Output:["id","val","val1"]
<-Map 3 [SIMPLE_EDGE]
SHUFFLE [RS_9]
PartitionCols:_col0, _col1
- Group By Operator [GBY_7] (rows=1 width=3)
+ Group By Operator [GBY_7] (rows=2 width=3)
Output:["_col0","_col1"],keys:_col0, _col1
- Select Operator [SEL_5] (rows=1 width=3)
+ Select Operator [SEL_5] (rows=2 width=3)
Output:["_col0","_col1"]
- Filter Operator [FIL_16] (rows=1 width=3)
- predicate:(((id = 100) = true) and (id = 200))
+ Filter Operator [FIL_16] (rows=2 width=3)
+ predicate:((id = 100) and (id = 100) is not null)
TableScan [TS_3] (rows=5 width=3)
default@table3,table3,Tbl:COMPLETE,Col:NONE,Output:["id"]
@@ -407,26 +407,26 @@ Stage-0
Stage-1
Reducer 2
File Output Operator [FS_12]
- Merge Join Operator [MERGEJOIN_17] (rows=2 width=22)
+ Merge Join Operator [MERGEJOIN_17] (rows=5 width=22)
Conds:RS_8.100, true=RS_9._col0, _col1(Left Semi),Output:["_col0","_col1","_col2"]
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_8]
PartitionCols:100, true
- Select Operator [SEL_2] (rows=2 width=20)
+ Select Operator [SEL_2] (rows=5 width=20)
Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_15] (rows=2 width=20)
- predicate:(((dimid = 100) = true) and (dimid = 100))
+ Filter Operator [FIL_15] (rows=5 width=20)
+ predicate:((dimid = 100) and (dimid = 100) is not null)
TableScan [TS_0] (rows=10 width=20)
default@table1,table1,Tbl:COMPLETE,Col:NONE,Output:["id","val","val1","dimid"]
<-Map 3 [SIMPLE_EDGE]
SHUFFLE [RS_9]
PartitionCols:_col0, _col1
- Group By Operator [GBY_7] (rows=1 width=3)
+ Group By Operator [GBY_7] (rows=2 width=3)
Output:["_col0","_col1"],keys:_col0, _col1
- Select Operator [SEL_5] (rows=1 width=3)
+ Select Operator [SEL_5] (rows=2 width=3)
Output:["_col0","_col1"]
- Filter Operator [FIL_16] (rows=1 width=3)
- predicate:(((id = 100) = true) and (id = 100))
+ Filter Operator [FIL_16] (rows=2 width=3)
+ predicate:((id = 100) and (id = 100) is not null)
TableScan [TS_3] (rows=5 width=3)
default@table3,table3,Tbl:COMPLETE,Col:NONE,Output:["id"]
@@ -458,14 +458,14 @@ Stage-0
Reducer 2
File Output Operator [FS_12]
Merge Join Operator [MERGEJOIN_17] (rows=5 width=22)
- Conds:RS_8._col3, true=RS_9._col0, _col1(Left Semi),Output:["_col0","_col1","_col2"]
+ Conds:RS_8.100, true=RS_9._col0, _col1(Left Semi),Output:["_col0","_col1","_col2"]
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_8]
- PartitionCols:_col3, true
+ PartitionCols:100, true
Select Operator [SEL_2] (rows=5 width=20)
- Output:["_col0","_col1","_col2","_col3"]
+ Output:["_col0","_col1","_col2"]
Filter Operator [FIL_15] (rows=5 width=20)
- predicate:(((dimid = 100) = true) and dimid is not null)
+ predicate:((dimid = 100) and (dimid = 100) is not null)
TableScan [TS_0] (rows=10 width=20)
default@table1,table1,Tbl:COMPLETE,Col:NONE,Output:["id","val","val1","dimid"]
<-Map 3 [SIMPLE_EDGE]
@@ -476,7 +476,7 @@ Stage-0
Select Operator [SEL_5] (rows=2 width=3)
Output:["_col0","_col1"]
Filter Operator [FIL_16] (rows=2 width=3)
- predicate:(((id = 100) = true) and id is not null)
+ predicate:((id = 100) and (id = 100) is not null)
TableScan [TS_3] (rows=5 width=3)
default@table3,table3,Tbl:COMPLETE,Col:NONE,Output:["id"]
http://git-wip-us.apache.org/repos/asf/hive/blob/f31f7495/ql/src/test/results/clientpositive/tez/cte_5.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/cte_5.q.out b/ql/src/test/results/clientpositive/tez/cte_5.q.out
index 579b4f3..73342f5 100644
--- a/ql/src/test/results/clientpositive/tez/cte_5.q.out
+++ b/ql/src/test/results/clientpositive/tez/cte_5.q.out
@@ -63,6 +63,7 @@ src_thrift
srcbucket
srcbucket2
srcpart
+Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product
PREHOOK: query: explain
with q1 as (select * from src where key= '5')
select a.colnum
@@ -86,26 +87,26 @@ Stage-0
Stage-1
Reducer 2
File Output Operator [FS_10]
- Merge Join Operator [MERGEJOIN_15] (rows=275 width=10)
- Conds:RS_6.UDFToDouble(_col0)=RS_7.5.0(Inner),Output:["_col0"]
- <-Map 1 [SIMPLE_EDGE]
- SHUFFLE [RS_6]
- PartitionCols:UDFToDouble(_col0)
- Select Operator [SEL_2] (rows=1 width=3)
- Output:["_col0"]
- Filter Operator [FIL_13] (rows=1 width=3)
- predicate:colnum is not null
- TableScan [TS_0] (rows=1 width=3)
- mydb@q1,a,Tbl:COMPLETE,Col:NONE,Output:["colnum"]
- <-Map 3 [SIMPLE_EDGE]
- SHUFFLE [RS_7]
- PartitionCols:5.0
- Select Operator [SEL_5] (rows=250 width=10)
- Filter Operator [FIL_14] (rows=250 width=10)
- predicate:(key = '5')
- TableScan [TS_3] (rows=500 width=10)
- default@src,src,Tbl:COMPLETE,Col:NONE,Output:["key"]
+ Select Operator [SEL_9] (rows=250 width=14)
+ Output:["_col0"]
+ Merge Join Operator [MERGEJOIN_13] (rows=250 width=14)
+ Conds:(Inner)
+ <-Map 1 [SIMPLE_EDGE]
+ SHUFFLE [RS_6]
+ Select Operator [SEL_2] (rows=1 width=3)
+ Filter Operator [FIL_11] (rows=1 width=3)
+ predicate:(UDFToDouble(colnum) = 5.0)
+ TableScan [TS_0] (rows=1 width=3)
+ mydb@q1,a,Tbl:COMPLETE,Col:NONE,Output:["colnum"]
+ <-Map 3 [SIMPLE_EDGE]
+ SHUFFLE [RS_7]
+ Select Operator [SEL_5] (rows=250 width=10)
+ Filter Operator [FIL_12] (rows=250 width=10)
+ predicate:(key = '5')
+ TableScan [TS_3] (rows=500 width=10)
+ default@src,src,Tbl:COMPLETE,Col:NONE,Output:["key"]
+Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product
PREHOOK: query: with q1 as (select * from src where key= '5')
select a.colnum
from mydb.q1 as a join q1 as b
http://git-wip-us.apache.org/repos/asf/hive/blob/f31f7495/ql/src/test/results/clientpositive/tez/dynamic_partition_pruning_2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/dynamic_partition_pruning_2.q.out b/ql/src/test/results/clientpositive/tez/dynamic_partition_pruning_2.q.out
index 5f8aab5..751382a 100644
--- a/ql/src/test/results/clientpositive/tez/dynamic_partition_pruning_2.q.out
+++ b/ql/src/test/results/clientpositive/tez/dynamic_partition_pruning_2.q.out
@@ -617,10 +617,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: d1
- filterExpr: (id = 1) (type: boolean)
+ filterExpr: (1 = id) (type: boolean)
Statistics: Num rows: 3 Data size: 15 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (id = 1) (type: boolean)
+ predicate: (1 = id) (type: boolean)
Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE
Select Operator
Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE
http://git-wip-us.apache.org/repos/asf/hive/blob/f31f7495/ql/src/test/results/clientpositive/tez/hybridgrace_hashjoin_1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/hybridgrace_hashjoin_1.q.out b/ql/src/test/results/clientpositive/tez/hybridgrace_hashjoin_1.q.out
index cffa870..9d7f7ae 100644
--- a/ql/src/test/results/clientpositive/tez/hybridgrace_hashjoin_1.q.out
+++ b/ql/src/test/results/clientpositive/tez/hybridgrace_hashjoin_1.q.out
@@ -1269,7 +1269,7 @@ STAGE PLANS:
alias: r
Statistics: Num rows: 12288 Data size: 2165060 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (cint = 6981) (type: boolean)
+ predicate: (6981 = cint) (type: boolean)
Statistics: Num rows: 6144 Data size: 1082530 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cdecimal2 (type: decimal(23,14))
@@ -1465,7 +1465,7 @@ STAGE PLANS:
alias: r
Statistics: Num rows: 12288 Data size: 2165060 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (cint = 6981) (type: boolean)
+ predicate: (6981 = cint) (type: boolean)
Statistics: Num rows: 6144 Data size: 1082530 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cdecimal2 (type: decimal(23,14))
http://git-wip-us.apache.org/repos/asf/hive/blob/f31f7495/ql/src/test/results/clientpositive/tez/mapjoin2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/mapjoin2.q.out b/ql/src/test/results/clientpositive/tez/mapjoin2.q.out
index 5ae1ac7..a7f7d15 100644
--- a/ql/src/test/results/clientpositive/tez/mapjoin2.q.out
+++ b/ql/src/test/results/clientpositive/tez/mapjoin2.q.out
@@ -26,6 +26,7 @@ POSTHOOK: Input: default@values__tmp__table__2
POSTHOOK: Output: default@tbl
POSTHOOK: Lineage: tbl.n EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
POSTHOOK: Lineage: tbl.t SIMPLE [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col2, type:string, comment:), ]
+Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Map 1' is a cross product
PREHOOK: query: select a.n, a.t, isnull(b.n), isnull(b.t) from (select * from tbl where n = 1) a left outer join (select * from tbl where 1 = 2) b on a.n = b.n
PREHOOK: type: QUERY
PREHOOK: Input: default@tbl
@@ -35,6 +36,7 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@tbl
#### A masked pattern was here ####
1 one true true
+Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Map 2' is a cross product
PREHOOK: query: select isnull(a.n), isnull(a.t), b.n, b.t from (select * from tbl where 2 = 1) a right outer join (select * from tbl where n = 2) b on a.n = b.n
PREHOOK: type: QUERY
PREHOOK: Input: default@tbl
@@ -44,6 +46,7 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@tbl
#### A masked pattern was here ####
true true 2 two
+Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product
PREHOOK: query: select isnull(a.n), isnull(a.t), isnull(b.n), isnull(b.t) from (select * from tbl where n = 1) a full outer join (select * from tbl where n = 2) b on a.n = b.n
PREHOOK: type: QUERY
PREHOOK: Input: default@tbl
http://git-wip-us.apache.org/repos/asf/hive/blob/f31f7495/ql/src/test/results/clientpositive/tez/mergejoin.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/mergejoin.q.out b/ql/src/test/results/clientpositive/tez/mergejoin.q.out
index b860b9b..5bc44fe 100644
--- a/ql/src/test/results/clientpositive/tez/mergejoin.q.out
+++ b/ql/src/test/results/clientpositive/tez/mergejoin.q.out
@@ -3023,6 +3023,7 @@ POSTHOOK: Input: default@tab_part
POSTHOOK: Input: default@tab_part@ds=2008-04-08
#### A masked pattern was here ####
480
+Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product
PREHOOK: query: select * from (select * from tab where tab.key = 0)a full outer join (select * from tab_part where tab_part.key = 98)b on a.key = b.key
PREHOOK: type: QUERY
PREHOOK: Input: default@tab
@@ -3042,6 +3043,7 @@ POSTHOOK: Input: default@tab_part@ds=2008-04-08
0 val_0 2008-04-08 NULL NULL NULL
NULL NULL NULL 98 val_98 2008-04-08
NULL NULL NULL 98 val_98 2008-04-08
+Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product
PREHOOK: query: select * from (select * from tab where tab.key = 0)a right outer join (select * from tab_part where tab_part.key = 98)b on a.key = b.key
PREHOOK: type: QUERY
PREHOOK: Input: default@tab
@@ -3056,13 +3058,16 @@ POSTHOOK: Input: default@tab_part@ds=2008-04-08
#### A masked pattern was here ####
NULL NULL NULL 98 val_98 2008-04-08
NULL NULL NULL 98 val_98 2008-04-08
+Warning: Shuffle Join MERGEJOIN[19][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 2' is a cross product
PREHOOK: query: select * from
(select * from tab where tab.key = 0)a
full outer join
(select * from tab_part where tab_part.key = 98)b join tab_part c on a.key = b.key and b.key = c.key
PREHOOK: type: QUERY
PREHOOK: Input: default@tab
+PREHOOK: Input: default@tab@ds=2008-04-08
PREHOOK: Input: default@tab_part
+PREHOOK: Input: default@tab_part@ds=2008-04-08
#### A masked pattern was here ####
POSTHOOK: query: select * from
(select * from tab where tab.key = 0)a
@@ -3070,9 +3075,12 @@ full outer join
(select * from tab_part where tab_part.key = 98)b join tab_part c on a.key = b.key and b.key = c.key
POSTHOOK: type: QUERY
POSTHOOK: Input: default@tab
+POSTHOOK: Input: default@tab@ds=2008-04-08
POSTHOOK: Input: default@tab_part
+POSTHOOK: Input: default@tab_part@ds=2008-04-08
#### A masked pattern was here ####
-Warning: Shuffle Join MERGEJOIN[22][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product
+Warning: Shuffle Join MERGEJOIN[20][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product
+Warning: Shuffle Join MERGEJOIN[21][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product
PREHOOK: query: select * from
(select * from tab where tab.key = 0)a
full outer join
@@ -3096,6 +3104,7 @@ NULL NULL NULL 98 val_98 2008-04-08 98 val_98 2008-04-08
NULL NULL NULL 98 val_98 2008-04-08 98 val_98 2008-04-08
NULL NULL NULL 98 val_98 2008-04-08 98 val_98 2008-04-08
Warning: Shuffle Join MERGEJOIN[18][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product
+Warning: Shuffle Join MERGEJOIN[19][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product
PREHOOK: query: select * from
(select * from tab where tab.key = 0)a
join
@@ -3622,12 +3631,15 @@ NULL NULL NULL NULL NULL NULL 97 val_97 2008-04-08
NULL NULL NULL NULL NULL NULL 97 val_97 2008-04-08
NULL NULL NULL NULL NULL NULL 98 val_98 2008-04-08
NULL NULL NULL NULL NULL NULL 98 val_98 2008-04-08
+Warning: Shuffle Join MERGEJOIN[19][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product
+Warning: Shuffle Join MERGEJOIN[20][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product
PREHOOK: query: select * from
(select * from tab where tab.key = 0)a
join
(select * from tab_part where tab_part.key = 98)b on a.key = b.key full outer join tab_part c on b.key = c.key
PREHOOK: type: QUERY
PREHOOK: Input: default@tab
+PREHOOK: Input: default@tab@ds=2008-04-08
PREHOOK: Input: default@tab_part
PREHOOK: Input: default@tab_part@ds=2008-04-08
#### A masked pattern was here ####
@@ -3637,6 +3649,7 @@ join
(select * from tab_part where tab_part.key = 98)b on a.key = b.key full outer join tab_part c on b.key = c.key
POSTHOOK: type: QUERY
POSTHOOK: Input: default@tab
+POSTHOOK: Input: default@tab@ds=2008-04-08
POSTHOOK: Input: default@tab_part
POSTHOOK: Input: default@tab_part@ds=2008-04-08
#### A masked pattern was here ####
http://git-wip-us.apache.org/repos/asf/hive/blob/f31f7495/ql/src/test/results/clientpositive/tez/tez_self_join.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/tez_self_join.q.out b/ql/src/test/results/clientpositive/tez/tez_self_join.q.out
index 93102be..da8d70d 100644
--- a/ql/src/test/results/clientpositive/tez/tez_self_join.q.out
+++ b/ql/src/test/results/clientpositive/tez/tez_self_join.q.out
@@ -100,7 +100,7 @@ STAGE PLANS:
alias: self2
Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (id3 = 'ab') (type: boolean)
+ predicate: ('ab' = id3) (type: boolean)
Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE
Select Operator
Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE
[39/50] [abbrv] hive git commit: HIVE-15874: Invalid position alias
in Group By when CBO failed (Walter Wu, reviewed by Pengcheng Xiong)
Posted by om...@apache.org.
HIVE-15874: Invalid position alias in Group By when CBO failed (Walter Wu, reviewed by Pengcheng Xiong)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/fb1c9fd7
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/fb1c9fd7
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/fb1c9fd7
Branch: refs/heads/branch-2.2
Commit: fb1c9fd7e6dc595bbe2b7d592e30bd7a490b93ef
Parents: ba56541
Author: Pengcheng Xiong <px...@apache.org>
Authored: Mon Feb 20 12:33:24 2017 -0800
Committer: Owen O'Malley <om...@apache.org>
Committed: Tue Mar 28 15:27:55 2017 -0700
----------------------------------------------------------------------
.../hadoop/hive/ql/parse/CalcitePlanner.java | 2 +
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 8 +-
ql/src/test/queries/clientpositive/masking_10.q | 25 ++
.../clientpositive/position_alias_test_1.q | 18 ++
.../results/clientpositive/masking_10.q.out | 244 +++++++++++++++++++
.../clientpositive/position_alias_test_1.q.out | 148 +++++++++++
6 files changed, 443 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/fb1c9fd7/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
index 51a517a..ca7583b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
@@ -298,6 +298,8 @@ public class CalcitePlanner extends SemanticAnalyzer {
public RelNode genLogicalPlan(ASTNode ast) throws SemanticException {
LOG.info("Starting generating logical plan");
PreCboCtx cboCtx = new PreCboCtx();
+ //change the location of position alias process here
+ processPositionAlias(ast);
if (!genResolvedParseTree(ast, cboCtx)) {
return null;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/fb1c9fd7/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index 6a66691..e71abc7 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -10912,7 +10912,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
ctesExpanded = new ArrayList<String>();
// 1. analyze and process the position alias
- processPositionAlias(ast);
+ // step processPositionAlias out of genResolvedParseTree
// 2. analyze create table command
if (ast.getToken().getType() == HiveParser.TOK_CREATETABLE) {
@@ -11027,6 +11027,8 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
void analyzeInternal(ASTNode ast, PlannerContext plannerCtx) throws SemanticException {
// 1. Generate Resolved Parse tree from syntax tree
LOG.info("Starting Semantic Analysis");
+ //change the location of position alias process here
+ processPositionAlias(ast);
if (!genResolvedParseTree(ast, plannerCtx)) {
return;
}
@@ -11040,6 +11042,8 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
if (tree != ast) {
ctx.setSkipTableMasking(true);
init(true);
+ //change the location of position alias process here
+ processPositionAlias(tree);
genResolvedParseTree(tree, plannerCtx);
if (this instanceof CalcitePlanner) {
((CalcitePlanner) this).resetCalciteConfiguration();
@@ -12197,7 +12201,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
}
// Process the position alias in GROUPBY and ORDERBY
- private void processPositionAlias(ASTNode ast) throws SemanticException {
+ public void processPositionAlias(ASTNode ast) throws SemanticException {
boolean isBothByPos = HiveConf.getBoolVar(conf, ConfVars.HIVE_GROUPBY_ORDERBY_POSITION_ALIAS);
boolean isGbyByPos = isBothByPos
|| HiveConf.getBoolVar(conf, ConfVars.HIVE_GROUPBY_POSITION_ALIAS);
http://git-wip-us.apache.org/repos/asf/hive/blob/fb1c9fd7/ql/src/test/queries/clientpositive/masking_10.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/masking_10.q b/ql/src/test/queries/clientpositive/masking_10.q
new file mode 100644
index 0000000..e933253
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/masking_10.q
@@ -0,0 +1,25 @@
+set hive.mapred.mode=nonstrict;
+set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest;
+
+drop table masking_test;
+
+create temporary table masking_test as select cast(key as int) as key, value from src;
+
+set hive.groupby.position.alias = true;
+set hive.cbo.enable=true;
+
+explain select 2017 as a, value from masking_test group by 1, 2;
+
+select 2017 as a, value from masking_test group by 1, 2;
+
+explain
+select * from
+ masking_test alias01
+ left join
+ (
+ select 2017 as a, value from masking_test group by 1, 2
+ ) alias02
+ on alias01.key = alias02.a
+ left join
+ masking_test alias03
+on alias01.key = alias03.key;
http://git-wip-us.apache.org/repos/asf/hive/blob/fb1c9fd7/ql/src/test/queries/clientpositive/position_alias_test_1.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/position_alias_test_1.q b/ql/src/test/queries/clientpositive/position_alias_test_1.q
new file mode 100644
index 0000000..599bc08
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/position_alias_test_1.q
@@ -0,0 +1,18 @@
+create table alias_test_01(a INT, b STRING) ;
+ create table alias_test_02(a INT, b STRING) ;
+ create table alias_test_03(a INT, b STRING) ;
+ set hive.groupby.position.alias = true;
+ set hive.cbo.enable=true;
+
+
+ explain
+ select * from
+ alias_test_01 alias01
+ left join
+ (
+ select 2017 as a, b from alias_test_02 group by 1, 2
+ ) alias02
+ on alias01.a = alias02.a
+ left join
+ alias_test_03 alias03
+ on alias01.a = alias03.a;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/fb1c9fd7/ql/src/test/results/clientpositive/masking_10.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/masking_10.q.out b/ql/src/test/results/clientpositive/masking_10.q.out
new file mode 100644
index 0000000..d6293e3
--- /dev/null
+++ b/ql/src/test/results/clientpositive/masking_10.q.out
@@ -0,0 +1,244 @@
+PREHOOK: query: drop table masking_test
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table masking_test
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create temporary table masking_test as select cast(key as int) as key, value from src
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@src
+PREHOOK: Output: database:default
+PREHOOK: Output: default@masking_test
+POSTHOOK: query: create temporary table masking_test as select cast(key as int) as key, value from src
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@src
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@masking_test
+PREHOOK: query: explain select 2017 as a, value from masking_test group by 1, 2
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select 2017 as a, value from masking_test group by 1, 2
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: masking_test
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (((key % 2) = 0) and (key < 10)) (type: boolean)
+ Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: reverse(value) (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ keys: _col0 (type: string)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
+ Reduce Operator Tree:
+ Group By Operator
+ keys: KEY._col0 (type: string)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: 2017 (type: int), _col0 (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select 2017 as a, value from masking_test group by 1, 2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@masking_test
+#### A masked pattern was here ####
+POSTHOOK: query: select 2017 as a, value from masking_test group by 1, 2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@masking_test
+#### A masked pattern was here ####
+2017 0_lav
+2017 2_lav
+2017 4_lav
+2017 8_lav
+Warning: Shuffle Join JOIN[34][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product
+PREHOOK: query: explain
+select * from
+ masking_test alias01
+ left join
+ (
+ select 2017 as a, value from masking_test group by 1, 2
+ ) alias02
+ on alias01.key = alias02.a
+ left join
+ masking_test alias03
+on alias01.key = alias03.key
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select * from
+ masking_test alias01
+ left join
+ (
+ select 2017 as a, value from masking_test group by 1, 2
+ ) alias02
+ on alias01.key = alias02.a
+ left join
+ masking_test alias03
+on alias01.key = alias03.key
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-3 is a root stage
+ Stage-1 depends on stages: Stage-3
+ Stage-2 depends on stages: Stage-1
+ Stage-0 depends on stages: Stage-2
+
+STAGE PLANS:
+ Stage: Stage-3
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: masking_test
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (((key % 2) = 0) and (key < 10)) (type: boolean)
+ Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: reverse(value) (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ keys: _col0 (type: string)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
+ Reduce Operator Tree:
+ Group By Operator
+ keys: KEY._col0 (type: string)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: 2017 (type: int), _col0 (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: masking_test
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (((key % 2) = 0) and (key < 10)) (type: boolean)
+ Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: int), reverse(value) (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: int), _col1 (type: string)
+ TableScan
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: int), _col1 (type: string)
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Left Outer Join0 to 1
+ filter predicates:
+ 0 {(VALUE._col0 = 2017)}
+ 1
+ keys:
+ 0
+ 1
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 3403 Data size: 75629 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-2
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 3403 Data size: 75629 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: string)
+ TableScan
+ alias: masking_test
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (((key % 2) = 0) and (key < 10)) (type: boolean)
+ Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: int), reverse(value) (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Left Outer Join0 to 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Statistics: Num rows: 3743 Data size: 83191 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 3743 Data size: 83191 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
http://git-wip-us.apache.org/repos/asf/hive/blob/fb1c9fd7/ql/src/test/results/clientpositive/position_alias_test_1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/position_alias_test_1.q.out b/ql/src/test/results/clientpositive/position_alias_test_1.q.out
new file mode 100644
index 0000000..9053bf1
--- /dev/null
+++ b/ql/src/test/results/clientpositive/position_alias_test_1.q.out
@@ -0,0 +1,148 @@
+PREHOOK: query: create table alias_test_01(a INT, b STRING)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@alias_test_01
+POSTHOOK: query: create table alias_test_01(a INT, b STRING)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@alias_test_01
+PREHOOK: query: create table alias_test_02(a INT, b STRING)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@alias_test_02
+POSTHOOK: query: create table alias_test_02(a INT, b STRING)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@alias_test_02
+PREHOOK: query: create table alias_test_03(a INT, b STRING)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@alias_test_03
+POSTHOOK: query: create table alias_test_03(a INT, b STRING)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@alias_test_03
+PREHOOK: query: explain
+ select * from
+ alias_test_01 alias01
+ left join
+ (
+ select 2017 as a, b from alias_test_02 group by 1, 2
+ ) alias02
+ on alias01.a = alias02.a
+ left join
+ alias_test_03 alias03
+ on alias01.a = alias03.a
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+ select * from
+ alias_test_01 alias01
+ left join
+ (
+ select 2017 as a, b from alias_test_02 group by 1, 2
+ ) alias02
+ on alias01.a = alias02.a
+ left join
+ alias_test_03 alias03
+ on alias01.a = alias03.a
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-0 depends on stages: Stage-2
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: alias_test_02
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Select Operator
+ expressions: b (type: string)
+ outputColumnNames: b
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Group By Operator
+ keys: 2017 (type: int), b (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Reduce Output Operator
+ key expressions: 2017 (type: int), _col1 (type: string)
+ sort order: ++
+ Map-reduce partition columns: 2017 (type: int), _col1 (type: string)
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Reduce Operator Tree:
+ Group By Operator
+ keys: 2017 (type: int), KEY._col1 (type: string)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Select Operator
+ expressions: 2017 (type: int), _col1 (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-2
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ value expressions: _col1 (type: string)
+ TableScan
+ alias: alias01
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Reduce Output Operator
+ key expressions: a (type: int)
+ sort order: +
+ Map-reduce partition columns: a (type: int)
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ value expressions: b (type: string)
+ TableScan
+ alias: alias03
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Reduce Output Operator
+ key expressions: a (type: int)
+ sort order: +
+ Map-reduce partition columns: a (type: int)
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ value expressions: b (type: string)
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Left Outer Join0 to 1
+ Left Outer Join0 to 2
+ keys:
+ 0 a (type: int)
+ 1 _col0 (type: int)
+ 2 a (type: int)
+ outputColumnNames: _col0, _col1, _col5, _col6, _col7, _col8
+ Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: int), _col1 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int), _col8 (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
[23/50] [abbrv] hive git commit: HIVE-14362: Support explain analyze
in Hive (Pengcheng Xiong, reviewed by Ashutosh Chauhan,
Gopal V and Gabor Szadovszky)
Posted by om...@apache.org.
HIVE-14362: Support explain analyze in Hive (Pengcheng Xiong, reviewed by Ashutosh Chauhan, Gopal V and Gabor Szadovszky)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/fd72ec4a
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/fd72ec4a
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/fd72ec4a
Branch: refs/heads/branch-2.2
Commit: fd72ec4a8cd494bc98542e62f4cfb25cb3c6b123
Parents: 47dbc00
Author: Pengcheng Xiong <px...@apache.org>
Authored: Tue Aug 30 13:43:25 2016 -0700
Committer: Owen O'Malley <om...@apache.org>
Committed: Tue Mar 28 15:27:51 2017 -0700
----------------------------------------------------------------------
.../hadoop/hive/common/StatsSetupConst.java | 2 +
.../test/resources/testconfiguration.properties | 5 +
.../resources/testconfiguration.properties.orig | 1373 ++
.../java/org/apache/hadoop/hive/ql/Context.java | 61 +-
.../org/apache/hadoop/hive/ql/Context.java.orig | 829 ++
.../java/org/apache/hadoop/hive/ql/Driver.java | 19 +-
.../hadoop/hive/ql/exec/ColumnStatsTask.java | 4 +
.../hadoop/hive/ql/exec/CommonJoinOperator.java | 1 +
.../org/apache/hadoop/hive/ql/exec/DDLTask.java | 4 +
.../apache/hadoop/hive/ql/exec/ExplainTask.java | 6 +-
.../hadoop/hive/ql/exec/FileSinkOperator.java | 4 +-
.../hadoop/hive/ql/exec/GroupByOperator.java | 2 +-
.../hadoop/hive/ql/exec/JoinOperator.java | 2 -
.../hadoop/hive/ql/exec/LimitOperator.java | 1 +
.../hadoop/hive/ql/exec/ListSinkOperator.java | 1 +
.../apache/hadoop/hive/ql/exec/MoveTask.java | 4 +
.../apache/hadoop/hive/ql/exec/Operator.java | 50 +-
.../hadoop/hive/ql/exec/ReduceSinkOperator.java | 1 +
.../hive/ql/exec/SerializationUtilities.java | 22 +
.../apache/hadoop/hive/ql/exec/StatsTask.java | 9 +-
.../hadoop/hive/ql/exec/TableScanOperator.java | 1 +
.../hadoop/hive/ql/exec/UDTFOperator.java | 1 +
.../apache/hadoop/hive/ql/hooks/ATSHook.java | 192 +-
.../hadoop/hive/ql/optimizer/Optimizer.java | 3 +-
.../physical/AnnotateRunTimeStatsOptimizer.java | 174 +
.../optimizer/physical/PhysicalOptimizer.java | 7 +-
.../ql/parse/ColumnStatsAutoGatherContext.java | 13 +-
.../ql/parse/ColumnStatsSemanticAnalyzer.java | 1 +
.../hive/ql/parse/ExplainConfiguration.java | 117 +
.../parse/ExplainSQRewriteSemanticAnalyzer.java | 3 +-
.../hive/ql/parse/ExplainSemanticAnalyzer.java | 137 +-
.../hadoop/hive/ql/parse/GenTezUtils.java | 7 +-
.../apache/hadoop/hive/ql/parse/HiveParser.g | 2 +-
.../hadoop/hive/ql/parse/MapReduceCompiler.java | 2 +-
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 17 +-
.../hive/ql/parse/SemanticAnalyzer.java.orig | 13038 +++++++++++++++++
.../hive/ql/parse/SubQueryDiagnostic.java | 2 +-
.../hadoop/hive/ql/parse/TaskCompiler.java | 5 +
.../hadoop/hive/ql/parse/TezCompiler.java | 12 +-
.../hive/ql/parse/spark/SparkCompiler.java | 8 +-
.../hive/ql/plan/AbstractOperatorDesc.java | 11 +
.../apache/hadoop/hive/ql/plan/ExplainWork.java | 66 +-
.../hadoop/hive/ql/plan/FileSinkDesc.java | 9 -
.../hadoop/hive/ql/plan/MergeJoinWork.java | 8 +-
.../hadoop/hive/ql/plan/OperatorDesc.java | 2 +
.../apache/hadoop/hive/ql/plan/Statistics.java | 22 +-
.../parse/TestUpdateDeleteSemanticAnalyzer.java | 4 +-
.../queries/clientpositive/explainanalyze_1.q | 38 +
.../queries/clientpositive/explainanalyze_2.q | 329 +
.../queries/clientpositive/explainanalyze_3.q | 158 +
.../queries/clientpositive/explainanalyze_4.q | 103 +
.../queries/clientpositive/explainanalyze_5.q | 81 +
.../clientpositive/columnstats_partlvl.q.out | 48 +
.../clientpositive/columnstats_partlvl_dp.q.out | 28 +
.../clientpositive/columnstats_quoting.q.out | 12 +
.../clientpositive/columnstats_tbllvl.q.out | 42 +
.../clientpositive/compute_stats_date.q.out | 6 +
.../clientpositive/constant_prop_2.q.out | 7 +
.../display_colstats_tbllvl.q.out | 18 +
.../dynpart_sort_optimization_acid.q.out | 80 +-
.../exec_parallel_column_stats.q.out | 6 +
.../temp_table_display_colstats_tbllvl.q.out | 18 +
.../clientpositive/tez/explainanalyze_1.q.out | 471 +
.../clientpositive/tez/explainanalyze_2.q.out | 4827 ++++++
.../clientpositive/tez/explainanalyze_3.q.out | 46 +-
.../clientpositive/tez/explainanalyze_4.q.out | 590 +
.../clientpositive/tez/explainanalyze_5.q.out | 445 +
67 files changed, 23338 insertions(+), 279 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/fd72ec4a/common/src/java/org/apache/hadoop/hive/common/StatsSetupConst.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/StatsSetupConst.java b/common/src/java/org/apache/hadoop/hive/common/StatsSetupConst.java
index 1466b69..6530cb8 100644
--- a/common/src/java/org/apache/hadoop/hive/common/StatsSetupConst.java
+++ b/common/src/java/org/apache/hadoop/hive/common/StatsSetupConst.java
@@ -83,6 +83,8 @@ public class StatsSetupConst {
*/
public static final String ROW_COUNT = "numRows";
+ public static final String RUN_TIME_ROW_COUNT = "runTimeNumRows";
+
/**
* The name of the statistic Raw Data Size to be published or gathered.
*/
http://git-wip-us.apache.org/repos/asf/hive/blob/fd72ec4a/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index 3ab2714..9720f76 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -388,6 +388,11 @@ minitez.query.files=acid_vectorization_missing_cols.q,\
dynamic_partition_pruning.q,\
dynamic_partition_pruning_2.q,\
bucketpruning1.q,\
+ explainanalyze_1.q,\
+ explainanalyze_2.q,\
+ explainanalyze_3.q,\
+ explainanalyze_4.q,\
+ explainanalyze_5.q,\
explainuser_1.q,\
explainuser_2.q,\
explainuser_3.q,\
http://git-wip-us.apache.org/repos/asf/hive/blob/fd72ec4a/itests/src/test/resources/testconfiguration.properties.orig
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties.orig b/itests/src/test/resources/testconfiguration.properties.orig
new file mode 100644
index 0000000..d5ee9ed
--- /dev/null
+++ b/itests/src/test/resources/testconfiguration.properties.orig
@@ -0,0 +1,1373 @@
+# NOTE: files should be listed in alphabetical order
+minimr.query.files=auto_sortmerge_join_16.q,\
+ bucket4.q,\
+ bucket5.q,\
+ bucket6.q,\
+ bucket_many.q,\
+ bucket_num_reducers.q,\
+ bucket_num_reducers2.q,\
+ bucketizedhiveinputformat.q,\
+ bucketmapjoin6.q,\
+ bucketmapjoin7.q,\
+ disable_merge_for_bucketing.q,\
+ empty_dir_in_table.q,\
+ exchgpartition2lel.q,\
+ external_table_with_space_in_location_path.q,\
+ file_with_header_footer.q,\
+ groupby2.q,\
+ import_exported_table.q,\
+ index_bitmap3.q,\
+ index_bitmap_auto.q,\
+ infer_bucket_sort_bucketed_table.q,\
+ infer_bucket_sort_dyn_part.q,\
+ infer_bucket_sort_map_operators.q,\
+ infer_bucket_sort_merge.q,\
+ infer_bucket_sort_num_buckets.q,\
+ infer_bucket_sort_reducers_power_two.q,\
+ input16_cc.q,\
+ insert_dir_distcp.q,\
+ join1.q,\
+ join_acid_non_acid.q,\
+ leftsemijoin_mr.q,\
+ list_bucket_dml_10.q,\
+ load_fs2.q,\
+ load_hdfs_file_with_space_in_the_name.q,\
+ non_native_window_udf.q, \
+ orc_merge_diff_fs.q,\
+ parallel_orderby.q,\
+ quotedid_smb.q,\
+ reduce_deduplicate.q,\
+ remote_script.q,\
+ root_dir_external_table.q,\
+ schemeAuthority.q,\
+ schemeAuthority2.q,\
+ scriptfile1.q,\
+ scriptfile1_win.q,\
+ skewjoin_onesideskew.q,\
+ table_nonprintable.q,\
+ temp_table_external.q,\
+ truncate_column_buckets.q,\
+ uber_reduce.q,\
+ udf_using.q
+
+# These tests are disabled for minimr
+# ql_rewrite_gbtoidx.q,\
+# ql_rewrite_gbtoidx_cbo_1.q,\
+# ql_rewrite_gbtoidx_cbo_2.q,\
+# smb_mapjoin_8.q,\
+
+
+# Tests that are not enabled for CLI Driver
+disabled.query.files=ql_rewrite_gbtoidx.q,\
+ ql_rewrite_gbtoidx_cbo_1.q,\
+ ql_rewrite_gbtoidx_cbo_2.q,\
+ rcfile_merge1.q,\
+ smb_mapjoin_8.q
+
+# NOTE: Add tests to minitez only if it is very
+# specific to tez and cannot be added to minillap.
+minitez.query.files.shared=delete_orig_table.q,\
+ orc_merge12.q,\
+ orc_vectorization_ppd.q,\
+ unionDistinct_2.q,\
+ update_orig_table.q,\
+ vector_join_part_col_char.q,\
+ vector_non_string_partition.q,\
+ vectorization_div0.q,\
+ vectorization_limit.q
+
+# NOTE: Add tests to minitez only if it is very
+# specific to tez and cannot be added to minillap.
+minitez.query.files=explainuser_3.q,\
+ hybridgrace_hashjoin_1.q,\
+ hybridgrace_hashjoin_2.q,\
+ partition_column_names_with_leading_and_trailing_spaces.q,\
+ stats_filemetadata.q,\
+ tez_union_with_udf.q
+
+minillap.shared.query.files=acid_globallimit.q,\
+ alter_merge_2_orc.q,\
+ alter_merge_orc.q,\
+ alter_merge_stats_orc.q,\
+ auto_join0.q,\
+ auto_join1.q,\
+ auto_join21.q,\
+ auto_join29.q,\
+ auto_join30.q,\
+ auto_join_filters.q,\
+ auto_join_nulls.q,\
+ auto_sortmerge_join_1.q,\
+ auto_sortmerge_join_10.q,\
+ auto_sortmerge_join_11.q,\
+ auto_sortmerge_join_12.q,\
+ auto_sortmerge_join_13.q,\
+ auto_sortmerge_join_14.q,\
+ auto_sortmerge_join_15.q,\
+ auto_sortmerge_join_16.q,\
+ auto_sortmerge_join_2.q,\
+ auto_sortmerge_join_3.q,\
+ auto_sortmerge_join_4.q,\
+ auto_sortmerge_join_5.q,\
+ auto_sortmerge_join_6.q,\
+ auto_sortmerge_join_7.q,\
+ auto_sortmerge_join_8.q,\
+ auto_sortmerge_join_9.q,\
+ bucket2.q,\
+ bucket3.q,\
+ bucket4.q,\
+ bucket_map_join_tez1.q,\
+ bucket_map_join_tez2.q,\
+ cbo_gby.q,\
+ cbo_gby_empty.q,\
+ cbo_join.q,\
+ cbo_limit.q,\
+ cbo_semijoin.q,\
+ cbo_simple_select.q,\
+ cbo_stats.q,\
+ cbo_subq_exists.q,\
+ cbo_subq_in.q,\
+ cbo_subq_not_in.q,\
+ cbo_udf_udaf.q,\
+ cbo_union.q,\
+ cbo_views.q,\
+ cbo_windowing.q,\
+ column_names_with_leading_and_trailing_spaces.q,\
+ constprog_dpp.q,\
+ constprog_semijoin.q,\
+ correlationoptimizer1.q,\
+ count.q,\
+ create_merge_compressed.q,\
+ cross_join.q,\
+ cross_product_check_1.q,\
+ cross_product_check_2.q,\
+ ctas.q,\
+ cte_1.q,\
+ cte_2.q,\
+ cte_3.q,\
+ cte_4.q,\
+ cte_5.q,\
+ cte_mat_1.q,\
+ cte_mat_2.q,\
+ cte_mat_3.q,\
+ cte_mat_4.q,\
+ cte_mat_5.q,\
+ custom_input_output_format.q,\
+ deleteAnalyze.q,\
+ delete_all_non_partitioned.q,\
+ delete_all_partitioned.q,\
+ delete_tmp_table.q,\
+ delete_where_no_match.q,\
+ delete_where_non_partitioned.q,\
+ delete_where_partitioned.q,\
+ delete_whole_partition.q,\
+ disable_merge_for_bucketing.q,\
+ dynamic_partition_pruning.q,\
+ dynamic_partition_pruning_2.q,\
+ dynpart_sort_opt_vectorization.q,\
+ dynpart_sort_optimization.q,\
+ dynpart_sort_optimization2.q,\
+ empty_join.q,\
+ enforce_order.q,\
+ filter_join_breaktask.q,\
+ filter_join_breaktask2.q,\
+ groupby1.q,\
+ groupby2.q,\
+ groupby3.q,\
+ having.q,\
+ identity_project_remove_skip.q,\
+ insert1.q,\
+ insert_into1.q,\
+ insert_into2.q,\
+ insert_orig_table.q,\
+ insert_update_delete.q,\
+ insert_values_dynamic_partitioned.q,\
+ insert_values_non_partitioned.q,\
+ insert_values_orig_table.,\
+ insert_values_partitioned.q,\
+ insert_values_tmp_table.q,\
+ join0.q,\
+ join1.q,\
+ join_nullsafe.q,\
+ leftsemijoin.q,\
+ limit_pushdown.q,\
+ llap_nullscan.q,\
+ llapdecider.q,\
+ load_dyn_part1.q,\
+ load_dyn_part2.q,\
+ load_dyn_part3.q,\
+ lvj_mapjoin.q,\
+ mapjoin2.q,\
+ mapjoin_decimal.q,\
+ mapjoin_mapjoin.q,\
+ mapreduce1.q,\
+ mapreduce2.q,\
+ merge1.q,\
+ merge2.q,\
+ mergejoin.q,\
+ metadata_only_queries.q,\
+ metadata_only_queries_with_filters.q,\
+ metadataonly1.q,\
+ mrr.q,\
+ nonmr_fetch_threshold.q,\
+ optimize_nullscan.q,\
+ orc_analyze.q,\
+ orc_merge1.q,\
+ orc_merge10.q,\
+ orc_merge11.q,\
+ orc_merge2.q,\
+ orc_merge3.q,\
+ orc_merge4.q,\
+ orc_merge5.q,\
+ orc_merge6.q,\
+ orc_merge7.q,\
+ orc_merge8.q,\
+ orc_merge9.q,\
+ orc_merge_incompat1.q,\
+ orc_merge_incompat2.q,\
+ orc_merge_incompat3.q,\
+ orc_ppd_basic.q,\
+ orc_ppd_schema_evol_1a.q,\
+ orc_ppd_schema_evol_1b.q,\
+ orc_ppd_schema_evol_2a.q,\
+ orc_ppd_schema_evol_2b.q,\
+ orc_ppd_schema_evol_3a.q,\
+ order_null.q,\
+ parallel.q,\
+ ptf.q,\
+ ptf_matchpath.q,\
+ ptf_streaming.q,\
+ sample1.q,\
+ script_env_var1.q,\
+ script_env_var2.q,\
+ script_pipe.q,\
+ scriptfile1.q,\
+ selectDistinctStar.q,\
+ select_dummy_source.q,\
+ skewjoin.q,\
+ stats_noscan_1.q,\
+ stats_only_null.q,\
+ subquery_exists.q,\
+ subquery_in.q,\
+ temp_table.q,\
+ tez_bmj_schema_evolution.q,\
+ tez_dml.q,\
+ tez_dynpart_hashjoin_1.q,\
+ tez_dynpart_hashjoin_2.q,\
+ tez_fsstat.q,\
+ tez_insert_overwrite_local_directory_1.q,\
+ tez_join.q,\
+ tez_join_hash.q,\
+ tez_join_result_complex.q,\
+ tez_join_tests.q,\
+ tez_joins_explain.q,\
+ tez_multi_union.q,\
+ tez_schema_evolution.q,\
+ tez_self_join.q,\
+ tez_smb_1.q,\
+ tez_smb_main.q,\
+ tez_union.q,\
+ tez_union2.q,\
+ tez_union_decimal.q,\
+ tez_union_dynamic_partition.q,\
+ tez_union_group_by.q,\
+ tez_union_multiinsert.q,\
+ tez_union_view.q,\
+ tez_vector_dynpart_hashjoin_1.q,\
+ tez_vector_dynpart_hashjoin_2.q,\
+ transform1.q,\
+ transform2.q,\
+ transform_ppr1.q,\
+ transform_ppr2.q,\
+ union2.q,\
+ union3.q,\
+ union4.q,\
+ union5.q,\
+ union6.q,\
+ union7.q,\
+ union8.q,\
+ union9.q,\
+ unionDistinct_1.q,\
+ union_fast_stats.q,\
+ union_stats.q,\
+ union_type_chk.q,\
+ update_after_multiple_inserts.q,\
+ update_all_non_partitioned.q,\
+ update_all_partitioned.q,\
+ update_all_types.q,\
+ update_tmp_table.q,\
+ update_two_cols.q,\
+ update_where_no_match.q,\
+ update_where_non_partitioned.q,\
+ update_where_partitioned.q,\
+ vector_acid3.q,\
+ vector_aggregate_9.q,\
+ vector_aggregate_without_gby.q,\
+ vector_auto_smb_mapjoin_14.q,\
+ vector_between_columns.q,\
+ vector_between_in.q,\
+ vector_binary_join_groupby.q,\
+ vector_bround.q,\
+ vector_bucket.q,\
+ vector_cast_constant.q,\
+ vector_char_2.q,\
+ vector_char_4.q,\
+ vector_char_cast.q,\
+ vector_char_mapjoin1.q,\
+ vector_char_simple.q,\
+ vector_coalesce.q,\
+ vector_coalesce_2.q,\
+ vector_complex_all.q,\
+ vector_complex_join.q,\
+ vector_count.q,\
+ vector_count_distinct.q,\
+ vector_data_types.q,\
+ vector_date_1.q,\
+ vector_decimal_1.q,\
+ vector_decimal_10_0.q,\
+ vector_decimal_2.q,\
+ vector_decimal_3.q,\
+ vector_decimal_4.q,\
+ vector_decimal_5.q,\
+ vector_decimal_6.q,\
+ vector_decimal_aggregate.q,\
+ vector_decimal_cast.q,\
+ vector_decimal_expressions.q,\
+ vector_decimal_mapjoin.q,\
+ vector_decimal_math_funcs.q,\
+ vector_decimal_precision.q,\
+ vector_decimal_round.q,\
+ vector_decimal_round_2.q,\
+ vector_decimal_trailing.q,\
+ vector_decimal_udf.q,\
+ vector_decimal_udf2.q,\
+ vector_distinct_2.q,\
+ vector_elt.q,\
+ vector_groupby4.q,\
+ vector_groupby6.q,\
+ vector_groupby_3.q,\
+ vector_groupby_mapjoin.q,\
+ vector_groupby_reduce.q,\
+ vector_grouping_sets.q,\
+ vector_if_expr.q,\
+ vector_include_no_sel.q,\
+ vector_inner_join.q,\
+ vector_interval_1.q,\
+ vector_interval_2.q,\
+ vector_interval_arithmetic.q,\
+ vector_interval_mapjoin.q,\
+ vector_join30.q,\
+ vector_join_filters.q,\
+ vector_join_nulls.q,\
+ vector_left_outer_join.q,\
+ vector_left_outer_join2.q,\
+ vector_leftsemi_mapjoin.q,\
+ vector_mapjoin_reduce.q,\
+ vector_mr_diff_schema_alias.q,\
+ vector_multi_insert.q,\
+ vector_null_projection.q,\
+ vector_nullsafe_join.q,\
+ vector_nvl.q,\
+ vector_orderby_5.q,\
+ vector_outer_join0.q,\
+ vector_outer_join1.q,\
+ vector_outer_join2.q,\
+ vector_outer_join3.q,\
+ vector_outer_join4.q,\
+ vector_outer_join5.q,\
+ vector_outer_join6.q,\
+ vector_partition_diff_num_cols.q,\
+ vector_partitioned_date_time.q,\
+ vector_reduce1.q,\
+ vector_reduce2.q,\
+ vector_reduce3.q,\
+ vector_reduce_groupby_decimal.q,\
+ vector_string_concat.q,\
+ vector_struct_in.q,\
+ vector_varchar_4.q,\
+ vector_varchar_mapjoin1.q,\
+ vector_varchar_simple.q,\
+ vector_when_case_null.q,\
+ vectorization_0.q,\
+ vectorization_1.q,\
+ vectorization_10.q,\
+ vectorization_11.q,\
+ vectorization_12.q,\
+ vectorization_13.q,\
+ vectorization_14.q,\
+ vectorization_15.q,\
+ vectorization_16.q,\
+ vectorization_17.q,\
+ vectorization_2.q,\
+ vectorization_3.q,\
+ vectorization_4.q,\
+ vectorization_5.q,\
+ vectorization_6.q,\
+ vectorization_7.q,\
+ vectorization_8.q,\
+ vectorization_9.q,\
+ vectorization_decimal_date.q,\
+ vectorization_nested_udf.q,\
+ vectorization_not.q,\
+ vectorization_part.q,\
+ vectorization_part_project.q,\
+ vectorization_part_varchar.q,\
+ vectorization_pushdown.q,\
+ vectorization_short_regress.q,\
+ vectorized_bucketmapjoin1.q,\
+ vectorized_case.q,\
+ vectorized_casts.q,\
+ vectorized_context.q,\
+ vectorized_date_funcs.q,\
+ vectorized_distinct_gby.q,\
+ vectorized_dynamic_partition_pruning.q,\
+ vectorized_mapjoin.q,\
+ vectorized_math_funcs.q,\
+ vectorized_nested_mapjoin.q,\
+ vectorized_parquet.q,\
+ vectorized_parquet_types.q,\
+ vectorized_ptf.q,\
+ vectorized_rcfile_columnar.q,\
+ vectorized_shufflejoin.q,\
+ vectorized_string_funcs.q,\
+ vectorized_timestamp.q,\
+ vectorized_timestamp_funcs.q,\
+ vectorized_timestamp_ints_casts.q
+
+minillap.query.files=acid_bucket_pruning.q,\
+ acid_vectorization_missing_cols.q,\
+ bucket_map_join_tez1.q,\
+ bucket_map_join_tez2.q,\
+ bucketpruning1.q,\
+ constprog_dpp.q,\
+ dynamic_partition_pruning.q,\
+ dynamic_partition_pruning_2.q,\
+ explainuser_1.q,\
+ explainuser_2.q,\
+ explainuser_4.q,\
+ hybridgrace_hashjoin_1.q,\
+ hybridgrace_hashjoin_2.q,\
+ llap_nullscan.q,\
+ llap_udf.q,\
+ llapdecider.q,\
+ lvj_mapjoin.q,\
+ mapjoin_decimal.q,\
+ mergejoin_3way.q,\
+ mrr.q,\
+ orc_llap.q,\
+ orc_llap_counters.q,\
+ orc_llap_counters1.q,\
+ orc_llap_nonvector.q,\
+ orc_merge_diff_fs.q,\
+ orc_ppd_basic.q,\
+ schema_evol_orc_acid_part.q,\
+ schema_evol_orc_acid_part_update.q,\
+ schema_evol_orc_acid_table.q,\
+ schema_evol_orc_acid_table_update.q,\
+ schema_evol_orc_acidvec_part.q,\
+ schema_evol_orc_acidvec_part_update.q,\
+ schema_evol_orc_acidvec_table.q,\
+ schema_evol_orc_acidvec_table_update.q,\
+ schema_evol_orc_nonvec_part.q,\
+ schema_evol_orc_nonvec_part_all_complex.q,\
+ schema_evol_orc_nonvec_part_all_primitive.q,\
+ schema_evol_orc_nonvec_table.q,\
+ schema_evol_orc_vec_part.q,\
+ schema_evol_orc_vec_part_all_complex.q,\
+ schema_evol_orc_vec_part_all_primitive.q,\
+ schema_evol_orc_vec_table.q,\
+ schema_evol_stats.q,\
+ schema_evol_text_nonvec_part.q,\
+ schema_evol_text_nonvec_part_all_complex.q,\
+ schema_evol_text_nonvec_part_all_primitive.q,\
+ schema_evol_text_nonvec_table.q,\
+ schema_evol_text_vec_part.q,\
+ schema_evol_text_vec_part_all_complex.q,\
+ schema_evol_text_vec_part_all_primitive.q,\
+ schema_evol_text_vec_table.q,\
+ schema_evol_text_vecrow_part.q,\
+ schema_evol_text_vecrow_part_all_complex.q,\
+ schema_evol_text_vecrow_part_all_primitive.q,\
+ schema_evol_text_vecrow_table.q,\
+ smb_cache.q,\
+ tez_aggr_part_stats.q,\
+ tez_bmj_schema_evolution.q,\
+ tez_dml.q,\
+ tez_dynpart_hashjoin_1.q,\
+ tez_dynpart_hashjoin_2.q,\
+ tez_dynpart_hashjoin_3.q,\
+ tez_fsstat.q,\
+ tez_insert_overwrite_local_directory_1.q,\
+ tez_join.q,\
+ tez_join_result_complex.q,\
+ tez_join_tests.q,\
+ tez_joins_explain.q,\
+ tez_multi_union.q,\
+ tez_schema_evolution.q,\
+ tez_self_join.q,\
+ tez_smb_1.q,\
+ tez_smb_empty.q,\
+ tez_smb_main.q,\
+ tez_union.q,\
+ tez_union2.q,\
+ tez_union_decimal.q,\
+ tez_union_dynamic_partition.q,\
+ tez_union_group_by.q,\
+ tez_union_multiinsert.q,\
+ tez_union_view.q,\
+ tez_vector_dynpart_hashjoin_1.q,\
+ tez_vector_dynpart_hashjoin_2.q,\
+ vectorized_dynamic_partition_pruning.q,\
+ windowing_gby.q
+
+encrypted.query.files=encryption_join_unencrypted_tbl.q,\
+ encryption_insert_partition_static.q,\
+ encryption_insert_partition_dynamic.q,\
+ encryption_join_with_different_encryption_keys.q,\
+ encryption_select_read_only_encrypted_tbl.q,\
+ encryption_select_read_only_unencrypted_tbl.q,\
+ encryption_load_data_to_encrypted_tables.q, \
+ encryption_unencrypted_nonhdfs_external_tables.q \
+ encryption_move_tbl.q \
+ encryption_drop_table.q \
+ encryption_insert_values.q \
+ encryption_drop_view.q \
+ encryption_drop_partition.q \
+ encryption_with_trash.q \
+ encryption_ctas.q
+
+beeline.positive.exclude=add_part_exist.q,\
+ alter1.q,\
+ alter2.q,\
+ alter4.q,\
+ alter5.q,\
+ alter_rename_partition.q,\
+ alter_rename_partition_authorization.q,\
+ archive.q,\
+ archive_corrupt.q,\
+ archive_mr_1806.q,\
+ archive_multi.q,\
+ archive_multi_mr_1806.q,\
+ authorization_1.q,\
+ authorization_2.q,\
+ authorization_4.q,\
+ authorization_5.q,\
+ authorization_6.q,\
+ authorization_7.q,\
+ ba_table1.q,\
+ ba_table2.q,\
+ ba_table3.q,\
+ ba_table_udfs.q,\
+ binary_table_bincolserde.q,\
+ binary_table_colserde.q,\
+ cluster.q,\
+ columnarserde_create_shortcut.q,\
+ combine2.q,\
+ constant_prop.q,\
+ create_nested_type.q,\
+ create_or_replace_view.q,\
+ create_struct_table.q,\
+ create_union_table.q,\
+ database.q,\
+ database_location.q,\
+ database_properties.q,\
+ describe_database_json.q,\
+ drop_database_removes_partition_dirs.q,\
+ escape1.q,\
+ escape2.q,\
+ exim_00_nonpart_empty.q,\
+ exim_01_nonpart.q,\
+ exim_02_00_part_empty.q,\
+ exim_02_part.q,\
+ exim_03_nonpart_over_compat.q,\
+ exim_04_all_part.q,\
+ exim_04_evolved_parts.q,\
+ exim_05_some_part.q,\
+ exim_06_one_part.q,\
+ exim_07_all_part_over_nonoverlap.q,\
+ exim_08_nonpart_rename.q,\
+ exim_09_part_spec_nonoverlap.q,\
+ exim_10_external_managed.q,\
+ exim_11_managed_external.q,\
+ exim_12_external_location.q,\
+ exim_13_managed_location.q,\
+ exim_14_managed_location_over_existing.q,\
+ exim_15_external_part.q,\
+ exim_16_part_external.q,\
+ exim_17_part_managed.q,\
+ exim_18_part_external.q,\
+ exim_19_00_part_external_location.q,\
+ exim_19_part_external_location.q,\
+ exim_20_part_managed_location.q,\
+ exim_21_export_authsuccess.q,\
+ exim_22_import_exist_authsuccess.q,\
+ exim_23_import_part_authsuccess.q,\
+ exim_24_import_nonexist_authsuccess.q,\
+ global_limit.q,\
+ groupby_complex_types.q,\
+ groupby_complex_types_multi_single_reducer.q,\
+ index_auth.q,\
+ index_auto.q,\
+ index_auto_empty.q,\
+ index_bitmap.q,\
+ index_bitmap1.q,\
+ index_bitmap2.q,\
+ index_bitmap3.q,\
+ index_bitmap_auto.q,\
+ index_bitmap_rc.q,\
+ index_compact.q,\
+ index_compact_1.q,\
+ index_compact_2.q,\
+ index_compact_3.q,\
+ index_stale_partitioned.q,\
+ init_file.q,\
+ input16.q,\
+ input16_cc.q,\
+ input46.q,\
+ input_columnarserde.q,\
+ input_dynamicserde.q,\
+ input_lazyserde.q,\
+ input_testxpath3.q,\
+ input_testxpath4.q,\
+ insert2_overwrite_partitions.q,\
+ insertexternal1.q,\
+ join_thrift.q,\
+ lateral_view.q,\
+ load_binary_data.q,\
+ load_exist_part_authsuccess.q,\
+ load_nonpart_authsuccess.q,\
+ load_part_authsuccess.q,\
+ loadpart_err.q,\
+ lock1.q,\
+ lock2.q,\
+ lock3.q,\
+ lock4.q,\
+ merge_dynamic_partition.q,\
+ multi_insert.q,\
+ multi_insert_move_tasks_share_dependencies.q,\
+ null_column.q,\
+ ppd_clusterby.q,\
+ query_with_semi.q,\
+ rename_column.q,\
+ sample6.q,\
+ sample_islocalmode_hook.q,\
+ set_processor_namespaces.q,\
+ show_tables.q,\
+ source.q,\
+ split_sample.q,\
+ str_to_map.q,\
+ transform1.q,\
+ udaf_collect_set.q,\
+ udaf_context_ngrams.q,\
+ udaf_histogram_numeric.q,\
+ udaf_ngrams.q,\
+ udaf_percentile_approx.q,\
+ udf_array.q,\
+ udf_bitmap_and.q,\
+ udf_bitmap_or.q,\
+ udf_explode.q,\
+ udf_format_number.q,\
+ udf_map.q,\
+ udf_map_keys.q,\
+ udf_map_values.q,\
+ udf_mask.q,\
+ udf_mask_first_n.q,\
+ udf_mask_hash.q,\
+ udf_mask_last_n.q,\
+ udf_mask_show_first_n.q,\
+ udf_mask_show_last_n.q,\
+ udf_max.q,\
+ udf_min.q,\
+ udf_named_struct.q,\
+ udf_percentile.q,\
+ udf_printf.q,\
+ udf_sentences.q,\
+ udf_sort_array.q,\
+ udf_split.q,\
+ udf_struct.q,\
+ udf_substr.q,\
+ udf_translate.q,\
+ udf_union.q,\
+ udf_xpath.q,\
+ udtf_stack.q,\
+ view.q,\
+ virtual_column.q
+
+minimr.query.negative.files=cluster_tasklog_retrieval.q,\
+ file_with_header_footer_negative.q,\
+ local_mapred_error_cache.q,\
+ mapreduce_stack_trace.q,\
+ mapreduce_stack_trace_hadoop20.q,\
+ mapreduce_stack_trace_turnoff.q,\
+ mapreduce_stack_trace_turnoff_hadoop20.q,\
+ minimr_broken_pipe.q,\
+ table_nonprintable_negative.q,\
+ udf_local_resource.q
+
+# tests are sorted use: perl -pe 's@\\\s*\n@ @g' testconfiguration.properties \
+# | awk -F= '/spark.query.files/{print $2}' | perl -pe 's@.q *, *@\n@g' \
+# | egrep -v '^ *$' | sort -V | uniq | perl -pe 's@\n@.q, \\\n@g' | perl -pe 's@^@ @g'
+spark.query.files=add_part_multiple.q, \
+ alter_merge_orc.q, \
+ alter_merge_stats_orc.q, \
+ annotate_stats_join.q, \
+ auto_join0.q, \
+ auto_join1.q, \
+ auto_join10.q, \
+ auto_join11.q, \
+ auto_join12.q, \
+ auto_join13.q, \
+ auto_join14.q, \
+ auto_join15.q, \
+ auto_join16.q, \
+ auto_join17.q, \
+ auto_join18.q, \
+ auto_join18_multi_distinct.q, \
+ auto_join19.q, \
+ auto_join2.q, \
+ auto_join20.q, \
+ auto_join21.q, \
+ auto_join22.q, \
+ auto_join23.q, \
+ auto_join24.q, \
+ auto_join26.q, \
+ auto_join27.q, \
+ auto_join28.q, \
+ auto_join29.q, \
+ auto_join3.q, \
+ auto_join30.q, \
+ auto_join31.q, \
+ auto_join4.q, \
+ auto_join5.q, \
+ auto_join6.q, \
+ auto_join7.q, \
+ auto_join8.q, \
+ auto_join9.q, \
+ auto_join_filters.q, \
+ auto_join_nulls.q, \
+ auto_join_reordering_values.q, \
+ auto_join_stats.q, \
+ auto_join_stats2.q, \
+ auto_join_without_localtask.q, \
+ auto_smb_mapjoin_14.q, \
+ auto_sortmerge_join_1.q, \
+ auto_sortmerge_join_10.q, \
+ auto_sortmerge_join_12.q, \
+ auto_sortmerge_join_13.q, \
+ auto_sortmerge_join_14.q, \
+ auto_sortmerge_join_15.q, \
+ auto_sortmerge_join_16.q, \
+ auto_sortmerge_join_3.q, \
+ auto_sortmerge_join_4.q, \
+ auto_sortmerge_join_5.q, \
+ auto_sortmerge_join_6.q, \
+ auto_sortmerge_join_7.q, \
+ auto_sortmerge_join_8.q, \
+ auto_sortmerge_join_9.q, \
+ avro_compression_enabled_native.q, \
+ avro_decimal_native.q, \
+ avro_joins.q, \
+ avro_joins_native.q, \
+ bucket2.q, \
+ bucket3.q, \
+ bucket4.q, \
+ bucket_map_join_1.q, \
+ bucket_map_join_2.q, \
+ bucket_map_join_spark1.q, \
+ bucket_map_join_spark2.q, \
+ bucket_map_join_spark3.q, \
+ bucket_map_join_spark4.q, \
+ bucket_map_join_tez1.q, \
+ bucket_map_join_tez2.q, \
+ bucketmapjoin1.q, \
+ bucketmapjoin10.q, \
+ bucketmapjoin11.q, \
+ bucketmapjoin12.q, \
+ bucketmapjoin13.q, \
+ bucketmapjoin2.q, \
+ bucketmapjoin3.q, \
+ bucketmapjoin4.q, \
+ bucketmapjoin5.q, \
+ bucketmapjoin7.q, \
+ bucketmapjoin8.q, \
+ bucketmapjoin9.q, \
+ bucketmapjoin_negative.q, \
+ bucketmapjoin_negative2.q, \
+ bucketmapjoin_negative3.q, \
+ bucketsortoptimize_insert_2.q, \
+ bucketsortoptimize_insert_4.q, \
+ bucketsortoptimize_insert_6.q, \
+ bucketsortoptimize_insert_7.q, \
+ bucketsortoptimize_insert_8.q, \
+ cbo_gby.q, \
+ cbo_gby_empty.q, \
+ cbo_limit.q, \
+ cbo_semijoin.q, \
+ cbo_simple_select.q, \
+ cbo_stats.q, \
+ cbo_subq_in.q, \
+ cbo_subq_not_in.q, \
+ cbo_udf_udaf.q, \
+ cbo_union.q, \
+ column_access_stats.q, \
+ count.q, \
+ create_merge_compressed.q, \
+ cross_join.q, \
+ cross_product_check_1.q, \
+ cross_product_check_2.q, \
+ ctas.q, \
+ custom_input_output_format.q, \
+ date_join1.q, \
+ date_udf.q, \
+ decimal_1_1.q, \
+ decimal_join.q, \
+ disable_merge_for_bucketing.q, \
+ dynamic_rdd_cache.q, \
+ enforce_order.q, \
+ escape_clusterby1.q, \
+ escape_distributeby1.q, \
+ escape_orderby1.q, \
+ escape_sortby1.q, \
+ filter_join_breaktask.q, \
+ filter_join_breaktask2.q, \
+ groupby1.q, \
+ groupby10.q, \
+ groupby11.q, \
+ groupby1_map.q, \
+ groupby1_map_nomap.q, \
+ groupby1_map_skew.q, \
+ groupby1_noskew.q, \
+ groupby2.q, \
+ groupby2_map.q, \
+ groupby2_map_multi_distinct.q, \
+ groupby2_map_skew.q, \
+ groupby2_noskew.q, \
+ groupby2_noskew_multi_distinct.q, \
+ groupby3.q, \
+ groupby3_map.q, \
+ groupby3_map_multi_distinct.q, \
+ groupby3_map_skew.q, \
+ groupby3_noskew.q, \
+ groupby3_noskew_multi_distinct.q, \
+ groupby4.q, \
+ groupby4_map.q, \
+ groupby4_map_skew.q, \
+ groupby4_noskew.q, \
+ groupby5.q, \
+ groupby5_map.q, \
+ groupby5_map_skew.q, \
+ groupby5_noskew.q, \
+ groupby6.q, \
+ groupby6_map.q, \
+ groupby6_map_skew.q, \
+ groupby6_noskew.q, \
+ groupby7.q, \
+ groupby7_map.q, \
+ groupby7_map_multi_single_reducer.q, \
+ groupby7_map_skew.q, \
+ groupby7_noskew.q, \
+ groupby7_noskew_multi_single_reducer.q, \
+ groupby8.q, \
+ groupby8_map.q, \
+ groupby8_map_skew.q, \
+ groupby8_noskew.q, \
+ groupby9.q, \
+ groupby_bigdata.q, \
+ groupby_complex_types.q, \
+ groupby_complex_types_multi_single_reducer.q, \
+ groupby_cube1.q, \
+ groupby_grouping_id2.q, \
+ groupby_map_ppr.q, \
+ groupby_map_ppr_multi_distinct.q, \
+ groupby_multi_insert_common_distinct.q, \
+ groupby_multi_single_reducer.q, \
+ groupby_multi_single_reducer2.q, \
+ groupby_multi_single_reducer3.q, \
+ groupby_position.q, \
+ groupby_ppr.q, \
+ groupby_ppr_multi_distinct.q, \
+ groupby_resolution.q, \
+ groupby_rollup1.q, \
+ groupby_sort_1_23.q, \
+ groupby_sort_skew_1.q, \
+ groupby_sort_skew_1_23.q, \
+ qroupby_limit_extrastep.q, \
+ having.q, \
+ identity_project_remove_skip.q, \
+ index_auto_self_join.q, \
+ innerjoin.q, \
+ input12.q, \
+ input13.q, \
+ input14.q, \
+ input17.q, \
+ input18.q, \
+ input1_limit.q, \
+ input_part2.q, \
+ insert_into1.q, \
+ insert_into2.q, \
+ insert_into3.q, \
+ join0.q, \
+ join1.q, \
+ join10.q, \
+ join11.q, \
+ join12.q, \
+ join13.q, \
+ join14.q, \
+ join15.q, \
+ join16.q, \
+ join17.q, \
+ join18.q, \
+ join18_multi_distinct.q, \
+ join19.q, \
+ join2.q, \
+ join20.q, \
+ join21.q, \
+ join22.q, \
+ join23.q, \
+ join24.q, \
+ join25.q, \
+ join26.q, \
+ join27.q, \
+ join28.q, \
+ join29.q, \
+ join3.q, \
+ join30.q, \
+ join31.q, \
+ join32.q, \
+ join32_lessSize.q, \
+ join33.q, \
+ join34.q, \
+ join35.q, \
+ join36.q, \
+ join37.q, \
+ join38.q, \
+ join39.q, \
+ join4.q, \
+ join41.q, \
+ join5.q, \
+ join6.q, \
+ join7.q, \
+ join8.q, \
+ join9.q, \
+ join_1to1.q, \
+ join_alt_syntax.q, \
+ join_array.q, \
+ join_casesensitive.q, \
+ join_cond_pushdown_1.q, \
+ join_cond_pushdown_2.q, \
+ join_cond_pushdown_3.q, \
+ join_cond_pushdown_4.q, \
+ join_cond_pushdown_unqual1.q, \
+ join_cond_pushdown_unqual2.q, \
+ join_cond_pushdown_unqual3.q, \
+ join_cond_pushdown_unqual4.q, \
+ join_filters_overlap.q, \
+ join_hive_626.q, \
+ join_literals.q, \
+ join_map_ppr.q, \
+ join_merge_multi_expressions.q, \
+ join_merging.q, \
+ join_nullsafe.q, \
+ join_rc.q, \
+ join_reorder.q, \
+ join_reorder2.q, \
+ join_reorder3.q, \
+ join_reorder4.q, \
+ join_star.q, \
+ join_thrift.q, \
+ join_vc.q, \
+ join_view.q, \
+ lateral_view_explode2.q, \
+ leftsemijoin.q, \
+ leftsemijoin_mr.q, \
+ limit_partition_metadataonly.q, \
+ limit_pushdown.q, \
+ list_bucket_dml_2.q, \
+ load_dyn_part1.q, \
+ load_dyn_part10.q, \
+ load_dyn_part11.q, \
+ load_dyn_part12.q, \
+ load_dyn_part13.q, \
+ load_dyn_part14.q, \
+ load_dyn_part15.q, \
+ load_dyn_part2.q, \
+ load_dyn_part3.q, \
+ load_dyn_part4.q, \
+ load_dyn_part5.q, \
+ load_dyn_part6.q, \
+ load_dyn_part7.q, \
+ load_dyn_part8.q, \
+ load_dyn_part9.q, \
+ louter_join_ppr.q, \
+ mapjoin1.q, \
+ mapjoin_addjar.q, \
+ mapjoin_decimal.q, \
+ mapjoin_distinct.q, \
+ mapjoin_filter_on_outerjoin.q, \
+ mapjoin_mapjoin.q, \
+ mapjoin_memcheck.q, \
+ mapjoin_subquery.q, \
+ mapjoin_subquery2.q, \
+ mapjoin_test_outer.q, \
+ mapreduce1.q, \
+ mapreduce2.q, \
+ merge1.q, \
+ merge2.q, \
+ mergejoins.q, \
+ mergejoins_mixed.q, \
+ metadata_only_queries.q, \
+ metadata_only_queries_with_filters.q, \
+ multi_insert.q, \
+ multi_insert_gby.q, \
+ multi_insert_gby2.q, \
+ multi_insert_gby3.q, \
+ multi_insert_lateral_view.q, \
+ multi_insert_mixed.q, \
+ multi_insert_move_tasks_share_dependencies.q, \
+ multi_insert_with_join.q, \
+ multi_join_union.q, \
+ multi_join_union_src.q, \
+ multigroupby_singlemr.q, \
+ nullgroup.q, \
+ nullgroup2.q, \
+ nullgroup4.q, \
+ nullgroup4_multi_distinct.q, \
+ optimize_nullscan.q, \
+ order.q, \
+ order2.q, \
+ outer_join_ppr.q, \
+ parallel.q, \
+ parallel_join0.q, \
+ parallel_join1.q, \
+ parquet_join.q, \
+ pcr.q, \
+ ppd_gby_join.q, \
+ ppd_join.q, \
+ ppd_join2.q, \
+ ppd_join3.q, \
+ ppd_join5.q, \
+ ppd_join_filter.q, \
+ ppd_multi_insert.q, \
+ ppd_outer_join1.q, \
+ ppd_outer_join2.q, \
+ ppd_outer_join3.q, \
+ ppd_outer_join4.q, \
+ ppd_outer_join5.q, \
+ ppd_transform.q, \
+ ptf.q, \
+ ptf_decimal.q, \
+ ptf_general_queries.q, \
+ ptf_matchpath.q, \
+ ptf_rcfile.q, \
+ ptf_register_tblfn.q, \
+ ptf_seqfile.q, \
+ ptf_streaming.q, \
+ rcfile_bigdata.q, \
+ reduce_deduplicate_exclude_join.q, \
+ router_join_ppr.q, \
+ runtime_skewjoin_mapjoin_spark.q, \
+ sample1.q, \
+ sample10.q, \
+ sample2.q, \
+ sample3.q, \
+ sample4.q, \
+ sample5.q, \
+ sample6.q, \
+ sample7.q, \
+ sample8.q, \
+ sample9.q, \
+ script_env_var1.q, \
+ script_env_var2.q, \
+ script_pipe.q, \
+ scriptfile1.q, \
+ semijoin.q, \
+ skewjoin.q, \
+ skewjoin_noskew.q, \
+ skewjoin_union_remove_1.q, \
+ skewjoin_union_remove_2.q, \
+ skewjoinopt1.q, \
+ skewjoinopt10.q, \
+ skewjoinopt11.q, \
+ skewjoinopt12.q, \
+ skewjoinopt13.q, \
+ skewjoinopt14.q, \
+ skewjoinopt15.q, \
+ skewjoinopt16.q, \
+ skewjoinopt17.q, \
+ skewjoinopt18.q, \
+ skewjoinopt19.q, \
+ skewjoinopt2.q, \
+ skewjoinopt20.q, \
+ skewjoinopt3.q, \
+ skewjoinopt4.q, \
+ skewjoinopt5.q, \
+ skewjoinopt6.q, \
+ skewjoinopt7.q, \
+ skewjoinopt8.q, \
+ skewjoinopt9.q, \
+ smb_mapjoin_1.q, \
+ smb_mapjoin_10.q, \
+ smb_mapjoin_11.q, \
+ smb_mapjoin_12.q, \
+ smb_mapjoin_13.q, \
+ smb_mapjoin_14.q, \
+ smb_mapjoin_15.q, \
+ smb_mapjoin_16.q, \
+ smb_mapjoin_17.q, \
+ smb_mapjoin_18.q, \
+ smb_mapjoin_19.q, \
+ smb_mapjoin_2.q, \
+ smb_mapjoin_20.q, \
+ smb_mapjoin_21.q, \
+ smb_mapjoin_22.q, \
+ smb_mapjoin_25.q, \
+ smb_mapjoin_3.q, \
+ smb_mapjoin_4.q, \
+ smb_mapjoin_5.q, \
+ smb_mapjoin_6.q, \
+ smb_mapjoin_7.q, \
+ smb_mapjoin_8.q, \
+ smb_mapjoin_9.q, \
+ sort.q, \
+ stats0.q, \
+ stats1.q, \
+ stats10.q, \
+ stats12.q, \
+ stats13.q, \
+ stats14.q, \
+ stats15.q, \
+ stats16.q, \
+ stats18.q, \
+ stats2.q, \
+ stats3.q, \
+ stats5.q, \
+ stats6.q, \
+ stats7.q, \
+ stats8.q, \
+ stats9.q, \
+ stats_noscan_1.q, \
+ stats_noscan_2.q, \
+ stats_only_null.q, \
+ stats_partscan_1_23.q, \
+ statsfs.q, \
+ subquery_exists.q, \
+ subquery_in.q, \
+ subquery_multiinsert.q, \
+ table_access_keys_stats.q, \
+ temp_table.q, \
+ temp_table_gb1.q, \
+ temp_table_join1.q, \
+ tez_join_tests.q, \
+ tez_joins_explain.q, \
+ timestamp_1.q, \
+ timestamp_2.q, \
+ timestamp_3.q, \
+ timestamp_comparison.q, \
+ timestamp_lazy.q, \
+ timestamp_null.q, \
+ timestamp_udf.q, \
+ transform2.q, \
+ transform_ppr1.q, \
+ transform_ppr2.q, \
+ udaf_collect_set.q, \
+ udf_example_add.q, \
+ udf_in_file.q, \
+ udf_max.q, \
+ udf_min.q, \
+ udf_percentile.q, \
+ union.q, \
+ union10.q, \
+ union11.q, \
+ union12.q, \
+ union13.q, \
+ union14.q, \
+ union15.q, \
+ union16.q, \
+ union17.q, \
+ union18.q, \
+ union19.q, \
+ union2.q, \
+ union20.q, \
+ union21.q, \
+ union22.q, \
+ union23.q, \
+ union24.q, \
+ union25.q, \
+ union26.q, \
+ union27.q, \
+ union28.q, \
+ union29.q, \
+ union3.q, \
+ union30.q, \
+ union31.q, \
+ union32.q, \
+ union33.q, \
+ union34.q, \
+ union4.q, \
+ union5.q, \
+ union6.q, \
+ union7.q, \
+ union8.q, \
+ union9.q, \
+ union_date.q, \
+ union_date_trim.q, \
+ union_lateralview.q, \
+ union_null.q, \
+ union_ppr.q, \
+ union_remove_1.q, \
+ union_remove_10.q, \
+ union_remove_11.q, \
+ union_remove_12.q, \
+ union_remove_13.q, \
+ union_remove_14.q, \
+ union_remove_15.q, \
+ union_remove_16.q, \
+ union_remove_17.q, \
+ union_remove_18.q, \
+ union_remove_19.q, \
+ union_remove_2.q, \
+ union_remove_20.q, \
+ union_remove_21.q, \
+ union_remove_22.q, \
+ union_remove_23.q, \
+ union_remove_24.q, \
+ union_remove_25.q, \
+ union_remove_3.q, \
+ union_remove_4.q, \
+ union_remove_5.q, \
+ union_remove_6.q, \
+ union_remove_6_subq.q, \
+ union_remove_7.q, \
+ union_remove_8.q, \
+ union_remove_9.q, \
+ union_script.q, \
+ union_top_level.q, \
+ union_view.q, \
+ uniquejoin.q, \
+ varchar_join1.q, \
+ vector_between_in.q, \
+ vector_cast_constant.q, \
+ vector_char_4.q, \
+ vector_count_distinct.q, \
+ vector_data_types.q, \
+ vector_decimal_aggregate.q, \
+ vector_decimal_mapjoin.q, \
+ vector_distinct_2.q, \
+ vector_elt.q, \
+ vector_groupby_3.q, \
+ vector_left_outer_join.q, \
+ vector_mapjoin_reduce.q, \
+ vector_orderby_5.q, \
+ vector_string_concat.q, \
+ vector_varchar_4.q, \
+ vectorization_0.q, \
+ vectorization_1.q, \
+ vectorization_10.q, \
+ vectorization_11.q, \
+ vectorization_12.q, \
+ vectorization_13.q, \
+ vectorization_14.q, \
+ vectorization_15.q, \
+ vectorization_16.q, \
+ vectorization_17.q, \
+ vectorization_2.q, \
+ vectorization_3.q, \
+ vectorization_4.q, \
+ vectorization_5.q, \
+ vectorization_6.q, \
+ vectorization_9.q, \
+ vectorization_decimal_date.q, \
+ vectorization_div0.q, \
+ vectorization_nested_udf.q, \
+ vectorization_not.q, \
+ vectorization_part.q, \
+ vectorization_part_project.q, \
+ vectorization_pushdown.q, \
+ vectorization_short_regress.q, \
+ vectorized_case.q, \
+ vectorized_mapjoin.q, \
+ vectorized_math_funcs.q, \
+ vectorized_nested_mapjoin.q, \
+ vectorized_ptf.q, \
+ vectorized_rcfile_columnar.q, \
+ vectorized_shufflejoin.q, \
+ vectorized_string_funcs.q, \
+ vectorized_timestamp_funcs.q, \
+ windowing.q
+
+# Unlike "spark.query.files" above, these tests only run
+# under Spark engine.
+spark.only.query.files=spark_dynamic_partition_pruning.q,\
+ spark_dynamic_partition_pruning_2.q,\
+ spark_vectorized_dynamic_partition_pruning.q
+
+miniSparkOnYarn.query.files=auto_sortmerge_join_16.q,\
+ bucket4.q,\
+ bucket5.q,\
+ bucket6.q,\
+ bucketizedhiveinputformat.q,\
+ bucketmapjoin6.q,\
+ bucketmapjoin7.q,\
+ constprog_partitioner.q,\
+ constprog_semijoin.q,\
+ disable_merge_for_bucketing.q,\
+ empty_dir_in_table.q,\
+ external_table_with_space_in_location_path.q,\
+ file_with_header_footer.q,\
+ gen_udf_example_add10.q,\
+ import_exported_table.q,\
+ index_bitmap3.q,\
+ index_bitmap_auto.q,\
+ infer_bucket_sort_bucketed_table.q,\
+ infer_bucket_sort_map_operators.q,\
+ infer_bucket_sort_merge.q,\
+ infer_bucket_sort_num_buckets.q,\
+ infer_bucket_sort_reducers_power_two.q,\
+ input16_cc.q,\
+ insert_overwrite_directory2.q,\
+ leftsemijoin_mr.q,\
+ list_bucket_dml_10.q,\
+ load_fs2.q,\
+ load_hdfs_file_with_space_in_the_name.q,\
+ orc_merge1.q,\
+ orc_merge2.q,\
+ orc_merge3.q,\
+ orc_merge4.q,\
+ orc_merge5.q,\
+ orc_merge6.q,\
+ orc_merge7.q,\
+ orc_merge8.q,\
+ orc_merge9.q,\
+ orc_merge_diff_fs.q,\
+ orc_merge_incompat1.q,\
+ orc_merge_incompat2.q,\
+ parallel_orderby.q,\
+ quotedid_smb.q,\
+ reduce_deduplicate.q,\
+ remote_script.q,\
+ root_dir_external_table.q,\
+ schemeAuthority.q,\
+ schemeAuthority2.q,\
+ scriptfile1.q,\
+ scriptfile1_win.q,\
+ temp_table_external.q,\
+ truncate_column_buckets.q,\
+ uber_reduce.q,\
+ vector_inner_join.q,\
+ vector_outer_join0.q,\
+ vector_outer_join1.q,\
+ vector_outer_join2.q,\
+ vector_outer_join3.q,\
+ vector_outer_join4.q,\
+ vector_outer_join5.q
+
+# These tests are removed from miniSparkOnYarn.query.files
+# ql_rewrite_gbtoidx.q,\
+# ql_rewrite_gbtoidx_cbo_1.q,\
+# smb_mapjoin_8.q,\
+
+
+spark.query.negative.files=groupby2_map_skew_multi_distinct.q,\
+ groupby2_multi_distinct.q,\
+ groupby3_map_skew_multi_distinct.q,\
+ groupby3_multi_distinct.q,\
+ groupby_grouping_sets7.q
http://git-wip-us.apache.org/repos/asf/hive/blob/fd72ec4a/ql/src/java/org/apache/hadoop/hive/ql/Context.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Context.java b/ql/src/java/org/apache/hadoop/hive/ql/Context.java
index 748c19a..f5befaf 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Context.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Context.java
@@ -50,6 +50,8 @@ import org.apache.hadoop.hive.ql.lockmgr.LockException;
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.parse.ASTNode;
import org.apache.hadoop.hive.ql.parse.HiveParser;
+import org.apache.hadoop.hive.ql.parse.ExplainConfiguration;
+import org.apache.hadoop.hive.ql.parse.ExplainConfiguration.AnalyzeState;
import org.apache.hadoop.hive.ql.plan.LoadTableDesc;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.shims.ShimLoader;
@@ -71,7 +73,7 @@ public class Context {
private int resDirFilesNum;
boolean initialized;
String originalTracker = null;
- private final CompilationOpContext opContext;
+ private CompilationOpContext opContext;
private final Map<String, ContentSummary> pathToCS = new ConcurrentHashMap<String, ContentSummary>();
// scratch path to use for all non-local (ie. hdfs) file system tmp folders
@@ -88,10 +90,9 @@ public class Context {
private final Configuration conf;
protected int pathid = 10000;
- protected boolean explain = false;
+ protected ExplainConfiguration explainConfig = null;
protected String cboInfo;
protected boolean cboSucceeded;
- protected boolean explainLogical = false;
protected String cmd = "";
// number of previous attempts
protected int tryCount = 0;
@@ -274,34 +275,25 @@ public class Context {
}
/**
- * Set the context on whether the current query is an explain query.
- * @param value true if the query is an explain query, false if not
+ * Find whether we should execute the current query due to explain
+ * @return true if the query needs to be executed, false if not
*/
- public void setExplain(boolean value) {
- explain = value;
- }
-
- /**
- * Find whether the current query is an explain query
- * @return true if the query is an explain query, false if not
- */
- public boolean getExplain() {
- return explain;
+ public boolean isExplainSkipExecution() {
+ return (explainConfig != null && explainConfig.getAnalyze() != AnalyzeState.RUNNING);
}
/**
* Find whether the current query is a logical explain query
*/
public boolean getExplainLogical() {
- return explainLogical;
+ return explainConfig != null && explainConfig.isLogical();
}
- /**
- * Set the context on whether the current query is a logical
- * explain query.
- */
- public void setExplainLogical(boolean explainLogical) {
- this.explainLogical = explainLogical;
+ public AnalyzeState getExplainAnalyze() {
+ if (explainConfig != null) {
+ return explainConfig.getAnalyze();
+ }
+ return null;
}
/**
@@ -448,7 +440,7 @@ public class Context {
// if we are executing entirely on the client side - then
// just (re)use the local scratch directory
if(isLocalOnlyExecutionMode()) {
- return getLocalScratchDir(!explain);
+ return getLocalScratchDir(!isExplainSkipExecution());
}
try {
@@ -456,7 +448,7 @@ public class Context {
URI uri = dir.toUri();
Path newScratchDir = getScratchDir(uri.getScheme(), uri.getAuthority(),
- !explain, uri.getPath());
+ !isExplainSkipExecution(), uri.getPath());
LOG.info("New scratch dir is " + newScratchDir);
return newScratchDir;
} catch (IOException e) {
@@ -468,7 +460,7 @@ public class Context {
}
private Path getExternalScratchDir(URI extURI) {
- return getStagingDir(new Path(extURI.getScheme(), extURI.getAuthority(), extURI.getPath()), !explain);
+ return getStagingDir(new Path(extURI.getScheme(), extURI.getAuthority(), extURI.getPath()), !isExplainSkipExecution());
}
/**
@@ -531,7 +523,7 @@ public class Context {
}
public Path getMRTmpPath(URI uri) {
- return new Path(getStagingDir(new Path(uri), !explain), MR_PREFIX + nextPathId());
+ return new Path(getStagingDir(new Path(uri), !isExplainSkipExecution()), MR_PREFIX + nextPathId());
}
/**
@@ -577,7 +569,7 @@ public class Context {
* path within /tmp
*/
public Path getExtTmpPathRelTo(Path path) {
- return new Path(getStagingDir(path, !explain), EXT_PREFIX + nextPathId());
+ return new Path(getStagingDir(path, !isExplainSkipExecution()), EXT_PREFIX + nextPathId());
}
/**
@@ -725,7 +717,7 @@ public class Context {
* the stream being used
*/
public void setTokenRewriteStream(TokenRewriteStream tokenRewriteStream) {
- assert (this.tokenRewriteStream == null);
+ assert (this.tokenRewriteStream == null || this.getExplainAnalyze() == AnalyzeState.RUNNING);
this.tokenRewriteStream = tokenRewriteStream;
}
@@ -908,4 +900,17 @@ public class Context {
public void setIsUpdateDeleteMerge(boolean isUpdate) {
this.isUpdateDeleteMerge = isUpdate;
}
+
+ public ExplainConfiguration getExplainConfig() {
+ return explainConfig;
+ }
+
+ public void setExplainConfig(ExplainConfiguration explainConfig) {
+ this.explainConfig = explainConfig;
+ }
+
+ public void resetOpContext(){
+ opContext = new CompilationOpContext();
+ sequencer = new AtomicInteger();
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/fd72ec4a/ql/src/java/org/apache/hadoop/hive/ql/Context.java.orig
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Context.java.orig b/ql/src/java/org/apache/hadoop/hive/ql/Context.java.orig
new file mode 100644
index 0000000..4667f68
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Context.java.orig
@@ -0,0 +1,829 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql;
+
+import java.io.DataInput;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.URI;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.antlr.runtime.TokenRewriteStream;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.ContentSummary;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hive.common.FileUtils;
+import org.apache.hadoop.hive.common.BlobStorageUtils;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.exec.TaskRunner;
+import org.apache.hadoop.hive.ql.exec.Utilities;
+import org.apache.hadoop.hive.ql.hooks.WriteEntity;
+import org.apache.hadoop.hive.ql.io.AcidUtils;
+import org.apache.hadoop.hive.ql.lockmgr.DbTxnManager.Heartbeater;
+import org.apache.hadoop.hive.ql.lockmgr.HiveLock;
+import org.apache.hadoop.hive.ql.lockmgr.HiveLockObj;
+import org.apache.hadoop.hive.ql.lockmgr.HiveTxnManager;
+import org.apache.hadoop.hive.ql.lockmgr.LockException;
+import org.apache.hadoop.hive.ql.metadata.Table;
+import org.apache.hadoop.hive.ql.plan.LoadTableDesc;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hadoop.hive.shims.ShimLoader;
+import org.apache.hadoop.util.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Context for Semantic Analyzers. Usage: not reusable - construct a new one for
+ * each query should call clear() at end of use to remove temporary folders
+ */
+public class Context {
+ private boolean isHDFSCleanup;
+ private Path resFile;
+ private Path resDir;
+ private FileSystem resFs;
+ private static final Logger LOG = LoggerFactory.getLogger("hive.ql.Context");
+ private Path[] resDirPaths;
+ private int resDirFilesNum;
+ boolean initialized;
+ String originalTracker = null;
+ private final CompilationOpContext opContext;
+ private final Map<String, ContentSummary> pathToCS = new ConcurrentHashMap<String, ContentSummary>();
+
+ // scratch path to use for all non-local (ie. hdfs) file system tmp folders
+ private final Path nonLocalScratchPath;
+
+ // scratch directory to use for local file system tmp folders
+ private final String localScratchDir;
+
+ // the permission to scratch directory (local and hdfs)
+ private final String scratchDirPermission;
+
+ // Keeps track of scratch directories created for different scheme/authority
+ private final Map<String, Path> fsScratchDirs = new HashMap<String, Path>();
+
+ private final Configuration conf;
+ protected int pathid = 10000;
+ protected boolean explain = false;
+ protected String cboInfo;
+ protected boolean cboSucceeded;
+ protected boolean explainLogical = false;
+ protected String cmd = "";
+ // number of previous attempts
+ protected int tryCount = 0;
+ private TokenRewriteStream tokenRewriteStream;
+
+ private String executionId;
+
+ // List of Locks for this query
+ protected List<HiveLock> hiveLocks;
+
+ // Transaction manager for this query
+ protected HiveTxnManager hiveTxnManager;
+
+ // Used to track what type of acid operation (insert, update, or delete) we are doing. Useful
+ // since we want to change where bucket columns are accessed in some operators and
+ // optimizations when doing updates and deletes.
+ private AcidUtils.Operation acidOperation = AcidUtils.Operation.NOT_ACID;
+
+ private boolean needLockMgr;
+
+ private AtomicInteger sequencer = new AtomicInteger();
+
+ private final Map<String, Table> cteTables = new HashMap<String, Table>();
+
+ // Keep track of the mapping from load table desc to the output and the lock
+ private final Map<LoadTableDesc, WriteEntity> loadTableOutputMap =
+ new HashMap<LoadTableDesc, WriteEntity>();
+ private final Map<WriteEntity, List<HiveLockObj>> outputLockObjects =
+ new HashMap<WriteEntity, List<HiveLockObj>>();
+
+ private final String stagingDir;
+
+ private Heartbeater heartbeater;
+
+ private boolean skipTableMasking;
+
+ public Context(Configuration conf) throws IOException {
+ this(conf, generateExecutionId());
+ }
+
+ /**
+ * Create a Context with a given executionId. ExecutionId, together with
+ * user name and conf, will determine the temporary directory locations.
+ */
+ public Context(Configuration conf, String executionId) {
+ this.conf = conf;
+ this.executionId = executionId;
+
+ // local & non-local tmp location is configurable. however it is the same across
+ // all external file systems
+ nonLocalScratchPath = new Path(SessionState.getHDFSSessionPath(conf), executionId);
+ localScratchDir = new Path(SessionState.getLocalSessionPath(conf), executionId).toUri().getPath();
+ scratchDirPermission = HiveConf.getVar(conf, HiveConf.ConfVars.SCRATCHDIRPERMISSION);
+ stagingDir = HiveConf.getVar(conf, HiveConf.ConfVars.STAGINGDIR);
+ opContext = new CompilationOpContext();
+ }
+
+
+ public Map<LoadTableDesc, WriteEntity> getLoadTableOutputMap() {
+ return loadTableOutputMap;
+ }
+
+ public Map<WriteEntity, List<HiveLockObj>> getOutputLockObjects() {
+ return outputLockObjects;
+ }
+
+ /**
+ * Set the context on whether the current query is an explain query.
+ * @param value true if the query is an explain query, false if not
+ */
+ public void setExplain(boolean value) {
+ explain = value;
+ }
+
+ /**
+ * Find whether the current query is an explain query
+ * @return true if the query is an explain query, false if not
+ */
+ public boolean getExplain() {
+ return explain;
+ }
+
+ /**
+ * Find whether the current query is a logical explain query
+ */
+ public boolean getExplainLogical() {
+ return explainLogical;
+ }
+
+ /**
+ * Set the context on whether the current query is a logical
+ * explain query.
+ */
+ public void setExplainLogical(boolean explainLogical) {
+ this.explainLogical = explainLogical;
+ }
+
+ /**
+ * Set the original query command.
+ * @param cmd the original query command string
+ */
+ public void setCmd(String cmd) {
+ this.cmd = cmd;
+ }
+
+ /**
+ * Find the original query command.
+ * @return the original query command string
+ */
+ public String getCmd () {
+ return cmd;
+ }
+
+ /**
+ * Gets a temporary staging directory related to a path.
+ * If a path already contains a staging directory, then returns the current directory; otherwise
+ * create the directory if needed.
+ *
+ * @param inputPath URI of the temporary directory
+ * @param mkdir Create the directory if True.
+ * @return A temporary path.
+ */
+ private Path getStagingDir(Path inputPath, boolean mkdir) {
+ final URI inputPathUri = inputPath.toUri();
+ final String inputPathName = inputPathUri.getPath();
+ final String fileSystem = inputPathUri.getScheme() + ":" + inputPathUri.getAuthority();
+ final FileSystem fs;
+
+ try {
+ fs = inputPath.getFileSystem(conf);
+ } catch (IOException e) {
+ throw new IllegalStateException("Error getting FileSystem for " + inputPath + ": "+ e, e);
+ }
+
+ String stagingPathName;
+ if (inputPathName.indexOf(stagingDir) == -1) {
+ stagingPathName = new Path(inputPathName, stagingDir).toString();
+ } else {
+ stagingPathName = inputPathName.substring(0, inputPathName.indexOf(stagingDir) + stagingDir.length());
+ }
+
+ final String key = fileSystem + "-" + stagingPathName + "-" + TaskRunner.getTaskRunnerID();
+
+ Path dir = fsScratchDirs.get(key);
+ if (dir == null) {
+ // Append task specific info to stagingPathName, instead of creating a sub-directory.
+ // This way we don't have to worry about deleting the stagingPathName separately at
+ // end of query execution.
+ dir = fs.makeQualified(new Path(stagingPathName + "_" + this.executionId + "-" + TaskRunner.getTaskRunnerID()));
+
+ LOG.debug("Created staging dir = " + dir + " for path = " + inputPath);
+
+ if (mkdir) {
+ try {
+ boolean inheritPerms = HiveConf.getBoolVar(conf,
+ HiveConf.ConfVars.HIVE_WAREHOUSE_SUBDIR_INHERIT_PERMS);
+ if (!FileUtils.mkdir(fs, dir, inheritPerms, conf)) {
+ throw new IllegalStateException("Cannot create staging directory '" + dir.toString() + "'");
+ }
+
+ if (isHDFSCleanup) {
+ fs.deleteOnExit(dir);
+ }
+ } catch (IOException e) {
+ throw new RuntimeException("Cannot create staging directory '" + dir.toString() + "': " + e.getMessage(), e);
+ }
+ }
+
+ fsScratchDirs.put(key, dir);
+ }
+
+ return dir;
+ }
+
+ /**
+ * Get a tmp directory on specified URI
+ *
+ * @param scheme Scheme of the target FS
+ * @param authority Authority of the target FS
+ * @param mkdir create the directory if true
+ * @param scratchDir path of tmp directory
+ */
+ private Path getScratchDir(String scheme, String authority,
+ boolean mkdir, String scratchDir) {
+
+ String fileSystem = scheme + ":" + authority;
+ Path dir = fsScratchDirs.get(fileSystem + "-" + TaskRunner.getTaskRunnerID());
+
+ if (dir == null) {
+ Path dirPath = new Path(scheme, authority,
+ scratchDir + "-" + TaskRunner.getTaskRunnerID());
+ if (mkdir) {
+ try {
+ FileSystem fs = dirPath.getFileSystem(conf);
+ dirPath = new Path(fs.makeQualified(dirPath).toString());
+ FsPermission fsPermission = new FsPermission(scratchDirPermission);
+
+ if (!fs.mkdirs(dirPath, fsPermission)) {
+ throw new RuntimeException("Cannot make directory: "
+ + dirPath.toString());
+ }
+ if (isHDFSCleanup) {
+ fs.deleteOnExit(dirPath);
+ }
+ } catch (IOException e) {
+ throw new RuntimeException (e);
+ }
+ }
+ dir = dirPath;
+ fsScratchDirs.put(fileSystem + "-" + TaskRunner.getTaskRunnerID(), dir);
+
+ }
+
+ return dir;
+ }
+
+
+ /**
+ * Create a local scratch directory on demand and return it.
+ */
+ public Path getLocalScratchDir(boolean mkdir) {
+ try {
+ FileSystem fs = FileSystem.getLocal(conf);
+ URI uri = fs.getUri();
+ return getScratchDir(uri.getScheme(), uri.getAuthority(),
+ mkdir, localScratchDir);
+ } catch (IOException e) {
+ throw new RuntimeException (e);
+ }
+ }
+
+
+ /**
+ * Create a map-reduce scratch directory on demand and return it.
+ *
+ */
+ public Path getMRScratchDir() {
+
+ // if we are executing entirely on the client side - then
+ // just (re)use the local scratch directory
+ if(isLocalOnlyExecutionMode()) {
+ return getLocalScratchDir(!explain);
+ }
+
+ try {
+ Path dir = FileUtils.makeQualified(nonLocalScratchPath, conf);
+ URI uri = dir.toUri();
+
+ Path newScratchDir = getScratchDir(uri.getScheme(), uri.getAuthority(),
+ !explain, uri.getPath());
+ LOG.info("New scratch dir is " + newScratchDir);
+ return newScratchDir;
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ } catch (IllegalArgumentException e) {
+ throw new RuntimeException("Error while making MR scratch "
+ + "directory - check filesystem config (" + e.getCause() + ")", e);
+ }
+ }
+
+ /**
+ * Create a temporary directory depending of the path specified.
+ * - If path is an Object store filesystem, then use the default MR scratch directory (HDFS)
+ * - If path is on HDFS, then create a staging directory inside the path
+ *
+ * @param path Path used to verify the Filesystem to use for temporary directory
+ * @return A path to the new temporary directory
+ */
+ public Path getTempDirForPath(Path path) {
+ boolean isLocal = isPathLocal(path);
+ if ((BlobStorageUtils.isBlobStoragePath(conf, path) && !BlobStorageUtils.isBlobStorageAsScratchDir(conf))
+ || isLocal) {
+ // For better write performance, we use HDFS for temporary data when object store is used.
+ // Note that the scratch directory configuration variable must use HDFS or any other non-blobstorage system
+ // to take advantage of this performance.
+ return getMRTmpPath();
+ } else {
+ return getExtTmpPathRelTo(path);
+ }
+ }
+
+ /*
+ * Checks if the path is for the local filesystem or not
+ */
+ private boolean isPathLocal(Path path) {
+ boolean isLocal = false;
+ if (path != null) {
+ String scheme = path.toUri().getScheme();
+ if (scheme != null) {
+ isLocal = scheme.equals(Utilities.HADOOP_LOCAL_FS_SCHEME);
+ }
+ }
+ return isLocal;
+ }
+
+ private Path getExternalScratchDir(URI extURI) {
+ return getStagingDir(new Path(extURI.getScheme(), extURI.getAuthority(), extURI.getPath()), !explain);
+ }
+
+ /**
+ * Remove any created scratch directories.
+ */
+ public void removeScratchDir() {
+ for (Map.Entry<String, Path> entry : fsScratchDirs.entrySet()) {
+ try {
+ Path p = entry.getValue();
+ FileSystem fs = p.getFileSystem(conf);
+ fs.delete(p, true);
+ fs.cancelDeleteOnExit(p);
+ } catch (Exception e) {
+ LOG.warn("Error Removing Scratch: "
+ + StringUtils.stringifyException(e));
+ }
+ }
+ fsScratchDirs.clear();
+ }
+
+ /**
+ * Remove any created directories for CTEs.
+ */
+ public void removeMaterializedCTEs() {
+ // clean CTE tables
+ for (Table materializedTable : cteTables.values()) {
+ Path location = materializedTable.getDataLocation();
+ try {
+ FileSystem fs = location.getFileSystem(conf);
+ boolean status = fs.delete(location, true);
+ LOG.info("Removed " + location + " for materialized "
+ + materializedTable.getTableName() + ", status=" + status);
+ } catch (IOException e) {
+ // ignore
+ LOG.warn("Error removing " + location + " for materialized " + materializedTable.getTableName() +
+ ": " + StringUtils.stringifyException(e));
+ }
+ }
+ cteTables.clear();
+ }
+
+ private String nextPathId() {
+ return Integer.toString(pathid++);
+ }
+
+
+ private static final String MR_PREFIX = "-mr-";
+ private static final String EXT_PREFIX = "-ext-";
+ private static final String LOCAL_PREFIX = "-local-";
+
+ /**
+ * Check if path is for intermediate data
+ * @return true if a uri is a temporary uri for map-reduce intermediate data,
+ * false otherwise
+ */
+ public boolean isMRTmpFileURI(String uriStr) {
+ return (uriStr.indexOf(executionId) != -1) &&
+ (uriStr.indexOf(MR_PREFIX) != -1);
+ }
+
+ public Path getMRTmpPath(URI uri) {
+ return new Path(getStagingDir(new Path(uri), !explain), MR_PREFIX + nextPathId());
+ }
+
+ /**
+ * Get a path to store map-reduce intermediate data in.
+ *
+ * @return next available path for map-red intermediate data
+ */
+ public Path getMRTmpPath() {
+ return new Path(getMRScratchDir(), MR_PREFIX +
+ nextPathId());
+ }
+
+ /**
+ * Get a tmp path on local host to store intermediate data.
+ *
+ * @return next available tmp path on local fs
+ */
+ public Path getLocalTmpPath() {
+ return new Path(getLocalScratchDir(true), LOCAL_PREFIX + nextPathId());
+ }
+
+ /**
+ * Get a path to store tmp data destined for external Path.
+ *
+ * @param path external Path to which the tmp data has to be eventually moved
+ * @return next available tmp path on the file system corresponding extURI
+ */
+ public Path getExternalTmpPath(Path path) {
+ URI extURI = path.toUri();
+ if (extURI.getScheme().equals("viewfs")) {
+ // if we are on viewfs we don't want to use /tmp as tmp dir since rename from /tmp/..
+ // to final /user/hive/warehouse/ will fail later, so instead pick tmp dir
+ // on same namespace as tbl dir.
+ return getExtTmpPathRelTo(path.getParent());
+ }
+ return new Path(getExternalScratchDir(extURI), EXT_PREFIX +
+ nextPathId());
+ }
+
+ /**
+ * This is similar to getExternalTmpPath() with difference being this method returns temp path
+ * within passed in uri, whereas getExternalTmpPath() ignores passed in path and returns temp
+ * path within /tmp
+ */
+ public Path getExtTmpPathRelTo(Path path) {
+ return new Path(getStagingDir(path, !explain), EXT_PREFIX + nextPathId());
+ }
+
+ /**
+ * @return the resFile
+ */
+ public Path getResFile() {
+ return resFile;
+ }
+
+ /**
+ * @param resFile
+ * the resFile to set
+ */
+ public void setResFile(Path resFile) {
+ this.resFile = resFile;
+ resDir = null;
+ resDirPaths = null;
+ resDirFilesNum = 0;
+ }
+
+ /**
+ * @return the resDir
+ */
+ public Path getResDir() {
+ return resDir;
+ }
+
+ /**
+ * @param resDir
+ * the resDir to set
+ */
+ public void setResDir(Path resDir) {
+ this.resDir = resDir;
+ resFile = null;
+
+ resDirFilesNum = 0;
+ resDirPaths = null;
+ }
+
+ public void clear() throws IOException {
+ if (resDir != null) {
+ try {
+ FileSystem fs = resDir.getFileSystem(conf);
+ fs.delete(resDir, true);
+ } catch (IOException e) {
+ LOG.info("Context clear error: " + StringUtils.stringifyException(e));
+ }
+ }
+
+ if (resFile != null) {
+ try {
+ FileSystem fs = resFile.getFileSystem(conf);
+ fs.delete(resFile, false);
+ } catch (IOException e) {
+ LOG.info("Context clear error: " + StringUtils.stringifyException(e));
+ }
+ }
+ removeMaterializedCTEs();
+ removeScratchDir();
+ originalTracker = null;
+ setNeedLockMgr(false);
+ }
+
+ public DataInput getStream() {
+ try {
+ if (!initialized) {
+ initialized = true;
+ if ((resFile == null) && (resDir == null)) {
+ return null;
+ }
+
+ if (resFile != null) {
+ return resFile.getFileSystem(conf).open(resFile);
+ }
+
+ resFs = resDir.getFileSystem(conf);
+ FileStatus status = resFs.getFileStatus(resDir);
+ assert status.isDir();
+ FileStatus[] resDirFS = resFs.globStatus(new Path(resDir + "/*"), FileUtils.HIDDEN_FILES_PATH_FILTER);
+ resDirPaths = new Path[resDirFS.length];
+ int pos = 0;
+ for (FileStatus resFS : resDirFS) {
+ if (!resFS.isDir()) {
+ resDirPaths[pos++] = resFS.getPath();
+ }
+ }
+ if (pos == 0) {
+ return null;
+ }
+
+ return resFs.open(resDirPaths[resDirFilesNum++]);
+ } else {
+ return getNextStream();
+ }
+ } catch (FileNotFoundException e) {
+ LOG.info("getStream error: " + StringUtils.stringifyException(e));
+ return null;
+ } catch (IOException e) {
+ LOG.info("getStream error: " + StringUtils.stringifyException(e));
+ return null;
+ }
+ }
+
+ private DataInput getNextStream() {
+ try {
+ if (resDir != null && resDirFilesNum < resDirPaths.length
+ && (resDirPaths[resDirFilesNum] != null)) {
+ return resFs.open(resDirPaths[resDirFilesNum++]);
+ }
+ } catch (FileNotFoundException e) {
+ LOG.info("getNextStream error: " + StringUtils.stringifyException(e));
+ return null;
+ } catch (IOException e) {
+ LOG.info("getNextStream error: " + StringUtils.stringifyException(e));
+ return null;
+ }
+
+ return null;
+ }
+
+ public void resetStream() {
+ if (initialized) {
+ resDirFilesNum = 0;
+ initialized = false;
+ }
+ }
+
+ /**
+ * Little abbreviation for StringUtils.
+ */
+ private static boolean strEquals(String str1, String str2) {
+ return org.apache.commons.lang.StringUtils.equals(str1, str2);
+ }
+
+ /**
+ * Set the token rewrite stream being used to parse the current top-level SQL
+ * statement. Note that this should <b>not</b> be used for other parsing
+ * activities; for example, when we encounter a reference to a view, we switch
+ * to a new stream for parsing the stored view definition from the catalog,
+ * but we don't clobber the top-level stream in the context.
+ *
+ * @param tokenRewriteStream
+ * the stream being used
+ */
+ public void setTokenRewriteStream(TokenRewriteStream tokenRewriteStream) {
+ assert (this.tokenRewriteStream == null);
+ this.tokenRewriteStream = tokenRewriteStream;
+ }
+
+ /**
+ * @return the token rewrite stream being used to parse the current top-level
+ * SQL statement, or null if it isn't available (e.g. for parser
+ * tests)
+ */
+ public TokenRewriteStream getTokenRewriteStream() {
+ return tokenRewriteStream;
+ }
+
+ /**
+ * Generate a unique executionId. An executionId, together with user name and
+ * the configuration, will determine the temporary locations of all intermediate
+ * files.
+ *
+ * In the future, users can use the executionId to resume a query.
+ */
+ public static String generateExecutionId() {
+ Random rand = new Random();
+ SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd_HH-mm-ss_SSS");
+ String executionId = "hive_" + format.format(new Date()) + "_"
+ + Math.abs(rand.nextLong());
+ return executionId;
+ }
+
+ /**
+ * Does Hive wants to run tasks entirely on the local machine
+ * (where the query is being compiled)?
+ *
+ * Today this translates into running hadoop jobs locally
+ */
+ public boolean isLocalOnlyExecutionMode() {
+ // Always allow spark to run in a cluster mode. Without this, depending on
+ // user's local hadoop settings, true may be returned, which causes plan to be
+ // stored in local path.
+ if (HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_EXECUTION_ENGINE).equals("spark")) {
+ return false;
+ }
+
+ return ShimLoader.getHadoopShims().isLocalMode(conf);
+ }
+
+ public List<HiveLock> getHiveLocks() {
+ return hiveLocks;
+ }
+
+ public void setHiveLocks(List<HiveLock> hiveLocks) {
+ this.hiveLocks = hiveLocks;
+ }
+
+ public HiveTxnManager getHiveTxnManager() {
+ return hiveTxnManager;
+ }
+
+ public void setHiveTxnManager(HiveTxnManager txnMgr) {
+ hiveTxnManager = txnMgr;
+ }
+
+ public void setOriginalTracker(String originalTracker) {
+ this.originalTracker = originalTracker;
+ }
+
+ public void restoreOriginalTracker() {
+ if (originalTracker != null) {
+ ShimLoader.getHadoopShims().setJobLauncherRpcAddress(conf, originalTracker);
+ originalTracker = null;
+ }
+ }
+
+ public void addCS(String path, ContentSummary cs) {
+ pathToCS.put(path, cs);
+ }
+
+ public ContentSummary getCS(Path path) {
+ return getCS(path.toString());
+ }
+
+ public ContentSummary getCS(String path) {
+ return pathToCS.get(path);
+ }
+
+ public Map<String, ContentSummary> getPathToCS() {
+ return pathToCS;
+ }
+
+ public Configuration getConf() {
+ return conf;
+ }
+
+ /**
+ * @return the isHDFSCleanup
+ */
+ public boolean isHDFSCleanup() {
+ return isHDFSCleanup;
+ }
+
+ /**
+ * @param isHDFSCleanup the isHDFSCleanup to set
+ */
+ public void setHDFSCleanup(boolean isHDFSCleanup) {
+ this.isHDFSCleanup = isHDFSCleanup;
+ }
+
+ public boolean isNeedLockMgr() {
+ return needLockMgr;
+ }
+
+ public void setNeedLockMgr(boolean needLockMgr) {
+ this.needLockMgr = needLockMgr;
+ }
+
+ public int getTryCount() {
+ return tryCount;
+ }
+
+ public void setTryCount(int tryCount) {
+ this.tryCount = tryCount;
+ }
+
+ public void setAcidOperation(AcidUtils.Operation op) {
+ acidOperation = op;
+ }
+
+ public AcidUtils.Operation getAcidOperation() {
+ return acidOperation;
+ }
+
+ public String getCboInfo() {
+ return cboInfo;
+ }
+
+ public void setCboInfo(String cboInfo) {
+ this.cboInfo = cboInfo;
+ }
+
+ public boolean isCboSucceeded() {
+ return cboSucceeded;
+ }
+
+ public void setCboSucceeded(boolean cboSucceeded) {
+ this.cboSucceeded = cboSucceeded;
+ }
+
+ public Table getMaterializedTable(String cteName) {
+ return cteTables.get(cteName);
+ }
+
+ public void addMaterializedTable(String cteName, Table table) {
+ cteTables.put(cteName, table);
+ }
+
+ public AtomicInteger getSequencer() {
+ return sequencer;
+ }
+
+ public CompilationOpContext getOpContext() {
+ return opContext;
+ }
+
+ public Heartbeater getHeartbeater() {
+ return heartbeater;
+ }
+
+ public void setHeartbeater(Heartbeater heartbeater) {
+ this.heartbeater = heartbeater;
+ }
+
+ public void checkHeartbeaterLockException() throws LockException {
+ if (getHeartbeater() != null && getHeartbeater().getLockException() != null) {
+ throw getHeartbeater().getLockException();
+ }
+ }
+
+ public boolean isSkipTableMasking() {
+ return skipTableMasking;
+ }
+
+ public void setSkipTableMasking(boolean skipTableMasking) {
+ this.skipTableMasking = skipTableMasking;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/fd72ec4a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
index d6f30e4..08bd040 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
@@ -82,6 +82,7 @@ import org.apache.hadoop.hive.ql.optimizer.ppr.PartitionPruner;
import org.apache.hadoop.hive.ql.parse.ASTNode;
import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer;
import org.apache.hadoop.hive.ql.parse.ColumnAccessInfo;
+import org.apache.hadoop.hive.ql.parse.ExplainSemanticAnalyzer;
import org.apache.hadoop.hive.ql.parse.HiveSemanticAnalyzerHook;
import org.apache.hadoop.hive.ql.parse.HiveSemanticAnalyzerHookContext;
import org.apache.hadoop.hive.ql.parse.HiveSemanticAnalyzerHookContextImpl;
@@ -92,6 +93,7 @@ import org.apache.hadoop.hive.ql.parse.ParseUtils;
import org.apache.hadoop.hive.ql.parse.PrunedPartitionList;
import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer;
import org.apache.hadoop.hive.ql.parse.SemanticAnalyzerFactory;
+import org.apache.hadoop.hive.ql.parse.ExplainConfiguration.AnalyzeState;
import org.apache.hadoop.hive.ql.plan.FileSinkDesc;
import org.apache.hadoop.hive.ql.plan.HiveOperation;
import org.apache.hadoop.hive.ql.plan.TableDesc;
@@ -324,6 +326,11 @@ public class Driver implements CommandProcessor {
this(new QueryState(conf), null);
}
+ public Driver(HiveConf conf, Context ctx) {
+ this(new QueryState(conf), null);
+ this.ctx = ctx;
+ }
+
public Driver(HiveConf conf, String userName) {
this(new QueryState(conf), userName);
}
@@ -388,14 +395,13 @@ public class Driver implements CommandProcessor {
LOG.warn("WARNING! Query command could not be redacted." + e);
}
- if (ctx != null) {
- closeInProcess(false);
+ if (ctx != null && ctx.getExplainAnalyze() != AnalyzeState.RUNNING) {
+ close();
}
-
if (isInterrupted()) {
return handleInterruption("at beginning of compilation."); //indicate if need clean resource
}
-
+
if (resetTaskIds) {
TaskFactory.resetId();
}
@@ -435,7 +441,10 @@ public class Driver implements CommandProcessor {
return handleInterruption("before parsing and analysing the query");
}
- ctx = new Context(conf);
+ if (ctx == null) {
+ ctx = new Context(conf);
+ }
+
ctx.setTryCount(getTryCount());
ctx.setCmd(command);
ctx.setHDFSCleanup(true);
http://git-wip-us.apache.org/repos/asf/hive/blob/fd72ec4a/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnStatsTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnStatsTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnStatsTask.java
index a183b9b..a899964 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnStatsTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnStatsTask.java
@@ -53,6 +53,7 @@ import org.apache.hadoop.hive.ql.QueryState;
import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.Table;
+import org.apache.hadoop.hive.ql.parse.ExplainConfiguration.AnalyzeState;
import org.apache.hadoop.hive.ql.plan.ColumnStatsWork;
import org.apache.hadoop.hive.ql.plan.api.StageType;
import org.apache.hadoop.hive.ql.session.SessionState;
@@ -407,6 +408,9 @@ public class ColumnStatsTask extends Task<ColumnStatsWork> implements Serializab
@Override
public int execute(DriverContext driverContext) {
+ if (driverContext.getCtx().getExplainAnalyze() == AnalyzeState.RUNNING) {
+ return 0;
+ }
try {
Hive db = getHive();
return persistColumnStats(db);
http://git-wip-us.apache.org/repos/asf/hive/blob/fd72ec4a/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java
index 42992bb..9510058 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java
@@ -956,6 +956,7 @@ public abstract class CommonJoinOperator<T extends JoinDesc> extends
}
}
Arrays.fill(storage, null);
+ super.closeOp(abort);
}
@Override
http://git-wip-us.apache.org/repos/asf/hive/blob/fd72ec4a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
index 2e04d1d..92859fe 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
@@ -140,6 +140,7 @@ import org.apache.hadoop.hive.ql.parse.DDLSemanticAnalyzer;
import org.apache.hadoop.hive.ql.parse.PreInsertTableDesc;
import org.apache.hadoop.hive.ql.parse.ReplicationSpec;
import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.parse.ExplainConfiguration.AnalyzeState;
import org.apache.hadoop.hive.ql.plan.AbortTxnsDesc;
import org.apache.hadoop.hive.ql.plan.AddPartitionDesc;
import org.apache.hadoop.hive.ql.plan.AlterDatabaseDesc;
@@ -292,6 +293,9 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
@Override
public int execute(DriverContext driverContext) {
+ if (driverContext.getCtx().getExplainAnalyze() == AnalyzeState.RUNNING) {
+ return 0;
+ }
// Create the db
Hive db;
http://git-wip-us.apache.org/repos/asf/hive/blob/fd72ec4a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java
index 403d57c..fb39188 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java
@@ -271,7 +271,7 @@ public class ExplainTask extends Task<ExplainWork> implements Serializable {
// Because of the implementation of the JsonParserFactory, we are sure
// that we can get a TezJsonParser.
JsonParser jsonParser = JsonParserFactory.getParser(conf);
- work.setFormatted(true);
+ work.getConfig().setFormatted(true);
JSONObject jsonPlan = getJSONPlan(out, work);
if (work.getCboInfo() != null) {
jsonPlan.put("cboInfo", work.getCboInfo());
@@ -282,8 +282,8 @@ public class ExplainTask extends Task<ExplainWork> implements Serializable {
// if there is anything wrong happen, we bail out.
LOG.error("Running explain user level has problem: " + e.toString()
+ ". Falling back to normal explain");
- work.setFormatted(false);
- work.setUserLevelExplain(false);
+ work.getConfig().setFormatted(false);
+ work.getConfig().setUserLevelExplain(false);
jsonPlan = getJSONPlan(out, work);
}
} else {
http://git-wip-us.apache.org/repos/asf/hive/blob/fd72ec4a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
index fbb4d6e..5f9ed4b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
@@ -651,6 +651,7 @@ public class FileSinkOperator extends TerminalOperator<FileSinkDesc> implements
@Override
public void process(Object row, int tag) throws HiveException {
+ runTimeNumRows++;
/* Create list bucketing sub-directory only if stored-as-directories is on. */
String lbDirName = null;
lbDirName = (lbCtx == null) ? null : generateListBucketingDirName(row);
@@ -1082,6 +1083,7 @@ public class FileSinkOperator extends TerminalOperator<FileSinkDesc> implements
}
}
fsp = prevFsp = null;
+ super.closeOp(abort);
}
/**
@@ -1226,7 +1228,7 @@ public class FileSinkOperator extends TerminalOperator<FileSinkDesc> implements
}
}
}
- sContext.setIndexForTezUnion(this.conf.getIndexInTezUnion());
+ sContext.setIndexForTezUnion(this.getIndexForTezUnion());
if (!statsPublisher.closeConnection(sContext)) {
// The original exception is lost.
// Not changing the interface to maintain backward compatibility
http://git-wip-us.apache.org/repos/asf/hive/blob/fd72ec4a/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java
index ef168fa..7c40b58 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java
@@ -1028,7 +1028,6 @@ public class GroupByOperator extends Operator<GroupByDesc> {
* @throws HiveException
*/
private void forward(Object[] keys, AggregationBuffer[] aggs) throws HiveException {
-
if (forwardCache == null) {
forwardCache = new Object[outputKeyLength + aggs.length];
}
@@ -1123,6 +1122,7 @@ public class GroupByOperator extends Operator<GroupByDesc> {
}
}
hashAggregations = null;
+ super.closeOp(abort);
}
// Group by contains the columns needed - no need to aggregate from children
http://git-wip-us.apache.org/repos/asf/hive/blob/fd72ec4a/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java
index 3a6fdd5..0282763 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java
@@ -20,9 +20,7 @@ package org.apache.hadoop.hive.ql.exec;
import java.io.IOException;
import java.io.Serializable;
-import java.util.Collection;
import java.util.List;
-import java.util.concurrent.Future;
import org.slf4j.Logger;
import org.apache.hadoop.conf.Configuration;
http://git-wip-us.apache.org/repos/asf/hive/blob/fd72ec4a/ql/src/java/org/apache/hadoop/hive/ql/exec/LimitOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/LimitOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/LimitOperator.java
index 9676d70..d4ebbd4 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/LimitOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/LimitOperator.java
@@ -88,6 +88,7 @@ public class LimitOperator extends Operator<LimitDesc> implements Serializable {
if (!isMap && currCount < leastRow) {
throw new HiveException("No sufficient row found");
}
+ super.closeOp(abort);
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/fd72ec4a/ql/src/java/org/apache/hadoop/hive/ql/exec/ListSinkOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ListSinkOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ListSinkOperator.java
index 9bf363c..0633854 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ListSinkOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ListSinkOperator.java
@@ -93,6 +93,7 @@ public class ListSinkOperator extends Operator<ListSinkDesc> {
try {
res.add(fetcher.convert(row, inputObjInspectors[0]));
numRows++;
+ runTimeNumRows++;
} catch (Exception e) {
throw new HiveException(e);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/fd72ec4a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
index 671b851..e1381be 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
@@ -58,6 +58,7 @@ import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.optimizer.physical.BucketingSortingCtx.BucketCol;
import org.apache.hadoop.hive.ql.optimizer.physical.BucketingSortingCtx.SortCol;
import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer;
+import org.apache.hadoop.hive.ql.parse.ExplainConfiguration.AnalyzeState;
import org.apache.hadoop.hive.ql.plan.DynamicPartitionCtx;
import org.apache.hadoop.hive.ql.plan.LoadFileDesc;
import org.apache.hadoop.hive.ql.plan.LoadMultiFilesDesc;
@@ -248,6 +249,9 @@ public class MoveTask extends Task<MoveWork> implements Serializable {
public int execute(DriverContext driverContext) {
try {
+ if (driverContext.getCtx().getExplainAnalyze() == AnalyzeState.RUNNING) {
+ return 0;
+ }
Hive db = getHive();
// Do any hive related operations like moving tables and files
[28/50] [abbrv] hive git commit: HIVE-15084: Flaky test:
TestMiniTezCliDriver:explainanalyze_1, 2, 3, 4, 5 (Pengcheng Xiong)
Posted by om...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/9f5bea3b/ql/src/test/results/clientpositive/tez/explainanalyze_2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/explainanalyze_2.q.out b/ql/src/test/results/clientpositive/tez/explainanalyze_2.q.out
index 37fc7d3..d537b19 100644
--- a/ql/src/test/results/clientpositive/tez/explainanalyze_2.q.out
+++ b/ql/src/test/results/clientpositive/tez/explainanalyze_2.q.out
@@ -1,2011 +1,23 @@
-PREHOOK: query: -- SORT_QUERY_RESULTS
-
-CREATE TABLE dest_j1(key STRING, value STRING, val2 STRING) STORED AS TEXTFILE
-PREHOOK: type: CREATETABLE
-PREHOOK: Output: database:default
-PREHOOK: Output: default@dest_j1
-POSTHOOK: query: -- SORT_QUERY_RESULTS
-
-CREATE TABLE dest_j1(key STRING, value STRING, val2 STRING) STORED AS TEXTFILE
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@dest_j1
-PREHOOK: query: CREATE TABLE ss(k1 STRING,v1 STRING,k2 STRING,v2 STRING,k3 STRING,v3 STRING) STORED AS TEXTFILE
-PREHOOK: type: CREATETABLE
-PREHOOK: Output: database:default
-PREHOOK: Output: default@ss
-POSTHOOK: query: CREATE TABLE ss(k1 STRING,v1 STRING,k2 STRING,v2 STRING,k3 STRING,v3 STRING) STORED AS TEXTFILE
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@ss
-PREHOOK: query: CREATE TABLE sr(k1 STRING,v1 STRING,k2 STRING,v2 STRING,k3 STRING,v3 STRING) STORED AS TEXTFILE
-PREHOOK: type: CREATETABLE
-PREHOOK: Output: database:default
-PREHOOK: Output: default@sr
-POSTHOOK: query: CREATE TABLE sr(k1 STRING,v1 STRING,k2 STRING,v2 STRING,k3 STRING,v3 STRING) STORED AS TEXTFILE
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@sr
-PREHOOK: query: CREATE TABLE cs(k1 STRING,v1 STRING,k2 STRING,v2 STRING,k3 STRING,v3 STRING) STORED AS TEXTFILE
-PREHOOK: type: CREATETABLE
-PREHOOK: Output: database:default
-PREHOOK: Output: default@cs
-POSTHOOK: query: CREATE TABLE cs(k1 STRING,v1 STRING,k2 STRING,v2 STRING,k3 STRING,v3 STRING) STORED AS TEXTFILE
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@cs
-PREHOOK: query: INSERT OVERWRITE TABLE ss
-SELECT x.key,x.value,y.key,y.value,z.key,z.value
-FROM src1 x
-JOIN src y ON (x.key = y.key)
-JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11)
-PREHOOK: type: QUERY
-PREHOOK: Input: default@src
-PREHOOK: Input: default@src1
-PREHOOK: Input: default@srcpart
-PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
-PREHOOK: Output: default@ss
-POSTHOOK: query: INSERT OVERWRITE TABLE ss
-SELECT x.key,x.value,y.key,y.value,z.key,z.value
-FROM src1 x
-JOIN src y ON (x.key = y.key)
-JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11)
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@src
-POSTHOOK: Input: default@src1
-POSTHOOK: Input: default@srcpart
-POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
-POSTHOOK: Output: default@ss
-POSTHOOK: Lineage: ss.k1 SIMPLE [(src1)x.FieldSchema(name:key, type:string, comment:default), ]
-POSTHOOK: Lineage: ss.k2 SIMPLE [(src)y.FieldSchema(name:key, type:string, comment:default), ]
-POSTHOOK: Lineage: ss.k3 SIMPLE [(srcpart)z.FieldSchema(name:key, type:string, comment:default), ]
-POSTHOOK: Lineage: ss.v1 SIMPLE [(src1)x.FieldSchema(name:value, type:string, comment:default), ]
-POSTHOOK: Lineage: ss.v2 SIMPLE [(src)y.FieldSchema(name:value, type:string, comment:default), ]
-POSTHOOK: Lineage: ss.v3 SIMPLE [(srcpart)z.FieldSchema(name:value, type:string, comment:default), ]
-PREHOOK: query: INSERT OVERWRITE TABLE sr
-SELECT x.key,x.value,y.key,y.value,z.key,z.value
-FROM src1 x
-JOIN src y ON (x.key = y.key)
-JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=12)
-PREHOOK: type: QUERY
-PREHOOK: Input: default@src
-PREHOOK: Input: default@src1
-PREHOOK: Input: default@srcpart
-PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
-PREHOOK: Output: default@sr
-POSTHOOK: query: INSERT OVERWRITE TABLE sr
-SELECT x.key,x.value,y.key,y.value,z.key,z.value
-FROM src1 x
-JOIN src y ON (x.key = y.key)
-JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=12)
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@src
-POSTHOOK: Input: default@src1
-POSTHOOK: Input: default@srcpart
-POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
-POSTHOOK: Output: default@sr
-POSTHOOK: Lineage: sr.k1 SIMPLE [(src1)x.FieldSchema(name:key, type:string, comment:default), ]
-POSTHOOK: Lineage: sr.k2 SIMPLE [(src)y.FieldSchema(name:key, type:string, comment:default), ]
-POSTHOOK: Lineage: sr.k3 SIMPLE [(srcpart)z.FieldSchema(name:key, type:string, comment:default), ]
-POSTHOOK: Lineage: sr.v1 SIMPLE [(src1)x.FieldSchema(name:value, type:string, comment:default), ]
-POSTHOOK: Lineage: sr.v2 SIMPLE [(src)y.FieldSchema(name:value, type:string, comment:default), ]
-POSTHOOK: Lineage: sr.v3 SIMPLE [(srcpart)z.FieldSchema(name:value, type:string, comment:default), ]
-PREHOOK: query: INSERT OVERWRITE TABLE cs
-SELECT x.key,x.value,y.key,y.value,z.key,z.value
-FROM src1 x
-JOIN src y ON (x.key = y.key)
-JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08')
-PREHOOK: type: QUERY
-PREHOOK: Input: default@src
-PREHOOK: Input: default@src1
-PREHOOK: Input: default@srcpart
-PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
-PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
-PREHOOK: Output: default@cs
-POSTHOOK: query: INSERT OVERWRITE TABLE cs
-SELECT x.key,x.value,y.key,y.value,z.key,z.value
-FROM src1 x
-JOIN src y ON (x.key = y.key)
-JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08')
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@src
-POSTHOOK: Input: default@src1
-POSTHOOK: Input: default@srcpart
-POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
-POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
-POSTHOOK: Output: default@cs
-POSTHOOK: Lineage: cs.k1 SIMPLE [(src1)x.FieldSchema(name:key, type:string, comment:default), ]
-POSTHOOK: Lineage: cs.k2 SIMPLE [(src)y.FieldSchema(name:key, type:string, comment:default), ]
-POSTHOOK: Lineage: cs.k3 SIMPLE [(srcpart)z.FieldSchema(name:key, type:string, comment:default), ]
-POSTHOOK: Lineage: cs.v1 SIMPLE [(src1)x.FieldSchema(name:value, type:string, comment:default), ]
-POSTHOOK: Lineage: cs.v2 SIMPLE [(src)y.FieldSchema(name:value, type:string, comment:default), ]
-POSTHOOK: Lineage: cs.v3 SIMPLE [(srcpart)z.FieldSchema(name:value, type:string, comment:default), ]
-PREHOOK: query: ANALYZE TABLE ss COMPUTE STATISTICS
-PREHOOK: type: QUERY
-PREHOOK: Input: default@ss
-PREHOOK: Output: default@ss
-POSTHOOK: query: ANALYZE TABLE ss COMPUTE STATISTICS
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@ss
-POSTHOOK: Output: default@ss
-PREHOOK: query: ANALYZE TABLE ss COMPUTE STATISTICS FOR COLUMNS k1,v1,k2,v2,k3,v3
-PREHOOK: type: QUERY
-PREHOOK: Input: default@ss
-#### A masked pattern was here ####
-POSTHOOK: query: ANALYZE TABLE ss COMPUTE STATISTICS FOR COLUMNS k1,v1,k2,v2,k3,v3
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@ss
-#### A masked pattern was here ####
-PREHOOK: query: ANALYZE TABLE sr COMPUTE STATISTICS
-PREHOOK: type: QUERY
-PREHOOK: Input: default@sr
-PREHOOK: Output: default@sr
-POSTHOOK: query: ANALYZE TABLE sr COMPUTE STATISTICS
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@sr
-POSTHOOK: Output: default@sr
-PREHOOK: query: ANALYZE TABLE sr COMPUTE STATISTICS FOR COLUMNS k1,v1,k2,v2,k3,v3
-PREHOOK: type: QUERY
-PREHOOK: Input: default@sr
-#### A masked pattern was here ####
-POSTHOOK: query: ANALYZE TABLE sr COMPUTE STATISTICS FOR COLUMNS k1,v1,k2,v2,k3,v3
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@sr
-#### A masked pattern was here ####
-PREHOOK: query: ANALYZE TABLE cs COMPUTE STATISTICS
-PREHOOK: type: QUERY
-PREHOOK: Input: default@cs
-PREHOOK: Output: default@cs
-POSTHOOK: query: ANALYZE TABLE cs COMPUTE STATISTICS
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@cs
-POSTHOOK: Output: default@cs
-PREHOOK: query: ANALYZE TABLE cs COMPUTE STATISTICS FOR COLUMNS k1,v1,k2,v2,k3,v3
-PREHOOK: type: QUERY
-PREHOOK: Input: default@cs
-#### A masked pattern was here ####
-POSTHOOK: query: ANALYZE TABLE cs COMPUTE STATISTICS FOR COLUMNS k1,v1,k2,v2,k3,v3
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@cs
-#### A masked pattern was here ####
-PREHOOK: query: SELECT x.key, z.value, y.value
-FROM src1 x JOIN src y ON (x.key = y.key)
-JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11)
-PREHOOK: type: QUERY
-PREHOOK: Input: default@src
-PREHOOK: Input: default@src1
-PREHOOK: Input: default@srcpart
-PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
-#### A masked pattern was here ####
-POSTHOOK: query: SELECT x.key, z.value, y.value
-FROM src1 x JOIN src y ON (x.key = y.key)
-JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11)
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@src
-POSTHOOK: Input: default@src1
-POSTHOOK: Input: default@srcpart
-POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
-#### A masked pattern was here ####
-PREHOOK: query: explain analyze
-SELECT x.key, z.value, y.value
-FROM src1 x JOIN src y ON (x.key = y.key)
-JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11)
-PREHOOK: type: QUERY
-POSTHOOK: query: explain analyze
-SELECT x.key, z.value, y.value
-FROM src1 x JOIN src y ON (x.key = y.key)
-JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11)
-POSTHOOK: type: QUERY
-Plan optimized by CBO.
-
-Vertex dependency in root stage
-Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE)
-Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE)
-
-Stage-0
- Fetch Operator
- limit:-1
- Stage-1
- Reducer 3
- File Output Operator [FS_16]
- Select Operator [SEL_15] (rows=141/85 width=268)
- Output:["_col0","_col1","_col2"]
- Merge Join Operator [MERGEJOIN_26] (rows=141/85 width=268)
- Conds:RS_12._col3=RS_13._col0(Inner),Output:["_col0","_col3","_col6"]
- <-Map 5 [SIMPLE_EDGE]
- SHUFFLE [RS_13]
- PartitionCols:_col0
- Select Operator [SEL_8] (rows=500/500 width=178)
- Output:["_col0","_col1"]
- Filter Operator [FIL_24] (rows=500/500 width=178)
- predicate:key is not null
- TableScan [TS_6] (rows=500/500 width=178)
- default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
- <-Reducer 2 [SIMPLE_EDGE]
- SHUFFLE [RS_12]
- PartitionCols:_col3
- Merge Join Operator [MERGEJOIN_25] (rows=58/41 width=177)
- Conds:RS_9._col0=RS_10._col1(Inner),Output:["_col0","_col3"]
- <-Map 1 [SIMPLE_EDGE]
- SHUFFLE [RS_9]
- PartitionCols:_col0
- Select Operator [SEL_2] (rows=500/500 width=369)
- Output:["_col0"]
- Filter Operator [FIL_22] (rows=500/500 width=91)
- predicate:value is not null
- TableScan [TS_0] (rows=500/500 width=91)
- default@srcpart,z,Tbl:COMPLETE,Col:COMPLETE,Output:["value"]
- <-Map 4 [SIMPLE_EDGE]
- SHUFFLE [RS_10]
- PartitionCols:_col1
- Select Operator [SEL_5] (rows=25/25 width=175)
- Output:["_col0","_col1"]
- Filter Operator [FIL_23] (rows=25/25 width=175)
- predicate:(key is not null and value is not null)
- TableScan [TS_3] (rows=25/25 width=175)
- default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
-
-PREHOOK: query: select
-ss.k1,sr.k2,cs.k3,count(ss.v1),count(sr.v2),count(cs.v3)
-FROM
-ss,sr,cs,src d1,src d2,src d3,src1,srcpart
-where
- ss.k1 = d1.key
-and sr.k1 = d2.key
-and cs.k1 = d3.key
-and ss.k2 = sr.k2
-and ss.k3 = sr.k3
-and ss.v1 = src1.value
-and ss.v2 = srcpart.value
-and sr.v2 = cs.v2
-and sr.v3 = cs.v3
-and ss.v3='ssv3'
-and sr.v1='srv1'
-and src1.key = 'src1key'
-and srcpart.key = 'srcpartkey'
-and d1.value = 'd1value'
-and d2.value in ('2000Q1','2000Q2','2000Q3')
-and d3.value in ('2000Q1','2000Q2','2000Q3')
-group by
-ss.k1,sr.k2,cs.k3
-order by
-ss.k1,sr.k2,cs.k3
-limit 100
-PREHOOK: type: QUERY
-PREHOOK: Input: default@cs
-PREHOOK: Input: default@sr
-PREHOOK: Input: default@src
-PREHOOK: Input: default@src1
-PREHOOK: Input: default@srcpart
-PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
-PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
-PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
-PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
-PREHOOK: Input: default@ss
-#### A masked pattern was here ####
-POSTHOOK: query: select
-ss.k1,sr.k2,cs.k3,count(ss.v1),count(sr.v2),count(cs.v3)
-FROM
-ss,sr,cs,src d1,src d2,src d3,src1,srcpart
-where
- ss.k1 = d1.key
-and sr.k1 = d2.key
-and cs.k1 = d3.key
-and ss.k2 = sr.k2
-and ss.k3 = sr.k3
-and ss.v1 = src1.value
-and ss.v2 = srcpart.value
-and sr.v2 = cs.v2
-and sr.v3 = cs.v3
-and ss.v3='ssv3'
-and sr.v1='srv1'
-and src1.key = 'src1key'
-and srcpart.key = 'srcpartkey'
-and d1.value = 'd1value'
-and d2.value in ('2000Q1','2000Q2','2000Q3')
-and d3.value in ('2000Q1','2000Q2','2000Q3')
-group by
-ss.k1,sr.k2,cs.k3
-order by
-ss.k1,sr.k2,cs.k3
-limit 100
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@cs
-POSTHOOK: Input: default@sr
-POSTHOOK: Input: default@src
-POSTHOOK: Input: default@src1
-POSTHOOK: Input: default@srcpart
-POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
-POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
-POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
-POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
-POSTHOOK: Input: default@ss
-#### A masked pattern was here ####
-PREHOOK: query: explain analyze
-select
-ss.k1,sr.k2,cs.k3,count(ss.v1),count(sr.v2),count(cs.v3)
-FROM
-ss,sr,cs,src d1,src d2,src d3,src1,srcpart
-where
- ss.k1 = d1.key
-and sr.k1 = d2.key
-and cs.k1 = d3.key
-and ss.k2 = sr.k2
-and ss.k3 = sr.k3
-and ss.v1 = src1.value
-and ss.v2 = srcpart.value
-and sr.v2 = cs.v2
-and sr.v3 = cs.v3
-and ss.v3='ssv3'
-and sr.v1='srv1'
-and src1.key = 'src1key'
-and srcpart.key = 'srcpartkey'
-and d1.value = 'd1value'
-and d2.value in ('2000Q1','2000Q2','2000Q3')
-and d3.value in ('2000Q1','2000Q2','2000Q3')
-group by
-ss.k1,sr.k2,cs.k3
-order by
-ss.k1,sr.k2,cs.k3
-limit 100
-PREHOOK: type: QUERY
-POSTHOOK: query: explain analyze
-select
-ss.k1,sr.k2,cs.k3,count(ss.v1),count(sr.v2),count(cs.v3)
-FROM
-ss,sr,cs,src d1,src d2,src d3,src1,srcpart
-where
- ss.k1 = d1.key
-and sr.k1 = d2.key
-and cs.k1 = d3.key
-and ss.k2 = sr.k2
-and ss.k3 = sr.k3
-and ss.v1 = src1.value
-and ss.v2 = srcpart.value
-and sr.v2 = cs.v2
-and sr.v3 = cs.v3
-and ss.v3='ssv3'
-and sr.v1='srv1'
-and src1.key = 'src1key'
-and srcpart.key = 'srcpartkey'
-and d1.value = 'd1value'
-and d2.value in ('2000Q1','2000Q2','2000Q3')
-and d3.value in ('2000Q1','2000Q2','2000Q3')
-group by
-ss.k1,sr.k2,cs.k3
-order by
-ss.k1,sr.k2,cs.k3
-limit 100
-POSTHOOK: type: QUERY
-Plan optimized by CBO.
-
-Vertex dependency in root stage
-Reducer 10 <- Map 14 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE)
-Reducer 11 <- Reducer 10 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE)
-Reducer 16 <- Map 15 (SIMPLE_EDGE), Map 17 (SIMPLE_EDGE)
-Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE)
-Reducer 3 <- Reducer 11 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE)
-Reducer 4 <- Reducer 3 (SIMPLE_EDGE)
-Reducer 5 <- Reducer 4 (SIMPLE_EDGE)
-Reducer 8 <- Map 12 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE)
-Reducer 9 <- Map 13 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE)
-
-Stage-0
- Fetch Operator
- limit:100
- Stage-1
- Reducer 5
- File Output Operator [FS_55]
- Limit [LIM_54] (rows=24/0 width=285)
- Number of rows:100
- Select Operator [SEL_53] (rows=24/0 width=285)
- Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
- <-Reducer 4 [SIMPLE_EDGE]
- SHUFFLE [RS_52]
- Group By Operator [GBY_50] (rows=24/0 width=285)
- Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["count(VALUE._col0)","count(VALUE._col1)","count(VALUE._col2)"],keys:KEY._col0, KEY._col1, KEY._col2
- <-Reducer 3 [SIMPLE_EDGE]
- SHUFFLE [RS_49]
- PartitionCols:_col0, _col1, _col2
- Group By Operator [GBY_48] (rows=24/0 width=285)
- Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["count(_col13)","count(_col21)","count(_col3)"],keys:_col12, _col20, _col2
- Select Operator [SEL_47] (rows=650/0 width=534)
- Output:["_col12","_col20","_col2","_col13","_col21","_col3"]
- Merge Join Operator [MERGEJOIN_97] (rows=650/0 width=534)
- Conds:RS_44._col1, _col3=RS_45._col15, _col17(Inner),Output:["_col2","_col3","_col12","_col13","_col20","_col21"]
- <-Reducer 11 [SIMPLE_EDGE]
- SHUFFLE [RS_45]
- PartitionCols:_col15, _col17
- Select Operator [SEL_40] (rows=190/0 width=447)
- Output:["_col14","_col15","_col17","_col6","_col7"]
- Merge Join Operator [MERGEJOIN_96] (rows=190/0 width=447)
- Conds:RS_37._col6, _col4=RS_38._col4, _col2(Inner),Output:["_col2","_col3","_col14","_col15","_col17"]
- <-Reducer 10 [SIMPLE_EDGE]
- SHUFFLE [RS_37]
- PartitionCols:_col6, _col4
- Merge Join Operator [MERGEJOIN_94] (rows=40/0 width=352)
- Conds:RS_34._col3=RS_35._col1(Inner),Output:["_col2","_col3","_col4","_col6"]
- <-Map 14 [SIMPLE_EDGE]
- SHUFFLE [RS_35]
- PartitionCols:_col1
- Select Operator [SEL_17] (rows=2/0 width=180)
- Output:["_col1"]
- Filter Operator [FIL_88] (rows=2/0 width=175)
- predicate:((key = 'src1key') and value is not null)
- TableScan [TS_15] (rows=25/25 width=175)
- default@src1,src1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
- <-Reducer 9 [SIMPLE_EDGE]
- SHUFFLE [RS_34]
- PartitionCols:_col3
- Merge Join Operator [MERGEJOIN_93] (rows=40/0 width=352)
- Conds:RS_31._col2=RS_32._col0(Inner),Output:["_col2","_col3","_col4","_col6"]
- <-Map 13 [SIMPLE_EDGE]
- SHUFFLE [RS_32]
- PartitionCols:_col0
- Select Operator [SEL_14] (rows=2/0 width=178)
- Output:["_col0"]
- Filter Operator [FIL_87] (rows=2/0 width=178)
- predicate:((value = 'd1value') and key is not null)
- TableScan [TS_12] (rows=500/500 width=178)
- default@src,d1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
- <-Reducer 8 [SIMPLE_EDGE]
- SHUFFLE [RS_31]
- PartitionCols:_col2
- Merge Join Operator [MERGEJOIN_92] (rows=40/0 width=352)
- Conds:RS_28._col1=RS_29._col3(Inner),Output:["_col2","_col3","_col4","_col6"]
- <-Map 12 [SIMPLE_EDGE]
- SHUFFLE [RS_29]
- PartitionCols:_col3
- Select Operator [SEL_11] (rows=8/0 width=531)
- Output:["_col0","_col1","_col2","_col3","_col4"]
- Filter Operator [FIL_86] (rows=8/0 width=534)
- predicate:((v3 = 'ssv3') and k2 is not null and k3 is not null and k1 is not null and v1 is not null and v2 is not null)
- TableScan [TS_9] (rows=85/85 width=534)
- default@ss,ss,Tbl:COMPLETE,Col:COMPLETE,Output:["k1","v1","k2","v2","k3","v3"]
- <-Map 7 [SIMPLE_EDGE]
- SHUFFLE [RS_28]
- PartitionCols:_col1
- Select Operator [SEL_8] (rows=10/0 width=185)
- Output:["_col1"]
- Filter Operator [FIL_85] (rows=10/0 width=178)
- predicate:((key = 'srcpartkey') and value is not null)
- TableScan [TS_6] (rows=2000/2000 width=178)
- default@srcpart,srcpart,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
- <-Reducer 16 [SIMPLE_EDGE]
- SHUFFLE [RS_38]
- PartitionCols:_col4, _col2
- Merge Join Operator [MERGEJOIN_95] (rows=19/0 width=356)
- Conds:RS_24._col0=RS_25._col0(Inner),Output:["_col2","_col3","_col4","_col5"]
- <-Map 15 [SIMPLE_EDGE]
- SHUFFLE [RS_24]
- PartitionCols:_col0
- Select Operator [SEL_20] (rows=8/0 width=531)
- Output:["_col0","_col2","_col3","_col4","_col5"]
- Filter Operator [FIL_89] (rows=8/0 width=534)
- predicate:((v1 = 'srv1') and k2 is not null and k3 is not null and v2 is not null and v3 is not null and k1 is not null)
- TableScan [TS_18] (rows=85/85 width=534)
- default@sr,sr,Tbl:COMPLETE,Col:COMPLETE,Output:["k1","v1","k2","v2","k3","v3"]
- <-Map 17 [SIMPLE_EDGE]
- SHUFFLE [RS_25]
- PartitionCols:_col0
- Select Operator [SEL_23] (rows=500/0 width=178)
- Output:["_col0"]
- Filter Operator [FIL_90] (rows=500/0 width=178)
- predicate:((value) IN ('2000Q1', '2000Q2', '2000Q3') and key is not null)
- TableScan [TS_21] (rows=500/500 width=178)
- default@src,d2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
- <-Reducer 2 [SIMPLE_EDGE]
- SHUFFLE [RS_44]
- PartitionCols:_col1, _col3
- Merge Join Operator [MERGEJOIN_91] (rows=414/0 width=269)
- Conds:RS_41._col0=RS_42._col0(Inner),Output:["_col1","_col2","_col3"]
- <-Map 1 [SIMPLE_EDGE]
- SHUFFLE [RS_41]
- PartitionCols:_col0
- Select Operator [SEL_2] (rows=170/170 width=356)
- Output:["_col0","_col1","_col2","_col3"]
- Filter Operator [FIL_83] (rows=170/170 width=356)
- predicate:(v2 is not null and v3 is not null and k1 is not null)
- TableScan [TS_0] (rows=170/170 width=356)
- default@cs,cs,Tbl:COMPLETE,Col:COMPLETE,Output:["k1","v2","k3","v3"]
- <-Map 6 [SIMPLE_EDGE]
- SHUFFLE [RS_42]
- PartitionCols:_col0
- Select Operator [SEL_5] (rows=500/0 width=178)
- Output:["_col0"]
- Filter Operator [FIL_84] (rows=500/0 width=178)
- predicate:((value) IN ('2000Q1', '2000Q2', '2000Q3') and key is not null)
- TableScan [TS_3] (rows=500/500 width=178)
- default@src,d3,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
-
-PREHOOK: query: SELECT x.key, z.value, y.value
-FROM src1 x JOIN src y ON (x.key = y.key)
-JOIN (select * from src1 union select * from src)z ON (x.value = z.value)
-union
-SELECT x.key, z.value, y.value
-FROM src1 x JOIN src y ON (x.key = y.key)
-JOIN (select * from src1 union select * from src)z ON (x.value = z.value)
-PREHOOK: type: QUERY
-PREHOOK: Input: default@src
-PREHOOK: Input: default@src1
-#### A masked pattern was here ####
-POSTHOOK: query: SELECT x.key, z.value, y.value
-FROM src1 x JOIN src y ON (x.key = y.key)
-JOIN (select * from src1 union select * from src)z ON (x.value = z.value)
-union
-SELECT x.key, z.value, y.value
-FROM src1 x JOIN src y ON (x.key = y.key)
-JOIN (select * from src1 union select * from src)z ON (x.value = z.value)
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@src
-POSTHOOK: Input: default@src1
-#### A masked pattern was here ####
-PREHOOK: query: explain analyze
-SELECT x.key, z.value, y.value
-FROM src1 x JOIN src y ON (x.key = y.key)
-JOIN (select * from src1 union select * from src)z ON (x.value = z.value)
-union
-SELECT x.key, z.value, y.value
-FROM src1 x JOIN src y ON (x.key = y.key)
-JOIN (select * from src1 union select * from src)z ON (x.value = z.value)
-PREHOOK: type: QUERY
-POSTHOOK: query: explain analyze
-SELECT x.key, z.value, y.value
-FROM src1 x JOIN src y ON (x.key = y.key)
-JOIN (select * from src1 union select * from src)z ON (x.value = z.value)
-union
-SELECT x.key, z.value, y.value
-FROM src1 x JOIN src y ON (x.key = y.key)
-JOIN (select * from src1 union select * from src)z ON (x.value = z.value)
-POSTHOOK: type: QUERY
-Plan optimized by CBO.
-
-Vertex dependency in root stage
-Map 1 <- Union 2 (CONTAINS)
-Map 11 <- Union 12 (CONTAINS)
-Map 16 <- Union 12 (CONTAINS)
-Map 8 <- Union 2 (CONTAINS)
-Reducer 13 <- Union 12 (SIMPLE_EDGE)
-Reducer 14 <- Map 17 (SIMPLE_EDGE), Reducer 13 (SIMPLE_EDGE)
-Reducer 15 <- Map 18 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE), Union 6 (CONTAINS)
-Reducer 3 <- Union 2 (SIMPLE_EDGE)
-Reducer 4 <- Map 9 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE)
-Reducer 5 <- Map 10 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE), Union 6 (CONTAINS)
-Reducer 7 <- Union 6 (SIMPLE_EDGE)
-
-Stage-0
- Fetch Operator
- limit:-1
- Stage-1
- Reducer 7
- File Output Operator [FS_59]
- Group By Operator [GBY_57] (rows=28/15 width=177)
- Output:["_col0","_col1"],keys:KEY._col0, KEY._col1
- <-Union 6 [SIMPLE_EDGE]
- <-Reducer 15 [CONTAINS]
- Reduce Output Operator [RS_56]
- PartitionCols:_col0, _col1
- Group By Operator [GBY_55] (rows=28/30 width=177)
- Output:["_col0","_col1"],keys:_col0, _col1
- Select Operator [SEL_51] (rows=73/61 width=177)
- Output:["_col0","_col1"]
- Merge Join Operator [MERGEJOIN_85] (rows=73/61 width=177)
- Conds:RS_48._col2=RS_49._col0(Inner),Output:["_col1","_col2"]
- <-Map 18 [SIMPLE_EDGE]
- SHUFFLE [RS_49]
- PartitionCols:_col0
- Select Operator [SEL_44] (rows=500/500 width=87)
- Output:["_col0"]
- Filter Operator [FIL_81] (rows=500/500 width=87)
- predicate:key is not null
- TableScan [TS_42] (rows=500/500 width=87)
- default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key"]
- <-Reducer 14 [SIMPLE_EDGE]
- SHUFFLE [RS_48]
- PartitionCols:_col2
- Merge Join Operator [MERGEJOIN_84] (rows=30/52 width=177)
- Conds:RS_45._col1=RS_46._col1(Inner),Output:["_col1","_col2"]
- <-Map 17 [SIMPLE_EDGE]
- SHUFFLE [RS_46]
- PartitionCols:_col1
- Select Operator [SEL_41] (rows=25/25 width=175)
- Output:["_col0","_col1"]
- Filter Operator [FIL_80] (rows=25/25 width=175)
- predicate:(key is not null and value is not null)
- TableScan [TS_39] (rows=25/25 width=175)
- default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
- <-Reducer 13 [SIMPLE_EDGE]
- SHUFFLE [RS_45]
- PartitionCols:_col1
- Select Operator [SEL_38] (rows=262/319 width=178)
- Output:["_col1"]
- Group By Operator [GBY_37] (rows=262/319 width=178)
- Output:["_col0","_col1"],keys:KEY._col0, KEY._col1
- <-Union 12 [SIMPLE_EDGE]
- <-Map 11 [CONTAINS]
- Reduce Output Operator [RS_36]
- PartitionCols:_col0, _col1
- Group By Operator [GBY_35] (rows=262/331 width=178)
- Output:["_col0","_col1"],keys:_col1, _col0
- Select Operator [SEL_28] (rows=25/25 width=175)
- Output:["_col0","_col1"]
- Filter Operator [FIL_78] (rows=25/25 width=175)
- predicate:value is not null
- TableScan [TS_26] (rows=25/25 width=175)
- Output:["key","value"]
- <-Map 16 [CONTAINS]
- Reduce Output Operator [RS_36]
- PartitionCols:_col0, _col1
- Group By Operator [GBY_35] (rows=262/331 width=178)
- Output:["_col0","_col1"],keys:_col1, _col0
- Select Operator [SEL_31] (rows=500/500 width=178)
- Output:["_col0","_col1"]
- Filter Operator [FIL_79] (rows=500/500 width=178)
- predicate:value is not null
- TableScan [TS_29] (rows=500/500 width=178)
- Output:["key","value"]
- <-Reducer 5 [CONTAINS]
- Reduce Output Operator [RS_56]
- PartitionCols:_col0, _col1
- Group By Operator [GBY_55] (rows=28/30 width=177)
- Output:["_col0","_col1"],keys:_col0, _col1
- Select Operator [SEL_25] (rows=73/61 width=177)
- Output:["_col0","_col1"]
- Merge Join Operator [MERGEJOIN_83] (rows=73/61 width=177)
- Conds:RS_22._col2=RS_23._col0(Inner),Output:["_col1","_col2"]
- <-Map 10 [SIMPLE_EDGE]
- SHUFFLE [RS_23]
- PartitionCols:_col0
- Select Operator [SEL_18] (rows=500/500 width=87)
- Output:["_col0"]
- Filter Operator [FIL_77] (rows=500/500 width=87)
- predicate:key is not null
- TableScan [TS_16] (rows=500/500 width=87)
- default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key"]
- <-Reducer 4 [SIMPLE_EDGE]
- SHUFFLE [RS_22]
- PartitionCols:_col2
- Merge Join Operator [MERGEJOIN_82] (rows=30/52 width=177)
- Conds:RS_19._col1=RS_20._col1(Inner),Output:["_col1","_col2"]
- <-Map 9 [SIMPLE_EDGE]
- SHUFFLE [RS_20]
- PartitionCols:_col1
- Select Operator [SEL_15] (rows=25/25 width=175)
- Output:["_col0","_col1"]
- Filter Operator [FIL_76] (rows=25/25 width=175)
- predicate:(key is not null and value is not null)
- TableScan [TS_13] (rows=25/25 width=175)
- default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
- <-Reducer 3 [SIMPLE_EDGE]
- SHUFFLE [RS_19]
- PartitionCols:_col1
- Select Operator [SEL_12] (rows=262/319 width=178)
- Output:["_col1"]
- Group By Operator [GBY_11] (rows=262/319 width=178)
- Output:["_col0","_col1"],keys:KEY._col0, KEY._col1
- <-Union 2 [SIMPLE_EDGE]
- <-Map 1 [CONTAINS]
- Reduce Output Operator [RS_10]
- PartitionCols:_col0, _col1
- Group By Operator [GBY_9] (rows=262/331 width=178)
- Output:["_col0","_col1"],keys:_col1, _col0
- Select Operator [SEL_2] (rows=25/25 width=175)
- Output:["_col0","_col1"]
- Filter Operator [FIL_74] (rows=25/25 width=175)
- predicate:value is not null
- TableScan [TS_0] (rows=25/25 width=175)
- Output:["key","value"]
- <-Map 8 [CONTAINS]
- Reduce Output Operator [RS_10]
- PartitionCols:_col0, _col1
- Group By Operator [GBY_9] (rows=262/331 width=178)
- Output:["_col0","_col1"],keys:_col1, _col0
- Select Operator [SEL_5] (rows=500/500 width=178)
- Output:["_col0","_col1"]
- Filter Operator [FIL_75] (rows=500/500 width=178)
- predicate:value is not null
- TableScan [TS_3] (rows=500/500 width=178)
- Output:["key","value"]
-
-PREHOOK: query: SELECT x.key, y.value
-FROM src1 x JOIN src y ON (x.key = y.key)
-JOIN (select * from src1 union select * from src)z ON (x.value = z.value)
-union
-SELECT x.key, y.value
-FROM src1 x JOIN src y ON (x.key = y.key)
-JOIN (select key, value from src1 union select key, value from src union select key, value from src)z ON (x.value = z.value)
-union
-SELECT x.key, y.value
-FROM src1 x JOIN src y ON (x.key = y.key)
-JOIN (select key, value from src1 union select key, value from src union select key, value from src union select key, value from src)z ON (x.value = z.value)
-PREHOOK: type: QUERY
-PREHOOK: Input: default@src
-PREHOOK: Input: default@src1
-#### A masked pattern was here ####
-POSTHOOK: query: SELECT x.key, y.value
-FROM src1 x JOIN src y ON (x.key = y.key)
-JOIN (select * from src1 union select * from src)z ON (x.value = z.value)
-union
-SELECT x.key, y.value
-FROM src1 x JOIN src y ON (x.key = y.key)
-JOIN (select key, value from src1 union select key, value from src union select key, value from src)z ON (x.value = z.value)
-union
-SELECT x.key, y.value
-FROM src1 x JOIN src y ON (x.key = y.key)
-JOIN (select key, value from src1 union select key, value from src union select key, value from src union select key, value from src)z ON (x.value = z.value)
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@src
-POSTHOOK: Input: default@src1
-#### A masked pattern was here ####
-PREHOOK: query: explain analyze
-SELECT x.key, y.value
-FROM src1 x JOIN src y ON (x.key = y.key)
-JOIN (select * from src1 union select * from src)z ON (x.value = z.value)
-union
-SELECT x.key, y.value
-FROM src1 x JOIN src y ON (x.key = y.key)
-JOIN (select key, value from src1 union select key, value from src union select key, value from src)z ON (x.value = z.value)
-union
-SELECT x.key, y.value
-FROM src1 x JOIN src y ON (x.key = y.key)
-JOIN (select key, value from src1 union select key, value from src union select key, value from src union select key, value from src)z ON (x.value = z.value)
-PREHOOK: type: QUERY
-POSTHOOK: query: explain analyze
-SELECT x.key, y.value
-FROM src1 x JOIN src y ON (x.key = y.key)
-JOIN (select * from src1 union select * from src)z ON (x.value = z.value)
-union
-SELECT x.key, y.value
-FROM src1 x JOIN src y ON (x.key = y.key)
-JOIN (select key, value from src1 union select key, value from src union select key, value from src)z ON (x.value = z.value)
-union
-SELECT x.key, y.value
-FROM src1 x JOIN src y ON (x.key = y.key)
-JOIN (select key, value from src1 union select key, value from src union select key, value from src union select key, value from src)z ON (x.value = z.value)
-POSTHOOK: type: QUERY
-Plan optimized by CBO.
-
-Vertex dependency in root stage
-Map 1 <- Union 2 (CONTAINS)
-Map 10 <- Union 2 (CONTAINS)
-Map 13 <- Union 14 (CONTAINS)
-Map 20 <- Union 14 (CONTAINS)
-Map 21 <- Union 16 (CONTAINS)
-Map 24 <- Union 25 (CONTAINS)
-Map 33 <- Union 25 (CONTAINS)
-Map 34 <- Union 27 (CONTAINS)
-Map 35 <- Union 29 (CONTAINS)
-Reducer 15 <- Union 14 (SIMPLE_EDGE), Union 16 (CONTAINS)
-Reducer 17 <- Union 16 (SIMPLE_EDGE)
-Reducer 18 <- Map 22 (SIMPLE_EDGE), Reducer 17 (SIMPLE_EDGE)
-Reducer 19 <- Map 23 (SIMPLE_EDGE), Reducer 18 (SIMPLE_EDGE), Union 6 (CONTAINS)
-Reducer 26 <- Union 25 (SIMPLE_EDGE), Union 27 (CONTAINS)
-Reducer 28 <- Union 27 (SIMPLE_EDGE), Union 29 (CONTAINS)
-Reducer 3 <- Union 2 (SIMPLE_EDGE)
-Reducer 30 <- Union 29 (SIMPLE_EDGE)
-Reducer 31 <- Map 36 (SIMPLE_EDGE), Reducer 30 (SIMPLE_EDGE)
-Reducer 32 <- Map 37 (SIMPLE_EDGE), Reducer 31 (SIMPLE_EDGE), Union 8 (CONTAINS)
-Reducer 4 <- Map 11 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE)
-Reducer 5 <- Map 12 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE), Union 6 (CONTAINS)
-Reducer 7 <- Union 6 (SIMPLE_EDGE), Union 8 (CONTAINS)
-Reducer 9 <- Union 8 (SIMPLE_EDGE)
-
-Stage-0
- Fetch Operator
- limit:-1
- Stage-1
- Reducer 9
- File Output Operator [FS_122]
- Group By Operator [GBY_120] (rows=107/15 width=177)
- Output:["_col0","_col1"],keys:KEY._col0, KEY._col1
- <-Union 8 [SIMPLE_EDGE]
- <-Reducer 32 [CONTAINS]
- Reduce Output Operator [RS_119]
- PartitionCols:_col0, _col1
- Group By Operator [GBY_118] (rows=107/30 width=177)
- Output:["_col0","_col1"],keys:_col0, _col1
- Select Operator [SEL_114] (rows=124/61 width=177)
- Output:["_col0","_col1"]
- Merge Join Operator [MERGEJOIN_170] (rows=124/61 width=177)
- Conds:RS_111._col2=RS_112._col0(Inner),Output:["_col2","_col5"]
- <-Map 37 [SIMPLE_EDGE]
- SHUFFLE [RS_112]
- PartitionCols:_col0
- Select Operator [SEL_107] (rows=500/500 width=178)
- Output:["_col0","_col1"]
- Filter Operator [FIL_164] (rows=500/500 width=178)
- predicate:key is not null
- TableScan [TS_105] (rows=500/500 width=178)
- default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
- <-Reducer 31 [SIMPLE_EDGE]
- SHUFFLE [RS_111]
- PartitionCols:_col2
- Merge Join Operator [MERGEJOIN_169] (rows=51/52 width=86)
- Conds:RS_108._col1=RS_109._col1(Inner),Output:["_col2"]
- <-Map 36 [SIMPLE_EDGE]
- SHUFFLE [RS_109]
- PartitionCols:_col1
- Select Operator [SEL_104] (rows=25/25 width=175)
- Output:["_col0","_col1"]
- Filter Operator [FIL_163] (rows=25/25 width=175)
- predicate:(key is not null and value is not null)
- TableScan [TS_102] (rows=25/25 width=175)
- default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
- <-Reducer 30 [SIMPLE_EDGE]
- SHUFFLE [RS_108]
- PartitionCols:_col1
- Select Operator [SEL_101] (rows=440/319 width=178)
- Output:["_col1"]
- Group By Operator [GBY_100] (rows=440/319 width=178)
- Output:["_col0","_col1"],keys:KEY._col0, KEY._col1
- <-Union 29 [SIMPLE_EDGE]
- <-Map 35 [CONTAINS]
- Reduce Output Operator [RS_99]
- PartitionCols:_col0, _col1
- Group By Operator [GBY_98] (rows=440/628 width=178)
- Output:["_col0","_col1"],keys:_col1, _col0
- Select Operator [SEL_94] (rows=500/500 width=178)
- Output:["_col0","_col1"]
- Filter Operator [FIL_162] (rows=500/500 width=178)
- predicate:value is not null
- TableScan [TS_92] (rows=500/500 width=178)
- Output:["key","value"]
- <-Reducer 28 [CONTAINS]
- Reduce Output Operator [RS_99]
- PartitionCols:_col0, _col1
- Group By Operator [GBY_98] (rows=440/628 width=178)
- Output:["_col0","_col1"],keys:_col1, _col0
- Select Operator [SEL_91] (rows=381/319 width=178)
- Output:["_col0","_col1"]
- Group By Operator [GBY_90] (rows=381/319 width=178)
- Output:["_col0","_col1"],keys:KEY._col0, KEY._col1
- <-Union 27 [SIMPLE_EDGE]
- <-Map 34 [CONTAINS]
- Reduce Output Operator [RS_89]
- PartitionCols:_col0, _col1
- Group By Operator [GBY_88] (rows=381/628 width=178)
- Output:["_col0","_col1"],keys:_col1, _col0
- Select Operator [SEL_84] (rows=500/500 width=178)
- Output:["_col0","_col1"]
- Filter Operator [FIL_161] (rows=500/500 width=178)
- predicate:value is not null
- TableScan [TS_82] (rows=500/500 width=178)
- Output:["key","value"]
- <-Reducer 26 [CONTAINS]
- Reduce Output Operator [RS_89]
- PartitionCols:_col0, _col1
- Group By Operator [GBY_88] (rows=381/628 width=178)
- Output:["_col0","_col1"],keys:_col1, _col0
- Select Operator [SEL_81] (rows=262/319 width=178)
- Output:["_col0","_col1"]
- Group By Operator [GBY_80] (rows=262/319 width=178)
- Output:["_col0","_col1"],keys:KEY._col0, KEY._col1
- <-Union 25 [SIMPLE_EDGE]
- <-Map 24 [CONTAINS]
- Reduce Output Operator [RS_79]
- PartitionCols:_col0, _col1
- Group By Operator [GBY_78] (rows=262/331 width=178)
- Output:["_col0","_col1"],keys:_col1, _col0
- Select Operator [SEL_71] (rows=25/25 width=175)
- Output:["_col0","_col1"]
- Filter Operator [FIL_159] (rows=25/25 width=175)
- predicate:value is not null
- TableScan [TS_69] (rows=25/25 width=175)
- Output:["key","value"]
- <-Map 33 [CONTAINS]
- Reduce Output Operator [RS_79]
- PartitionCols:_col0, _col1
- Group By Operator [GBY_78] (rows=262/331 width=178)
- Output:["_col0","_col1"],keys:_col1, _col0
- Select Operator [SEL_74] (rows=500/500 width=178)
- Output:["_col0","_col1"]
- Filter Operator [FIL_160] (rows=500/500 width=178)
- predicate:value is not null
- TableScan [TS_72] (rows=500/500 width=178)
- Output:["key","value"]
- <-Reducer 7 [CONTAINS]
- Reduce Output Operator [RS_119]
- PartitionCols:_col0, _col1
- Group By Operator [GBY_118] (rows=107/30 width=177)
- Output:["_col0","_col1"],keys:_col0, _col1
- Group By Operator [GBY_67] (rows=90/15 width=177)
- Output:["_col0","_col1"],keys:KEY._col0, KEY._col1
- <-Union 6 [SIMPLE_EDGE]
- <-Reducer 19 [CONTAINS]
- Reduce Output Operator [RS_66]
- PartitionCols:_col0, _col1
- Group By Operator [GBY_65] (rows=90/30 width=177)
- Output:["_col0","_col1"],keys:_col0, _col1
- Select Operator [SEL_61] (rows=107/61 width=177)
- Output:["_col0","_col1"]
- Merge Join Operator [MERGEJOIN_168] (rows=107/61 width=177)
- Conds:RS_58._col2=RS_59._col0(Inner),Output:["_col2","_col5"]
- <-Map 23 [SIMPLE_EDGE]
- SHUFFLE [RS_59]
- PartitionCols:_col0
- Select Operator [SEL_54] (rows=500/500 width=178)
- Output:["_col0","_col1"]
- Filter Operator [FIL_158] (rows=500/500 width=178)
- predicate:key is not null
- TableScan [TS_52] (rows=500/500 width=178)
- default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
- <-Reducer 18 [SIMPLE_EDGE]
- SHUFFLE [RS_58]
- PartitionCols:_col2
- Merge Join Operator [MERGEJOIN_167] (rows=44/52 width=86)
- Conds:RS_55._col1=RS_56._col1(Inner),Output:["_col2"]
- <-Map 22 [SIMPLE_EDGE]
- SHUFFLE [RS_56]
- PartitionCols:_col1
- Select Operator [SEL_51] (rows=25/25 width=175)
- Output:["_col0","_col1"]
- Filter Operator [FIL_157] (rows=25/25 width=175)
- predicate:(key is not null and value is not null)
- TableScan [TS_49] (rows=25/25 width=175)
- default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
- <-Reducer 17 [SIMPLE_EDGE]
- SHUFFLE [RS_55]
- PartitionCols:_col1
- Select Operator [SEL_48] (rows=381/319 width=178)
- Output:["_col1"]
- Group By Operator [GBY_47] (rows=381/319 width=178)
- Output:["_col0","_col1"],keys:KEY._col0, KEY._col1
- <-Union 16 [SIMPLE_EDGE]
- <-Map 21 [CONTAINS]
- Reduce Output Operator [RS_46]
- PartitionCols:_col0, _col1
- Group By Operator [GBY_45] (rows=381/628 width=178)
- Output:["_col0","_col1"],keys:_col1, _col0
- Select Operator [SEL_41] (rows=500/500 width=178)
- Output:["_col0","_col1"]
- Filter Operator [FIL_156] (rows=500/500 width=178)
- predicate:value is not null
- TableScan [TS_39] (rows=500/500 width=178)
- Output:["key","value"]
- <-Reducer 15 [CONTAINS]
- Reduce Output Operator [RS_46]
- PartitionCols:_col0, _col1
- Group By Operator [GBY_45] (rows=381/628 width=178)
- Output:["_col0","_col1"],keys:_col1, _col0
- Select Operator [SEL_38] (rows=262/319 width=178)
- Output:["_col0","_col1"]
- Group By Operator [GBY_37] (rows=262/319 width=178)
- Output:["_col0","_col1"],keys:KEY._col0, KEY._col1
- <-Union 14 [SIMPLE_EDGE]
- <-Map 13 [CONTAINS]
- Reduce Output Operator [RS_36]
- PartitionCols:_col0, _col1
- Group By Operator [GBY_35] (rows=262/331 width=178)
- Output:["_col0","_col1"],keys:_col1, _col0
- Select Operator [SEL_28] (rows=25/25 width=175)
- Output:["_col0","_col1"]
- Filter Operator [FIL_154] (rows=25/25 width=175)
- predicate:value is not null
- TableScan [TS_26] (rows=25/25 width=175)
- Output:["key","value"]
- <-Map 20 [CONTAINS]
- Reduce Output Operator [RS_36]
- PartitionCols:_col0, _col1
- Group By Operator [GBY_35] (rows=262/331 width=178)
- Output:["_col0","_col1"],keys:_col1, _col0
- Select Operator [SEL_31] (rows=500/500 width=178)
- Output:["_col0","_col1"]
- Filter Operator [FIL_155] (rows=500/500 width=178)
- predicate:value is not null
- TableScan [TS_29] (rows=500/500 width=178)
- Output:["key","value"]
- <-Reducer 5 [CONTAINS]
- Reduce Output Operator [RS_66]
- PartitionCols:_col0, _col1
- Group By Operator [GBY_65] (rows=90/30 width=177)
- Output:["_col0","_col1"],keys:_col0, _col1
- Select Operator [SEL_25] (rows=73/61 width=177)
- Output:["_col0","_col1"]
- Merge Join Operator [MERGEJOIN_166] (rows=73/61 width=177)
- Conds:RS_22._col2=RS_23._col0(Inner),Output:["_col2","_col5"]
- <-Map 12 [SIMPLE_EDGE]
- SHUFFLE [RS_23]
- PartitionCols:_col0
- Select Operator [SEL_18] (rows=500/500 width=178)
- Output:["_col0","_col1"]
- Filter Operator [FIL_153] (rows=500/500 width=178)
- predicate:key is not null
- TableScan [TS_16] (rows=500/500 width=178)
- default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
- <-Reducer 4 [SIMPLE_EDGE]
- SHUFFLE [RS_22]
- PartitionCols:_col2
- Merge Join Operator [MERGEJOIN_165] (rows=30/52 width=86)
- Conds:RS_19._col1=RS_20._col1(Inner),Output:["_col2"]
- <-Map 11 [SIMPLE_EDGE]
- SHUFFLE [RS_20]
- PartitionCols:_col1
- Select Operator [SEL_15] (rows=25/25 width=175)
- Output:["_col0","_col1"]
- Filter Operator [FIL_152] (rows=25/25 width=175)
- predicate:(key is not null and value is not null)
- TableScan [TS_13] (rows=25/25 width=175)
- default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
- <-Reducer 3 [SIMPLE_EDGE]
- SHUFFLE [RS_19]
- PartitionCols:_col1
- Select Operator [SEL_12] (rows=262/319 width=178)
- Output:["_col1"]
- Group By Operator [GBY_11] (rows=262/319 width=178)
- Output:["_col0","_col1"],keys:KEY._col0, KEY._col1
- <-Union 2 [SIMPLE_EDGE]
- <-Map 1 [CONTAINS]
- Reduce Output Operator [RS_10]
- PartitionCols:_col0, _col1
- Group By Operator [GBY_9] (rows=262/331 width=178)
- Output:["_col0","_col1"],keys:_col1, _col0
- Select Operator [SEL_2] (rows=25/25 width=175)
- Output:["_col0","_col1"]
- Filter Operator [FIL_150] (rows=25/25 width=175)
- predicate:value is not null
- TableScan [TS_0] (rows=25/25 width=175)
- Output:["key","value"]
- <-Map 10 [CONTAINS]
- Reduce Output Operator [RS_10]
- PartitionCols:_col0, _col1
- Group By Operator [GBY_9] (rows=262/331 width=178)
- Output:["_col0","_col1"],keys:_col1, _col0
- Select Operator [SEL_5] (rows=500/500 width=178)
- Output:["_col0","_col1"]
- Filter Operator [FIL_151] (rows=500/500 width=178)
- predicate:value is not null
- TableScan [TS_3] (rows=500/500 width=178)
- Output:["key","value"]
-
-PREHOOK: query: SELECT x.key, z.value, y.value
-FROM src1 x JOIN src y ON (x.key = y.key)
-JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11)
-PREHOOK: type: QUERY
-PREHOOK: Input: default@src
-PREHOOK: Input: default@src1
-PREHOOK: Input: default@srcpart
-PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
-#### A masked pattern was here ####
-POSTHOOK: query: SELECT x.key, z.value, y.value
-FROM src1 x JOIN src y ON (x.key = y.key)
-JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11)
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@src
-POSTHOOK: Input: default@src1
-POSTHOOK: Input: default@srcpart
-POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
-#### A masked pattern was here ####
-PREHOOK: query: explain analyze
-SELECT x.key, z.value, y.value
-FROM src1 x JOIN src y ON (x.key = y.key)
-JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11)
-PREHOOK: type: QUERY
-POSTHOOK: query: explain analyze
-SELECT x.key, z.value, y.value
-FROM src1 x JOIN src y ON (x.key = y.key)
-JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11)
-POSTHOOK: type: QUERY
-Plan optimized by CBO.
-
-Vertex dependency in root stage
-Map 1 <- Map 2 (BROADCAST_EDGE), Map 3 (BROADCAST_EDGE)
-
-Stage-0
- Fetch Operator
- limit:-1
- Stage-1
- Map 1
- File Output Operator [FS_16]
- Select Operator [SEL_15] (rows=605/85 width=10)
- Output:["_col0","_col1","_col2"]
- Map Join Operator [MAPJOIN_26] (rows=605/85 width=10)
- Conds:MAPJOIN_25._col3=RS_13._col0(Inner),HybridGraceHashJoin:true,Output:["_col0","_col3","_col6"]
- <-Map 3 [BROADCAST_EDGE]
- BROADCAST [RS_13]
- PartitionCols:_col0
- Select Operator [SEL_8] (rows=500/500 width=10)
- Output:["_col0","_col1"]
- Filter Operator [FIL_24] (rows=500/500 width=10)
- predicate:key is not null
- TableScan [TS_6] (rows=500/500 width=10)
- default@src,y,Tbl:COMPLETE,Col:NONE,Output:["key","value"]
- <-Map Join Operator [MAPJOIN_25] (rows=550/41 width=10)
- Conds:SEL_2._col0=RS_10._col1(Inner),HybridGraceHashJoin:true,Output:["_col0","_col3"]
- <-Map 2 [BROADCAST_EDGE]
- BROADCAST [RS_10]
- PartitionCols:_col1
- Select Operator [SEL_5] (rows=25/25 width=7)
- Output:["_col0","_col1"]
- Filter Operator [FIL_23] (rows=25/25 width=7)
- predicate:(key is not null and value is not null)
- TableScan [TS_3] (rows=25/25 width=7)
- default@src1,x,Tbl:COMPLETE,Col:NONE,Output:["key","value"]
- <-Select Operator [SEL_2] (rows=500/500 width=10)
- Output:["_col0"]
- Filter Operator [FIL_22] (rows=500/500 width=10)
- predicate:value is not null
- TableScan [TS_0] (rows=500/500 width=10)
- default@srcpart,z,Tbl:COMPLETE,Col:NONE,Output:["value"]
-
-PREHOOK: query: select
-ss.k1,sr.k2,cs.k3,count(ss.v1),count(sr.v2),count(cs.v3)
-FROM
-ss,sr,cs,src d1,src d2,src d3,src1,srcpart
-where
- ss.k1 = d1.key
-and sr.k1 = d2.key
-and cs.k1 = d3.key
-and ss.k2 = sr.k2
-and ss.k3 = sr.k3
-and ss.v1 = src1.value
-and ss.v2 = srcpart.value
-and sr.v2 = cs.v2
-and sr.v3 = cs.v3
-and ss.v3='ssv3'
-and sr.v1='srv1'
-and src1.key = 'src1key'
-and srcpart.key = 'srcpartkey'
-and d1.value = 'd1value'
-and d2.value in ('2000Q1','2000Q2','2000Q3')
-and d3.value in ('2000Q1','2000Q2','2000Q3')
-group by
-ss.k1,sr.k2,cs.k3
-order by
-ss.k1,sr.k2,cs.k3
-limit 100
-PREHOOK: type: QUERY
-PREHOOK: Input: default@cs
-PREHOOK: Input: default@sr
-PREHOOK: Input: default@src
-PREHOOK: Input: default@src1
-PREHOOK: Input: default@srcpart
-PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
-PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
-PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
-PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
-PREHOOK: Input: default@ss
-#### A masked pattern was here ####
-POSTHOOK: query: select
-ss.k1,sr.k2,cs.k3,count(ss.v1),count(sr.v2),count(cs.v3)
-FROM
-ss,sr,cs,src d1,src d2,src d3,src1,srcpart
-where
- ss.k1 = d1.key
-and sr.k1 = d2.key
-and cs.k1 = d3.key
-and ss.k2 = sr.k2
-and ss.k3 = sr.k3
-and ss.v1 = src1.value
-and ss.v2 = srcpart.value
-and sr.v2 = cs.v2
-and sr.v3 = cs.v3
-and ss.v3='ssv3'
-and sr.v1='srv1'
-and src1.key = 'src1key'
-and srcpart.key = 'srcpartkey'
-and d1.value = 'd1value'
-and d2.value in ('2000Q1','2000Q2','2000Q3')
-and d3.value in ('2000Q1','2000Q2','2000Q3')
-group by
-ss.k1,sr.k2,cs.k3
-order by
-ss.k1,sr.k2,cs.k3
-limit 100
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@cs
-POSTHOOK: Input: default@sr
-POSTHOOK: Input: default@src
-POSTHOOK: Input: default@src1
-POSTHOOK: Input: default@srcpart
-POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
-POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
-POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
-POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
-POSTHOOK: Input: default@ss
-#### A masked pattern was here ####
-PREHOOK: query: explain analyze
-select
-ss.k1,sr.k2,cs.k3,count(ss.v1),count(sr.v2),count(cs.v3)
-FROM
-ss,sr,cs,src d1,src d2,src d3,src1,srcpart
-where
- ss.k1 = d1.key
-and sr.k1 = d2.key
-and cs.k1 = d3.key
-and ss.k2 = sr.k2
-and ss.k3 = sr.k3
-and ss.v1 = src1.value
-and ss.v2 = srcpart.value
-and sr.v2 = cs.v2
-and sr.v3 = cs.v3
-and ss.v3='ssv3'
-and sr.v1='srv1'
-and src1.key = 'src1key'
-and srcpart.key = 'srcpartkey'
-and d1.value = 'd1value'
-and d2.value in ('2000Q1','2000Q2','2000Q3')
-and d3.value in ('2000Q1','2000Q2','2000Q3')
-group by
-ss.k1,sr.k2,cs.k3
-order by
-ss.k1,sr.k2,cs.k3
-limit 100
-PREHOOK: type: QUERY
-POSTHOOK: query: explain analyze
-select
-ss.k1,sr.k2,cs.k3,count(ss.v1),count(sr.v2),count(cs.v3)
-FROM
-ss,sr,cs,src d1,src d2,src d3,src1,srcpart
-where
- ss.k1 = d1.key
-and sr.k1 = d2.key
-and cs.k1 = d3.key
-and ss.k2 = sr.k2
-and ss.k3 = sr.k3
-and ss.v1 = src1.value
-and ss.v2 = srcpart.value
-and sr.v2 = cs.v2
-and sr.v3 = cs.v3
-and ss.v3='ssv3'
-and sr.v1='srv1'
-and src1.key = 'src1key'
-and srcpart.key = 'srcpartkey'
-and d1.value = 'd1value'
-and d2.value in ('2000Q1','2000Q2','2000Q3')
-and d3.value in ('2000Q1','2000Q2','2000Q3')
-group by
-ss.k1,sr.k2,cs.k3
-order by
-ss.k1,sr.k2,cs.k3
-limit 100
-POSTHOOK: type: QUERY
-Plan optimized by CBO.
-
-Vertex dependency in root stage
-Map 10 <- Map 9 (BROADCAST_EDGE)
-Map 2 <- Map 1 (BROADCAST_EDGE)
-Map 3 <- Map 10 (BROADCAST_EDGE), Map 2 (BROADCAST_EDGE), Map 6 (BROADCAST_EDGE), Map 7 (BROADCAST_EDGE), Map 8 (BROADCAST_EDGE)
-Reducer 4 <- Map 3 (SIMPLE_EDGE)
-Reducer 5 <- Reducer 4 (SIMPLE_EDGE)
-
-Stage-0
- Fetch Operator
- limit:100
- Stage-1
- Reducer 5
- File Output Operator [FS_55]
- Limit [LIM_54] (rows=100/0 width=10)
- Number of rows:100
- Select Operator [SEL_53] (rows=805/0 width=10)
- Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
- <-Reducer 4 [SIMPLE_EDGE]
- SHUFFLE [RS_52]
- Group By Operator [GBY_50] (rows=805/0 width=10)
- Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["count(VALUE._col0)","count(VALUE._col1)","count(VALUE._col2)"],keys:KEY._col0, KEY._col1, KEY._col2
- <-Map 3 [SIMPLE_EDGE]
- SHUFFLE [RS_49]
- PartitionCols:_col0, _col1, _col2
- Group By Operator [GBY_48] (rows=1610/0 width=10)
- Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["count(_col13)","count(_col21)","count(_col3)"],keys:_col12, _col20, _col2
- Select Operator [SEL_47] (rows=1610/0 width=10)
- Output:["_col12","_col20","_col2","_col13","_col21","_col3"]
- Map Join Operator [MAPJOIN_97] (rows=1610/0 width=10)
- Conds:RS_44._col1, _col3=SEL_40._col15, _col17(Inner),HybridGraceHashJoin:true,Output:["_col2","_col3","_col12","_col13","_col20","_col21"]
- <-Map 2 [BROADCAST_EDGE]
- BROADCAST [RS_44]
- PartitionCols:_col1, _col3
- Map Join Operator [MAPJOIN_91] (rows=275/0 width=10)
- Conds:RS_41._col0=SEL_5._col0(Inner),HybridGraceHashJoin:true,Output:["_col1","_col2","_col3"]
- <-Map 1 [BROADCAST_EDGE]
- BROADCAST [RS_41]
- PartitionCols:_col0
- Select Operator [SEL_2] (rows=170/170 width=34)
- Output:["_col0","_col1","_col2","_col3"]
- Filter Operator [FIL_83] (rows=170/170 width=34)
- predicate:(v2 is not null and v3 is not null and k1 is not null)
- TableScan [TS_0] (rows=170/170 width=34)
- default@cs,cs,Tbl:COMPLETE,Col:NONE,Output:["k1","v2","k3","v3"]
- <-Select Operator [SEL_5] (rows=250/0 width=10)
- Output:["_col0"]
- Filter Operator [FIL_84] (rows=250/0 width=10)
- predicate:((value) IN ('2000Q1', '2000Q2', '2000Q3') and key is not null)
- TableScan [TS_3] (rows=500/500 width=10)
- default@src,d3,Tbl:COMPLETE,Col:NONE,Output:["key","value"]
- <-Select Operator [SEL_40] (rows=1464/0 width=10)
- Output:["_col14","_col15","_col17","_col6","_col7"]
- Map Join Operator [MAPJOIN_96] (rows=1464/0 width=10)
- Conds:MAPJOIN_94._col6, _col4=RS_38._col4, _col2(Inner),HybridGraceHashJoin:true,Output:["_col2","_col3","_col14","_col15","_col17"]
- <-Map 10 [BROADCAST_EDGE]
- BROADCAST [RS_38]
- PartitionCols:_col4, _col2
- Map Join Operator [MAPJOIN_95] (rows=275/0 width=10)
- Conds:RS_24._col0=SEL_23._col0(Inner),HybridGraceHashJoin:true,Output:["_col2","_col3","_col4","_col5"]
- <-Map 9 [BROADCAST_EDGE]
- BROADCAST [RS_24]
- PartitionCols:_col0
- Select Operator [SEL_20] (rows=42/0 width=34)
- Output:["_col0","_col2","_col3","_col4","_col5"]
- Filter Operator [FIL_89] (rows=42/0 width=34)
- predicate:((v1 = 'srv1') and k2 is not null and k3 is not null and v2 is not null and v3 is not null and k1 is not null)
- TableScan [TS_18] (rows=85/85 width=34)
- default@sr,sr,Tbl:COMPLETE,Col:NONE,Output:["k1","v1","k2","v2","k3","v3"]
- <-Select Operator [SEL_23] (rows=250/0 width=10)
- Output:["_col0"]
- Filter Operator [FIL_90] (rows=250/0 width=10)
- predicate:((value) IN ('2000Q1', '2000Q2', '2000Q3') and key is not null)
- TableScan [TS_21] (rows=500/500 width=10)
- default@src,d2,Tbl:COMPLETE,Col:NONE,Output:["key","value"]
- <-Map Join Operator [MAPJOIN_94] (rows=1331/0 width=10)
- Conds:MAPJOIN_93._col3=RS_35._col1(Inner),HybridGraceHashJoin:true,Output:["_col2","_col3","_col4","_col6"]
- <-Map 8 [BROADCAST_EDGE]
- BROADCAST [RS_35]
- PartitionCols:_col1
- Select Operator [SEL_17] (rows=12/0 width=7)
- Output:["_col1"]
- Filter Operator [FIL_88] (rows=12/0 width=7)
- predicate:((key = 'src1key') and value is not null)
- TableScan [TS_15] (rows=25/25 width=7)
- default@src1,src1,Tbl:COMPLETE,Col:NONE,Output:["key","value"]
- <-Map Join Operator [MAPJOIN_93] (rows=1210/0 width=10)
- Conds:MAPJOIN_92._col2=RS_32._col0(Inner),HybridGraceHashJoin:true,Output:["_col2","_col3","_col4","_col6"]
- <-Map 7 [BROADCAST_EDGE]
- BROADCAST [RS_32]
- PartitionCols:_col0
- Select Operator [SEL_14] (rows=250/0 width=10)
- Output:["_col0"]
- Filter Operator [FIL_87] (rows=250/0 width=10)
- predicate:((value = 'd1value') and key is not null)
- TableScan [TS_12] (rows=500/500 width=10)
- default@src,d1,Tbl:COMPLETE,Col:NONE,Output:["key","value"]
- <-Map Join Operator [MAPJOIN_92] (rows=1100/0 width=10)
- Conds:SEL_8._col1=RS_29._col3(Inner),HybridGraceHashJoin:true,Output:["_col2","_col3","_col4","_col6"]
- <-Map 6 [BROADCAST_EDGE]
- BROADCAST [RS_29]
- PartitionCols:_col3
- Select Operator [SEL_11] (rows=42/0 width=34)
- Output:["_col0","_col1","_col2","_col3","_col4"]
- Filter Operator [FIL_86] (rows=42/0 width=34)
- predicate:((v3 = 'ssv3') and k2 is not null and k3 is not null and k1 is not null and v1 is not null and v2 is not null)
- TableScan [TS_9] (rows=85/85 width=34)
- default@ss,ss,Tbl:COMPLETE,Col:NONE,Output:["k1","v1","k2","v2","k3","v3"]
- <-Select Operator [SEL_8] (rows=1000/0 width=10)
- Output:["_col1"]
- Filter Operator [FIL_85] (rows=1000/0 width=10)
- predicate:((key = 'srcpartkey') and value is not null)
- TableScan [TS_6] (rows=2000/2000 width=10)
- default@srcpart,srcpart,Tbl:COMPLETE,Col:NONE,Output:["key","value"]
-
-PREHOOK: query: SELECT x.key, z.value, y.value
-FROM src1 x JOIN src y ON (x.key = y.key)
-JOIN (select * from src1 union select * from src)z ON (x.value = z.value)
-union
-SELECT x.key, z.value, y.value
-FROM src1 x JOIN src y ON (x.key = y.key)
-JOIN (select * from src1 union select * from src)z ON (x.value = z.value)
-PREHOOK: type: QUERY
-PREHOOK: Input: default@src
-PREHOOK: Input: default@src1
-#### A masked pattern was here ####
-POSTHOOK: query: SELECT x.key, z.value, y.value
-FROM src1 x JOIN src y ON (x.key = y.key)
-JOIN (select * from src1 union select * from src)z ON (x.value = z.value)
-union
-SELECT x.key, z.value, y.value
-FROM src1 x JOIN src y ON (x.key = y.key)
-JOIN (select * from src1 union select * from src)z ON (x.value = z.value)
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@src
-POSTHOOK: Input: default@src1
-#### A masked pattern was here ####
-PREHOOK: query: explain analyze
-SELECT x.key, z.value, y.value
-FROM src1 x JOIN src y ON (x.key = y.key)
-JOIN (select * from src1 union select * from src)z ON (x.value = z.value)
-union
-SELECT x.key, z.value, y.value
-FROM src1 x JOIN src y ON (x.key = y.key)
-JOIN (select * from src1 union select * from src)z ON (x.value = z.value)
-PREHOOK: type: QUERY
-POSTHOOK: query: explain analyze
-SELECT x.key, z.value, y.value
-FROM src1 x JOIN src y ON (x.key = y.key)
-JOIN (select * from src1 union select * from src)z ON (x.value = z.value)
-union
-SELECT x.key, z.value, y.value
-FROM src1 x JOIN src y ON (x.key = y.key)
-JOIN (select * from src1 union select * from src)z ON (x.value = z.value)
-POSTHOOK: type: QUERY
-Plan optimized by CBO.
-
-Vertex dependency in root stage
-Map 1 <- Union 2 (CONTAINS)
-Map 12 <- Union 10 (CONTAINS)
-Map 6 <- Union 2 (CONTAINS)
-Map 9 <- Union 10 (CONTAINS)
-Reducer 11 <- Map 13 (BROADCAST_EDGE), Map 14 (BROADCAST_EDGE), Union 10 (SIMPLE_EDGE), Union 4 (CONTAINS)
-Reducer 3 <- Map 7 (BROADCAST_EDGE), Map 8 (BROADCAST_EDGE), Union 2 (SIMPLE_EDGE), Union 4 (CONTAINS)
-Reducer 5 <- Union 4 (SIMPLE_EDGE)
-
-Stage-0
- Fetch Operator
- limit:-1
- Stage-1
- Reducer 5
- File Output Operator [FS_59]
- Group By Operator [GBY_57] (rows=550/15 width=10)
- Output:["_col0","_col1"],keys:KEY._col0, KEY._col1
- <-Union 4 [SIMPLE_EDGE]
- <-Reducer 11 [CONTAINS]
- Reduce Output Operator [RS_56]
- PartitionCols:_col0, _col1
- Group By Operator [GBY_55] (rows=1100/30 width=10)
- Output:["_col0","_col1"],keys:_col0, _col1
- Select Operator [SEL_51] (rows=550/61 width=10)
- Output:["_col0","_col1"]
- Map Join Operator [MAPJOIN_85] (rows=550/61 width=10)
- Conds:MAPJOIN_84._col2=RS_49._col0(Inner),HybridGraceHashJoin:true,Output:["_col1","_col2"]
- <-Map 14 [BROADCAST_EDGE]
- BROADCAST [RS_49]
- PartitionCols:_col0
- Select Operator [SEL_44] (rows=500/500 width=10)
- Output:["_col0"]
- Filter Operator [FIL_81] (rows=500/500 width=10)
- predicate:key is not null
- TableScan [TS_42] (rows=500/500 width=10)
- default@src,y,Tbl:COMPLETE,Col:NONE,Output:["key"]
- <-Map Join Operator [MAPJOIN_84] (rows=288/52 width=10)
- Conds:SEL_38._col1=RS_46._col1(Inner),HybridGraceHashJoin:true,Output:["_col1","_col2"]
- <-Map 13 [BROADCAST_EDGE]
- BROADCAST [RS_46]
- PartitionCols:_col1
- Select Operator [SEL_41] (rows=25/25 width=7)
- Output:["_col0","_col1"]
- Filter Operator [FIL_80] (rows=25/25 width=7)
- predicate:(key is not null and value is not null)
- TableScan [TS_39] (rows=25/25 width=7)
- default@src1,x,Tbl:COMPLETE,Col:NONE,Output:["key","value"]
- <-Select Operator [SEL_38] (rows=262/319 width=10)
- Output:["_col1"]
- Group By Operator [GBY_37] (rows=262/319 width=10)
- Output:["_col0","_col1"],keys:KEY._col0, KEY._col1
- <-Union 10 [SIMPLE_EDGE]
- <-Map 12 [CONTAINS]
- Reduce Output Operator [RS_36]
- PartitionCols:_col0, _col1
- Group By Operator [GBY_35] (rows=525/331 width=10)
- Output:["_col0","_col1"],keys:_col1, _col0
- Select Operator [SEL_31] (rows=500/500 width=10)
- Output:["_col0","_col1"]
- Filter Operator [FIL_79] (rows=500/500 width=10)
- predicate:value is not null
- TableScan [TS_29] (rows=500/500 width=10)
- Output:["key","value"]
- <-Map 9 [CONTAINS]
- Reduce Output Operator [RS_36]
- PartitionCols:_col0, _col1
- Group By Operator [GBY_35] (rows=525/331 width=10)
- Output:["_col0","_col1"],keys:_col1, _col0
- Select Operator [SEL_28] (rows=25/25 width=7)
- Output:["_col0","_col1"]
- Filter Operator [FIL_78] (rows=25/25 width=7)
- predicate:value is not null
- TableScan [TS_26] (rows=25/25 width=7)
- Output:["key","value"]
- <-Reducer 3 [CONTAINS]
- Reduce Output Operator [RS_56]
- PartitionCols:_col0, _col1
- Group By Operator [GBY_55] (rows=1100/30 width=10)
- Output:["_col0","_col1"],keys:_col0, _col1
- Select Operator [SEL_25] (rows=550/61 width=10)
- Output:["_col0","_col1"]
- Map Join Operator [MAPJOIN_83] (rows=550/61 width=10)
- Conds:MAPJOIN_82._col2=RS_23._col0(Inner),HybridGraceHashJoin:true,Output:["_col1","_col2"]
- <-Map 8 [BROADCAST_EDGE]
- BROADCAST [RS_23]
- PartitionCols:_col0
- Select Operator [SEL_18] (rows=500/500 width=10)
- Output:["_col0"]
- Filter Operator [FIL_77] (rows=500/500 width=10)
- predicate:key is not null
- TableScan [TS_16] (rows=500/500 width=10)
- default@src,y,Tbl:COMPLETE,Col:NONE,Output:["key"]
- <-Map Join Operator [MAPJOIN_82] (rows=288/52 width=10)
- Conds:SEL_12._col1=RS_20._col1(Inner),HybridGraceHashJoin:true,Output:["_col1","_col2"]
- <-Map 7 [BROADCAST_EDGE]
- BROADCAST [RS_20]
- PartitionCols:_col1
- Select Operator [SEL_15] (rows=25/25 width=7)
- Output:["_col0","_col1"]
- Filter Operator [FIL_76] (rows=25/25 width=7)
- predicate:(key is not null and value is not null)
- TableScan [TS_13] (rows=25/25 width=7)
- default@src1,x,Tbl:COMPLETE,Col:NONE,Output:["key","value"]
- <-Select Operator [SEL_12] (rows=262/319 width=10)
- Output:["_col1"]
- Group By Operator [GBY_11] (rows=262/319 width=10)
- Output:["_col0","_col1"],keys:KEY._col0, KEY._col1
- <-Union 2 [SIMPLE_EDGE]
- <-Map 1 [CONTAINS]
- Reduce Output Operator [RS_10]
- PartitionCols:_col0, _col1
- Group By Operator [GBY_9] (rows=525/331 width=10)
- Output:["_col0","_col1"],keys:_col1, _col0
- Select Operator [SEL_2] (rows=25/25 width=7)
- Output:["_col0","_col1"]
- Filter Operator [FIL_74] (rows=25/25 width=7)
- predicate:value is not null
- TableScan [TS_0] (rows=25/25 width=7)
- Output:["key","value"]
- <-Map 6 [CONTAINS]
- Reduce Output Operator [RS_10]
- PartitionCols:_col0, _col1
- Group By Operator [GBY_9] (rows=525/331 width=10)
- Output:["_col0","_col1"],keys:_col1, _col0
- Select Operator [SEL_5] (rows=500/500 width=10)
- Output:["_col0","_col1"]
- Filter Operator [FIL_75] (rows=500/500 width=10)
- predicate:value is not null
- TableScan [TS_3] (rows=500/500 width=10)
- Output:["key","value"]
-
-PREHOOK: query: SELECT x.key, y.value
-FROM src1 x JOIN src y ON (x.key = y.key)
-JOIN (select * from src1 union select * from src)z ON (x.value = z.value)
-union
-SELECT x.key, y.value
-FROM src1 x JOIN src y ON (x.key = y.key)
-JOIN (select key, value from src1 union select key, value from src union select key, value from src)z ON (x.value = z.value)
-union
-SELECT x.key, y.value
-FROM src1 x JOIN src y ON (x.key = y.key)
-JOIN (select key, value from src1 union select key, value from src union select key, value from src union select key, value from src)z ON (x.value = z.value)
-PREHOOK: type: QUERY
-PREHOOK: Input: default@src
-PREHOOK: Input: default@src1
-#### A masked pattern was here ####
-POSTHOOK: query: SELECT x.key, y.value
-FROM src1 x JOIN src y ON (x.key = y.key)
-JOIN (select * from src1 union select * from src)z ON (x.value = z.value)
-union
-SELECT x.key, y.value
-FROM src1 x JOIN src y ON (x.key = y.key)
-JOIN (select key, value from src1 union select key, value from src union select key, value from src)z ON (x.value = z.value)
-union
-SELECT x.key, y.value
-FROM src1 x JOIN src y ON (x.key = y.key)
-JOIN (select key, value from src1 union select key, value from src union select key, value from src union select key, value from src)z ON (x.value = z.value)
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@src
-POSTHOOK: Input: default@src1
-#### A masked pattern was here ####
-PREHOOK: query: explain analyze
-SELECT x.key, y.value
-FROM src1 x JOIN src y ON (x.key = y.key)
-JOIN (select * from src1 union select * from src)z ON (x.value = z.value)
-union
-SELECT x.key, y.value
-FROM src1 x JOIN src y ON (x.key = y.key)
-JOIN (select key, value from src1 union select key, value from src union select key, value from src)z ON (x.value = z.value)
-union
-SELECT x.key, y.value
-FROM src1 x JOIN src y ON (x.key = y.key)
-JOIN (select key, value from src1 union select key, value from src union select key, value from src union select key, value from src)z ON (x.value = z.value)
-PREHOOK: type: QUERY
-POSTHOOK: query: explain analyze
-SELECT x.key, y.value
-FROM src1 x JOIN src y ON (x.key = y.key)
-JOIN (select * from src1 union select * from src)z ON (x.value = z.value)
-union
-SELECT x.key, y.value
-FROM src1 x JOIN src y ON (x.key = y.key)
-JOIN (select key, value from src1 union select key, value from src union select key, value from src)z ON (x.value = z.value)
-union
-SELECT x.key, y.value
-FROM src1 x JOIN src y ON (x.key = y.key)
-JOIN (select key, value from src1 union select key, value from src union select key, value from src union select key, value from src)z ON (x.value = z.value)
-POSTHOOK: type: QUERY
-Plan optimized by CBO.
-
-Vertex dependency in root stage
-Map 1 <- Union 2 (CONTAINS)
-Map 11 <- Union 12 (CONTAINS)
-Map 16 <- Union 12 (CONTAINS)
-Map 17 <- Union 14 (CONTAINS)
-Map 20 <- Union 21 (CONTAINS)
-Map 27 <- Union 21 (CONTAINS)
-Map 28 <- Union 23 (CONTAINS)
-Map 29 <- Union 25 (CONTAINS)
-Map 8 <- Union 2 (CONTAINS)
-Reducer 13 <- Union 12 (SIMPLE_EDGE), Union 14 (CONTAINS)
-Reducer 15 <- Map 18 (BROADCAST_EDGE), Map 19 (BROADCAST_EDGE), Union 14 (SIMPLE_EDGE), Union 4 (CONTAINS)
-Reducer 22 <- Union 21 (SIMPLE_EDGE), Union 23 (CONTAINS)
-Reducer 24 <- Union 23 (SIMPLE_EDGE), Union 25 (CONTAINS)
-Reducer 26 <- Map 30 (BROADCAST_EDGE), Map 31 (BROADCAST_EDGE), Union 25 (SIMPLE_EDGE), Union 6 (CONTAINS)
-Reducer 3 <- Map 10 (BROADCAST_EDGE), Map 9 (BROADCAST_EDGE), Union 2 (SIMPLE_EDGE), Union 4 (CONTAINS)
-Reducer 5 <- Union 4 (SIMPLE_EDGE), Union 6 (CONTAINS)
-Reducer 7 <- Union 6 (SIMPLE_EDGE)
-
-Stage-0
- Fetch Operator
- limit:-1
- Stage-1
- Reducer 7
- File Output Operator [FS_122]
- Group By Operator [GBY_120] (rows=550/15 width=10)
- Output:["_col0","_col1"],keys:KEY._col0, KEY._col1
- <-Union 6 [SIMPLE_EDGE]
- <-Reducer 26 [CONTAINS]
- Reduce Output Operator [RS_119]
- PartitionCols:_col0, _col1
- Group By Operator [GBY_118] (rows=1100/30 width=10)
- Output:["_col0","_col1"],keys:_col0, _col1
- Select Operator [SEL_114] (rows=550/61 width=10)
- Output:["_col0","_col1"]
- Map Join Operator [MAPJOIN_170] (rows=550/61 width=10)
- Conds:MAPJOIN_169._col2=RS_112._col0(Inner),HybridGraceHashJoin:true,Output:["_col2","_col5"]
- <-Map 31 [BROADCAST_EDGE]
- BROADCAST [RS_112]
- PartitionCols:_col0
- Select Operator [SEL_107] (rows=500/500 width=10)
- Output:["_col0","_col1"]
- Filter Operator [FIL_164] (rows=500/500 width=10)
- predicate:key is not null
- TableScan [TS_105] (rows=500/500 width=10)
- default@src,y,Tbl:COMPLETE,Col:NONE,Output:["key","value"]
- <-Map Join Operator [MAPJOIN_169] (rows=484/52 width=10)
- Conds:SEL_101._col1=RS_109._col1(Inner),HybridGraceHashJoin:true,Output:["_col2"]
- <-Map 30 [BROADCAST_EDGE]
- BROADCAST [RS_109]
- PartitionCols:_col1
- Select Operator [SEL_104] (rows=25/25 width=7)
- Output:["_col0","_col1"]
- Filter Operator [FIL_163] (rows=25/25 width=7)
- predicate:(key is not null and value is not null)
- TableScan [TS_102] (rows=25/25 width=7)
- default@src1,x,Tbl:COMPLETE,Col:NONE,Output:["key","value"]
- <-Select Operator [SEL_101] (rows=440/319 width=10)
- Output:["_col1"]
- Group By Operator [GBY_100] (rows=440/319 width=10)
-
<TRUNCATED>
[37/50] [abbrv] hive git commit: HIVE-16022: BloomFilter check not
showing up in MERGE statement queries (Jason Dere reviewed by Gunt her
Hagleitner)
Posted by om...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/c3d311f4/ql/src/test/results/clientpositive/llap/vectorized_dynamic_semijoin_reduction2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vectorized_dynamic_semijoin_reduction2.q.out b/ql/src/test/results/clientpositive/llap/vectorized_dynamic_semijoin_reduction2.q.out
index d53ca00..33c2302 100644
--- a/ql/src/test/results/clientpositive/llap/vectorized_dynamic_semijoin_reduction2.q.out
+++ b/ql/src/test/results/clientpositive/llap/vectorized_dynamic_semijoin_reduction2.q.out
@@ -109,10 +109,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: a
- filterExpr: (partkey_bigint is not null and partkey_bigint BETWEEN DynamicValue(RS_7_b_partkey_bigint_min) AND DynamicValue(RS_7_b_partkey_bigint_max) and in_bloom_filter(partkey_bigint, DynamicValue(RS_7_b_partkey_bigint_bloom_filter))) (type: boolean)
+ filterExpr: (partkey_bigint is not null and (partkey_bigint BETWEEN DynamicValue(RS_7_b_partkey_bigint_min) AND DynamicValue(RS_7_b_partkey_bigint_max) and in_bloom_filter(partkey_bigint, DynamicValue(RS_7_b_partkey_bigint_bloom_filter)))) (type: boolean)
Statistics: Num rows: 100 Data size: 50600 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (partkey_bigint is not null and partkey_bigint BETWEEN DynamicValue(RS_7_b_partkey_bigint_min) AND DynamicValue(RS_7_b_partkey_bigint_max) and in_bloom_filter(partkey_bigint, DynamicValue(RS_7_b_partkey_bigint_bloom_filter))) (type: boolean)
+ predicate: (partkey_bigint is not null and (partkey_bigint BETWEEN DynamicValue(RS_7_b_partkey_bigint_min) AND DynamicValue(RS_7_b_partkey_bigint_max) and in_bloom_filter(partkey_bigint, DynamicValue(RS_7_b_partkey_bigint_bloom_filter)))) (type: boolean)
Statistics: Num rows: 100 Data size: 50600 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: partkey_bigint (type: bigint)
@@ -245,10 +245,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: a
- filterExpr: (partkey_decimal is not null and partkey_decimal BETWEEN DynamicValue(RS_7_b_partkey_decimal_min) AND DynamicValue(RS_7_b_partkey_decimal_max) and in_bloom_filter(partkey_decimal, DynamicValue(RS_7_b_partkey_decimal_bloom_filter))) (type: boolean)
+ filterExpr: (partkey_decimal is not null and (partkey_decimal BETWEEN DynamicValue(RS_7_b_partkey_decimal_min) AND DynamicValue(RS_7_b_partkey_decimal_max) and in_bloom_filter(partkey_decimal, DynamicValue(RS_7_b_partkey_decimal_bloom_filter)))) (type: boolean)
Statistics: Num rows: 100 Data size: 50600 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (partkey_decimal is not null and partkey_decimal BETWEEN DynamicValue(RS_7_b_partkey_decimal_min) AND DynamicValue(RS_7_b_partkey_decimal_max) and in_bloom_filter(partkey_decimal, DynamicValue(RS_7_b_partkey_decimal_bloom_filter))) (type: boolean)
+ predicate: (partkey_decimal is not null and (partkey_decimal BETWEEN DynamicValue(RS_7_b_partkey_decimal_min) AND DynamicValue(RS_7_b_partkey_decimal_max) and in_bloom_filter(partkey_decimal, DynamicValue(RS_7_b_partkey_decimal_bloom_filter)))) (type: boolean)
Statistics: Num rows: 100 Data size: 50600 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: partkey_decimal (type: decimal(10,1))
@@ -381,10 +381,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: a
- filterExpr: (partkey_double is not null and partkey_double BETWEEN DynamicValue(RS_7_b_partkey_double_min) AND DynamicValue(RS_7_b_partkey_double_max) and in_bloom_filter(partkey_double, DynamicValue(RS_7_b_partkey_double_bloom_filter))) (type: boolean)
+ filterExpr: (partkey_double is not null and (partkey_double BETWEEN DynamicValue(RS_7_b_partkey_double_min) AND DynamicValue(RS_7_b_partkey_double_max) and in_bloom_filter(partkey_double, DynamicValue(RS_7_b_partkey_double_bloom_filter)))) (type: boolean)
Statistics: Num rows: 100 Data size: 50600 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (partkey_double is not null and partkey_double BETWEEN DynamicValue(RS_7_b_partkey_double_min) AND DynamicValue(RS_7_b_partkey_double_max) and in_bloom_filter(partkey_double, DynamicValue(RS_7_b_partkey_double_bloom_filter))) (type: boolean)
+ predicate: (partkey_double is not null and (partkey_double BETWEEN DynamicValue(RS_7_b_partkey_double_min) AND DynamicValue(RS_7_b_partkey_double_max) and in_bloom_filter(partkey_double, DynamicValue(RS_7_b_partkey_double_bloom_filter)))) (type: boolean)
Statistics: Num rows: 100 Data size: 50600 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: partkey_double (type: double)
@@ -517,10 +517,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: a
- filterExpr: (shipdate_date is not null and shipdate_date BETWEEN DynamicValue(RS_7_b_shipdate_date_min) AND DynamicValue(RS_7_b_shipdate_date_max) and in_bloom_filter(shipdate_date, DynamicValue(RS_7_b_shipdate_date_bloom_filter))) (type: boolean)
+ filterExpr: (shipdate_date is not null and (shipdate_date BETWEEN DynamicValue(RS_7_b_shipdate_date_min) AND DynamicValue(RS_7_b_shipdate_date_max) and in_bloom_filter(shipdate_date, DynamicValue(RS_7_b_shipdate_date_bloom_filter)))) (type: boolean)
Statistics: Num rows: 100 Data size: 50600 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (shipdate_date is not null and shipdate_date BETWEEN DynamicValue(RS_7_b_shipdate_date_min) AND DynamicValue(RS_7_b_shipdate_date_max) and in_bloom_filter(shipdate_date, DynamicValue(RS_7_b_shipdate_date_bloom_filter))) (type: boolean)
+ predicate: (shipdate_date is not null and (shipdate_date BETWEEN DynamicValue(RS_7_b_shipdate_date_min) AND DynamicValue(RS_7_b_shipdate_date_max) and in_bloom_filter(shipdate_date, DynamicValue(RS_7_b_shipdate_date_bloom_filter)))) (type: boolean)
Statistics: Num rows: 100 Data size: 50600 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: shipdate_date (type: date)
@@ -653,10 +653,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: a
- filterExpr: (shipdate_ts is not null and shipdate_ts BETWEEN DynamicValue(RS_7_b_shipdate_ts_min) AND DynamicValue(RS_7_b_shipdate_ts_max) and in_bloom_filter(shipdate_ts, DynamicValue(RS_7_b_shipdate_ts_bloom_filter))) (type: boolean)
+ filterExpr: (shipdate_ts is not null and (shipdate_ts BETWEEN DynamicValue(RS_7_b_shipdate_ts_min) AND DynamicValue(RS_7_b_shipdate_ts_max) and in_bloom_filter(shipdate_ts, DynamicValue(RS_7_b_shipdate_ts_bloom_filter)))) (type: boolean)
Statistics: Num rows: 100 Data size: 50600 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (shipdate_ts is not null and shipdate_ts BETWEEN DynamicValue(RS_7_b_shipdate_ts_min) AND DynamicValue(RS_7_b_shipdate_ts_max) and in_bloom_filter(shipdate_ts, DynamicValue(RS_7_b_shipdate_ts_bloom_filter))) (type: boolean)
+ predicate: (shipdate_ts is not null and (shipdate_ts BETWEEN DynamicValue(RS_7_b_shipdate_ts_min) AND DynamicValue(RS_7_b_shipdate_ts_max) and in_bloom_filter(shipdate_ts, DynamicValue(RS_7_b_shipdate_ts_bloom_filter)))) (type: boolean)
Statistics: Num rows: 100 Data size: 50600 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: shipdate_ts (type: timestamp)
@@ -789,10 +789,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: a
- filterExpr: (shipdate_string is not null and shipdate_string BETWEEN DynamicValue(RS_7_b_shipdate_string_min) AND DynamicValue(RS_7_b_shipdate_string_max) and in_bloom_filter(shipdate_string, DynamicValue(RS_7_b_shipdate_string_bloom_filter))) (type: boolean)
+ filterExpr: (shipdate_string is not null and (shipdate_string BETWEEN DynamicValue(RS_7_b_shipdate_string_min) AND DynamicValue(RS_7_b_shipdate_string_max) and in_bloom_filter(shipdate_string, DynamicValue(RS_7_b_shipdate_string_bloom_filter)))) (type: boolean)
Statistics: Num rows: 100 Data size: 50600 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (shipdate_string is not null and shipdate_string BETWEEN DynamicValue(RS_7_b_shipdate_string_min) AND DynamicValue(RS_7_b_shipdate_string_max) and in_bloom_filter(shipdate_string, DynamicValue(RS_7_b_shipdate_string_bloom_filter))) (type: boolean)
+ predicate: (shipdate_string is not null and (shipdate_string BETWEEN DynamicValue(RS_7_b_shipdate_string_min) AND DynamicValue(RS_7_b_shipdate_string_max) and in_bloom_filter(shipdate_string, DynamicValue(RS_7_b_shipdate_string_bloom_filter)))) (type: boolean)
Statistics: Num rows: 100 Data size: 50600 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: shipdate_string (type: string)
@@ -925,10 +925,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: a
- filterExpr: (shipdate_char is not null and shipdate_char BETWEEN DynamicValue(RS_7_b_shipdate_char_min) AND DynamicValue(RS_7_b_shipdate_char_max) and in_bloom_filter(shipdate_char, DynamicValue(RS_7_b_shipdate_char_bloom_filter))) (type: boolean)
+ filterExpr: (shipdate_char is not null and (shipdate_char BETWEEN DynamicValue(RS_7_b_shipdate_char_min) AND DynamicValue(RS_7_b_shipdate_char_max) and in_bloom_filter(shipdate_char, DynamicValue(RS_7_b_shipdate_char_bloom_filter)))) (type: boolean)
Statistics: Num rows: 100 Data size: 50600 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (shipdate_char is not null and shipdate_char BETWEEN DynamicValue(RS_7_b_shipdate_char_min) AND DynamicValue(RS_7_b_shipdate_char_max) and in_bloom_filter(shipdate_char, DynamicValue(RS_7_b_shipdate_char_bloom_filter))) (type: boolean)
+ predicate: (shipdate_char is not null and (shipdate_char BETWEEN DynamicValue(RS_7_b_shipdate_char_min) AND DynamicValue(RS_7_b_shipdate_char_max) and in_bloom_filter(shipdate_char, DynamicValue(RS_7_b_shipdate_char_bloom_filter)))) (type: boolean)
Statistics: Num rows: 100 Data size: 50600 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: shipdate_char (type: char(10))
@@ -1061,10 +1061,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: a
- filterExpr: (shipdate_varchar is not null and shipdate_varchar BETWEEN DynamicValue(RS_7_b_shipdate_varchar_min) AND DynamicValue(RS_7_b_shipdate_varchar_max) and in_bloom_filter(shipdate_varchar, DynamicValue(RS_7_b_shipdate_varchar_bloom_filter))) (type: boolean)
+ filterExpr: (shipdate_varchar is not null and (shipdate_varchar BETWEEN DynamicValue(RS_7_b_shipdate_varchar_min) AND DynamicValue(RS_7_b_shipdate_varchar_max) and in_bloom_filter(shipdate_varchar, DynamicValue(RS_7_b_shipdate_varchar_bloom_filter)))) (type: boolean)
Statistics: Num rows: 100 Data size: 50600 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (shipdate_varchar is not null and shipdate_varchar BETWEEN DynamicValue(RS_7_b_shipdate_varchar_min) AND DynamicValue(RS_7_b_shipdate_varchar_max) and in_bloom_filter(shipdate_varchar, DynamicValue(RS_7_b_shipdate_varchar_bloom_filter))) (type: boolean)
+ predicate: (shipdate_varchar is not null and (shipdate_varchar BETWEEN DynamicValue(RS_7_b_shipdate_varchar_min) AND DynamicValue(RS_7_b_shipdate_varchar_max) and in_bloom_filter(shipdate_varchar, DynamicValue(RS_7_b_shipdate_varchar_bloom_filter)))) (type: boolean)
Statistics: Num rows: 100 Data size: 50600 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: shipdate_varchar (type: varchar(10))
http://git-wip-us.apache.org/repos/asf/hive/blob/c3d311f4/ql/src/test/results/clientpositive/tez/dynamic_partition_pruning.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/dynamic_partition_pruning.q.out b/ql/src/test/results/clientpositive/tez/dynamic_partition_pruning.q.out
index ff424a3..8bce87b 100644
--- a/ql/src/test/results/clientpositive/tez/dynamic_partition_pruning.q.out
+++ b/ql/src/test/results/clientpositive/tez/dynamic_partition_pruning.q.out
@@ -3038,10 +3038,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: srcpart_hour
- filterExpr: ((UDFToDouble(hr) = 13.0) and hr BETWEEN DynamicValue(RS_12_srcpart_hr_min) AND DynamicValue(RS_12_srcpart_hr_max) and in_bloom_filter(hr, DynamicValue(RS_12_srcpart_hr_bloom_filter))) (type: boolean)
+ filterExpr: ((UDFToDouble(hr) = 13.0) and (hr BETWEEN DynamicValue(RS_12_srcpart_hr_min) AND DynamicValue(RS_12_srcpart_hr_max) and in_bloom_filter(hr, DynamicValue(RS_12_srcpart_hr_bloom_filter)))) (type: boolean)
Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((UDFToDouble(hr) = 13.0) and hr BETWEEN DynamicValue(RS_12_srcpart_hr_min) AND DynamicValue(RS_12_srcpart_hr_max) and in_bloom_filter(hr, DynamicValue(RS_12_srcpart_hr_bloom_filter))) (type: boolean)
+ predicate: ((UDFToDouble(hr) = 13.0) and (hr BETWEEN DynamicValue(RS_12_srcpart_hr_min) AND DynamicValue(RS_12_srcpart_hr_max) and in_bloom_filter(hr, DynamicValue(RS_12_srcpart_hr_bloom_filter)))) (type: boolean)
Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: hr (type: string)
http://git-wip-us.apache.org/repos/asf/hive/blob/c3d311f4/ql/src/test/results/clientpositive/tez/explainuser_3.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/explainuser_3.q.out b/ql/src/test/results/clientpositive/tez/explainuser_3.q.out
index 1fa21fe..9eb65cf 100644
--- a/ql/src/test/results/clientpositive/tez/explainuser_3.q.out
+++ b/ql/src/test/results/clientpositive/tez/explainuser_3.q.out
@@ -697,10 +697,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: b
- filterExpr: (key is not null and key BETWEEN DynamicValue(RS_6_a_key_min) AND DynamicValue(RS_6_a_key_max) and in_bloom_filter(key, DynamicValue(RS_6_a_key_bloom_filter))) (type: boolean)
+ filterExpr: (key is not null and (key BETWEEN DynamicValue(RS_6_a_key_min) AND DynamicValue(RS_6_a_key_max) and in_bloom_filter(key, DynamicValue(RS_6_a_key_bloom_filter)))) (type: boolean)
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (key is not null and key BETWEEN DynamicValue(RS_6_a_key_min) AND DynamicValue(RS_6_a_key_max) and in_bloom_filter(key, DynamicValue(RS_6_a_key_bloom_filter))) (type: boolean)
+ predicate: (key is not null and (key BETWEEN DynamicValue(RS_6_a_key_min) AND DynamicValue(RS_6_a_key_max) and in_bloom_filter(key, DynamicValue(RS_6_a_key_bloom_filter)))) (type: boolean)
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: int), value (type: string)
http://git-wip-us.apache.org/repos/asf/hive/blob/c3d311f4/ql/src/test/results/clientpositive/tez/mergejoin.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/mergejoin.q.out b/ql/src/test/results/clientpositive/tez/mergejoin.q.out
index 5853b2b..b860b9b 100644
--- a/ql/src/test/results/clientpositive/tez/mergejoin.q.out
+++ b/ql/src/test/results/clientpositive/tez/mergejoin.q.out
@@ -22,10 +22,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: a
- filterExpr: (key is not null and key BETWEEN DynamicValue(RS_7_b_key_min) AND DynamicValue(RS_7_b_key_max) and in_bloom_filter(key, DynamicValue(RS_7_b_key_bloom_filter))) (type: boolean)
+ filterExpr: (key is not null and (key BETWEEN DynamicValue(RS_7_b_key_min) AND DynamicValue(RS_7_b_key_max) and in_bloom_filter(key, DynamicValue(RS_7_b_key_bloom_filter)))) (type: boolean)
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (key is not null and key BETWEEN DynamicValue(RS_7_b_key_min) AND DynamicValue(RS_7_b_key_max) and in_bloom_filter(key, DynamicValue(RS_7_b_key_bloom_filter))) (type: boolean)
+ predicate: (key is not null and (key BETWEEN DynamicValue(RS_7_b_key_min) AND DynamicValue(RS_7_b_key_max) and in_bloom_filter(key, DynamicValue(RS_7_b_key_bloom_filter)))) (type: boolean)
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
@@ -321,10 +321,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: b
- filterExpr: (key is not null and key BETWEEN DynamicValue(RS_6_a_key_min) AND DynamicValue(RS_6_a_key_max) and in_bloom_filter(key, DynamicValue(RS_6_a_key_bloom_filter))) (type: boolean)
+ filterExpr: (key is not null and (key BETWEEN DynamicValue(RS_6_a_key_min) AND DynamicValue(RS_6_a_key_max) and in_bloom_filter(key, DynamicValue(RS_6_a_key_bloom_filter)))) (type: boolean)
Statistics: Num rows: 500 Data size: 47000 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (key is not null and key BETWEEN DynamicValue(RS_6_a_key_min) AND DynamicValue(RS_6_a_key_max) and in_bloom_filter(key, DynamicValue(RS_6_a_key_bloom_filter))) (type: boolean)
+ predicate: (key is not null and (key BETWEEN DynamicValue(RS_6_a_key_min) AND DynamicValue(RS_6_a_key_max) and in_bloom_filter(key, DynamicValue(RS_6_a_key_bloom_filter)))) (type: boolean)
Statistics: Num rows: 500 Data size: 47000 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: int)
@@ -1433,7 +1433,7 @@ STAGE PLANS:
alias: b
Statistics: Num rows: 500 Data size: 47000 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: key BETWEEN DynamicValue(RS_4_a_key_min) AND DynamicValue(RS_4_a_key_max) (type: boolean)
+ predicate: (key BETWEEN DynamicValue(RS_4_a_key_min) AND DynamicValue(RS_4_a_key_max) and in_bloom_filter(key, DynamicValue(RS_4_a_key_bloom_filter))) (type: boolean)
Statistics: Num rows: 500 Data size: 47000 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: int)
@@ -1543,7 +1543,7 @@ STAGE PLANS:
alias: a
Statistics: Num rows: 242 Data size: 22748 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: key BETWEEN DynamicValue(RS_5_b_key_min) AND DynamicValue(RS_5_b_key_max) (type: boolean)
+ predicate: (key BETWEEN DynamicValue(RS_5_b_key_min) AND DynamicValue(RS_5_b_key_max) and in_bloom_filter(key, DynamicValue(RS_5_b_key_bloom_filter))) (type: boolean)
Statistics: Num rows: 242 Data size: 22748 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: int)
@@ -1786,10 +1786,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: a
- filterExpr: (key is not null and value is not null and value BETWEEN DynamicValue(RS_13_c_value_min) AND DynamicValue(RS_13_c_value_max) and in_bloom_filter(value, DynamicValue(RS_13_c_value_bloom_filter))) (type: boolean)
+ filterExpr: (key is not null and value is not null and (value BETWEEN DynamicValue(RS_13_c_value_min) AND DynamicValue(RS_13_c_value_max) and in_bloom_filter(value, DynamicValue(RS_13_c_value_bloom_filter)))) (type: boolean)
Statistics: Num rows: 242 Data size: 22748 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (key is not null and value is not null and value BETWEEN DynamicValue(RS_13_c_value_min) AND DynamicValue(RS_13_c_value_max) and in_bloom_filter(value, DynamicValue(RS_13_c_value_bloom_filter))) (type: boolean)
+ predicate: (key is not null and value is not null and (value BETWEEN DynamicValue(RS_13_c_value_min) AND DynamicValue(RS_13_c_value_max) and in_bloom_filter(value, DynamicValue(RS_13_c_value_bloom_filter)))) (type: boolean)
Statistics: Num rows: 242 Data size: 22748 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: int), value (type: string)
@@ -1819,10 +1819,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: b
- filterExpr: (key is not null and key BETWEEN DynamicValue(RS_9_a_key_min) AND DynamicValue(RS_9_a_key_max) and in_bloom_filter(key, DynamicValue(RS_9_a_key_bloom_filter))) (type: boolean)
+ filterExpr: (key is not null and (key BETWEEN DynamicValue(RS_9_a_key_min) AND DynamicValue(RS_9_a_key_max) and in_bloom_filter(key, DynamicValue(RS_9_a_key_bloom_filter)))) (type: boolean)
Statistics: Num rows: 500 Data size: 47000 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (key is not null and key BETWEEN DynamicValue(RS_9_a_key_min) AND DynamicValue(RS_9_a_key_max) and in_bloom_filter(key, DynamicValue(RS_9_a_key_bloom_filter))) (type: boolean)
+ predicate: (key is not null and (key BETWEEN DynamicValue(RS_9_a_key_min) AND DynamicValue(RS_9_a_key_max) and in_bloom_filter(key, DynamicValue(RS_9_a_key_bloom_filter)))) (type: boolean)
Statistics: Num rows: 500 Data size: 47000 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: int)
@@ -2016,10 +2016,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: b
- filterExpr: (value is not null and value BETWEEN DynamicValue(RS_6_a_value_min) AND DynamicValue(RS_6_a_value_max) and in_bloom_filter(value, DynamicValue(RS_6_a_value_bloom_filter))) (type: boolean)
+ filterExpr: (value is not null and (value BETWEEN DynamicValue(RS_6_a_value_min) AND DynamicValue(RS_6_a_value_max) and in_bloom_filter(value, DynamicValue(RS_6_a_value_bloom_filter)))) (type: boolean)
Statistics: Num rows: 500 Data size: 47000 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (value is not null and value BETWEEN DynamicValue(RS_6_a_value_min) AND DynamicValue(RS_6_a_value_max) and in_bloom_filter(value, DynamicValue(RS_6_a_value_bloom_filter))) (type: boolean)
+ predicate: (value is not null and (value BETWEEN DynamicValue(RS_6_a_value_min) AND DynamicValue(RS_6_a_value_max) and in_bloom_filter(value, DynamicValue(RS_6_a_value_bloom_filter)))) (type: boolean)
Statistics: Num rows: 500 Data size: 47000 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: value (type: string)
@@ -2201,10 +2201,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: b
- filterExpr: (key is not null and key BETWEEN DynamicValue(RS_18_s1_key_min) AND DynamicValue(RS_18_s1_key_max) and in_bloom_filter(key, DynamicValue(RS_18_s1_key_bloom_filter))) (type: boolean)
+ filterExpr: (key is not null and (key BETWEEN DynamicValue(RS_18_s1_key_min) AND DynamicValue(RS_18_s1_key_max) and in_bloom_filter(key, DynamicValue(RS_18_s1_key_bloom_filter)))) (type: boolean)
Statistics: Num rows: 500 Data size: 47000 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (key is not null and key BETWEEN DynamicValue(RS_18_s1_key_min) AND DynamicValue(RS_18_s1_key_max) and in_bloom_filter(key, DynamicValue(RS_18_s1_key_bloom_filter))) (type: boolean)
+ predicate: (key is not null and (key BETWEEN DynamicValue(RS_18_s1_key_min) AND DynamicValue(RS_18_s1_key_max) and in_bloom_filter(key, DynamicValue(RS_18_s1_key_bloom_filter)))) (type: boolean)
Statistics: Num rows: 500 Data size: 47000 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: int)
@@ -2353,10 +2353,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: b
- filterExpr: (value is not null and value BETWEEN DynamicValue(RS_6_a_value_min) AND DynamicValue(RS_6_a_value_max) and in_bloom_filter(value, DynamicValue(RS_6_a_value_bloom_filter))) (type: boolean)
+ filterExpr: (value is not null and (value BETWEEN DynamicValue(RS_6_a_value_min) AND DynamicValue(RS_6_a_value_max) and in_bloom_filter(value, DynamicValue(RS_6_a_value_bloom_filter)))) (type: boolean)
Statistics: Num rows: 500 Data size: 47000 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (value is not null and value BETWEEN DynamicValue(RS_6_a_value_min) AND DynamicValue(RS_6_a_value_max) and in_bloom_filter(value, DynamicValue(RS_6_a_value_bloom_filter))) (type: boolean)
+ predicate: (value is not null and (value BETWEEN DynamicValue(RS_6_a_value_min) AND DynamicValue(RS_6_a_value_max) and in_bloom_filter(value, DynamicValue(RS_6_a_value_bloom_filter)))) (type: boolean)
Statistics: Num rows: 500 Data size: 47000 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: value (type: string)
@@ -2461,10 +2461,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: a
- filterExpr: (key is not null and value is not null and value BETWEEN DynamicValue(RS_13_c_value_min) AND DynamicValue(RS_13_c_value_max) and in_bloom_filter(value, DynamicValue(RS_13_c_value_bloom_filter))) (type: boolean)
+ filterExpr: (key is not null and value is not null and (value BETWEEN DynamicValue(RS_13_c_value_min) AND DynamicValue(RS_13_c_value_max) and in_bloom_filter(value, DynamicValue(RS_13_c_value_bloom_filter)))) (type: boolean)
Statistics: Num rows: 242 Data size: 22748 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (key is not null and value is not null and value BETWEEN DynamicValue(RS_13_c_value_min) AND DynamicValue(RS_13_c_value_max) and in_bloom_filter(value, DynamicValue(RS_13_c_value_bloom_filter))) (type: boolean)
+ predicate: (key is not null and value is not null and (value BETWEEN DynamicValue(RS_13_c_value_min) AND DynamicValue(RS_13_c_value_max) and in_bloom_filter(value, DynamicValue(RS_13_c_value_bloom_filter)))) (type: boolean)
Statistics: Num rows: 242 Data size: 22748 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: int), value (type: string)
@@ -2494,10 +2494,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: b
- filterExpr: (key is not null and key BETWEEN DynamicValue(RS_9_a_key_min) AND DynamicValue(RS_9_a_key_max) and in_bloom_filter(key, DynamicValue(RS_9_a_key_bloom_filter))) (type: boolean)
+ filterExpr: (key is not null and (key BETWEEN DynamicValue(RS_9_a_key_min) AND DynamicValue(RS_9_a_key_max) and in_bloom_filter(key, DynamicValue(RS_9_a_key_bloom_filter)))) (type: boolean)
Statistics: Num rows: 500 Data size: 47000 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (key is not null and key BETWEEN DynamicValue(RS_9_a_key_min) AND DynamicValue(RS_9_a_key_max) and in_bloom_filter(key, DynamicValue(RS_9_a_key_bloom_filter))) (type: boolean)
+ predicate: (key is not null and (key BETWEEN DynamicValue(RS_9_a_key_min) AND DynamicValue(RS_9_a_key_max) and in_bloom_filter(key, DynamicValue(RS_9_a_key_bloom_filter)))) (type: boolean)
Statistics: Num rows: 500 Data size: 47000 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: int)
@@ -2739,10 +2739,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: b
- filterExpr: (key is not null and key BETWEEN DynamicValue(RS_18_s1_key_min) AND DynamicValue(RS_18_s1_key_max) and in_bloom_filter(key, DynamicValue(RS_18_s1_key_bloom_filter))) (type: boolean)
+ filterExpr: (key is not null and (key BETWEEN DynamicValue(RS_18_s1_key_min) AND DynamicValue(RS_18_s1_key_max) and in_bloom_filter(key, DynamicValue(RS_18_s1_key_bloom_filter)))) (type: boolean)
Statistics: Num rows: 500 Data size: 47000 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (key is not null and key BETWEEN DynamicValue(RS_18_s1_key_min) AND DynamicValue(RS_18_s1_key_max) and in_bloom_filter(key, DynamicValue(RS_18_s1_key_bloom_filter))) (type: boolean)
+ predicate: (key is not null and (key BETWEEN DynamicValue(RS_18_s1_key_min) AND DynamicValue(RS_18_s1_key_max) and in_bloom_filter(key, DynamicValue(RS_18_s1_key_bloom_filter)))) (type: boolean)
Statistics: Num rows: 500 Data size: 47000 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: int)
@@ -2893,10 +2893,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: t2
- filterExpr: (key is not null and key BETWEEN DynamicValue(RS_12_t1_key_min) AND DynamicValue(RS_12_t1_key_max) and in_bloom_filter(key, DynamicValue(RS_12_t1_key_bloom_filter))) (type: boolean)
+ filterExpr: (key is not null and (key BETWEEN DynamicValue(RS_12_t1_key_min) AND DynamicValue(RS_12_t1_key_max) and in_bloom_filter(key, DynamicValue(RS_12_t1_key_bloom_filter)))) (type: boolean)
Statistics: Num rows: 500 Data size: 47000 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (key is not null and key BETWEEN DynamicValue(RS_12_t1_key_min) AND DynamicValue(RS_12_t1_key_max) and in_bloom_filter(key, DynamicValue(RS_12_t1_key_bloom_filter))) (type: boolean)
+ predicate: (key is not null and (key BETWEEN DynamicValue(RS_12_t1_key_min) AND DynamicValue(RS_12_t1_key_max) and in_bloom_filter(key, DynamicValue(RS_12_t1_key_bloom_filter)))) (type: boolean)
Statistics: Num rows: 500 Data size: 47000 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: int), value (type: string)
http://git-wip-us.apache.org/repos/asf/hive/blob/c3d311f4/ql/src/test/results/clientpositive/tez/vectorized_dynamic_partition_pruning.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/vectorized_dynamic_partition_pruning.q.out b/ql/src/test/results/clientpositive/tez/vectorized_dynamic_partition_pruning.q.out
index 1e688eb..d9df49d 100644
--- a/ql/src/test/results/clientpositive/tez/vectorized_dynamic_partition_pruning.q.out
+++ b/ql/src/test/results/clientpositive/tez/vectorized_dynamic_partition_pruning.q.out
@@ -2853,10 +2853,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: srcpart_hour
- filterExpr: ((UDFToDouble(hr) = 13.0) and hr BETWEEN DynamicValue(RS_12_srcpart_hr_min) AND DynamicValue(RS_12_srcpart_hr_max) and in_bloom_filter(hr, DynamicValue(RS_12_srcpart_hr_bloom_filter))) (type: boolean)
+ filterExpr: ((UDFToDouble(hr) = 13.0) and (hr BETWEEN DynamicValue(RS_12_srcpart_hr_min) AND DynamicValue(RS_12_srcpart_hr_max) and in_bloom_filter(hr, DynamicValue(RS_12_srcpart_hr_bloom_filter)))) (type: boolean)
Statistics: Num rows: 2 Data size: 344 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((UDFToDouble(hr) = 13.0) and hr BETWEEN DynamicValue(RS_12_srcpart_hr_min) AND DynamicValue(RS_12_srcpart_hr_max) and in_bloom_filter(hr, DynamicValue(RS_12_srcpart_hr_bloom_filter))) (type: boolean)
+ predicate: ((UDFToDouble(hr) = 13.0) and (hr BETWEEN DynamicValue(RS_12_srcpart_hr_min) AND DynamicValue(RS_12_srcpart_hr_max) and in_bloom_filter(hr, DynamicValue(RS_12_srcpart_hr_bloom_filter)))) (type: boolean)
Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: hr (type: string)
[24/50] [abbrv] hive git commit: HIVE-16020: LLAP : Reduce IPC
connection misses (Rajesh Balamohan, Siddharth Seth,
reviewed by Sergey Shelukhin)
Posted by om...@apache.org.
HIVE-16020: LLAP : Reduce IPC connection misses (Rajesh Balamohan, Siddharth Seth, reviewed by Sergey Shelukhin)
Change-Id: Ibf7f4c1a9840fa1463a7c7c4a777e5a276254e2e
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/59cde667
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/59cde667
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/59cde667
Branch: refs/heads/branch-2.2
Commit: 59cde667d77b549409898fdb30a549223a251e32
Parents: 5be5391
Author: Prasanth Jayachandran <pj...@hortonworks.com>
Authored: Mon Feb 27 19:44:19 2017 -0800
Committer: Owen O'Malley <om...@apache.org>
Committed: Tue Mar 28 15:27:52 2017 -0700
----------------------------------------------------------------------
.../hive/llap/daemon/impl/AMReporter.java | 5 ++--
.../llap/daemon/impl/ContainerRunnerImpl.java | 16 ++++++----
.../hive/llap/daemon/impl/LlapDaemon.java | 8 +++--
.../hadoop/hive/llap/daemon/impl/QueryInfo.java | 29 ++++++++++++++++++
.../hive/llap/daemon/impl/QueryTracker.java | 4 ++-
.../llap/daemon/impl/TaskRunnerCallable.java | 31 +++++++++++---------
.../daemon/impl/TaskExecutorTestHelpers.java | 4 ++-
7 files changed, 72 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/59cde667/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/AMReporter.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/AMReporter.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/AMReporter.java
index 32f070a..ede7e00 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/AMReporter.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/AMReporter.java
@@ -105,7 +105,8 @@ public class AMReporter extends AbstractService {
private final DaemonId daemonId;
public AMReporter(int numExecutors, int maxThreads, AtomicReference<InetSocketAddress>
- localAddress, QueryFailedHandler queryFailedHandler, Configuration conf, DaemonId daemonId) {
+ localAddress, QueryFailedHandler queryFailedHandler, Configuration conf, DaemonId daemonId,
+ SocketFactory socketFactory) {
super(AMReporter.class.getName());
this.localAddress = localAddress;
this.queryFailedHandler = queryFailedHandler;
@@ -137,7 +138,7 @@ public class AMReporter extends AbstractService {
.retryUpToMaximumTimeWithFixedSleep(retryTimeout, retrySleep,
TimeUnit.MILLISECONDS);
- this.socketFactory = NetUtils.getDefaultSocketFactory(conf);
+ this.socketFactory = socketFactory;
LOG.info("Setting up AMReporter with " +
"heartbeatInterval(ms)=" + heartbeatInterval +
http://git-wip-us.apache.org/repos/asf/hive/blob/59cde667/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/ContainerRunnerImpl.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/ContainerRunnerImpl.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/ContainerRunnerImpl.java
index 6908138..cc4eff0 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/ContainerRunnerImpl.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/ContainerRunnerImpl.java
@@ -83,6 +83,8 @@ import com.google.common.base.Preconditions;
import com.google.protobuf.ByteString;
import com.google.protobuf.InvalidProtocolBufferException;
+import javax.net.SocketFactory;
+
public class ContainerRunnerImpl extends CompositeService implements ContainerRunner, FragmentCompletionHandler, QueryFailedHandler {
// TODO Setup a set of threads to process incoming requests.
@@ -107,12 +109,14 @@ public class ContainerRunnerImpl extends CompositeService implements ContainerRu
private final String clusterId;
private final DaemonId daemonId;
private final UgiFactory fsUgiFactory;
+ private final SocketFactory socketFactory;
public ContainerRunnerImpl(Configuration conf, int numExecutors, int waitQueueSize,
boolean enablePreemption, String[] localDirsBase, AtomicReference<Integer> localShufflePort,
AtomicReference<InetSocketAddress> localAddress,
long totalMemoryAvailableBytes, LlapDaemonExecutorMetrics metrics,
- AMReporter amReporter, ClassLoader classLoader, DaemonId daemonId, UgiFactory fsUgiFactory) {
+ AMReporter amReporter, ClassLoader classLoader, DaemonId daemonId, UgiFactory fsUgiFactory,
+ SocketFactory socketFactory) {
super("ContainerRunnerImpl");
Preconditions.checkState(numExecutors > 0,
"Invalid number of executors: " + numExecutors + ". Must be > 0");
@@ -122,6 +126,7 @@ public class ContainerRunnerImpl extends CompositeService implements ContainerRu
this.signer = UserGroupInformation.isSecurityEnabled()
? new LlapSignerImpl(conf, daemonId.getClusterString()) : null;
this.fsUgiFactory = fsUgiFactory;
+ this.socketFactory = socketFactory;
this.clusterId = daemonId.getClusterString();
this.daemonId = daemonId;
@@ -239,7 +244,8 @@ public class ContainerRunnerImpl extends CompositeService implements ContainerRu
queryIdentifier, qIdProto.getApplicationIdString(), dagId,
vertex.getDagName(), vertex.getHiveQueryId(), dagIdentifier,
vertex.getVertexName(), request.getFragmentNumber(), request.getAttemptNumber(),
- vertex.getUser(), vertex, jobToken, fragmentIdString, tokenInfo);
+ vertex.getUser(), vertex, jobToken, fragmentIdString, tokenInfo, request.getAmHost(),
+ request.getAmPort());
String[] localDirs = fragmentInfo.getLocalDirs();
Preconditions.checkNotNull(localDirs);
@@ -250,12 +256,12 @@ public class ContainerRunnerImpl extends CompositeService implements ContainerRu
// Used for re-localization, to add the user specified configuration (conf_pb_binary_stream)
Configuration callableConf = new Configuration(getConfig());
- UserGroupInformation taskUgi = fsUgiFactory == null ? null : fsUgiFactory.createUgi();
+ UserGroupInformation fsTaskUgi = fsUgiFactory == null ? null : fsUgiFactory.createUgi();
TaskRunnerCallable callable = new TaskRunnerCallable(request, fragmentInfo, callableConf,
new ExecutionContextImpl(localAddress.get().getHostName()), env,
credentials, memoryPerExecutor, amReporter, confParams, metrics, killedTaskHandler,
- this, tezHadoopShim, attemptId, vertex, initialEvent, taskUgi,
- completionListener);
+ this, tezHadoopShim, attemptId, vertex, initialEvent, fsTaskUgi,
+ completionListener, socketFactory);
submissionState = executorService.schedule(callable);
if (LOG.isInfoEnabled()) {
http://git-wip-us.apache.org/repos/asf/hive/blob/59cde667/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
index 0494f0d..1ede5a1 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
@@ -29,6 +29,7 @@ import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
import javax.management.ObjectName;
+import javax.net.SocketFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.common.LogUtils;
@@ -63,6 +64,7 @@ import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge.UdfWhitelistChecker;
import org.apache.hadoop.metrics2.util.MBeans;
+import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.service.CompositeService;
@@ -106,6 +108,7 @@ public class LlapDaemon extends CompositeService implements ContainerRunner, Lla
private final long maxJvmMemory;
private final String[] localDirs;
private final DaemonId daemonId;
+ private final SocketFactory socketFactory;
// TODO Not the best way to share the address
private final AtomicReference<InetSocketAddress> srvAddress = new AtomicReference<>(),
@@ -257,8 +260,9 @@ public class LlapDaemon extends CompositeService implements ContainerRunner, Lla
" sessionId: " + sessionId);
int maxAmReporterThreads = HiveConf.getIntVar(daemonConf, ConfVars.LLAP_DAEMON_AM_REPORTER_MAX_THREADS);
+ this.socketFactory = NetUtils.getDefaultSocketFactory(daemonConf);
this.amReporter = new AMReporter(numExecutors, maxAmReporterThreads, srvAddress,
- new QueryFailedHandlerProxy(), daemonConf, daemonId);
+ new QueryFailedHandlerProxy(), daemonConf, daemonId, socketFactory);
SecretManager sm = null;
if (UserGroupInformation.isSecurityEnabled()) {
@@ -276,7 +280,7 @@ public class LlapDaemon extends CompositeService implements ContainerRunner, Lla
}
this.containerRunner = new ContainerRunnerImpl(daemonConf, numExecutors, waitQueueSize,
enablePreemption, localDirs, this.shufflePort, srvAddress, executorMemoryPerInstance, metrics,
- amReporter, executorClassLoader, daemonId, fsUgiFactory);
+ amReporter, executorClassLoader, daemonId, fsUgiFactory, socketFactory);
addIfService(containerRunner);
// Not adding the registry as a service, since we need to control when it is initialized - conf used to pickup properties.
http://git-wip-us.apache.org/repos/asf/hive/blob/59cde667/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/QueryInfo.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/QueryInfo.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/QueryInfo.java
index 5d07e90..5f0271f 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/QueryInfo.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/QueryInfo.java
@@ -16,6 +16,7 @@ package org.apache.hadoop.hive.llap.daemon.impl;
import java.io.File;
import java.io.IOException;
+import java.net.InetSocketAddress;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
@@ -25,6 +26,7 @@ import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.locks.ReentrantLock;
import com.google.common.base.Preconditions;
@@ -37,6 +39,11 @@ import org.apache.hadoop.hive.llap.LlapUtil;
import org.apache.hadoop.hive.llap.daemon.FinishableStateUpdateHandler;
import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SignableVertexSpec;
import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SourceStateProto;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.tez.common.security.JobTokenIdentifier;
public class QueryInfo {
private final QueryIdentifier queryIdentifier;
@@ -58,6 +65,7 @@ public class QueryInfo {
private final FinishableStateTracker finishableStateTracker = new FinishableStateTracker();
private final String tokenUserName, appId;
+ private final AtomicReference<UserGroupInformation> umbilicalUgi;
public QueryInfo(QueryIdentifier queryIdentifier, String appIdString, String dagIdString,
String dagName, String hiveQueryIdString,
@@ -77,6 +85,7 @@ public class QueryInfo {
this.localFs = localFs;
this.tokenUserName = tokenUserName;
this.appId = tokenAppId;
+ this.umbilicalUgi = new AtomicReference<>();
}
public QueryIdentifier getQueryIdentifier() {
@@ -298,4 +307,24 @@ public class QueryInfo {
public String getTokenAppId() {
return appId;
}
+
+ public void setupUmbilicalUgi(String umbilicalUser, Token<JobTokenIdentifier> appToken, String amHost, int amPort) {
+ synchronized (umbilicalUgi) {
+ if (umbilicalUgi.get() == null) {
+ UserGroupInformation taskOwner =
+ UserGroupInformation.createRemoteUser(umbilicalUser);
+ final InetSocketAddress address =
+ NetUtils.createSocketAddrForHost(amHost, amPort);
+ SecurityUtil.setTokenService(appToken, address);
+ taskOwner.addToken(appToken);
+ umbilicalUgi.set(taskOwner);
+ }
+ }
+ }
+
+ public UserGroupInformation getUmbilicalUgi() {
+ synchronized (umbilicalUgi) {
+ return umbilicalUgi.get();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/59cde667/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/QueryTracker.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/QueryTracker.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/QueryTracker.java
index 9eaddd2..5cf3a38 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/QueryTracker.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/QueryTracker.java
@@ -139,7 +139,7 @@ public class QueryTracker extends AbstractService {
QueryFragmentInfo registerFragment(QueryIdentifier queryIdentifier, String appIdString, String dagIdString,
String dagName, String hiveQueryIdString, int dagIdentifier, String vertexName, int fragmentNumber, int attemptNumber,
String user, SignableVertexSpec vertex, Token<JobTokenIdentifier> appToken,
- String fragmentIdString, LlapTokenInfo tokenInfo) throws IOException {
+ String fragmentIdString, LlapTokenInfo tokenInfo, String amHost, int amPort) throws IOException {
ReadWriteLock dagLock = getDagLock(queryIdentifier);
// Note: This is a readLock to prevent a race with queryComplete. Operations
@@ -174,6 +174,8 @@ public class QueryTracker extends AbstractService {
if (old != null) {
queryInfo = old;
} else {
+ // Ensure the UGI is setup once.
+ queryInfo.setupUmbilicalUgi(vertex.getTokenIdentifier(), appToken, amHost, amPort);
isExistingQueryInfo = false;
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/59cde667/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/TaskRunnerCallable.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/TaskRunnerCallable.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/TaskRunnerCallable.java
index fe124ff..8739d5b 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/TaskRunnerCallable.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/TaskRunnerCallable.java
@@ -41,7 +41,6 @@ import org.apache.hadoop.hive.ql.io.IOContextMap;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.log4j.MDC;
@@ -65,6 +64,7 @@ import org.apache.tez.runtime.task.TezTaskRunner2;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import javax.net.SocketFactory;
import java.net.InetSocketAddress;
import java.nio.ByteBuffer;
import java.security.PrivilegedExceptionAction;
@@ -116,7 +116,8 @@ public class TaskRunnerCallable extends CallableWithNdc<TaskRunner2Result> {
private final SignableVertexSpec vertex;
private final TezEvent initialEvent;
private final SchedulerFragmentCompletingListener completionListener;
- private UserGroupInformation taskUgi;
+ private UserGroupInformation fsTaskUgi;
+ private final SocketFactory socketFactory;
@VisibleForTesting
public TaskRunnerCallable(SubmitWorkRequestProto request, QueryFragmentInfo fragmentInfo,
@@ -125,7 +126,8 @@ public class TaskRunnerCallable extends CallableWithNdc<TaskRunner2Result> {
LlapDaemonExecutorMetrics metrics, KilledTaskHandler killedTaskHandler,
FragmentCompletionHandler fragmentCompleteHandler, HadoopShim tezHadoopShim,
TezTaskAttemptID attemptId, SignableVertexSpec vertex, TezEvent initialEvent,
- UserGroupInformation taskUgi, SchedulerFragmentCompletingListener completionListener) {
+ UserGroupInformation fsTaskUgi, SchedulerFragmentCompletingListener completionListener,
+ SocketFactory socketFactory) {
this.request = request;
this.fragmentInfo = fragmentInfo;
this.conf = conf;
@@ -153,8 +155,9 @@ public class TaskRunnerCallable extends CallableWithNdc<TaskRunner2Result> {
this.fragmentCompletionHanler = fragmentCompleteHandler;
this.tezHadoopShim = tezHadoopShim;
this.initialEvent = initialEvent;
- this.taskUgi = taskUgi;
+ this.fsTaskUgi = fsTaskUgi;
this.completionListener = completionListener;
+ this.socketFactory = socketFactory;
}
public long getStartTime() {
@@ -195,27 +198,27 @@ public class TaskRunnerCallable extends CallableWithNdc<TaskRunner2Result> {
// TODO Consolidate this code with TezChild.
runtimeWatch.start();
- if (taskUgi == null) {
- taskUgi = UserGroupInformation.createRemoteUser(vertex.getUser());
+ if (fsTaskUgi == null) {
+ fsTaskUgi = UserGroupInformation.createRemoteUser(vertex.getUser());
}
- taskUgi.addCredentials(credentials);
+ fsTaskUgi.addCredentials(credentials);
Map<String, ByteBuffer> serviceConsumerMetadata = new HashMap<>();
serviceConsumerMetadata.put(TezConstants.TEZ_SHUFFLE_HANDLER_SERVICE_ID,
TezCommonUtils.convertJobTokenToBytes(jobToken));
Multimap<String, String> startedInputsMap = createStartedInputMap(vertex);
- UserGroupInformation taskOwner =
- UserGroupInformation.createRemoteUser(vertex.getTokenIdentifier());
+ final UserGroupInformation taskOwner = fragmentInfo.getQueryInfo().getUmbilicalUgi();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("taskOwner hashCode:" + taskOwner.hashCode());
+ }
final InetSocketAddress address =
NetUtils.createSocketAddrForHost(request.getAmHost(), request.getAmPort());
- SecurityUtil.setTokenService(jobToken, address);
- taskOwner.addToken(jobToken);
umbilical = taskOwner.doAs(new PrivilegedExceptionAction<LlapTaskUmbilicalProtocol>() {
@Override
public LlapTaskUmbilicalProtocol run() throws Exception {
return RPC.getProxy(LlapTaskUmbilicalProtocol.class,
- LlapTaskUmbilicalProtocol.versionID, address, conf);
+ LlapTaskUmbilicalProtocol.versionID, address, taskOwner, conf, socketFactory);
}
});
@@ -237,7 +240,7 @@ public class TaskRunnerCallable extends CallableWithNdc<TaskRunner2Result> {
try {
synchronized (this) {
if (shouldRunTask) {
- taskRunner = new TezTaskRunner2(conf, taskUgi, fragmentInfo.getLocalDirs(),
+ taskRunner = new TezTaskRunner2(conf, fsTaskUgi, fragmentInfo.getLocalDirs(),
taskSpec,
vertex.getQueryIdentifier().getAppAttemptNumber(),
serviceConsumerMetadata, envMap, startedInputsMap, taskReporter, executor,
@@ -259,7 +262,7 @@ public class TaskRunnerCallable extends CallableWithNdc<TaskRunner2Result> {
isCompleted.set(true);
return result;
} finally {
- FileSystem.closeAllForUGI(taskUgi);
+ FileSystem.closeAllForUGI(fsTaskUgi);
LOG.info("ExecutionTime for Container: " + request.getContainerIdString() + "=" +
runtimeWatch.stop().elapsedMillis());
if (LOG.isDebugEnabled()) {
http://git-wip-us.apache.org/repos/asf/hive/blob/59cde667/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TaskExecutorTestHelpers.java
----------------------------------------------------------------------
diff --git a/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TaskExecutorTestHelpers.java b/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TaskExecutorTestHelpers.java
index 5dc1be5..ae3328a 100644
--- a/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TaskExecutorTestHelpers.java
+++ b/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TaskExecutorTestHelpers.java
@@ -44,6 +44,8 @@ import org.apache.tez.runtime.task.TaskRunner2Result;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import javax.net.SocketFactory;
+
public class TaskExecutorTestHelpers {
private static final Logger LOG = LoggerFactory.getLogger(TestTaskExecutorService.class);
@@ -184,7 +186,7 @@ public class TaskExecutorTestHelpers {
mock(KilledTaskHandler.class), mock(
FragmentCompletionHandler.class), new DefaultHadoopShim(), null,
requestProto.getWorkSpec().getVertex(), initialEvent, null, mock(
- SchedulerFragmentCompletingListener.class));
+ SchedulerFragmentCompletingListener.class), mock(SocketFactory.class));
this.workTime = workTime;
this.canFinish = canFinish;
}
[33/50] [abbrv] hive git commit: HIVE-15958: LLAP: IPC connections
are not being reused for umbilical protocol (Prasanth Jayachandran reviewed
by Siddharth Seth)
Posted by om...@apache.org.
HIVE-15958: LLAP: IPC connections are not being reused for umbilical protocol (Prasanth Jayachandran reviewed by Siddharth Seth)
Change-Id: Ib4e8ba9881cc560142dc3f75a130060b29ea7c57
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/badea3fa
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/badea3fa
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/badea3fa
Branch: refs/heads/branch-2.2
Commit: badea3fad5b8c6ef734e16446218ce01fa3a1904
Parents: 59cde66
Author: Prasanth Jayachandran <pj...@hortonworks.com>
Authored: Mon Feb 27 19:45:37 2017 -0800
Committer: Owen O'Malley <om...@apache.org>
Committed: Tue Mar 28 15:27:53 2017 -0700
----------------------------------------------------------------------
.../hive/llap/daemon/impl/AMReporter.java | 74 ++++++++++++--------
.../llap/daemon/impl/ContainerRunnerImpl.java | 21 +++---
.../hadoop/hive/llap/daemon/impl/QueryInfo.java | 17 +++--
.../hive/llap/daemon/impl/QueryTracker.java | 20 +++---
.../daemon/impl/TaskExecutorTestHelpers.java | 2 +-
5 files changed, 82 insertions(+), 52 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/badea3fa/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/AMReporter.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/AMReporter.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/AMReporter.java
index ede7e00..b01a495 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/AMReporter.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/AMReporter.java
@@ -113,9 +113,9 @@ public class AMReporter extends AbstractService {
this.conf = conf;
this.daemonId = daemonId;
if (maxThreads < numExecutors) {
- maxThreads = numExecutors;
LOG.warn("maxThreads={} is less than numExecutors={}. Setting maxThreads=numExecutors",
- maxThreads, numExecutors);
+ maxThreads, numExecutors);
+ maxThreads = numExecutors;
}
ExecutorService rawExecutor =
new ThreadPoolExecutor(numExecutors, maxThreads,
@@ -227,12 +227,15 @@ public class AMReporter extends AbstractService {
public void taskKilled(String amLocation, int port, String user, Token<JobTokenIdentifier> jobToken,
final QueryIdentifier queryIdentifier, final TezTaskAttemptID taskAttemptId) {
- // Not re-using the connection for the AM heartbeat - which may or may not be open by this point.
- // knownAppMasters is used for sending heartbeats for queued tasks. Killed messages use a new connection.
LlapNodeId amNodeId = LlapNodeId.getInstance(amLocation, port);
- AMNodeInfo amNodeInfo =
- new AMNodeInfo(amNodeId, user, jobToken, queryIdentifier, retryPolicy, retryTimeout, socketFactory,
- conf);
+ AMNodeInfo amNodeInfo;
+ synchronized (knownAppMasters) {
+ amNodeInfo = knownAppMasters.get(amNodeId);
+ if (amNodeInfo == null) {
+ amNodeInfo = new AMNodeInfo(amNodeId, user, jobToken, queryIdentifier, retryPolicy, retryTimeout, socketFactory,
+ conf);
+ }
+ }
// Even if the service hasn't started up. It's OK to make this invocation since this will
// only happen after the AtomicReference address has been populated. Not adding an additional check.
@@ -252,6 +255,20 @@ public class AMReporter extends AbstractService {
});
}
+ public void queryComplete(LlapNodeId llapNodeId) {
+ if (llapNodeId != null) {
+ synchronized (knownAppMasters) {
+ AMNodeInfo amNodeInfo = knownAppMasters.remove(llapNodeId);
+ // TODO: not stopping umbilical explicitly as some taskKill requests may get scheduled during queryComplete
+ // which will be using the umbilical. HIVE-16021 should fix this, until then leave umbilical open and wait for
+ // it to be closed after max idle timeout (10s default)
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Query complete received. Removed {}.", amNodeInfo);
+ }
+ }
+ }
+ }
+
private class QueueLookupCallable extends CallableWithNdc<Void> {
@Override
@@ -259,7 +276,7 @@ public class AMReporter extends AbstractService {
while (!isShutdown.get() && !Thread.currentThread().isInterrupted()) {
try {
final AMNodeInfo amNodeInfo = pendingHeartbeatQueeu.take();
- if (amNodeInfo.getTaskCount() == 0 || amNodeInfo.hasAmFailed()) {
+ if (amNodeInfo.hasAmFailed()) {
synchronized (knownAppMasters) {
if (LOG.isDebugEnabled()) {
LOG.debug(
@@ -269,28 +286,29 @@ public class AMReporter extends AbstractService {
}
knownAppMasters.remove(amNodeInfo.amNodeId);
}
- amNodeInfo.stopUmbilical();
} else {
- // Add back to the queue for the next heartbeat, and schedule the actual heartbeat
- long next = System.currentTimeMillis() + heartbeatInterval;
- amNodeInfo.setNextHeartbeatTime(next);
- pendingHeartbeatQueeu.add(amNodeInfo);
- ListenableFuture<Void> future = executor.submit(new AMHeartbeatCallable(amNodeInfo));
- Futures.addCallback(future, new FutureCallback<Void>() {
- @Override
- public void onSuccess(Void result) {
- // Nothing to do.
- }
-
- @Override
- public void onFailure(Throwable t) {
- QueryIdentifier currentQueryIdentifier = amNodeInfo.getCurrentQueryIdentifier();
- amNodeInfo.setAmFailed(true);
- LOG.warn("Heartbeat failed to AM {}. Killing all other tasks for the query={}",
+ if (amNodeInfo.getTaskCount() > 0) {
+ // Add back to the queue for the next heartbeat, and schedule the actual heartbeat
+ long next = System.currentTimeMillis() + heartbeatInterval;
+ amNodeInfo.setNextHeartbeatTime(next);
+ pendingHeartbeatQueeu.add(amNodeInfo);
+ ListenableFuture<Void> future = executor.submit(new AMHeartbeatCallable(amNodeInfo));
+ Futures.addCallback(future, new FutureCallback<Void>() {
+ @Override
+ public void onSuccess(Void result) {
+ // Nothing to do.
+ }
+
+ @Override
+ public void onFailure(Throwable t) {
+ QueryIdentifier currentQueryIdentifier = amNodeInfo.getCurrentQueryIdentifier();
+ amNodeInfo.setAmFailed(true);
+ LOG.warn("Heartbeat failed to AM {}. Killing all other tasks for the query={}",
amNodeInfo.amNodeId, currentQueryIdentifier, t);
- queryFailedHandler.queryFailed(currentQueryIdentifier);
- }
- });
+ queryFailedHandler.queryFailed(currentQueryIdentifier);
+ }
+ });
+ }
}
} catch (InterruptedException e) {
if (isShutdown.get()) {
http://git-wip-us.apache.org/repos/asf/hive/blob/badea3fa/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/ContainerRunnerImpl.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/ContainerRunnerImpl.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/ContainerRunnerImpl.java
index cc4eff0..1176e5e 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/ContainerRunnerImpl.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/ContainerRunnerImpl.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.hive.common.UgiFactory;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.llap.DaemonId;
+import org.apache.hadoop.hive.llap.LlapNodeId;
import org.apache.hadoop.hive.llap.NotTezEventHelper;
import org.apache.hadoop.hive.llap.daemon.ContainerRunner;
import org.apache.hadoop.hive.llap.daemon.FragmentCompletionHandler;
@@ -240,12 +241,12 @@ public class ContainerRunnerImpl extends CompositeService implements ContainerRu
Token<JobTokenIdentifier> jobToken = TokenCache.getSessionToken(credentials);
+ LlapNodeId amNodeId = LlapNodeId.getInstance(request.getAmHost(), request.getAmPort());
QueryFragmentInfo fragmentInfo = queryTracker.registerFragment(
queryIdentifier, qIdProto.getApplicationIdString(), dagId,
vertex.getDagName(), vertex.getHiveQueryId(), dagIdentifier,
vertex.getVertexName(), request.getFragmentNumber(), request.getAttemptNumber(),
- vertex.getUser(), vertex, jobToken, fragmentIdString, tokenInfo, request.getAmHost(),
- request.getAmPort());
+ vertex.getUser(), vertex, jobToken, fragmentIdString, tokenInfo, amNodeId);
String[] localDirs = fragmentInfo.getLocalDirs();
Preconditions.checkNotNull(localDirs);
@@ -388,14 +389,18 @@ public class ContainerRunnerImpl extends CompositeService implements ContainerRu
new QueryIdentifier(request.getQueryIdentifier().getApplicationIdString(),
request.getQueryIdentifier().getDagIndex());
LOG.info("Processing queryComplete notification for {}", queryIdentifier);
- List<QueryFragmentInfo> knownFragments = queryTracker.queryComplete(
- queryIdentifier, request.getDeleteDelay(), false);
- LOG.info("DBG: Pending fragment count for completed query {} = {}", queryIdentifier,
+ QueryInfo queryInfo = queryTracker.queryComplete(queryIdentifier, request.getDeleteDelay(), false);
+ if (queryInfo != null) {
+ List<QueryFragmentInfo> knownFragments = queryInfo.getRegisteredFragments();
+ LOG.info("DBG: Pending fragment count for completed query {} = {}", queryIdentifier,
knownFragments.size());
- for (QueryFragmentInfo fragmentInfo : knownFragments) {
- LOG.info("Issuing killFragment for completed query {} {}", queryIdentifier,
+ for (QueryFragmentInfo fragmentInfo : knownFragments) {
+ LOG.info("Issuing killFragment for completed query {} {}", queryIdentifier,
fragmentInfo.getFragmentIdentifierString());
- executorService.killFragment(fragmentInfo.getFragmentIdentifierString());
+ executorService.killFragment(fragmentInfo.getFragmentIdentifierString());
+ }
+ LlapNodeId amNodeId = queryInfo.getAmNodeId();
+ amReporter.queryComplete(amNodeId);
}
return QueryCompleteResponseProto.getDefaultInstance();
}
http://git-wip-us.apache.org/repos/asf/hive/blob/badea3fa/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/QueryInfo.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/QueryInfo.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/QueryInfo.java
index 5f0271f..90641d4 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/QueryInfo.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/QueryInfo.java
@@ -35,6 +35,7 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Multimap;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.llap.LlapNodeId;
import org.apache.hadoop.hive.llap.LlapUtil;
import org.apache.hadoop.hive.llap.daemon.FinishableStateUpdateHandler;
import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SignableVertexSpec;
@@ -56,6 +57,7 @@ public class QueryInfo {
private final String[] localDirsBase;
private final FileSystem localFs;
private String[] localDirs;
+ private final LlapNodeId amNodeId;
// Map of states for different vertices.
private final Set<QueryFragmentInfo> knownFragments =
@@ -68,11 +70,11 @@ public class QueryInfo {
private final AtomicReference<UserGroupInformation> umbilicalUgi;
public QueryInfo(QueryIdentifier queryIdentifier, String appIdString, String dagIdString,
- String dagName, String hiveQueryIdString,
- int dagIdentifier, String user,
- ConcurrentMap<String, SourceStateProto> sourceStateMap,
- String[] localDirsBase, FileSystem localFs, String tokenUserName,
- String tokenAppId) {
+ String dagName, String hiveQueryIdString,
+ int dagIdentifier, String user,
+ ConcurrentMap<String, SourceStateProto> sourceStateMap,
+ String[] localDirsBase, FileSystem localFs, String tokenUserName,
+ String tokenAppId, final LlapNodeId amNodeId) {
this.queryIdentifier = queryIdentifier;
this.appIdString = appIdString;
this.dagIdString = dagIdString;
@@ -86,6 +88,7 @@ public class QueryInfo {
this.tokenUserName = tokenUserName;
this.appId = tokenAppId;
this.umbilicalUgi = new AtomicReference<>();
+ this.amNodeId = amNodeId;
}
public QueryIdentifier getQueryIdentifier() {
@@ -116,6 +119,10 @@ public class QueryInfo {
return sourceStateMap;
}
+ public LlapNodeId getAmNodeId() {
+ return amNodeId;
+ }
+
public QueryFragmentInfo registerFragment(String vertexName, int fragmentNumber,
int attemptNumber, SignableVertexSpec vertexSpec, String fragmentIdString) {
QueryFragmentInfo fragmentInfo = new QueryFragmentInfo(
http://git-wip-us.apache.org/repos/asf/hive/blob/badea3fa/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/QueryTracker.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/QueryTracker.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/QueryTracker.java
index 5cf3a38..7e646c5 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/QueryTracker.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/QueryTracker.java
@@ -21,6 +21,7 @@ import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.llap.LlapNodeId;
import org.apache.hadoop.hive.llap.log.Log4jQueryCompleteMarker;
import org.apache.hadoop.hive.llap.log.LogHelpers;
import org.apache.hadoop.security.UserGroupInformation;
@@ -69,8 +70,6 @@ public class QueryTracker extends AbstractService {
private final ConcurrentHashMap<QueryIdentifier, QueryInfo> queryInfoMap = new ConcurrentHashMap<>();
-
-
private final String[] localDirsBase;
private final FileSystem localFs;
private final String clusterId;
@@ -137,9 +136,10 @@ public class QueryTracker extends AbstractService {
* Register a new fragment for a specific query
*/
QueryFragmentInfo registerFragment(QueryIdentifier queryIdentifier, String appIdString, String dagIdString,
- String dagName, String hiveQueryIdString, int dagIdentifier, String vertexName, int fragmentNumber, int attemptNumber,
- String user, SignableVertexSpec vertex, Token<JobTokenIdentifier> appToken,
- String fragmentIdString, LlapTokenInfo tokenInfo, String amHost, int amPort) throws IOException {
+ String dagName, String hiveQueryIdString, int dagIdentifier, String vertexName, int fragmentNumber,
+ int attemptNumber,
+ String user, SignableVertexSpec vertex, Token<JobTokenIdentifier> appToken,
+ String fragmentIdString, LlapTokenInfo tokenInfo, final LlapNodeId amNodeId) throws IOException {
ReadWriteLock dagLock = getDagLock(queryIdentifier);
// Note: This is a readLock to prevent a race with queryComplete. Operations
@@ -169,13 +169,13 @@ public class QueryTracker extends AbstractService {
new QueryInfo(queryIdentifier, appIdString, dagIdString, dagName, hiveQueryIdString,
dagIdentifier, user,
getSourceCompletionMap(queryIdentifier), localDirsBase, localFs,
- tokenInfo.userName, tokenInfo.appId);
+ tokenInfo.userName, tokenInfo.appId, amNodeId);
QueryInfo old = queryInfoMap.putIfAbsent(queryIdentifier, queryInfo);
if (old != null) {
queryInfo = old;
} else {
// Ensure the UGI is setup once.
- queryInfo.setupUmbilicalUgi(vertex.getTokenIdentifier(), appToken, amHost, amPort);
+ queryInfo.setupUmbilicalUgi(vertex.getTokenIdentifier(), appToken, amNodeId.getHostname(), amNodeId.getPort());
isExistingQueryInfo = false;
}
}
@@ -238,7 +238,7 @@ public class QueryTracker extends AbstractService {
* @param queryIdentifier
* @param deleteDelay
*/
- List<QueryFragmentInfo> queryComplete(QueryIdentifier queryIdentifier, long deleteDelay,
+ QueryInfo queryComplete(QueryIdentifier queryIdentifier, long deleteDelay,
boolean isInternal) throws IOException {
if (deleteDelay == -1) {
deleteDelay = defaultDeleteDelaySeconds;
@@ -255,7 +255,7 @@ public class QueryTracker extends AbstractService {
if (queryInfo == null) {
// Should not happen.
LOG.warn("Ignoring query complete for unknown dag: {}", queryIdentifier);
- return Collections.emptyList();
+ return null;
}
String[] localDirs = queryInfo.getLocalDirsNoCreate();
if (localDirs != null) {
@@ -292,7 +292,7 @@ public class QueryTracker extends AbstractService {
if (savedQueryId != null) {
ObjectCacheFactory.removeLlapQueryCache(savedQueryId);
}
- return queryInfo.getRegisteredFragments();
+ return queryInfo;
} finally {
dagLock.writeLock().unlock();
}
http://git-wip-us.apache.org/repos/asf/hive/blob/badea3fa/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TaskExecutorTestHelpers.java
----------------------------------------------------------------------
diff --git a/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TaskExecutorTestHelpers.java b/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TaskExecutorTestHelpers.java
index ae3328a..d1fce19 100644
--- a/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TaskExecutorTestHelpers.java
+++ b/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TaskExecutorTestHelpers.java
@@ -93,7 +93,7 @@ public class TaskExecutorTestHelpers {
new QueryInfo(queryIdentifier, "fake_app_id_string", "fake_dag_id_string", "fake_dag_name",
"fakeHiveQueryId", 1, "fakeUser",
new ConcurrentHashMap<String, LlapDaemonProtocolProtos.SourceStateProto>(),
- new String[0], null, "fakeUser", null);
+ new String[0], null, "fakeUser", null, null);
return queryInfo;
}
[42/50] [abbrv] hive git commit: HIVE-16050: Regression: Union of
null with non-null (Gopal V, reviewed by Sergey Shelukhin)
Posted by om...@apache.org.
HIVE-16050: Regression: Union of null with non-null (Gopal V, reviewed by Sergey Shelukhin)
(cherry-picked: a6366fc07931f374af77d0ab208404027bf165a3)
Change-Id: I65d05a2385f712bec0f41269c2498136b60fba0d
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/72ca985e
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/72ca985e
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/72ca985e
Branch: refs/heads/branch-2.2
Commit: 72ca985e6151542b290e44fa5e8d2ed6d800196a
Parents: 5f533bc
Author: Gopal V <go...@apache.org>
Authored: Wed Mar 1 17:57:54 2017 +0530
Committer: Owen O'Malley <om...@apache.org>
Committed: Tue Mar 28 15:27:56 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hive/ql/exec/FunctionRegistry.java | 9 +++++++++
ql/src/test/queries/clientpositive/union_null.q | 3 +++
ql/src/test/results/clientpositive/spark/union_null.q.out | 10 ++++++++++
ql/src/test/results/clientpositive/union_null.q.out | 10 ++++++++++
4 files changed, 32 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/72ca985e/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
index d8198c6..84c218f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
@@ -725,6 +725,15 @@ public final class FunctionRegistry {
PrimitiveGrouping pgA = PrimitiveObjectInspectorUtils.getPrimitiveGrouping(pcA);
PrimitiveGrouping pgB = PrimitiveObjectInspectorUtils.getPrimitiveGrouping(pcB);
+
+ // untyped nulls
+ if (pgA == PrimitiveGrouping.VOID_GROUP) {
+ return b;
+ }
+ if (pgB == PrimitiveGrouping.VOID_GROUP) {
+ return a;
+ }
+
if (pgA != pgB) {
return null;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/72ca985e/ql/src/test/queries/clientpositive/union_null.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/union_null.q b/ql/src/test/queries/clientpositive/union_null.q
index 45448b4..aacf310 100644
--- a/ql/src/test/queries/clientpositive/union_null.q
+++ b/ql/src/test/queries/clientpositive/union_null.q
@@ -8,3 +8,6 @@ select x from (select * from (select value as x from src order by x limit 5)a un
set hive.cbo.returnpath.hiveop=false;
-- HIVE-4837
select * from (select * from (select cast(null as string) as N from src1 group by key)a UNION ALL select * from (select cast(null as string) as N from src1 group by key)b ) a;
+
+-- HIVE-16050
+select null as c1 UNION ALL select 1 as c1;
http://git-wip-us.apache.org/repos/asf/hive/blob/72ca985e/ql/src/test/results/clientpositive/spark/union_null.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/union_null.q.out b/ql/src/test/results/clientpositive/spark/union_null.q.out
index 49482b2..76f5ac8 100644
--- a/ql/src/test/results/clientpositive/spark/union_null.q.out
+++ b/ql/src/test/results/clientpositive/spark/union_null.q.out
@@ -95,3 +95,13 @@ NULL
NULL
NULL
NULL
+PREHOOK: query: select null as c1 UNION ALL select 1 as c1
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+POSTHOOK: query: select null as c1 UNION ALL select 1 as c1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+NULL
+1
http://git-wip-us.apache.org/repos/asf/hive/blob/72ca985e/ql/src/test/results/clientpositive/union_null.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/union_null.q.out b/ql/src/test/results/clientpositive/union_null.q.out
index e196ff3..00bd9d9 100644
--- a/ql/src/test/results/clientpositive/union_null.q.out
+++ b/ql/src/test/results/clientpositive/union_null.q.out
@@ -74,3 +74,13 @@ NULL
NULL
NULL
NULL
+PREHOOK: query: select null as c1 UNION ALL select 1 as c1
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+POSTHOOK: query: select null as c1 UNION ALL select 1 as c1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+NULL
+1
[14/50] [abbrv] hive git commit: HIVE-14367 : Estimated size for
constant nulls is 0 (Ashutosh Chauhan via Jesus Camacho Rodriguez)
Posted by om...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out b/ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out
index 0b021f2..b2955ed 100644
--- a/ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out
+++ b/ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out
@@ -2228,7 +2228,7 @@ STAGE PLANS:
Tez
#### A masked pattern was here ####
Edges:
- Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE)
+ Reducer 2 <- Map 1 (XPROD_EDGE), Reducer 5 (XPROD_EDGE)
Reducer 3 <- Reducer 2 (SIMPLE_EDGE)
Reducer 5 <- Map 4 (SIMPLE_EDGE)
#### A masked pattern was here ####
@@ -2361,7 +2361,7 @@ STAGE PLANS:
Tez
#### A masked pattern was here ####
Edges:
- Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE)
+ Reducer 2 <- Map 1 (XPROD_EDGE), Map 4 (XPROD_EDGE)
Reducer 3 <- Reducer 2 (SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
@@ -3027,6 +3027,21 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ keys: _col0 (type: string)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE
+ Dynamic Partitioning Event Operator
+ Target column: hr (string)
+ Target Input: srcpart
+ Partition key expr: hr
+ Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE
+ Target Vertex: Map 1
Execution mode: llap
LLAP IO: no inputs
Reducer 2
@@ -3120,9 +3135,11 @@ STAGE PLANS:
Tez
#### A masked pattern was here ####
Edges:
- Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE)
- Reducer 3 <- Map 6 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE)
+ Map 7 <- Reducer 5 (BROADCAST_EDGE)
+ Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE)
+ Reducer 3 <- Map 7 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE)
Reducer 4 <- Reducer 3 (SIMPLE_EDGE)
+ Reducer 5 <- Reducer 2 (SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -3146,7 +3163,7 @@ STAGE PLANS:
value expressions: _col1 (type: string)
Execution mode: llap
LLAP IO: unknown
- Map 5
+ Map 6
Map Operator Tree:
TableScan
alias: srcpart_date
@@ -3166,14 +3183,14 @@ STAGE PLANS:
Statistics: Num rows: 1 Data size: 21 Basic stats: COMPLETE Column stats: NONE
Execution mode: llap
LLAP IO: no inputs
- Map 6
+ Map 7
Map Operator Tree:
TableScan
alias: srcpart_hour
- filterExpr: (UDFToDouble(hr) = 13.0) (type: boolean)
+ filterExpr: ((UDFToDouble(hr) = 13.0) and hr BETWEEN DynamicValue(RS_12_srcpart_hr_min) AND DynamicValue(RS_12_srcpart_hr_max) and in_bloom_filter(hr, DynamicValue(RS_12_srcpart_hr_bloom_filter))) (type: boolean)
Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (UDFToDouble(hr) = 13.0) (type: boolean)
+ predicate: ((UDFToDouble(hr) = 13.0) and hr BETWEEN DynamicValue(RS_12_srcpart_hr_min) AND DynamicValue(RS_12_srcpart_hr_max) and in_bloom_filter(hr, DynamicValue(RS_12_srcpart_hr_bloom_filter))) (type: boolean)
Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: hr (type: string)
@@ -3202,6 +3219,19 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col1 (type: string)
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Select Operator
+ expressions: _col1 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Group By Operator
+ aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=1)
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary)
Reducer 3
Execution mode: llap
Reduce Operator Tree:
@@ -3236,6 +3266,18 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Reducer 5
+ Execution mode: llap
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: min(VALUE._col0), max(VALUE._col1), bloom_filter(VALUE._col2, expectedEntries=1)
+ mode: final
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary)
Stage: Stage-0
Fetch Operator
@@ -3308,10 +3350,10 @@ STAGE PLANS:
aggregations: max(ds)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string)
Execution mode: llap
LLAP IO: no inputs
@@ -3328,10 +3370,10 @@ STAGE PLANS:
aggregations: min(ds)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string)
Execution mode: llap
LLAP IO: no inputs
@@ -3376,20 +3418,20 @@ STAGE PLANS:
aggregations: max(VALUE._col0)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: _col0 is not null (type: boolean)
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Reducer 7
Execution mode: llap
Reduce Operator Tree:
@@ -3397,26 +3439,26 @@ STAGE PLANS:
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Dynamic Partitioning Event Operator
Target column: ds (string)
Target Input: srcpart
Partition key expr: ds
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Target Vertex: Map 1
Reducer 9
Execution mode: llap
@@ -3425,20 +3467,20 @@ STAGE PLANS:
aggregations: min(VALUE._col0)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: _col0 is not null (type: boolean)
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Union 6
Vertex: Union 6
@@ -3515,10 +3557,10 @@ STAGE PLANS:
aggregations: max(ds)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string)
Execution mode: llap
LLAP IO: no inputs
@@ -3535,10 +3577,10 @@ STAGE PLANS:
aggregations: min(ds)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string)
Execution mode: llap
LLAP IO: no inputs
@@ -3585,20 +3627,20 @@ STAGE PLANS:
aggregations: max(VALUE._col0)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: _col0 is not null (type: boolean)
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Reducer 7
Execution mode: llap
Reduce Operator Tree:
@@ -3606,26 +3648,26 @@ STAGE PLANS:
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Dynamic Partitioning Event Operator
Target column: ds (string)
Target Input: srcpart
Partition key expr: ds
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Target Vertex: Map 1
Reducer 9
Execution mode: llap
@@ -3634,20 +3676,20 @@ STAGE PLANS:
aggregations: min(VALUE._col0)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: _col0 is not null (type: boolean)
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Union 6
Vertex: Union 6
@@ -3726,10 +3768,10 @@ STAGE PLANS:
aggregations: min(ds)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string)
Execution mode: llap
LLAP IO: no inputs
@@ -3764,10 +3806,10 @@ STAGE PLANS:
aggregations: max(ds)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string)
Execution mode: llap
LLAP IO: no inputs
@@ -3778,49 +3820,49 @@ STAGE PLANS:
aggregations: min(VALUE._col0)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: _col0 is not null (type: boolean)
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Dynamic Partitioning Event Operator
Target column: ds (string)
Target Input: srcpart
Partition key expr: ds
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Target Vertex: Map 1
Select Operator
expressions: _col0 (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Dynamic Partitioning Event Operator
Target column: ds (string)
Target Input: srcpart
Partition key expr: ds
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Target Vertex: Map 5
Reducer 2
Execution mode: llap
@@ -3873,49 +3915,49 @@ STAGE PLANS:
aggregations: max(VALUE._col0)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: _col0 is not null (type: boolean)
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Dynamic Partitioning Event Operator
Target column: ds (string)
Target Input: srcpart
Partition key expr: ds
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Target Vertex: Map 1
Select Operator
expressions: _col0 (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Dynamic Partitioning Event Operator
Target column: ds (string)
Target Input: srcpart
Partition key expr: ds
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Target Vertex: Map 5
Union 3
Vertex: Union 3
@@ -5421,6 +5463,21 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ keys: _col0 (type: string)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE
+ Dynamic Partitioning Event Operator
+ Target column: hr (string)
+ Target Input: srcpart
+ Partition key expr: hr
+ Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE
+ Target Vertex: Map 1
Execution mode: llap
LLAP IO: no inputs
Reducer 2
@@ -5638,10 +5695,10 @@ STAGE PLANS:
aggregations: max(ds)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string)
Execution mode: llap
LLAP IO: no inputs
@@ -5658,10 +5715,10 @@ STAGE PLANS:
aggregations: min(ds)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string)
Execution mode: llap
LLAP IO: no inputs
@@ -5672,20 +5729,20 @@ STAGE PLANS:
aggregations: max(VALUE._col0)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: _col0 is not null (type: boolean)
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Reducer 5
Execution mode: llap
Reduce Operator Tree:
@@ -5693,7 +5750,7 @@ STAGE PLANS:
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Inner Join 0 to 1
@@ -5736,20 +5793,20 @@ STAGE PLANS:
aggregations: min(VALUE._col0)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: _col0 is not null (type: boolean)
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Union 4
Vertex: Union 4
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/llap/vectorized_dynamic_partition_pruning.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vectorized_dynamic_partition_pruning.q.out b/ql/src/test/results/clientpositive/llap/vectorized_dynamic_partition_pruning.q.out
index d4d2f82..99e5a42 100644
--- a/ql/src/test/results/clientpositive/llap/vectorized_dynamic_partition_pruning.q.out
+++ b/ql/src/test/results/clientpositive/llap/vectorized_dynamic_partition_pruning.q.out
@@ -2984,10 +2984,10 @@ STAGE PLANS:
aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=1)
mode: hash
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary)
Reducer 3
Execution mode: llap
@@ -3030,10 +3030,10 @@ STAGE PLANS:
aggregations: min(VALUE._col0), max(VALUE._col1), bloom_filter(VALUE._col2, expectedEntries=1)
mode: final
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary)
Stage: Stage-0
@@ -3107,10 +3107,10 @@ STAGE PLANS:
aggregations: max(ds)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string)
Execution mode: llap
LLAP IO: no inputs
@@ -3127,10 +3127,10 @@ STAGE PLANS:
aggregations: min(ds)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string)
Execution mode: llap
LLAP IO: no inputs
@@ -3175,20 +3175,20 @@ STAGE PLANS:
aggregations: max(VALUE._col0)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: _col0 is not null (type: boolean)
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Reducer 7
Execution mode: vectorized, llap
Reduce Operator Tree:
@@ -3196,26 +3196,26 @@ STAGE PLANS:
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Dynamic Partitioning Event Operator
Target column: ds (string)
Target Input: srcpart
Partition key expr: ds
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Target Vertex: Map 1
Reducer 9
Execution mode: vectorized, llap
@@ -3224,20 +3224,20 @@ STAGE PLANS:
aggregations: min(VALUE._col0)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: _col0 is not null (type: boolean)
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Union 6
Vertex: Union 6
@@ -3314,10 +3314,10 @@ STAGE PLANS:
aggregations: max(ds)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string)
Execution mode: llap
LLAP IO: no inputs
@@ -3334,10 +3334,10 @@ STAGE PLANS:
aggregations: min(ds)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string)
Execution mode: llap
LLAP IO: no inputs
@@ -3384,20 +3384,20 @@ STAGE PLANS:
aggregations: max(VALUE._col0)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: _col0 is not null (type: boolean)
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Reducer 7
Execution mode: vectorized, llap
Reduce Operator Tree:
@@ -3405,26 +3405,26 @@ STAGE PLANS:
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Dynamic Partitioning Event Operator
Target column: ds (string)
Target Input: srcpart
Partition key expr: ds
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Target Vertex: Map 1
Reducer 9
Execution mode: vectorized, llap
@@ -3433,20 +3433,20 @@ STAGE PLANS:
aggregations: min(VALUE._col0)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: _col0 is not null (type: boolean)
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Union 6
Vertex: Union 6
@@ -3525,10 +3525,10 @@ STAGE PLANS:
aggregations: min(ds)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string)
Execution mode: llap
LLAP IO: no inputs
@@ -3563,10 +3563,10 @@ STAGE PLANS:
aggregations: max(ds)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string)
Execution mode: llap
LLAP IO: no inputs
@@ -3577,49 +3577,49 @@ STAGE PLANS:
aggregations: min(VALUE._col0)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: _col0 is not null (type: boolean)
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Dynamic Partitioning Event Operator
Target column: ds (string)
Target Input: srcpart
Partition key expr: ds
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Target Vertex: Map 1
Select Operator
expressions: _col0 (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Dynamic Partitioning Event Operator
Target column: ds (string)
Target Input: srcpart
Partition key expr: ds
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Target Vertex: Map 5
Reducer 2
Execution mode: vectorized, llap
@@ -3672,49 +3672,49 @@ STAGE PLANS:
aggregations: max(VALUE._col0)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: _col0 is not null (type: boolean)
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Dynamic Partitioning Event Operator
Target column: ds (string)
Target Input: srcpart
Partition key expr: ds
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Target Vertex: Map 1
Select Operator
expressions: _col0 (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Dynamic Partitioning Event Operator
Target column: ds (string)
Target Input: srcpart
Partition key expr: ds
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Target Vertex: Map 5
Union 3
Vertex: Union 3
@@ -5329,10 +5329,10 @@ STAGE PLANS:
aggregations: max(ds)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string)
Execution mode: llap
LLAP IO: no inputs
@@ -5349,10 +5349,10 @@ STAGE PLANS:
aggregations: min(ds)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string)
Execution mode: llap
LLAP IO: no inputs
@@ -5363,20 +5363,20 @@ STAGE PLANS:
aggregations: max(VALUE._col0)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: _col0 is not null (type: boolean)
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Reducer 5
Execution mode: vectorized, llap
Reduce Operator Tree:
@@ -5384,7 +5384,7 @@ STAGE PLANS:
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Inner Join 0 to 1
@@ -5427,20 +5427,20 @@ STAGE PLANS:
aggregations: min(VALUE._col0)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: _col0 is not null (type: boolean)
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Union 4
Vertex: Union 4
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/metadataonly1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/metadataonly1.q.out b/ql/src/test/results/clientpositive/metadataonly1.q.out
index d2a15c3..8c0a258 100644
--- a/ql/src/test/results/clientpositive/metadataonly1.q.out
+++ b/ql/src/test/results/clientpositive/metadataonly1.q.out
@@ -30,11 +30,11 @@ STAGE PLANS:
aggregations: max(ds)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
null sort order:
sort order:
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
tag: -1
value expressions: _col0 (type: string)
auto parallelism: false
@@ -44,13 +44,13 @@ STAGE PLANS:
aggregations: max(VALUE._col0)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
GlobalTableId: 0
#### A masked pattern was here ####
NumFilesPerFileSink: 1
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
#### A masked pattern was here ####
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
@@ -114,11 +114,11 @@ STAGE PLANS:
aggregations: max(ds)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
null sort order:
sort order:
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
tag: -1
value expressions: _col0 (type: string)
auto parallelism: false
@@ -177,13 +177,13 @@ STAGE PLANS:
aggregations: max(VALUE._col0)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
GlobalTableId: 0
#### A masked pattern was here ####
NumFilesPerFileSink: 1
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
#### A masked pattern was here ####
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
@@ -511,11 +511,11 @@ STAGE PLANS:
aggregations: max(ds)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
null sort order:
sort order:
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
tag: -1
value expressions: _col0 (type: string)
auto parallelism: false
@@ -620,11 +620,11 @@ STAGE PLANS:
aggregations: max(VALUE._col0)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Filter Operator
isSamplingPred: false
predicate: _col0 is not null (type: boolean)
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
GlobalTableId: 0
@@ -669,7 +669,7 @@ STAGE PLANS:
null sort order: a
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
tag: 1
auto parallelism: false
Path -> Alias:
@@ -1420,11 +1420,11 @@ STAGE PLANS:
aggregations: max(ds)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
null sort order:
sort order:
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
tag: -1
value expressions: _col0 (type: string)
auto parallelism: false
@@ -1529,13 +1529,13 @@ STAGE PLANS:
aggregations: max(VALUE._col0)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
GlobalTableId: 0
#### A masked pattern was here ####
NumFilesPerFileSink: 1
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
#### A masked pattern was here ####
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/num_op_type_conv.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/num_op_type_conv.q.out b/ql/src/test/results/clientpositive/num_op_type_conv.q.out
index 013a153..00a4ca9 100644
--- a/ql/src/test/results/clientpositive/num_op_type_conv.q.out
+++ b/ql/src/test/results/clientpositive/num_op_type_conv.q.out
@@ -22,10 +22,10 @@ STAGE PLANS:
Select Operator
expressions: null (type: double), null (type: double), null (type: double), 1 (type: bigint), 0 (type: bigint), 0.0 (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
- Statistics: Num rows: 500 Data size: 12000 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 500 Data size: 12024 Basic stats: COMPLETE Column stats: COMPLETE
Limit
Number of rows: 1
- Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
ListSink
PREHOOK: query: SELECT null + 7, 1.0 - null, null + null,
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/perf/query13.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/query13.q.out b/ql/src/test/results/clientpositive/perf/query13.q.out
index 7c144af..b145d14 100644
--- a/ql/src/test/results/clientpositive/perf/query13.q.out
+++ b/ql/src/test/results/clientpositive/perf/query13.q.out
@@ -122,11 +122,11 @@ Stage-0
Stage-1
Reducer 3
File Output Operator [FS_43]
- Group By Operator [GBY_41] (rows=1 width=344)
+ Group By Operator [GBY_41] (rows=1 width=764)
Output:["_col0","_col1","_col2","_col3"],aggregations:["avg(VALUE._col0)","avg(VALUE._col1)","avg(VALUE._col2)","sum(VALUE._col3)"]
<-Reducer 2 [SIMPLE_EDGE]
SHUFFLE [RS_40]
- Group By Operator [GBY_39] (rows=1 width=112)
+ Group By Operator [GBY_39] (rows=1 width=764)
Output:["_col0","_col1","_col2","_col3"],aggregations:["avg(_col7)","avg(_col9)","avg(_col10)","sum(_col10)"]
Merge Join Operator [MERGEJOIN_74] (rows=2016666 width=1014)
Conds:RS_35._col0=RS_36._col0(Inner),Output:["_col7","_col9","_col10"]
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/perf/query28.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/query28.q.out b/ql/src/test/results/clientpositive/perf/query28.q.out
index e92a1b3..cb9d73e 100644
--- a/ql/src/test/results/clientpositive/perf/query28.q.out
+++ b/ql/src/test/results/clientpositive/perf/query28.q.out
@@ -118,15 +118,15 @@ Stage-0
Stage-1
Reducer 3
File Output Operator [FS_51]
- Limit [LIM_50] (rows=1 width=817)
+ Limit [LIM_50] (rows=1 width=2497)
Number of rows:100
- Select Operator [SEL_49] (rows=1 width=817)
+ Select Operator [SEL_49] (rows=1 width=2497)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"]
- Merge Join Operator [MERGEJOIN_58] (rows=1 width=817)
+ Merge Join Operator [MERGEJOIN_58] (rows=1 width=2497)
Conds:(Inner),(Inner),(Inner),(Inner),(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"]
<-Reducer 11 [SIMPLE_EDGE]
SHUFFLE [RS_46]
- Group By Operator [GBY_33] (rows=1 width=136)
+ Group By Operator [GBY_33] (rows=1 width=416)
Output:["_col0","_col1","_col2"],aggregations:["avg(VALUE._col0)","count(VALUE._col1)","count(DISTINCT KEY._col0:0._col0)"]
<-Map 10 [SIMPLE_EDGE]
SHUFFLE [RS_32]
@@ -140,7 +140,7 @@ Stage-0
default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_quantity","ss_wholesale_cost","ss_list_price","ss_coupon_amt"]
<-Reducer 13 [SIMPLE_EDGE]
SHUFFLE [RS_47]
- Group By Operator [GBY_40] (rows=1 width=136)
+ Group By Operator [GBY_40] (rows=1 width=416)
Output:["_col0","_col1","_col2"],aggregations:["avg(VALUE._col0)","count(VALUE._col1)","count(DISTINCT KEY._col0:0._col0)"]
<-Map 12 [SIMPLE_EDGE]
SHUFFLE [RS_39]
@@ -154,7 +154,7 @@ Stage-0
default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_quantity","ss_wholesale_cost","ss_list_price","ss_coupon_amt"]
<-Reducer 2 [SIMPLE_EDGE]
SHUFFLE [RS_42]
- Group By Operator [GBY_5] (rows=1 width=136)
+ Group By Operator [GBY_5] (rows=1 width=416)
Output:["_col0","_col1","_col2"],aggregations:["avg(VALUE._col0)","count(VALUE._col1)","count(DISTINCT KEY._col0:0._col0)"]
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_4]
@@ -168,7 +168,7 @@ Stage-0
default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_quantity","ss_wholesale_cost","ss_list_price","ss_coupon_amt"]
<-Reducer 5 [SIMPLE_EDGE]
SHUFFLE [RS_43]
- Group By Operator [GBY_12] (rows=1 width=136)
+ Group By Operator [GBY_12] (rows=1 width=416)
Output:["_col0","_col1","_col2"],aggregations:["avg(VALUE._col0)","count(VALUE._col1)","count(DISTINCT KEY._col0:0._col0)"]
<-Map 4 [SIMPLE_EDGE]
SHUFFLE [RS_11]
@@ -182,7 +182,7 @@ Stage-0
default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_quantity","ss_wholesale_cost","ss_list_price","ss_coupon_amt"]
<-Reducer 7 [SIMPLE_EDGE]
SHUFFLE [RS_44]
- Group By Operator [GBY_19] (rows=1 width=136)
+ Group By Operator [GBY_19] (rows=1 width=416)
Output:["_col0","_col1","_col2"],aggregations:["avg(VALUE._col0)","count(VALUE._col1)","count(DISTINCT KEY._col0:0._col0)"]
<-Map 6 [SIMPLE_EDGE]
SHUFFLE [RS_18]
@@ -196,7 +196,7 @@ Stage-0
default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_quantity","ss_wholesale_cost","ss_list_price","ss_coupon_amt"]
<-Reducer 9 [SIMPLE_EDGE]
SHUFFLE [RS_45]
- Group By Operator [GBY_26] (rows=1 width=136)
+ Group By Operator [GBY_26] (rows=1 width=416)
Output:["_col0","_col1","_col2"],aggregations:["avg(VALUE._col0)","count(VALUE._col1)","count(DISTINCT KEY._col0:0._col0)"]
<-Map 8 [SIMPLE_EDGE]
SHUFFLE [RS_25]
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/reduceSinkDeDuplication_pRS_key_empty.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/reduceSinkDeDuplication_pRS_key_empty.q.out b/ql/src/test/results/clientpositive/reduceSinkDeDuplication_pRS_key_empty.q.out
index 4a848f2..3ad09e8 100644
--- a/ql/src/test/results/clientpositive/reduceSinkDeDuplication_pRS_key_empty.q.out
+++ b/ql/src/test/results/clientpositive/reduceSinkDeDuplication_pRS_key_empty.q.out
@@ -67,15 +67,15 @@ STAGE PLANS:
aggregations: avg(DISTINCT KEY._col0:0._col0), max(KEY._col0:0._col0), variance(KEY._col0:0._col0), var_samp(KEY._col0:0._col0)
mode: complete
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 208 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: compute_stats(_col0, 16), compute_stats(_col1, 16), compute_stats(_col2, 16), compute_stats(_col3, 16)
mode: complete
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 2004 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 2004 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -157,7 +157,7 @@ STAGE PLANS:
aggregations: avg(DISTINCT KEY._col0:0._col0), max(KEY._col0:0._col0), variance(KEY._col0:0._col0), var_samp(KEY._col0:0._col0)
mode: complete
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 208 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
table:
@@ -171,17 +171,17 @@ STAGE PLANS:
TableScan
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 208 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: double), _col1 (type: string), _col2 (type: double), _col3 (type: double)
Reduce Operator Tree:
Group By Operator
aggregations: compute_stats(VALUE._col0, 16), compute_stats(VALUE._col2, 16), compute_stats(VALUE._col3, 16), compute_stats(VALUE._col4, 16)
mode: complete
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 2004 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 2004 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
[19/50] [abbrv] hive git commit: HIVE-14362: Support explain analyze
in Hive (Pengcheng Xiong, reviewed by Ashutosh Chauhan,
Gopal V and Gabor Szadovszky)
Posted by om...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/fd72ec4a/ql/src/test/results/clientpositive/columnstats_tbllvl.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/columnstats_tbllvl.q.out b/ql/src/test/results/clientpositive/columnstats_tbllvl.q.out
index 8d280c1..d56ec6f 100644
--- a/ql/src/test/results/clientpositive/columnstats_tbllvl.q.out
+++ b/ql/src/test/results/clientpositive/columnstats_tbllvl.q.out
@@ -54,23 +54,29 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: uservisits_web_text_none
+ Statistics: Num rows: 65 Data size: 7060 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: sourceip (type: string), avgtimeonsite (type: int), adrevenue (type: float)
outputColumnNames: sourceip, avgtimeonsite, adrevenue
+ Statistics: Num rows: 65 Data size: 7060 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: compute_stats(sourceip, 16), compute_stats(avgtimeonsite, 16), compute_stats(adrevenue, 16)
mode: hash
outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 1444 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
+ Statistics: Num rows: 1 Data size: 1444 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: struct<columntype:string,maxlength:bigint,sumlength:bigint,count:bigint,countnulls:bigint,bitvector:string,numbitvectors:int>), _col1 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,bitvector:string,numbitvectors:int>), _col2 (type: struct<columntype:string,min:double,max:double,countnulls:bigint,bitvector:string,numbitvectors:int>)
Reduce Operator Tree:
Group By Operator
aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 1452 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
+ Statistics: Num rows: 1 Data size: 1452 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -99,17 +105,21 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: uservisits_web_text_none
+ Statistics: Num rows: 65 Data size: 7060 Basic stats: COMPLETE Column stats: NONE
GatherStats: false
Select Operator
expressions: sourceip (type: string), avgtimeonsite (type: int), adrevenue (type: float)
outputColumnNames: sourceip, avgtimeonsite, adrevenue
+ Statistics: Num rows: 65 Data size: 7060 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: compute_stats(sourceip, 16), compute_stats(avgtimeonsite, 16), compute_stats(adrevenue, 16)
mode: hash
outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 1444 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
null sort order:
sort order:
+ Statistics: Num rows: 1 Data size: 1444 Basic stats: COMPLETE Column stats: NONE
tag: -1
value expressions: _col0 (type: struct<columntype:string,maxlength:bigint,sumlength:bigint,count:bigint,countnulls:bigint,bitvector:string,numbitvectors:int>), _col1 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,bitvector:string,numbitvectors:int>), _col2 (type: struct<columntype:string,min:double,max:double,countnulls:bigint,bitvector:string,numbitvectors:int>)
auto parallelism: false
@@ -168,11 +178,13 @@ STAGE PLANS:
aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 1452 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
GlobalTableId: 0
#### A masked pattern was here ####
NumFilesPerFileSink: 1
+ Statistics: Num rows: 1 Data size: 1452 Basic stats: COMPLETE Column stats: NONE
#### A masked pattern was here ####
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
@@ -222,23 +234,29 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: uservisits_web_text_none
+ Statistics: Num rows: 9 Data size: 7060 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: sourceip (type: string), desturl (type: string), visitdate (type: string), adrevenue (type: float), useragent (type: string), ccode (type: string), lcode (type: string), skeyword (type: string), avgtimeonsite (type: int)
outputColumnNames: sourceip, desturl, visitdate, adrevenue, useragent, ccode, lcode, skeyword, avgtimeonsite
+ Statistics: Num rows: 9 Data size: 7060 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: compute_stats(sourceip, 16), compute_stats(desturl, 16), compute_stats(visitdate, 16), compute_stats(adrevenue, 16), compute_stats(useragent, 16), compute_stats(ccode, 16), compute_stats(lcode, 16), compute_stats(skeyword, 16), compute_stats(avgtimeonsite, 16)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
+ Statistics: Num rows: 1 Data size: 4396 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
+ Statistics: Num rows: 1 Data size: 4396 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: struct<columntype:string,maxlength:bigint,sumlength:bigint,count:bigint,countnulls:bigint,bitvector:string,numbitvectors:int>), _col1 (type: struct<columntype:string,maxlength:bigint,sumlength:bigint,count:bigint,countnulls:bigint,bitvector:string,numbitvectors:int>), _col2 (type: struct<columntype:string,maxlength:bigint,sumlength:bigint,count:bigint,countnulls:bigint,bitvector:string,numbitvectors:int>), _col3 (type: struct<columntype:string,min:double,max:double,countnulls:bigint,bitvector:string,numbitvectors:int>), _col4 (type: struct<columntype:string,maxlength:bigint,sumlength:bigint,count:bigint,countnulls:bigint,bitvector:string,numbitvectors:int>), _col5 (type: struct<columntype:string,maxlength:bigint,sumlength:bigint,count:bigint,countnulls:bigint,bitvector:string,numbitvectors:int>), _col6 (type: struct<columntype:string,maxlength:bigint,sumlength:bigint,count:bigint,countnulls:bigint,bitvector:string,numbitvectors:int>)
, _col7 (type: struct<columntype:string,maxlength:bigint,sumlength:bigint,count:bigint,countnulls:bigint,bitvector:string,numbitvectors:int>), _col8 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,bitvector:string,numbitvectors:int>)
Reduce Operator Tree:
Group By Operator
aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2), compute_stats(VALUE._col3), compute_stats(VALUE._col4), compute_stats(VALUE._col5), compute_stats(VALUE._col6), compute_stats(VALUE._col7), compute_stats(VALUE._col8)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
+ Statistics: Num rows: 1 Data size: 4404 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
+ Statistics: Num rows: 1 Data size: 4404 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -322,23 +340,29 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: empty_tab
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: a (type: int), b (type: double), c (type: string), d (type: boolean), e (type: binary)
outputColumnNames: a, b, c, d, e
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Group By Operator
aggregations: compute_stats(a, 16), compute_stats(b, 16), compute_stats(c, 16), compute_stats(d, 16), compute_stats(e, 16)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4
+ Statistics: Num rows: 1 Data size: 2004 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
+ Statistics: Num rows: 1 Data size: 2004 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,bitvector:string,numbitvectors:int>), _col1 (type: struct<columntype:string,min:double,max:double,countnulls:bigint,bitvector:string,numbitvectors:int>), _col2 (type: struct<columntype:string,maxlength:bigint,sumlength:bigint,count:bigint,countnulls:bigint,bitvector:string,numbitvectors:int>), _col3 (type: struct<columntype:string,counttrues:bigint,countfalses:bigint,countnulls:bigint>), _col4 (type: struct<columntype:string,maxlength:bigint,sumlength:bigint,count:bigint,countnulls:bigint>)
Reduce Operator Tree:
Group By Operator
aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2), compute_stats(VALUE._col3), compute_stats(VALUE._col4)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4
+ Statistics: Num rows: 1 Data size: 2012 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
+ Statistics: Num rows: 1 Data size: 2012 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -446,23 +470,29 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: uservisits_in_dummy_db
+ Statistics: Num rows: 65 Data size: 7060 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: sourceip (type: string), avgtimeonsite (type: int), adrevenue (type: float)
outputColumnNames: sourceip, avgtimeonsite, adrevenue
+ Statistics: Num rows: 65 Data size: 7060 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: compute_stats(sourceip, 16), compute_stats(avgtimeonsite, 16), compute_stats(adrevenue, 16)
mode: hash
outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 1444 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
+ Statistics: Num rows: 1 Data size: 1444 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: struct<columntype:string,maxlength:bigint,sumlength:bigint,count:bigint,countnulls:bigint,bitvector:string,numbitvectors:int>), _col1 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,bitvector:string,numbitvectors:int>), _col2 (type: struct<columntype:string,min:double,max:double,countnulls:bigint,bitvector:string,numbitvectors:int>)
Reduce Operator Tree:
Group By Operator
aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 1452 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
+ Statistics: Num rows: 1 Data size: 1452 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -491,17 +521,21 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: uservisits_in_dummy_db
+ Statistics: Num rows: 65 Data size: 7060 Basic stats: COMPLETE Column stats: NONE
GatherStats: false
Select Operator
expressions: sourceip (type: string), avgtimeonsite (type: int), adrevenue (type: float)
outputColumnNames: sourceip, avgtimeonsite, adrevenue
+ Statistics: Num rows: 65 Data size: 7060 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: compute_stats(sourceip, 16), compute_stats(avgtimeonsite, 16), compute_stats(adrevenue, 16)
mode: hash
outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 1444 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
null sort order:
sort order:
+ Statistics: Num rows: 1 Data size: 1444 Basic stats: COMPLETE Column stats: NONE
tag: -1
value expressions: _col0 (type: struct<columntype:string,maxlength:bigint,sumlength:bigint,count:bigint,countnulls:bigint,bitvector:string,numbitvectors:int>), _col1 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,bitvector:string,numbitvectors:int>), _col2 (type: struct<columntype:string,min:double,max:double,countnulls:bigint,bitvector:string,numbitvectors:int>)
auto parallelism: false
@@ -560,11 +594,13 @@ STAGE PLANS:
aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 1452 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
GlobalTableId: 0
#### A masked pattern was here ####
NumFilesPerFileSink: 1
+ Statistics: Num rows: 1 Data size: 1452 Basic stats: COMPLETE Column stats: NONE
#### A masked pattern was here ####
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
@@ -614,23 +650,29 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: uservisits_in_dummy_db
+ Statistics: Num rows: 9 Data size: 7060 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: sourceip (type: string), desturl (type: string), visitdate (type: string), adrevenue (type: float), useragent (type: string), ccode (type: string), lcode (type: string), skeyword (type: string), avgtimeonsite (type: int)
outputColumnNames: sourceip, desturl, visitdate, adrevenue, useragent, ccode, lcode, skeyword, avgtimeonsite
+ Statistics: Num rows: 9 Data size: 7060 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: compute_stats(sourceip, 16), compute_stats(desturl, 16), compute_stats(visitdate, 16), compute_stats(adrevenue, 16), compute_stats(useragent, 16), compute_stats(ccode, 16), compute_stats(lcode, 16), compute_stats(skeyword, 16), compute_stats(avgtimeonsite, 16)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
+ Statistics: Num rows: 1 Data size: 4396 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
+ Statistics: Num rows: 1 Data size: 4396 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: struct<columntype:string,maxlength:bigint,sumlength:bigint,count:bigint,countnulls:bigint,bitvector:string,numbitvectors:int>), _col1 (type: struct<columntype:string,maxlength:bigint,sumlength:bigint,count:bigint,countnulls:bigint,bitvector:string,numbitvectors:int>), _col2 (type: struct<columntype:string,maxlength:bigint,sumlength:bigint,count:bigint,countnulls:bigint,bitvector:string,numbitvectors:int>), _col3 (type: struct<columntype:string,min:double,max:double,countnulls:bigint,bitvector:string,numbitvectors:int>), _col4 (type: struct<columntype:string,maxlength:bigint,sumlength:bigint,count:bigint,countnulls:bigint,bitvector:string,numbitvectors:int>), _col5 (type: struct<columntype:string,maxlength:bigint,sumlength:bigint,count:bigint,countnulls:bigint,bitvector:string,numbitvectors:int>), _col6 (type: struct<columntype:string,maxlength:bigint,sumlength:bigint,count:bigint,countnulls:bigint,bitvector:string,numbitvectors:int>)
, _col7 (type: struct<columntype:string,maxlength:bigint,sumlength:bigint,count:bigint,countnulls:bigint,bitvector:string,numbitvectors:int>), _col8 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,bitvector:string,numbitvectors:int>)
Reduce Operator Tree:
Group By Operator
aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2), compute_stats(VALUE._col3), compute_stats(VALUE._col4), compute_stats(VALUE._col5), compute_stats(VALUE._col6), compute_stats(VALUE._col7), compute_stats(VALUE._col8)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
+ Statistics: Num rows: 1 Data size: 4404 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
+ Statistics: Num rows: 1 Data size: 4404 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/fd72ec4a/ql/src/test/results/clientpositive/compute_stats_date.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/compute_stats_date.q.out b/ql/src/test/results/clientpositive/compute_stats_date.q.out
index d1a7c1a..73f08ad 100644
--- a/ql/src/test/results/clientpositive/compute_stats_date.q.out
+++ b/ql/src/test/results/clientpositive/compute_stats_date.q.out
@@ -60,23 +60,29 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: tab_date
+ Statistics: Num rows: 13 Data size: 778 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: fl_date (type: date)
outputColumnNames: fl_date
+ Statistics: Num rows: 13 Data size: 778 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: compute_stats(fl_date, 16)
mode: hash
outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 572 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
+ Statistics: Num rows: 1 Data size: 572 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: struct<columntype:string,min:date,max:date,countnulls:bigint,bitvector:string,numbitvectors:int>)
Reduce Operator Tree:
Group By Operator
aggregations: compute_stats(VALUE._col0)
mode: mergepartial
outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 576 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
+ Statistics: Num rows: 1 Data size: 576 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/fd72ec4a/ql/src/test/results/clientpositive/constant_prop_2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/constant_prop_2.q.out b/ql/src/test/results/clientpositive/constant_prop_2.q.out
index c1de559..24be518 100644
--- a/ql/src/test/results/clientpositive/constant_prop_2.q.out
+++ b/ql/src/test/results/clientpositive/constant_prop_2.q.out
@@ -37,18 +37,22 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: srcpart
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
outputColumnNames: key, value
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: compute_stats(key, 16), compute_stats(value, 16)
keys: '2008-04-08' (type: string), '11' (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: '2008-04-08' (type: string), '11' (type: string)
sort order: ++
Map-reduce partition columns: '2008-04-08' (type: string), '11' (type: string)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
value expressions: _col2 (type: struct<columntype:string,maxlength:bigint,sumlength:bigint,count:bigint,countnulls:bigint,bitvector:string,numbitvectors:int>), _col3 (type: struct<columntype:string,maxlength:bigint,sumlength:bigint,count:bigint,countnulls:bigint,bitvector:string,numbitvectors:int>)
Reduce Operator Tree:
Group By Operator
@@ -56,11 +60,14 @@ STAGE PLANS:
keys: '2008-04-08' (type: string), '11' (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col2 (type: struct<columntype:string,maxlength:bigint,avglength:double,countnulls:bigint,numdistinctvalues:bigint,ndvbitvector:string>), _col3 (type: struct<columntype:string,maxlength:bigint,avglength:double,countnulls:bigint,numdistinctvalues:bigint,ndvbitvector:string>), '2008-04-08' (type: string), '11' (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/fd72ec4a/ql/src/test/results/clientpositive/display_colstats_tbllvl.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/display_colstats_tbllvl.q.out b/ql/src/test/results/clientpositive/display_colstats_tbllvl.q.out
index 42aeb6f..d974dbc 100644
--- a/ql/src/test/results/clientpositive/display_colstats_tbllvl.q.out
+++ b/ql/src/test/results/clientpositive/display_colstats_tbllvl.q.out
@@ -70,23 +70,29 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: uservisits_web_text_none
+ Statistics: Num rows: 65 Data size: 7060 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: sourceip (type: string), avgtimeonsite (type: int), adrevenue (type: float)
outputColumnNames: sourceip, avgtimeonsite, adrevenue
+ Statistics: Num rows: 65 Data size: 7060 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: compute_stats(sourceip, 16), compute_stats(avgtimeonsite, 16), compute_stats(adrevenue, 16)
mode: hash
outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 1444 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
+ Statistics: Num rows: 1 Data size: 1444 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: struct<columntype:string,maxlength:bigint,sumlength:bigint,count:bigint,countnulls:bigint,bitvector:string,numbitvectors:int>), _col1 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,bitvector:string,numbitvectors:int>), _col2 (type: struct<columntype:string,min:double,max:double,countnulls:bigint,bitvector:string,numbitvectors:int>)
Reduce Operator Tree:
Group By Operator
aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 1452 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
+ Statistics: Num rows: 1 Data size: 1452 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -115,17 +121,21 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: uservisits_web_text_none
+ Statistics: Num rows: 65 Data size: 7060 Basic stats: COMPLETE Column stats: NONE
GatherStats: false
Select Operator
expressions: sourceip (type: string), avgtimeonsite (type: int), adrevenue (type: float)
outputColumnNames: sourceip, avgtimeonsite, adrevenue
+ Statistics: Num rows: 65 Data size: 7060 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: compute_stats(sourceip, 16), compute_stats(avgtimeonsite, 16), compute_stats(adrevenue, 16)
mode: hash
outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 1444 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
null sort order:
sort order:
+ Statistics: Num rows: 1 Data size: 1444 Basic stats: COMPLETE Column stats: NONE
tag: -1
value expressions: _col0 (type: struct<columntype:string,maxlength:bigint,sumlength:bigint,count:bigint,countnulls:bigint,bitvector:string,numbitvectors:int>), _col1 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,bitvector:string,numbitvectors:int>), _col2 (type: struct<columntype:string,min:double,max:double,countnulls:bigint,bitvector:string,numbitvectors:int>)
auto parallelism: false
@@ -184,11 +194,13 @@ STAGE PLANS:
aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 1452 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
GlobalTableId: 0
#### A masked pattern was here ####
NumFilesPerFileSink: 1
+ Statistics: Num rows: 1 Data size: 1452 Basic stats: COMPLETE Column stats: NONE
#### A masked pattern was here ####
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
@@ -294,23 +306,29 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: empty_tab
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: a (type: int), b (type: double), c (type: string), d (type: boolean), e (type: binary)
outputColumnNames: a, b, c, d, e
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Group By Operator
aggregations: compute_stats(a, 16), compute_stats(b, 16), compute_stats(c, 16), compute_stats(d, 16), compute_stats(e, 16)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4
+ Statistics: Num rows: 1 Data size: 2004 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
+ Statistics: Num rows: 1 Data size: 2004 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,bitvector:string,numbitvectors:int>), _col1 (type: struct<columntype:string,min:double,max:double,countnulls:bigint,bitvector:string,numbitvectors:int>), _col2 (type: struct<columntype:string,maxlength:bigint,sumlength:bigint,count:bigint,countnulls:bigint,bitvector:string,numbitvectors:int>), _col3 (type: struct<columntype:string,counttrues:bigint,countfalses:bigint,countnulls:bigint>), _col4 (type: struct<columntype:string,maxlength:bigint,sumlength:bigint,count:bigint,countnulls:bigint>)
Reduce Operator Tree:
Group By Operator
aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2), compute_stats(VALUE._col3), compute_stats(VALUE._col4)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4
+ Statistics: Num rows: 1 Data size: 2012 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
+ Statistics: Num rows: 1 Data size: 2012 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/fd72ec4a/ql/src/test/results/clientpositive/dynpart_sort_optimization_acid.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/dynpart_sort_optimization_acid.q.out b/ql/src/test/results/clientpositive/dynpart_sort_optimization_acid.q.out
index f463c40..d1dc317 100644
--- a/ql/src/test/results/clientpositive/dynpart_sort_optimization_acid.q.out
+++ b/ql/src/test/results/clientpositive/dynpart_sort_optimization_acid.q.out
@@ -78,21 +78,27 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: acid
+ Statistics: Num rows: 1725 Data size: 5177 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (key = 'foo') (type: boolean)
+ Statistics: Num rows: 862 Data size: 2586 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ROW__ID (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>)
outputColumnNames: _col0
+ Statistics: Num rows: 862 Data size: 2586 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>)
sort order: +
Map-reduce partition columns: UDFToInteger(_col0) (type: int)
+ Statistics: Num rows: 862 Data size: 2586 Basic stats: COMPLETE Column stats: NONE
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>), 'foo' (type: string), 'bar' (type: string), '2008-04-08' (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 862 Data size: 2586 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
+ Statistics: Num rows: 862 Data size: 2586 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
@@ -150,22 +156,28 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: acid
+ Statistics: Num rows: 1945 Data size: 5835 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (key = 'foo') (type: boolean)
+ Statistics: Num rows: 972 Data size: 2916 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ROW__ID (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>), ds (type: string)
outputColumnNames: _col0, _col3
+ Statistics: Num rows: 972 Data size: 2916 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>)
sort order: +
Map-reduce partition columns: UDFToInteger(_col0) (type: int)
+ Statistics: Num rows: 972 Data size: 2916 Basic stats: COMPLETE Column stats: NONE
value expressions: _col3 (type: string)
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>), 'foo' (type: string), 'bar' (type: string), VALUE._col1 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 972 Data size: 2916 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
+ Statistics: Num rows: 972 Data size: 2916 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
@@ -313,21 +325,27 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: acid
+ Statistics: Num rows: 1566 Data size: 4698 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (key = 'foo') (type: boolean)
+ Statistics: Num rows: 783 Data size: 2349 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ROW__ID (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>)
outputColumnNames: _col0
+ Statistics: Num rows: 783 Data size: 2349 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>)
sort order: +
Map-reduce partition columns: UDFToInteger(_col0) (type: int)
+ Statistics: Num rows: 783 Data size: 2349 Basic stats: COMPLETE Column stats: NONE
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>), 'foo' (type: string), 'bar' (type: string), '2008-04-08' (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 783 Data size: 2349 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
+ Statistics: Num rows: 783 Data size: 2349 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
@@ -385,21 +403,27 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: acid
+ Statistics: Num rows: 1785 Data size: 5357 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (key = 'foo') (type: boolean)
+ Statistics: Num rows: 892 Data size: 2676 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ROW__ID (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>), ds (type: string)
outputColumnNames: _col0, _col3
+ Statistics: Num rows: 892 Data size: 2676 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col3 (type: string), '_bucket_number' (type: string), _col0 (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>)
sort order: +++
Map-reduce partition columns: _col3 (type: string)
+ Statistics: Num rows: 892 Data size: 2676 Basic stats: COMPLETE Column stats: NONE
Reduce Operator Tree:
Select Operator
- expressions: KEY._col0 (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>), 'foo' (type: string), 'bar' (type: string), KEY._col3 (type: string)
- outputColumnNames: _col0, _col1, _col2, _col3
+ expressions: KEY._col0 (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>), 'foo' (type: string), 'bar' (type: string), KEY._col3 (type: string), KEY.'_bucket_number' (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3, '_bucket_number'
+ Statistics: Num rows: 892 Data size: 2676 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
+ Statistics: Num rows: 892 Data size: 2676 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
@@ -553,21 +577,27 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: acid
+ Statistics: Num rows: 1547 Data size: 4642 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (key = 'foo') (type: boolean)
+ Statistics: Num rows: 773 Data size: 2319 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ROW__ID (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>)
outputColumnNames: _col0
+ Statistics: Num rows: 773 Data size: 2319 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>)
sort order: +
Map-reduce partition columns: UDFToInteger(_col0) (type: int)
+ Statistics: Num rows: 773 Data size: 2319 Basic stats: COMPLETE Column stats: NONE
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>), 'foo' (type: string), 'bar' (type: string), '2008-04-08' (type: string), 11 (type: int)
outputColumnNames: _col0, _col1, _col2, _col3, _col4
+ Statistics: Num rows: 773 Data size: 2319 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
+ Statistics: Num rows: 773 Data size: 2319 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
@@ -626,22 +656,28 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: acid
+ Statistics: Num rows: 3032 Data size: 9099 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (key = 'foo') (type: boolean)
+ Statistics: Num rows: 1516 Data size: 4549 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ROW__ID (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>), hr (type: int)
outputColumnNames: _col0, _col4
+ Statistics: Num rows: 1516 Data size: 4549 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>)
sort order: +
Map-reduce partition columns: UDFToInteger(_col0) (type: int)
+ Statistics: Num rows: 1516 Data size: 4549 Basic stats: COMPLETE Column stats: NONE
value expressions: _col4 (type: int)
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>), 'foo' (type: string), 'bar' (type: string), '2008-04-08' (type: string), VALUE._col2 (type: int)
outputColumnNames: _col0, _col1, _col2, _col3, _col4
+ Statistics: Num rows: 1516 Data size: 4549 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
+ Statistics: Num rows: 1516 Data size: 4549 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
@@ -896,21 +932,27 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: acid
+ Statistics: Num rows: 1548 Data size: 4644 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (key = 'foo') (type: boolean)
+ Statistics: Num rows: 774 Data size: 2322 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ROW__ID (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>)
outputColumnNames: _col0
+ Statistics: Num rows: 774 Data size: 2322 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>)
sort order: +
Map-reduce partition columns: UDFToInteger(_col0) (type: int)
+ Statistics: Num rows: 774 Data size: 2322 Basic stats: COMPLETE Column stats: NONE
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>), 'foo' (type: string), 'bar' (type: string), '2008-04-08' (type: string), 11 (type: int)
outputColumnNames: _col0, _col1, _col2, _col3, _col4
+ Statistics: Num rows: 774 Data size: 2322 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
+ Statistics: Num rows: 774 Data size: 2322 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
@@ -969,21 +1011,27 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: acid
+ Statistics: Num rows: 3034 Data size: 9103 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (key = 'foo') (type: boolean)
+ Statistics: Num rows: 1517 Data size: 4551 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ROW__ID (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>), hr (type: int)
outputColumnNames: _col0, _col4
+ Statistics: Num rows: 1517 Data size: 4551 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: '2008-04-08' (type: string), _col4 (type: int), '_bucket_number' (type: string), _col0 (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>)
sort order: ++++
Map-reduce partition columns: '2008-04-08' (type: string), _col4 (type: int)
+ Statistics: Num rows: 1517 Data size: 4551 Basic stats: COMPLETE Column stats: NONE
Reduce Operator Tree:
Select Operator
- expressions: KEY._col0 (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>), 'foo' (type: string), 'bar' (type: string), '2008-04-08' (type: string), KEY._col4 (type: int)
- outputColumnNames: _col0, _col1, _col2, _col3, _col4
+ expressions: KEY._col0 (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>), 'foo' (type: string), 'bar' (type: string), '2008-04-08' (type: string), KEY._col4 (type: int), KEY.'_bucket_number' (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, '_bucket_number'
+ Statistics: Num rows: 1517 Data size: 4551 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
+ Statistics: Num rows: 1517 Data size: 4551 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
@@ -1082,8 +1130,8 @@ STAGE PLANS:
Map-reduce partition columns: _col1 (type: string), _col2 (type: int)
Reduce Operator Tree:
Select Operator
- expressions: KEY._col0 (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>), KEY._col1 (type: string), KEY._col2 (type: int)
- outputColumnNames: _col0, _col1, _col2
+ expressions: KEY._col0 (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>), KEY._col1 (type: string), KEY._col2 (type: int), KEY.'_bucket_number' (type: string)
+ outputColumnNames: _col0, _col1, _col2, '_bucket_number'
File Output Operator
compressed: false
table:
@@ -1237,22 +1285,28 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: acid
+ Statistics: Num rows: 46 Data size: 4644 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (key = 'foo') (type: boolean)
+ Statistics: Num rows: 23 Data size: 2322 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ROW__ID (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>), key (type: string), ds (type: string), hr (type: int)
outputColumnNames: _col0, _col1, _col3, _col4
+ Statistics: Num rows: 23 Data size: 2322 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col3 (type: string), _col4 (type: int), '_bucket_number' (type: string), _col0 (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>)
sort order: ++++
Map-reduce partition columns: _col3 (type: string), _col4 (type: int)
+ Statistics: Num rows: 23 Data size: 2322 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: string), 'bar' (type: string)
Reduce Operator Tree:
Select Operator
- expressions: KEY._col0 (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>), VALUE._col1 (type: string), VALUE._col2 (type: string), KEY._col3 (type: string), KEY._col4 (type: int)
- outputColumnNames: _col0, _col1, _col2, _col3, _col4
+ expressions: KEY._col0 (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>), VALUE._col1 (type: string), VALUE._col2 (type: string), KEY._col3 (type: string), KEY._col4 (type: int), KEY.'_bucket_number' (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, '_bucket_number'
+ Statistics: Num rows: 23 Data size: 2322 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
+ Statistics: Num rows: 23 Data size: 2322 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
@@ -1311,22 +1365,28 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: acid
+ Statistics: Num rows: 90 Data size: 9101 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (key = 'foo') (type: boolean)
+ Statistics: Num rows: 45 Data size: 4550 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ROW__ID (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>), key (type: string), ds (type: string), hr (type: int)
outputColumnNames: _col0, _col1, _col3, _col4
+ Statistics: Num rows: 45 Data size: 4550 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col3 (type: string), _col4 (type: int), '_bucket_number' (type: string), _col0 (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>)
sort order: ++++
Map-reduce partition columns: _col3 (type: string), _col4 (type: int)
+ Statistics: Num rows: 45 Data size: 4550 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: string), 'bar' (type: string)
Reduce Operator Tree:
Select Operator
- expressions: KEY._col0 (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>), VALUE._col1 (type: string), VALUE._col2 (type: string), KEY._col3 (type: string), KEY._col4 (type: int)
- outputColumnNames: _col0, _col1, _col2, _col3, _col4
+ expressions: KEY._col0 (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>), VALUE._col1 (type: string), VALUE._col2 (type: string), KEY._col3 (type: string), KEY._col4 (type: int), KEY.'_bucket_number' (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, '_bucket_number'
+ Statistics: Num rows: 45 Data size: 4550 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
+ Statistics: Num rows: 45 Data size: 4550 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/fd72ec4a/ql/src/test/results/clientpositive/exec_parallel_column_stats.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/exec_parallel_column_stats.q.out b/ql/src/test/results/clientpositive/exec_parallel_column_stats.q.out
index 4fcde91..f256ec1 100644
--- a/ql/src/test/results/clientpositive/exec_parallel_column_stats.q.out
+++ b/ql/src/test/results/clientpositive/exec_parallel_column_stats.q.out
@@ -12,23 +12,29 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: src
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
outputColumnNames: key, value
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: compute_stats(key, 16), compute_stats(value, 16)
mode: hash
outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 984 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
+ Statistics: Num rows: 1 Data size: 984 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: struct<columntype:string,maxlength:bigint,sumlength:bigint,count:bigint,countnulls:bigint,bitvector:string,numbitvectors:int>), _col1 (type: struct<columntype:string,maxlength:bigint,sumlength:bigint,count:bigint,countnulls:bigint,bitvector:string,numbitvectors:int>)
Reduce Operator Tree:
Group By Operator
aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1)
mode: mergepartial
outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 984 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
+ Statistics: Num rows: 1 Data size: 984 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/fd72ec4a/ql/src/test/results/clientpositive/temp_table_display_colstats_tbllvl.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/temp_table_display_colstats_tbllvl.q.out b/ql/src/test/results/clientpositive/temp_table_display_colstats_tbllvl.q.out
index 4dc052c..c46ba12 100644
--- a/ql/src/test/results/clientpositive/temp_table_display_colstats_tbllvl.q.out
+++ b/ql/src/test/results/clientpositive/temp_table_display_colstats_tbllvl.q.out
@@ -74,23 +74,29 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: uservisits_web_text_none
+ Statistics: Num rows: 65 Data size: 7060 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: sourceip (type: string), avgtimeonsite (type: int), adrevenue (type: float)
outputColumnNames: sourceip, avgtimeonsite, adrevenue
+ Statistics: Num rows: 65 Data size: 7060 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: compute_stats(sourceip, 16), compute_stats(avgtimeonsite, 16), compute_stats(adrevenue, 16)
mode: hash
outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 1444 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
+ Statistics: Num rows: 1 Data size: 1444 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: struct<columntype:string,maxlength:bigint,sumlength:bigint,count:bigint,countnulls:bigint,bitvector:string,numbitvectors:int>), _col1 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,bitvector:string,numbitvectors:int>), _col2 (type: struct<columntype:string,min:double,max:double,countnulls:bigint,bitvector:string,numbitvectors:int>)
Reduce Operator Tree:
Group By Operator
aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 1452 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
+ Statistics: Num rows: 1 Data size: 1452 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -119,17 +125,21 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: uservisits_web_text_none
+ Statistics: Num rows: 65 Data size: 7060 Basic stats: COMPLETE Column stats: NONE
GatherStats: false
Select Operator
expressions: sourceip (type: string), avgtimeonsite (type: int), adrevenue (type: float)
outputColumnNames: sourceip, avgtimeonsite, adrevenue
+ Statistics: Num rows: 65 Data size: 7060 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: compute_stats(sourceip, 16), compute_stats(avgtimeonsite, 16), compute_stats(adrevenue, 16)
mode: hash
outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 1444 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
null sort order:
sort order:
+ Statistics: Num rows: 1 Data size: 1444 Basic stats: COMPLETE Column stats: NONE
tag: -1
value expressions: _col0 (type: struct<columntype:string,maxlength:bigint,sumlength:bigint,count:bigint,countnulls:bigint,bitvector:string,numbitvectors:int>), _col1 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,bitvector:string,numbitvectors:int>), _col2 (type: struct<columntype:string,min:double,max:double,countnulls:bigint,bitvector:string,numbitvectors:int>)
auto parallelism: false
@@ -184,11 +194,13 @@ STAGE PLANS:
aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 1452 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
GlobalTableId: 0
#### A masked pattern was here ####
NumFilesPerFileSink: 1
+ Statistics: Num rows: 1 Data size: 1452 Basic stats: COMPLETE Column stats: NONE
#### A masked pattern was here ####
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
@@ -294,23 +306,29 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: empty_tab
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: a (type: int), b (type: double), c (type: string), d (type: boolean), e (type: binary)
outputColumnNames: a, b, c, d, e
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Group By Operator
aggregations: compute_stats(a, 16), compute_stats(b, 16), compute_stats(c, 16), compute_stats(d, 16), compute_stats(e, 16)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4
+ Statistics: Num rows: 1 Data size: 2004 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
+ Statistics: Num rows: 1 Data size: 2004 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,bitvector:string,numbitvectors:int>), _col1 (type: struct<columntype:string,min:double,max:double,countnulls:bigint,bitvector:string,numbitvectors:int>), _col2 (type: struct<columntype:string,maxlength:bigint,sumlength:bigint,count:bigint,countnulls:bigint,bitvector:string,numbitvectors:int>), _col3 (type: struct<columntype:string,counttrues:bigint,countfalses:bigint,countnulls:bigint>), _col4 (type: struct<columntype:string,maxlength:bigint,sumlength:bigint,count:bigint,countnulls:bigint>)
Reduce Operator Tree:
Group By Operator
aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2), compute_stats(VALUE._col3), compute_stats(VALUE._col4)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4
+ Statistics: Num rows: 1 Data size: 2012 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
+ Statistics: Num rows: 1 Data size: 2012 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/fd72ec4a/ql/src/test/results/clientpositive/tez/explainanalyze_1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/explainanalyze_1.q.out b/ql/src/test/results/clientpositive/tez/explainanalyze_1.q.out
new file mode 100644
index 0000000..6fe287a
--- /dev/null
+++ b/ql/src/test/results/clientpositive/tez/explainanalyze_1.q.out
@@ -0,0 +1,471 @@
+PREHOOK: query: select * from src a union all select * from src b limit 10
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: select * from src a union all select * from src b limit 10
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+PREHOOK: query: explain analyze select * from src a union all select * from src b limit 10
+PREHOOK: type: QUERY
+POSTHOOK: query: explain analyze select * from src a union all select * from src b limit 10
+POSTHOOK: type: QUERY
+Plan optimized by CBO.
+
+Vertex dependency in root stage
+Map 1 <- Union 2 (CONTAINS)
+Map 3 <- Union 2 (CONTAINS)
+
+Stage-0
+ Fetch Operator
+ limit:10
+ Stage-1
+ Union 2
+ <-Map 1 [CONTAINS]
+ File Output Operator [FS_7]
+ Limit [LIM_6] (rows=10/20 width=178)
+ Number of rows:10
+ Select Operator [SEL_1] (rows=500/12 width=178)
+ Output:["_col0","_col1"]
+ TableScan [TS_0] (rows=500/13 width=178)
+ Output:["key","value"]
+ <-Map 3 [CONTAINS]
+ File Output Operator [FS_7]
+ Limit [LIM_6] (rows=10/20 width=178)
+ Number of rows:10
+ Select Operator [SEL_3] (rows=500/12 width=178)
+ Output:["_col0","_col1"]
+ TableScan [TS_2] (rows=500/13 width=178)
+ Output:["key","value"]
+
+PREHOOK: query: select key from src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: select key from src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+PREHOOK: query: explain analyze select key from src
+PREHOOK: type: QUERY
+POSTHOOK: query: explain analyze select key from src
+POSTHOOK: type: QUERY
+Plan optimized by CBO.
+
+Stage-0
+ Fetch Operator
+ limit:-1
+ Select Operator [SEL_1]
+ Output:["_col0"]
+ TableScan [TS_0]
+ Output:["key"]
+
+PREHOOK: query: create table t as select key from src
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@src
+PREHOOK: Output: database:default
+PREHOOK: Output: default@t
+POSTHOOK: query: create table t as select key from src
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@src
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@t
+PREHOOK: query: explain analyze create table t as select key from src
+PREHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: query: explain analyze create table t as select key from src
+POSTHOOK: type: CREATETABLE_AS_SELECT
+Plan optimized by CBO.
+
+Stage-3
+ Stats-Aggr Operator
+ Stage-4
+ Create Table Operator:
+ name:default.t
+ Stage-2
+ Dependency Collection{}
+ Stage-1
+ Map 1
+ File Output Operator [FS_2]
+ table:{"name:":"default.t"}
+ Select Operator [SEL_1] (rows=500/500 width=87)
+ Output:["_col0"]
+ TableScan [TS_0] (rows=500/500 width=87)
+ default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key"]
+ Stage-0
+ Move Operator
+ Please refer to the previous Stage-1
+
+PREHOOK: query: create table t as select key from src
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@src
+PREHOOK: Output: database:default
+PREHOOK: Output: default@t
+POSTHOOK: query: create table t as select key from src
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@src
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@t
+POSTHOOK: Lineage: t.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+PREHOOK: query: insert overwrite table t select key from src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@t
+POSTHOOK: query: insert overwrite table t select key from src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@t
+PREHOOK: query: explain analyze insert overwrite table t select key from src
+PREHOOK: type: QUERY
+POSTHOOK: query: explain analyze insert overwrite table t select key from src
+POSTHOOK: type: QUERY
+Plan optimized by CBO.
+
+Stage-3
+ Stats-Aggr Operator
+ Stage-0
+ Move Operator
+ table:{"name:":"default.t"}
+ Stage-2
+ Dependency Collection{}
+ Stage-1
+ Map 1
+ File Output Operator [FS_2]
+ table:{"name:":"default.t"}
+ Select Operator [SEL_1] (rows=500/500 width=87)
+ Output:["_col0"]
+ TableScan [TS_0] (rows=500/500 width=87)
+ default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key"]
+
+PREHOOK: query: select key from src limit 10
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: select key from src limit 10
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+PREHOOK: query: explain analyze select key from src limit 10
+PREHOOK: type: QUERY
+POSTHOOK: query: explain analyze select key from src limit 10
+POSTHOOK: type: QUERY
+Plan optimized by CBO.
+
+Stage-0
+ Fetch Operator
+ limit:10
+ Limit [LIM_2]
+ Number of rows:10
+ Select Operator [SEL_1]
+ Output:["_col0"]
+ TableScan [TS_0]
+ Output:["key"]
+
+PREHOOK: query: select key from src where value < 10
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: select key from src where value < 10
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+PREHOOK: query: explain analyze select key from src where value < 10
+PREHOOK: type: QUERY
+POSTHOOK: query: explain analyze select key from src where value < 10
+POSTHOOK: type: QUERY
+Plan optimized by CBO.
+
+Stage-0
+ Fetch Operator
+ limit:-1
+ Select Operator [SEL_2]
+ Output:["_col0"]
+ Filter Operator [FIL_4]
+ predicate:(UDFToDouble(value) < 10.0)
+ TableScan [TS_0]
+ Output:["key","value"]
+
+PREHOOK: query: select key from src where key < 10
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: select key from src where key < 10
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+PREHOOK: query: explain analyze select key from src where key < 10
+PREHOOK: type: QUERY
+POSTHOOK: query: explain analyze select key from src where key < 10
+POSTHOOK: type: QUERY
+Plan optimized by CBO.
+
+Stage-0
+ Fetch Operator
+ limit:-1
+ Select Operator [SEL_2]
+ Output:["_col0"]
+ Filter Operator [FIL_4]
+ predicate:(UDFToDouble(key) < 10.0)
+ TableScan [TS_0]
+ Output:["key"]
+
+PREHOOK: query: select count(*) from (select key from src where key < 10)subq
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from (select key from src where key < 10)subq
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+10
+PREHOOK: query: select key, count(key) from src group by key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: select key, count(key) from src group by key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+PREHOOK: query: explain analyze select key, count(key) from src group by key
+PREHOOK: type: QUERY
+POSTHOOK: query: explain analyze select key, count(key) from src group by key
+POSTHOOK: type: QUERY
+Plan optimized by CBO.
+
+Vertex dependency in root stage
+Reducer 2 <- Map 1 (SIMPLE_EDGE)
+
+Stage-0
+ Fetch Operator
+ limit:-1
+ Stage-1
+ Reducer 2
+ File Output Operator [FS_6]
+ Group By Operator [GBY_4] (rows=205/309 width=95)
+ Output:["_col0","_col1"],aggregations:["count(VALUE._col0)"],keys:KEY._col0
+ <-Map 1 [SIMPLE_EDGE]
+ SHUFFLE [RS_3]
+ PartitionCols:_col0
+ Group By Operator [GBY_2] (rows=205/309 width=95)
+ Output:["_col0","_col1"],aggregations:["count(key)"],keys:key
+ Select Operator [SEL_1] (rows=500/500 width=87)
+ Output:["key"]
+ TableScan [TS_0] (rows=500/500 width=87)
+ default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key"]
+
+PREHOOK: query: select count(*) from (select key, count(key) from src group by key)subq
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from (select key, count(key) from src group by key)subq
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+309
+PREHOOK: query: select count(*) from src a join src b on a.key = b.value where a.key > 0
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from src a join src b on a.key = b.value where a.key > 0
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+PREHOOK: query: explain analyze select count(*) from src a join src b on a.key = b.value where a.key > 0
+PREHOOK: type: QUERY
+POSTHOOK: query: explain analyze select count(*) from src a join src b on a.key = b.value where a.key > 0
+POSTHOOK: type: QUERY
+Plan optimized by CBO.
+
+Vertex dependency in root stage
+Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE)
+Reducer 3 <- Reducer 2 (SIMPLE_EDGE)
+
+Stage-0
+ Fetch Operator
+ limit:-1
+ Stage-1
+ Reducer 3
+ File Output Operator [FS_14]
+ Group By Operator [GBY_12] (rows=1/1 width=8)
+ Output:["_col0"],aggregations:["count(VALUE._col0)"]
+ <-Reducer 2 [SIMPLE_EDGE]
+ SHUFFLE [RS_11]
+ Group By Operator [GBY_10] (rows=1/1 width=8)
+ Output:["_col0"],aggregations:["count()"]
+ Merge Join Operator [MERGEJOIN_19] (rows=382/0 width=8)
+ Conds:RS_6._col0=RS_7._col0(Inner)
+ <-Map 1 [SIMPLE_EDGE]
+ SHUFFLE [RS_6]
+ PartitionCols:_col0
+ Select Operator [SEL_2] (rows=166/497 width=87)
+ Output:["_col0"]
+ Filter Operator [FIL_17] (rows=166/497 width=87)
+ predicate:(UDFToDouble(key) > 0.0)
+ TableScan [TS_0] (rows=500/500 width=87)
+ default@src,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key"]
+ <-Map 4 [SIMPLE_EDGE]
+ SHUFFLE [RS_7]
+ PartitionCols:_col0
+ Select Operator [SEL_5] (rows=166/0 width=91)
+ Output:["_col0"]
+ Filter Operator [FIL_18] (rows=166/0 width=91)
+ predicate:(UDFToDouble(value) > 0.0)
+ TableScan [TS_3] (rows=500/500 width=91)
+ default@src,b,Tbl:COMPLETE,Col:COMPLETE,Output:["value"]
+
+PREHOOK: query: select count(*) from src a join src b on a.key = b.key where a.key > 0
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from src a join src b on a.key = b.key where a.key > 0
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+PREHOOK: query: explain analyze select count(*) from src a join src b on a.key = b.key where a.key > 0
+PREHOOK: type: QUERY
+POSTHOOK: query: explain analyze select count(*) from src a join src b on a.key = b.key where a.key > 0
+POSTHOOK: type: QUERY
+Plan optimized by CBO.
+
+Vertex dependency in root stage
+Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE)
+Reducer 3 <- Reducer 2 (SIMPLE_EDGE)
+
+Stage-0
+ Fetch Operator
+ limit:-1
+ Stage-1
+ Reducer 3
+ File Output Operator [FS_14]
+ Group By Operator [GBY_12] (rows=1/1 width=8)
+ Output:["_col0"],aggregations:["count(VALUE._col0)"]
+ <-Reducer 2 [SIMPLE_EDGE]
+ SHUFFLE [RS_11]
+ Group By Operator [GBY_10] (rows=1/1 width=8)
+ Output:["_col0"],aggregations:["count()"]
+ Merge Join Operator [MERGEJOIN_19] (rows=399/1019 width=8)
+ Conds:RS_6._col0=RS_7._col0(Inner)
+ <-Map 1 [SIMPLE_EDGE]
+ SHUFFLE [RS_6]
+ PartitionCols:_col0
+ Select Operator [SEL_2] (rows=166/497 width=87)
+ Output:["_col0"]
+ Filter Operator [FIL_17] (rows=166/497 width=87)
+ predicate:(UDFToDouble(key) > 0.0)
+ TableScan [TS_0] (rows=500/500 width=87)
+ default@src,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key"]
+ <-Map 4 [SIMPLE_EDGE]
+ SHUFFLE [RS_7]
+ PartitionCols:_col0
+ Select Operator [SEL_5] (rows=166/497 width=87)
+ Output:["_col0"]
+ Filter Operator [FIL_18] (rows=166/497 width=87)
+ predicate:(UDFToDouble(key) > 0.0)
+ TableScan [TS_3] (rows=500/500 width=87)
+ default@src,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key"]
+
+PREHOOK: query: select count(*) from src a join src b on a.key = b.key where a.key > 0
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from src a join src b on a.key = b.key where a.key > 0
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+1019
+PREHOOK: query: select * from src a union all select * from src b
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: select * from src a union all select * from src b
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+PREHOOK: query: explain analyze select * from src a union all select * from src b
+PREHOOK: type: QUERY
+POSTHOOK: query: explain analyze select * from src a union all select * from src b
+POSTHOOK: type: QUERY
+Plan optimized by CBO.
+
+Vertex dependency in root stage
+Map 1 <- Union 2 (CONTAINS)
+Map 3 <- Union 2 (CONTAINS)
+
+Stage-0
+ Fetch Operator
+ limit:-1
+ Stage-1
+ Union 2
+ <-Map 1 [CONTAINS]
+ File Output Operator [FS_6]
+ Select Operator [SEL_1] (rows=500/500 width=178)
+ Output:["_col0","_col1"]
+ TableScan [TS_0] (rows=500/500 width=178)
+ Output:["key","value"]
+ <-Map 3 [CONTAINS]
+ File Output Operator [FS_6]
+ Select Operator [SEL_3] (rows=500/500 width=178)
+ Output:["_col0","_col1"]
+ TableScan [TS_2] (rows=500/500 width=178)
+ Output:["key","value"]
+
+PREHOOK: query: select count(*) from (select * from src a union all select * from src b)subq
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from (select * from src a union all select * from src b)subq
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+1000
+PREHOOK: query: SELECT x.key, y.value
+FROM src x JOIN src y ON (x.key = y.key)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT x.key, y.value
+FROM src x JOIN src y ON (x.key = y.key)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+PREHOOK: query: EXPLAIN analyze
+SELECT x.key, y.value
+FROM src x JOIN src y ON (x.key = y.key)
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN analyze
+SELECT x.key, y.value
+FROM src x JOIN src y ON (x.key = y.key)
+POSTHOOK: type: QUERY
+Plan optimized by CBO.
+
+Vertex dependency in root stage
+Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE)
+
+Stage-0
+ Fetch Operator
+ limit:-1
+ Stage-1
+ Reducer 2
+ File Output Operator [FS_10]
+ Select Operator [SEL_9] (rows=1219/1028 width=178)
+ Output:["_col0","_col1"]
+ Merge Join Operator [MERGEJOIN_15] (rows=1219/1028 width=178)
+ Conds:RS_6._col0=RS_7._col0(Inner),Output:["_col0","_col2"]
+ <-Map 1 [SIMPLE_EDGE]
+ SHUFFLE [RS_6]
+ PartitionCols:_col0
+ Select Operator [SEL_2] (rows=500/500 width=87)
+ Output:["_col0"]
+ Filter Operator [FIL_13] (rows=500/500 width=87)
+ predicate:key is not null
+ TableScan [TS_0] (rows=500/500 width=87)
+ default@src,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key"]
+ <-Map 3 [SIMPLE_EDGE]
+ SHUFFLE [RS_7]
+ PartitionCols:_col0
+ Select Operator [SEL_5] (rows=500/500 width=178)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_14] (rows=500/500 width=178)
+ predicate:key is not null
+ TableScan [TS_3] (rows=500/500 width=178)
+ default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
+
[22/50] [abbrv] hive git commit: HIVE-14362: Support explain analyze
in Hive (Pengcheng Xiong, reviewed by Ashutosh Chauhan,
Gopal V and Gabor Szadovszky)
Posted by om...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/fd72ec4a/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java
index 1688524..3cf8b71 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java
@@ -35,7 +35,9 @@ import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.StatsSetupConst;
import org.apache.hadoop.hive.ql.CompilationOpContext;
+import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.exec.mr.ExecMapperContext;
import org.apache.hadoop.hive.ql.lib.Node;
import org.apache.hadoop.hive.ql.metadata.HiveException;
@@ -46,6 +48,9 @@ import org.apache.hadoop.hive.ql.plan.OpTraits;
import org.apache.hadoop.hive.ql.plan.OperatorDesc;
import org.apache.hadoop.hive.ql.plan.Statistics;
import org.apache.hadoop.hive.ql.plan.api.OperatorType;
+import org.apache.hadoop.hive.ql.stats.StatsCollectionContext;
+import org.apache.hadoop.hive.ql.stats.StatsPublisher;
+import org.apache.hadoop.hive.ql.stats.fs.FSStatsPublisher;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
@@ -77,6 +82,9 @@ public abstract class Operator<T extends OperatorDesc> implements Serializable,C
protected final AtomicBoolean abortOp;
private transient ExecMapperContext execContext;
private transient boolean rootInitializeCalled = false;
+ protected transient long runTimeNumRows;
+ protected int indexForTezUnion = -1;
+ private transient Configuration hconf;
protected final transient Collection<Future<?>> asyncInitOperations = new HashSet<>();
// It can be optimized later so that an operator operator (init/close) is performed
@@ -476,7 +484,9 @@ public abstract class Operator<T extends OperatorDesc> implements Serializable,C
* Operator specific initialization.
*/
protected void initializeOp(Configuration hconf) throws HiveException {
+ this.hconf = hconf;
rootInitializeCalled = true;
+ runTimeNumRows = 0;
}
/**
@@ -715,6 +725,10 @@ public abstract class Operator<T extends OperatorDesc> implements Serializable,C
* should overwrite this funtion for their specific cleanup routine.
*/
protected void closeOp(boolean abort) throws HiveException {
+ if (conf != null && conf.getRuntimeStatsTmpDir() != null) {
+ publishRunTimeStats();
+ }
+ runTimeNumRows = 0;
}
private boolean jobCloseDone = false;
@@ -869,7 +883,7 @@ public abstract class Operator<T extends OperatorDesc> implements Serializable,C
protected void forward(Object row, ObjectInspector rowInspector)
throws HiveException {
-
+ runTimeNumRows++;
if (getDone()) {
return;
}
@@ -1425,4 +1439,38 @@ public abstract class Operator<T extends OperatorDesc> implements Serializable,C
public CompilationOpContext getCompilationOpContext() {
return cContext;
}
+
+ private void publishRunTimeStats() throws HiveException {
+ StatsPublisher statsPublisher = new FSStatsPublisher();
+ StatsCollectionContext sContext = new StatsCollectionContext(hconf);
+ sContext.setIndexForTezUnion(indexForTezUnion);
+ sContext.setStatsTmpDir(conf.getRuntimeStatsTmpDir());
+
+ if (!statsPublisher.connect(sContext)) {
+ LOG.error("StatsPublishing error: cannot connect to database");
+ throw new HiveException(ErrorMsg.STATSPUBLISHER_CONNECTION_ERROR.getErrorCodedMsg());
+ }
+
+ String prefix = "";
+ Map<String, String> statsToPublish = new HashMap<String, String>();
+ statsToPublish.put(StatsSetupConst.RUN_TIME_ROW_COUNT, Long.toString(runTimeNumRows));
+ if (!statsPublisher.publishStat(prefix, statsToPublish)) {
+ // The original exception is lost.
+ // Not changing the interface to maintain backward compatibility
+ throw new HiveException(ErrorMsg.STATSPUBLISHER_PUBLISHING_ERROR.getErrorCodedMsg());
+ }
+ if (!statsPublisher.closeConnection(sContext)) {
+ // The original exception is lost.
+ // Not changing the interface to maintain backward compatibility
+ throw new HiveException(ErrorMsg.STATSPUBLISHER_CLOSING_ERROR.getErrorCodedMsg());
+ }
+ }
+
+ public int getIndexForTezUnion() {
+ return indexForTezUnion;
+ }
+
+ public void setIndexForTezUnion(int indexForTezUnion) {
+ this.indexForTezUnion = indexForTezUnion;
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/fd72ec4a/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java
index fa23b01..c4490f2 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java
@@ -548,6 +548,7 @@ public class ReduceSinkOperator extends TerminalOperator<ReduceSinkDesc>
// forward is not called
if (null != out) {
numRows++;
+ runTimeNumRows++;
if (isLogInfoEnabled) {
if (numRows == cntr) {
cntr = logEveryNRows == 0 ? cntr * 10 : numRows + logEveryNRows;
http://git-wip-us.apache.org/repos/asf/hive/blob/fd72ec4a/ql/src/java/org/apache/hadoop/hive/ql/exec/SerializationUtilities.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/SerializationUtilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/SerializationUtilities.java
index 277683e..247d589 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/SerializationUtilities.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/SerializationUtilities.java
@@ -546,6 +546,28 @@ public class SerializationUtilities {
return result;
}
+ public static List<Operator<?>> cloneOperatorTree(List<Operator<?>> roots, int indexForTezUnion) {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream(4096);
+ CompilationOpContext ctx = roots.isEmpty() ? null : roots.get(0).getCompilationOpContext();
+ serializePlan(roots, baos, true);
+ @SuppressWarnings("unchecked")
+ List<Operator<?>> result =
+ deserializePlan(new ByteArrayInputStream(baos.toByteArray()),
+ roots.getClass(), true);
+ // Restore the context.
+ LinkedList<Operator<?>> newOps = new LinkedList<>(result);
+ while (!newOps.isEmpty()) {
+ Operator<?> newOp = newOps.poll();
+ newOp.setIndexForTezUnion(indexForTezUnion);
+ newOp.setCompilationOpContext(ctx);
+ List<Operator<?>> children = newOp.getChildOperators();
+ if (children != null) {
+ newOps.addAll(children);
+ }
+ }
+ return result;
+ }
+
/**
* Clones using the powers of XML. Do not use unless necessary.
* @param plan The plan.
http://git-wip-us.apache.org/repos/asf/hive/blob/fd72ec4a/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java
index 602ed1a..483fc3a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java
@@ -46,6 +46,7 @@ import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.Partition;
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.TableSpec;
+import org.apache.hadoop.hive.ql.parse.ExplainConfiguration.AnalyzeState;
import org.apache.hadoop.hive.ql.plan.DynamicPartitionCtx;
import org.apache.hadoop.hive.ql.plan.LoadTableDesc;
import org.apache.hadoop.hive.ql.plan.StatsWork;
@@ -87,7 +88,9 @@ public class StatsTask extends Task<StatsWork> implements Serializable {
@Override
public int execute(DriverContext driverContext) {
-
+ if (driverContext.getCtx().getExplainAnalyze() == AnalyzeState.RUNNING) {
+ return 0;
+ }
LOG.info("Executing stats task");
// Make sure that it is either an ANALYZE, INSERT OVERWRITE (maybe load) or CTAS command
short workComponentsPresent = 0;
@@ -147,7 +150,7 @@ public class StatsTask extends Task<StatsWork> implements Serializable {
if (!getWork().getNoStatsAggregator() && !getWork().isNoScanAnalyzeCommand()) {
try {
scc = getContext();
- statsAggregator = createStatsAggregator(scc);
+ statsAggregator = createStatsAggregator(scc, conf);
} catch (HiveException e) {
if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_STATS_RELIABLE)) {
throw e;
@@ -294,7 +297,7 @@ public class StatsTask extends Task<StatsWork> implements Serializable {
return prefix;
}
- private StatsAggregator createStatsAggregator(StatsCollectionContext scc) throws HiveException {
+ private StatsAggregator createStatsAggregator(StatsCollectionContext scc, HiveConf conf) throws HiveException {
String statsImpl = HiveConf.getVar(conf, HiveConf.ConfVars.HIVESTATSDBCLASS);
StatsFactory factory = StatsFactory.newFactory(statsImpl, conf);
if (factory == null) {
http://git-wip-us.apache.org/repos/asf/hive/blob/fd72ec4a/ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java
index 6afe957..0f02222 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java
@@ -267,6 +267,7 @@ public class TableScanOperator extends Operator<TableScanDesc> implements
publishStats();
}
}
+ super.closeOp(abort);
}
/**
http://git-wip-us.apache.org/repos/asf/hive/blob/fd72ec4a/ql/src/java/org/apache/hadoop/hive/ql/exec/UDTFOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/UDTFOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/UDTFOperator.java
index a75b52a..0d1fa31 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/UDTFOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/UDTFOperator.java
@@ -150,5 +150,6 @@ public class UDTFOperator extends Operator<UDTFDesc> implements Serializable {
@Override
protected void closeOp(boolean abort) throws HiveException {
conf.getGenericUDTF().close();
+ super.closeOp(abort);
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/fd72ec4a/ql/src/java/org/apache/hadoop/hive/ql/hooks/ATSHook.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/hooks/ATSHook.java b/ql/src/java/org/apache/hadoop/hive/ql/hooks/ATSHook.java
index 5b8afb5..2dd03f5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/hooks/ATSHook.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/hooks/ATSHook.java
@@ -41,6 +41,7 @@ import org.apache.hadoop.hive.ql.exec.TaskFactory;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.exec.tez.TezTask;
import org.apache.hadoop.hive.ql.log.PerfLogger;
+import org.apache.hadoop.hive.ql.parse.ExplainConfiguration;
import org.apache.hadoop.hive.ql.plan.ExplainWork;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
@@ -137,98 +138,111 @@ public class ATSHook implements ExecuteWithHookContext {
@Override
public void run(final HookContext hookContext) throws Exception {
- final long currentTime = System.currentTimeMillis();
- final HiveConf conf = new HiveConf(hookContext.getConf());
- final QueryState queryState = hookContext.getQueryState();
- final String queryId = queryState.getQueryId();
-
- final Map<String, Long> durations = new HashMap<String, Long>();
- for (String key : hookContext.getPerfLogger().getEndTimes().keySet()) {
- durations.put(key, hookContext.getPerfLogger().getDuration(key));
- }
-
- try {
- setupAtsExecutor(conf);
+ final long currentTime = System.currentTimeMillis();
+ final HiveConf conf = new HiveConf(hookContext.getConf());
+ final QueryState queryState = hookContext.getQueryState();
+ final String queryId = queryState.getQueryId();
+
+ final Map<String, Long> durations = new HashMap<String, Long>();
+ for (String key : hookContext.getPerfLogger().getEndTimes().keySet()) {
+ durations.put(key, hookContext.getPerfLogger().getDuration(key));
+ }
- executor.submit(new Runnable() {
- @Override
- public void run() {
- try {
- QueryPlan plan = hookContext.getQueryPlan();
- if (plan == null) {
- return;
- }
- String queryId = plan.getQueryId();
- String opId = hookContext.getOperationId();
- long queryStartTime = plan.getQueryStartTime();
- String user = hookContext.getUgi().getShortUserName();
- String requestuser = hookContext.getUserName();
- if (hookContext.getUserName() == null ){
- requestuser = hookContext.getUgi().getUserName() ;
- }
- int numMrJobs = Utilities.getMRTasks(plan.getRootTasks()).size();
- int numTezJobs = Utilities.getTezTasks(plan.getRootTasks()).size();
- if (numMrJobs + numTezJobs <= 0) {
- return; // ignore client only queries
- }
-
- switch(hookContext.getHookType()) {
- case PRE_EXEC_HOOK:
- ExplainWork work = new ExplainWork(null,// resFile
- null,// pCtx
- plan.getRootTasks(),// RootTasks
- plan.getFetchTask(),// FetchTask
- null,// analyzer
- false,// extended
- true,// formatted
- false,// dependency
- false,// logical
- false,// authorize
- false,// userLevelExplain
- null// cboInfo
- );
- @SuppressWarnings("unchecked")
- ExplainTask explain = (ExplainTask) TaskFactory.get(work, conf);
- explain.initialize(queryState, plan, null, null);
- String query = plan.getQueryStr();
- JSONObject explainPlan = explain.getJSONPlan(null, work);
- String logID = conf.getLogIdVar(hookContext.getSessionId());
- List<String> tablesRead = getTablesFromEntitySet(hookContext.getInputs());
- List<String> tablesWritten = getTablesFromEntitySet(hookContext.getOutputs());
- String executionMode = getExecutionMode(plan).name();
- String hiveInstanceAddress = hookContext.getHiveInstanceAddress();
- if (hiveInstanceAddress == null) {
- hiveInstanceAddress = InetAddress.getLocalHost().getHostAddress();
- }
- String hiveInstanceType = hookContext.isHiveServerQuery() ? "HS2" : "CLI";
- ApplicationId llapId = determineLlapId(conf, plan);
- fireAndForget(
- createPreHookEvent(queryId, query, explainPlan, queryStartTime,
- user, requestuser, numMrJobs, numTezJobs, opId,
- hookContext.getIpAddress(), hiveInstanceAddress, hiveInstanceType,
- hookContext.getSessionId(), logID, hookContext.getThreadId(), executionMode,
- tablesRead, tablesWritten, conf));
- break;
- case POST_EXEC_HOOK:
- fireAndForget(createPostHookEvent(queryId, currentTime, user, requestuser, true, opId, durations));
- break;
- case ON_FAILURE_HOOK:
- fireAndForget(createPostHookEvent(queryId, currentTime, user, requestuser , false, opId, durations));
- break;
- default:
- //ignore
- break;
- }
- } catch (Exception e) {
- LOG.warn("Failed to submit plan to ATS for " + queryId, e);
- }
+ try {
+ setupAtsExecutor(conf);
+
+ executor.submit(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ QueryPlan plan = hookContext.getQueryPlan();
+ if (plan == null) {
+ return;
+ }
+ String queryId = plan.getQueryId();
+ String opId = hookContext.getOperationId();
+ long queryStartTime = plan.getQueryStartTime();
+ String user = hookContext.getUgi().getUserName();
+ String requestuser = hookContext.getUserName();
+ if (hookContext.getUserName() == null) {
+ requestuser = hookContext.getUgi().getUserName();
+ }
+ int numMrJobs = Utilities.getMRTasks(
+ plan.getRootTasks()).size();
+ int numTezJobs = Utilities.getTezTasks(
+ plan.getRootTasks()).size();
+ if (numMrJobs + numTezJobs <= 0) {
+ return; // ignore client only queries
+ }
+
+ switch (hookContext.getHookType()) {
+ case PRE_EXEC_HOOK:
+ ExplainConfiguration config = new ExplainConfiguration();
+ config.setFormatted(true);
+ ExplainWork work = new ExplainWork(null,// resFile
+ null,// pCtx
+ plan.getRootTasks(),// RootTasks
+ plan.getFetchTask(),// FetchTask
+ null,// analyzer
+ config, // explainConfig
+ null// cboInfo
+ );
+ @SuppressWarnings("unchecked")
+ ExplainTask explain = (ExplainTask) TaskFactory
+ .get(work, conf);
+ explain.initialize(queryState, plan, null, null);
+ String query = plan.getQueryStr();
+ JSONObject explainPlan = explain.getJSONPlan(null,
+ work);
+ String logID = conf.getLogIdVar(hookContext.getSessionId());
+ List<String> tablesRead = getTablesFromEntitySet(hookContext
+ .getInputs());
+ List<String> tablesWritten = getTablesFromEntitySet(hookContext
+ .getOutputs());
+ String executionMode = getExecutionMode(plan)
+ .name();
+ String hiveInstanceAddress = hookContext
+ .getHiveInstanceAddress();
+ if (hiveInstanceAddress == null) {
+ hiveInstanceAddress = InetAddress
+ .getLocalHost().getHostAddress();
+ }
+ String hiveInstanceType = hookContext
+ .isHiveServerQuery() ? "HS2" : "CLI";
+ ApplicationId llapId = determineLlapId(conf, plan);
+ fireAndForget(createPreHookEvent(queryId, query,
+ explainPlan, queryStartTime, user,
+ requestuser, numMrJobs, numTezJobs, opId,
+ hookContext.getIpAddress(),
+ hiveInstanceAddress, hiveInstanceType,
+ hookContext.getSessionId(),
+ logID,
+ hookContext.getThreadId(), executionMode,
+ tablesRead, tablesWritten, conf));
+ break;
+ case POST_EXEC_HOOK:
+ fireAndForget(createPostHookEvent(queryId,
+ currentTime, user, requestuser, true, opId,
+ durations));
+ break;
+ case ON_FAILURE_HOOK:
+ fireAndForget(createPostHookEvent(queryId,
+ currentTime, user, requestuser, false,
+ opId, durations));
+ break;
+ default:
+ // ignore
+ break;
+ }
+ } catch (Exception e) {
+ LOG.warn("Failed to submit to ATS for " + queryId, e);
+ }
+ }
+ });
+ } catch (Exception e) {
+ LOG.warn("Failed to submit to ATS for " + queryId, e);
}
- });
- } catch (Exception e) {
- LOG.warn("Failed to submit to ATS for " + queryId, e);
- }
}
-
protected List<String> getTablesFromEntitySet(Set<? extends Entity> entities) {
List<String> tableNames = new ArrayList<String>();
for (Entity entity : entities) {
http://git-wip-us.apache.org/repos/asf/hive/blob/fd72ec4a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java
index 8c44933..c6287e4 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java
@@ -219,7 +219,7 @@ public class Optimizer {
if(HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVEOPTIMIZEMETADATAQUERIES)) {
transformations.add(new StatsOptimizer());
}
- if (pctx.getContext().getExplain() && !isTezExecEngine && !isSparkExecEngine) {
+ if (pctx.getContext().isExplainSkipExecution() && !isTezExecEngine && !isSparkExecEngine) {
transformations.add(new AnnotateWithStatistics());
transformations.add(new AnnotateWithOpTraits());
}
@@ -231,6 +231,7 @@ public class Optimizer {
if (HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVEFETCHTASKAGGR)) {
transformations.add(new SimpleFetchAggregation());
}
+
}
/**
http://git-wip-us.apache.org/repos/asf/hive/blob/fd72ec4a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/AnnotateRunTimeStatsOptimizer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/AnnotateRunTimeStatsOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/AnnotateRunTimeStatsOptimizer.java
new file mode 100644
index 0000000..ee67443
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/AnnotateRunTimeStatsOptimizer.java
@@ -0,0 +1,174 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.optimizer.physical;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.Stack;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.ql.ErrorMsg;
+import org.apache.hadoop.hive.ql.exec.Operator;
+import org.apache.hadoop.hive.ql.exec.Task;
+import org.apache.hadoop.hive.ql.exec.mr.MapRedTask;
+import org.apache.hadoop.hive.ql.exec.spark.SparkTask;
+import org.apache.hadoop.hive.ql.exec.tez.TezTask;
+import org.apache.hadoop.hive.ql.lib.DefaultGraphWalker;
+import org.apache.hadoop.hive.ql.lib.Dispatcher;
+import org.apache.hadoop.hive.ql.lib.GraphWalker;
+import org.apache.hadoop.hive.ql.lib.Node;
+import org.apache.hadoop.hive.ql.lib.NodeProcessor;
+import org.apache.hadoop.hive.ql.lib.Rule;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.parse.ExplainConfiguration.AnalyzeState;
+import org.apache.hadoop.hive.ql.parse.ParseContext;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.plan.BaseWork;
+import org.apache.hadoop.hive.ql.plan.OperatorDesc;
+import org.apache.hadoop.hive.ql.plan.SparkWork;
+import org.apache.hadoop.hive.ql.plan.TezWork;
+import org.apache.hadoop.hive.ql.stats.StatsCollectionContext;
+import org.apache.hadoop.hive.ql.stats.StatsPublisher;
+import org.apache.hadoop.hive.ql.stats.fs.FSStatsPublisher;
+
+public class AnnotateRunTimeStatsOptimizer implements PhysicalPlanResolver {
+ private static final Logger LOG = LoggerFactory.getLogger(AnnotateRunTimeStatsOptimizer.class);
+
+ private class AnnotateRunTimeStatsDispatcher implements Dispatcher {
+
+ private final PhysicalContext physicalContext;
+
+ public AnnotateRunTimeStatsDispatcher(PhysicalContext context, Map<Rule, NodeProcessor> rules) {
+ super();
+ physicalContext = context;
+ }
+
+ @Override
+ public Object dispatch(Node nd, Stack<Node> stack, Object... nodeOutputs)
+ throws SemanticException {
+ Task<? extends Serializable> currTask = (Task<? extends Serializable>) nd;
+ Set<Operator<? extends OperatorDesc>> ops = new HashSet<>();
+
+ if (currTask instanceof MapRedTask) {
+ MapRedTask mr = (MapRedTask) currTask;
+ ops.addAll(mr.getWork().getAllOperators());
+ } else if (currTask instanceof TezTask) {
+ TezWork work = ((TezTask) currTask).getWork();
+ for (BaseWork w : work.getAllWork()) {
+ ops.addAll(w.getAllOperators());
+ }
+ } else if (currTask instanceof SparkTask) {
+ SparkWork sparkWork = (SparkWork) currTask.getWork();
+ for (BaseWork w : sparkWork.getAllWork()) {
+ ops.addAll(w.getAllOperators());
+ }
+ }
+
+ setOrAnnotateStats(ops, physicalContext.getParseContext());
+ return null;
+ }
+
+ }
+
+ public static void setOrAnnotateStats(Set<Operator<? extends OperatorDesc>> ops, ParseContext pctx)
+ throws SemanticException {
+ for (Operator<? extends OperatorDesc> op : ops) {
+ if (pctx.getContext().getExplainAnalyze() == AnalyzeState.RUNNING) {
+ setRuntimeStatsDir(op, pctx);
+ } else if (pctx.getContext().getExplainAnalyze() == AnalyzeState.ANALYZING) {
+ annotateRuntimeStats(op, pctx);
+ } else {
+ throw new SemanticException("Unexpected stats in AnnotateWithRunTimeStatistics.");
+ }
+ }
+ }
+
+ private static void setRuntimeStatsDir(Operator<? extends OperatorDesc> op, ParseContext pctx)
+ throws SemanticException {
+ try {
+ OperatorDesc conf = op.getConf();
+ if (conf != null) {
+ LOG.info("setRuntimeStatsDir for " + op.getOperatorId());
+ String path = new Path(pctx.getContext().getExplainConfig().getExplainRootPath(),
+ op.getOperatorId()).toString();
+ StatsPublisher statsPublisher = new FSStatsPublisher();
+ StatsCollectionContext runtimeStatsContext = new StatsCollectionContext(pctx.getConf());
+ runtimeStatsContext.setStatsTmpDir(path);
+ if (!statsPublisher.init(runtimeStatsContext)) {
+ LOG.error("StatsPublishing error: StatsPublisher is not initialized.");
+ throw new HiveException(ErrorMsg.STATSPUBLISHER_NOT_OBTAINED.getErrorCodedMsg());
+ }
+ conf.setRuntimeStatsTmpDir(path);
+ } else {
+ LOG.debug("skip setRuntimeStatsDir for " + op.getOperatorId()
+ + " because OperatorDesc is null");
+ }
+ } catch (HiveException e) {
+ throw new SemanticException(e);
+ }
+ }
+
+ private static void annotateRuntimeStats(Operator<? extends OperatorDesc> op, ParseContext pctx) {
+ Long runTimeNumRows = pctx.getContext().getExplainConfig().getOpIdToRuntimeNumRows()
+ .get(op.getOperatorId());
+ if (op.getConf() != null && op.getConf().getStatistics() != null && runTimeNumRows != null) {
+ LOG.info("annotateRuntimeStats for " + op.getOperatorId());
+ op.getConf().getStatistics().setRunTimeNumRows(runTimeNumRows);
+ } else {
+ LOG.debug("skip annotateRuntimeStats for " + op.getOperatorId());
+ }
+ }
+
+ @Override
+ public PhysicalContext resolve(PhysicalContext pctx) throws SemanticException {
+ Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>();
+ Dispatcher disp = new AnnotateRunTimeStatsDispatcher(pctx, opRules);
+ GraphWalker ogw = new DefaultGraphWalker(disp);
+ ArrayList<Node> topNodes = new ArrayList<Node>();
+ topNodes.addAll(pctx.getRootTasks());
+ ogw.startWalking(topNodes, null);
+ return pctx;
+ }
+
+ public void resolve(Set<Operator<?>> opSet, ParseContext pctx) throws SemanticException {
+ Set<Operator<?>> ops = getAllOperatorsForSimpleFetch(opSet);
+ setOrAnnotateStats(ops, pctx);
+ }
+
+ private Set<Operator<?>> getAllOperatorsForSimpleFetch(Set<Operator<?>> opSet) {
+ Set<Operator<?>> returnSet = new LinkedHashSet<Operator<?>>();
+ Stack<Operator<?>> opStack = new Stack<Operator<?>>();
+ // add all children
+ opStack.addAll(opSet);
+ while (!opStack.empty()) {
+ Operator<?> op = opStack.pop();
+ returnSet.add(op);
+ if (op.getChildOperators() != null) {
+ opStack.addAll(op.getChildOperators());
+ }
+ }
+ return returnSet;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/fd72ec4a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/PhysicalOptimizer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/PhysicalOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/PhysicalOptimizer.java
index 49706b1..9377563 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/PhysicalOptimizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/PhysicalOptimizer.java
@@ -88,12 +88,17 @@ public class PhysicalOptimizer {
// Vectorization should be the last optimization, because it doesn't modify the plan
// or any operators. It makes a very low level transformation to the expressions to
// run in the vectorized mode.
- if (hiveConf.getBoolVar(HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED)) {
+ if (hiveConf.getBoolVar(HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED)
+ && pctx.getContext().getExplainAnalyze() == null) {
resolvers.add(new Vectorizer());
}
if (!"none".equalsIgnoreCase(hiveConf.getVar(HiveConf.ConfVars.HIVESTAGEIDREARRANGE))) {
resolvers.add(new StageIDsRearranger());
}
+
+ if (pctx.getContext().getExplainAnalyze() != null) {
+ resolvers.add(new AnnotateRunTimeStatsOptimizer());
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/hive/blob/fd72ec4a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsAutoGatherContext.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsAutoGatherContext.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsAutoGatherContext.java
index c3e8bf5..80e62c1 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsAutoGatherContext.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsAutoGatherContext.java
@@ -67,10 +67,11 @@ public class ColumnStatsAutoGatherContext {
private boolean isInsertInto;
private Table tbl;
private Map<String, String> partSpec;
+ private Context origCtx;
public ColumnStatsAutoGatherContext(SemanticAnalyzer sa, HiveConf conf,
Operator<? extends OperatorDesc> op, Table tbl, Map<String, String> partSpec,
- boolean isInsertInto) throws SemanticException {
+ boolean isInsertInto, Context ctx) throws SemanticException {
super();
this.sa = sa;
this.conf = conf;
@@ -78,6 +79,7 @@ public class ColumnStatsAutoGatherContext {
this.tbl = tbl;
this.partSpec = partSpec;
this.isInsertInto = isInsertInto;
+ this.origCtx = ctx;
columns = tbl.getCols();
partitionColumns = tbl.getPartCols();
}
@@ -107,7 +109,7 @@ public class ColumnStatsAutoGatherContext {
// 2. Based on the statement, generate the selectOperator
Operator<?> selOp = null;
try {
- selOp = genSelOpForAnalyze(analyzeCommand);
+ selOp = genSelOpForAnalyze(analyzeCommand, origCtx);
} catch (IOException | ParseException e) {
throw new SemanticException(e);
}
@@ -126,10 +128,13 @@ public class ColumnStatsAutoGatherContext {
}
@SuppressWarnings("rawtypes")
- private Operator genSelOpForAnalyze(String analyzeCommand) throws IOException, ParseException, SemanticException{
+ private Operator genSelOpForAnalyze(String analyzeCommand, Context origCtx) throws IOException, ParseException, SemanticException{
//0. initialization
Context ctx = new Context(conf);
- ASTNode tree = ParseUtils.parse(analyzeCommand, ctx);
+ ctx.setExplainConfig(origCtx.getExplainConfig());
+ ParseDriver pd = new ParseDriver();
+ ASTNode tree = pd.parse(analyzeCommand, ctx);
+ tree = ParseUtils.findRootNonNullToken(tree);
//1. get the ColumnStatsSemanticAnalyzer
BaseSemanticAnalyzer baseSem = SemanticAnalyzerFactory.get(new QueryState(conf), tree);
http://git-wip-us.apache.org/repos/asf/hive/blob/fd72ec4a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java
index 242a880..93b8183 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java
@@ -405,6 +405,7 @@ public class ColumnStatsSemanticAnalyzer extends SemanticAnalyzer {
analyzeRewrite.setColType(colType);
qbp.setAnalyzeRewrite(analyzeRewrite);
initCtx(ctx);
+ ctx.setExplainConfig(origCtx.getExplainConfig());
LOG.info("Invoking analyze on rewritten query");
analyzeInternal(rewrittenTree);
} else {
http://git-wip-us.apache.org/repos/asf/hive/blob/fd72ec4a/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainConfiguration.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainConfiguration.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainConfiguration.java
new file mode 100644
index 0000000..4a8ff15
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainConfiguration.java
@@ -0,0 +1,117 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.parse;
+
+import java.util.Map;
+
+import org.apache.hadoop.fs.Path;
+
+/**
+ * ExplainConfiguration
+ */
+
+public class ExplainConfiguration {
+ private boolean extended = false;
+ private boolean formatted = false;
+ private boolean dependency = false;
+ private boolean logical = false;
+ private boolean authorize = false;
+ private boolean userLevelExplain = false;
+ private Path explainRootPath;
+ private Map<String, Long> opIdToRuntimeNumRows;
+
+ public enum AnalyzeState {
+ RUNNING, ANALYZING
+ };
+
+ private AnalyzeState analyze = null;
+
+ public boolean isExtended() {
+ return extended;
+ }
+
+ public void setExtended(boolean extended) {
+ this.extended = extended;
+ }
+
+ public boolean isFormatted() {
+ return formatted;
+ }
+
+ public void setFormatted(boolean formatted) {
+ this.formatted = formatted;
+ }
+
+ public boolean isDependency() {
+ return dependency;
+ }
+
+ public void setDependency(boolean dependency) {
+ this.dependency = dependency;
+ }
+
+ public boolean isLogical() {
+ return logical;
+ }
+
+ public void setLogical(boolean logical) {
+ this.logical = logical;
+ }
+
+ public boolean isAuthorize() {
+ return authorize;
+ }
+
+ public void setAuthorize(boolean authorize) {
+ this.authorize = authorize;
+ }
+
+ public AnalyzeState getAnalyze() {
+ return analyze;
+ }
+
+ public void setAnalyze(AnalyzeState analyze) {
+ this.analyze = analyze;
+ }
+
+ public boolean isUserLevelExplain() {
+ return userLevelExplain;
+ }
+
+ public void setUserLevelExplain(boolean userLevelExplain) {
+ this.userLevelExplain = userLevelExplain;
+ }
+
+ public Path getExplainRootPath() {
+ return explainRootPath;
+ }
+
+ public void setExplainRootPath(Path explainRootPath) {
+ this.explainRootPath = explainRootPath;
+ }
+
+ public Map<String, Long> getOpIdToRuntimeNumRows() {
+ return opIdToRuntimeNumRows;
+ }
+
+ public void setOpIdToRuntimeNumRows(Map<String, Long> opIdToRuntimeNumRows) {
+ this.opIdToRuntimeNumRows = opIdToRuntimeNumRows;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/fd72ec4a/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSQRewriteSemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSQRewriteSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSQRewriteSemanticAnalyzer.java
index 8d7fd92..7e24364 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSQRewriteSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSQRewriteSemanticAnalyzer.java
@@ -36,8 +36,7 @@ public class ExplainSQRewriteSemanticAnalyzer extends BaseSemanticAnalyzer {
@Override
public void analyzeInternal(ASTNode ast) throws SemanticException {
-
- ctx.setExplain(true);
+ ctx.setExplainConfig(new ExplainConfiguration());
// Create a semantic analyzer for the query
ASTNode input = (ASTNode) ast.getChild(0);
http://git-wip-us.apache.org/repos/asf/hive/blob/fd72ec4a/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java
index 75753b0..e0a1d3c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java
@@ -18,18 +18,40 @@
package org.apache.hadoop.hive.ql.parse;
+import java.io.IOException;
import java.io.Serializable;
+import java.util.ArrayList;
import java.util.Collections;
+import java.util.HashMap;
import java.util.List;
-
+import java.util.Map;
+
+import org.antlr.runtime.TokenRewriteStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.common.FileUtils;
+import org.apache.hadoop.hive.common.StatsSetupConst;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.ql.CommandNeedRetryException;
+import org.apache.hadoop.hive.ql.Context;
+import org.apache.hadoop.hive.ql.Driver;
import org.apache.hadoop.hive.ql.QueryState;
import org.apache.hadoop.hive.ql.exec.ExplainTask;
import org.apache.hadoop.hive.ql.exec.FetchTask;
+import org.apache.hadoop.hive.ql.exec.StatsTask;
import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.exec.TaskFactory;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.parse.ExplainConfiguration.AnalyzeState;
import org.apache.hadoop.hive.ql.plan.ExplainWork;
+import org.apache.hadoop.hive.ql.processors.CommandProcessor;
+import org.apache.hadoop.hive.ql.processors.CommandProcessorFactory;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hadoop.hive.ql.stats.StatsAggregator;
+import org.apache.hadoop.hive.ql.stats.StatsCollectionContext;
+import org.apache.hadoop.hive.ql.stats.fs.FSStatsAggregator;
/**
* ExplainSemanticAnalyzer.
@@ -37,40 +59,67 @@ import org.apache.hadoop.hive.ql.plan.ExplainWork;
*/
public class ExplainSemanticAnalyzer extends BaseSemanticAnalyzer {
List<FieldSchema> fieldList;
+ ExplainConfiguration config;
public ExplainSemanticAnalyzer(QueryState queryState) throws SemanticException {
super(queryState);
+ config = new ExplainConfiguration();
}
@SuppressWarnings("unchecked")
@Override
public void analyzeInternal(ASTNode ast) throws SemanticException {
-
- boolean extended = false;
- boolean formatted = false;
- boolean dependency = false;
- boolean logical = false;
- boolean authorize = false;
for (int i = 1; i < ast.getChildCount(); i++) {
int explainOptions = ast.getChild(i).getType();
if (explainOptions == HiveParser.KW_FORMATTED) {
- formatted = true;
+ config.setFormatted(true);
} else if (explainOptions == HiveParser.KW_EXTENDED) {
- extended = true;
+ config.setExtended(true);
} else if (explainOptions == HiveParser.KW_DEPENDENCY) {
- dependency = true;
+ config.setDependency(true);
} else if (explainOptions == HiveParser.KW_LOGICAL) {
- logical = true;
+ config.setLogical(true);
} else if (explainOptions == HiveParser.KW_AUTHORIZATION) {
- authorize = true;
+ config.setAuthorize(true);
+ } else if (explainOptions == HiveParser.KW_ANALYZE) {
+ config.setAnalyze(AnalyzeState.RUNNING);
+ config.setExplainRootPath(ctx.getMRTmpPath());
}
}
- ctx.setExplain(true);
- ctx.setExplainLogical(logical);
+ ctx.setExplainConfig(config);
- // Create a semantic analyzer for the query
ASTNode input = (ASTNode) ast.getChild(0);
+ // explain analyze is composed of two steps
+ // step 1 (ANALYZE_STATE.RUNNING), run the query and collect the runtime #rows
+ // step 2 (ANALYZE_STATE.ANALYZING), explain the query and provide the runtime #rows collected.
+ if (config.getAnalyze() == AnalyzeState.RUNNING) {
+ String query = ctx.getTokenRewriteStream().toString(input.getTokenStartIndex(),
+ input.getTokenStopIndex());
+ LOG.info("Explain analyze (running phase) for query " + query);
+ Context runCtx = null;
+ try {
+ runCtx = new Context(conf);
+ // runCtx and ctx share the configuration
+ runCtx.setExplainConfig(config);
+ Driver driver = new Driver(conf, runCtx);
+ driver.run(query);
+ // Note that we need to call getResults for simple fetch optimization.
+ // However, we need to skip all the results.
+ while (driver.getResults(new ArrayList<String>())) {
+ }
+ config.setOpIdToRuntimeNumRows(aggregateStats(config.getExplainRootPath()));
+ } catch (IOException e1) {
+ throw new SemanticException(e1);
+ } catch (CommandNeedRetryException e) {
+ throw new SemanticException(e);
+ }
+ ctx.resetOpContext();
+ ctx.resetStream();
+ TaskFactory.resetId();
+ LOG.info("Explain analyze (analyzing phase) for query " + query);
+ config.setAnalyze(AnalyzeState.ANALYZING);
+ }
BaseSemanticAnalyzer sem = SemanticAnalyzerFactory.get(queryState, input);
sem.analyze(input, ctx);
sem.validate();
@@ -92,24 +141,20 @@ public class ExplainSemanticAnalyzer extends BaseSemanticAnalyzer {
pCtx = ((SemanticAnalyzer)sem).getParseContext();
}
- boolean userLevelExplain = !extended
- && !formatted
- && !dependency
- && !logical
- && !authorize
+ config.setUserLevelExplain(!config.isExtended()
+ && !config.isFormatted()
+ && !config.isDependency()
+ && !config.isLogical()
+ && !config.isAuthorize()
&& (HiveConf.getBoolVar(ctx.getConf(), HiveConf.ConfVars.HIVE_EXPLAIN_USER) && HiveConf
- .getVar(conf, HiveConf.ConfVars.HIVE_EXECUTION_ENGINE).equals("tez"));
+ .getVar(conf, HiveConf.ConfVars.HIVE_EXECUTION_ENGINE).equals("tez")));
+
ExplainWork work = new ExplainWork(ctx.getResFile(),
pCtx,
tasks,
fetchTask,
sem,
- extended,
- formatted,
- dependency,
- logical,
- authorize,
- userLevelExplain,
+ config,
ctx.getCboInfo());
work.setAppendTaskType(
@@ -121,6 +166,43 @@ public class ExplainSemanticAnalyzer extends BaseSemanticAnalyzer {
rootTasks.add(explTask);
}
+ private Map<String, Long> aggregateStats(Path localTmpPath) {
+ Map<String, Long> opIdToRuntimeNumRows = new HashMap<String, Long>();
+ // localTmpPath is the root of all the stats.
+ // Under it, there will be SEL_1/statsfiles, SEL_2/statsfiles etc where SEL_1 and SEL_2 are the op ids.
+ FileSystem fs;
+ FileStatus[] statuses = null;
+ try {
+ fs = localTmpPath.getFileSystem(conf);
+ statuses = fs.listStatus(localTmpPath, FileUtils.HIDDEN_FILES_PATH_FILTER);
+ // statuses can be null if it is DDL, etc
+ } catch (IOException e) {
+ LOG.warn(e.toString());
+ }
+ if (statuses != null) {
+ for (FileStatus status : statuses) {
+ if (status.isDir()) {
+ StatsCollectionContext scc = new StatsCollectionContext(conf);
+ String[] names = status.getPath().toString().split(Path.SEPARATOR);
+ String opId = names[names.length - 1];
+ scc.setStatsTmpDir(status.getPath().toString());
+ StatsAggregator statsAggregator = new FSStatsAggregator();
+ if (!statsAggregator.connect(scc)) {
+ // -1 means that there is no stats
+ opIdToRuntimeNumRows.put(opId, -1L);
+ } else {
+ String value = statsAggregator.aggregateStats("", StatsSetupConst.RUN_TIME_ROW_COUNT);
+ opIdToRuntimeNumRows.put(opId, Long.parseLong(value));
+ }
+ if (statsAggregator != null) {
+ statsAggregator.closeConnection(scc);
+ }
+ }
+ }
+ }
+ return opIdToRuntimeNumRows;
+ }
+
@Override
public List<FieldSchema> getResultSchema() {
return fieldList;
@@ -133,4 +215,5 @@ public class ExplainSemanticAnalyzer extends BaseSemanticAnalyzer {
Task task = rootTasks.get(0);
return task instanceof ExplainTask && ((ExplainTask)task).getWork().isAuthorize();
}
+
}
http://git-wip-us.apache.org/repos/asf/hive/blob/fd72ec4a/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java
index bb9b53d..c88d537 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java
@@ -194,7 +194,7 @@ public class GenTezUtils {
}
// removes any union operator and clones the plan
- public static void removeUnionOperators(GenTezProcContext context, BaseWork work)
+ public static void removeUnionOperators(GenTezProcContext context, BaseWork work, int indexForTezUnion)
throws SemanticException {
List<Operator<?>> roots = new ArrayList<Operator<?>>();
@@ -205,7 +205,7 @@ public class GenTezUtils {
roots.addAll(context.eventOperatorSet);
// need to clone the plan.
- List<Operator<?>> newRoots = SerializationUtilities.cloneOperatorTree(roots);
+ List<Operator<?>> newRoots = SerializationUtilities.cloneOperatorTree(roots, indexForTezUnion);
// we're cloning the operator plan but we're retaining the original work. That means
// that root operators have to be replaced with the cloned ops. The replacement map
@@ -294,8 +294,7 @@ public class GenTezUtils {
linked = context.linkedFileSinks.get(path);
linked.add(desc);
- desc.setIndexInTezUnion(linked.size());
- desc.setDirName(new Path(path, "" + desc.getIndexInTezUnion()));
+ desc.setDirName(new Path(path, "" + linked.size()));
desc.setLinkedFileSink(true);
desc.setParentDir(path);
desc.setLinkedFileSinkDesc(linked);
http://git-wip-us.apache.org/repos/asf/hive/blob/fd72ec4a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
index c85efec..00b872d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
@@ -735,7 +735,7 @@ explainStatement
explainOption
@init { msgs.push("explain option"); }
@after { msgs.pop(); }
- : KW_EXTENDED|KW_FORMATTED|KW_DEPENDENCY|KW_LOGICAL|KW_AUTHORIZATION
+ : KW_EXTENDED|KW_FORMATTED|KW_DEPENDENCY|KW_LOGICAL|KW_AUTHORIZATION|KW_ANALYZE
;
execStatement
http://git-wip-us.apache.org/repos/asf/hive/blob/fd72ec4a/ql/src/java/org/apache/hadoop/hive/ql/parse/MapReduceCompiler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/MapReduceCompiler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/MapReduceCompiler.java
index 5b08ed2..d7a56e5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/MapReduceCompiler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/MapReduceCompiler.java
@@ -178,7 +178,7 @@ public class MapReduceCompiler extends TaskCompiler {
throws SemanticException {
// bypass for explain queries for now
- if (ctx.getExplain()) {
+ if (ctx.isExplainSkipExecution()) {
return;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/fd72ec4a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index 252a916..6a66691 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -138,6 +138,7 @@ import org.apache.hadoop.hive.ql.optimizer.lineage.Generator;
import org.apache.hadoop.hive.ql.optimizer.unionproc.UnionProcContext;
import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.TableSpec.SpecType;
import org.apache.hadoop.hive.ql.parse.CalcitePlanner.ASTSearcher;
+import org.apache.hadoop.hive.ql.parse.ExplainConfiguration.AnalyzeState;
import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.OrderExpression;
import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.OrderSpec;
import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.PTFInputSpec;
@@ -7246,7 +7247,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
LOG.info("Generate an operator pipleline to autogather column stats for table " + tableName
+ " in query " + ctx.getCmd());
ColumnStatsAutoGatherContext columnStatsAutoGatherContext = null;
- columnStatsAutoGatherContext = new ColumnStatsAutoGatherContext(this, conf, curr, table, partSpec, isInsertInto);
+ columnStatsAutoGatherContext = new ColumnStatsAutoGatherContext(this, conf, curr, table, partSpec, isInsertInto, ctx);
columnStatsAutoGatherContext.insertAnalyzePipeline();
columnStatsAutoGatherContexts.add(columnStatsAutoGatherContext);
}
@@ -11076,10 +11077,14 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
// 5. Take care of view creation
if (createVwDesc != null) {
- if (!ctx.isCboSucceeded()) {
- saveViewDefinition();
+ if (ctx.getExplainAnalyze() == AnalyzeState.RUNNING) {
+ return;
}
+ if (!ctx.isCboSucceeded()) {
+ saveViewDefinition();
+ }
+
// validate the create view statement at this point, the createVwDesc gets
// all the information for semanticcheck
validateCreateView(createVwDesc);
@@ -11168,7 +11173,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
}
// 11. if desired check we're not going over partition scan limits
- if (!ctx.getExplain()) {
+ if (!ctx.isExplainSkipExecution()) {
enforceScanLimits(pCtx, origFetchTask);
}
@@ -11964,7 +11969,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
case CTAS: // create table as select
if (isTemporary) {
- if (!ctx.getExplain() && !isMaterialization) {
+ if (!ctx.isExplainSkipExecution() && !isMaterialization) {
String dbName = qualifiedTabName[0];
String tblName = qualifiedTabName[1];
SessionState ss = SessionState.get();
@@ -11983,7 +11988,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
// dumpTable is only used to check the conflict for non-temporary tables
try {
Table dumpTable = db.newTable(dbDotTab);
- if (null != db.getTable(dumpTable.getDbName(), dumpTable.getTableName(), false) && !ctx.getExplain()) {
+ if (null != db.getTable(dumpTable.getDbName(), dumpTable.getTableName(), false) && !ctx.isExplainSkipExecution()) {
throw new SemanticException(ErrorMsg.TABLE_ALREADY_EXISTS.getMsg(dbDotTab));
}
} catch (HiveException e) {
[12/50] [abbrv] hive git commit: HIVE-14367 : Estimated size for
constant nulls is 0 (Ashutosh Chauhan via Jesus Camacho Rodriguez)
Posted by om...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/tez/dynamic_partition_pruning.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/dynamic_partition_pruning.q.out b/ql/src/test/results/clientpositive/tez/dynamic_partition_pruning.q.out
index 71709d4..ff424a3 100644
--- a/ql/src/test/results/clientpositive/tez/dynamic_partition_pruning.q.out
+++ b/ql/src/test/results/clientpositive/tez/dynamic_partition_pruning.q.out
@@ -3075,10 +3075,10 @@ STAGE PLANS:
aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=1)
mode: hash
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary)
Reducer 3
Reduce Operator Tree:
@@ -3118,10 +3118,10 @@ STAGE PLANS:
aggregations: min(VALUE._col0), max(VALUE._col1), bloom_filter(VALUE._col2, expectedEntries=1)
mode: final
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary)
Stage: Stage-0
@@ -3193,10 +3193,10 @@ STAGE PLANS:
aggregations: max(ds)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string)
Map 8
Map Operator Tree:
@@ -3211,10 +3211,10 @@ STAGE PLANS:
aggregations: min(ds)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string)
Reducer 2
Reduce Operator Tree:
@@ -3254,46 +3254,46 @@ STAGE PLANS:
aggregations: max(VALUE._col0)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: _col0 is not null (type: boolean)
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Reducer 7
Reduce Operator Tree:
Group By Operator
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Dynamic Partitioning Event Operator
Target column: ds (string)
Target Input: srcpart
Partition key expr: ds
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Target Vertex: Map 1
Reducer 9
Reduce Operator Tree:
@@ -3301,20 +3301,20 @@ STAGE PLANS:
aggregations: min(VALUE._col0)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: _col0 is not null (type: boolean)
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Union 6
Vertex: Union 6
@@ -3389,10 +3389,10 @@ STAGE PLANS:
aggregations: max(ds)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string)
Map 8
Map Operator Tree:
@@ -3407,10 +3407,10 @@ STAGE PLANS:
aggregations: min(ds)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string)
Reducer 2
Reduce Operator Tree:
@@ -3452,46 +3452,46 @@ STAGE PLANS:
aggregations: max(VALUE._col0)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: _col0 is not null (type: boolean)
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Reducer 7
Reduce Operator Tree:
Group By Operator
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Dynamic Partitioning Event Operator
Target column: ds (string)
Target Input: srcpart
Partition key expr: ds
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Target Vertex: Map 1
Reducer 9
Reduce Operator Tree:
@@ -3499,20 +3499,20 @@ STAGE PLANS:
aggregations: min(VALUE._col0)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: _col0 is not null (type: boolean)
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Union 6
Vertex: Union 6
@@ -3589,10 +3589,10 @@ STAGE PLANS:
aggregations: min(ds)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string)
Map 5
Map Operator Tree:
@@ -3623,10 +3623,10 @@ STAGE PLANS:
aggregations: max(ds)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string)
Reducer 11
Reduce Operator Tree:
@@ -3634,49 +3634,49 @@ STAGE PLANS:
aggregations: min(VALUE._col0)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: _col0 is not null (type: boolean)
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Dynamic Partitioning Event Operator
Target column: ds (string)
Target Input: srcpart
Partition key expr: ds
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Target Vertex: Map 1
Select Operator
expressions: _col0 (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Dynamic Partitioning Event Operator
Target column: ds (string)
Target Input: srcpart
Partition key expr: ds
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Target Vertex: Map 5
Reducer 2
Reduce Operator Tree:
@@ -3725,49 +3725,49 @@ STAGE PLANS:
aggregations: max(VALUE._col0)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: _col0 is not null (type: boolean)
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Dynamic Partitioning Event Operator
Target column: ds (string)
Target Input: srcpart
Partition key expr: ds
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Target Vertex: Map 1
Select Operator
expressions: _col0 (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Dynamic Partitioning Event Operator
Target column: ds (string)
Target Input: srcpart
Partition key expr: ds
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Target Vertex: Map 5
Union 3
Vertex: Union 3
@@ -5446,10 +5446,10 @@ STAGE PLANS:
aggregations: max(ds)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string)
Map 7
Map Operator Tree:
@@ -5464,10 +5464,10 @@ STAGE PLANS:
aggregations: min(ds)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string)
Reducer 3
Reduce Operator Tree:
@@ -5475,27 +5475,27 @@ STAGE PLANS:
aggregations: max(VALUE._col0)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: _col0 is not null (type: boolean)
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Reducer 5
Reduce Operator Tree:
Group By Operator
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Inner Join 0 to 1
@@ -5537,20 +5537,20 @@ STAGE PLANS:
aggregations: min(VALUE._col0)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: _col0 is not null (type: boolean)
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Union 4
Vertex: Union 4
[43/50] [abbrv] hive git commit: HIVE-15884: Optimize not between for
vectorization (Pengcheng Xiong, reviewed by Ashutosh Chauhan)
Posted by om...@apache.org.
HIVE-15884: Optimize not between for vectorization (Pengcheng Xiong, reviewed by Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/5f533bce
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/5f533bce
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/5f533bce
Branch: refs/heads/branch-2.2
Commit: 5f533bcebfdfe0d79a2b41de7fc72bd8c1b366c4
Parents: fb1c9fd
Author: Pengcheng Xiong <px...@apache.org>
Authored: Thu Mar 2 11:16:34 2017 -0800
Committer: Owen O'Malley <om...@apache.org>
Committed: Tue Mar 28 15:27:56 2017 -0700
----------------------------------------------------------------------
.../ql/exec/vector/VectorizationContext.java | 24 +++++++++++++++++++-
.../clientpositive/llap/vector_between_in.q.out | 6 ++---
.../spark/vector_between_in.q.out | 12 +++++-----
.../clientpositive/tez/vector_between_in.q.out | 6 ++---
.../clientpositive/vector_between_in.q.out | 6 ++---
5 files changed, 38 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/5f533bce/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
index f81a0fb..d8387bf 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
@@ -102,7 +102,6 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDynamicValueDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
import org.apache.hadoop.hive.ql.plan.GroupByDesc;
-import org.apache.hadoop.hive.ql.udf.SettableUDF;
import org.apache.hadoop.hive.ql.udf.*;
import org.apache.hadoop.hive.ql.udf.generic.*;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.Mode;
@@ -582,6 +581,29 @@ public class VectorizationContext {
ve = getColumnVectorExpression((ExprNodeColumnDesc) exprDesc, mode);
} else if (exprDesc instanceof ExprNodeGenericFuncDesc) {
ExprNodeGenericFuncDesc expr = (ExprNodeGenericFuncDesc) exprDesc;
+ // push not through between...
+ if ("not".equals(expr.getFuncText())) {
+ if (expr.getChildren() != null && expr.getChildren().size() == 1) {
+ ExprNodeDesc child = expr.getChildren().get(0);
+ if (child instanceof ExprNodeGenericFuncDesc) {
+ ExprNodeGenericFuncDesc childExpr = (ExprNodeGenericFuncDesc) child;
+ if ("between".equals(childExpr.getFuncText())) {
+ ExprNodeConstantDesc flag = (ExprNodeConstantDesc) childExpr.getChildren().get(0);
+ List<ExprNodeDesc> newChildren = new ArrayList<>();
+ if (Boolean.TRUE.equals(flag.getValue())) {
+ newChildren.add(new ExprNodeConstantDesc(Boolean.FALSE));
+ } else {
+ newChildren.add(new ExprNodeConstantDesc(Boolean.TRUE));
+ }
+ newChildren
+ .addAll(childExpr.getChildren().subList(1, childExpr.getChildren().size()));
+ expr.setTypeInfo(childExpr.getTypeInfo());
+ expr.setGenericUDF(childExpr.getGenericUDF());
+ expr.setChildren(newChildren);
+ }
+ }
+ }
+ }
if (isCustomUDF(expr)) {
ve = getCustomUDFExpression(expr, mode);
} else {
http://git-wip-us.apache.org/repos/asf/hive/blob/5f533bce/ql/src/test/results/clientpositive/llap/vector_between_in.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vector_between_in.q.out b/ql/src/test/results/clientpositive/llap/vector_between_in.q.out
index 88d97f5..7f9067d 100644
--- a/ql/src/test/results/clientpositive/llap/vector_between_in.q.out
+++ b/ql/src/test/results/clientpositive/llap/vector_between_in.q.out
@@ -317,7 +317,7 @@ STAGE PLANS:
alias: decimal_date_test
Statistics: Num rows: 12288 Data size: 2467616 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (not cdate BETWEEN 1968-05-01 AND 1971-09-01) (type: boolean)
+ predicate: cdate NOT BETWEEN 1968-05-01 AND 1971-09-01 (type: boolean)
Statistics: Num rows: 10923 Data size: 2193503 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cdate (type: date)
@@ -427,7 +427,7 @@ STAGE PLANS:
alias: decimal_date_test
Statistics: Num rows: 12288 Data size: 2467616 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (not cdecimal1 BETWEEN -2000 AND 4390.1351351351) (type: boolean)
+ predicate: cdecimal1 NOT BETWEEN -2000 AND 4390.1351351351 (type: boolean)
Statistics: Num rows: 10923 Data size: 2193503 Basic stats: COMPLETE Column stats: NONE
Select Operator
Statistics: Num rows: 10923 Data size: 2193503 Basic stats: COMPLETE Column stats: NONE
@@ -917,7 +917,7 @@ STAGE PLANS:
alias: decimal_date_test
Statistics: Num rows: 12288 Data size: 2467616 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: (not cdecimal1 BETWEEN -2000 AND 4390.1351351351) (type: boolean)
+ expressions: cdecimal1 NOT BETWEEN -2000 AND 4390.1351351351 (type: boolean)
outputColumnNames: _col0
Statistics: Num rows: 12288 Data size: 2467616 Basic stats: COMPLETE Column stats: NONE
Group By Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/5f533bce/ql/src/test/results/clientpositive/spark/vector_between_in.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/vector_between_in.q.out b/ql/src/test/results/clientpositive/spark/vector_between_in.q.out
index 689707f..efbca8c 100644
--- a/ql/src/test/results/clientpositive/spark/vector_between_in.q.out
+++ b/ql/src/test/results/clientpositive/spark/vector_between_in.q.out
@@ -311,8 +311,8 @@ STAGE PLANS:
Filter Vectorization:
className: VectorFilterOperator
native: true
- predicateExpression: SelectColumnIsFalse(col 4)(children: VectorUDFAdaptor(cdate BETWEEN 1968-05-01 AND 1971-09-01) -> 4:boolean) -> boolean
- predicate: (not cdate BETWEEN 1968-05-01 AND 1971-09-01) (type: boolean)
+ predicateExpression: FilterLongColumnNotBetween(col 3, left -610, right 608) -> boolean
+ predicate: cdate NOT BETWEEN 1968-05-01 AND 1971-09-01 (type: boolean)
Statistics: Num rows: 10923 Data size: 2193503 Basic stats: COMPLETE Column stats: NONE
>>>>>>> eb1da30... HIVE-15388: HiveParser spends lots of time in parsing queries with lots of '(' (Pengcheng Xiong, reviewed by Ashutosh Chauhan, Gunther Hagleitner)
Select Operator
@@ -332,7 +332,7 @@ STAGE PLANS:
groupByVectorOutput: true
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
allNative: false
- usesVectorUDFAdaptor: true
+ usesVectorUDFAdaptor: false
vectorized: true
>>>>>>> eb1da30... HIVE-15388: HiveParser spends lots of time in parsing queries with lots of '(' (Pengcheng Xiong, reviewed by Ashutosh Chauhan, Gunther Hagleitner)
Reducer 2
@@ -437,8 +437,8 @@ STAGE PLANS:
Filter Vectorization:
className: VectorFilterOperator
native: true
- predicateExpression: SelectColumnIsFalse(col 4)(children: VectorUDFAdaptor(cdecimal1 BETWEEN -2000 AND 4390.1351351351) -> 4:boolean) -> boolean
- predicate: (not cdecimal1 BETWEEN -2000 AND 4390.1351351351) (type: boolean)
+ predicateExpression: FilterDecimalColumnNotBetween(col 1, left -2000, right 4390.1351351351) -> boolean
+ predicate: cdecimal1 NOT BETWEEN -2000 AND 4390.1351351351 (type: boolean)
Statistics: Num rows: 10923 Data size: 2193503 Basic stats: COMPLETE Column stats: NONE
>>>>>>> eb1da30... HIVE-15388: HiveParser spends lots of time in parsing queries with lots of '(' (Pengcheng Xiong, reviewed by Ashutosh Chauhan, Gunther Hagleitner)
Select Operator
@@ -461,7 +461,7 @@ STAGE PLANS:
groupByVectorOutput: true
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
allNative: false
- usesVectorUDFAdaptor: true
+ usesVectorUDFAdaptor: false
vectorized: true
>>>>>>> eb1da30... HIVE-15388: HiveParser spends lots of time in parsing queries with lots of '(' (Pengcheng Xiong, reviewed by Ashutosh Chauhan, Gunther Hagleitner)
Reducer 2
http://git-wip-us.apache.org/repos/asf/hive/blob/5f533bce/ql/src/test/results/clientpositive/tez/vector_between_in.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/vector_between_in.q.out b/ql/src/test/results/clientpositive/tez/vector_between_in.q.out
index 8903337..57e6d1a 100644
--- a/ql/src/test/results/clientpositive/tez/vector_between_in.q.out
+++ b/ql/src/test/results/clientpositive/tez/vector_between_in.q.out
@@ -310,7 +310,7 @@ STAGE PLANS:
alias: decimal_date_test
Statistics: Num rows: 12288 Data size: 2467616 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (not cdate BETWEEN 1968-05-01 AND 1971-09-01) (type: boolean)
+ predicate: cdate NOT BETWEEN 1968-05-01 AND 1971-09-01 (type: boolean)
Statistics: Num rows: 10923 Data size: 2193503 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cdate (type: date)
@@ -418,7 +418,7 @@ STAGE PLANS:
alias: decimal_date_test
Statistics: Num rows: 12288 Data size: 2467616 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (not cdecimal1 BETWEEN -2000 AND 4390.1351351351) (type: boolean)
+ predicate: cdecimal1 NOT BETWEEN -2000 AND 4390.1351351351 (type: boolean)
Statistics: Num rows: 10923 Data size: 2193503 Basic stats: COMPLETE Column stats: NONE
Select Operator
Statistics: Num rows: 10923 Data size: 2193503 Basic stats: COMPLETE Column stats: NONE
@@ -903,7 +903,7 @@ STAGE PLANS:
alias: decimal_date_test
Statistics: Num rows: 12288 Data size: 2467616 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: (not cdecimal1 BETWEEN -2000 AND 4390.1351351351) (type: boolean)
+ expressions: cdecimal1 NOT BETWEEN -2000 AND 4390.1351351351 (type: boolean)
outputColumnNames: _col0
Statistics: Num rows: 12288 Data size: 2467616 Basic stats: COMPLETE Column stats: NONE
Group By Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/5f533bce/ql/src/test/results/clientpositive/vector_between_in.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vector_between_in.q.out b/ql/src/test/results/clientpositive/vector_between_in.q.out
index 270de4b..cad29dd 100644
--- a/ql/src/test/results/clientpositive/vector_between_in.q.out
+++ b/ql/src/test/results/clientpositive/vector_between_in.q.out
@@ -264,7 +264,7 @@ STAGE PLANS:
alias: decimal_date_test
Statistics: Num rows: 12288 Data size: 2467616 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (not cdate BETWEEN 1968-05-01 AND 1971-09-01) (type: boolean)
+ predicate: cdate NOT BETWEEN 1968-05-01 AND 1971-09-01 (type: boolean)
Statistics: Num rows: 10923 Data size: 2193503 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cdate (type: date)
@@ -356,7 +356,7 @@ STAGE PLANS:
alias: decimal_date_test
Statistics: Num rows: 12288 Data size: 2467616 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (not cdecimal1 BETWEEN -2000 AND 4390.1351351351) (type: boolean)
+ predicate: cdecimal1 NOT BETWEEN -2000 AND 4390.1351351351 (type: boolean)
Statistics: Num rows: 10923 Data size: 2193503 Basic stats: COMPLETE Column stats: NONE
Select Operator
Statistics: Num rows: 10923 Data size: 2193503 Basic stats: COMPLETE Column stats: NONE
@@ -809,7 +809,7 @@ STAGE PLANS:
alias: decimal_date_test
Statistics: Num rows: 12288 Data size: 2467616 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: (not cdecimal1 BETWEEN -2000 AND 4390.1351351351) (type: boolean)
+ expressions: cdecimal1 NOT BETWEEN -2000 AND 4390.1351351351 (type: boolean)
outputColumnNames: _col0
Statistics: Num rows: 12288 Data size: 2467616 Basic stats: COMPLETE Column stats: NONE
Group By Operator
[13/50] [abbrv] hive git commit: HIVE-14367 : Estimated size for
constant nulls is 0 (Ashutosh Chauhan via Jesus Camacho Rodriguez)
Posted by om...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/remove_exprs_stats.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/remove_exprs_stats.q.out b/ql/src/test/results/clientpositive/remove_exprs_stats.q.out
index b1ba942..567e6b2 100644
--- a/ql/src/test/results/clientpositive/remove_exprs_stats.q.out
+++ b/ql/src/test/results/clientpositive/remove_exprs_stats.q.out
@@ -74,11 +74,11 @@ STAGE PLANS:
Processor Tree:
TableScan
alias: loc_orc
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: int)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
ListSink
PREHOOK: query: explain select * from loc_orc where locid > 30
@@ -95,7 +95,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: loc_orc
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: false (type: boolean)
Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE
@@ -131,11 +131,11 @@ STAGE PLANS:
Processor Tree:
TableScan
alias: loc_orc
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: int)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
ListSink
PREHOOK: query: explain select * from loc_orc where locid >= 30
@@ -152,7 +152,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: loc_orc
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: false (type: boolean)
Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE
@@ -188,7 +188,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: loc_orc
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: (locid < 6) (type: boolean)
Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE
@@ -224,7 +224,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: loc_orc
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: false (type: boolean)
Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE
@@ -260,11 +260,11 @@ STAGE PLANS:
Processor Tree:
TableScan
alias: loc_orc
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: int)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
ListSink
PREHOOK: query: explain select * from loc_orc where locid >= 6
@@ -281,7 +281,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: loc_orc
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: (locid >= 6) (type: boolean)
Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE
@@ -317,7 +317,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: loc_orc
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: false (type: boolean)
Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE
@@ -353,7 +353,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: loc_orc
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: (locid > 1) (type: boolean)
Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE
@@ -389,7 +389,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: loc_orc
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: (locid <= 1) (type: boolean)
Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE
@@ -425,11 +425,11 @@ STAGE PLANS:
Processor Tree:
TableScan
alias: loc_orc
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: int)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
ListSink
PREHOOK: query: explain select * from loc_orc where locid IN (-4,5,30,40)
@@ -446,7 +446,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: loc_orc
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: (locid) IN (5) (type: boolean)
Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE
@@ -482,17 +482,17 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: loc_orc
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: (locid) IN (5, 2, 3) (type: boolean)
- Statistics: Num rows: 5 Data size: 498 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 5 Data size: 510 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: int)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 5 Data size: 498 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 5 Data size: 510 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 5 Data size: 498 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 5 Data size: 510 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -518,7 +518,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: loc_orc
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: (locid) IN (1, 6) (type: boolean)
Statistics: Num rows: 3 Data size: 306 Basic stats: COMPLETE Column stats: COMPLETE
@@ -554,7 +554,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: loc_orc
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: false (type: boolean)
Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/spark/annotate_stats_join.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/annotate_stats_join.q.out b/ql/src/test/results/clientpositive/spark/annotate_stats_join.q.out
index 5f06358..31ccc5c 100644
--- a/ql/src/test/results/clientpositive/spark/annotate_stats_join.q.out
+++ b/ql/src/test/results/clientpositive/spark/annotate_stats_join.q.out
@@ -565,19 +565,19 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: l
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: locid is not null (type: boolean)
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: int)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col1 (type: int)
sort order: +
Map-reduce partition columns: _col1 (type: int)
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col0 (type: string), _col2 (type: bigint), _col3 (type: int)
Reducer 2
Reduce Operator Tree:
@@ -659,19 +659,19 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: l
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: (state is not null and locid is not null) (type: boolean)
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: int)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: int)
sort order: ++
Map-reduce partition columns: _col0 (type: string), _col1 (type: int)
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col2 (type: bigint), _col3 (type: int)
Reducer 2
Reduce Operator Tree:
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/spark/groupby3.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/groupby3.q.out b/ql/src/test/results/clientpositive/spark/groupby3.q.out
index e48018c..23871ba 100644
--- a/ql/src/test/results/clientpositive/spark/groupby3.q.out
+++ b/ql/src/test/results/clientpositive/spark/groupby3.q.out
@@ -65,10 +65,10 @@ STAGE PLANS:
aggregations: sum(KEY._col0:0._col0), avg(KEY._col0:0._col0), avg(DISTINCT KEY._col0:0._col0), max(KEY._col0:0._col0), min(KEY._col0:0._col0), std(KEY._col0:0._col0), stddev_samp(KEY._col0:0._col0), variance(KEY._col0:0._col0), var_samp(KEY._col0:0._col0)
mode: partial1
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
- Statistics: Num rows: 1 Data size: 176 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 1208 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 176 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 1208 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: double), _col1 (type: struct<count:bigint,sum:double,input:string>), _col2 (type: struct<count:bigint,sum:double,input:string>), _col3 (type: string), _col4 (type: string), _col5 (type: struct<count:bigint,sum:double,variance:double>), _col6 (type: struct<count:bigint,sum:double,variance:double>), _col7 (type: struct<count:bigint,sum:double,variance:double>), _col8 (type: struct<count:bigint,sum:double,variance:double>)
Reducer 3
Reduce Operator Tree:
@@ -76,14 +76,14 @@ STAGE PLANS:
aggregations: sum(VALUE._col0), avg(VALUE._col1), avg(VALUE._col2), max(VALUE._col3), min(VALUE._col4), std(VALUE._col5), stddev_samp(VALUE._col6), variance(VALUE._col7), var_samp(VALUE._col8)
mode: final
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
- Statistics: Num rows: 1 Data size: 224 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 1208 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: double), _col1 (type: double), _col2 (type: double), UDFToDouble(_col3) (type: double), UDFToDouble(_col4) (type: double), _col5 (type: double), _col6 (type: double), _col7 (type: double), _col8 (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
- Statistics: Num rows: 1 Data size: 224 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 1208 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 224 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 1208 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/spark/groupby3_map.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/groupby3_map.q.out b/ql/src/test/results/clientpositive/spark/groupby3_map.q.out
index f806303..71f8dc0 100644
--- a/ql/src/test/results/clientpositive/spark/groupby3_map.q.out
+++ b/ql/src/test/results/clientpositive/spark/groupby3_map.q.out
@@ -70,14 +70,14 @@ STAGE PLANS:
aggregations: sum(VALUE._col0), avg(VALUE._col1), avg(DISTINCT KEY._col0:0._col0), max(VALUE._col3), min(VALUE._col4), std(VALUE._col5), stddev_samp(VALUE._col6), variance(VALUE._col7), var_samp(VALUE._col8)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
- Statistics: Num rows: 1 Data size: 224 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 1216 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: double), _col1 (type: double), _col2 (type: double), UDFToDouble(_col3) (type: double), UDFToDouble(_col4) (type: double), _col5 (type: double), _col6 (type: double), _col7 (type: double), _col8 (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
- Statistics: Num rows: 1 Data size: 224 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 1216 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 224 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 1216 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/spark/groupby3_map_multi_distinct.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/groupby3_map_multi_distinct.q.out b/ql/src/test/results/clientpositive/spark/groupby3_map_multi_distinct.q.out
index 3b31dfe..47ef5cb 100644
--- a/ql/src/test/results/clientpositive/spark/groupby3_map_multi_distinct.q.out
+++ b/ql/src/test/results/clientpositive/spark/groupby3_map_multi_distinct.q.out
@@ -74,14 +74,14 @@ STAGE PLANS:
aggregations: sum(VALUE._col0), avg(VALUE._col1), avg(DISTINCT KEY._col0:0._col0), max(VALUE._col3), min(VALUE._col4), std(VALUE._col5), stddev_samp(VALUE._col6), variance(VALUE._col7), var_samp(VALUE._col8), sum(DISTINCT KEY._col0:1._col0), count(DISTINCT KEY._col0:2._col0)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10
- Statistics: Num rows: 1 Data size: 248 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 1232 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: double), _col1 (type: double), _col2 (type: double), UDFToDouble(_col3) (type: double), UDFToDouble(_col4) (type: double), _col5 (type: double), _col6 (type: double), _col7 (type: double), _col8 (type: double), _col9 (type: double), UDFToDouble(_col10) (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10
- Statistics: Num rows: 1 Data size: 248 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 1232 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 248 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 1232 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/spark/groupby3_map_skew.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/groupby3_map_skew.q.out b/ql/src/test/results/clientpositive/spark/groupby3_map_skew.q.out
index bbad6e7..7cfca81 100644
--- a/ql/src/test/results/clientpositive/spark/groupby3_map_skew.q.out
+++ b/ql/src/test/results/clientpositive/spark/groupby3_map_skew.q.out
@@ -72,10 +72,10 @@ STAGE PLANS:
aggregations: sum(VALUE._col0), avg(VALUE._col1), avg(DISTINCT KEY._col0:0._col0), max(VALUE._col3), min(VALUE._col4), std(VALUE._col5), stddev_samp(VALUE._col6), variance(VALUE._col7), var_samp(VALUE._col8)
mode: partials
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
- Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 1464 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 1464 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: double), _col1 (type: struct<count:bigint,sum:double,input:string>), _col2 (type: struct<count:bigint,sum:double,input:string>), _col3 (type: string), _col4 (type: string), _col5 (type: struct<count:bigint,sum:double,variance:double>), _col6 (type: struct<count:bigint,sum:double,variance:double>), _col7 (type: struct<count:bigint,sum:double,variance:double>), _col8 (type: struct<count:bigint,sum:double,variance:double>)
Reducer 3
Reduce Operator Tree:
@@ -83,14 +83,14 @@ STAGE PLANS:
aggregations: sum(VALUE._col0), avg(VALUE._col1), avg(VALUE._col2), max(VALUE._col3), min(VALUE._col4), std(VALUE._col5), stddev_samp(VALUE._col6), variance(VALUE._col7), var_samp(VALUE._col8)
mode: final
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
- Statistics: Num rows: 1 Data size: 224 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 1464 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: double), _col1 (type: double), _col2 (type: double), UDFToDouble(_col3) (type: double), UDFToDouble(_col4) (type: double), _col5 (type: double), _col6 (type: double), _col7 (type: double), _col8 (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
- Statistics: Num rows: 1 Data size: 224 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 1464 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 224 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 1464 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/spark/groupby3_noskew.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/groupby3_noskew.q.out b/ql/src/test/results/clientpositive/spark/groupby3_noskew.q.out
index 6868eff..b2993a6 100644
--- a/ql/src/test/results/clientpositive/spark/groupby3_noskew.q.out
+++ b/ql/src/test/results/clientpositive/spark/groupby3_noskew.q.out
@@ -63,14 +63,14 @@ STAGE PLANS:
aggregations: sum(KEY._col0:0._col0), avg(KEY._col0:0._col0), avg(DISTINCT KEY._col0:0._col0), max(KEY._col0:0._col0), min(KEY._col0:0._col0), std(KEY._col0:0._col0), stddev_samp(KEY._col0:0._col0), variance(KEY._col0:0._col0), var_samp(KEY._col0:0._col0)
mode: complete
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
- Statistics: Num rows: 1 Data size: 224 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 424 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: double), _col1 (type: double), _col2 (type: double), UDFToDouble(_col3) (type: double), UDFToDouble(_col4) (type: double), _col5 (type: double), _col6 (type: double), _col7 (type: double), _col8 (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
- Statistics: Num rows: 1 Data size: 224 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 424 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 224 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 424 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/spark/groupby3_noskew_multi_distinct.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/groupby3_noskew_multi_distinct.q.out b/ql/src/test/results/clientpositive/spark/groupby3_noskew_multi_distinct.q.out
index 399bfd8..d152a07 100644
--- a/ql/src/test/results/clientpositive/spark/groupby3_noskew_multi_distinct.q.out
+++ b/ql/src/test/results/clientpositive/spark/groupby3_noskew_multi_distinct.q.out
@@ -67,14 +67,14 @@ STAGE PLANS:
aggregations: sum(KEY._col0:0._col0), avg(KEY._col0:0._col0), avg(DISTINCT KEY._col0:0._col0), max(KEY._col0:0._col0), min(KEY._col0:0._col0), std(KEY._col0:0._col0), stddev_samp(KEY._col0:0._col0), variance(KEY._col0:0._col0), var_samp(KEY._col0:0._col0), sum(DISTINCT KEY._col0:1._col0), count(DISTINCT KEY._col0:2._col0)
mode: complete
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10
- Statistics: Num rows: 1 Data size: 240 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 440 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: double), _col1 (type: double), _col2 (type: double), UDFToDouble(_col3) (type: double), UDFToDouble(_col4) (type: double), _col5 (type: double), _col6 (type: double), _col7 (type: double), _col8 (type: double), _col9 (type: double), UDFToDouble(_col10) (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10
- Statistics: Num rows: 1 Data size: 240 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 440 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 240 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 440 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/spark/subquery_in.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/subquery_in.q.out b/ql/src/test/results/clientpositive/spark/subquery_in.q.out
index 8923a4d..e40d6e4 100644
--- a/ql/src/test/results/clientpositive/spark/subquery_in.q.out
+++ b/ql/src/test/results/clientpositive/spark/subquery_in.q.out
@@ -354,10 +354,10 @@ STAGE PLANS:
aggregations: avg(_col0)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 76 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 76 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: struct<count:bigint,sum:double,input:int>)
Reducer 5
Reduce Operator Tree:
@@ -366,19 +366,24 @@ STAGE PLANS:
mode: mergepartial
outputColumnNames: _col0
<<<<<<< HEAD
+<<<<<<< HEAD
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+=======
+ Statistics: Num rows: 1 Data size: 76 Basic stats: COMPLETE Column stats: NONE
+>>>>>>> 8efe6f7... HIVE-14367 : Estimated size for constant nulls is 0 (Ashutosh Chauhan via Jesus Camacho Rodriguez)
Filter Operator
predicate: _col0 is not null (type: boolean)
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 76 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: double)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 76 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: double)
sort order: +
Map-reduce partition columns: _col0 (type: double)
+<<<<<<< HEAD
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
=======
Statistics: Num rows: 1 Data size: 76 Basic stats: COMPLETE Column stats: NONE
@@ -411,6 +416,9 @@ STAGE PLANS:
>>>>>>> 382dc20... HIVE-15192 : Use Calcite to de-correlate and plan subqueries (Vineet Garg via Ashutosh Chauhan)
=======
>>>>>>> f63dc2d... HIVE-15458 : Fix semi-join conversion rule for subquery (Vineet Garg via Ashutosh Chauhan)
+=======
+ Statistics: Num rows: 1 Data size: 76 Basic stats: COMPLETE Column stats: NONE
+>>>>>>> 8efe6f7... HIVE-14367 : Estimated size for constant nulls is 0 (Ashutosh Chauhan via Jesus Camacho Rodriguez)
Stage: Stage-0
Fetch Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/spark/union_remove_6_subq.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/union_remove_6_subq.q.out b/ql/src/test/results/clientpositive/spark/union_remove_6_subq.q.out
index ef9dfc4..8ef0d60 100644
--- a/ql/src/test/results/clientpositive/spark/union_remove_6_subq.q.out
+++ b/ql/src/test/results/clientpositive/spark/union_remove_6_subq.q.out
@@ -279,10 +279,10 @@ STAGE PLANS:
aggregations: avg(_col0)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col0 (type: struct<count:bigint,sum:double,input:bigint>)
Reducer 3
Reduce Operator Tree:
@@ -309,10 +309,10 @@ STAGE PLANS:
aggregations: avg(_col0)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col0 (type: struct<count:bigint,sum:double,input:bigint>)
Stage: Stage-0
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/spark/vectorization_0.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/vectorization_0.q.out b/ql/src/test/results/clientpositive/spark/vectorization_0.q.out
index 2c6c52d..efca361 100644
--- a/ql/src/test/results/clientpositive/spark/vectorization_0.q.out
+++ b/ql/src/test/results/clientpositive/spark/vectorization_0.q.out
@@ -236,10 +236,10 @@ STAGE PLANS:
aggregations: avg(ctinyint), variance(ctinyint), var_pop(ctinyint), var_samp(ctinyint), std(ctinyint), stddev(ctinyint), stddev_pop(ctinyint), stddev_samp(ctinyint)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 636 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 636 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: struct<count:bigint,sum:double,input:tinyint>), _col1 (type: struct<count:bigint,sum:double,variance:double>), _col2 (type: struct<count:bigint,sum:double,variance:double>), _col3 (type: struct<count:bigint,sum:double,variance:double>), _col4 (type: struct<count:bigint,sum:double,variance:double>), _col5 (type: struct<count:bigint,sum:double,variance:double>), _col6 (type: struct<count:bigint,sum:double,variance:double>), _col7 (type: struct<count:bigint,sum:double,variance:double>)
Execution mode: vectorized
Reducer 2
@@ -248,11 +248,11 @@ STAGE PLANS:
aggregations: avg(VALUE._col0), variance(VALUE._col1), var_pop(VALUE._col2), var_samp(VALUE._col3), std(VALUE._col4), stddev(VALUE._col5), stddev_pop(VALUE._col6), stddev_samp(VALUE._col7)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
- Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 636 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: double)
sort order: +
- Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 636 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: double), _col2 (type: double), _col3 (type: double), _col4 (type: double), _col5 (type: double), _col6 (type: double), _col7 (type: double)
Reducer 3
Execution mode: vectorized
@@ -260,10 +260,10 @@ STAGE PLANS:
Select Operator
expressions: KEY.reducesinkkey0 (type: double), VALUE._col0 (type: double), VALUE._col1 (type: double), VALUE._col2 (type: double), VALUE._col3 (type: double), VALUE._col4 (type: double), VALUE._col5 (type: double), VALUE._col6 (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
- Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 636 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 636 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -542,10 +542,10 @@ STAGE PLANS:
aggregations: avg(cbigint), variance(cbigint), var_pop(cbigint), var_samp(cbigint), std(cbigint), stddev(cbigint), stddev_pop(cbigint), stddev_samp(cbigint)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 640 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 640 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: struct<count:bigint,sum:double,input:bigint>), _col1 (type: struct<count:bigint,sum:double,variance:double>), _col2 (type: struct<count:bigint,sum:double,variance:double>), _col3 (type: struct<count:bigint,sum:double,variance:double>), _col4 (type: struct<count:bigint,sum:double,variance:double>), _col5 (type: struct<count:bigint,sum:double,variance:double>), _col6 (type: struct<count:bigint,sum:double,variance:double>), _col7 (type: struct<count:bigint,sum:double,variance:double>)
Execution mode: vectorized
Reducer 2
@@ -554,11 +554,11 @@ STAGE PLANS:
aggregations: avg(VALUE._col0), variance(VALUE._col1), var_pop(VALUE._col2), var_samp(VALUE._col3), std(VALUE._col4), stddev(VALUE._col5), stddev_pop(VALUE._col6), stddev_samp(VALUE._col7)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
- Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 640 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: double)
sort order: +
- Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 640 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: double), _col2 (type: double), _col3 (type: double), _col4 (type: double), _col5 (type: double), _col6 (type: double), _col7 (type: double)
Reducer 3
Execution mode: vectorized
@@ -566,10 +566,10 @@ STAGE PLANS:
Select Operator
expressions: KEY.reducesinkkey0 (type: double), VALUE._col0 (type: double), VALUE._col1 (type: double), VALUE._col2 (type: double), VALUE._col3 (type: double), VALUE._col4 (type: double), VALUE._col5 (type: double), VALUE._col6 (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
- Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 640 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 640 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -848,10 +848,10 @@ STAGE PLANS:
aggregations: avg(cfloat), variance(cfloat), var_pop(cfloat), var_samp(cfloat), std(cfloat), stddev(cfloat), stddev_pop(cfloat), stddev_samp(cfloat)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 636 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 636 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: struct<count:bigint,sum:double,input:float>), _col1 (type: struct<count:bigint,sum:double,variance:double>), _col2 (type: struct<count:bigint,sum:double,variance:double>), _col3 (type: struct<count:bigint,sum:double,variance:double>), _col4 (type: struct<count:bigint,sum:double,variance:double>), _col5 (type: struct<count:bigint,sum:double,variance:double>), _col6 (type: struct<count:bigint,sum:double,variance:double>), _col7 (type: struct<count:bigint,sum:double,variance:double>)
Execution mode: vectorized
Reducer 2
@@ -860,11 +860,11 @@ STAGE PLANS:
aggregations: avg(VALUE._col0), variance(VALUE._col1), var_pop(VALUE._col2), var_samp(VALUE._col3), std(VALUE._col4), stddev(VALUE._col5), stddev_pop(VALUE._col6), stddev_samp(VALUE._col7)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
- Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 636 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: double)
sort order: +
- Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 636 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: double), _col2 (type: double), _col3 (type: double), _col4 (type: double), _col5 (type: double), _col6 (type: double), _col7 (type: double)
Reducer 3
Execution mode: vectorized
@@ -872,10 +872,10 @@ STAGE PLANS:
Select Operator
expressions: KEY.reducesinkkey0 (type: double), VALUE._col0 (type: double), VALUE._col1 (type: double), VALUE._col2 (type: double), VALUE._col3 (type: double), VALUE._col4 (type: double), VALUE._col5 (type: double), VALUE._col6 (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
- Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 636 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 636 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -998,10 +998,10 @@ STAGE PLANS:
aggregations: avg(cbigint), stddev_pop(cbigint), var_samp(cbigint), count(), sum(cfloat), min(ctinyint)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
- Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 260 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 260 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: struct<count:bigint,sum:double,input:bigint>), _col1 (type: struct<count:bigint,sum:double,variance:double>), _col2 (type: struct<count:bigint,sum:double,variance:double>), _col3 (type: bigint), _col4 (type: double), _col5 (type: tinyint)
Execution mode: vectorized
Reducer 2
@@ -1010,14 +1010,14 @@ STAGE PLANS:
aggregations: avg(VALUE._col0), stddev_pop(VALUE._col1), var_samp(VALUE._col2), count(VALUE._col3), sum(VALUE._col4), min(VALUE._col5)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
- Statistics: Num rows: 1 Data size: 44 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 260 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: double), (- _col0) (type: double), (-6432.0 + _col0) (type: double), _col1 (type: double), (- (-6432.0 + _col0)) (type: double), ((- (-6432.0 + _col0)) + (-6432.0 + _col0)) (type: double), _col2 (type: double), (- (-6432.0 + _col0)) (type: double), (-6432.0 + (- (-6432.0 + _col0))) (type: double), (- (-6432.0 + _col0)) (type: double), ((- (-6432.0 + _col0)) / (- (-6432.0 + _col0))) (type: double), _col3 (type: bigint), _col4 (type: double), (_col2 % _col1) (type: double), (- _col2) (type: double), ((- (-6432.0 + _col0)) * (- _col0)) (type: double), _col5 (type: tinyint), (- _col5) (type: tinyint)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17
- Statistics: Num rows: 1 Data size: 44 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 260 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 44 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 260 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/spark/vectorization_pushdown.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/vectorization_pushdown.q.out b/ql/src/test/results/clientpositive/spark/vectorization_pushdown.q.out
index ef7df76..59a3be0 100644
--- a/ql/src/test/results/clientpositive/spark/vectorization_pushdown.q.out
+++ b/ql/src/test/results/clientpositive/spark/vectorization_pushdown.q.out
@@ -31,10 +31,10 @@ STAGE PLANS:
aggregations: avg(cbigint)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: struct<count:bigint,sum:double,input:bigint>)
Execution mode: vectorized
Reducer 2
@@ -43,10 +43,10 @@ STAGE PLANS:
aggregations: avg(VALUE._col0)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out b/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out
index a969029..20524f1 100644
--- a/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out
+++ b/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out
@@ -134,10 +134,10 @@ STAGE PLANS:
aggregations: avg(cint), sum(cdouble), stddev_pop(cint), stddev_samp(csmallint), var_samp(cint), avg(cfloat), stddev_samp(cint), min(ctinyint), count(csmallint)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
- Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 492 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 492 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: struct<count:bigint,sum:double,input:int>), _col1 (type: double), _col2 (type: struct<count:bigint,sum:double,variance:double>), _col3 (type: struct<count:bigint,sum:double,variance:double>), _col4 (type: struct<count:bigint,sum:double,variance:double>), _col5 (type: struct<count:bigint,sum:double,input:float>), _col6 (type: struct<count:bigint,sum:double,variance:double>), _col7 (type: tinyint), _col8 (type: bigint)
Execution mode: vectorized
Reducer 2
@@ -146,14 +146,14 @@ STAGE PLANS:
aggregations: avg(VALUE._col0), sum(VALUE._col1), stddev_pop(VALUE._col2), stddev_samp(VALUE._col3), var_samp(VALUE._col4), avg(VALUE._col5), stddev_samp(VALUE._col6), min(VALUE._col7), count(VALUE._col8)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
- Statistics: Num rows: 1 Data size: 68 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 492 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: double), (_col0 + -3728.0) (type: double), (- (_col0 + -3728.0)) (type: double), (- (- (_col0 + -3728.0))) (type: double), ((- (- (_col0 + -3728.0))) * (_col0 + -3728.0)) (type: double), _col1 (type: double), (- _col0) (type: double), _col2 (type: double), (((- (- (_col0 + -3728.0))) * (_col0 + -3728.0)) * (- (- (_col0 + -3728.0)))) (type: double), _col3 (type: double), (- _col2) (type: double), (_col2 - (- (- (_col0 + -3728.0)))) (type: double), ((_col2 - (- (- (_col0 + -3728.0)))) * _col2) (type: double), _col4 (type: double), _col5 (type: double), (10.175 - _col4) (type: double), (- (10.175 - _col4)) (type: double), ((- _col2) / -563.0) (type: double), _col6 (type: double), (- ((- _col2) / -563.0)) (type: double), (_col0 / _col1) (type: double), _col7 (type: tinyint), _col8 (type: bigint), (UDFToDouble(_col7) / ((- _col2) / -563.0)) (type: double), (- (_col0 / _col1)) (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24
- Statistics: Num rows: 1 Data size: 68 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 492 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 68 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 492 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -336,10 +336,10 @@ STAGE PLANS:
aggregations: max(cint), var_pop(cbigint), stddev_pop(csmallint), max(cdouble), avg(ctinyint), min(cint), min(cdouble), stddev_samp(csmallint), var_samp(cint)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
- Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 420 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 420 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: int), _col1 (type: struct<count:bigint,sum:double,variance:double>), _col2 (type: struct<count:bigint,sum:double,variance:double>), _col3 (type: double), _col4 (type: struct<count:bigint,sum:double,input:tinyint>), _col5 (type: int), _col6 (type: double), _col7 (type: struct<count:bigint,sum:double,variance:double>), _col8 (type: struct<count:bigint,sum:double,variance:double>)
Execution mode: vectorized
Reducer 2
@@ -348,14 +348,14 @@ STAGE PLANS:
aggregations: max(VALUE._col0), var_pop(VALUE._col1), stddev_pop(VALUE._col2), max(VALUE._col3), avg(VALUE._col4), min(VALUE._col5), min(VALUE._col6), stddev_samp(VALUE._col7), var_samp(VALUE._col8)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
- Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 420 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: int), (UDFToDouble(_col0) / -3728.0) (type: double), (_col0 * -3728) (type: int), _col1 (type: double), (- (_col0 * -3728)) (type: int), _col2 (type: double), (-563 % (_col0 * -3728)) (type: int), (_col1 / _col2) (type: double), (- _col2) (type: double), _col3 (type: double), _col4 (type: double), (_col2 - 10.175) (type: double), _col5 (type: int), (UDFToDouble((_col0 * -3728)) % (_col2 - 10.175)) (type: double), (- _col3) (type: double), _col6 (type: double), (_col3 % -26.28) (type: double), _col7 (type: double), (- (UDFToDouble(_col0) / -3728.0)) (type: double), ((- (_col0 * -3728)) % (-563 % (_col0 * -3728))) (type: int), ((UDFToDouble(_col0) / -3728.0) - _col4) (type: double), (- (_col0 * -3728)) (type: int), _col8 (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22
- Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 420 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 420 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -529,10 +529,10 @@ STAGE PLANS:
aggregations: var_pop(cbigint), count(), max(ctinyint), stddev_pop(csmallint), max(cint), stddev_samp(cdouble), count(ctinyint), avg(ctinyint)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
- Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 340 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 340 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: struct<count:bigint,sum:double,variance:double>), _col1 (type: bigint), _col2 (type: tinyint), _col3 (type: struct<count:bigint,sum:double,variance:double>), _col4 (type: int), _col5 (type: struct<count:bigint,sum:double,variance:double>), _col6 (type: bigint), _col7 (type: struct<count:bigint,sum:double,input:tinyint>)
Execution mode: vectorized
Reducer 2
@@ -541,14 +541,14 @@ STAGE PLANS:
aggregations: var_pop(VALUE._col0), count(VALUE._col1), max(VALUE._col2), stddev_pop(VALUE._col3), max(VALUE._col4), stddev_samp(VALUE._col5), count(VALUE._col6), avg(VALUE._col7)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
- Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 340 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: double), (- _col0) (type: double), (_col0 - (- _col0)) (type: double), _col1 (type: bigint), (UDFToDouble(_col1) % 79.553) (type: double), _col2 (type: tinyint), (UDFToDouble(_col1) - (- _col0)) (type: double), (- (- _col0)) (type: double), (-1.0 % (- _col0)) (type: double), _col1 (type: bigint), (- _col1) (type: bigint), _col3 (type: double), (- (- (- _col0))) (type: double), (762 * (- _col1)) (type: bigint), _col4 (type: int), (UDFToLong(_col2) + (762 * (- _col1))) (type: bigint), ((- _col0) + UDFToDouble(_col4)) (type: double), _col5 (type: double), ((- _col1) % _col1) (type: bigint), _col6 (type: bigint), _col7 (type: double), (-3728 % (UDFToLong(_col2) + (762 * (- _col1)))) (type: bigint)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21
- Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 340 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 340 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -701,10 +701,10 @@ STAGE PLANS:
aggregations: avg(ctinyint), max(cbigint), stddev_samp(cint), var_pop(cint), var_pop(cbigint), max(cfloat)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
- Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 328 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 328 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: struct<count:bigint,sum:double,input:tinyint>), _col1 (type: bigint), _col2 (type: struct<count:bigint,sum:double,variance:double>), _col3 (type: struct<count:bigint,sum:double,variance:double>), _col4 (type: struct<count:bigint,sum:double,variance:double>), _col5 (type: float)
Execution mode: vectorized
Reducer 2
@@ -713,14 +713,14 @@ STAGE PLANS:
aggregations: avg(VALUE._col0), max(VALUE._col1), stddev_samp(VALUE._col2), var_pop(VALUE._col3), var_pop(VALUE._col4), max(VALUE._col5)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
- Statistics: Num rows: 1 Data size: 44 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 328 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: double), (_col0 + 6981.0) (type: double), ((_col0 + 6981.0) + _col0) (type: double), _col1 (type: bigint), (((_col0 + 6981.0) + _col0) / _col0) (type: double), (- (_col0 + 6981.0)) (type: double), _col2 (type: double), (_col0 % (- (_col0 + 6981.0))) (type: double), _col3 (type: double), _col4 (type: double), (- _col1) (type: bigint), (UDFToDouble((- _col1)) / _col2) (type: double), _col5 (type: float), (_col4 * -26.28) (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13
- Statistics: Num rows: 1 Data size: 44 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 328 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 44 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 328 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/spark/vectorized_mapjoin.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/vectorized_mapjoin.q.out b/ql/src/test/results/clientpositive/spark/vectorized_mapjoin.q.out
index cf5f55a..d605185 100644
--- a/ql/src/test/results/clientpositive/spark/vectorized_mapjoin.q.out
+++ b/ql/src/test/results/clientpositive/spark/vectorized_mapjoin.q.out
@@ -72,10 +72,10 @@ STAGE PLANS:
aggregations: count(_col0), max(_col1), min(_col0), avg(_col2)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: bigint), _col1 (type: int), _col2 (type: int), _col3 (type: struct<count:bigint,sum:double,input:int>)
Execution mode: vectorized
Local Work:
@@ -86,10 +86,10 @@ STAGE PLANS:
aggregations: count(VALUE._col0), max(VALUE._col1), min(VALUE._col2), avg(VALUE._col3)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/spark/vectorized_shufflejoin.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/vectorized_shufflejoin.q.out b/ql/src/test/results/clientpositive/spark/vectorized_shufflejoin.q.out
index 21e47b2..97f12d4 100644
--- a/ql/src/test/results/clientpositive/spark/vectorized_shufflejoin.q.out
+++ b/ql/src/test/results/clientpositive/spark/vectorized_shufflejoin.q.out
@@ -73,10 +73,10 @@ STAGE PLANS:
aggregations: count(_col0), max(_col1), min(_col0), avg(_col2)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: bigint), _col1 (type: int), _col2 (type: int), _col3 (type: struct<count:bigint,sum:double,input:int>)
Reducer 3
Reduce Operator Tree:
@@ -84,11 +84,11 @@ STAGE PLANS:
aggregations: count(VALUE._col0), max(VALUE._col1), min(VALUE._col2), avg(VALUE._col3)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: bigint)
sort order: +
- Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: double)
Reducer 4
Execution mode: vectorized
@@ -96,10 +96,10 @@ STAGE PLANS:
Select Operator
expressions: KEY.reducesinkkey0 (type: bigint), VALUE._col0 (type: int), VALUE._col1 (type: int), VALUE._col2 (type: double)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/spark/vectorized_timestamp_funcs.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/vectorized_timestamp_funcs.q.out b/ql/src/test/results/clientpositive/spark/vectorized_timestamp_funcs.q.out
index c43e769..9948112 100644
--- a/ql/src/test/results/clientpositive/spark/vectorized_timestamp_funcs.q.out
+++ b/ql/src/test/results/clientpositive/spark/vectorized_timestamp_funcs.q.out
@@ -805,10 +805,10 @@ STAGE PLANS:
aggregations: avg(ctimestamp1), variance(ctimestamp1), var_pop(ctimestamp1), var_samp(ctimestamp1), std(ctimestamp1), stddev(ctimestamp1), stddev_pop(ctimestamp1), stddev_samp(ctimestamp1)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 672 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 672 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: struct<count:bigint,sum:double,input:timestamp>), _col1 (type: struct<count:bigint,sum:double,variance:double>), _col2 (type: struct<count:bigint,sum:double,variance:double>), _col3 (type: struct<count:bigint,sum:double,variance:double>), _col4 (type: struct<count:bigint,sum:double,variance:double>), _col5 (type: struct<count:bigint,sum:double,variance:double>), _col6 (type: struct<count:bigint,sum:double,variance:double>), _col7 (type: struct<count:bigint,sum:double,variance:double>)
Execution mode: vectorized
Reducer 2
@@ -817,14 +817,14 @@ STAGE PLANS:
aggregations: avg(VALUE._col0), variance(VALUE._col1), var_pop(VALUE._col2), var_samp(VALUE._col3), std(VALUE._col4), stddev(VALUE._col5), stddev_pop(VALUE._col6), stddev_samp(VALUE._col7)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
- Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 672 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: round(_col0, 0) (type: double), _col1 BETWEEN 8.97077295279421E19 AND 8.97077295279422E19 (type: boolean), _col2 BETWEEN 8.97077295279421E19 AND 8.97077295279422E19 (type: boolean), _col3 BETWEEN 9.20684592523616E19 AND 9.20684592523617E19 (type: boolean), round(_col4, 3) (type: double), round(_col5, 3) (type: double), round(_col6, 3) (type: double), round(_col7, 3) (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
- Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 672 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 672 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/subquery_in.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/subquery_in.q.out b/ql/src/test/results/clientpositive/subquery_in.q.out
index caf2470..e2bfae0 100644
--- a/ql/src/test/results/clientpositive/subquery_in.q.out
+++ b/ql/src/test/results/clientpositive/subquery_in.q.out
@@ -284,7 +284,7 @@ STAGE PLANS:
aggregations: avg(_col0)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 76 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
table:
@@ -298,22 +298,22 @@ STAGE PLANS:
TableScan
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 76 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: struct<count:bigint,sum:double,input:int>)
Reduce Operator Tree:
Group By Operator
aggregations: avg(VALUE._col0)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 76 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: _col0 is not null (type: boolean)
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 76 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: double)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 76 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
table:
@@ -345,7 +345,7 @@ STAGE PLANS:
key expressions: _col0 (type: double)
sort order: +
Map-reduce partition columns: _col0 (type: double)
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 76 Basic stats: COMPLETE Column stats: NONE
Reduce Operator Tree:
Join Operator
condition map:
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/subquery_notin.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/subquery_notin.q.out b/ql/src/test/results/clientpositive/subquery_notin.q.out
index 4398c69..30ead87 100644
--- a/ql/src/test/results/clientpositive/subquery_notin.q.out
+++ b/ql/src/test/results/clientpositive/subquery_notin.q.out
@@ -713,7 +713,7 @@ STAGE PLANS:
aggregations: avg(_col0)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 76 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
table:
@@ -727,19 +727,19 @@ STAGE PLANS:
TableScan
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 76 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: struct<count:bigint,sum:double,input:int>)
Reduce Operator Tree:
Group By Operator
aggregations: avg(VALUE._col0)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 76 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(), count(_col0)
mode: complete
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
table:
@@ -764,7 +764,7 @@ STAGE PLANS:
TableScan
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: bigint), _col1 (type: bigint)
Reduce Operator Tree:
Join Operator
@@ -774,7 +774,7 @@ STAGE PLANS:
0
1
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 26 Data size: 3589 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 26 Data size: 5357 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
table:
@@ -790,14 +790,14 @@ STAGE PLANS:
key expressions: UDFToDouble(_col1) (type: double)
sort order: +
Map-reduce partition columns: UDFToDouble(_col1) (type: double)
- Statistics: Num rows: 26 Data size: 3589 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 26 Data size: 5357 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string), _col1 (type: int), _col2 (type: bigint), _col3 (type: bigint)
TableScan
Reduce Output Operator
key expressions: _col0 (type: double)
sort order: +
Map-reduce partition columns: _col0 (type: double)
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 76 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: boolean)
Reduce Operator Tree:
Join Operator
@@ -807,17 +807,17 @@ STAGE PLANS:
0 UDFToDouble(_col1) (type: double)
1 _col0 (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col5
- Statistics: Num rows: 28 Data size: 3947 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 28 Data size: 5892 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: ((_col2 = 0) or (_col5 is null and _col1 is not null and (_col3 >= _col2))) (type: boolean)
- Statistics: Num rows: 18 Data size: 2537 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 18 Data size: 3787 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: string), _col1 (type: int)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 18 Data size: 2537 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 18 Data size: 3787 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 18 Data size: 2537 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 18 Data size: 3787 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -872,7 +872,7 @@ STAGE PLANS:
aggregations: avg(_col0)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 76 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
table:
@@ -886,19 +886,19 @@ STAGE PLANS:
TableScan
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 76 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: struct<count:bigint,sum:double,input:int>)
Reduce Operator Tree:
Group By Operator
aggregations: avg(VALUE._col0)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 76 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: double)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 76 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
table:
@@ -914,17 +914,17 @@ STAGE PLANS:
key expressions: _col0 (type: double)
sort order: +
Map-reduce partition columns: _col0 (type: double)
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 76 Basic stats: COMPLETE Column stats: NONE
Reduce Operator Tree:
Group By Operator
keys: KEY._col0 (type: double)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 76 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: double), true (type: boolean)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 76 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
table:
[07/50] [abbrv] hive git commit: HIVE-12492: MapJoin: 4 million
unique integers seems to be a probe plateau (Jesus Camacho Rodriguez,
reviewed by Ashutosh Chauhan)
Posted by om...@apache.org.
HIVE-12492: MapJoin: 4 million unique integers seems to be a probe plateau (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
Change-Id: Ifd892eba13e6253bfbd37a8304f1acd8d1319d68
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/da395b57
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/da395b57
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/da395b57
Branch: refs/heads/branch-2.2
Commit: da395b57ef8a73e63509d5671492c407949e0b25
Parents: f9eb0bd
Author: Jesus Camacho Rodriguez <jc...@apache.org>
Authored: Mon Feb 27 15:45:53 2017 +0000
Committer: Owen O'Malley <om...@apache.org>
Committed: Tue Mar 28 15:27:50 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hive/conf/HiveConf.java | 5 +
.../test/resources/testconfiguration.properties | 1 +
.../hive/ql/optimizer/ConvertJoinMapJoin.java | 106 +++-
.../stats/annotation/StatsRulesProcFactory.java | 44 +-
.../apache/hadoop/hive/ql/stats/StatsUtils.java | 6 +-
.../queries/clientpositive/join_max_hashtable.q | 38 ++
.../llap/join_max_hashtable.q.out | 490 +++++++++++++++++++
7 files changed, 679 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/da395b57/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index eefb590..c6c5284 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -1432,6 +1432,11 @@ public class HiveConf extends Configuration {
"This controls how many partitions can be scanned for each partitioned table.\n" +
"The default value \"-1\" means no limit."),
+ HIVECONVERTJOINMAXENTRIESHASHTABLE("hive.auto.convert.join.hashtable.max.entries", 4194304L,
+ "If hive.auto.convert.join.noconditionaltask is off, this parameter does not take affect. \n" +
+ "However, if it is on, and the predicated number of entries in hashtable for a given join \n" +
+ "input is larger than this number, the join will not be converted to a mapjoin. \n" +
+ "The value \"-1\" means no limit."),
HIVEHASHTABLEKEYCOUNTADJUSTMENT("hive.hashtable.key.count.adjustment", 1.0f,
"Adjustment to mapjoin hashtable size derived from table and column statistics; the estimate" +
" of the number of keys is divided by this value. If the value is 0, statistics are not used" +
http://git-wip-us.apache.org/repos/asf/hive/blob/da395b57/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index b2abc74..3ab2714 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -575,6 +575,7 @@ minillap.query.files=cross_prod_1.q,\
insert_dir_distcp.q,\
join1.q,\
join_acid_non_acid.q,\
+ join_max_hashtable.q,\
leftsemijoin_mr.q,\
list_bucket_dml_10.q,\
load_fs2.q,\
http://git-wip-us.apache.org/repos/asf/hive/blob/da395b57/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
index 8ea5268..467d134 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
@@ -19,7 +19,6 @@
package org.apache.hadoop.hive.ql.optimizer;
import java.util.ArrayList;
-import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -53,6 +52,7 @@ import org.apache.hadoop.hive.ql.parse.GenTezUtils;
import org.apache.hadoop.hive.ql.parse.OptimizeTezProcContext;
import org.apache.hadoop.hive.ql.parse.ParseContext;
import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.plan.ColStatistics;
import org.apache.hadoop.hive.ql.plan.CommonMergeJoinDesc;
import org.apache.hadoop.hive.ql.plan.DynamicPruningEventDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
@@ -63,6 +63,7 @@ import org.apache.hadoop.hive.ql.plan.MapJoinDesc;
import org.apache.hadoop.hive.ql.plan.OpTraits;
import org.apache.hadoop.hive.ql.plan.OperatorDesc;
import org.apache.hadoop.hive.ql.plan.Statistics;
+import org.apache.hadoop.hive.ql.stats.StatsUtils;
import org.apache.hadoop.util.ReflectionUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -117,7 +118,7 @@ public class ConvertJoinMapJoin implements NodeProcessor {
numBuckets = 1;
}
LOG.info("Estimated number of buckets " + numBuckets);
- int mapJoinConversionPos = getMapJoinConversionPos(joinOp, context, numBuckets, false, maxSize);
+ int mapJoinConversionPos = getMapJoinConversionPos(joinOp, context, numBuckets, false, maxSize, true);
if (mapJoinConversionPos < 0) {
Object retval = checkAndConvertSMBJoin(context, joinOp, tezBucketJoinProcCtx);
if (retval == null) {
@@ -141,7 +142,7 @@ public class ConvertJoinMapJoin implements NodeProcessor {
// check if we can convert to map join no bucket scaling.
LOG.info("Convert to non-bucketed map join");
if (numBuckets != 1) {
- mapJoinConversionPos = getMapJoinConversionPos(joinOp, context, 1, false, maxSize);
+ mapJoinConversionPos = getMapJoinConversionPos(joinOp, context, 1, false, maxSize, true);
}
if (mapJoinConversionPos < 0) {
// we are just converting to a common merge join operator. The shuffle
@@ -515,8 +516,22 @@ public class ConvertJoinMapJoin implements NodeProcessor {
return false;
}
+ /**
+ * Obtain big table position for join.
+ *
+ * @param joinOp join operator
+ * @param context optimization context
+ * @param buckets bucket count for Bucket Map Join conversion consideration or reduce count
+ * for Dynamic Hash Join conversion consideration
+ * @param skipJoinTypeChecks whether to skip join type checking
+ * @param maxSize size threshold for Map Join conversion
+ * @param checkHashTableEntries whether to check threshold for distinct keys in hash table for Map Join
+ * @return returns big table position or -1 if it cannot be determined
+ * @throws SemanticException
+ */
public int getMapJoinConversionPos(JoinOperator joinOp, OptimizeTezProcContext context,
- int buckets, boolean skipJoinTypeChecks, long maxSize) throws SemanticException {
+ int buckets, boolean skipJoinTypeChecks, long maxSize, boolean checkHashTableEntries)
+ throws SemanticException {
if (!skipJoinTypeChecks) {
/*
* HIVE-9038: Join tests fail in tez when we have more than 1 join on the same key and there is
@@ -624,10 +639,20 @@ public class ConvertJoinMapJoin implements NodeProcessor {
// We are replacing the current big table with a new one, thus
// we need to count the current one as a map table then.
totalSize += bigInputStat.getDataSize();
+ // Check if number of distinct keys is larger than given max
+ // number of entries for HashMap. If it is, we do not convert.
+ if (checkHashTableEntries && !checkNumberOfEntriesForHashTable(joinOp, bigTablePosition, context)) {
+ return -1;
+ }
} else if (!selectedBigTable) {
// This is not the first table and we are not using it as big table,
// in fact, we're adding this table as a map table
totalSize += inputSize;
+ // Check if number of distinct keys is larger than given max
+ // number of entries for HashMap. If it is, we do not convert.
+ if (checkHashTableEntries && !checkNumberOfEntriesForHashTable(joinOp, pos, context)) {
+ return -1;
+ }
}
if (totalSize/buckets > maxSize) {
@@ -901,8 +926,8 @@ public class ConvertJoinMapJoin implements NodeProcessor {
int numReducers = estimateNumBuckets(joinOp, false);
LOG.info("Try dynamic partitioned hash join with estimated " + numReducers + " reducers");
int bigTablePos = getMapJoinConversionPos(joinOp, context, numReducers, false,
- context.conf.getLongVar(
- HiveConf.ConfVars.HIVECONVERTJOINNOCONDITIONALTASKTHRESHOLD));
+ context.conf.getLongVar(HiveConf.ConfVars.HIVECONVERTJOINNOCONDITIONALTASKTHRESHOLD),
+ false);
if (bigTablePos >= 0) {
// Now that we have the big table index, get real numReducers value based on big table RS
ReduceSinkOperator bigTableParentRS =
@@ -947,7 +972,7 @@ public class ConvertJoinMapJoin implements NodeProcessor {
}
int pos = getMapJoinConversionPos(joinOp, context, estimateNumBuckets(joinOp, false),
- true, Long.MAX_VALUE);
+ true, Long.MAX_VALUE, false);
if (pos < 0) {
LOG.info("Could not get a valid join position. Defaulting to position 0");
pos = 0;
@@ -957,4 +982,71 @@ public class ConvertJoinMapJoin implements NodeProcessor {
LOG.info("Fallback to common merge join operator");
convertJoinSMBJoin(joinOp, context, pos, 0, false);
}
+
+ /* Returns true if it passes the test, false otherwise. */
+ private boolean checkNumberOfEntriesForHashTable(JoinOperator joinOp, int position,
+ OptimizeTezProcContext context) {
+ long max = HiveConf.getLongVar(context.parseContext.getConf(),
+ HiveConf.ConfVars.HIVECONVERTJOINMAXENTRIESHASHTABLE);
+ if (max < 1) {
+ // Max is disabled, we can safely return true
+ return true;
+ }
+ // Calculate number of different entries and evaluate
+ ReduceSinkOperator rsOp = (ReduceSinkOperator) joinOp.getParentOperators().get(position);
+ List<String> keys = StatsUtils.getQualifedReducerKeyNames(rsOp.getConf().getOutputKeyColumnNames());
+ Statistics inputStats = rsOp.getStatistics();
+ List<ColStatistics> columnStats = new ArrayList<>();
+ for (String key : keys) {
+ ColStatistics cs = inputStats.getColumnStatisticsFromColName(key);
+ if (cs == null) {
+ LOG.debug("Couldn't get statistics for: {}", key);
+ return true;
+ }
+ columnStats.add(cs);
+ }
+ long numRows = inputStats.getNumRows();
+ long estimation = estimateNDV(numRows, columnStats);
+ LOG.debug("Estimated NDV for input {}: {}; Max NDV for MapJoin conversion: {}",
+ position, estimation, max);
+ if (estimation > max) {
+ // Estimation larger than max
+ LOG.debug("Number of different entries for HashTable is greater than the max; "
+ + "we do not converting to MapJoin");
+ return false;
+ }
+ // We can proceed with the conversion
+ return true;
+ }
+
+ private static long estimateNDV(long numRows, List<ColStatistics> columnStats) {
+ // If there is a single column, return the number of distinct values
+ if (columnStats.size() == 1) {
+ return columnStats.get(0).getCountDistint();
+ }
+
+ // The expected number of distinct values when choosing p values
+ // with replacement from n integers is n . (1 - ((n - 1) / n) ^ p).
+ //
+ // If we have several uniformly distributed attributes A1 ... Am
+ // with N1 ... Nm distinct values, they behave as one uniformly
+ // distributed attribute with N1 * ... * Nm distinct values.
+ long n = 1L;
+ for (ColStatistics cs : columnStats) {
+ final long ndv = cs.getCountDistint();
+ if (ndv > 1) {
+ n = StatsUtils.safeMult(n, ndv);
+ }
+ }
+ final double nn = (double) n;
+ final double a = (nn - 1d) / nn;
+ if (a == 1d) {
+ // A under-flows if nn is large.
+ return numRows;
+ }
+ final double v = nn * (1d - Math.pow(a, numRows));
+ // Cap at fact-row-count, because numerical artifacts can cause it
+ // to go a few % over.
+ return Math.min(Math.round(v), numRows);
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/da395b57/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
index faf26c5..33778cd 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
@@ -51,10 +51,43 @@ import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.parse.PrunedPartitionList;
import org.apache.hadoop.hive.ql.parse.SemanticException;
-import org.apache.hadoop.hive.ql.plan.*;
+import org.apache.hadoop.hive.ql.plan.AggregationDesc;
+import org.apache.hadoop.hive.ql.plan.ColStatistics;
+import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeColumnListDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc.ExprNodeDescEqualityWrapper;
+import org.apache.hadoop.hive.ql.plan.ExprNodeDescUtils;
+import org.apache.hadoop.hive.ql.plan.ExprNodeDynamicListDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeDynamicValueDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeFieldDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
+import org.apache.hadoop.hive.ql.plan.GroupByDesc;
+import org.apache.hadoop.hive.ql.plan.JoinCondDesc;
+import org.apache.hadoop.hive.ql.plan.JoinDesc;
+import org.apache.hadoop.hive.ql.plan.MapJoinDesc;
+import org.apache.hadoop.hive.ql.plan.OperatorDesc;
+import org.apache.hadoop.hive.ql.plan.Statistics;
import org.apache.hadoop.hive.ql.stats.StatsUtils;
-import org.apache.hadoop.hive.ql.udf.generic.*;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBetween;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFIn;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFInBloomFilter;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPAnd;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqual;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqualNS;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqualOrGreaterThan;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqualOrLessThan;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPGreaterThan;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPLessThan;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNot;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNotEqual;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNotNull;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNull;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPOr;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFStruct;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
@@ -1423,7 +1456,12 @@ public class StatsRulesProcFactory {
// get the join keys from parent ReduceSink operators
for (int pos = 0; pos < parents.size(); pos++) {
ReduceSinkOperator parent = (ReduceSinkOperator) jop.getParentOperators().get(pos);
- Statistics parentStats = parent.getStatistics();
+ Statistics parentStats;
+ try {
+ parentStats = parent.getStatistics().clone();
+ } catch (CloneNotSupportedException e) {
+ throw new SemanticException(ErrorMsg.STATISTICS_CLONING_FAILED.getMsg());
+ }
keyExprs = StatsUtils.getQualifedReducerKeyNames(parent.getConf()
.getOutputKeyColumnNames());
http://git-wip-us.apache.org/repos/asf/hive/blob/da395b57/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
index afa7d82..0a2e4a5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
@@ -1274,7 +1274,11 @@ public class StatsUtils {
ColStatistics colStats = parentStats.getColumnStatisticsFromColName(colName);
if (colStats != null) {
/* If statistics for the column already exist use it. */
- return colStats;
+ try {
+ return colStats.clone();
+ } catch (CloneNotSupportedException e) {
+ return null;
+ }
}
// virtual columns
http://git-wip-us.apache.org/repos/asf/hive/blob/da395b57/ql/src/test/queries/clientpositive/join_max_hashtable.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/join_max_hashtable.q b/ql/src/test/queries/clientpositive/join_max_hashtable.q
new file mode 100644
index 0000000..c3c4434
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/join_max_hashtable.q
@@ -0,0 +1,38 @@
+set hive.auto.convert.join=true;
+set hive.stats.fetch.column.stats=true;
+set hive.optimize.dynamic.partition.hashjoin=true;
+set hive.auto.convert.join.hashtable.max.entries=500;
+
+-- CONVERT
+EXPLAIN
+SELECT x.key, x.value
+FROM src x JOIN src y ON (x.key = y.key);
+
+-- CONVERT
+EXPLAIN
+SELECT x.key, x.value
+FROM src x JOIN src y ON (x.key = y.key AND x.value = y.value);
+
+set hive.auto.convert.join.hashtable.max.entries=300;
+
+-- CONVERT
+EXPLAIN
+SELECT x.key, x.value
+FROM src x JOIN src y ON (x.key = y.key);
+
+-- DO NOT CONVERT
+EXPLAIN
+SELECT x.key, x.value
+FROM src x JOIN src y ON (x.key = y.key AND x.value = y.value);
+
+set hive.auto.convert.join.hashtable.max.entries=10;
+
+-- DO NOT CONVERT
+EXPLAIN
+SELECT x.key, x.value
+FROM src x JOIN src y ON (x.key = y.key);
+
+-- DO NOT CONVERT
+EXPLAIN
+SELECT x.key, x.value
+FROM src x JOIN src y ON (x.key = y.key AND x.value = y.value);
http://git-wip-us.apache.org/repos/asf/hive/blob/da395b57/ql/src/test/results/clientpositive/llap/join_max_hashtable.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/join_max_hashtable.q.out b/ql/src/test/results/clientpositive/llap/join_max_hashtable.q.out
new file mode 100644
index 0000000..85d45fe
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/join_max_hashtable.q.out
@@ -0,0 +1,490 @@
+PREHOOK: query: EXPLAIN
+SELECT x.key, x.value
+FROM src x JOIN src y ON (x.key = y.key)
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN
+SELECT x.key, x.value
+FROM src x JOIN src y ON (x.key = y.key)
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Map 1 <- Map 2 (BROADCAST_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: x
+ Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col0, _col1
+ input vertices:
+ 1 Map 2
+ Statistics: Num rows: 1219 Data size: 216982 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1219 Data size: 216982 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: llap
+ LLAP IO: no inputs
+ Map 2
+ Map Operator Tree:
+ TableScan
+ alias: y
+ Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
+ Execution mode: llap
+ LLAP IO: no inputs
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: EXPLAIN
+SELECT x.key, x.value
+FROM src x JOIN src y ON (x.key = y.key AND x.value = y.value)
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN
+SELECT x.key, x.value
+FROM src x JOIN src y ON (x.key = y.key AND x.value = y.value)
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Map 1 <- Map 2 (BROADCAST_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: x
+ Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
+ Filter Operator
+ predicate: (key is not null and value is not null) (type: boolean)
+ Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: string), _col1 (type: string)
+ 1 _col0 (type: string), _col1 (type: string)
+ outputColumnNames: _col0, _col1
+ input vertices:
+ 1 Map 2
+ Statistics: Num rows: 5 Data size: 890 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 5 Data size: 890 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: llap
+ LLAP IO: no inputs
+ Map 2
+ Map Operator Tree:
+ TableScan
+ alias: y
+ Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
+ Filter Operator
+ predicate: (key is not null and value is not null) (type: boolean)
+ Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string), _col1 (type: string)
+ sort order: ++
+ Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
+ Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
+ Execution mode: llap
+ LLAP IO: no inputs
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: EXPLAIN
+SELECT x.key, x.value
+FROM src x JOIN src y ON (x.key = y.key)
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN
+SELECT x.key, x.value
+FROM src x JOIN src y ON (x.key = y.key)
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Map 1 <- Map 2 (BROADCAST_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: x
+ Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col0, _col1
+ input vertices:
+ 1 Map 2
+ Statistics: Num rows: 1219 Data size: 216982 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1219 Data size: 216982 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: llap
+ LLAP IO: no inputs
+ Map 2
+ Map Operator Tree:
+ TableScan
+ alias: y
+ Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
+ Execution mode: llap
+ LLAP IO: no inputs
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: EXPLAIN
+SELECT x.key, x.value
+FROM src x JOIN src y ON (x.key = y.key AND x.value = y.value)
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN
+SELECT x.key, x.value
+FROM src x JOIN src y ON (x.key = y.key AND x.value = y.value)
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: x
+ Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
+ Filter Operator
+ predicate: (key is not null and value is not null) (type: boolean)
+ Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string), _col1 (type: string)
+ sort order: ++
+ Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
+ Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
+ Execution mode: llap
+ LLAP IO: no inputs
+ Map 3
+ Map Operator Tree:
+ TableScan
+ alias: y
+ Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
+ Filter Operator
+ predicate: (key is not null and value is not null) (type: boolean)
+ Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string), _col1 (type: string)
+ sort order: ++
+ Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
+ Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
+ Execution mode: llap
+ LLAP IO: no inputs
+ Reducer 2
+ Execution mode: llap
+ Reduce Operator Tree:
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string)
+ 1 KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string)
+ outputColumnNames: _col0, _col1
+ input vertices:
+ 1 Map 3
+ Statistics: Num rows: 5 Data size: 890 Basic stats: COMPLETE Column stats: COMPLETE
+ HybridGraceHashJoin: true
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 5 Data size: 890 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: EXPLAIN
+SELECT x.key, x.value
+FROM src x JOIN src y ON (x.key = y.key)
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN
+SELECT x.key, x.value
+FROM src x JOIN src y ON (x.key = y.key)
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: x
+ Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col1 (type: string)
+ Execution mode: llap
+ LLAP IO: no inputs
+ Map 3
+ Map Operator Tree:
+ TableScan
+ alias: y
+ Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
+ Execution mode: llap
+ LLAP IO: no inputs
+ Reducer 2
+ Execution mode: llap
+ Reduce Operator Tree:
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 KEY.reducesinkkey0 (type: string)
+ 1 KEY.reducesinkkey0 (type: string)
+ outputColumnNames: _col0, _col1
+ input vertices:
+ 1 Map 3
+ Statistics: Num rows: 1219 Data size: 216982 Basic stats: COMPLETE Column stats: COMPLETE
+ HybridGraceHashJoin: true
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1219 Data size: 216982 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: EXPLAIN
+SELECT x.key, x.value
+FROM src x JOIN src y ON (x.key = y.key AND x.value = y.value)
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN
+SELECT x.key, x.value
+FROM src x JOIN src y ON (x.key = y.key AND x.value = y.value)
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: x
+ Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
+ Filter Operator
+ predicate: (key is not null and value is not null) (type: boolean)
+ Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string), _col1 (type: string)
+ sort order: ++
+ Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
+ Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
+ Execution mode: llap
+ LLAP IO: no inputs
+ Map 3
+ Map Operator Tree:
+ TableScan
+ alias: y
+ Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
+ Filter Operator
+ predicate: (key is not null and value is not null) (type: boolean)
+ Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string), _col1 (type: string)
+ sort order: ++
+ Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
+ Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
+ Execution mode: llap
+ LLAP IO: no inputs
+ Reducer 2
+ Execution mode: llap
+ Reduce Operator Tree:
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string)
+ 1 KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string)
+ outputColumnNames: _col0, _col1
+ input vertices:
+ 1 Map 3
+ Statistics: Num rows: 5 Data size: 890 Basic stats: COMPLETE Column stats: COMPLETE
+ HybridGraceHashJoin: true
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 5 Data size: 890 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
[21/50] [abbrv] hive git commit: HIVE-14362: Support explain analyze
in Hive (Pengcheng Xiong, reviewed by Ashutosh Chauhan,
Gopal V and Gabor Szadovszky)
Posted by om...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/fd72ec4a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java.orig
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java.orig b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java.orig
new file mode 100644
index 0000000..699bb11
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java.orig
@@ -0,0 +1,13038 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.parse;
+
+import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.HIVESTATSDBCLASS;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.Serializable;
+import java.security.AccessControlException;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Deque;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Queue;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.UUID;
+import java.util.regex.Pattern;
+import java.util.regex.PatternSyntaxException;
+
+import org.antlr.runtime.ClassicToken;
+import org.antlr.runtime.CommonToken;
+import org.antlr.runtime.Token;
+import org.antlr.runtime.tree.Tree;
+import org.antlr.runtime.tree.TreeWizard;
+import org.antlr.runtime.tree.TreeWizard.ContextVisitor;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.hive.common.BlobStorageUtils;
+import org.apache.hadoop.hive.ql.plan.AlterTableDesc;
+import org.apache.hadoop.hive.common.FileUtils;
+import org.apache.hadoop.hive.common.ObjectPair;
+import org.apache.hadoop.hive.common.StatsSetupConst;
+import org.apache.hadoop.hive.common.StatsSetupConst.StatDB;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.conf.HiveConf.StrictChecks;
+import org.apache.hadoop.hive.metastore.MetaStoreUtils;
+import org.apache.hadoop.hive.metastore.TableType;
+import org.apache.hadoop.hive.metastore.Warehouse;
+import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.Order;
+import org.apache.hadoop.hive.metastore.api.SQLForeignKey;
+import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey;
+import org.apache.hadoop.hive.ql.CompilationOpContext;
+import org.apache.hadoop.hive.ql.ErrorMsg;
+import org.apache.hadoop.hive.ql.QueryProperties;
+import org.apache.hadoop.hive.ql.QueryState;
+import org.apache.hadoop.hive.ql.exec.AbstractMapJoinOperator;
+import org.apache.hadoop.hive.ql.exec.ArchiveUtils;
+import org.apache.hadoop.hive.ql.exec.ColumnInfo;
+import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluatorFactory;
+import org.apache.hadoop.hive.ql.exec.FetchTask;
+import org.apache.hadoop.hive.ql.exec.FileSinkOperator;
+import org.apache.hadoop.hive.ql.exec.FilterOperator;
+import org.apache.hadoop.hive.ql.exec.FunctionInfo;
+import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
+import org.apache.hadoop.hive.ql.exec.GroupByOperator;
+import org.apache.hadoop.hive.ql.exec.JoinOperator;
+import org.apache.hadoop.hive.ql.exec.Operator;
+import org.apache.hadoop.hive.ql.exec.OperatorFactory;
+import org.apache.hadoop.hive.ql.exec.RecordReader;
+import org.apache.hadoop.hive.ql.exec.RecordWriter;
+import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator;
+import org.apache.hadoop.hive.ql.exec.RowSchema;
+import org.apache.hadoop.hive.ql.exec.SMBMapJoinOperator;
+import org.apache.hadoop.hive.ql.exec.SelectOperator;
+import org.apache.hadoop.hive.ql.exec.TableScanOperator;
+import org.apache.hadoop.hive.ql.exec.Task;
+import org.apache.hadoop.hive.ql.exec.TaskFactory;
+import org.apache.hadoop.hive.ql.exec.UnionOperator;
+import org.apache.hadoop.hive.ql.exec.Utilities;
+import org.apache.hadoop.hive.ql.hooks.ReadEntity;
+import org.apache.hadoop.hive.ql.hooks.WriteEntity;
+import org.apache.hadoop.hive.ql.io.AcidOutputFormat;
+import org.apache.hadoop.hive.ql.io.AcidUtils;
+import org.apache.hadoop.hive.ql.io.AcidUtils.Operation;
+import org.apache.hadoop.hive.ql.io.CombineHiveInputFormat;
+import org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat;
+import org.apache.hadoop.hive.ql.io.HiveOutputFormat;
+import org.apache.hadoop.hive.ql.io.NullRowsInputFormat;
+import org.apache.hadoop.hive.ql.io.RCFileInputFormat;
+import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat;
+import org.apache.hadoop.hive.ql.lib.DefaultGraphWalker;
+import org.apache.hadoop.hive.ql.lib.Dispatcher;
+import org.apache.hadoop.hive.ql.lib.GraphWalker;
+import org.apache.hadoop.hive.ql.lib.Node;
+import org.apache.hadoop.hive.ql.metadata.DummyPartition;
+import org.apache.hadoop.hive.ql.metadata.Hive;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.metadata.HiveUtils;
+import org.apache.hadoop.hive.ql.metadata.InvalidTableException;
+import org.apache.hadoop.hive.ql.metadata.Partition;
+import org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient;
+import org.apache.hadoop.hive.ql.metadata.Table;
+import org.apache.hadoop.hive.ql.metadata.VirtualColumn;
+import org.apache.hadoop.hive.ql.optimizer.Optimizer;
+import org.apache.hadoop.hive.ql.optimizer.Transform;
+import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException;
+import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException.UnsupportedFeature;
+import org.apache.hadoop.hive.ql.optimizer.calcite.translator.HiveOpConverterPostProc;
+import org.apache.hadoop.hive.ql.optimizer.lineage.Generator;
+import org.apache.hadoop.hive.ql.optimizer.unionproc.UnionProcContext;
+import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.TableSpec.SpecType;
+import org.apache.hadoop.hive.ql.parse.CalcitePlanner.ASTSearcher;
+import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.OrderExpression;
+import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.OrderSpec;
+import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.PTFInputSpec;
+import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.PTFQueryInputSpec;
+import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.PTFQueryInputType;
+import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.PartitionExpression;
+import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.PartitionSpec;
+import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.PartitionedTableFunctionSpec;
+import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.PartitioningSpec;
+import org.apache.hadoop.hive.ql.parse.QBSubQuery.SubQueryType;
+import org.apache.hadoop.hive.ql.parse.SubQueryUtils.ISubQueryJoinInfo;
+import org.apache.hadoop.hive.ql.parse.WindowingSpec.BoundarySpec;
+import org.apache.hadoop.hive.ql.parse.WindowingSpec.CurrentRowSpec;
+import org.apache.hadoop.hive.ql.parse.WindowingSpec.Direction;
+import org.apache.hadoop.hive.ql.parse.WindowingSpec.RangeBoundarySpec;
+import org.apache.hadoop.hive.ql.parse.WindowingSpec.ValueBoundarySpec;
+import org.apache.hadoop.hive.ql.parse.WindowingSpec.WindowExpressionSpec;
+import org.apache.hadoop.hive.ql.parse.WindowingSpec.WindowFrameSpec;
+import org.apache.hadoop.hive.ql.parse.WindowingSpec.WindowFunctionSpec;
+import org.apache.hadoop.hive.ql.parse.WindowingSpec.WindowSpec;
+import org.apache.hadoop.hive.ql.plan.AggregationDesc;
+import org.apache.hadoop.hive.ql.plan.AlterTableDesc.AlterTableTypes;
+import org.apache.hadoop.hive.ql.plan.CreateTableDesc;
+import org.apache.hadoop.hive.ql.plan.CreateTableLikeDesc;
+import org.apache.hadoop.hive.ql.plan.CreateViewDesc;
+import org.apache.hadoop.hive.ql.plan.DDLWork;
+import org.apache.hadoop.hive.ql.plan.DynamicPartitionCtx;
+import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeColumnListDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeDescUtils;
+import org.apache.hadoop.hive.ql.plan.ExprNodeFieldDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
+import org.apache.hadoop.hive.ql.plan.FileSinkDesc;
+import org.apache.hadoop.hive.ql.plan.FilterDesc;
+import org.apache.hadoop.hive.ql.plan.FilterDesc.SampleDesc;
+import org.apache.hadoop.hive.ql.plan.ForwardDesc;
+import org.apache.hadoop.hive.ql.plan.GroupByDesc;
+import org.apache.hadoop.hive.ql.plan.HiveOperation;
+import org.apache.hadoop.hive.ql.plan.JoinCondDesc;
+import org.apache.hadoop.hive.ql.plan.JoinDesc;
+import org.apache.hadoop.hive.ql.plan.LateralViewForwardDesc;
+import org.apache.hadoop.hive.ql.plan.LateralViewJoinDesc;
+import org.apache.hadoop.hive.ql.plan.LimitDesc;
+import org.apache.hadoop.hive.ql.plan.ListBucketingCtx;
+import org.apache.hadoop.hive.ql.plan.LoadFileDesc;
+import org.apache.hadoop.hive.ql.plan.LoadTableDesc;
+import org.apache.hadoop.hive.ql.plan.MapJoinDesc;
+import org.apache.hadoop.hive.ql.plan.OperatorDesc;
+import org.apache.hadoop.hive.ql.plan.PTFDesc;
+import org.apache.hadoop.hive.ql.plan.PlanUtils;
+import org.apache.hadoop.hive.ql.plan.ReduceSinkDesc;
+import org.apache.hadoop.hive.ql.plan.ScriptDesc;
+import org.apache.hadoop.hive.ql.plan.SelectDesc;
+import org.apache.hadoop.hive.ql.plan.TableDesc;
+import org.apache.hadoop.hive.ql.plan.TableScanDesc;
+import org.apache.hadoop.hive.ql.plan.UDTFDesc;
+import org.apache.hadoop.hive.ql.plan.UnionDesc;
+import org.apache.hadoop.hive.ql.plan.ptf.OrderExpressionDef;
+import org.apache.hadoop.hive.ql.plan.ptf.PTFExpressionDef;
+import org.apache.hadoop.hive.ql.plan.ptf.PartitionedTableFunctionDef;
+import org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrivilegeObject;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hadoop.hive.ql.session.SessionState.ResourceType;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.Mode;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFHash;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPOr;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDTF;
+import org.apache.hadoop.hive.ql.util.ResourceDownloader;
+import org.apache.hadoop.hive.serde.serdeConstants;
+import org.apache.hadoop.hive.serde2.Deserializer;
+import org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe;
+import org.apache.hadoop.hive.serde2.NoOpFetchFormatter;
+import org.apache.hadoop.hive.serde2.NullStructSerDe;
+import org.apache.hadoop.hive.serde2.SerDeException;
+import org.apache.hadoop.hive.serde2.SerDeUtils;
+import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe;
+import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.StandardStructObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.StructField;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.hive.serde2.thrift.ThriftJDBCBinarySerDe;
+import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
+import org.apache.hadoop.hive.shims.HadoopShims;
+import org.apache.hadoop.hive.shims.Utils;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.InputFormat;
+import org.apache.hadoop.mapred.OutputFormat;
+import org.apache.hadoop.security.UserGroupInformation;
+
+import com.google.common.collect.Sets;
+
+/**
+ * Implementation of the semantic analyzer. It generates the query plan.
+ * There are other specific semantic analyzers for some hive operations such as
+ * DDLSemanticAnalyzer for ddl operations.
+ */
+
+public class SemanticAnalyzer extends BaseSemanticAnalyzer {
+
+ public static final String DUMMY_DATABASE = "_dummy_database";
+ public static final String DUMMY_TABLE = "_dummy_table";
+ public static final String SUBQUERY_TAG_1 = "-subquery1";
+ public static final String SUBQUERY_TAG_2 = "-subquery2";
+
+ // Max characters when auto generating the column name with func name
+ private static final int AUTOGEN_COLALIAS_PRFX_MAXLENGTH = 20;
+
+ private static final String VALUES_TMP_TABLE_NAME_PREFIX = "Values__Tmp__Table__";
+
+ static final String MATERIALIZATION_MARKER = "$MATERIALIZATION";
+
+ private HashMap<TableScanOperator, ExprNodeDesc> opToPartPruner;
+ private HashMap<TableScanOperator, PrunedPartitionList> opToPartList;
+ protected HashMap<String, TableScanOperator> topOps;
+ protected LinkedHashMap<Operator<? extends OperatorDesc>, OpParseContext> opParseCtx;
+ private List<LoadTableDesc> loadTableWork;
+ private List<LoadFileDesc> loadFileWork;
+ private List<ColumnStatsAutoGatherContext> columnStatsAutoGatherContexts;
+ private final Map<JoinOperator, QBJoinTree> joinContext;
+ private final Map<SMBMapJoinOperator, QBJoinTree> smbMapJoinContext;
+ private final HashMap<TableScanOperator, Table> topToTable;
+ private final Map<FileSinkOperator, Table> fsopToTable;
+ private final List<ReduceSinkOperator> reduceSinkOperatorsAddedByEnforceBucketingSorting;
+ private final HashMap<TableScanOperator, Map<String, String>> topToTableProps;
+ private QB qb;
+ private ASTNode ast;
+ private int destTableId;
+ private UnionProcContext uCtx;
+ List<AbstractMapJoinOperator<? extends MapJoinDesc>> listMapJoinOpsNoReducer;
+ private HashMap<TableScanOperator, SampleDesc> opToSamplePruner;
+ private final Map<TableScanOperator, Map<String, ExprNodeDesc>> opToPartToSkewedPruner;
+ private Map<SelectOperator, Table> viewProjectToTableSchema;
+ /**
+ * a map for the split sampling, from alias to an instance of SplitSample
+ * that describes percentage and number.
+ */
+ private final HashMap<String, SplitSample> nameToSplitSample;
+ Map<GroupByOperator, Set<String>> groupOpToInputTables;
+ Map<String, PrunedPartitionList> prunedPartitions;
+ protected List<FieldSchema> resultSchema;
+ private CreateViewDesc createVwDesc;
+ private ArrayList<String> viewsExpanded;
+ private ASTNode viewSelect;
+ protected final UnparseTranslator unparseTranslator;
+ private final GlobalLimitCtx globalLimitCtx;
+
+ // prefix for column names auto generated by hive
+ private final String autogenColAliasPrfxLbl;
+ private final boolean autogenColAliasPrfxIncludeFuncName;
+
+ // Keep track of view alias to read entity corresponding to the view
+ // For eg: for a query like 'select * from V3', where V3 -> V2, V2 -> V1, V1 -> T
+ // keeps track of aliases for V3, V3:V2, V3:V2:V1.
+ // This is used when T is added as an input for the query, the parents of T is
+ // derived from the alias V3:V2:V1:T
+ private final Map<String, ReadEntity> viewAliasToInput;
+
+ //need merge isDirect flag to input even if the newInput does not have a parent
+ private boolean mergeIsDirect;
+
+ // flag for no scan during analyze ... compute statistics
+ protected boolean noscan;
+
+ //flag for partial scan during analyze ... compute statistics
+ protected boolean partialscan;
+
+ protected volatile boolean disableJoinMerge = false;
+
+ /*
+ * Capture the CTE definitions in a Query.
+ */
+ final Map<String, CTEClause> aliasToCTEs;
+
+ /*
+ * Used to check recursive CTE invocations. Similar to viewsExpanded
+ */
+ ArrayList<String> ctesExpanded;
+
+ /*
+ * Whether root tasks after materialized CTE linkage have been resolved
+ */
+ boolean rootTasksResolved;
+
+ protected TableMask tableMask;
+
+ CreateTableDesc tableDesc;
+
+ /** Not thread-safe. */
+ final ASTSearcher astSearcher = new ASTSearcher();
+
+ protected AnalyzeRewriteContext analyzeRewrite;
+
+ // A mapping from a tableName to a table object in metastore.
+ Map<String, Table> tabNameToTabObject;
+
+ // The tokens we should ignore when we are trying to do table masking.
+ private final Set<Integer> ignoredTokens = Sets.newHashSet(HiveParser.TOK_GROUPBY,
+ HiveParser.TOK_ORDERBY, HiveParser.TOK_WINDOWSPEC, HiveParser.TOK_CLUSTERBY,
+ HiveParser.TOK_DISTRIBUTEBY, HiveParser.TOK_SORTBY);
+
+ static class Phase1Ctx {
+ String dest;
+ int nextNum;
+ }
+
+ public SemanticAnalyzer(QueryState queryState) throws SemanticException {
+ super(queryState);
+ opToPartPruner = new HashMap<TableScanOperator, ExprNodeDesc>();
+ opToPartList = new HashMap<TableScanOperator, PrunedPartitionList>();
+ opToSamplePruner = new HashMap<TableScanOperator, SampleDesc>();
+ nameToSplitSample = new HashMap<String, SplitSample>();
+ // Must be deterministic order maps - see HIVE-8707
+ topOps = new LinkedHashMap<String, TableScanOperator>();
+ loadTableWork = new ArrayList<LoadTableDesc>();
+ loadFileWork = new ArrayList<LoadFileDesc>();
+ columnStatsAutoGatherContexts = new ArrayList<ColumnStatsAutoGatherContext>();
+ opParseCtx = new LinkedHashMap<Operator<? extends OperatorDesc>, OpParseContext>();
+ joinContext = new HashMap<JoinOperator, QBJoinTree>();
+ smbMapJoinContext = new HashMap<SMBMapJoinOperator, QBJoinTree>();
+ // Must be deterministic order map for consistent q-test output across Java versions
+ topToTable = new LinkedHashMap<TableScanOperator, Table>();
+ fsopToTable = new HashMap<FileSinkOperator, Table>();
+ reduceSinkOperatorsAddedByEnforceBucketingSorting = new ArrayList<ReduceSinkOperator>();
+ topToTableProps = new HashMap<TableScanOperator, Map<String, String>>();
+ destTableId = 1;
+ uCtx = null;
+ listMapJoinOpsNoReducer = new ArrayList<AbstractMapJoinOperator<? extends MapJoinDesc>>();
+ groupOpToInputTables = new HashMap<GroupByOperator, Set<String>>();
+ prunedPartitions = new HashMap<String, PrunedPartitionList>();
+ tabNameToTabObject = new HashMap<String, Table>();
+ unparseTranslator = new UnparseTranslator(conf);
+ autogenColAliasPrfxLbl = HiveConf.getVar(conf,
+ HiveConf.ConfVars.HIVE_AUTOGEN_COLUMNALIAS_PREFIX_LABEL);
+ autogenColAliasPrfxIncludeFuncName = HiveConf.getBoolVar(conf,
+ HiveConf.ConfVars.HIVE_AUTOGEN_COLUMNALIAS_PREFIX_INCLUDEFUNCNAME);
+ queryProperties = new QueryProperties();
+ opToPartToSkewedPruner = new HashMap<TableScanOperator, Map<String, ExprNodeDesc>>();
+ aliasToCTEs = new HashMap<String, CTEClause>();
+ globalLimitCtx = new GlobalLimitCtx();
+ viewAliasToInput = new HashMap<String, ReadEntity>();
+ mergeIsDirect = true;
+ noscan = partialscan = false;
+ tabNameToTabObject = new HashMap<>();
+ }
+
+ @Override
+ protected void reset(boolean clearPartsCache) {
+ super.reset(true);
+ if(clearPartsCache) {
+ prunedPartitions.clear();
+
+ //When init(true) combine with genResolvedParseTree, it will generate Resolved Parse tree from syntax tree
+ //ReadEntity created under these conditions should be all relevant to the syntax tree even the ones without parents
+ //set mergeIsDirect to true here.
+ mergeIsDirect = true;
+ } else {
+ mergeIsDirect = false;
+ }
+ tabNameToTabObject.clear();
+ loadTableWork.clear();
+ loadFileWork.clear();
+ columnStatsAutoGatherContexts.clear();
+ topOps.clear();
+ destTableId = 1;
+ idToTableNameMap.clear();
+ qb = null;
+ ast = null;
+ uCtx = null;
+ joinContext.clear();
+ smbMapJoinContext.clear();
+ opParseCtx.clear();
+ groupOpToInputTables.clear();
+ disableJoinMerge = false;
+ aliasToCTEs.clear();
+ topToTable.clear();
+ opToPartPruner.clear();
+ opToPartList.clear();
+ opToPartToSkewedPruner.clear();
+ opToSamplePruner.clear();
+ nameToSplitSample.clear();
+ fsopToTable.clear();
+ resultSchema = null;
+ createVwDesc = null;
+ viewsExpanded = null;
+ viewSelect = null;
+ ctesExpanded = null;
+ globalLimitCtx.disableOpt();
+ viewAliasToInput.clear();
+ reduceSinkOperatorsAddedByEnforceBucketingSorting.clear();
+ topToTableProps.clear();
+ listMapJoinOpsNoReducer.clear();
+ unparseTranslator.clear();
+ queryProperties.clear();
+ outputs.clear();
+ }
+
+ public void initParseCtx(ParseContext pctx) {
+ opToPartPruner = pctx.getOpToPartPruner();
+ opToPartList = pctx.getOpToPartList();
+ opToSamplePruner = pctx.getOpToSamplePruner();
+ topOps = pctx.getTopOps();
+ loadTableWork = pctx.getLoadTableWork();
+ loadFileWork = pctx.getLoadFileWork();
+ ctx = pctx.getContext();
+ destTableId = pctx.getDestTableId();
+ idToTableNameMap = pctx.getIdToTableNameMap();
+ uCtx = pctx.getUCtx();
+ listMapJoinOpsNoReducer = pctx.getListMapJoinOpsNoReducer();
+ prunedPartitions = pctx.getPrunedPartitions();
+ tabNameToTabObject = pctx.getTabNameToTabObject();
+ fetchTask = pctx.getFetchTask();
+ setLineageInfo(pctx.getLineageInfo());
+ }
+
+ public ParseContext getParseContext() {
+ // Make sure the basic query properties are initialized
+ copyInfoToQueryProperties(queryProperties);
+ return new ParseContext(queryState, opToPartPruner, opToPartList, topOps,
+ new HashSet<JoinOperator>(joinContext.keySet()),
+ new HashSet<SMBMapJoinOperator>(smbMapJoinContext.keySet()),
+ loadTableWork, loadFileWork, columnStatsAutoGatherContexts, ctx, idToTableNameMap, destTableId, uCtx,
+ listMapJoinOpsNoReducer, prunedPartitions, tabNameToTabObject,
+ opToSamplePruner, globalLimitCtx, nameToSplitSample, inputs, rootTasks,
+ opToPartToSkewedPruner, viewAliasToInput, reduceSinkOperatorsAddedByEnforceBucketingSorting,
+ analyzeRewrite, tableDesc, queryProperties, viewProjectToTableSchema, acidFileSinks);
+ }
+
+ public CompilationOpContext getOpContext() {
+ return ctx.getOpContext();
+ }
+
+ public void doPhase1QBExpr(ASTNode ast, QBExpr qbexpr, String id, String alias)
+ throws SemanticException {
+ doPhase1QBExpr(ast, qbexpr, id, alias, false);
+ }
+ @SuppressWarnings("nls")
+ public void doPhase1QBExpr(ASTNode ast, QBExpr qbexpr, String id, String alias, boolean insideView)
+ throws SemanticException {
+
+ assert (ast.getToken() != null);
+ switch (ast.getToken().getType()) {
+ case HiveParser.TOK_QUERY: {
+ QB qb = new QB(id, alias, true);
+ qb.setInsideView(insideView);
+ Phase1Ctx ctx_1 = initPhase1Ctx();
+ doPhase1(ast, qb, ctx_1, null);
+
+ qbexpr.setOpcode(QBExpr.Opcode.NULLOP);
+ qbexpr.setQB(qb);
+ }
+ break;
+ case HiveParser.TOK_UNIONALL: {
+ qbexpr.setOpcode(QBExpr.Opcode.UNION);
+ // query 1
+ assert (ast.getChild(0) != null);
+ QBExpr qbexpr1 = new QBExpr(alias + SUBQUERY_TAG_1);
+ doPhase1QBExpr((ASTNode) ast.getChild(0), qbexpr1, id + SUBQUERY_TAG_1,
+ alias + SUBQUERY_TAG_1, insideView);
+ qbexpr.setQBExpr1(qbexpr1);
+
+ // query 2
+ assert (ast.getChild(1) != null);
+ QBExpr qbexpr2 = new QBExpr(alias + SUBQUERY_TAG_2);
+ doPhase1QBExpr((ASTNode) ast.getChild(1), qbexpr2, id + SUBQUERY_TAG_2,
+ alias + SUBQUERY_TAG_2, insideView);
+ qbexpr.setQBExpr2(qbexpr2);
+ }
+ break;
+ }
+ }
+
+ private LinkedHashMap<String, ASTNode> doPhase1GetAggregationsFromSelect(
+ ASTNode selExpr, QB qb, String dest) throws SemanticException {
+
+ // Iterate over the selects search for aggregation Trees.
+ // Use String as keys to eliminate duplicate trees.
+ LinkedHashMap<String, ASTNode> aggregationTrees = new LinkedHashMap<String, ASTNode>();
+ List<ASTNode> wdwFns = new ArrayList<ASTNode>();
+ for (int i = 0; i < selExpr.getChildCount(); ++i) {
+ ASTNode function = (ASTNode) selExpr.getChild(i);
+ if (function.getType() == HiveParser.TOK_SELEXPR ||
+ function.getType() == HiveParser.TOK_SUBQUERY_EXPR) {
+ function = (ASTNode)function.getChild(0);
+ }
+ doPhase1GetAllAggregations(function, aggregationTrees, wdwFns);
+ }
+
+ // window based aggregations are handled differently
+ for (ASTNode wdwFn : wdwFns) {
+ WindowingSpec spec = qb.getWindowingSpec(dest);
+ if(spec == null) {
+ queryProperties.setHasWindowing(true);
+ spec = new WindowingSpec();
+ qb.addDestToWindowingSpec(dest, spec);
+ }
+ HashMap<String, ASTNode> wExprsInDest = qb.getParseInfo().getWindowingExprsForClause(dest);
+ int wColIdx = spec.getWindowExpressions() == null ? 0 : spec.getWindowExpressions().size();
+ WindowFunctionSpec wFnSpec = processWindowFunction(wdwFn,
+ (ASTNode)wdwFn.getChild(wdwFn.getChildCount()-1));
+ // If this is a duplicate invocation of a function; don't add to WindowingSpec.
+ if ( wExprsInDest != null &&
+ wExprsInDest.containsKey(wFnSpec.getExpression().toStringTree())) {
+ continue;
+ }
+ wFnSpec.setAlias(wFnSpec.getName() + "_window_" + wColIdx);
+ spec.addWindowFunction(wFnSpec);
+ qb.getParseInfo().addWindowingExprToClause(dest, wFnSpec.getExpression());
+ }
+
+ return aggregationTrees;
+ }
+
+ private void doPhase1GetColumnAliasesFromSelect(
+ ASTNode selectExpr, QBParseInfo qbp) {
+ for (int i = 0; i < selectExpr.getChildCount(); ++i) {
+ ASTNode selExpr = (ASTNode) selectExpr.getChild(i);
+ if ((selExpr.getToken().getType() == HiveParser.TOK_SELEXPR)
+ && (selExpr.getChildCount() == 2)) {
+ String columnAlias = unescapeIdentifier(selExpr.getChild(1).getText());
+ qbp.setExprToColumnAlias((ASTNode) selExpr.getChild(0), columnAlias);
+ }
+ }
+ }
+
+ /**
+ * DFS-scan the expressionTree to find all aggregation subtrees and put them
+ * in aggregations.
+ *
+ * @param expressionTree
+ * @param aggregations
+ * the key to the HashTable is the toStringTree() representation of
+ * the aggregation subtree.
+ * @throws SemanticException
+ */
+ private void doPhase1GetAllAggregations(ASTNode expressionTree,
+ HashMap<String, ASTNode> aggregations, List<ASTNode> wdwFns) throws SemanticException {
+ int exprTokenType = expressionTree.getToken().getType();
+ if (exprTokenType == HiveParser.TOK_FUNCTION
+ || exprTokenType == HiveParser.TOK_FUNCTIONDI
+ || exprTokenType == HiveParser.TOK_FUNCTIONSTAR) {
+ assert (expressionTree.getChildCount() != 0);
+ if (expressionTree.getChild(expressionTree.getChildCount()-1).getType()
+ == HiveParser.TOK_WINDOWSPEC) {
+ // If it is a windowing spec, we include it in the list
+ // Further, we will examine its children AST nodes to check whether
+ // there are aggregation functions within
+ wdwFns.add(expressionTree);
+ doPhase1GetAllAggregations((ASTNode) expressionTree.getChild(expressionTree.getChildCount()-1),
+ aggregations, wdwFns);
+ return;
+ }
+ if (expressionTree.getChild(0).getType() == HiveParser.Identifier) {
+ String functionName = unescapeIdentifier(expressionTree.getChild(0)
+ .getText());
+ // Validate the function name
+ if (FunctionRegistry.getFunctionInfo(functionName) == null) {
+ throw new SemanticException(ErrorMsg.INVALID_FUNCTION.getMsg(functionName));
+ }
+ if(FunctionRegistry.impliesOrder(functionName)) {
+ throw new SemanticException(ErrorMsg.MISSING_OVER_CLAUSE.getMsg(functionName));
+ }
+ if (FunctionRegistry.getGenericUDAFResolver(functionName) != null) {
+ if(containsLeadLagUDF(expressionTree)) {
+ throw new SemanticException(ErrorMsg.MISSING_OVER_CLAUSE.getMsg(functionName));
+ }
+ aggregations.put(expressionTree.toStringTree(), expressionTree);
+ FunctionInfo fi = FunctionRegistry.getFunctionInfo(functionName);
+ if (!fi.isNative()) {
+ unparseTranslator.addIdentifierTranslation((ASTNode) expressionTree
+ .getChild(0));
+ }
+ return;
+ }
+ }
+ }
+ for (int i = 0; i < expressionTree.getChildCount(); i++) {
+ doPhase1GetAllAggregations((ASTNode) expressionTree.getChild(i),
+ aggregations, wdwFns);
+ }
+ }
+
+ private List<ASTNode> doPhase1GetDistinctFuncExprs(
+ HashMap<String, ASTNode> aggregationTrees) throws SemanticException {
+ List<ASTNode> exprs = new ArrayList<ASTNode>();
+ for (Map.Entry<String, ASTNode> entry : aggregationTrees.entrySet()) {
+ ASTNode value = entry.getValue();
+ assert (value != null);
+ if (value.getToken().getType() == HiveParser.TOK_FUNCTIONDI) {
+ exprs.add(value);
+ }
+ }
+ return exprs;
+ }
+
+ public static String generateErrorMessage(ASTNode ast, String message) {
+ StringBuilder sb = new StringBuilder();
+ if (ast == null) {
+ sb.append(message).append(". Cannot tell the position of null AST.");
+ return sb.toString();
+ }
+ sb.append(ast.getLine());
+ sb.append(":");
+ sb.append(ast.getCharPositionInLine());
+ sb.append(" ");
+ sb.append(message);
+ sb.append(". Error encountered near token '");
+ sb.append(ErrorMsg.getText(ast));
+ sb.append("'");
+ return sb.toString();
+ }
+
+ ASTNode getAST() {
+ return this.ast;
+ }
+
+ protected void setAST(ASTNode newAST) {
+ this.ast = newAST;
+ }
+
+ /**
+ * Goes though the tabref tree and finds the alias for the table. Once found,
+ * it records the table name-> alias association in aliasToTabs. It also makes
+ * an association from the alias to the table AST in parse info.
+ *
+ * @return the alias of the table
+ */
+ private String processTable(QB qb, ASTNode tabref) throws SemanticException {
+ // For each table reference get the table name
+ // and the alias (if alias is not present, the table name
+ // is used as an alias)
+ int aliasIndex = 0;
+ int propsIndex = -1;
+ int tsampleIndex = -1;
+ int ssampleIndex = -1;
+ for (int index = 1; index < tabref.getChildCount(); index++) {
+ ASTNode ct = (ASTNode) tabref.getChild(index);
+ if (ct.getToken().getType() == HiveParser.TOK_TABLEBUCKETSAMPLE) {
+ tsampleIndex = index;
+ } else if (ct.getToken().getType() == HiveParser.TOK_TABLESPLITSAMPLE) {
+ ssampleIndex = index;
+ } else if (ct.getToken().getType() == HiveParser.TOK_TABLEPROPERTIES) {
+ propsIndex = index;
+ } else {
+ aliasIndex = index;
+ }
+ }
+
+ ASTNode tableTree = (ASTNode) (tabref.getChild(0));
+
+ String tabIdName = getUnescapedName(tableTree).toLowerCase();
+
+ String alias;
+ if (aliasIndex != 0) {
+ alias = unescapeIdentifier(tabref.getChild(aliasIndex).getText());
+ }
+ else {
+ alias = getUnescapedUnqualifiedTableName(tableTree);
+ }
+
+ if (propsIndex >= 0) {
+ Tree propsAST = tabref.getChild(propsIndex);
+ Map<String, String> props = DDLSemanticAnalyzer.getProps((ASTNode) propsAST.getChild(0));
+ // We get the information from Calcite.
+ if ("TRUE".equals(props.get("insideView"))) {
+ qb.getAliasInsideView().add(alias.toLowerCase());
+ }
+ qb.setTabProps(alias, props);
+ }
+
+ // If the alias is already there then we have a conflict
+ if (qb.exists(alias)) {
+ throw new SemanticException(ErrorMsg.AMBIGUOUS_TABLE_ALIAS.getMsg(tabref
+ .getChild(aliasIndex)));
+ }
+ if (tsampleIndex >= 0) {
+ ASTNode sampleClause = (ASTNode) tabref.getChild(tsampleIndex);
+ ArrayList<ASTNode> sampleCols = new ArrayList<ASTNode>();
+ if (sampleClause.getChildCount() > 2) {
+ for (int i = 2; i < sampleClause.getChildCount(); i++) {
+ sampleCols.add((ASTNode) sampleClause.getChild(i));
+ }
+ }
+ // TODO: For now only support sampling on up to two columns
+ // Need to change it to list of columns
+ if (sampleCols.size() > 2) {
+ throw new SemanticException(generateErrorMessage(
+ (ASTNode) tabref.getChild(0),
+ ErrorMsg.SAMPLE_RESTRICTION.getMsg()));
+ }
+ TableSample tabSample = new TableSample(
+ unescapeIdentifier(sampleClause.getChild(0).getText()),
+ unescapeIdentifier(sampleClause.getChild(1).getText()),
+ sampleCols);
+ qb.getParseInfo().setTabSample(alias, tabSample);
+ if (unparseTranslator.isEnabled()) {
+ for (ASTNode sampleCol : sampleCols) {
+ unparseTranslator.addIdentifierTranslation((ASTNode) sampleCol
+ .getChild(0));
+ }
+ }
+ } else if (ssampleIndex >= 0) {
+ ASTNode sampleClause = (ASTNode) tabref.getChild(ssampleIndex);
+
+ Tree type = sampleClause.getChild(0);
+ Tree numerator = sampleClause.getChild(1);
+ String value = unescapeIdentifier(numerator.getText());
+
+
+ SplitSample sample;
+ if (type.getType() == HiveParser.TOK_PERCENT) {
+ assertCombineInputFormat(numerator, "Percentage");
+ Double percent = Double.valueOf(value).doubleValue();
+ if (percent < 0 || percent > 100) {
+ throw new SemanticException(generateErrorMessage((ASTNode) numerator,
+ "Sampling percentage should be between 0 and 100"));
+ }
+ int seedNum = conf.getIntVar(ConfVars.HIVESAMPLERANDOMNUM);
+ sample = new SplitSample(percent, seedNum);
+ } else if (type.getType() == HiveParser.TOK_ROWCOUNT) {
+ sample = new SplitSample(Integer.parseInt(value));
+ } else {
+ assert type.getType() == HiveParser.TOK_LENGTH;
+ assertCombineInputFormat(numerator, "Total Length");
+ long length = Integer.parseInt(value.substring(0, value.length() - 1));
+ char last = value.charAt(value.length() - 1);
+ if (last == 'k' || last == 'K') {
+ length <<= 10;
+ } else if (last == 'm' || last == 'M') {
+ length <<= 20;
+ } else if (last == 'g' || last == 'G') {
+ length <<= 30;
+ }
+ int seedNum = conf.getIntVar(ConfVars.HIVESAMPLERANDOMNUM);
+ sample = new SplitSample(length, seedNum);
+ }
+ String alias_id = getAliasId(alias, qb);
+ nameToSplitSample.put(alias_id, sample);
+ }
+ // Insert this map into the stats
+ qb.setTabAlias(alias, tabIdName);
+ if (qb.isInsideView()) {
+ qb.getAliasInsideView().add(alias.toLowerCase());
+ }
+ qb.addAlias(alias);
+
+ qb.getParseInfo().setSrcForAlias(alias, tableTree);
+
+ // if alias to CTE contains the alias, we do not do the translation because
+ // cte is actually a subquery.
+ if (!this.aliasToCTEs.containsKey(alias)) {
+ unparseTranslator.addTableNameTranslation(tableTree, SessionState.get().getCurrentDatabase());
+ if (aliasIndex != 0) {
+ unparseTranslator.addIdentifierTranslation((ASTNode) tabref.getChild(aliasIndex));
+ }
+ }
+
+ return alias;
+ }
+
+ Map<String, SplitSample> getNameToSplitSampleMap() {
+ return this.nameToSplitSample;
+ }
+
+ /**
+ * Convert a string to Text format and write its bytes in the same way TextOutputFormat would do.
+ * This is needed to properly encode non-ascii characters.
+ */
+ private static void writeAsText(String text, FSDataOutputStream out) throws IOException {
+ Text to = new Text(text);
+ out.write(to.getBytes(), 0, to.getLength());
+ }
+
+ /**
+ * Generate a temp table out of a value clause
+ * See also {@link #preProcessForInsert(ASTNode, QB)}
+ */
+ private ASTNode genValuesTempTable(ASTNode originalFrom, QB qb) throws SemanticException {
+ Path dataDir = null;
+ if(!qb.getEncryptedTargetTablePaths().isEmpty()) {
+ //currently only Insert into T values(...) is supported thus only 1 values clause
+ //and only 1 target table are possible. If/when support for
+ //select ... from values(...) is added an insert statement may have multiple
+ //encrypted target tables.
+ dataDir = ctx.getMRTmpPath(qb.getEncryptedTargetTablePaths().get(0).toUri());
+ }
+ // Pick a name for the table
+ SessionState ss = SessionState.get();
+ String tableName = VALUES_TMP_TABLE_NAME_PREFIX + ss.getNextValuesTempTableSuffix();
+
+ // Step 1, parse the values clause we were handed
+ List<? extends Node> fromChildren = originalFrom.getChildren();
+ // First child should be the virtual table ref
+ ASTNode virtualTableRef = (ASTNode)fromChildren.get(0);
+ assert virtualTableRef.getToken().getType() == HiveParser.TOK_VIRTUAL_TABREF :
+ "Expected first child of TOK_VIRTUAL_TABLE to be TOK_VIRTUAL_TABREF but was " +
+ virtualTableRef.getName();
+
+ List<? extends Node> virtualTableRefChildren = virtualTableRef.getChildren();
+ // First child of this should be the table name. If it's anonymous,
+ // then we don't have a table name.
+ ASTNode tabName = (ASTNode)virtualTableRefChildren.get(0);
+ if (tabName.getToken().getType() != HiveParser.TOK_ANONYMOUS) {
+ // TODO, if you want to make select ... from (values(...) as foo(...) work,
+ // you need to parse this list of columns names and build it into the table
+ throw new SemanticException(ErrorMsg.VALUES_TABLE_CONSTRUCTOR_NOT_SUPPORTED.getMsg());
+ }
+
+ // The second child of the TOK_VIRTUAL_TABLE should be TOK_VALUES_TABLE
+ ASTNode valuesTable = (ASTNode)fromChildren.get(1);
+ assert valuesTable.getToken().getType() == HiveParser.TOK_VALUES_TABLE :
+ "Expected second child of TOK_VIRTUAL_TABLE to be TOK_VALUE_TABLE but was " +
+ valuesTable.getName();
+ // Each of the children of TOK_VALUES_TABLE will be a TOK_VALUE_ROW
+ List<? extends Node> valuesTableChildren = valuesTable.getChildren();
+
+ // Now that we're going to start reading through the rows, open a file to write the rows too
+ // If we leave this method before creating the temporary table we need to be sure to clean up
+ // this file.
+ Path tablePath = null;
+ FileSystem fs = null;
+ FSDataOutputStream out = null;
+ try {
+ if(dataDir == null) {
+ tablePath = Warehouse.getDnsPath(new Path(ss.getTempTableSpace(), tableName), conf);
+ }
+ else {
+ //if target table of insert is encrypted, make sure temporary table data is stored
+ //similarly encrypted
+ tablePath = Warehouse.getDnsPath(new Path(dataDir, tableName), conf);
+ }
+ fs = tablePath.getFileSystem(conf);
+ fs.mkdirs(tablePath);
+ Path dataFile = new Path(tablePath, "data_file");
+ out = fs.create(dataFile);
+ List<FieldSchema> fields = new ArrayList<FieldSchema>();
+
+ boolean firstRow = true;
+ for (Node n : valuesTableChildren) {
+ ASTNode valuesRow = (ASTNode) n;
+ assert valuesRow.getToken().getType() == HiveParser.TOK_VALUE_ROW :
+ "Expected child of TOK_VALUE_TABLE to be TOK_VALUE_ROW but was " + valuesRow.getName();
+ // Each of the children of this should be a literal
+ List<? extends Node> valuesRowChildren = valuesRow.getChildren();
+ boolean isFirst = true;
+ int nextColNum = 1;
+ for (Node n1 : valuesRowChildren) {
+ ASTNode value = (ASTNode) n1;
+ if (firstRow) {
+ fields.add(new FieldSchema("tmp_values_col" + nextColNum++, "string", ""));
+ }
+ if (isFirst) isFirst = false;
+ else writeAsText("\u0001", out);
+ writeAsText(unparseExprForValuesClause(value), out);
+ }
+ writeAsText("\n", out);
+ firstRow = false;
+ }
+
+ // Step 2, create a temp table, using the created file as the data
+ StorageFormat format = new StorageFormat(conf);
+ format.processStorageFormat("TextFile");
+ Table table = db.newTable(tableName);
+ table.setSerializationLib(format.getSerde());
+ table.setFields(fields);
+ table.setDataLocation(tablePath);
+ table.getTTable().setTemporary(true);
+ table.setStoredAsSubDirectories(false);
+ table.setInputFormatClass(format.getInputFormat());
+ table.setOutputFormatClass(format.getOutputFormat());
+ db.createTable(table, false);
+ } catch (Exception e) {
+ String errMsg = ErrorMsg.INSERT_CANNOT_CREATE_TEMP_FILE.getMsg() + e.getMessage();
+ LOG.error(errMsg);
+ // Try to delete the file
+ if (fs != null && tablePath != null) {
+ try {
+ fs.delete(tablePath, false);
+ } catch (IOException swallowIt) {}
+ }
+ throw new SemanticException(errMsg, e);
+ } finally {
+ IOUtils.closeStream(out);
+ }
+
+ // Step 3, return a new subtree with a from clause built around that temp table
+ // The form of the tree is TOK_TABREF->TOK_TABNAME->identifier(tablename)
+ Token t = new ClassicToken(HiveParser.TOK_TABREF);
+ ASTNode tabRef = new ASTNode(t);
+ t = new ClassicToken(HiveParser.TOK_TABNAME);
+ ASTNode tabNameNode = new ASTNode(t);
+ tabRef.addChild(tabNameNode);
+ t = new ClassicToken(HiveParser.Identifier, tableName);
+ ASTNode identifier = new ASTNode(t);
+ tabNameNode.addChild(identifier);
+ return tabRef;
+ }
+
+ // Take an expression in the values clause and turn it back into a string. This is far from
+ // comprehensive. At the moment it only supports:
+ // * literals (all types)
+ // * unary negatives
+ // * true/false
+ private String unparseExprForValuesClause(ASTNode expr) throws SemanticException {
+ switch (expr.getToken().getType()) {
+ case HiveParser.Number:
+ return expr.getText();
+
+ case HiveParser.StringLiteral:
+ return BaseSemanticAnalyzer.unescapeSQLString(expr.getText());
+
+ case HiveParser.KW_FALSE:
+ // UDFToBoolean casts any non-empty string to true, so set this to false
+ return "";
+
+ case HiveParser.KW_TRUE:
+ return "TRUE";
+
+ case HiveParser.MINUS:
+ return "-" + unparseExprForValuesClause((ASTNode)expr.getChildren().get(0));
+
+ case HiveParser.TOK_NULL:
+ // Hive's text input will translate this as a null
+ return "\\N";
+
+ default:
+ throw new SemanticException("Expression of type " + expr.getText() +
+ " not supported in insert/values");
+ }
+
+ }
+
+ private void assertCombineInputFormat(Tree numerator, String message) throws SemanticException {
+ String inputFormat = conf.getVar(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE).equals("tez") ?
+ HiveConf.getVar(conf, HiveConf.ConfVars.HIVETEZINPUTFORMAT):
+ HiveConf.getVar(conf, HiveConf.ConfVars.HIVEINPUTFORMAT);
+ if (!inputFormat.equals(CombineHiveInputFormat.class.getName())) {
+ throw new SemanticException(generateErrorMessage((ASTNode) numerator,
+ message + " sampling is not supported in " + inputFormat));
+ }
+ }
+
+ private String processSubQuery(QB qb, ASTNode subq) throws SemanticException {
+
+ // This is a subquery and must have an alias
+ if (subq.getChildCount() != 2) {
+ throw new SemanticException(ErrorMsg.NO_SUBQUERY_ALIAS.getMsg(subq));
+ }
+ ASTNode subqref = (ASTNode) subq.getChild(0);
+ String alias = unescapeIdentifier(subq.getChild(1).getText());
+
+ // Recursively do the first phase of semantic analysis for the subquery
+ QBExpr qbexpr = new QBExpr(alias);
+
+ doPhase1QBExpr(subqref, qbexpr, qb.getId(), alias);
+
+ // If the alias is already there then we have a conflict
+ if (qb.exists(alias)) {
+ throw new SemanticException(ErrorMsg.AMBIGUOUS_TABLE_ALIAS.getMsg(subq
+ .getChild(1)));
+ }
+ // Insert this map into the stats
+ qb.setSubqAlias(alias, qbexpr);
+ qb.addAlias(alias);
+
+ unparseTranslator.addIdentifierTranslation((ASTNode) subq.getChild(1));
+
+ return alias;
+ }
+
+ /*
+ * Phase1: hold onto any CTE definitions in aliasToCTE.
+ * CTE definitions are global to the Query.
+ */
+ private void processCTE(QB qb, ASTNode ctes) throws SemanticException {
+
+ int numCTEs = ctes.getChildCount();
+
+ for(int i=0; i <numCTEs; i++) {
+ ASTNode cte = (ASTNode) ctes.getChild(i);
+ ASTNode cteQry = (ASTNode) cte.getChild(0);
+ String alias = unescapeIdentifier(cte.getChild(1).getText());
+
+ String qName = qb.getId() == null ? "" : qb.getId() + ":";
+ qName += alias.toLowerCase();
+
+ if ( aliasToCTEs.containsKey(qName)) {
+ throw new SemanticException(ErrorMsg.AMBIGUOUS_TABLE_ALIAS.getMsg(cte.getChild(1)));
+ }
+ aliasToCTEs.put(qName, new CTEClause(qName, cteQry));
+ }
+ }
+
+ /*
+ * We allow CTE definitions in views. So we can end up with a hierarchy of CTE definitions:
+ * - at the top level of a query statement
+ * - where a view is referenced.
+ * - views may refer to other views.
+ *
+ * The scoping rules we use are: to search for a CTE from the current QB outwards. In order to
+ * disambiguate between CTES are different levels we qualify(prefix) them with the id of the QB
+ * they appear in when adding them to the <code>aliasToCTEs</code> map.
+ *
+ */
+ private CTEClause findCTEFromName(QB qb, String cteName) {
+ StringBuilder qId = new StringBuilder();
+ if (qb.getId() != null) {
+ qId.append(qb.getId());
+ }
+
+ while (qId.length() > 0) {
+ String nm = qId + ":" + cteName;
+ CTEClause cte = aliasToCTEs.get(nm);
+ if (cte != null) {
+ return cte;
+ }
+ int lastIndex = qId.lastIndexOf(":");
+ lastIndex = lastIndex < 0 ? 0 : lastIndex;
+ qId.setLength(lastIndex);
+ }
+ return aliasToCTEs.get(cteName);
+ }
+
+ /*
+ * If a CTE is referenced in a QueryBlock:
+ * - add it as a SubQuery for now.
+ * - SQ.alias is the alias used in QB. (if no alias is specified,
+ * it used the CTE name. Works just like table references)
+ * - Adding SQ done by:
+ * - copying AST of CTE
+ * - setting ASTOrigin on cloned AST.
+ * - trigger phase 1 on new QBExpr.
+ * - update QB data structs: remove this as a table reference, move it to a SQ invocation.
+ */
+ private void addCTEAsSubQuery(QB qb, String cteName, String cteAlias)
+ throws SemanticException {
+ cteAlias = cteAlias == null ? cteName : cteAlias;
+ CTEClause cte = findCTEFromName(qb, cteName);
+ ASTNode cteQryNode = cte.cteNode;
+ QBExpr cteQBExpr = new QBExpr(cteAlias);
+ doPhase1QBExpr(cteQryNode, cteQBExpr, qb.getId(), cteAlias);
+ qb.rewriteCTEToSubq(cteAlias, cteName, cteQBExpr);
+ }
+
+ private final CTEClause rootClause = new CTEClause(null, null);
+
+ @Override
+ public List<Task<? extends Serializable>> getAllRootTasks() {
+ if (!rootTasksResolved) {
+ rootTasks = toRealRootTasks(rootClause.asExecutionOrder());
+ rootTasksResolved = true;
+ }
+ return rootTasks;
+ }
+
+ @Override
+ public HashSet<ReadEntity> getAllInputs() {
+ HashSet<ReadEntity> readEntities = new HashSet<ReadEntity>(getInputs());
+ for (CTEClause cte : rootClause.asExecutionOrder()) {
+ if (cte.source != null) {
+ readEntities.addAll(cte.source.getInputs());
+ }
+ }
+ return readEntities;
+ }
+
+ @Override
+ public HashSet<WriteEntity> getAllOutputs() {
+ HashSet<WriteEntity> writeEntities = new HashSet<WriteEntity>(getOutputs());
+ for (CTEClause cte : rootClause.asExecutionOrder()) {
+ if (cte.source != null) {
+ writeEntities.addAll(cte.source.getOutputs());
+ }
+ }
+ return writeEntities;
+ }
+
+ class CTEClause {
+ CTEClause(String alias, ASTNode cteNode) {
+ this.alias = alias;
+ this.cteNode = cteNode;
+ }
+ String alias;
+ ASTNode cteNode;
+ boolean materialize;
+ int reference;
+ QBExpr qbExpr;
+ List<CTEClause> parents = new ArrayList<CTEClause>();
+
+ // materialized
+ Table table;
+ SemanticAnalyzer source;
+
+ List<Task<? extends Serializable>> getTasks() {
+ return source == null ? null : source.rootTasks;
+ }
+
+ List<CTEClause> asExecutionOrder() {
+ List<CTEClause> execution = new ArrayList<CTEClause>();
+ asExecutionOrder(new HashSet<CTEClause>(), execution);
+ return execution;
+ }
+
+ void asExecutionOrder(Set<CTEClause> visited, List<CTEClause> execution) {
+ for (CTEClause parent : parents) {
+ if (visited.add(parent)) {
+ parent.asExecutionOrder(visited, execution);
+ }
+ }
+ execution.add(this);
+ }
+
+ @Override
+ public String toString() {
+ return alias == null ? "<root>" : alias;
+ }
+ }
+
+ private List<Task<? extends Serializable>> toRealRootTasks(List<CTEClause> execution) {
+ List<Task<? extends Serializable>> cteRoots = new ArrayList<>();
+ List<Task<? extends Serializable>> cteLeafs = new ArrayList<>();
+ List<Task<? extends Serializable>> curTopRoots = null;
+ List<Task<? extends Serializable>> curBottomLeafs = null;
+ for (int i = 0; i < execution.size(); i++) {
+ CTEClause current = execution.get(i);
+ if (current.parents.isEmpty() && curTopRoots != null) {
+ cteRoots.addAll(curTopRoots);
+ cteLeafs.addAll(curBottomLeafs);
+ curTopRoots = curBottomLeafs = null;
+ }
+ List<Task<? extends Serializable>> curTasks = current.getTasks();
+ if (curTasks == null) {
+ continue;
+ }
+ if (curTopRoots == null) {
+ curTopRoots = curTasks;
+ }
+ if (curBottomLeafs != null) {
+ for (Task<?> topLeafTask : curBottomLeafs) {
+ for (Task<?> currentRootTask : curTasks) {
+ topLeafTask.addDependentTask(currentRootTask);
+ }
+ }
+ }
+ curBottomLeafs = Task.findLeafs(curTasks);
+ }
+ if (curTopRoots != null) {
+ cteRoots.addAll(curTopRoots);
+ cteLeafs.addAll(curBottomLeafs);
+ }
+
+ if (cteRoots.isEmpty()) {
+ return rootTasks;
+ }
+ for (Task<?> cteLeafTask : cteLeafs) {
+ for (Task<?> mainRootTask : rootTasks) {
+ cteLeafTask.addDependentTask(mainRootTask);
+ }
+ }
+ return cteRoots;
+ }
+
+ Table materializeCTE(String cteName, CTEClause cte) throws HiveException {
+
+ ASTNode createTable = new ASTNode(new ClassicToken(HiveParser.TOK_CREATETABLE));
+
+ ASTNode tableName = new ASTNode(new ClassicToken(HiveParser.TOK_TABNAME));
+ tableName.addChild(new ASTNode(new ClassicToken(HiveParser.Identifier, cteName)));
+
+ ASTNode temporary = new ASTNode(new ClassicToken(HiveParser.KW_TEMPORARY, MATERIALIZATION_MARKER));
+
+ createTable.addChild(tableName);
+ createTable.addChild(temporary);
+ createTable.addChild(cte.cteNode);
+
+ SemanticAnalyzer analyzer = new SemanticAnalyzer(queryState);
+ analyzer.initCtx(ctx);
+ analyzer.init(false);
+
+ // should share cte contexts
+ analyzer.aliasToCTEs.putAll(aliasToCTEs);
+
+ HiveOperation operation = queryState.getHiveOperation();
+ try {
+ analyzer.analyzeInternal(createTable);
+ } finally {
+ queryState.setCommandType(operation);
+ }
+
+ Table table = analyzer.tableDesc.toTable(conf);
+ Path location = table.getDataLocation();
+ try {
+ location.getFileSystem(conf).mkdirs(location);
+ } catch (IOException e) {
+ throw new HiveException(e);
+ }
+ table.setMaterializedTable(true);
+
+ LOG.info(cteName + " will be materialized into " + location);
+ cte.table = table;
+ cte.source = analyzer;
+
+ ctx.addMaterializedTable(cteName, table);
+
+ return table;
+ }
+
+
+ static boolean isJoinToken(ASTNode node) {
+ if ((node.getToken().getType() == HiveParser.TOK_JOIN)
+ || (node.getToken().getType() == HiveParser.TOK_CROSSJOIN)
+ || isOuterJoinToken(node)
+ || (node.getToken().getType() == HiveParser.TOK_LEFTSEMIJOIN)
+ || (node.getToken().getType() == HiveParser.TOK_UNIQUEJOIN)) {
+ return true;
+ }
+
+ return false;
+ }
+
+ static private boolean isOuterJoinToken(ASTNode node) {
+ return (node.getToken().getType() == HiveParser.TOK_LEFTOUTERJOIN)
+ || (node.getToken().getType() == HiveParser.TOK_RIGHTOUTERJOIN)
+ || (node.getToken().getType() == HiveParser.TOK_FULLOUTERJOIN);
+ }
+
+ /**
+ * Given the AST with TOK_JOIN as the root, get all the aliases for the tables
+ * or subqueries in the join.
+ *
+ * @param qb
+ * @param join
+ * @throws SemanticException
+ */
+ @SuppressWarnings("nls")
+ private void processJoin(QB qb, ASTNode join) throws SemanticException {
+ int numChildren = join.getChildCount();
+ if ((numChildren != 2) && (numChildren != 3)
+ && join.getToken().getType() != HiveParser.TOK_UNIQUEJOIN) {
+ throw new SemanticException(generateErrorMessage(join,
+ "Join with multiple children"));
+ }
+
+ queryProperties.incrementJoinCount(isOuterJoinToken(join));
+ for (int num = 0; num < numChildren; num++) {
+ ASTNode child = (ASTNode) join.getChild(num);
+ if (child.getToken().getType() == HiveParser.TOK_TABREF) {
+ processTable(qb, child);
+ } else if (child.getToken().getType() == HiveParser.TOK_SUBQUERY) {
+ processSubQuery(qb, child);
+ } else if (child.getToken().getType() == HiveParser.TOK_PTBLFUNCTION) {
+ queryProperties.setHasPTF(true);
+ processPTF(qb, child);
+ PTFInvocationSpec ptfInvocationSpec = qb.getPTFInvocationSpec(child);
+ String inputAlias = ptfInvocationSpec == null ? null :
+ ptfInvocationSpec.getFunction().getAlias();;
+ if ( inputAlias == null ) {
+ throw new SemanticException(generateErrorMessage(child,
+ "PTF invocation in a Join must have an alias"));
+ }
+
+ } else if (child.getToken().getType() == HiveParser.TOK_LATERAL_VIEW ||
+ child.getToken().getType() == HiveParser.TOK_LATERAL_VIEW_OUTER) {
+ // SELECT * FROM src1 LATERAL VIEW udtf() AS myTable JOIN src2 ...
+ // is not supported. Instead, the lateral view must be in a subquery
+ // SELECT * FROM (SELECT * FROM src1 LATERAL VIEW udtf() AS myTable) a
+ // JOIN src2 ...
+ throw new SemanticException(ErrorMsg.LATERAL_VIEW_WITH_JOIN
+ .getMsg(join));
+ } else if (isJoinToken(child)) {
+ processJoin(qb, child);
+ }
+ }
+ }
+
+ /**
+ * Given the AST with TOK_LATERAL_VIEW as the root, get the alias for the
+ * table or subquery in the lateral view and also make a mapping from the
+ * alias to all the lateral view AST's.
+ *
+ * @param qb
+ * @param lateralView
+ * @return the alias for the table/subquery
+ * @throws SemanticException
+ */
+
+ private String processLateralView(QB qb, ASTNode lateralView)
+ throws SemanticException {
+ int numChildren = lateralView.getChildCount();
+
+ assert (numChildren == 2);
+ ASTNode next = (ASTNode) lateralView.getChild(1);
+
+ String alias = null;
+
+ switch (next.getToken().getType()) {
+ case HiveParser.TOK_TABREF:
+ alias = processTable(qb, next);
+ break;
+ case HiveParser.TOK_SUBQUERY:
+ alias = processSubQuery(qb, next);
+ break;
+ case HiveParser.TOK_LATERAL_VIEW:
+ case HiveParser.TOK_LATERAL_VIEW_OUTER:
+ alias = processLateralView(qb, next);
+ break;
+ default:
+ throw new SemanticException(ErrorMsg.LATERAL_VIEW_INVALID_CHILD
+ .getMsg(lateralView));
+ }
+ alias = alias.toLowerCase();
+ qb.getParseInfo().addLateralViewForAlias(alias, lateralView);
+ qb.addAlias(alias);
+ return alias;
+ }
+
+ /**
+ * Phase 1: (including, but not limited to):
+ *
+ * 1. Gets all the aliases for all the tables / subqueries and makes the
+ * appropriate mapping in aliasToTabs, aliasToSubq 2. Gets the location of the
+ * destination and names the clause "inclause" + i 3. Creates a map from a
+ * string representation of an aggregation tree to the actual aggregation AST
+ * 4. Creates a mapping from the clause name to the select expression AST in
+ * destToSelExpr 5. Creates a mapping from a table alias to the lateral view
+ * AST's in aliasToLateralViews
+ *
+ * @param ast
+ * @param qb
+ * @param ctx_1
+ * @throws SemanticException
+ */
+ @SuppressWarnings({"fallthrough", "nls"})
+ public boolean doPhase1(ASTNode ast, QB qb, Phase1Ctx ctx_1, PlannerContext plannerCtx)
+ throws SemanticException {
+
+ boolean phase1Result = true;
+ QBParseInfo qbp = qb.getParseInfo();
+ boolean skipRecursion = false;
+
+ if (ast.getToken() != null) {
+ skipRecursion = true;
+ switch (ast.getToken().getType()) {
+ case HiveParser.TOK_SELECTDI:
+ qb.countSelDi();
+ // fall through
+ case HiveParser.TOK_SELECT:
+ qb.countSel();
+ qbp.setSelExprForClause(ctx_1.dest, ast);
+
+ int posn = 0;
+ if (((ASTNode) ast.getChild(0)).getToken().getType() == HiveParser.TOK_HINTLIST) {
+ qbp.setHints((ASTNode) ast.getChild(0));
+ posn++;
+ }
+
+ if ((ast.getChild(posn).getChild(0).getType() == HiveParser.TOK_TRANSFORM))
+ queryProperties.setUsesScript(true);
+
+ LinkedHashMap<String, ASTNode> aggregations = doPhase1GetAggregationsFromSelect(ast,
+ qb, ctx_1.dest);
+ doPhase1GetColumnAliasesFromSelect(ast, qbp);
+ qbp.setAggregationExprsForClause(ctx_1.dest, aggregations);
+ qbp.setDistinctFuncExprsForClause(ctx_1.dest,
+ doPhase1GetDistinctFuncExprs(aggregations));
+ break;
+
+ case HiveParser.TOK_WHERE:
+ qbp.setWhrExprForClause(ctx_1.dest, ast);
+ if (!SubQueryUtils.findSubQueries((ASTNode) ast.getChild(0)).isEmpty())
+ queryProperties.setFilterWithSubQuery(true);
+ break;
+
+ case HiveParser.TOK_INSERT_INTO:
+ String currentDatabase = SessionState.get().getCurrentDatabase();
+ String tab_name = getUnescapedName((ASTNode) ast.getChild(0).getChild(0), currentDatabase);
+ qbp.addInsertIntoTable(tab_name, ast);
+
+ case HiveParser.TOK_DESTINATION:
+ ctx_1.dest = "insclause-" + ctx_1.nextNum;
+ ctx_1.nextNum++;
+ boolean isTmpFileDest = false;
+ if (ast.getChildCount() > 0 && ast.getChild(0) instanceof ASTNode) {
+ ASTNode ch = (ASTNode) ast.getChild(0);
+ if (ch.getToken().getType() == HiveParser.TOK_DIR && ch.getChildCount() > 0
+ && ch.getChild(0) instanceof ASTNode) {
+ ch = (ASTNode) ch.getChild(0);
+ isTmpFileDest = ch.getToken().getType() == HiveParser.TOK_TMP_FILE;
+ } else {
+ if (ast.getToken().getType() == HiveParser.TOK_DESTINATION
+ && ast.getChild(0).getType() == HiveParser.TOK_TAB) {
+ String fullTableName = getUnescapedName((ASTNode) ast.getChild(0).getChild(0),
+ SessionState.get().getCurrentDatabase());
+ qbp.getInsertOverwriteTables().put(fullTableName, ast);
+ }
+ }
+ }
+
+ // is there a insert in the subquery
+ if (qbp.getIsSubQ() && !isTmpFileDest) {
+ throw new SemanticException(ErrorMsg.NO_INSERT_INSUBQUERY.getMsg(ast));
+ }
+
+ if (plannerCtx != null) {
+ plannerCtx.setInsertToken(ast, isTmpFileDest);
+ }
+
+ qbp.setDestForClause(ctx_1.dest, (ASTNode) ast.getChild(0));
+ handleInsertStatementSpecPhase1(ast, qbp, ctx_1);
+ if (qbp.getClauseNamesForDest().size() > 1) {
+ queryProperties.setMultiDestQuery(true);
+ }
+ break;
+
+ case HiveParser.TOK_FROM:
+ int child_count = ast.getChildCount();
+ if (child_count != 1) {
+ throw new SemanticException(generateErrorMessage(ast,
+ "Multiple Children " + child_count));
+ }
+
+ // Check if this is a subquery / lateral view
+ ASTNode frm = (ASTNode) ast.getChild(0);
+ if (frm.getToken().getType() == HiveParser.TOK_TABREF) {
+ processTable(qb, frm);
+ } else if (frm.getToken().getType() == HiveParser.TOK_VIRTUAL_TABLE) {
+ // Create a temp table with the passed values in it then rewrite this portion of the
+ // tree to be from that table.
+ ASTNode newFrom = genValuesTempTable(frm, qb);
+ ast.setChild(0, newFrom);
+ processTable(qb, newFrom);
+ } else if (frm.getToken().getType() == HiveParser.TOK_SUBQUERY) {
+ processSubQuery(qb, frm);
+ } else if (frm.getToken().getType() == HiveParser.TOK_LATERAL_VIEW ||
+ frm.getToken().getType() == HiveParser.TOK_LATERAL_VIEW_OUTER) {
+ queryProperties.setHasLateralViews(true);
+ processLateralView(qb, frm);
+ } else if (isJoinToken(frm)) {
+ processJoin(qb, frm);
+ qbp.setJoinExpr(frm);
+ }else if(frm.getToken().getType() == HiveParser.TOK_PTBLFUNCTION){
+ queryProperties.setHasPTF(true);
+ processPTF(qb, frm);
+ }
+ break;
+
+ case HiveParser.TOK_CLUSTERBY:
+ // Get the clusterby aliases - these are aliased to the entries in the
+ // select list
+ queryProperties.setHasClusterBy(true);
+ qbp.setClusterByExprForClause(ctx_1.dest, ast);
+ break;
+
+ case HiveParser.TOK_DISTRIBUTEBY:
+ // Get the distribute by aliases - these are aliased to the entries in
+ // the
+ // select list
+ queryProperties.setHasDistributeBy(true);
+ qbp.setDistributeByExprForClause(ctx_1.dest, ast);
+ if (qbp.getClusterByForClause(ctx_1.dest) != null) {
+ throw new SemanticException(generateErrorMessage(ast,
+ ErrorMsg.CLUSTERBY_DISTRIBUTEBY_CONFLICT.getMsg()));
+ } else if (qbp.getOrderByForClause(ctx_1.dest) != null) {
+ throw new SemanticException(generateErrorMessage(ast,
+ ErrorMsg.ORDERBY_DISTRIBUTEBY_CONFLICT.getMsg()));
+ }
+ break;
+
+ case HiveParser.TOK_SORTBY:
+ // Get the sort by aliases - these are aliased to the entries in the
+ // select list
+ queryProperties.setHasSortBy(true);
+ qbp.setSortByExprForClause(ctx_1.dest, ast);
+ if (qbp.getClusterByForClause(ctx_1.dest) != null) {
+ throw new SemanticException(generateErrorMessage(ast,
+ ErrorMsg.CLUSTERBY_SORTBY_CONFLICT.getMsg()));
+ } else if (qbp.getOrderByForClause(ctx_1.dest) != null) {
+ throw new SemanticException(generateErrorMessage(ast,
+ ErrorMsg.ORDERBY_SORTBY_CONFLICT.getMsg()));
+ }
+
+ break;
+
+ case HiveParser.TOK_ORDERBY:
+ // Get the order by aliases - these are aliased to the entries in the
+ // select list
+ queryProperties.setHasOrderBy(true);
+ qbp.setOrderByExprForClause(ctx_1.dest, ast);
+ if (qbp.getClusterByForClause(ctx_1.dest) != null) {
+ throw new SemanticException(generateErrorMessage(ast,
+ ErrorMsg.CLUSTERBY_ORDERBY_CONFLICT.getMsg()));
+ }
+ break;
+
+ case HiveParser.TOK_GROUPBY:
+ case HiveParser.TOK_ROLLUP_GROUPBY:
+ case HiveParser.TOK_CUBE_GROUPBY:
+ case HiveParser.TOK_GROUPING_SETS:
+ // Get the groupby aliases - these are aliased to the entries in the
+ // select list
+ queryProperties.setHasGroupBy(true);
+ if (qbp.getJoinExpr() != null) {
+ queryProperties.setHasJoinFollowedByGroupBy(true);
+ }
+ if (qbp.getSelForClause(ctx_1.dest).getToken().getType() == HiveParser.TOK_SELECTDI) {
+ throw new SemanticException(generateErrorMessage(ast,
+ ErrorMsg.SELECT_DISTINCT_WITH_GROUPBY.getMsg()));
+ }
+ qbp.setGroupByExprForClause(ctx_1.dest, ast);
+ skipRecursion = true;
+
+ // Rollup and Cubes are syntactic sugar on top of grouping sets
+ if (ast.getToken().getType() == HiveParser.TOK_ROLLUP_GROUPBY) {
+ qbp.getDestRollups().add(ctx_1.dest);
+ } else if (ast.getToken().getType() == HiveParser.TOK_CUBE_GROUPBY) {
+ qbp.getDestCubes().add(ctx_1.dest);
+ } else if (ast.getToken().getType() == HiveParser.TOK_GROUPING_SETS) {
+ qbp.getDestGroupingSets().add(ctx_1.dest);
+ }
+ break;
+
+ case HiveParser.TOK_HAVING:
+ qbp.setHavingExprForClause(ctx_1.dest, ast);
+ qbp.addAggregationExprsForClause(ctx_1.dest,
+ doPhase1GetAggregationsFromSelect(ast, qb, ctx_1.dest));
+ break;
+
+ case HiveParser.KW_WINDOW:
+ if (!qb.hasWindowingSpec(ctx_1.dest) ) {
+ throw new SemanticException(generateErrorMessage(ast,
+ "Query has no Cluster/Distribute By; but has a Window definition"));
+ }
+ handleQueryWindowClauses(qb, ctx_1, ast);
+ break;
+
+ case HiveParser.TOK_LIMIT:
+ if (ast.getChildCount() == 2) {
+ qbp.setDestLimit(ctx_1.dest,
+ new Integer(ast.getChild(0).getText()),
+ new Integer(ast.getChild(1).getText()));
+ } else {
+ qbp.setDestLimit(ctx_1.dest, new Integer(0),
+ new Integer(ast.getChild(0).getText()));
+ }
+ break;
+
+ case HiveParser.TOK_ANALYZE:
+ // Case of analyze command
+
+ String table_name = getUnescapedName((ASTNode) ast.getChild(0).getChild(0)).toLowerCase();
+
+
+ qb.setTabAlias(table_name, table_name);
+ qb.addAlias(table_name);
+ qb.getParseInfo().setIsAnalyzeCommand(true);
+ qb.getParseInfo().setNoScanAnalyzeCommand(this.noscan);
+ qb.getParseInfo().setPartialScanAnalyzeCommand(this.partialscan);
+ // Allow analyze the whole table and dynamic partitions
+ HiveConf.setVar(conf, HiveConf.ConfVars.DYNAMICPARTITIONINGMODE, "nonstrict");
+ HiveConf.setVar(conf, HiveConf.ConfVars.HIVEMAPREDMODE, "nonstrict");
+
+ break;
+
+ case HiveParser.TOK_UNIONALL:
+ if (!qbp.getIsSubQ()) {
+ // this shouldn't happen. The parser should have converted the union to be
+ // contained in a subquery. Just in case, we keep the error as a fallback.
+ throw new SemanticException(generateErrorMessage(ast,
+ ErrorMsg.UNION_NOTIN_SUBQ.getMsg()));
+ }
+ skipRecursion = false;
+ break;
+
+ case HiveParser.TOK_INSERT:
+ ASTNode destination = (ASTNode) ast.getChild(0);
+ Tree tab = destination.getChild(0);
+
+ // Proceed if AST contains partition & If Not Exists
+ if (destination.getChildCount() == 2 &&
+ tab.getChildCount() == 2 &&
+ destination.getChild(1).getType() == HiveParser.TOK_IFNOTEXISTS) {
+ String tableName = tab.getChild(0).getChild(0).getText();
+
+ Tree partitions = tab.getChild(1);
+ int childCount = partitions.getChildCount();
+ HashMap<String, String> partition = new HashMap<String, String>();
+ for (int i = 0; i < childCount; i++) {
+ String partitionName = partitions.getChild(i).getChild(0).getText();
+ Tree pvalue = partitions.getChild(i).getChild(1);
+ if (pvalue == null) {
+ break;
+ }
+ String partitionVal = stripQuotes(pvalue.getText());
+ partition.put(partitionName, partitionVal);
+ }
+ // if it is a dynamic partition throw the exception
+ if (childCount != partition.size()) {
+ throw new SemanticException(ErrorMsg.INSERT_INTO_DYNAMICPARTITION_IFNOTEXISTS
+ .getMsg(partition.toString()));
+ }
+ Table table = null;
+ try {
+ table = this.getTableObjectByName(tableName);
+ } catch (HiveException ex) {
+ throw new SemanticException(ex);
+ }
+ try {
+ Partition parMetaData = db.getPartition(table, partition, false);
+ // Check partition exists if it exists skip the overwrite
+ if (parMetaData != null) {
+ phase1Result = false;
+ skipRecursion = true;
+ LOG.info("Partition already exists so insert into overwrite " +
+ "skipped for partition : " + parMetaData.toString());
+ break;
+ }
+ } catch (HiveException e) {
+ LOG.info("Error while getting metadata : ", e);
+ }
+ validatePartSpec(table, partition, (ASTNode)tab, conf, false);
+ }
+ skipRecursion = false;
+ break;
+ case HiveParser.TOK_LATERAL_VIEW:
+ case HiveParser.TOK_LATERAL_VIEW_OUTER:
+ // todo: nested LV
+ assert ast.getChildCount() == 1;
+ qb.getParseInfo().getDestToLateralView().put(ctx_1.dest, ast);
+ break;
+ case HiveParser.TOK_CTE:
+ processCTE(qb, ast);
+ break;
+ default:
+ skipRecursion = false;
+ break;
+ }
+ }
+
+ if (!skipRecursion) {
+ // Iterate over the rest of the children
+ int child_count = ast.getChildCount();
+ for (int child_pos = 0; child_pos < child_count && phase1Result; ++child_pos) {
+ // Recurse
+ phase1Result = phase1Result && doPhase1(
+ (ASTNode)ast.getChild(child_pos), qb, ctx_1, plannerCtx);
+ }
+ }
+ return phase1Result;
+ }
+
+ /**
+ * This is phase1 of supporting specifying schema in insert statement
+ * insert into foo(z,y) select a,b from bar;
+ * @see #handleInsertStatementSpec(java.util.List, String, RowResolver, RowResolver, QB, ASTNode)
+ * @throws SemanticException
+ */
+ private void handleInsertStatementSpecPhase1(ASTNode ast, QBParseInfo qbp, Phase1Ctx ctx_1) throws SemanticException {
+ ASTNode tabColName = (ASTNode)ast.getChild(1);
+ if(ast.getType() == HiveParser.TOK_INSERT_INTO && tabColName != null && tabColName.getType() == HiveParser.TOK_TABCOLNAME) {
+ //we have "insert into foo(a,b)..."; parser will enforce that 1+ columns are listed if TOK_TABCOLNAME is present
+ List<String> targetColNames = new ArrayList<String>();
+ for(Node col : tabColName.getChildren()) {
+ assert ((ASTNode)col).getType() == HiveParser.Identifier :
+ "expected token " + HiveParser.Identifier + " found " + ((ASTNode)col).getType();
+ targetColNames.add(((ASTNode)col).getText());
+ }
+ String fullTableName = getUnescapedName((ASTNode) ast.getChild(0).getChild(0),
+ SessionState.get().getCurrentDatabase());
+ qbp.setDestSchemaForClause(ctx_1.dest, targetColNames);
+ Set<String> targetColumns = new HashSet<String>();
+ targetColumns.addAll(targetColNames);
+ if(targetColNames.size() != targetColumns.size()) {
+ throw new SemanticException(generateErrorMessage(tabColName,
+ "Duplicate column name detected in " + fullTableName + " table schema specification"));
+ }
+ Table targetTable = null;
+ try {
+ targetTable = db.getTable(fullTableName, false);
+ }
+ catch (HiveException ex) {
+ LOG.error("Error processing HiveParser.TOK_DESTINATION: " + ex.getMessage(), ex);
+ throw new SemanticException(ex);
+ }
+ if(targetTable == null) {
+ throw new SemanticException(generateErrorMessage(ast,
+ "Unable to access metadata for table " + fullTableName));
+ }
+ for(FieldSchema f : targetTable.getCols()) {
+ //parser only allows foo(a,b), not foo(foo.a, foo.b)
+ targetColumns.remove(f.getName());
+ }
+ if(!targetColumns.isEmpty()) {//here we need to see if remaining columns are dynamic partition columns
+ /* We just checked the user specified schema columns among regular table column and found some which are not
+ 'regular'. Now check is they are dynamic partition columns
+ For dynamic partitioning,
+ Given "create table multipart(a int, b int) partitioned by (c int, d int);"
+ for "insert into multipart partition(c='1',d)(d,a) values(2,3);" we expect parse tree to look like this
+ (TOK_INSERT_INTO
+ (TOK_TAB
+ (TOK_TABNAME multipart)
+ (TOK_PARTSPEC
+ (TOK_PARTVAL c '1')
+ (TOK_PARTVAL d)
+ )
+ )
+ (TOK_TABCOLNAME d a)
+ )*/
+ List<String> dynamicPartitionColumns = new ArrayList<String>();
+ if(ast.getChild(0) != null && ast.getChild(0).getType() == HiveParser.TOK_TAB) {
+ ASTNode tokTab = (ASTNode)ast.getChild(0);
+ ASTNode tokPartSpec = (ASTNode)tokTab.getFirstChildWithType(HiveParser.TOK_PARTSPEC);
+ if(tokPartSpec != null) {
+ for(Node n : tokPartSpec.getChildren()) {
+ ASTNode tokPartVal = null;
+ if(n instanceof ASTNode) {
+ tokPartVal = (ASTNode)n;
+ }
+ if(tokPartVal != null && tokPartVal.getType() == HiveParser.TOK_PARTVAL && tokPartVal.getChildCount() == 1) {
+ assert tokPartVal.getChild(0).getType() == HiveParser.Identifier :
+ "Expected column name; found tokType=" + tokPartVal.getType();
+ dynamicPartitionColumns.add(tokPartVal.getChild(0).getText());
+ }
+ }
+ }
+ }
+ for(String colName : dynamicPartitionColumns) {
+ targetColumns.remove(colName);
+ }
+ if(!targetColumns.isEmpty()) {
+ //Found some columns in user specified schema which are neither regular not dynamic partition columns
+ throw new SemanticException(generateErrorMessage(tabColName,
+ "'" + (targetColumns.size() == 1 ? targetColumns.iterator().next() : targetColumns) +
+ "' in insert schema specification " + (targetColumns.size() == 1 ? "is" : "are") +
+ " not found among regular columns of " +
+ fullTableName + " nor dynamic partition columns."));
+ }
+ }
+ }
+ }
+
+ public void getMaterializationMetadata(QB qb) throws SemanticException {
+ try {
+ gatherCTEReferences(qb, rootClause);
+ int threshold = HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVE_CTE_MATERIALIZE_THRESHOLD);
+ for (CTEClause cte : Sets.newHashSet(aliasToCTEs.values())) {
+ if (threshold >= 0 && cte.reference >= threshold) {
+ cte.materialize = true;
+ }
+ }
+ } catch (HiveException e) {
+ // Has to use full name to make sure it does not conflict with
+ // org.apache.commons.lang.StringUtils
+ LOG.error(org.apache.hadoop.util.StringUtils.stringifyException(e));
+ if (e instanceof SemanticException) {
+ throw (SemanticException)e;
+ }
+ throw new SemanticException(e.getMessage(), e);
+ }
+ }
+
+ private void gatherCTEReferences(QBExpr qbexpr, CTEClause parent) throws HiveException {
+ if (qbexpr.getOpcode() == QBExpr.Opcode.NULLOP) {
+ gatherCTEReferences(qbexpr.getQB(), parent);
+ } else {
+ gatherCTEReferences(qbexpr.getQBExpr1(), parent);
+ gatherCTEReferences(qbexpr.getQBExpr2(), parent);
+ }
+ }
+
+ // TODO: check view references, too
+ private void gatherCTEReferences(QB qb, CTEClause current) throws HiveException {
+ for (String alias : qb.getTabAliases()) {
+ String tabName = qb.getTabNameForAlias(alias);
+ String cteName = tabName.toLowerCase();
+
+ CTEClause cte = findCTEFromName(qb, cteName);
+ if (cte != null) {
+ if (ctesExpanded.contains(cteName)) {
+ throw new SemanticException("Recursive cte " + cteName +
+ " detected (cycle: " + StringUtils.join(ctesExpanded, " -> ") +
+ " -> " + cteName + ").");
+ }
+ cte.reference++;
+ current.parents.add(cte);
+ if (cte.qbExpr != null) {
+ continue;
+ }
+ cte.qbExpr = new QBExpr(cteName);
+ doPhase1QBExpr(cte.cteNode, cte.qbExpr, qb.getId(), cteName);
+
+ ctesExpanded.add(cteName);
+ gatherCTEReferences(cte.qbExpr, cte);
+ ctesExpanded.remove(ctesExpanded.size() - 1);
+ }
+ }
+ for (String alias : qb.getSubqAliases()) {
+ gatherCTEReferences(qb.getSubqForAlias(alias), current);
+ }
+ }
+
+ public void getMetaData(QB qb) throws SemanticException {
+ getMetaData(qb, false);
+ }
+
+ public void getMetaData(QB qb, boolean enableMaterialization) throws SemanticException {
+ try {
+ if (enableMaterialization) {
+ getMaterializationMetadata(qb);
+ }
+ getMetaData(qb, null);
+ } catch (HiveException e) {
+ // Has to use full name to make sure it does not conflict with
+ // org.apache.commons.lang.StringUtils
+ LOG.error(org.apache.hadoop.util.StringUtils.stringifyException(e));
+ if (e instanceof SemanticException) {
+ throw (SemanticException)e;
+ }
+ throw new SemanticException(e.getMessage(), e);
+ }
+ }
+
+ private void getMetaData(QBExpr qbexpr, ReadEntity parentInput)
+ throws HiveException {
+ if (qbexpr.getOpcode() == QBExpr.Opcode.NULLOP) {
+ getMetaData(qbexpr.getQB(), parentInput);
+ } else {
+ getMetaData(qbexpr.getQBExpr1(), parentInput);
+ getMetaData(qbexpr.getQBExpr2(), parentInput);
+ }
+ }
+
+ @SuppressWarnings("nls")
+ private void getMetaData(QB qb, ReadEntity parentInput)
+ throws HiveException {
+ LOG.info("Get metadata for source tables");
+
+ // Go over the tables and populate the related structures.
+ // We have to materialize the table alias list since we might
+ // modify it in the middle for view rewrite.
+ List<String> tabAliases = new ArrayList<String>(qb.getTabAliases());
+
+ // Keep track of view alias to view name and read entity
+ // For eg: for a query like 'select * from V3', where V3 -> V2, V2 -> V1, V1 -> T
+ // keeps track of full view name and read entity corresponding to alias V3, V3:V2, V3:V2:V1.
+ // This is needed for tracking the dependencies for inputs, along with their parents.
+ Map<String, ObjectPair<String, ReadEntity>> aliasToViewInfo =
+ new HashMap<String, ObjectPair<String, ReadEntity>>();
+
+ /*
+ * used to capture view to SQ conversions. This is used to check for
+ * recursive CTE invocations.
+ */
+ Map<String, String> sqAliasToCTEName = new HashMap<String, String>();
+
+ for (String alias : tabAliases) {
+ String tabName = qb.getTabNameForAlias(alias);
+ String cteName = tabName.toLowerCase();
+
+ Table tab = db.getTable(tabName, false);
+ if (tab == null ||
+ tab.getDbName().equals(SessionState.get().getCurrentDatabase())) {
+ Table materializedTab = ctx.getMaterializedTable(cteName);
+ if (materializedTab == null) {
+ // we first look for this alias from CTE, and then from catalog.
+ CTEClause cte = findCTEFromName(qb, cteName);
+ if (cte != null) {
+ if (!cte.materialize) {
+ addCTEAsSubQuery(qb, cteName, alias);
+ sqAliasToCTEName.put(alias, cteName);
+ continue;
+ }
+ tab = materializeCTE(cteName, cte);
+ }
+ } else {
+ tab = materializedTab;
+ }
+ }
+
+ if (tab == null) {
+ ASTNode src = qb.getParseInfo().getSrcForAlias(alias);
+ if (null != src) {
+ throw new SemanticException(ErrorMsg.INVALID_TABLE.getMsg(src));
+ } else {
+ throw new SemanticException(ErrorMsg.INVALID_TABLE.getMsg(alias));
+ }
+ }
+
+ // Disallow INSERT INTO on bucketized tables
+ boolean isAcid = AcidUtils.isAcidTable(tab);
+ boolean isTableWrittenTo = qb.getParseInfo().isInsertIntoTable(tab.getDbName(), tab.getTableName());
+ if (isTableWrittenTo &&
+ tab.getNumBuckets() > 0 && !isAcid) {
+ throw new SemanticException(ErrorMsg.INSERT_INTO_BUCKETIZED_TABLE.
+ getMsg("Table: " + tabName));
+ }
+ // Disallow update and delete on non-acid tables
+ if ((updating() || deleting()) && !isAcid && isTableWrittenTo) {
+ //isTableWrittenTo: delete from acidTbl where a in (select id from nonAcidTable)
+ //so only assert this if we are actually writing to this table
+ // Whether we are using an acid compliant transaction manager has already been caught in
+ // UpdateDeleteSemanticAnalyzer, so if we are updating or deleting and getting nonAcid
+ // here, it means the table itself doesn't support it.
+ throw new SemanticException(ErrorMsg.ACID_OP_ON_NONACID_TABLE, tabName);
+ }
+
+ if (tab.isView()) {
+ if (qb.getParseInfo().isAnalyzeCommand()) {
+ throw new SemanticException(ErrorMsg.ANALYZE_VIEW.getMsg());
+ }
+ String fullViewName = tab.getDbName() + "." + tab.getTableName();
+ // Prevent view cycles
+ if (viewsExpanded.contains(fullViewName)) {
+ throw new SemanticException("Recursive view " + fullViewName +
+ " detected (cycle: " + StringUtils.join(viewsExpanded, " -> ") +
+ " -> " + fullViewName + ").");
+ }
+ replaceViewReferenceWithDefinition(qb, tab, tabName, alias);
+ // This is the last time we'll see the Table objects for views, so add it to the inputs
+ // now. isInsideView will tell if this view is embedded in another view.
+ ReadEntity viewInput = new ReadEntity(tab, parentInput, !qb.isInsideView());
+ viewInput = PlanUtils.addInput(inputs, viewInput);
+ aliasToViewInfo.put(alias, new ObjectPair<String, ReadEntity>(fullViewName, viewInput));
+ String aliasId = getAliasId(alias, qb);
+ if (aliasId != null) {
+ aliasId = aliasId.replace(SemanticAnalyzer.SUBQUERY_TAG_1, "")
+ .replace(SemanticAnalyzer.SUBQUERY_TAG_2, "");
+ }
+ viewAliasToInput.put(aliasId, viewInput);
+ continue;
+ }
+
+ if (!InputFormat.class.isAssignableFrom(tab.getInputFormatClass())) {
+ throw new SemanticException(generateErrorMessage(
+ qb.getParseInfo().getSrcForAlias(alias),
+ ErrorMsg.INVALID_INPUT_FORMAT_TYPE.getMsg()));
+ }
+
+ qb.getMetaData().setSrcForAlias(alias, tab);
+
+ if (qb.getParseInfo().isAnalyzeCommand()) {
+ // allow partial partition specification for nonscan since noscan is fast.
+ TableSpec ts = new TableSpec(db, conf, (ASTNode) ast.getChild(0), true, this.noscan);
+ if (ts.specType == SpecType.DYNAMIC_PARTITION) { // dynamic partitions
+ try {
+ ts.partitions = db.getPartitionsByNames(ts.tableHandle, ts.partSpec);
+ } catch (HiveException e) {
+ throw new SemanticException(generateErrorMessage(
+ qb.getParseInfo().getSrcForAlias(alias),
+ "Cannot get partitions for " + ts.partSpec), e);
+ }
+ }
+ // validate partial scan command
+ QBParseInfo qbpi = qb.getParseInfo();
+ if (qbpi.isPartialScanAnalyzeCommand()) {
+ Class<? extends InputFormat> inputFormatClass = null;
+ switch (ts.specType) {
+ case TABLE_ONLY:
+ case DYNAMIC_PARTITION:
+ inputFormatClass = ts.tableHandle.getInputFormatClass();
+ break;
+ case STATIC_PARTITION:
+ inputFormatClass = ts.partHandle.getInputFormatClass();
+ break;
+ default:
+ assert false;
+ }
+ // throw a HiveException for formats other than rcfile or orcfile.
+ if (!(inputFormatClass.equals(RCFileInputFormat.class) || inputFormatClass
+ .equals(OrcInputFormat.class))) {
+ throw new SemanticException(ErrorMsg.ANALYZE_TABLE_PARTIALSCAN_NON_RCFILE.getMsg());
+ }
+ }
+
+ tab.setTableSpec(ts);
+ qb.getParseInfo().addTableSpec(alias, ts);
+ }
+
+ ReadEntity parentViewInfo = PlanUtils.getParentViewInfo(getAliasId(alias, qb), viewAliasToInput);
+ PlanUtils.addInput(inputs,
+ new ReadEntity(tab, parentViewInfo, parentViewInfo == null),mergeIsDirect);
+ }
+
+ LOG.info("Get metadata for subqueries");
+ // Go over the subqueries and getMetaData for these
+ for (String alias : qb.getSubqAliases()) {
+ boolean wasView = aliasToViewInfo.containsKey(alias);
+ boolean wasCTE = sqAliasToCTEName.containsKey(alias);
+ ReadEntity newParentInput = null;
+ if (wasView) {
+ viewsExpanded.add(aliasToViewInfo.get(alias).getFirst());
+ newParentInput = aliasToViewInfo.get(alias).getSecond();
+ } else if (wasCTE) {
+ ctesExpanded.add(sqAliasToCTEName.get(alias));
+ }
+ QBExpr qbexpr = qb.getSubqForAlias(alias);
+ getMetaData(qbexpr, newParentInput);
+ if (wasView) {
+ viewsExpanded.remove(viewsExpanded.size() - 1);
+ } else if (wasCTE) {
+ ctesExpanded.remove(ctesExpanded.size() - 1);
+ }
+ }
+
+ RowFormatParams rowFormatParams = new RowFormatParams();
+ StorageFormat storageFormat = new StorageFormat(conf);
+
+ LOG.info("Get metadata for destination tables");
+ // Go over all the destination structures and populate the related
+ // metadata
+ QBParseInfo qbp = qb.getParseInfo();
+
+ for (String name : qbp.getClauseNamesForDest()) {
+ ASTNode ast = qbp.getDestForClause(name);
+ switch (ast.getToken().getType()) {
+ case HiveParser.TOK_TAB: {
+ TableSpec ts = new TableSpec(db, conf, ast);
+ if (ts.tableHandle.isView()) {
+ throw new SemanticException(ErrorMsg.DML_AGAINST_VIEW.getMsg());
+ }
+
+ Class<?> outputFormatClass = ts.tableHandle.getOutputFormatClass();
+ if (!ts.tableHandle.isNonNative() &&
+ !HiveOutputFormat.class.isAssignableFrom(outputFormatClass)) {
+ throw new SemanticException(ErrorMsg.INVALID_OUTPUT_FORMAT_TYPE
+ .getMsg(ast, "The class is " + outputFormatClass.toString()));
+ }
+
+ // TableSpec ts is got from the query (user specified),
+ // which means the user didn't specify partitions in their query,
+ // but whether the table itself is partitioned is not know.
+ if (ts.specType != SpecType.STATIC_PARTITION) {
+ // This is a table or dynamic partition
+ qb.getMetaData().setDestForAlias(name, ts.tableHandle);
+ // has dynamic as well as static partitions
+ if (ts.partSpec != null && ts.partSpec.size() > 0) {
+ qb.getMetaData().setPartSpecForAlias(name, ts.partSpec);
+ }
+ } else {
+ // This is a partition
+ qb.getMetaData().setDestForAlias(name, ts.partHandle);
+ }
+ if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVESTATSAUTOGATHER)) {
+ // Add the table spec for the destination table.
+ qb.getParseInfo().addTableSpec(ts.tableName.toLowerCase(), ts);
+ }
+ break;
+ }
+
+ case HiveParser.TOK_DIR: {
+ // This is a dfs file
+ String fname = stripQuotes(ast.getChild(0).getText());
+ if ((!qb.getParseInfo().getIsSubQ())
+ && (((ASTNode) ast.getChild(0)).getToken().getType() == HiveParser.TOK_TMP_FILE)) {
+
+ if (qb.isCTAS()) {
+ qb.setIsQuery(false);
+ ctx.setResDir(null);
+ ctx.setResFile(null);
+
+ // allocate a t
<TRUNCATED>
[45/50] [abbrv] hive git commit: HIVE-16094. queued containers may
timeout if they don't get to run for a long time.
Posted by om...@apache.org.
HIVE-16094. queued containers may timeout if they don't get to run for a long time.
Change-Id: Ieb412a66dbe53c6709f7bd840b3dfa543225d826
(cherry picked from commit 3def1d7f19982b4a710e7cb867a8b6b7bbf8fb97)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/2f3c949c
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/2f3c949c
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/2f3c949c
Branch: refs/heads/branch-2.2
Commit: 2f3c949c20ca51ac835bedd9bff2ac1c07e000a1
Parents: 29db808
Author: Siddharth Seth <ss...@apache.org>
Authored: Thu Mar 2 22:27:39 2017 -0800
Committer: Owen O'Malley <om...@apache.org>
Committed: Tue Mar 28 15:27:57 2017 -0700
----------------------------------------------------------------------
.../hive/llap/daemon/impl/AMReporter.java | 95 ++++++++++++--------
.../llap/daemon/impl/ContainerRunnerImpl.java | 3 +-
.../llap/daemon/impl/TaskRunnerCallable.java | 7 +-
.../llap/tezplugins/LlapTaskCommunicator.java | 2 +-
4 files changed, 64 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/2f3c949c/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/AMReporter.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/AMReporter.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/AMReporter.java
index b01a495..af4a1f2 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/AMReporter.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/AMReporter.java
@@ -100,7 +100,7 @@ public class AMReporter extends AbstractService {
private final AtomicBoolean isShutdown = new AtomicBoolean(false);
// Tracks appMasters to which heartbeats are being sent. This should not be used for any other
// messages like taskKilled, etc.
- private final Map<LlapNodeId, AMNodeInfo> knownAppMasters = new HashMap<>();
+ private final Map<QueryIdentifier, AMNodeInfo> knownAppMasters = new HashMap<>();
volatile ListenableFuture<Void> queueLookupFuture;
private final DaemonId daemonId;
@@ -186,35 +186,42 @@ public class AMReporter extends AbstractService {
public void registerTask(String amLocation, int port, String user,
Token<JobTokenIdentifier> jobToken, QueryIdentifier queryIdentifier) {
if (LOG.isTraceEnabled()) {
- LOG.trace("Registering for heartbeat: " + amLocation + ":" + port + " for queryIdentifier=" + queryIdentifier);
+ LOG.trace(
+ "Registering for heartbeat: {}, queryIdentifier={}",
+ (amLocation + ":" + port), queryIdentifier);
}
AMNodeInfo amNodeInfo;
+
+ // Since we don't have an explicit AM end signal yet - we're going to create
+ // and discard AMNodeInfo instances per query.
synchronized (knownAppMasters) {
LlapNodeId amNodeId = LlapNodeId.getInstance(amLocation, port);
- amNodeInfo = knownAppMasters.get(amNodeId);
+ amNodeInfo = knownAppMasters.get(queryIdentifier);
if (amNodeInfo == null) {
amNodeInfo =
new AMNodeInfo(amNodeId, user, jobToken, queryIdentifier, retryPolicy, retryTimeout, socketFactory,
conf);
- knownAppMasters.put(amNodeId, amNodeInfo);
+ knownAppMasters.put(queryIdentifier, amNodeInfo);
// Add to the queue only the first time this is registered, and on
// subsequent instances when it's taken off the queue.
amNodeInfo.setNextHeartbeatTime(System.currentTimeMillis() + heartbeatInterval);
pendingHeartbeatQueeu.add(amNodeInfo);
+ // AMNodeInfo will only be cleared when a queryComplete is received for this query, or
+ // when we detect a failure on the AM side (failure to heartbeat).
+ // A single queueLookupCallable is added here. We have to make sure one instance stays
+ // in the queue till the query completes.
}
- amNodeInfo.setCurrentQueryIdentifier(queryIdentifier);
amNodeInfo.incrementAndGetTaskCount();
}
}
- public void unregisterTask(String amLocation, int port) {
+ public void unregisterTask(String amLocation, int port, QueryIdentifier queryIdentifier) {
if (LOG.isTraceEnabled()) {
- LOG.trace("Un-registering for heartbeat: " + amLocation + ":" + port);
+ LOG.trace("Un-registering for heartbeat: {}", (amLocation + ":" + port));
}
AMNodeInfo amNodeInfo;
- LlapNodeId amNodeId = LlapNodeId.getInstance(amLocation, port);
synchronized (knownAppMasters) {
- amNodeInfo = knownAppMasters.get(amNodeId);
+ amNodeInfo = knownAppMasters.get(queryIdentifier);
if (amNodeInfo == null) {
LOG.info(("Ignoring duplicate unregisterRequest for am at: " + amLocation + ":" + port));
} else {
@@ -230,7 +237,7 @@ public class AMReporter extends AbstractService {
LlapNodeId amNodeId = LlapNodeId.getInstance(amLocation, port);
AMNodeInfo amNodeInfo;
synchronized (knownAppMasters) {
- amNodeInfo = knownAppMasters.get(amNodeId);
+ amNodeInfo = knownAppMasters.get(queryIdentifier);
if (amNodeInfo == null) {
amNodeInfo = new AMNodeInfo(amNodeId, user, jobToken, queryIdentifier, retryPolicy, retryTimeout, socketFactory,
conf);
@@ -255,10 +262,17 @@ public class AMReporter extends AbstractService {
});
}
- public void queryComplete(LlapNodeId llapNodeId) {
- if (llapNodeId != null) {
+ public void queryComplete(QueryIdentifier queryIdentifier) {
+ if (queryIdentifier != null) {
synchronized (knownAppMasters) {
- AMNodeInfo amNodeInfo = knownAppMasters.remove(llapNodeId);
+ AMNodeInfo amNodeInfo = knownAppMasters.remove(queryIdentifier);
+
+ // The AM can be used for multiple queries. This is an indication that a single query is complete.
+ // We don't have a good mechanism to know when an app ends. Removing this right now ensures
+ // that a new one gets created for the next query on the same AM.
+ if (amNodeInfo != null) {
+ amNodeInfo.setIsDone(true);
+ }
// TODO: not stopping umbilical explicitly as some taskKill requests may get scheduled during queryComplete
// which will be using the umbilical. HIVE-16021 should fix this, until then leave umbilical open and wait for
// it to be closed after max idle timeout (10s default)
@@ -276,22 +290,26 @@ public class AMReporter extends AbstractService {
while (!isShutdown.get() && !Thread.currentThread().isInterrupted()) {
try {
final AMNodeInfo amNodeInfo = pendingHeartbeatQueeu.take();
- if (amNodeInfo.hasAmFailed()) {
+ if (amNodeInfo.hasAmFailed() || amNodeInfo.isDone()) {
synchronized (knownAppMasters) {
if (LOG.isDebugEnabled()) {
LOG.debug(
- "Removing am {} with last associated dag {} from heartbeat with taskCount={}, amFailed={}",
- amNodeInfo.amNodeId, amNodeInfo.getCurrentQueryIdentifier(), amNodeInfo.getTaskCount(),
- amNodeInfo.hasAmFailed(), amNodeInfo);
+ "Removing am {} with last associated dag {} from heartbeat with taskCount={}, amFailed={}, isDone={}",
+ amNodeInfo.amNodeId, amNodeInfo.getQueryIdentifier(), amNodeInfo.getTaskCount(),
+ amNodeInfo.hasAmFailed(), amNodeInfo.isDone());
}
- knownAppMasters.remove(amNodeInfo.amNodeId);
+ knownAppMasters.remove(amNodeInfo.getQueryIdentifier());
}
} else {
+ // Always re-schedule the next callable - irrespective of task count,
+ // in case new tasks come in later.
+ long next = System.currentTimeMillis() + heartbeatInterval;
+ amNodeInfo.setNextHeartbeatTime(next);
+ pendingHeartbeatQueeu.add(amNodeInfo);
+
+ // Send an actual heartbeat only if the task count is > 0
if (amNodeInfo.getTaskCount() > 0) {
// Add back to the queue for the next heartbeat, and schedule the actual heartbeat
- long next = System.currentTimeMillis() + heartbeatInterval;
- amNodeInfo.setNextHeartbeatTime(next);
- pendingHeartbeatQueeu.add(amNodeInfo);
ListenableFuture<Void> future = executor.submit(new AMHeartbeatCallable(amNodeInfo));
Futures.addCallback(future, new FutureCallback<Void>() {
@Override
@@ -301,9 +319,9 @@ public class AMReporter extends AbstractService {
@Override
public void onFailure(Throwable t) {
- QueryIdentifier currentQueryIdentifier = amNodeInfo.getCurrentQueryIdentifier();
+ QueryIdentifier currentQueryIdentifier = amNodeInfo.getQueryIdentifier();
amNodeInfo.setAmFailed(true);
- LOG.warn("Heartbeat failed to AM {}. Killing all other tasks for the query={}",
+ LOG.warn("Heartbeat failed to AM {}. Marking query as failed. query={}",
amNodeInfo.amNodeId, currentQueryIdentifier, t);
queryFailedHandler.queryFailed(currentQueryIdentifier);
}
@@ -369,7 +387,7 @@ public class AMReporter extends AbstractService {
amNodeInfo.getUmbilical().nodeHeartbeat(new Text(nodeId.getHostname()),
new Text(daemonId.getUniqueNodeIdInCluster()), nodeId.getPort());
} catch (IOException e) {
- QueryIdentifier currentQueryIdentifier = amNodeInfo.getCurrentQueryIdentifier();
+ QueryIdentifier currentQueryIdentifier = amNodeInfo.getQueryIdentifier();
amNodeInfo.setAmFailed(true);
LOG.warn("Failed to communicated with AM at {}. Killing remaining fragments for query {}",
amNodeInfo.amNodeId, currentQueryIdentifier, e);
@@ -379,11 +397,7 @@ public class AMReporter extends AbstractService {
LOG.warn("Interrupted while trying to send heartbeat to AM {}", amNodeInfo.amNodeId, e);
}
}
- } else {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Skipping node heartbeat to AM: " + amNodeInfo + ", since ref count is 0");
- }
- }
+ }
return null;
}
}
@@ -400,9 +414,10 @@ public class AMReporter extends AbstractService {
private final long timeout;
private final SocketFactory socketFactory;
private final AtomicBoolean amFailed = new AtomicBoolean(false);
- private QueryIdentifier currentQueryIdentifier;
+ private final QueryIdentifier queryIdentifier;
private LlapTaskUmbilicalProtocol umbilical;
private long nextHeartbeatTime;
+ private final AtomicBoolean isDone = new AtomicBoolean(false);
public AMNodeInfo(LlapNodeId amNodeId, String user,
@@ -414,7 +429,7 @@ public class AMReporter extends AbstractService {
Configuration conf) {
this.user = user;
this.jobToken = jobToken;
- this.currentQueryIdentifier = currentQueryIdentifier;
+ this.queryIdentifier = currentQueryIdentifier;
this.retryPolicy = retryPolicy;
this.timeout = timeout;
this.socketFactory = socketFactory;
@@ -465,16 +480,20 @@ public class AMReporter extends AbstractService {
return amFailed.get();
}
- int getTaskCount() {
- return taskCount.get();
+ void setIsDone(boolean val) {
+ isDone.set(val);
}
- public synchronized QueryIdentifier getCurrentQueryIdentifier() {
- return currentQueryIdentifier;
+ boolean isDone() {
+ return isDone.get();
+ }
+
+ int getTaskCount() {
+ return taskCount.get();
}
- public synchronized void setCurrentQueryIdentifier(QueryIdentifier queryIdentifier) {
- this.currentQueryIdentifier = queryIdentifier;
+ public QueryIdentifier getQueryIdentifier() {
+ return queryIdentifier;
}
synchronized void setNextHeartbeatTime(long nextTime) {
@@ -500,7 +519,7 @@ public class AMReporter extends AbstractService {
@Override
public String toString() {
- return "AMInfo: " + amNodeId + ", taskCount=" + getTaskCount();
+ return "AMInfo: " + amNodeId + ", taskCount=" + getTaskCount() + ", queryIdentifier=" + queryIdentifier;
}
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/2f3c949c/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/ContainerRunnerImpl.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/ContainerRunnerImpl.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/ContainerRunnerImpl.java
index 1176e5e..2b9ece9 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/ContainerRunnerImpl.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/ContainerRunnerImpl.java
@@ -399,8 +399,7 @@ public class ContainerRunnerImpl extends CompositeService implements ContainerRu
fragmentInfo.getFragmentIdentifierString());
executorService.killFragment(fragmentInfo.getFragmentIdentifierString());
}
- LlapNodeId amNodeId = queryInfo.getAmNodeId();
- amReporter.queryComplete(amNodeId);
+ amReporter.queryComplete(queryIdentifier);
}
return QueryCompleteResponseProto.getDefaultInstance();
}
http://git-wip-us.apache.org/repos/asf/hive/blob/2f3c949c/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/TaskRunnerCallable.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/TaskRunnerCallable.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/TaskRunnerCallable.java
index 8739d5b..716d05b 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/TaskRunnerCallable.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/TaskRunnerCallable.java
@@ -178,7 +178,8 @@ public class TaskRunnerCallable extends CallableWithNdc<TaskRunner2Result> {
}
// Unregister from the AMReporter, since the task is now running.
- this.amReporter.unregisterTask(request.getAmHost(), request.getAmPort());
+ this.amReporter.unregisterTask(request.getAmHost(), request.getAmPort(),
+ fragmentInfo.getQueryInfo().getQueryIdentifier());
synchronized (this) {
if (!shouldRunTask) {
@@ -326,7 +327,9 @@ public class TaskRunnerCallable extends CallableWithNdc<TaskRunner2Result> {
// If the task hasn't started - inform about fragment completion immediately. It's possible for
// the callable to never run.
fragmentCompletionHanler.fragmentComplete(fragmentInfo);
- this.amReporter.unregisterTask(request.getAmHost(), request.getAmPort());
+ this.amReporter
+ .unregisterTask(request.getAmHost(), request.getAmPort(),
+ fragmentInfo.getQueryInfo().getQueryIdentifier());
}
}
} else {
http://git-wip-us.apache.org/repos/asf/hive/blob/2f3c949c/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskCommunicator.java
----------------------------------------------------------------------
diff --git a/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskCommunicator.java b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskCommunicator.java
index c716c5e..f6166a6 100644
--- a/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskCommunicator.java
+++ b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskCommunicator.java
@@ -744,10 +744,10 @@ public class LlapTaskCommunicator extends TezTaskCommunicatorImpl {
@Override
public void nodeHeartbeat(Text hostname, Text uniqueId, int port) throws IOException {
- nodePinged(hostname.toString(), uniqueId.toString(), port);
if (LOG.isDebugEnabled()) {
LOG.debug("Received heartbeat from [" + hostname + ":" + port +" (" + uniqueId +")]");
}
+ nodePinged(hostname.toString(), uniqueId.toString(), port);
}
@Override
[03/50] [abbrv] hive git commit: HIVE-14959: Fix DISTINCT with
windowing when CBO is enabled/disabled (Jesus Camacho Rodriguez,
reviewed by Ashutosh Chauhan)
Posted by om...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/9e7daaf4/ql/src/test/results/clientpositive/llap/windowing.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/windowing.q.out b/ql/src/test/results/clientpositive/llap/windowing.q.out
new file mode 100644
index 0000000..0c61103
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/windowing.q.out
@@ -0,0 +1,2348 @@
+PREHOOK: query: select p_mfgr, p_name, p_size,
+rank() over(distribute by p_mfgr sort by p_name) as r,
+dense_rank() over(distribute by p_mfgr sort by p_name) as dr,
+sum(p_retailprice) over (distribute by p_mfgr sort by p_name rows between unbounded preceding and current row) as s1
+from part
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part
+#### A masked pattern was here ####
+POSTHOOK: query: select p_mfgr, p_name, p_size,
+rank() over(distribute by p_mfgr sort by p_name) as r,
+dense_rank() over(distribute by p_mfgr sort by p_name) as dr,
+sum(p_retailprice) over (distribute by p_mfgr sort by p_name rows between unbounded preceding and current row) as s1
+from part
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part
+#### A masked pattern was here ####
+Manufacturer#1 almond antique burnished rose metallic 2 1 1 1173.15
+Manufacturer#1 almond antique burnished rose metallic 2 1 1 2346.3
+Manufacturer#1 almond antique chartreuse lavender yellow 34 3 2 4100.06
+Manufacturer#1 almond antique salmon chartreuse burlywood 6 4 3 5702.650000000001
+Manufacturer#1 almond aquamarine burnished black steel 28 5 4 7117.070000000001
+Manufacturer#1 almond aquamarine pink moccasin thistle 42 6 5 8749.730000000001
+Manufacturer#2 almond antique violet chocolate turquoise 14 1 1 1690.68
+Manufacturer#2 almond antique violet turquoise frosted 40 2 2 3491.38
+Manufacturer#2 almond aquamarine midnight light salmon 2 3 3 5523.360000000001
+Manufacturer#2 almond aquamarine rose maroon antique 25 4 4 7222.02
+Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 5 5 8923.62
+Manufacturer#3 almond antique chartreuse khaki white 17 1 1 1671.68
+Manufacturer#3 almond antique forest lavender goldenrod 14 2 2 2861.95
+Manufacturer#3 almond antique metallic orange dim 19 3 3 4272.34
+Manufacturer#3 almond antique misty red olive 1 4 4 6195.32
+Manufacturer#3 almond antique olive coral navajo 45 5 5 7532.61
+Manufacturer#4 almond antique gainsboro frosted violet 10 1 1 1620.67
+Manufacturer#4 almond antique violet mint lemon 39 2 2 2996.09
+Manufacturer#4 almond aquamarine floral ivory bisque 27 3 3 4202.35
+Manufacturer#4 almond aquamarine yellow dodger mint 7 4 4 6047.27
+Manufacturer#4 almond azure aquamarine papaya violet 12 5 5 7337.620000000001
+Manufacturer#5 almond antique blue firebrick mint 31 1 1 1789.69
+Manufacturer#5 almond antique medium spring khaki 6 2 2 3401.3500000000004
+Manufacturer#5 almond antique sky peru orange 2 3 3 5190.08
+Manufacturer#5 almond aquamarine dodger light gainsboro 46 4 4 6208.18
+Manufacturer#5 almond azure blanched chiffon midnight 23 5 5 7672.66
+PREHOOK: query: select p_mfgr, p_name, p_size,
+min(p_retailprice),
+rank() over(distribute by p_mfgr sort by p_name)as r,
+dense_rank() over(distribute by p_mfgr sort by p_name) as dr,
+p_size, p_size - lag(p_size,1,p_size) over(distribute by p_mfgr sort by p_name) as deltaSz
+from part
+group by p_mfgr, p_name, p_size
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part
+#### A masked pattern was here ####
+POSTHOOK: query: select p_mfgr, p_name, p_size,
+min(p_retailprice),
+rank() over(distribute by p_mfgr sort by p_name)as r,
+dense_rank() over(distribute by p_mfgr sort by p_name) as dr,
+p_size, p_size - lag(p_size,1,p_size) over(distribute by p_mfgr sort by p_name) as deltaSz
+from part
+group by p_mfgr, p_name, p_size
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part
+#### A masked pattern was here ####
+Manufacturer#1 almond antique burnished rose metallic 2 1173.15 1 1 2 0
+Manufacturer#1 almond antique chartreuse lavender yellow 34 1753.76 2 2 34 32
+Manufacturer#1 almond antique salmon chartreuse burlywood 6 1602.59 3 3 6 -28
+Manufacturer#1 almond aquamarine burnished black steel 28 1414.42 4 4 28 22
+Manufacturer#1 almond aquamarine pink moccasin thistle 42 1632.66 5 5 42 14
+Manufacturer#2 almond antique violet chocolate turquoise 14 1690.68 1 1 14 0
+Manufacturer#2 almond antique violet turquoise frosted 40 1800.7 2 2 40 26
+Manufacturer#2 almond aquamarine midnight light salmon 2 2031.98 3 3 2 -38
+Manufacturer#2 almond aquamarine rose maroon antique 25 1698.66 4 4 25 23
+Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 1701.6 5 5 18 -7
+Manufacturer#3 almond antique chartreuse khaki white 17 1671.68 1 1 17 0
+Manufacturer#3 almond antique forest lavender goldenrod 14 1190.27 2 2 14 -3
+Manufacturer#3 almond antique metallic orange dim 19 1410.39 3 3 19 5
+Manufacturer#3 almond antique misty red olive 1 1922.98 4 4 1 -18
+Manufacturer#3 almond antique olive coral navajo 45 1337.29 5 5 45 44
+Manufacturer#4 almond antique gainsboro frosted violet 10 1620.67 1 1 10 0
+Manufacturer#4 almond antique violet mint lemon 39 1375.42 2 2 39 29
+Manufacturer#4 almond aquamarine floral ivory bisque 27 1206.26 3 3 27 -12
+Manufacturer#4 almond aquamarine yellow dodger mint 7 1844.92 4 4 7 -20
+Manufacturer#4 almond azure aquamarine papaya violet 12 1290.35 5 5 12 5
+Manufacturer#5 almond antique blue firebrick mint 31 1789.69 1 1 31 0
+Manufacturer#5 almond antique medium spring khaki 6 1611.66 2 2 6 -25
+Manufacturer#5 almond antique sky peru orange 2 1788.73 3 3 2 -4
+Manufacturer#5 almond aquamarine dodger light gainsboro 46 1018.1 4 4 46 44
+Manufacturer#5 almond azure blanched chiffon midnight 23 1464.48 5 5 23 -23
+PREHOOK: query: select p_mfgr, p_name, p_size, min(p_retailprice),
+rank() over(distribute by p_mfgr sort by p_name) as r,
+dense_rank() over(distribute by p_mfgr sort by p_name) as dr,
+p_size, p_size - lag(p_size,1,p_size) over(distribute by p_mfgr sort by p_name) as deltaSz
+from part
+group by p_mfgr, p_name, p_size
+having p_size > 0
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part
+#### A masked pattern was here ####
+POSTHOOK: query: select p_mfgr, p_name, p_size, min(p_retailprice),
+rank() over(distribute by p_mfgr sort by p_name) as r,
+dense_rank() over(distribute by p_mfgr sort by p_name) as dr,
+p_size, p_size - lag(p_size,1,p_size) over(distribute by p_mfgr sort by p_name) as deltaSz
+from part
+group by p_mfgr, p_name, p_size
+having p_size > 0
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part
+#### A masked pattern was here ####
+Manufacturer#1 almond antique burnished rose metallic 2 1173.15 1 1 2 0
+Manufacturer#1 almond antique chartreuse lavender yellow 34 1753.76 2 2 34 32
+Manufacturer#1 almond antique salmon chartreuse burlywood 6 1602.59 3 3 6 -28
+Manufacturer#1 almond aquamarine burnished black steel 28 1414.42 4 4 28 22
+Manufacturer#1 almond aquamarine pink moccasin thistle 42 1632.66 5 5 42 14
+Manufacturer#2 almond antique violet chocolate turquoise 14 1690.68 1 1 14 0
+Manufacturer#2 almond antique violet turquoise frosted 40 1800.7 2 2 40 26
+Manufacturer#2 almond aquamarine midnight light salmon 2 2031.98 3 3 2 -38
+Manufacturer#2 almond aquamarine rose maroon antique 25 1698.66 4 4 25 23
+Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 1701.6 5 5 18 -7
+Manufacturer#3 almond antique chartreuse khaki white 17 1671.68 1 1 17 0
+Manufacturer#3 almond antique forest lavender goldenrod 14 1190.27 2 2 14 -3
+Manufacturer#3 almond antique metallic orange dim 19 1410.39 3 3 19 5
+Manufacturer#3 almond antique misty red olive 1 1922.98 4 4 1 -18
+Manufacturer#3 almond antique olive coral navajo 45 1337.29 5 5 45 44
+Manufacturer#4 almond antique gainsboro frosted violet 10 1620.67 1 1 10 0
+Manufacturer#4 almond antique violet mint lemon 39 1375.42 2 2 39 29
+Manufacturer#4 almond aquamarine floral ivory bisque 27 1206.26 3 3 27 -12
+Manufacturer#4 almond aquamarine yellow dodger mint 7 1844.92 4 4 7 -20
+Manufacturer#4 almond azure aquamarine papaya violet 12 1290.35 5 5 12 5
+Manufacturer#5 almond antique blue firebrick mint 31 1789.69 1 1 31 0
+Manufacturer#5 almond antique medium spring khaki 6 1611.66 2 2 6 -25
+Manufacturer#5 almond antique sky peru orange 2 1788.73 3 3 2 -4
+Manufacturer#5 almond aquamarine dodger light gainsboro 46 1018.1 4 4 46 44
+Manufacturer#5 almond azure blanched chiffon midnight 23 1464.48 5 5 23 -23
+PREHOOK: query: select p_mfgr, p_name,
+count(p_size) over(distribute by p_mfgr sort by p_name) as cd
+from part
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part
+#### A masked pattern was here ####
+POSTHOOK: query: select p_mfgr, p_name,
+count(p_size) over(distribute by p_mfgr sort by p_name) as cd
+from part
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part
+#### A masked pattern was here ####
+Manufacturer#1 almond antique burnished rose metallic 2
+Manufacturer#1 almond antique burnished rose metallic 2
+Manufacturer#1 almond antique chartreuse lavender yellow 3
+Manufacturer#1 almond antique salmon chartreuse burlywood 4
+Manufacturer#1 almond aquamarine burnished black steel 5
+Manufacturer#1 almond aquamarine pink moccasin thistle 6
+Manufacturer#2 almond antique violet chocolate turquoise 1
+Manufacturer#2 almond antique violet turquoise frosted 2
+Manufacturer#2 almond aquamarine midnight light salmon 3
+Manufacturer#2 almond aquamarine rose maroon antique 4
+Manufacturer#2 almond aquamarine sandy cyan gainsboro 5
+Manufacturer#3 almond antique chartreuse khaki white 1
+Manufacturer#3 almond antique forest lavender goldenrod 2
+Manufacturer#3 almond antique metallic orange dim 3
+Manufacturer#3 almond antique misty red olive 4
+Manufacturer#3 almond antique olive coral navajo 5
+Manufacturer#4 almond antique gainsboro frosted violet 1
+Manufacturer#4 almond antique violet mint lemon 2
+Manufacturer#4 almond aquamarine floral ivory bisque 3
+Manufacturer#4 almond aquamarine yellow dodger mint 4
+Manufacturer#4 almond azure aquamarine papaya violet 5
+Manufacturer#5 almond antique blue firebrick mint 1
+Manufacturer#5 almond antique medium spring khaki 2
+Manufacturer#5 almond antique sky peru orange 3
+Manufacturer#5 almond aquamarine dodger light gainsboro 4
+Manufacturer#5 almond azure blanched chiffon midnight 5
+PREHOOK: query: select p_mfgr, p_name,
+rank() over(distribute by p_mfgr sort by p_name) as r,
+dense_rank() over(distribute by p_mfgr sort by p_name) as dr,
+count(p_size) over(distribute by p_mfgr sort by p_name) as cd,
+p_retailprice, sum(p_retailprice) over (distribute by p_mfgr sort by p_name rows between unbounded preceding and current row) as s1,
+p_size, p_size - lag(p_size,1,p_size) over(distribute by p_mfgr sort by p_name) as deltaSz
+from part
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part
+#### A masked pattern was here ####
+POSTHOOK: query: select p_mfgr, p_name,
+rank() over(distribute by p_mfgr sort by p_name) as r,
+dense_rank() over(distribute by p_mfgr sort by p_name) as dr,
+count(p_size) over(distribute by p_mfgr sort by p_name) as cd,
+p_retailprice, sum(p_retailprice) over (distribute by p_mfgr sort by p_name rows between unbounded preceding and current row) as s1,
+p_size, p_size - lag(p_size,1,p_size) over(distribute by p_mfgr sort by p_name) as deltaSz
+from part
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part
+#### A masked pattern was here ####
+Manufacturer#1 almond antique burnished rose metallic 1 1 2 1173.15 1173.15 2 0
+Manufacturer#1 almond antique burnished rose metallic 1 1 2 1173.15 2346.3 2 0
+Manufacturer#1 almond antique chartreuse lavender yellow 3 2 3 1753.76 4100.06 34 32
+Manufacturer#1 almond antique salmon chartreuse burlywood 4 3 4 1602.59 5702.650000000001 6 -28
+Manufacturer#1 almond aquamarine burnished black steel 5 4 5 1414.42 7117.070000000001 28 22
+Manufacturer#1 almond aquamarine pink moccasin thistle 6 5 6 1632.66 8749.730000000001 42 14
+Manufacturer#2 almond antique violet chocolate turquoise 1 1 1 1690.68 1690.68 14 0
+Manufacturer#2 almond antique violet turquoise frosted 2 2 2 1800.7 3491.38 40 26
+Manufacturer#2 almond aquamarine midnight light salmon 3 3 3 2031.98 5523.360000000001 2 -38
+Manufacturer#2 almond aquamarine rose maroon antique 4 4 4 1698.66 7222.02 25 23
+Manufacturer#2 almond aquamarine sandy cyan gainsboro 5 5 5 1701.6 8923.62 18 -7
+Manufacturer#3 almond antique chartreuse khaki white 1 1 1 1671.68 1671.68 17 0
+Manufacturer#3 almond antique forest lavender goldenrod 2 2 2 1190.27 2861.95 14 -3
+Manufacturer#3 almond antique metallic orange dim 3 3 3 1410.39 4272.34 19 5
+Manufacturer#3 almond antique misty red olive 4 4 4 1922.98 6195.32 1 -18
+Manufacturer#3 almond antique olive coral navajo 5 5 5 1337.29 7532.61 45 44
+Manufacturer#4 almond antique gainsboro frosted violet 1 1 1 1620.67 1620.67 10 0
+Manufacturer#4 almond antique violet mint lemon 2 2 2 1375.42 2996.09 39 29
+Manufacturer#4 almond aquamarine floral ivory bisque 3 3 3 1206.26 4202.35 27 -12
+Manufacturer#4 almond aquamarine yellow dodger mint 4 4 4 1844.92 6047.27 7 -20
+Manufacturer#4 almond azure aquamarine papaya violet 5 5 5 1290.35 7337.620000000001 12 5
+Manufacturer#5 almond antique blue firebrick mint 1 1 1 1789.69 1789.69 31 0
+Manufacturer#5 almond antique medium spring khaki 2 2 2 1611.66 3401.3500000000004 6 -25
+Manufacturer#5 almond antique sky peru orange 3 3 3 1788.73 5190.08 2 -4
+Manufacturer#5 almond aquamarine dodger light gainsboro 4 4 4 1018.1 6208.18 46 44
+Manufacturer#5 almond azure blanched chiffon midnight 5 5 5 1464.48 7672.66 23 -23
+PREHOOK: query: select sub1.r, sub1.dr, sub1.cd, sub1.s1, sub1.deltaSz
+from (select p_mfgr, p_name,
+rank() over(distribute by p_mfgr sort by p_name) as r,
+dense_rank() over(distribute by p_mfgr sort by p_name) as dr,
+count(p_size) over(distribute by p_mfgr sort by p_name) as cd,
+p_retailprice, sum(p_retailprice) over (distribute by p_mfgr sort by p_name rows between unbounded preceding and current row) as s1,
+p_size, p_size - lag(p_size,1,p_size) over(distribute by p_mfgr sort by p_name) as deltaSz
+from part
+) sub1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part
+#### A masked pattern was here ####
+POSTHOOK: query: select sub1.r, sub1.dr, sub1.cd, sub1.s1, sub1.deltaSz
+from (select p_mfgr, p_name,
+rank() over(distribute by p_mfgr sort by p_name) as r,
+dense_rank() over(distribute by p_mfgr sort by p_name) as dr,
+count(p_size) over(distribute by p_mfgr sort by p_name) as cd,
+p_retailprice, sum(p_retailprice) over (distribute by p_mfgr sort by p_name rows between unbounded preceding and current row) as s1,
+p_size, p_size - lag(p_size,1,p_size) over(distribute by p_mfgr sort by p_name) as deltaSz
+from part
+) sub1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part
+#### A masked pattern was here ####
+1 1 1 1620.67 0
+1 1 1 1671.68 0
+1 1 1 1690.68 0
+1 1 1 1789.69 0
+1 1 2 1173.15 0
+1 1 2 2346.3 0
+2 2 2 2861.95 -3
+2 2 2 2996.09 29
+2 2 2 3401.3500000000004 -25
+2 2 2 3491.38 26
+3 2 3 4100.06 32
+3 3 3 4202.35 -12
+3 3 3 4272.34 5
+3 3 3 5190.08 -4
+3 3 3 5523.360000000001 -38
+4 3 4 5702.650000000001 -28
+4 4 4 6047.27 -20
+4 4 4 6195.32 -18
+4 4 4 6208.18 44
+4 4 4 7222.02 23
+5 4 5 7117.070000000001 22
+5 5 5 7337.620000000001 5
+5 5 5 7532.61 44
+5 5 5 7672.66 -23
+5 5 5 8923.62 -7
+6 5 6 8749.730000000001 14
+PREHOOK: query: select abc.p_mfgr, abc.p_name,
+rank() over(distribute by abc.p_mfgr sort by abc.p_name) as r,
+dense_rank() over(distribute by abc.p_mfgr sort by abc.p_name) as dr,
+abc.p_retailprice, sum(abc.p_retailprice) over (distribute by abc.p_mfgr sort by abc.p_name rows between unbounded preceding and current row) as s1,
+abc.p_size, abc.p_size - lag(abc.p_size,1,abc.p_size) over(distribute by abc.p_mfgr sort by abc.p_name) as deltaSz
+from noop(on part
+partition by p_mfgr
+order by p_name
+) abc join part p1 on abc.p_partkey = p1.p_partkey
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part
+#### A masked pattern was here ####
+POSTHOOK: query: select abc.p_mfgr, abc.p_name,
+rank() over(distribute by abc.p_mfgr sort by abc.p_name) as r,
+dense_rank() over(distribute by abc.p_mfgr sort by abc.p_name) as dr,
+abc.p_retailprice, sum(abc.p_retailprice) over (distribute by abc.p_mfgr sort by abc.p_name rows between unbounded preceding and current row) as s1,
+abc.p_size, abc.p_size - lag(abc.p_size,1,abc.p_size) over(distribute by abc.p_mfgr sort by abc.p_name) as deltaSz
+from noop(on part
+partition by p_mfgr
+order by p_name
+) abc join part p1 on abc.p_partkey = p1.p_partkey
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part
+#### A masked pattern was here ####
+Manufacturer#1 almond antique burnished rose metallic 1 1 1173.15 1173.15 2 0
+Manufacturer#1 almond antique burnished rose metallic 1 1 1173.15 2346.3 2 0
+Manufacturer#1 almond antique burnished rose metallic 1 1 1173.15 3519.4500000000003 2 0
+Manufacturer#1 almond antique burnished rose metallic 1 1 1173.15 4692.6 2 0
+Manufacturer#1 almond antique chartreuse lavender yellow 5 2 1753.76 6446.360000000001 34 32
+Manufacturer#1 almond antique salmon chartreuse burlywood 6 3 1602.59 8048.950000000001 6 -28
+Manufacturer#1 almond aquamarine burnished black steel 7 4 1414.42 9463.37 28 22
+Manufacturer#1 almond aquamarine pink moccasin thistle 8 5 1632.66 11096.03 42 14
+Manufacturer#2 almond antique violet chocolate turquoise 1 1 1690.68 1690.68 14 0
+Manufacturer#2 almond antique violet turquoise frosted 2 2 1800.7 3491.38 40 26
+Manufacturer#2 almond aquamarine midnight light salmon 3 3 2031.98 5523.360000000001 2 -38
+Manufacturer#2 almond aquamarine rose maroon antique 4 4 1698.66 7222.02 25 23
+Manufacturer#2 almond aquamarine sandy cyan gainsboro 5 5 1701.6 8923.62 18 -7
+Manufacturer#3 almond antique chartreuse khaki white 1 1 1671.68 1671.68 17 0
+Manufacturer#3 almond antique forest lavender goldenrod 2 2 1190.27 2861.95 14 -3
+Manufacturer#3 almond antique metallic orange dim 3 3 1410.39 4272.34 19 5
+Manufacturer#3 almond antique misty red olive 4 4 1922.98 6195.32 1 -18
+Manufacturer#3 almond antique olive coral navajo 5 5 1337.29 7532.61 45 44
+Manufacturer#4 almond antique gainsboro frosted violet 1 1 1620.67 1620.67 10 0
+Manufacturer#4 almond antique violet mint lemon 2 2 1375.42 2996.09 39 29
+Manufacturer#4 almond aquamarine floral ivory bisque 3 3 1206.26 4202.35 27 -12
+Manufacturer#4 almond aquamarine yellow dodger mint 4 4 1844.92 6047.27 7 -20
+Manufacturer#4 almond azure aquamarine papaya violet 5 5 1290.35 7337.620000000001 12 5
+Manufacturer#5 almond antique blue firebrick mint 1 1 1789.69 1789.69 31 0
+Manufacturer#5 almond antique medium spring khaki 2 2 1611.66 3401.3500000000004 6 -25
+Manufacturer#5 almond antique sky peru orange 3 3 1788.73 5190.08 2 -4
+Manufacturer#5 almond aquamarine dodger light gainsboro 4 4 1018.1 6208.18 46 44
+Manufacturer#5 almond azure blanched chiffon midnight 5 5 1464.48 7672.66 23 -23
+PREHOOK: query: select p_mfgr, p_name, p_size,
+rank() over(distribute by p_mfgr sort by p_name, p_size desc) as R
+from part
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part
+#### A masked pattern was here ####
+POSTHOOK: query: select p_mfgr, p_name, p_size,
+rank() over(distribute by p_mfgr sort by p_name, p_size desc) as R
+from part
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part
+#### A masked pattern was here ####
+Manufacturer#1 almond antique burnished rose metallic 2 1
+Manufacturer#1 almond antique burnished rose metallic 2 1
+Manufacturer#1 almond antique chartreuse lavender yellow 34 3
+Manufacturer#1 almond antique salmon chartreuse burlywood 6 4
+Manufacturer#1 almond aquamarine burnished black steel 28 5
+Manufacturer#1 almond aquamarine pink moccasin thistle 42 6
+Manufacturer#2 almond antique violet chocolate turquoise 14 1
+Manufacturer#2 almond antique violet turquoise frosted 40 2
+Manufacturer#2 almond aquamarine midnight light salmon 2 3
+Manufacturer#2 almond aquamarine rose maroon antique 25 4
+Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 5
+Manufacturer#3 almond antique chartreuse khaki white 17 1
+Manufacturer#3 almond antique forest lavender goldenrod 14 2
+Manufacturer#3 almond antique metallic orange dim 19 3
+Manufacturer#3 almond antique misty red olive 1 4
+Manufacturer#3 almond antique olive coral navajo 45 5
+Manufacturer#4 almond antique gainsboro frosted violet 10 1
+Manufacturer#4 almond antique violet mint lemon 39 2
+Manufacturer#4 almond aquamarine floral ivory bisque 27 3
+Manufacturer#4 almond aquamarine yellow dodger mint 7 4
+Manufacturer#4 almond azure aquamarine papaya violet 12 5
+Manufacturer#5 almond antique blue firebrick mint 31 1
+Manufacturer#5 almond antique medium spring khaki 6 2
+Manufacturer#5 almond antique sky peru orange 2 3
+Manufacturer#5 almond aquamarine dodger light gainsboro 46 4
+Manufacturer#5 almond azure blanched chiffon midnight 23 5
+PREHOOK: query: select p_mfgr, p_name, p_size,
+rank() over(distribute by p_mfgr sort by p_name) as r,
+dense_rank() over(distribute by p_mfgr sort by p_name) as dr,
+sum(p_retailprice) over (distribute by p_mfgr sort by p_name rows between unbounded preceding and current row) as s1
+from part
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part
+#### A masked pattern was here ####
+POSTHOOK: query: select p_mfgr, p_name, p_size,
+rank() over(distribute by p_mfgr sort by p_name) as r,
+dense_rank() over(distribute by p_mfgr sort by p_name) as dr,
+sum(p_retailprice) over (distribute by p_mfgr sort by p_name rows between unbounded preceding and current row) as s1
+from part
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part
+#### A masked pattern was here ####
+Manufacturer#1 almond antique burnished rose metallic 2 1 1 1173.15
+Manufacturer#1 almond antique burnished rose metallic 2 1 1 2346.3
+Manufacturer#1 almond antique chartreuse lavender yellow 34 3 2 4100.06
+Manufacturer#1 almond antique salmon chartreuse burlywood 6 4 3 5702.650000000001
+Manufacturer#1 almond aquamarine burnished black steel 28 5 4 7117.070000000001
+Manufacturer#1 almond aquamarine pink moccasin thistle 42 6 5 8749.730000000001
+Manufacturer#2 almond antique violet chocolate turquoise 14 1 1 1690.68
+Manufacturer#2 almond antique violet turquoise frosted 40 2 2 3491.38
+Manufacturer#2 almond aquamarine midnight light salmon 2 3 3 5523.360000000001
+Manufacturer#2 almond aquamarine rose maroon antique 25 4 4 7222.02
+Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 5 5 8923.62
+Manufacturer#3 almond antique chartreuse khaki white 17 1 1 1671.68
+Manufacturer#3 almond antique forest lavender goldenrod 14 2 2 2861.95
+Manufacturer#3 almond antique metallic orange dim 19 3 3 4272.34
+Manufacturer#3 almond antique misty red olive 1 4 4 6195.32
+Manufacturer#3 almond antique olive coral navajo 45 5 5 7532.61
+Manufacturer#4 almond antique gainsboro frosted violet 10 1 1 1620.67
+Manufacturer#4 almond antique violet mint lemon 39 2 2 2996.09
+Manufacturer#4 almond aquamarine floral ivory bisque 27 3 3 4202.35
+Manufacturer#4 almond aquamarine yellow dodger mint 7 4 4 6047.27
+Manufacturer#4 almond azure aquamarine papaya violet 12 5 5 7337.620000000001
+Manufacturer#5 almond antique blue firebrick mint 31 1 1 1789.69
+Manufacturer#5 almond antique medium spring khaki 6 2 2 3401.3500000000004
+Manufacturer#5 almond antique sky peru orange 2 3 3 5190.08
+Manufacturer#5 almond aquamarine dodger light gainsboro 46 4 4 6208.18
+Manufacturer#5 almond azure blanched chiffon midnight 23 5 5 7672.66
+PREHOOK: query: select p_mfgr, p_name, p_size,
+rank() over(distribute by p_mfgr sort by p_name) as r,
+dense_rank() over(distribute by p_mfgr sort by p_name) as dr,
+sum(p_retailprice) over (distribute by p_mfgr sort by p_name rows between unbounded preceding and current row) as s1
+from part
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part
+#### A masked pattern was here ####
+POSTHOOK: query: select p_mfgr, p_name, p_size,
+rank() over(distribute by p_mfgr sort by p_name) as r,
+dense_rank() over(distribute by p_mfgr sort by p_name) as dr,
+sum(p_retailprice) over (distribute by p_mfgr sort by p_name rows between unbounded preceding and current row) as s1
+from part
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part
+#### A masked pattern was here ####
+Manufacturer#1 almond antique burnished rose metallic 2 1 1 1173.15
+Manufacturer#1 almond antique burnished rose metallic 2 1 1 2346.3
+Manufacturer#1 almond antique chartreuse lavender yellow 34 3 2 4100.06
+Manufacturer#1 almond antique salmon chartreuse burlywood 6 4 3 5702.650000000001
+Manufacturer#1 almond aquamarine burnished black steel 28 5 4 7117.070000000001
+Manufacturer#1 almond aquamarine pink moccasin thistle 42 6 5 8749.730000000001
+Manufacturer#2 almond antique violet chocolate turquoise 14 1 1 1690.68
+Manufacturer#2 almond antique violet turquoise frosted 40 2 2 3491.38
+Manufacturer#2 almond aquamarine midnight light salmon 2 3 3 5523.360000000001
+Manufacturer#2 almond aquamarine rose maroon antique 25 4 4 7222.02
+Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 5 5 8923.62
+Manufacturer#3 almond antique chartreuse khaki white 17 1 1 1671.68
+Manufacturer#3 almond antique forest lavender goldenrod 14 2 2 2861.95
+Manufacturer#3 almond antique metallic orange dim 19 3 3 4272.34
+Manufacturer#3 almond antique misty red olive 1 4 4 6195.32
+Manufacturer#3 almond antique olive coral navajo 45 5 5 7532.61
+Manufacturer#4 almond antique gainsboro frosted violet 10 1 1 1620.67
+Manufacturer#4 almond antique violet mint lemon 39 2 2 2996.09
+Manufacturer#4 almond aquamarine floral ivory bisque 27 3 3 4202.35
+Manufacturer#4 almond aquamarine yellow dodger mint 7 4 4 6047.27
+Manufacturer#4 almond azure aquamarine papaya violet 12 5 5 7337.620000000001
+Manufacturer#5 almond antique blue firebrick mint 31 1 1 1789.69
+Manufacturer#5 almond antique medium spring khaki 6 2 2 3401.3500000000004
+Manufacturer#5 almond antique sky peru orange 2 3 3 5190.08
+Manufacturer#5 almond aquamarine dodger light gainsboro 46 4 4 6208.18
+Manufacturer#5 almond azure blanched chiffon midnight 23 5 5 7672.66
+PREHOOK: query: select p_mfgr,p_name, p_size,
+sum(p_size) over (distribute by p_mfgr sort by p_name rows between current row and current row) as s2,
+first_value(p_size) over w1 as f,
+last_value(p_size, false) over w1 as l
+from part
+window w1 as (distribute by p_mfgr sort by p_name rows between 2 preceding and 2 following)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part
+#### A masked pattern was here ####
+POSTHOOK: query: select p_mfgr,p_name, p_size,
+sum(p_size) over (distribute by p_mfgr sort by p_name rows between current row and current row) as s2,
+first_value(p_size) over w1 as f,
+last_value(p_size, false) over w1 as l
+from part
+window w1 as (distribute by p_mfgr sort by p_name rows between 2 preceding and 2 following)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part
+#### A masked pattern was here ####
+Manufacturer#1 almond antique burnished rose metallic 2 2 2 34
+Manufacturer#1 almond antique burnished rose metallic 2 2 2 6
+Manufacturer#1 almond antique chartreuse lavender yellow 34 34 2 28
+Manufacturer#1 almond antique salmon chartreuse burlywood 6 6 2 42
+Manufacturer#1 almond aquamarine burnished black steel 28 28 34 42
+Manufacturer#1 almond aquamarine pink moccasin thistle 42 42 6 42
+Manufacturer#2 almond antique violet chocolate turquoise 14 14 14 2
+Manufacturer#2 almond antique violet turquoise frosted 40 40 14 25
+Manufacturer#2 almond aquamarine midnight light salmon 2 2 14 18
+Manufacturer#2 almond aquamarine rose maroon antique 25 25 40 18
+Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 18 2 18
+Manufacturer#3 almond antique chartreuse khaki white 17 17 17 19
+Manufacturer#3 almond antique forest lavender goldenrod 14 14 17 1
+Manufacturer#3 almond antique metallic orange dim 19 19 17 45
+Manufacturer#3 almond antique misty red olive 1 1 14 45
+Manufacturer#3 almond antique olive coral navajo 45 45 19 45
+Manufacturer#4 almond antique gainsboro frosted violet 10 10 10 27
+Manufacturer#4 almond antique violet mint lemon 39 39 10 7
+Manufacturer#4 almond aquamarine floral ivory bisque 27 27 10 12
+Manufacturer#4 almond aquamarine yellow dodger mint 7 7 39 12
+Manufacturer#4 almond azure aquamarine papaya violet 12 12 27 12
+Manufacturer#5 almond antique blue firebrick mint 31 31 31 2
+Manufacturer#5 almond antique medium spring khaki 6 6 31 46
+Manufacturer#5 almond antique sky peru orange 2 2 31 23
+Manufacturer#5 almond aquamarine dodger light gainsboro 46 46 6 23
+Manufacturer#5 almond azure blanched chiffon midnight 23 23 2 23
+PREHOOK: query: select p_mfgr,p_name, p_size,
+rank() over(distribute by p_mfgr sort by p_name) as r,
+sum(p_size) over (distribute by p_mfgr sort by p_name rows between current row and current row) as s2,
+first_value(p_size) over w1 as f,
+last_value(p_size, false) over w1 as l
+from part
+where p_mfgr = 'Manufacturer#3'
+window w1 as (distribute by p_mfgr sort by p_name rows between 2 preceding and 2 following)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part
+#### A masked pattern was here ####
+POSTHOOK: query: select p_mfgr,p_name, p_size,
+rank() over(distribute by p_mfgr sort by p_name) as r,
+sum(p_size) over (distribute by p_mfgr sort by p_name rows between current row and current row) as s2,
+first_value(p_size) over w1 as f,
+last_value(p_size, false) over w1 as l
+from part
+where p_mfgr = 'Manufacturer#3'
+window w1 as (distribute by p_mfgr sort by p_name rows between 2 preceding and 2 following)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part
+#### A masked pattern was here ####
+Manufacturer#3 almond antique chartreuse khaki white 17 1 17 17 19
+Manufacturer#3 almond antique forest lavender goldenrod 14 2 14 17 1
+Manufacturer#3 almond antique metallic orange dim 19 3 19 17 45
+Manufacturer#3 almond antique misty red olive 1 4 1 14 45
+Manufacturer#3 almond antique olive coral navajo 45 5 45 19 45
+PREHOOK: query: select p_mfgr,p_name, p_size,
+sum(p_size) over w1 as s1,
+sum(p_size) over (distribute by p_mfgr sort by p_name rows between current row and current row) as s2
+from part
+window w1 as (distribute by p_mfgr sort by p_name rows between 2 preceding and 2 following)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part
+#### A masked pattern was here ####
+POSTHOOK: query: select p_mfgr,p_name, p_size,
+sum(p_size) over w1 as s1,
+sum(p_size) over (distribute by p_mfgr sort by p_name rows between current row and current row) as s2
+from part
+window w1 as (distribute by p_mfgr sort by p_name rows between 2 preceding and 2 following)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part
+#### A masked pattern was here ####
+Manufacturer#1 almond antique burnished rose metallic 2 38 2
+Manufacturer#1 almond antique burnished rose metallic 2 44 2
+Manufacturer#1 almond antique chartreuse lavender yellow 34 72 34
+Manufacturer#1 almond antique salmon chartreuse burlywood 6 112 6
+Manufacturer#1 almond aquamarine burnished black steel 28 110 28
+Manufacturer#1 almond aquamarine pink moccasin thistle 42 76 42
+Manufacturer#2 almond antique violet chocolate turquoise 14 56 14
+Manufacturer#2 almond antique violet turquoise frosted 40 81 40
+Manufacturer#2 almond aquamarine midnight light salmon 2 99 2
+Manufacturer#2 almond aquamarine rose maroon antique 25 85 25
+Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 45 18
+Manufacturer#3 almond antique chartreuse khaki white 17 50 17
+Manufacturer#3 almond antique forest lavender goldenrod 14 51 14
+Manufacturer#3 almond antique metallic orange dim 19 96 19
+Manufacturer#3 almond antique misty red olive 1 79 1
+Manufacturer#3 almond antique olive coral navajo 45 65 45
+Manufacturer#4 almond antique gainsboro frosted violet 10 76 10
+Manufacturer#4 almond antique violet mint lemon 39 83 39
+Manufacturer#4 almond aquamarine floral ivory bisque 27 95 27
+Manufacturer#4 almond aquamarine yellow dodger mint 7 85 7
+Manufacturer#4 almond azure aquamarine papaya violet 12 46 12
+Manufacturer#5 almond antique blue firebrick mint 31 39 31
+Manufacturer#5 almond antique medium spring khaki 6 85 6
+Manufacturer#5 almond antique sky peru orange 2 108 2
+Manufacturer#5 almond aquamarine dodger light gainsboro 46 77 46
+Manufacturer#5 almond azure blanched chiffon midnight 23 71 23
+PREHOOK: query: select p_mfgr,p_name, p_size,
+rank() over(distribute by p_mfgr sort by p_name) as r, dense_rank() over(distribute by p_mfgr sort by p_name) as dr
+from part
+window w1 as (distribute by p_mfgr sort by p_name rows between 2 preceding and 2 following)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part
+#### A masked pattern was here ####
+POSTHOOK: query: select p_mfgr,p_name, p_size,
+rank() over(distribute by p_mfgr sort by p_name) as r, dense_rank() over(distribute by p_mfgr sort by p_name) as dr
+from part
+window w1 as (distribute by p_mfgr sort by p_name rows between 2 preceding and 2 following)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part
+#### A masked pattern was here ####
+Manufacturer#1 almond antique burnished rose metallic 2 1 1
+Manufacturer#1 almond antique burnished rose metallic 2 1 1
+Manufacturer#1 almond antique chartreuse lavender yellow 34 3 2
+Manufacturer#1 almond antique salmon chartreuse burlywood 6 4 3
+Manufacturer#1 almond aquamarine burnished black steel 28 5 4
+Manufacturer#1 almond aquamarine pink moccasin thistle 42 6 5
+Manufacturer#2 almond antique violet chocolate turquoise 14 1 1
+Manufacturer#2 almond antique violet turquoise frosted 40 2 2
+Manufacturer#2 almond aquamarine midnight light salmon 2 3 3
+Manufacturer#2 almond aquamarine rose maroon antique 25 4 4
+Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 5 5
+Manufacturer#3 almond antique chartreuse khaki white 17 1 1
+Manufacturer#3 almond antique forest lavender goldenrod 14 2 2
+Manufacturer#3 almond antique metallic orange dim 19 3 3
+Manufacturer#3 almond antique misty red olive 1 4 4
+Manufacturer#3 almond antique olive coral navajo 45 5 5
+Manufacturer#4 almond antique gainsboro frosted violet 10 1 1
+Manufacturer#4 almond antique violet mint lemon 39 2 2
+Manufacturer#4 almond aquamarine floral ivory bisque 27 3 3
+Manufacturer#4 almond aquamarine yellow dodger mint 7 4 4
+Manufacturer#4 almond azure aquamarine papaya violet 12 5 5
+Manufacturer#5 almond antique blue firebrick mint 31 1 1
+Manufacturer#5 almond antique medium spring khaki 6 2 2
+Manufacturer#5 almond antique sky peru orange 2 3 3
+Manufacturer#5 almond aquamarine dodger light gainsboro 46 4 4
+Manufacturer#5 almond azure blanched chiffon midnight 23 5 5
+PREHOOK: query: select p_mfgr,p_name, p_size,
+rank() over(distribute by p_mfgr sort by p_name) as r,
+dense_rank() over(distribute by p_mfgr sort by p_name) as dr,
+cume_dist() over(distribute by p_mfgr sort by p_name) as cud,
+percent_rank() over(distribute by p_mfgr sort by p_name) as pr,
+ntile(3) over(distribute by p_mfgr sort by p_name) as nt,
+count(p_size) over(distribute by p_mfgr sort by p_name) as ca,
+avg(p_size) over(distribute by p_mfgr sort by p_name) as avg,
+stddev(p_size) over(distribute by p_mfgr sort by p_name) as st,
+first_value(p_size % 5) over(distribute by p_mfgr sort by p_name) as fv,
+last_value(p_size) over(distribute by p_mfgr sort by p_name) as lv,
+first_value(p_size) over w1 as fvW1
+from part
+window w1 as (distribute by p_mfgr sort by p_mfgr, p_name rows between 2 preceding and 2 following)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part
+#### A masked pattern was here ####
+POSTHOOK: query: select p_mfgr,p_name, p_size,
+rank() over(distribute by p_mfgr sort by p_name) as r,
+dense_rank() over(distribute by p_mfgr sort by p_name) as dr,
+cume_dist() over(distribute by p_mfgr sort by p_name) as cud,
+percent_rank() over(distribute by p_mfgr sort by p_name) as pr,
+ntile(3) over(distribute by p_mfgr sort by p_name) as nt,
+count(p_size) over(distribute by p_mfgr sort by p_name) as ca,
+avg(p_size) over(distribute by p_mfgr sort by p_name) as avg,
+stddev(p_size) over(distribute by p_mfgr sort by p_name) as st,
+first_value(p_size % 5) over(distribute by p_mfgr sort by p_name) as fv,
+last_value(p_size) over(distribute by p_mfgr sort by p_name) as lv,
+first_value(p_size) over w1 as fvW1
+from part
+window w1 as (distribute by p_mfgr sort by p_mfgr, p_name rows between 2 preceding and 2 following)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part
+#### A masked pattern was here ####
+Manufacturer#1 almond antique burnished rose metallic 2 1 1 0.3333333333333333 0.0 1 2 2.0 0.0 2 2 2
+Manufacturer#1 almond antique burnished rose metallic 2 1 1 0.3333333333333333 0.0 1 2 2.0 0.0 2 2 2
+Manufacturer#1 almond antique chartreuse lavender yellow 34 3 2 0.5 0.4 2 3 12.666666666666666 15.084944665313014 2 34 2
+Manufacturer#1 almond antique salmon chartreuse burlywood 6 4 3 0.6666666666666666 0.6 2 4 11.0 13.379088160259652 2 6 2
+Manufacturer#1 almond aquamarine burnished black steel 28 5 4 0.8333333333333334 0.8 3 5 14.4 13.763720427268202 2 28 34
+Manufacturer#1 almond aquamarine pink moccasin thistle 42 6 5 1.0 1.0 3 6 19.0 16.237815945091466 2 42 6
+Manufacturer#2 almond antique violet chocolate turquoise 14 1 1 0.2 0.0 1 1 14.0 0.0 4 14 14
+Manufacturer#2 almond antique violet turquoise frosted 40 2 2 0.4 0.25 1 2 27.0 13.0 4 40 14
+Manufacturer#2 almond aquamarine midnight light salmon 2 3 3 0.6 0.5 2 3 18.666666666666668 15.86050300449376 4 2 14
+Manufacturer#2 almond aquamarine rose maroon antique 25 4 4 0.8 0.75 2 4 20.25 14.00669482783144 4 25 40
+Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 5 5 1.0 1.0 3 5 19.8 12.560254774486067 4 18 2
+Manufacturer#3 almond antique chartreuse khaki white 17 1 1 0.2 0.0 1 1 17.0 0.0 2 17 17
+Manufacturer#3 almond antique forest lavender goldenrod 14 2 2 0.4 0.25 1 2 15.5 1.5 2 14 17
+Manufacturer#3 almond antique metallic orange dim 19 3 3 0.6 0.5 2 3 16.666666666666668 2.0548046676563256 2 19 17
+Manufacturer#3 almond antique misty red olive 1 4 4 0.8 0.75 2 4 12.75 7.013380069552769 2 1 14
+Manufacturer#3 almond antique olive coral navajo 45 5 5 1.0 1.0 3 5 19.2 14.344336861632886 2 45 19
+Manufacturer#4 almond antique gainsboro frosted violet 10 1 1 0.2 0.0 1 1 10.0 0.0 0 10 10
+Manufacturer#4 almond antique violet mint lemon 39 2 2 0.4 0.25 1 2 24.5 14.5 0 39 10
+Manufacturer#4 almond aquamarine floral ivory bisque 27 3 3 0.6 0.5 2 3 25.333333333333332 11.897712198383164 0 27 10
+Manufacturer#4 almond aquamarine yellow dodger mint 7 4 4 0.8 0.75 2 4 20.75 13.007209539328564 0 7 39
+Manufacturer#4 almond azure aquamarine papaya violet 12 5 5 1.0 1.0 3 5 19.0 12.149074038789951 0 12 27
+Manufacturer#5 almond antique blue firebrick mint 31 1 1 0.2 0.0 1 1 31.0 0.0 1 31 31
+Manufacturer#5 almond antique medium spring khaki 6 2 2 0.4 0.25 1 2 18.5 12.5 1 6 31
+Manufacturer#5 almond antique sky peru orange 2 3 3 0.6 0.5 2 3 13.0 12.832251036613439 1 2 31
+Manufacturer#5 almond aquamarine dodger light gainsboro 46 4 4 0.8 0.75 2 4 21.25 18.102140757380052 1 46 6
+Manufacturer#5 almond azure blanched chiffon midnight 23 5 5 1.0 1.0 3 5 21.6 16.206171663906314 1 23 2
+PREHOOK: query: select p_mfgr,p_name, p_size,
+ rank() over(distribute by p_mfgr sort by p_name) as r,
+ dense_rank() over(distribute by p_mfgr sort by p_name) as dr,
+cume_dist() over(distribute by p_mfgr sort by p_name) as cud,
+sum(p_size) over (distribute by p_mfgr sort by p_name range between unbounded preceding and current row) as s1,
+sum(p_size) over (distribute by p_mfgr sort by p_size range between 5 preceding and current row) as s2,
+first_value(p_size) over w1 as fv1
+from part
+window w1 as (distribute by p_mfgr sort by p_mfgr, p_name rows between 2 preceding and 2 following)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part
+#### A masked pattern was here ####
+POSTHOOK: query: select p_mfgr,p_name, p_size,
+ rank() over(distribute by p_mfgr sort by p_name) as r,
+ dense_rank() over(distribute by p_mfgr sort by p_name) as dr,
+cume_dist() over(distribute by p_mfgr sort by p_name) as cud,
+sum(p_size) over (distribute by p_mfgr sort by p_name range between unbounded preceding and current row) as s1,
+sum(p_size) over (distribute by p_mfgr sort by p_size range between 5 preceding and current row) as s2,
+first_value(p_size) over w1 as fv1
+from part
+window w1 as (distribute by p_mfgr sort by p_mfgr, p_name rows between 2 preceding and 2 following)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part
+#### A masked pattern was here ####
+Manufacturer#1 almond antique burnished rose metallic 2 1 1 0.3333333333333333 4 4 2
+Manufacturer#1 almond antique burnished rose metallic 2 1 1 0.3333333333333333 4 4 2
+Manufacturer#1 almond antique chartreuse lavender yellow 34 3 2 0.5 38 34 2
+Manufacturer#1 almond antique salmon chartreuse burlywood 6 4 3 0.6666666666666666 44 10 2
+Manufacturer#1 almond aquamarine burnished black steel 28 5 4 0.8333333333333334 72 28 34
+Manufacturer#1 almond aquamarine pink moccasin thistle 42 6 5 1.0 114 42 6
+Manufacturer#2 almond antique violet chocolate turquoise 14 1 1 0.2 14 14 14
+Manufacturer#2 almond antique violet turquoise frosted 40 2 2 0.4 54 40 14
+Manufacturer#2 almond aquamarine midnight light salmon 2 3 3 0.6 56 2 14
+Manufacturer#2 almond aquamarine rose maroon antique 25 4 4 0.8 81 25 40
+Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 5 5 1.0 99 32 2
+Manufacturer#3 almond antique chartreuse khaki white 17 1 1 0.2 17 31 17
+Manufacturer#3 almond antique forest lavender goldenrod 14 2 2 0.4 31 14 17
+Manufacturer#3 almond antique metallic orange dim 19 3 3 0.6 50 50 17
+Manufacturer#3 almond antique misty red olive 1 4 4 0.8 51 1 14
+Manufacturer#3 almond antique olive coral navajo 45 5 5 1.0 96 45 19
+Manufacturer#4 almond antique gainsboro frosted violet 10 1 1 0.2 10 17 10
+Manufacturer#4 almond antique violet mint lemon 39 2 2 0.4 49 39 10
+Manufacturer#4 almond aquamarine floral ivory bisque 27 3 3 0.6 76 27 10
+Manufacturer#4 almond aquamarine yellow dodger mint 7 4 4 0.8 83 7 39
+Manufacturer#4 almond azure aquamarine papaya violet 12 5 5 1.0 95 29 27
+Manufacturer#5 almond antique blue firebrick mint 31 1 1 0.2 31 31 31
+Manufacturer#5 almond antique medium spring khaki 6 2 2 0.4 37 8 31
+Manufacturer#5 almond antique sky peru orange 2 3 3 0.6 39 2 31
+Manufacturer#5 almond aquamarine dodger light gainsboro 46 4 4 0.8 85 46 6
+Manufacturer#5 almond azure blanched chiffon midnight 23 5 5 1.0 108 23 2
+PREHOOK: query: select p_mfgr,p_name, p_size,
+count(*) over(distribute by p_mfgr sort by p_name ) as c,
+count(p_size) over(distribute by p_mfgr sort by p_name) as ca,
+first_value(p_size) over w1 as fvW1
+from part
+window w1 as (distribute by p_mfgr sort by p_mfgr, p_name rows between 2 preceding and 2 following)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part
+#### A masked pattern was here ####
+POSTHOOK: query: select p_mfgr,p_name, p_size,
+count(*) over(distribute by p_mfgr sort by p_name ) as c,
+count(p_size) over(distribute by p_mfgr sort by p_name) as ca,
+first_value(p_size) over w1 as fvW1
+from part
+window w1 as (distribute by p_mfgr sort by p_mfgr, p_name rows between 2 preceding and 2 following)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part
+#### A masked pattern was here ####
+Manufacturer#1 almond antique burnished rose metallic 2 2 2 2
+Manufacturer#1 almond antique burnished rose metallic 2 2 2 2
+Manufacturer#1 almond antique chartreuse lavender yellow 34 3 3 2
+Manufacturer#1 almond antique salmon chartreuse burlywood 6 4 4 2
+Manufacturer#1 almond aquamarine burnished black steel 28 5 5 34
+Manufacturer#1 almond aquamarine pink moccasin thistle 42 6 6 6
+Manufacturer#2 almond antique violet chocolate turquoise 14 1 1 14
+Manufacturer#2 almond antique violet turquoise frosted 40 2 2 14
+Manufacturer#2 almond aquamarine midnight light salmon 2 3 3 14
+Manufacturer#2 almond aquamarine rose maroon antique 25 4 4 40
+Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 5 5 2
+Manufacturer#3 almond antique chartreuse khaki white 17 1 1 17
+Manufacturer#3 almond antique forest lavender goldenrod 14 2 2 17
+Manufacturer#3 almond antique metallic orange dim 19 3 3 17
+Manufacturer#3 almond antique misty red olive 1 4 4 14
+Manufacturer#3 almond antique olive coral navajo 45 5 5 19
+Manufacturer#4 almond antique gainsboro frosted violet 10 1 1 10
+Manufacturer#4 almond antique violet mint lemon 39 2 2 10
+Manufacturer#4 almond aquamarine floral ivory bisque 27 3 3 10
+Manufacturer#4 almond aquamarine yellow dodger mint 7 4 4 39
+Manufacturer#4 almond azure aquamarine papaya violet 12 5 5 27
+Manufacturer#5 almond antique blue firebrick mint 31 1 1 31
+Manufacturer#5 almond antique medium spring khaki 6 2 2 31
+Manufacturer#5 almond antique sky peru orange 2 3 3 31
+Manufacturer#5 almond aquamarine dodger light gainsboro 46 4 4 6
+Manufacturer#5 almond azure blanched chiffon midnight 23 5 5 2
+PREHOOK: query: select p_mfgr,p_name, p_size,
+sum(p_retailprice) over w1 as s,
+min(p_retailprice) over w1 as mi,
+max(p_retailprice) over w1 as ma,
+avg(p_retailprice) over w1 as ag
+from part
+window w1 as (distribute by p_mfgr sort by p_mfgr, p_name rows between 2 preceding and 2 following)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part
+#### A masked pattern was here ####
+POSTHOOK: query: select p_mfgr,p_name, p_size,
+sum(p_retailprice) over w1 as s,
+min(p_retailprice) over w1 as mi,
+max(p_retailprice) over w1 as ma,
+avg(p_retailprice) over w1 as ag
+from part
+window w1 as (distribute by p_mfgr sort by p_mfgr, p_name rows between 2 preceding and 2 following)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part
+#### A masked pattern was here ####
+Manufacturer#1 almond antique burnished rose metallic 2 4100.06 1173.15 1753.76 1366.6866666666667
+Manufacturer#1 almond antique burnished rose metallic 2 5702.650000000001 1173.15 1753.76 1425.6625000000001
+Manufacturer#1 almond antique chartreuse lavender yellow 34 7117.070000000001 1173.15 1753.76 1423.4140000000002
+Manufacturer#1 almond antique salmon chartreuse burlywood 6 7576.580000000002 1173.15 1753.76 1515.3160000000003
+Manufacturer#1 almond aquamarine burnished black steel 28 6403.430000000001 1414.42 1753.76 1600.8575000000003
+Manufacturer#1 almond aquamarine pink moccasin thistle 42 4649.670000000001 1414.42 1632.66 1549.8900000000003
+Manufacturer#2 almond antique violet chocolate turquoise 14 5523.360000000001 1690.68 2031.98 1841.1200000000001
+Manufacturer#2 almond antique violet turquoise frosted 40 7222.02 1690.68 2031.98 1805.505
+Manufacturer#2 almond aquamarine midnight light salmon 2 8923.62 1690.68 2031.98 1784.7240000000002
+Manufacturer#2 almond aquamarine rose maroon antique 25 7232.9400000000005 1698.66 2031.98 1808.2350000000001
+Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 5432.240000000001 1698.66 2031.98 1810.746666666667
+Manufacturer#3 almond antique chartreuse khaki white 17 4272.34 1190.27 1671.68 1424.1133333333335
+Manufacturer#3 almond antique forest lavender goldenrod 14 6195.32 1190.27 1922.98 1548.83
+Manufacturer#3 almond antique metallic orange dim 19 7532.61 1190.27 1922.98 1506.522
+Manufacturer#3 almond antique misty red olive 1 5860.929999999999 1190.27 1922.98 1465.2324999999998
+Manufacturer#3 almond antique olive coral navajo 45 4670.66 1337.29 1922.98 1556.8866666666665
+Manufacturer#4 almond antique gainsboro frosted violet 10 4202.35 1206.26 1620.67 1400.7833333333335
+Manufacturer#4 almond antique violet mint lemon 39 6047.27 1206.26 1844.92 1511.8175
+Manufacturer#4 almond aquamarine floral ivory bisque 27 7337.620000000001 1206.26 1844.92 1467.5240000000001
+Manufacturer#4 almond aquamarine yellow dodger mint 7 5716.950000000001 1206.26 1844.92 1429.2375000000002
+Manufacturer#4 almond azure aquamarine papaya violet 12 4341.530000000001 1206.26 1844.92 1447.176666666667
+Manufacturer#5 almond antique blue firebrick mint 31 5190.08 1611.66 1789.69 1730.0266666666666
+Manufacturer#5 almond antique medium spring khaki 6 6208.18 1018.1 1789.69 1552.045
+Manufacturer#5 almond antique sky peru orange 2 7672.66 1018.1 1789.69 1534.532
+Manufacturer#5 almond aquamarine dodger light gainsboro 46 5882.969999999999 1018.1 1788.73 1470.7424999999998
+Manufacturer#5 almond azure blanched chiffon midnight 23 4271.3099999999995 1018.1 1788.73 1423.7699999999998
+PREHOOK: query: select p_mfgr,p_name, p_size, p_retailprice,
+sum(p_retailprice) over w1 as s,
+min(p_retailprice) as mi ,
+max(p_retailprice) as ma ,
+avg(p_retailprice) over w1 as ag
+from part
+group by p_mfgr,p_name, p_size, p_retailprice
+window w1 as (distribute by p_mfgr sort by p_mfgr, p_name rows between 2 preceding and 2 following)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part
+#### A masked pattern was here ####
+POSTHOOK: query: select p_mfgr,p_name, p_size, p_retailprice,
+sum(p_retailprice) over w1 as s,
+min(p_retailprice) as mi ,
+max(p_retailprice) as ma ,
+avg(p_retailprice) over w1 as ag
+from part
+group by p_mfgr,p_name, p_size, p_retailprice
+window w1 as (distribute by p_mfgr sort by p_mfgr, p_name rows between 2 preceding and 2 following)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part
+#### A masked pattern was here ####
+Manufacturer#1 almond antique burnished rose metallic 2 1173.15 4529.5 1173.15 1173.15 1509.8333333333333
+Manufacturer#1 almond antique chartreuse lavender yellow 34 1753.76 5943.92 1753.76 1753.76 1485.98
+Manufacturer#1 almond antique salmon chartreuse burlywood 6 1602.59 7576.58 1602.59 1602.59 1515.316
+Manufacturer#1 almond aquamarine burnished black steel 28 1414.42 6403.43 1414.42 1414.42 1600.8575
+Manufacturer#1 almond aquamarine pink moccasin thistle 42 1632.66 4649.67 1632.66 1632.66 1549.89
+Manufacturer#2 almond antique violet chocolate turquoise 14 1690.68 5523.360000000001 1690.68 1690.68 1841.1200000000001
+Manufacturer#2 almond antique violet turquoise frosted 40 1800.7 7222.02 1800.7 1800.7 1805.505
+Manufacturer#2 almond aquamarine midnight light salmon 2 2031.98 8923.62 2031.98 2031.98 1784.7240000000002
+Manufacturer#2 almond aquamarine rose maroon antique 25 1698.66 7232.9400000000005 1698.66 1698.66 1808.2350000000001
+Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 1701.6 5432.240000000001 1701.6 1701.6 1810.746666666667
+Manufacturer#3 almond antique chartreuse khaki white 17 1671.68 4272.34 1671.68 1671.68 1424.1133333333335
+Manufacturer#3 almond antique forest lavender goldenrod 14 1190.27 6195.32 1190.27 1190.27 1548.83
+Manufacturer#3 almond antique metallic orange dim 19 1410.39 7532.61 1410.39 1410.39 1506.522
+Manufacturer#3 almond antique misty red olive 1 1922.98 5860.929999999999 1922.98 1922.98 1465.2324999999998
+Manufacturer#3 almond antique olive coral navajo 45 1337.29 4670.66 1337.29 1337.29 1556.8866666666665
+Manufacturer#4 almond antique gainsboro frosted violet 10 1620.67 4202.35 1620.67 1620.67 1400.7833333333335
+Manufacturer#4 almond antique violet mint lemon 39 1375.42 6047.27 1375.42 1375.42 1511.8175
+Manufacturer#4 almond aquamarine floral ivory bisque 27 1206.26 7337.620000000001 1206.26 1206.26 1467.5240000000001
+Manufacturer#4 almond aquamarine yellow dodger mint 7 1844.92 5716.950000000001 1844.92 1844.92 1429.2375000000002
+Manufacturer#4 almond azure aquamarine papaya violet 12 1290.35 4341.530000000001 1290.35 1290.35 1447.176666666667
+Manufacturer#5 almond antique blue firebrick mint 31 1789.69 5190.08 1789.69 1789.69 1730.0266666666666
+Manufacturer#5 almond antique medium spring khaki 6 1611.66 6208.18 1611.66 1611.66 1552.045
+Manufacturer#5 almond antique sky peru orange 2 1788.73 7672.66 1788.73 1788.73 1534.532
+Manufacturer#5 almond aquamarine dodger light gainsboro 46 1018.1 5882.969999999999 1018.1 1018.1 1470.7424999999998
+Manufacturer#5 almond azure blanched chiffon midnight 23 1464.48 4271.3099999999995 1464.48 1464.48 1423.7699999999998
+PREHOOK: query: select p_mfgr,p_name, p_size,
+stddev(p_retailprice) over w1 as sdev,
+stddev_pop(p_retailprice) over w1 as sdev_pop,
+collect_set(p_size) over w1 as uniq_size,
+variance(p_retailprice) over w1 as var,
+corr(p_size, p_retailprice) over w1 as cor,
+covar_pop(p_size, p_retailprice) over w1 as covarp
+from part
+window w1 as (distribute by p_mfgr sort by p_mfgr, p_name rows between 2 preceding and 2 following)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part
+#### A masked pattern was here ####
+POSTHOOK: query: select p_mfgr,p_name, p_size,
+stddev(p_retailprice) over w1 as sdev,
+stddev_pop(p_retailprice) over w1 as sdev_pop,
+collect_set(p_size) over w1 as uniq_size,
+variance(p_retailprice) over w1 as var,
+corr(p_size, p_retailprice) over w1 as cor,
+covar_pop(p_size, p_retailprice) over w1 as covarp
+from part
+window w1 as (distribute by p_mfgr sort by p_mfgr, p_name rows between 2 preceding and 2 following)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part
+#### A masked pattern was here ####
+Manufacturer#1 almond antique burnished rose metallic 2 258.10677784349235 258.10677784349235 [2,34,6] 66619.10876874991 0.811328754177887 2801.7074999999995
+Manufacturer#1 almond antique burnished rose metallic 2 273.70217881648074 273.70217881648074 [2,34] 74912.8826888888 1.0 4128.782222222221
+Manufacturer#1 almond antique chartreuse lavender yellow 34 230.90151585470358 230.90151585470358 [2,34,6,28] 53315.51002399992 0.695639377397664 2210.7864
+Manufacturer#1 almond antique salmon chartreuse burlywood 6 202.73109328368946 202.73109328368946 [2,34,6,28,42] 41099.896184 0.630785977101214 2009.9536000000007
+Manufacturer#1 almond aquamarine burnished black steel 28 121.6064517973862 121.6064517973862 [34,6,28,42] 14788.129118750014 0.2036684720435979 331.1337500000004
+Manufacturer#1 almond aquamarine pink moccasin thistle 42 96.5751586416853 96.5751586416853 [6,28,42] 9326.761266666683 -1.4442181184933883E-4 -0.20666666666708502
+Manufacturer#2 almond antique violet chocolate turquoise 14 142.2363169751898 142.2363169751898 [14,40,2] 20231.169866666663 -0.49369526554523185 -1113.7466666666658
+Manufacturer#2 almond antique violet turquoise frosted 40 137.76306498840682 137.76306498840682 [14,40,2,25] 18978.662075 -0.5205630897335946 -1004.4812499999995
+Manufacturer#2 almond aquamarine midnight light salmon 2 130.03972279269132 130.03972279269132 [14,40,2,25,18] 16910.329504000005 -0.46908967495720255 -766.1791999999995
+Manufacturer#2 almond aquamarine rose maroon antique 25 135.55100986344584 135.55100986344584 [40,2,25,18] 18374.07627499999 -0.6091405874714462 -1128.1787499999987
+Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 156.44019460768044 156.44019460768044 [2,25,18] 24473.534488888927 -0.9571686373491608 -1441.4466666666676
+Manufacturer#3 almond antique chartreuse khaki white 17 196.7742266885805 196.7742266885805 [17,14,19] 38720.09628888887 0.5557168646224995 224.6944444444446
+Manufacturer#3 almond antique forest lavender goldenrod 14 275.14144189852607 275.14144189852607 [17,14,19,1] 75702.81305 -0.6720833036576083 -1296.9000000000003
+Manufacturer#3 almond antique metallic orange dim 19 260.23473614412046 260.23473614412046 [17,14,19,1,45] 67722.117896 -0.5703526513979519 -2129.0664
+Manufacturer#3 almond antique misty red olive 1 275.9139962356932 275.9139962356932 [14,19,1,45] 76128.53331875012 -0.577476899644802 -2547.7868749999993
+Manufacturer#3 almond antique olive coral navajo 45 260.5815918713796 260.5815918713796 [19,1,45] 67902.76602222225 -0.8710736366736884 -4099.731111111111
+Manufacturer#4 almond antique gainsboro frosted violet 10 170.13011889596618 170.13011889596618 [10,39,27] 28944.25735555559 -0.6656975320098423 -1347.4777777777779
+Manufacturer#4 almond antique violet mint lemon 39 242.26834609323197 242.26834609323197 [10,39,27,7] 58693.95151875002 -0.8051852719193339 -2537.328125
+Manufacturer#4 almond aquamarine floral ivory bisque 27 234.10001662537326 234.10001662537326 [10,39,27,7,12] 54802.817784000035 -0.6046935574240581 -1719.8079999999995
+Manufacturer#4 almond aquamarine yellow dodger mint 7 247.3342714197732 247.3342714197732 [39,27,7,12] 61174.24181875003 -0.5508665654707869 -1719.0368749999975
+Manufacturer#4 almond azure aquamarine papaya violet 12 283.3344330566893 283.3344330566893 [27,7,12] 80278.40095555557 -0.7755740084632333 -1867.4888888888881
+Manufacturer#5 almond antique blue firebrick mint 31 83.69879024746363 83.69879024746363 [31,6,2] 7005.487488888913 0.39004303087285047 418.9233333333353
+Manufacturer#5 almond antique medium spring khaki 6 316.68049612345885 316.68049612345885 [31,6,2,46] 100286.53662500004 -0.713612911776183 -4090.853749999999
+Manufacturer#5 almond antique sky peru orange 2 285.40506298242155 285.40506298242155 [31,6,2,46,23] 81456.04997600002 -0.712858514567818 -3297.2011999999986
+Manufacturer#5 almond aquamarine dodger light gainsboro 46 285.43749038756283 285.43749038756283 [6,2,46,23] 81474.56091875004 -0.984128787153391 -4871.028125000002
+Manufacturer#5 almond azure blanched chiffon midnight 23 315.9225931564038 315.9225931564038 [2,46,23] 99807.08486666664 -0.9978877469246936 -5664.856666666666
+PREHOOK: query: select p_mfgr,p_name, p_size,
+histogram_numeric(p_retailprice, 5) over w1 as hist,
+percentile(p_partkey, 0.5) over w1 as per,
+row_number() over(distribute by p_mfgr sort by p_mfgr, p_name) as rn
+from part
+window w1 as (distribute by p_mfgr sort by p_mfgr, p_name rows between 2 preceding and 2 following)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part
+#### A masked pattern was here ####
+POSTHOOK: query: select p_mfgr,p_name, p_size,
+histogram_numeric(p_retailprice, 5) over w1 as hist,
+percentile(p_partkey, 0.5) over w1 as per,
+row_number() over(distribute by p_mfgr sort by p_mfgr, p_name) as rn
+from part
+window w1 as (distribute by p_mfgr sort by p_mfgr, p_name rows between 2 preceding and 2 following)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part
+#### A masked pattern was here ####
+Manufacturer#1 almond antique burnished rose metallic 2 [{"x":1173.15,"y":2.0},{"x":1602.59,"y":1.0},{"x":1753.76,"y":1.0}] 115872.0 2
+Manufacturer#1 almond antique burnished rose metallic 2 [{"x":1173.15,"y":2.0},{"x":1753.76,"y":1.0}] 121152.0 1
+Manufacturer#1 almond antique chartreuse lavender yellow 34 [{"x":1173.15,"y":2.0},{"x":1414.42,"y":1.0},{"x":1602.59,"y":1.0},{"x":1753.76,"y":1.0}] 110592.0 3
+Manufacturer#1 almond antique salmon chartreuse burlywood 6 [{"x":1173.15,"y":1.0},{"x":1414.42,"y":1.0},{"x":1602.59,"y":1.0},{"x":1632.66,"y":1.0},{"x":1753.76,"y":1.0}] 86428.0 4
+Manufacturer#1 almond aquamarine burnished black steel 28 [{"x":1414.42,"y":1.0},{"x":1602.59,"y":1.0},{"x":1632.66,"y":1.0},{"x":1753.76,"y":1.0}] 86098.0 5
+Manufacturer#1 almond aquamarine pink moccasin thistle 42 [{"x":1414.42,"y":1.0},{"x":1602.59,"y":1.0},{"x":1632.66,"y":1.0}] 86428.0 6
+Manufacturer#2 almond antique violet chocolate turquoise 14 [{"x":1690.68,"y":1.0},{"x":1800.7,"y":1.0},{"x":2031.98,"y":1.0}] 146985.0 1
+Manufacturer#2 almond antique violet turquoise frosted 40 [{"x":1690.68,"y":1.0},{"x":1698.66,"y":1.0},{"x":1800.7,"y":1.0},{"x":2031.98,"y":1.0}] 139825.5 2
+Manufacturer#2 almond aquamarine midnight light salmon 2 [{"x":1690.68,"y":1.0},{"x":1698.66,"y":1.0},{"x":1701.6,"y":1.0},{"x":1800.7,"y":1.0},{"x":2031.98,"y":1.0}] 146985.0 3
+Manufacturer#2 almond aquamarine rose maroon antique 25 [{"x":1698.66,"y":1.0},{"x":1701.6,"y":1.0},{"x":1800.7,"y":1.0},{"x":2031.98,"y":1.0}] 169347.0 4
+Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 [{"x":1698.66,"y":1.0},{"x":1701.6,"y":1.0},{"x":2031.98,"y":1.0}] 146985.0 5
+Manufacturer#3 almond antique chartreuse khaki white 17 [{"x":1190.27,"y":1.0},{"x":1410.39,"y":1.0},{"x":1671.68,"y":1.0}] 90681.0 1
+Manufacturer#3 almond antique forest lavender goldenrod 14 [{"x":1190.27,"y":1.0},{"x":1410.39,"y":1.0},{"x":1671.68,"y":1.0},{"x":1922.98,"y":1.0}] 65831.5 2
+Manufacturer#3 almond antique metallic orange dim 19 [{"x":1190.27,"y":1.0},{"x":1337.29,"y":1.0},{"x":1410.39,"y":1.0},{"x":1671.68,"y":1.0},{"x":1922.98,"y":1.0}] 90681.0 3
+Manufacturer#3 almond antique misty red olive 1 [{"x":1190.27,"y":1.0},{"x":1337.29,"y":1.0},{"x":1410.39,"y":1.0},{"x":1922.98,"y":1.0}] 76690.0 4
+Manufacturer#3 almond antique olive coral navajo 45 [{"x":1337.29,"y":1.0},{"x":1410.39,"y":1.0},{"x":1922.98,"y":1.0}] 112398.0 5
+Manufacturer#4 almond antique gainsboro frosted violet 10 [{"x":1206.26,"y":1.0},{"x":1375.42,"y":1.0},{"x":1620.67,"y":1.0}] 48427.0 1
+Manufacturer#4 almond antique violet mint lemon 39 [{"x":1206.26,"y":1.0},{"x":1375.42,"y":1.0},{"x":1620.67,"y":1.0},{"x":1844.92,"y":1.0}] 46844.0 2
+Manufacturer#4 almond aquamarine floral ivory bisque 27 [{"x":1206.26,"y":1.0},{"x":1290.35,"y":1.0},{"x":1375.42,"y":1.0},{"x":1620.67,"y":1.0},{"x":1844.92,"y":1.0}] 45261.0 3
+Manufacturer#4 almond aquamarine yellow dodger mint 7 [{"x":1206.26,"y":1.0},{"x":1290.35,"y":1.0},{"x":1375.42,"y":1.0},{"x":1844.92,"y":1.0}] 39309.0 4
+Manufacturer#4 almond azure aquamarine papaya violet 12 [{"x":1206.26,"y":1.0},{"x":1290.35,"y":1.0},{"x":1844.92,"y":1.0}] 33357.0 5
+Manufacturer#5 almond antique blue firebrick mint 31 [{"x":1611.66,"y":1.0},{"x":1788.73,"y":1.0},{"x":1789.69,"y":1.0}] 155733.0 1
+Manufacturer#5 almond antique medium spring khaki 6 [{"x":1018.1,"y":1.0},{"x":1611.66,"y":1.0},{"x":1788.73,"y":1.0},{"x":1789.69,"y":1.0}] 99201.0 2
+Manufacturer#5 almond antique sky peru orange 2 [{"x":1018.1,"y":1.0},{"x":1464.48,"y":1.0},{"x":1611.66,"y":1.0},{"x":1788.73,"y":1.0},{"x":1789.69,"y":1.0}] 78486.0 3
+Manufacturer#5 almond aquamarine dodger light gainsboro 46 [{"x":1018.1,"y":1.0},{"x":1464.48,"y":1.0},{"x":1611.66,"y":1.0},{"x":1788.73,"y":1.0}] 60577.5 4
+Manufacturer#5 almond azure blanched chiffon midnight 23 [{"x":1018.1,"y":1.0},{"x":1464.48,"y":1.0},{"x":1788.73,"y":1.0}] 78486.0 5
+PREHOOK: query: create view IF NOT EXISTS mfgr_price_view as
+select p_mfgr, p_brand,
+round(sum(p_retailprice),2) as s
+from part
+group by p_mfgr, p_brand
+PREHOOK: type: CREATEVIEW
+PREHOOK: Input: default@part
+PREHOOK: Output: database:default
+PREHOOK: Output: default@mfgr_price_view
+POSTHOOK: query: create view IF NOT EXISTS mfgr_price_view as
+select p_mfgr, p_brand,
+round(sum(p_retailprice),2) as s
+from part
+group by p_mfgr, p_brand
+POSTHOOK: type: CREATEVIEW
+POSTHOOK: Input: default@part
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@mfgr_price_view
+POSTHOOK: Lineage: mfgr_price_view.p_brand SIMPLE [(part)part.FieldSchema(name:p_brand, type:string, comment:null), ]
+POSTHOOK: Lineage: mfgr_price_view.p_mfgr SIMPLE [(part)part.FieldSchema(name:p_mfgr, type:string, comment:null), ]
+POSTHOOK: Lineage: mfgr_price_view.s EXPRESSION [(part)part.FieldSchema(name:p_retailprice, type:double, comment:null), ]
+PREHOOK: query: select *
+from (
+select p_mfgr, p_brand, s,
+round(sum(s) over w1 , 2) as s1
+from mfgr_price_view
+window w1 as (distribute by p_mfgr sort by p_mfgr )
+) sq
+order by p_mfgr, p_brand
+PREHOOK: type: QUERY
+PREHOOK: Input: default@mfgr_price_view
+PREHOOK: Input: default@part
+#### A masked pattern was here ####
+POSTHOOK: query: select *
+from (
+select p_mfgr, p_brand, s,
+round(sum(s) over w1 , 2) as s1
+from mfgr_price_view
+window w1 as (distribute by p_mfgr sort by p_mfgr )
+) sq
+order by p_mfgr, p_brand
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@mfgr_price_view
+POSTHOOK: Input: default@part
+#### A masked pattern was here ####
+Manufacturer#1 Brand#12 4800.84 8749.73
+Manufacturer#1 Brand#14 2346.3 8749.73
+Manufacturer#1 Brand#15 1602.59 8749.73
+Manufacturer#2 Brand#22 3491.38 8923.62
+Manufacturer#2 Brand#23 2031.98 8923.62
+Manufacturer#2 Brand#24 1698.66 8923.62
+Manufacturer#2 Brand#25 1701.6 8923.62
+Manufacturer#3 Brand#31 1671.68 7532.61
+Manufacturer#3 Brand#32 3333.37 7532.61
+Manufacturer#3 Brand#34 1337.29 7532.61
+Manufacturer#3 Brand#35 1190.27 7532.61
+Manufacturer#4 Brand#41 4755.94 7337.62
+Manufacturer#4 Brand#42 2581.68 7337.62
+Manufacturer#5 Brand#51 1611.66 7672.66
+Manufacturer#5 Brand#52 3254.17 7672.66
+Manufacturer#5 Brand#53 2806.83 7672.66
+PREHOOK: query: select p_mfgr, p_brand, s,
+round(sum(s) over w1 ,2) as s1
+from mfgr_price_view
+window w1 as (distribute by p_mfgr sort by p_brand rows between 2 preceding and current row)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@mfgr_price_view
+PREHOOK: Input: default@part
+#### A masked pattern was here ####
+POSTHOOK: query: select p_mfgr, p_brand, s,
+round(sum(s) over w1 ,2) as s1
+from mfgr_price_view
+window w1 as (distribute by p_mfgr sort by p_brand rows between 2 preceding and current row)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@mfgr_price_view
+POSTHOOK: Input: default@part
+#### A masked pattern was here ####
+Manufacturer#1 Brand#12 4800.84 4800.84
+Manufacturer#1 Brand#14 2346.3 7147.14
+Manufacturer#1 Brand#15 1602.59 8749.73
+Manufacturer#2 Brand#22 3491.38 3491.38
+Manufacturer#2 Brand#23 2031.98 5523.36
+Manufacturer#2 Brand#24 1698.66 7222.02
+Manufacturer#2 Brand#25 1701.6 5432.24
+Manufacturer#3 Brand#31 1671.68 1671.68
+Manufacturer#3 Brand#32 3333.37 5005.05
+Manufacturer#3 Brand#34 1337.29 6342.34
+Manufacturer#3 Brand#35 1190.27 5860.93
+Manufacturer#4 Brand#41 4755.94 4755.94
+Manufacturer#4 Brand#42 2581.68 7337.62
+Manufacturer#5 Brand#51 1611.66 1611.66
+Manufacturer#5 Brand#52 3254.17 4865.83
+Manufacturer#5 Brand#53 2806.83 7672.66
+PREHOOK: query: create view IF NOT EXISTS mfgr_brand_price_view as
+select p_mfgr, p_brand,
+sum(p_retailprice) over w1 as s
+from part
+window w1 as (distribute by p_mfgr sort by p_name rows between 2 preceding and current row)
+PREHOOK: type: CREATEVIEW
+PREHOOK: Input: default@part
+PREHOOK: Output: database:default
+PREHOOK: Output: default@mfgr_brand_price_view
+POSTHOOK: query: create view IF NOT EXISTS mfgr_brand_price_view as
+select p_mfgr, p_brand,
+sum(p_retailprice) over w1 as s
+from part
+window w1 as (distribute by p_mfgr sort by p_name rows between 2 preceding and current row)
+POSTHOOK: type: CREATEVIEW
+POSTHOOK: Input: default@part
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@mfgr_brand_price_view
+POSTHOOK: Lineage: mfgr_brand_price_view.p_brand SIMPLE [(part)part.FieldSchema(name:p_brand, type:string, comment:null), ]
+POSTHOOK: Lineage: mfgr_brand_price_view.p_mfgr SIMPLE [(part)part.FieldSchema(name:p_mfgr, type:string, comment:null), ]
+POSTHOOK: Lineage: mfgr_brand_price_view.s SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ]
+PREHOOK: query: select * from mfgr_brand_price_view
+PREHOOK: type: QUERY
+PREHOOK: Input: default@mfgr_brand_price_view
+PREHOOK: Input: default@part
+#### A masked pattern was here ####
+POSTHOOK: query: select * from mfgr_brand_price_view
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@mfgr_brand_price_view
+POSTHOOK: Input: default@part
+#### A masked pattern was here ####
+Manufacturer#1 Brand#12 4100.06
+Manufacturer#1 Brand#12 4649.670000000001
+Manufacturer#1 Brand#12 4770.77
+Manufacturer#1 Brand#14 1173.15
+Manufacturer#1 Brand#14 2346.3
+Manufacturer#1 Brand#15 4529.5
+Manufacturer#2 Brand#22 1690.68
+Manufacturer#2 Brand#22 3491.38
+Manufacturer#2 Brand#23 5523.360000000001
+Manufacturer#2 Brand#24 5531.34
+Manufacturer#2 Brand#25 5432.240000000001
+Manufacturer#3 Brand#31 1671.68
+Manufacturer#3 Brand#32 4272.34
+Manufacturer#3 Brand#32 4523.639999999999
+Manufacturer#3 Brand#34 4670.66
+Manufacturer#3 Brand#35 2861.95
+Manufacturer#4 Brand#41 1620.67
+Manufacturer#4 Brand#41 4341.530000000001
+Manufacturer#4 Brand#41 4426.6
+Manufacturer#4 Brand#42 2996.09
+Manufacturer#4 Brand#42 4202.35
+Manufacturer#5 Brand#51 3401.3500000000004
+Manufacturer#5 Brand#52 1789.69
+Manufacturer#5 Brand#52 4271.3099999999995
+Manufacturer#5 Brand#53 4418.49
+Manufacturer#5 Brand#53 5190.08
+PREHOOK: query: select p_mfgr, p_name,
+lv_col, p_size, sum(p_size) over w1 as s
+from (select p_mfgr, p_name, p_size, array(1,2,3) arr from part) p
+lateral view explode(arr) part_lv as lv_col
+window w1 as (distribute by p_mfgr sort by p_size, lv_col rows between 2 preceding and current row)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part
+#### A masked pattern was here ####
+POSTHOOK: query: select p_mfgr, p_name,
+lv_col, p_size, sum(p_size) over w1 as s
+from (select p_mfgr, p_name, p_size, array(1,2,3) arr from part) p
+lateral view explode(arr) part_lv as lv_col
+window w1 as (distribute by p_mfgr sort by p_size, lv_col rows between 2 preceding and current row)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part
+#### A masked pattern was here ####
+Manufacturer#1 almond antique burnished rose metallic 1 2 2
+Manufacturer#1 almond antique burnished rose metallic 1 2 4
+Manufacturer#1 almond antique burnished rose metallic 2 2 6
+Manufacturer#1 almond antique burnished rose metallic 2 2 6
+Manufacturer#1 almond antique burnished rose metallic 3 2 6
+Manufacturer#1 almond antique burnished rose metallic 3 2 6
+Manufacturer#1 almond antique chartreuse lavender yellow 1 34 90
+Manufacturer#1 almond antique chartreuse lavender yellow 2 34 96
+Manufacturer#1 almond antique chartreuse lavender yellow 3 34 102
+Manufacturer#1 almond antique salmon chartreuse burlywood 1 6 10
+Manufacturer#1 almond antique salmon chartreuse burlywood 2 6 14
+Manufacturer#1 almond antique salmon chartreuse burlywood 3 6 18
+Manufacturer#1 almond aquamarine burnished black steel 1 28 40
+Manufacturer#1 almond aquamarine burnished black steel 2 28 62
+Manufacturer#1 almond aquamarine burnished black steel 3 28 84
+Manufacturer#1 almond aquamarine pink moccasin thistle 1 42 110
+Manufacturer#1 almond aquamarine pink moccasin thistle 2 42 118
+Manufacturer#1 almond aquamarine pink moccasin thistle 3 42 126
+Manufacturer#2 almond antique violet chocolate turquoise 1 14 18
+Manufacturer#2 almond antique violet chocolate turquoise 2 14 30
+Manufacturer#2 almond antique violet chocolate turquoise 3 14 42
+Manufacturer#2 almond antique violet turquoise frosted 1 40 90
+Manufacturer#2 almond antique violet turquoise frosted 2 40 105
+Manufacturer#2 almond antique violet turquoise frosted 3 40 120
+Manufacturer#2 almond aquamarine midnight light salmon 1 2 2
+Manufacturer#2 almond aquamarine midnight light salmon 2 2 4
+Manufacturer#2 almond aquamarine midnight light salmon 3 2 6
+Manufacturer#2 almond aquamarine rose maroon antique 1 25 61
+Manufacturer#2 almond aquamarine rose maroon antique 2 25 68
+Manufacturer#2 almond aquamarine rose maroon antique 3 25 75
+Manufacturer#2 almond aquamarine sandy cyan gainsboro 1 18 46
+Manufacturer#2 almond aquamarine sandy cyan gainsboro 2 18 50
+Manufacturer#2 almond aquamarine sandy cyan gainsboro 3 18 54
+Manufacturer#3 almond antique chartreuse khaki white 1 17 45
+Manufacturer#3 almond antique chartreuse khaki white 2 17 48
+Manufacturer#3 almond antique chartreuse khaki white 3 17 51
+Manufacturer#3 almond antique forest lavender goldenrod 1 14 16
+Manufacturer#3 almond antique forest lavender goldenrod 2 14 29
+Manufacturer#3 almond antique forest lavender goldenrod 3 14 42
+Manufacturer#3 almond antique metallic orange dim 1 19 53
+Manufacturer#3 almond antique metallic orange dim 2 19 55
+Manufacturer#3 almond antique metallic orange dim 3 19 57
+Manufacturer#3 almond antique misty red olive 1 1 1
+Manufacturer#3 almond antique misty red olive 2 1 2
+Manufacturer#3 almond antique misty red olive 3 1 3
+Manufacturer#3 almond antique olive coral navajo 1 45 83
+Manufacturer#3 almond antique olive coral navajo 2 45 109
+Manufacturer#3 almond antique olive coral navajo 3 45 135
+Manufacturer#4 almond antique gainsboro frosted violet 1 10 24
+Manufacturer#4 almond antique gainsboro frosted violet 2 10 27
+Manufacturer#4 almond antique gainsboro frosted violet 3 10 30
+Manufacturer#4 almond antique violet mint lemon 1 39 93
+Manufacturer#4 almond antique violet mint lemon 2 39 105
+Manufacturer#4 almond antique violet mint lemon 3 39 117
+Manufacturer#4 almond aquamarine floral ivory bisque 1 27 51
+Manufacturer#4 almond aquamarine floral ivory bisque 2 27 66
+Manufacturer#4 almond aquamarine floral ivory bisque 3 27 81
+Manufacturer#4 almond aquamarine yellow dodger mint 1 7 7
+Manufacturer#4 almond aquamarine yellow dodger mint 2 7 14
+Manufacturer#4 almond aquamarine yellow dodger mint 3 7 21
+Manufacturer#4 almond azure aquamarine papaya violet 1 12 32
+Manufacturer#4 almond azure aquamarine papaya violet 2 12 34
+Manufacturer#4 almond azure aquamarine papaya violet 3 12 36
+Manufacturer#5 almond antique blue firebrick mint 1 31 77
+Manufacturer#5 almond antique blue firebrick mint 2 31 85
+Manufacturer#5 almond antique blue firebrick mint 3 31 93
+Manufacturer#5 almond antique medium spring khaki 1 6 10
+Manufacturer#5 almond antique medium spring khaki 2 6 14
+Manufacturer#5 almond antique medium spring khaki 3 6 18
+Manufacturer#5 almond antique sky peru orange 1 2 2
+Manufacturer#5 almond antique sky peru orange 2 2 4
+Manufacturer#5 almond antique sky peru orange 3 2 6
+Manufacturer#5 almond aquamarine dodger light gainsboro 1 46 108
+Manufacturer#5 almond aquamarine dodger light gainsboro 2 46 123
+Manufacturer#5 almond aquamarine dodger light gainsboro 3 46 138
+Manufacturer#5 almond azure blanched chiffon midnight 1 23 35
+Manufacturer#5 almond azure blanched chiffon midnight 2 23 52
+Manufacturer#5 almond azure blanched chiffon midnight 3 23 69
+PREHOOK: query: CREATE TABLE part_1(
+p_mfgr STRING,
+p_name STRING,
+p_size INT,
+r INT,
+dr INT,
+s DOUBLE)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@part_1
+POSTHOOK: query: CREATE TABLE part_1(
+p_mfgr STRING,
+p_name STRING,
+p_size INT,
+r INT,
+dr INT,
+s DOUBLE)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@part_1
+PREHOOK: query: CREATE TABLE part_2(
+p_mfgr STRING,
+p_name STRING,
+p_size INT,
+r INT,
+dr INT,
+cud INT,
+s2 DOUBLE,
+fv1 INT)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@part_2
+POSTHOOK: query: CREATE TABLE part_2(
+p_mfgr STRING,
+p_name STRING,
+p_size INT,
+r INT,
+dr INT,
+cud INT,
+s2 DOUBLE,
+fv1 INT)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@part_2
+PREHOOK: query: CREATE TABLE part_3(
+p_mfgr STRING,
+p_name STRING,
+p_size INT,
+c INT,
+ca INT,
+fv INT)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@part_3
+POSTHOOK: query: CREATE TABLE part_3(
+p_mfgr STRING,
+p_name STRING,
+p_size INT,
+c INT,
+ca INT,
+fv INT)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@part_3
+PREHOOK: query: from part
+INSERT OVERWRITE TABLE part_1
+select p_mfgr, p_name, p_size,
+rank() over(distribute by p_mfgr sort by p_name ) as r,
+dense_rank() over(distribute by p_mfgr sort by p_name ) as dr,
+sum(p_retailprice) over (distribute by p_mfgr sort by p_name rows between unbounded preceding and current row) as s
+INSERT OVERWRITE TABLE part_2
+select p_mfgr,p_name, p_size,
+rank() over(distribute by p_mfgr sort by p_name) as r,
+dense_rank() over(distribute by p_mfgr sort by p_name) as dr,
+cume_dist() over(distribute by p_mfgr sort by p_name) as cud,
+round(sum(p_size) over (distribute by p_mfgr sort by p_size range between 5 preceding and current row),1) as s2,
+first_value(p_size) over w1 as fv1
+window w1 as (distribute by p_mfgr sort by p_mfgr, p_name rows between 2 preceding and 2 following)
+INSERT OVERWRITE TABLE part_3
+select p_mfgr,p_name, p_size,
+count(*) over(distribute by p_mfgr sort by p_name) as c,
+count(p_size) over(distribute by p_mfgr sort by p_name) as ca,
+first_value(p_size) over w1 as fv
+window w1 as (distribute by p_mfgr sort by p_mfgr, p_name rows between 2 preceding and 2 following)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part
+PREHOOK: Output: default@part_1
+PREHOOK: Output: default@part_2
+PREHOOK: Output: default@part_3
+POSTHOOK: query: from part
+INSERT OVERWRITE TABLE part_1
+select p_mfgr, p_name, p_size,
+rank() over(distribute by p_mfgr sort by p_name ) as r,
+dense_rank() over(distribute by p_mfgr sort by p_name ) as dr,
+sum(p_retailprice) over (distribute by p_mfgr sort by p_name rows between unbounded preceding and current row) as s
+INSERT OVERWRITE TABLE part_2
+select p_mfgr,p_name, p_size,
+rank() over(distribute by p_mfgr sort by p_name) as r,
+dense_rank() over(distribute by p_mfgr sort by p_name) as dr,
+cume_dist() over(distribute by p_mfgr sort by p_name) as cud,
+round(sum(p_size) over (distribute by p_mfgr sort by p_size range between 5 preceding and current row),1) as s2,
+first_value(p_size) over w1 as fv1
+window w1 as (distribute by p_mfgr sort by p_mfgr, p_name rows between 2 preceding and 2 following)
+INSERT OVERWRITE TABLE part_3
+select p_mfgr,p_name, p_size,
+count(*) over(distribute by p_mfgr sort by p_name) as c,
+count(p_size) over(distribute by p_mfgr sort by p_name) as ca,
+first_value(p_size) over w1 as fv
+window w1 as (distribute by p_mfgr sort by p_mfgr, p_name rows between 2 preceding and 2 following)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part
+POSTHOOK: Output: default@part_1
+POSTHOOK: Output: default@part_2
+POSTHOOK: Output: default@part_3
+POSTHOOK: Lineage: part_1.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ]
+POSTHOOK: Lineage: part_1.p_mfgr SIMPLE [(part)part.FieldSchema(name:p_mfgr, type:string, comment:null), ]
+POSTHOOK: Lineage: part_1.p_name SIMPLE [(part)part.FieldSchema(name:p_name, type:string, comment:null), ]
+POSTHOOK: Lineage: part_1.p_size SIMPLE [(part)part.FieldSchema(name:p_size, type:int, comment:null), ]
+POSTHOOK: Lineage: part_1.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ]
+POSTHOOK: Lineage: part_1.s SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ]
+POSTHOOK: Lineage: part_2.cud SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ]
+POSTHOOK: Lineage: part_2.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ]
+POSTHOOK: Lineage: part_2.fv1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ]
+POSTHOOK: Lineage: part_2.p_mfgr SIMPLE [(part)part.FieldSchema(name:p_mfgr, type:string, comment:null), ]
+POSTHOOK: Lineage: part_2.p_name SIMPLE [(part)part.FieldSchema(name:p_name, type:string, comment:null), ]
+POSTHOOK: Lineage: part_2.p_size SIMPLE [(part)part.FieldSchema(name:p_size, type:int, comment:null), ]
+POSTHOOK: Lineage: part_2.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ]
+POSTHOOK: Lineage: part_2.s2 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ]
+POSTHOOK: Lineage: part_3.c SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ]
+POSTHOOK: Lineage: part_3.ca SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ]
+POSTHOOK: Lineage: part_3.fv SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ]
+POSTHOOK: Lineage: part_3.p_mfgr SIMPLE [(part)part.FieldSchema(name:p_mfgr, type:string, comment:null), ]
+POSTHOOK: Lineage: part_3.p_name SIMPLE [(part)part.FieldSchema(name:p_name, type:string, comment:null), ]
+POSTHOOK: Lineage: part_3.p_size SIMPLE [(part)part.FieldSchema(name:p_size, type:int, comment:null), ]
+PREHOOK: query: select * from part_1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part_1
+#### A masked pattern was here ####
+POSTHOOK: query: select * from part_1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part_1
+#### A masked pattern was here ####
+Manufacturer#1 almond antique burnished rose metallic 2 1 1 1173.15
+Manufacturer#1 almond antique burnished rose metallic 2 1 1 2346.3
+Manufacturer#1 almond antique chartreuse lavender yellow 34 3 2 4100.06
+Manufacturer#1 almond antique salmon chartreuse burlywood 6 4 3 5702.650000000001
+Manufacturer#1 almond aquamarine burnished black steel 28 5 4 7117.070000000001
+Manufacturer#1 almond aquamarine pink moccasin thistle 42 6 5 8749.730000000001
+Manufacturer#2 almond antique violet chocolate turquoise 14 1 1 1690.68
+Manufacturer#2 almond antique violet turquoise frosted 40 2 2 3491.38
+Manufacturer#2 almond aquamarine midnight light salmon 2 3 3 5523.360000000001
+Manufacturer#2 almond aquamarine rose maroon antique 25 4 4 7222.02
+Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 5 5 8923.62
+Manufacturer#3 almond antique chartreuse khaki white 17 1 1 1671.68
+Manufacturer#3 almond antique forest lavender goldenrod 14 2 2 2861.95
+Manufacturer#3 almond antique metallic orange dim 19 3 3 4272.34
+Manufacturer#3 almond antique misty red olive 1 4 4 6195.32
+Manufacturer#3 almond antique olive coral navajo 45 5 5 7532.61
+Manufacturer#4 almond antique gainsboro frosted violet 10 1 1 1620.67
+Manufacturer#4 almond antique violet mint lemon 39 2 2 2996.09
+Manufacturer#4 almond aquamarine floral ivory bisque 27 3 3 4202.35
+Manufacturer#4 almond aquamarine yellow dodger mint 7 4 4 6047.27
+Manufacturer#4 almond azure aquamarine papaya violet 12 5 5 7337.620000000001
+Manufacturer#5 almond antique blue firebrick mint 31 1 1 1789.69
+Manufacturer#5 almond antique medium spring khaki 6 2 2 3401.3500000000004
+Manufacturer#5 almond antique sky peru orange 2 3 3 5190.08
+Manufacturer#5 almond aquamarine dodger light gainsboro 46 4 4 6208.18
+Manufacturer#5 almond azure blanched chiffon midnight 23 5 5 7672.66
+PREHOOK: query: select * from part_2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part_2
+#### A masked pattern was here ####
+POSTHOOK: query: select * from part_2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part_2
+#### A masked pattern was here ####
+Manufacturer#1 almond antique burnished rose metallic 2 1 1 0 4.0 2
+Manufacturer#1 almond antique burnished rose metallic 2 1 1 0 4.0 2
+Manufacturer#1 almond antique chartreuse lavender yellow 34 3 2 0 34.0 2
+Manufacturer#1 almond antique salmon chartreuse burlywood 6 4 3 0 10.0 2
+Manufacturer#1 almond aquamarine burnished black steel 28 5 4 0 28.0 34
+Manufacturer#1 almond aquamarine pink moccasin thistle 42 6 5 1 42.0 6
+Manufacturer#2 almond antique violet chocolate turquoise 14 1 1 0 14.0 14
+Manufacturer#2 almond antique violet turquoise frosted 40 2 2 0 40.0 14
+Manufacturer#2 almond aquamarine midnight light salmon 2 3 3 0 2.0 14
+Manufacturer#2 almond aquamarine rose maroon antique 25 4 4 0 25.0 40
+Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 5 5 1 32.0 2
+Manufacturer#3 almond antique chartreuse khaki white 17 1 1 0 31.0 17
+Manufacturer#3 almond antique f
<TRUNCATED>
[08/50] [abbrv] hive git commit: HIVE-14367 : Estimated size for
constant nulls is 0 (Ashutosh Chauhan via Jesus Camacho Rodriguez)
Posted by om...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/vector_decimal_udf.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vector_decimal_udf.q.out b/ql/src/test/results/clientpositive/vector_decimal_udf.q.out
index 48654ca..165bae9 100644
--- a/ql/src/test/results/clientpositive/vector_decimal_udf.q.out
+++ b/ql/src/test/results/clientpositive/vector_decimal_udf.q.out
@@ -2374,20 +2374,20 @@ STAGE PLANS:
aggregations: histogram_numeric(_col0, 3)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 160 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 160 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: array<double>)
Reduce Operator Tree:
Group By Operator
aggregations: histogram_numeric(VALUE._col0)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 720 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 720 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/vector_elt.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vector_elt.q.out b/ql/src/test/results/clientpositive/vector_elt.q.out
index d494296..08ca167 100644
--- a/ql/src/test/results/clientpositive/vector_elt.q.out
+++ b/ql/src/test/results/clientpositive/vector_elt.q.out
@@ -104,10 +104,10 @@ STAGE PLANS:
Select Operator
expressions: 'defg' (type: string), 'cc' (type: string), 'abc' (type: string), '2' (type: string), '12345' (type: string), '123456789012' (type: string), '1.25' (type: string), '16.0' (type: string), null (type: string), null (type: string)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9
- Statistics: Num rows: 12288 Data size: 8687616 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 12288 Data size: 8687784 Basic stats: COMPLETE Column stats: COMPLETE
Limit
Number of rows: 1
- Statistics: Num rows: 1 Data size: 707 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 875 Basic stats: COMPLETE Column stats: COMPLETE
ListSink
PREHOOK: query: SELECT elt(2, 'abc', 'defg'),
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/vector_interval_arithmetic.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vector_interval_arithmetic.q.out b/ql/src/test/results/clientpositive/vector_interval_arithmetic.q.out
index ad37d13..dd4b7d1 100644
--- a/ql/src/test/results/clientpositive/vector_interval_arithmetic.q.out
+++ b/ql/src/test/results/clientpositive/vector_interval_arithmetic.q.out
@@ -478,24 +478,24 @@ STAGE PLANS:
alias: interval_arithmetic_1
Statistics: Num rows: 50 Data size: 4800 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
- Statistics: Num rows: 50 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 50 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: CAST( 5-5 AS INTERVAL YEAR TO MONTH) (type: interval_year_month)
sort order: +
- Statistics: Num rows: 50 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 50 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE
TopN Hash Memory Usage: 0.1
Execution mode: vectorized
Reduce Operator Tree:
Select Operator
expressions: 5-5 (type: interval_year_month), -1-1 (type: interval_year_month)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 50 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 50 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE
Limit
Number of rows: 2
- Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -975,13 +975,13 @@ STAGE PLANS:
Select Operator
expressions: 109 20:30:40.246913578 (type: interval_day_time), 89 02:14:26.000000000 (type: interval_day_time)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 50 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 50 Data size: 1200 Basic stats: COMPLETE Column stats: COMPLETE
Limit
Number of rows: 2
- Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/vector_null_projection.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vector_null_projection.q.out b/ql/src/test/results/clientpositive/vector_null_projection.q.out
index b9a4b05..f289fdd 100644
--- a/ql/src/test/results/clientpositive/vector_null_projection.q.out
+++ b/ql/src/test/results/clientpositive/vector_null_projection.q.out
@@ -52,10 +52,10 @@ STAGE PLANS:
Select Operator
expressions: null (type: void)
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -94,53 +94,53 @@ STAGE PLANS:
alias: a
Statistics: Num rows: 1 Data size: 87 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
Union
- Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
- Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
keys: null (type: void)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: void)
sort order: +
Map-reduce partition columns: _col0 (type: void)
- Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
TableScan
alias: b
Statistics: Num rows: 1 Data size: 87 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
Union
- Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
- Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
keys: null (type: void)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: void)
sort order: +
Map-reduce partition columns: _col0 (type: void)
- Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Operator Tree:
Group By Operator
keys: KEY._col0 (type: void)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: null (type: void)
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/vector_nvl.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vector_nvl.q.out b/ql/src/test/results/clientpositive/vector_nvl.q.out
index 69712aa..8330810 100644
--- a/ql/src/test/results/clientpositive/vector_nvl.q.out
+++ b/ql/src/test/results/clientpositive/vector_nvl.q.out
@@ -203,10 +203,10 @@ STAGE PLANS:
Select Operator
expressions: null (type: void)
outputColumnNames: _col0
- Statistics: Num rows: 12288 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 12288 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
Limit
Number of rows: 10
- Statistics: Num rows: 10 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 10 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
ListSink
PREHOOK: query: SELECT nvl(null, null) as n
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/vector_udf1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vector_udf1.q.out b/ql/src/test/results/clientpositive/vector_udf1.q.out
index a66e369..7ffd8a5 100644
--- a/ql/src/test/results/clientpositive/vector_udf1.q.out
+++ b/ql/src/test/results/clientpositive/vector_udf1.q.out
@@ -1710,20 +1710,20 @@ STAGE PLANS:
aggregations: compute_stats(_col0, 16), compute_stats(_col2, 16)
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 984 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 984 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: struct<columntype:string,maxlength:bigint,sumlength:bigint,count:bigint,countnulls:bigint,bitvector:string,numbitvectors:int>), _col1 (type: struct<columntype:string,maxlength:bigint,sumlength:bigint,count:bigint,countnulls:bigint,bitvector:string,numbitvectors:int>)
Reduce Operator Tree:
Group By Operator
aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 984 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 984 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -1781,10 +1781,10 @@ STAGE PLANS:
aggregations: min(c2), min(c4)
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 288 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 288 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string), _col1 (type: varchar(20))
Execution mode: vectorized
Reduce Operator Tree:
@@ -1792,10 +1792,10 @@ STAGE PLANS:
aggregations: min(VALUE._col0), min(VALUE._col1)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 288 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 288 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -1853,10 +1853,10 @@ STAGE PLANS:
aggregations: max(c2), max(c4)
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 288 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 288 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string), _col1 (type: varchar(20))
Execution mode: vectorized
Reduce Operator Tree:
@@ -1864,10 +1864,10 @@ STAGE PLANS:
aggregations: max(VALUE._col0), max(VALUE._col1)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 288 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 288 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/vectorization_0.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vectorization_0.q.out b/ql/src/test/results/clientpositive/vectorization_0.q.out
index 0564195..dabb643 100644
--- a/ql/src/test/results/clientpositive/vectorization_0.q.out
+++ b/ql/src/test/results/clientpositive/vectorization_0.q.out
@@ -235,10 +235,10 @@ STAGE PLANS:
aggregations: avg(ctinyint), variance(ctinyint), var_pop(ctinyint), var_samp(ctinyint), std(ctinyint), stddev(ctinyint), stddev_pop(ctinyint), stddev_samp(ctinyint)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 636 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 636 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: struct<count:bigint,sum:double,input:tinyint>), _col1 (type: struct<count:bigint,sum:double,variance:double>), _col2 (type: struct<count:bigint,sum:double,variance:double>), _col3 (type: struct<count:bigint,sum:double,variance:double>), _col4 (type: struct<count:bigint,sum:double,variance:double>), _col5 (type: struct<count:bigint,sum:double,variance:double>), _col6 (type: struct<count:bigint,sum:double,variance:double>), _col7 (type: struct<count:bigint,sum:double,variance:double>)
Execution mode: vectorized
Reduce Operator Tree:
@@ -246,7 +246,7 @@ STAGE PLANS:
aggregations: avg(VALUE._col0), variance(VALUE._col1), var_pop(VALUE._col2), var_samp(VALUE._col3), std(VALUE._col4), stddev(VALUE._col5), stddev_pop(VALUE._col6), stddev_samp(VALUE._col7)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
- Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 636 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
table:
@@ -261,16 +261,16 @@ STAGE PLANS:
Reduce Output Operator
key expressions: _col0 (type: double)
sort order: +
- Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 636 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: double), _col2 (type: double), _col3 (type: double), _col4 (type: double), _col5 (type: double), _col6 (type: double), _col7 (type: double)
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: double), VALUE._col0 (type: double), VALUE._col1 (type: double), VALUE._col2 (type: double), VALUE._col3 (type: double), VALUE._col4 (type: double), VALUE._col5 (type: double), VALUE._col6 (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
- Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 636 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 636 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -548,10 +548,10 @@ STAGE PLANS:
aggregations: avg(cbigint), variance(cbigint), var_pop(cbigint), var_samp(cbigint), std(cbigint), stddev(cbigint), stddev_pop(cbigint), stddev_samp(cbigint)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 640 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 640 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: struct<count:bigint,sum:double,input:bigint>), _col1 (type: struct<count:bigint,sum:double,variance:double>), _col2 (type: struct<count:bigint,sum:double,variance:double>), _col3 (type: struct<count:bigint,sum:double,variance:double>), _col4 (type: struct<count:bigint,sum:double,variance:double>), _col5 (type: struct<count:bigint,sum:double,variance:double>), _col6 (type: struct<count:bigint,sum:double,variance:double>), _col7 (type: struct<count:bigint,sum:double,variance:double>)
Execution mode: vectorized
Reduce Operator Tree:
@@ -559,7 +559,7 @@ STAGE PLANS:
aggregations: avg(VALUE._col0), variance(VALUE._col1), var_pop(VALUE._col2), var_samp(VALUE._col3), std(VALUE._col4), stddev(VALUE._col5), stddev_pop(VALUE._col6), stddev_samp(VALUE._col7)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
- Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 640 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
table:
@@ -574,16 +574,16 @@ STAGE PLANS:
Reduce Output Operator
key expressions: _col0 (type: double)
sort order: +
- Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 640 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: double), _col2 (type: double), _col3 (type: double), _col4 (type: double), _col5 (type: double), _col6 (type: double), _col7 (type: double)
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: double), VALUE._col0 (type: double), VALUE._col1 (type: double), VALUE._col2 (type: double), VALUE._col3 (type: double), VALUE._col4 (type: double), VALUE._col5 (type: double), VALUE._col6 (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
- Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 640 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 640 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -861,10 +861,10 @@ STAGE PLANS:
aggregations: avg(cfloat), variance(cfloat), var_pop(cfloat), var_samp(cfloat), std(cfloat), stddev(cfloat), stddev_pop(cfloat), stddev_samp(cfloat)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 636 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 636 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: struct<count:bigint,sum:double,input:float>), _col1 (type: struct<count:bigint,sum:double,variance:double>), _col2 (type: struct<count:bigint,sum:double,variance:double>), _col3 (type: struct<count:bigint,sum:double,variance:double>), _col4 (type: struct<count:bigint,sum:double,variance:double>), _col5 (type: struct<count:bigint,sum:double,variance:double>), _col6 (type: struct<count:bigint,sum:double,variance:double>), _col7 (type: struct<count:bigint,sum:double,variance:double>)
Execution mode: vectorized
Reduce Operator Tree:
@@ -872,7 +872,7 @@ STAGE PLANS:
aggregations: avg(VALUE._col0), variance(VALUE._col1), var_pop(VALUE._col2), var_samp(VALUE._col3), std(VALUE._col4), stddev(VALUE._col5), stddev_pop(VALUE._col6), stddev_samp(VALUE._col7)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
- Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 636 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
table:
@@ -887,16 +887,16 @@ STAGE PLANS:
Reduce Output Operator
key expressions: _col0 (type: double)
sort order: +
- Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 636 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: double), _col2 (type: double), _col3 (type: double), _col4 (type: double), _col5 (type: double), _col6 (type: double), _col7 (type: double)
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: double), VALUE._col0 (type: double), VALUE._col1 (type: double), VALUE._col2 (type: double), VALUE._col3 (type: double), VALUE._col4 (type: double), VALUE._col5 (type: double), VALUE._col6 (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
- Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 636 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 636 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -1014,10 +1014,10 @@ STAGE PLANS:
aggregations: avg(cbigint), stddev_pop(cbigint), var_samp(cbigint), count(), sum(cfloat), min(ctinyint)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
- Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 260 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 260 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: struct<count:bigint,sum:double,input:bigint>), _col1 (type: struct<count:bigint,sum:double,variance:double>), _col2 (type: struct<count:bigint,sum:double,variance:double>), _col3 (type: bigint), _col4 (type: double), _col5 (type: tinyint)
Execution mode: vectorized
Reduce Operator Tree:
@@ -1025,14 +1025,14 @@ STAGE PLANS:
aggregations: avg(VALUE._col0), stddev_pop(VALUE._col1), var_samp(VALUE._col2), count(VALUE._col3), sum(VALUE._col4), min(VALUE._col5)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
- Statistics: Num rows: 1 Data size: 44 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 260 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: double), (- _col0) (type: double), (-6432.0 + _col0) (type: double), _col1 (type: double), (- (-6432.0 + _col0)) (type: double), ((- (-6432.0 + _col0)) + (-6432.0 + _col0)) (type: double), _col2 (type: double), (- (-6432.0 + _col0)) (type: double), (-6432.0 + (- (-6432.0 + _col0))) (type: double), (- (-6432.0 + _col0)) (type: double), ((- (-6432.0 + _col0)) / (- (-6432.0 + _col0))) (type: double), _col3 (type: bigint), _col4 (type: double), (_col2 % _col1) (type: double), (- _col2) (type: double), ((- (-6432.0 + _col0)) * (- _col0)) (type: double), _col5 (type: tinyint), (- _col5) (type: tinyint)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17
- Statistics: Num rows: 1 Data size: 44 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 260 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 44 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 260 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/vectorization_pushdown.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vectorization_pushdown.q.out b/ql/src/test/results/clientpositive/vectorization_pushdown.q.out
index 6bbfed6..04780c4 100644
--- a/ql/src/test/results/clientpositive/vectorization_pushdown.q.out
+++ b/ql/src/test/results/clientpositive/vectorization_pushdown.q.out
@@ -26,10 +26,10 @@ STAGE PLANS:
aggregations: avg(cbigint)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: struct<count:bigint,sum:double,input:bigint>)
Execution mode: vectorized
Reduce Operator Tree:
@@ -37,10 +37,10 @@ STAGE PLANS:
aggregations: avg(VALUE._col0)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/vectorization_short_regress.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vectorization_short_regress.q.out b/ql/src/test/results/clientpositive/vectorization_short_regress.q.out
index 4ae2ad5..5dd1bb3 100644
--- a/ql/src/test/results/clientpositive/vectorization_short_regress.q.out
+++ b/ql/src/test/results/clientpositive/vectorization_short_regress.q.out
@@ -93,10 +93,10 @@ STAGE PLANS:
aggregations: avg(cint), sum(cdouble), stddev_pop(cint), stddev_samp(csmallint), var_samp(cint), avg(cfloat), stddev_samp(cint), min(ctinyint), count(csmallint)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
- Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 492 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 492 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: struct<count:bigint,sum:double,input:int>), _col1 (type: double), _col2 (type: struct<count:bigint,sum:double,variance:double>), _col3 (type: struct<count:bigint,sum:double,variance:double>), _col4 (type: struct<count:bigint,sum:double,variance:double>), _col5 (type: struct<count:bigint,sum:double,input:float>), _col6 (type: struct<count:bigint,sum:double,variance:double>), _col7 (type: tinyint), _col8 (type: bigint)
Execution mode: vectorized
Reduce Operator Tree:
@@ -104,14 +104,14 @@ STAGE PLANS:
aggregations: avg(VALUE._col0), sum(VALUE._col1), stddev_pop(VALUE._col2), stddev_samp(VALUE._col3), var_samp(VALUE._col4), avg(VALUE._col5), stddev_samp(VALUE._col6), min(VALUE._col7), count(VALUE._col8)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
- Statistics: Num rows: 1 Data size: 68 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 492 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: double), (_col0 + -3728.0) (type: double), (- (_col0 + -3728.0)) (type: double), (- (- (_col0 + -3728.0))) (type: double), ((- (- (_col0 + -3728.0))) * (_col0 + -3728.0)) (type: double), _col1 (type: double), (- _col0) (type: double), _col2 (type: double), (((- (- (_col0 + -3728.0))) * (_col0 + -3728.0)) * (- (- (_col0 + -3728.0)))) (type: double), _col3 (type: double), (- _col2) (type: double), (_col2 - (- (- (_col0 + -3728.0)))) (type: double), ((_col2 - (- (- (_col0 + -3728.0)))) * _col2) (type: double), _col4 (type: double), _col5 (type: double), (10.175 - _col4) (type: double), (- (10.175 - _col4)) (type: double), ((- _col2) / -563.0) (type: double), _col6 (type: double), (- ((- _col2) / -563.0)) (type: double), (_col0 / _col1) (type: double), _col7 (type: tinyint), _col8 (type: bigint), (UDFToDouble(_col7) / ((- _col2) / -563.0)) (type: double), (- (_col0 / _col1)) (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24
- Statistics: Num rows: 1 Data size: 68 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 492 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 68 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 492 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -289,10 +289,10 @@ STAGE PLANS:
aggregations: max(cint), var_pop(cbigint), stddev_pop(csmallint), max(cdouble), avg(ctinyint), min(cint), min(cdouble), stddev_samp(csmallint), var_samp(cint)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
- Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 420 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 420 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: int), _col1 (type: struct<count:bigint,sum:double,variance:double>), _col2 (type: struct<count:bigint,sum:double,variance:double>), _col3 (type: double), _col4 (type: struct<count:bigint,sum:double,input:tinyint>), _col5 (type: int), _col6 (type: double), _col7 (type: struct<count:bigint,sum:double,variance:double>), _col8 (type: struct<count:bigint,sum:double,variance:double>)
Execution mode: vectorized
Reduce Operator Tree:
@@ -300,14 +300,14 @@ STAGE PLANS:
aggregations: max(VALUE._col0), var_pop(VALUE._col1), stddev_pop(VALUE._col2), max(VALUE._col3), avg(VALUE._col4), min(VALUE._col5), min(VALUE._col6), stddev_samp(VALUE._col7), var_samp(VALUE._col8)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
- Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 420 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: int), (UDFToDouble(_col0) / -3728.0) (type: double), (_col0 * -3728) (type: int), _col1 (type: double), (- (_col0 * -3728)) (type: int), _col2 (type: double), (-563 % (_col0 * -3728)) (type: int), (_col1 / _col2) (type: double), (- _col2) (type: double), _col3 (type: double), _col4 (type: double), (_col2 - 10.175) (type: double), _col5 (type: int), (UDFToDouble((_col0 * -3728)) % (_col2 - 10.175)) (type: double), (- _col3) (type: double), _col6 (type: double), (_col3 % -26.28) (type: double), _col7 (type: double), (- (UDFToDouble(_col0) / -3728.0)) (type: double), ((- (_col0 * -3728)) % (-563 % (_col0 * -3728))) (type: int), ((UDFToDouble(_col0) / -3728.0) - _col4) (type: double), (- (_col0 * -3728)) (type: int), _col8 (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22
- Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 420 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 420 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -476,10 +476,10 @@ STAGE PLANS:
aggregations: var_pop(cbigint), count(), max(ctinyint), stddev_pop(csmallint), max(cint), stddev_samp(cdouble), count(ctinyint), avg(ctinyint)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
- Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 340 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 340 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: struct<count:bigint,sum:double,variance:double>), _col1 (type: bigint), _col2 (type: tinyint), _col3 (type: struct<count:bigint,sum:double,variance:double>), _col4 (type: int), _col5 (type: struct<count:bigint,sum:double,variance:double>), _col6 (type: bigint), _col7 (type: struct<count:bigint,sum:double,input:tinyint>)
Execution mode: vectorized
Reduce Operator Tree:
@@ -487,14 +487,14 @@ STAGE PLANS:
aggregations: var_pop(VALUE._col0), count(VALUE._col1), max(VALUE._col2), stddev_pop(VALUE._col3), max(VALUE._col4), stddev_samp(VALUE._col5), count(VALUE._col6), avg(VALUE._col7)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
- Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 340 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: double), (- _col0) (type: double), (_col0 - (- _col0)) (type: double), _col1 (type: bigint), (UDFToDouble(_col1) % 79.553) (type: double), _col2 (type: tinyint), (UDFToDouble(_col1) - (- _col0)) (type: double), (- (- _col0)) (type: double), (-1.0 % (- _col0)) (type: double), _col1 (type: bigint), (- _col1) (type: bigint), _col3 (type: double), (- (- (- _col0))) (type: double), (762 * (- _col1)) (type: bigint), _col4 (type: int), (UDFToLong(_col2) + (762 * (- _col1))) (type: bigint), ((- _col0) + UDFToDouble(_col4)) (type: double), _col5 (type: double), ((- _col1) % _col1) (type: bigint), _col6 (type: bigint), _col7 (type: double), (-3728 % (UDFToLong(_col2) + (762 * (- _col1)))) (type: bigint)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21
- Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 340 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 340 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -642,10 +642,10 @@ STAGE PLANS:
aggregations: avg(ctinyint), max(cbigint), stddev_samp(cint), var_pop(cint), var_pop(cbigint), max(cfloat)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
- Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 328 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 328 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: struct<count:bigint,sum:double,input:tinyint>), _col1 (type: bigint), _col2 (type: struct<count:bigint,sum:double,variance:double>), _col3 (type: struct<count:bigint,sum:double,variance:double>), _col4 (type: struct<count:bigint,sum:double,variance:double>), _col5 (type: float)
Execution mode: vectorized
Reduce Operator Tree:
@@ -653,14 +653,14 @@ STAGE PLANS:
aggregations: avg(VALUE._col0), max(VALUE._col1), stddev_samp(VALUE._col2), var_pop(VALUE._col3), var_pop(VALUE._col4), max(VALUE._col5)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
- Statistics: Num rows: 1 Data size: 44 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 328 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: double), (_col0 + 6981.0) (type: double), ((_col0 + 6981.0) + _col0) (type: double), _col1 (type: bigint), (((_col0 + 6981.0) + _col0) / _col0) (type: double), (- (_col0 + 6981.0)) (type: double), _col2 (type: double), (_col0 % (- (_col0 + 6981.0))) (type: double), _col3 (type: double), _col4 (type: double), (- _col1) (type: bigint), (UDFToDouble((- _col1)) / _col2) (type: double), _col5 (type: float), (_col4 * -26.28) (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13
- Statistics: Num rows: 1 Data size: 44 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 328 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 44 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 328 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/vectorized_distinct_gby.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vectorized_distinct_gby.q.out b/ql/src/test/results/clientpositive/vectorized_distinct_gby.q.out
index 64e4b01..f8ae962 100644
--- a/ql/src/test/results/clientpositive/vectorized_distinct_gby.q.out
+++ b/ql/src/test/results/clientpositive/vectorized_distinct_gby.q.out
@@ -111,10 +111,10 @@ STAGE PLANS:
aggregations: sum(DISTINCT KEY._col0:0._col0), count(DISTINCT KEY._col0:1._col0), avg(DISTINCT KEY._col0:2._col0), std(DISTINCT KEY._col0:3._col0)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 1 Data size: 32 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 180 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 32 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 180 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/vectorized_mapjoin.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vectorized_mapjoin.q.out b/ql/src/test/results/clientpositive/vectorized_mapjoin.q.out
index 46494db..a11e058 100644
--- a/ql/src/test/results/clientpositive/vectorized_mapjoin.q.out
+++ b/ql/src/test/results/clientpositive/vectorized_mapjoin.q.out
@@ -64,10 +64,10 @@ STAGE PLANS:
aggregations: count(_col0), max(_col1), min(_col0), avg(_col2)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: bigint), _col1 (type: int), _col2 (type: int), _col3 (type: struct<count:bigint,sum:double,input:int>)
Execution mode: vectorized
Local Work:
@@ -77,10 +77,10 @@ STAGE PLANS:
aggregations: count(VALUE._col0), max(VALUE._col1), min(VALUE._col2), avg(VALUE._col3)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/vectorized_shufflejoin.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vectorized_shufflejoin.q.out b/ql/src/test/results/clientpositive/vectorized_shufflejoin.q.out
index 4fd0aeb..992a1ef 100644
--- a/ql/src/test/results/clientpositive/vectorized_shufflejoin.q.out
+++ b/ql/src/test/results/clientpositive/vectorized_shufflejoin.q.out
@@ -63,7 +63,7 @@ STAGE PLANS:
aggregations: count(_col0), max(_col1), min(_col0), avg(_col2)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
table:
@@ -77,14 +77,14 @@ STAGE PLANS:
TableScan
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: bigint), _col1 (type: int), _col2 (type: int), _col3 (type: struct<count:bigint,sum:double,input:int>)
Reduce Operator Tree:
Group By Operator
aggregations: count(VALUE._col0), max(VALUE._col1), min(VALUE._col2), avg(VALUE._col3)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
table:
@@ -99,16 +99,16 @@ STAGE PLANS:
Reduce Output Operator
key expressions: _col0 (type: bigint)
sort order: +
- Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: double)
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: bigint), VALUE._col0 (type: int), VALUE._col1 (type: int), VALUE._col2 (type: double)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/vectorized_timestamp_funcs.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vectorized_timestamp_funcs.q.out b/ql/src/test/results/clientpositive/vectorized_timestamp_funcs.q.out
index baedd67..03c5ee4 100644
--- a/ql/src/test/results/clientpositive/vectorized_timestamp_funcs.q.out
+++ b/ql/src/test/results/clientpositive/vectorized_timestamp_funcs.q.out
@@ -758,10 +758,10 @@ STAGE PLANS:
aggregations: avg(ctimestamp1), variance(ctimestamp1), var_pop(ctimestamp1), var_samp(ctimestamp1), std(ctimestamp1), stddev(ctimestamp1), stddev_pop(ctimestamp1), stddev_samp(ctimestamp1)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 672 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 672 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: struct<count:bigint,sum:double,input:timestamp>), _col1 (type: struct<count:bigint,sum:double,variance:double>), _col2 (type: struct<count:bigint,sum:double,variance:double>), _col3 (type: struct<count:bigint,sum:double,variance:double>), _col4 (type: struct<count:bigint,sum:double,variance:double>), _col5 (type: struct<count:bigint,sum:double,variance:double>), _col6 (type: struct<count:bigint,sum:double,variance:double>), _col7 (type: struct<count:bigint,sum:double,variance:double>)
Execution mode: vectorized
Reduce Operator Tree:
@@ -769,14 +769,14 @@ STAGE PLANS:
aggregations: avg(VALUE._col0), variance(VALUE._col1), var_pop(VALUE._col2), var_samp(VALUE._col3), std(VALUE._col4), stddev(VALUE._col5), stddev_pop(VALUE._col6), stddev_samp(VALUE._col7)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
- Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 672 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: round(_col0, 0) (type: double), _col1 BETWEEN 8.97077295279421E19 AND 8.97077295279422E19 (type: boolean), _col2 BETWEEN 8.97077295279421E19 AND 8.97077295279422E19 (type: boolean), _col3 BETWEEN 9.20684592523616E19 AND 9.20684592523617E19 (type: boolean), round(_col4, 3) (type: double), round(_col5, 3) (type: double), round(_col6, 3) (type: double), round(_col7, 3) (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
- Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 672 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 672 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
[16/50] [abbrv] hive git commit: HIVE-14367 : Estimated size for
constant nulls is 0 (Ashutosh Chauhan via Jesus Camacho Rodriguez)
Posted by om...@apache.org.
HIVE-14367 : Estimated size for constant nulls is 0 (Ashutosh Chauhan via Jesus Camacho Rodriguez)
Signed-off-by: Ashutosh Chauhan <ha...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/47dbc005
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/47dbc005
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/47dbc005
Branch: refs/heads/branch-2.2
Commit: 47dbc005cdf9761852ef717fea8d89ca54459a43
Parents: da395b5
Author: Ashutosh Chauhan <ha...@apache.org>
Authored: Wed Jul 27 19:21:11 2016 -0700
Committer: Owen O'Malley <om...@apache.org>
Committed: Tue Mar 28 15:27:50 2017 -0700
----------------------------------------------------------------------
.../clientpositive/udaf_example_avg.q.out | 8 +-
.../clientpositive/udaf_example_max.q.out | 8 +-
.../clientpositive/udaf_example_max_n.q.out | 8 +-
.../clientpositive/udaf_example_min.q.out | 8 +-
.../clientpositive/udaf_example_min_n.q.out | 8 +-
.../stats/annotation/StatsRulesProcFactory.java | 2 +-
.../apache/hadoop/hive/ql/stats/StatsUtils.java | 49 +--
.../hive/ql/udf/generic/GenericUDAFAverage.java | 15 +-
.../hive/ql/udf/generic/GenericUDAFMax.java | 11 +-
.../hive/ql/udf/generic/GenericUDAFMin.java | 10 +
.../queries/clientpositive/vector_coalesce.q | 1 +
.../clientnegative/udf_assert_true.q.out | 24 +-
.../clientpositive/acid_table_stats.q.out | 8 +-
.../annotate_stats_deep_filters.q.out | 6 +-
.../clientpositive/annotate_stats_filter.q.out | 112 +++---
.../clientpositive/annotate_stats_groupby.q.out | 12 +-
.../clientpositive/annotate_stats_join.q.out | 16 +-
.../annotate_stats_join_pkfk.q.out | 74 ++--
.../clientpositive/annotate_stats_limit.q.out | 20 +-
.../clientpositive/annotate_stats_select.q.out | 4 +-
.../clientpositive/annotate_stats_union.q.out | 20 +-
.../clientpositive/autoColumnStats_4.q.out | 8 +-
.../clientpositive/autoColumnStats_7.q.out | 8 +-
.../clientpositive/autoColumnStats_9.q.out | 10 +-
.../cbo_rp_annotate_stats_groupby.q.out | 12 +-
.../clientpositive/cbo_rp_auto_join0.q.out | 24 +-
.../cbo_rp_groupby3_noskew_multi_distinct.q.out | 6 +-
.../results/clientpositive/cbo_rp_join0.q.out | 50 +--
.../cbo_rp_udaf_percentile_approx_23.q.out | 16 +-
.../clientpositive/constantfolding.q.out | 4 +-
.../clientpositive/create_genericudaf.q.out | 8 +-
.../clientpositive/decimal_precision.q.out | 8 +-
.../results/clientpositive/decimal_stats.q.out | 12 +-
.../results/clientpositive/decimal_udf.q.out | 8 +-
.../clientpositive/fetch_aggregation.q.out | 4 +-
.../test/results/clientpositive/fold_case.q.out | 6 +-
.../test/results/clientpositive/groupby3.q.out | 10 +-
.../results/clientpositive/groupby3_map.q.out | 6 +-
.../groupby3_map_multi_distinct.q.out | 6 +-
.../clientpositive/groupby3_map_skew.q.out | 10 +-
.../clientpositive/groupby3_noskew.q.out | 6 +-
.../groupby3_noskew_multi_distinct.q.out | 6 +-
.../clientpositive/interval_arithmetic.q.out | 8 +-
.../clientpositive/literal_decimal.q.out | 4 +-
.../llap/dynamic_partition_pruning.q.out | 215 +++++++-----
.../vectorized_dynamic_partition_pruning.q.out | 150 ++++----
.../results/clientpositive/metadataonly1.q.out | 34 +-
.../clientpositive/num_op_type_conv.q.out | 4 +-
.../results/clientpositive/perf/query13.q.out | 4 +-
.../results/clientpositive/perf/query28.q.out | 18 +-
.../reduceSinkDeDuplication_pRS_key_empty.q.out | 14 +-
.../clientpositive/remove_exprs_stats.q.out | 46 +--
.../spark/annotate_stats_join.q.out | 16 +-
.../results/clientpositive/spark/groupby3.q.out | 10 +-
.../clientpositive/spark/groupby3_map.q.out | 6 +-
.../spark/groupby3_map_multi_distinct.q.out | 6 +-
.../spark/groupby3_map_skew.q.out | 10 +-
.../clientpositive/spark/groupby3_noskew.q.out | 6 +-
.../spark/groupby3_noskew_multi_distinct.q.out | 6 +-
.../clientpositive/spark/subquery_in.q.out | 16 +-
.../spark/union_remove_6_subq.q.out | 8 +-
.../clientpositive/spark/vectorization_0.q.out | 46 +--
.../spark/vectorization_pushdown.q.out | 8 +-
.../spark/vectorization_short_regress.q.out | 40 +--
.../spark/vectorized_mapjoin.q.out | 8 +-
.../spark/vectorized_shufflejoin.q.out | 12 +-
.../spark/vectorized_timestamp_funcs.q.out | 10 +-
.../results/clientpositive/subquery_in.q.out | 12 +-
.../results/clientpositive/subquery_notin.q.out | 38 +-
.../tez/dynamic_partition_pruning.q.out | 150 ++++----
.../clientpositive/tez/explainuser_1.q.out | 346 +++++++++----------
.../clientpositive/tez/explainuser_3.q.out | 4 +-
.../results/clientpositive/tez/groupby3.q.out | 10 +-
.../clientpositive/tez/metadataonly1.q.out | 40 +--
.../clientpositive/tez/subquery_in.q.out | 12 +-
.../clientpositive/tez/vector_aggregate_9.q.out | 8 +-
.../tez/vector_aggregate_without_gby.q.out | 4 +-
.../clientpositive/tez/vector_coalesce.q.out | 70 ++--
.../tez/vector_decimal_precision.q.out | 8 +-
.../clientpositive/tez/vector_decimal_udf.q.out | 8 +-
.../tez/vector_interval_arithmetic.q.out | 16 +-
.../tez/vector_null_projection.q.out | 26 +-
.../clientpositive/tez/vectorization_0.q.out | 46 +--
.../tez/vectorization_pushdown.q.out | 8 +-
.../tez/vectorization_short_regress.q.out | 40 +--
.../tez/vectorized_distinct_gby.q.out | 8 +-
.../vectorized_dynamic_partition_pruning.q.out | 150 ++++----
.../clientpositive/tez/vectorized_mapjoin.q.out | 8 +-
.../tez/vectorized_shufflejoin.q.out | 12 +-
.../tez/vectorized_timestamp_funcs.q.out | 10 +-
.../clientpositive/udaf_number_format.q.out | 4 +-
.../udaf_percentile_approx_23.q.out | 16 +-
ql/src/test/results/clientpositive/udf3.q.out | 4 +-
ql/src/test/results/clientpositive/udf4.q.out | 4 +-
ql/src/test/results/clientpositive/udf7.q.out | 2 +-
ql/src/test/results/clientpositive/udf8.q.out | 8 +-
.../test/results/clientpositive/udf_case.q.out | 2 +-
.../results/clientpositive/udf_coalesce.q.out | 2 +-
.../test/results/clientpositive/udf_elt.q.out | 2 +-
.../results/clientpositive/udf_greatest.q.out | 2 +-
ql/src/test/results/clientpositive/udf_if.q.out | 2 +-
.../test/results/clientpositive/udf_instr.q.out | 2 +-
.../test/results/clientpositive/udf_least.q.out | 2 +-
.../results/clientpositive/udf_locate.q.out | 2 +-
.../test/results/clientpositive/udf_trunc.q.out | 4 +-
.../test/results/clientpositive/udf_when.q.out | 2 +-
.../results/clientpositive/udtf_stack.q.out | 6 +-
.../clientpositive/union_remove_6_subq.q.out | 8 +-
.../clientpositive/vector_aggregate_9.q.out | 8 +-
.../vector_aggregate_without_gby.q.out | 8 +-
.../clientpositive/vector_coalesce.q.out | 80 ++---
.../vector_decimal_precision.q.out | 8 +-
.../clientpositive/vector_decimal_udf.q.out | 8 +-
.../results/clientpositive/vector_elt.q.out | 4 +-
.../vector_interval_arithmetic.q.out | 16 +-
.../clientpositive/vector_null_projection.q.out | 30 +-
.../results/clientpositive/vector_nvl.q.out | 4 +-
.../results/clientpositive/vector_udf1.q.out | 24 +-
.../clientpositive/vectorization_0.q.out | 46 +--
.../clientpositive/vectorization_pushdown.q.out | 8 +-
.../vectorization_short_regress.q.out | 40 +--
.../vectorized_distinct_gby.q.out | 4 +-
.../clientpositive/vectorized_mapjoin.q.out | 8 +-
.../clientpositive/vectorized_shufflejoin.q.out | 12 +-
.../vectorized_timestamp_funcs.q.out | 10 +-
125 files changed, 1430 insertions(+), 1353 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/contrib/src/test/results/clientpositive/udaf_example_avg.q.out
----------------------------------------------------------------------
diff --git a/contrib/src/test/results/clientpositive/udaf_example_avg.q.out b/contrib/src/test/results/clientpositive/udaf_example_avg.q.out
index 61926d4..4e6cb99 100644
--- a/contrib/src/test/results/clientpositive/udaf_example_avg.q.out
+++ b/contrib/src/test/results/clientpositive/udaf_example_avg.q.out
@@ -33,20 +33,20 @@ STAGE PLANS:
aggregations: example_avg(_col0), example_avg(_col1)
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 128 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 128 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: struct<mcount:bigint,msum:double>), _col1 (type: struct<mcount:bigint,msum:double>)
Reduce Operator Tree:
Group By Operator
aggregations: example_avg(VALUE._col0), example_avg(VALUE._col1)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 128 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 128 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/contrib/src/test/results/clientpositive/udaf_example_max.q.out
----------------------------------------------------------------------
diff --git a/contrib/src/test/results/clientpositive/udaf_example_max.q.out b/contrib/src/test/results/clientpositive/udaf_example_max.q.out
index 932d8df..fc5a896 100644
--- a/contrib/src/test/results/clientpositive/udaf_example_max.q.out
+++ b/contrib/src/test/results/clientpositive/udaf_example_max.q.out
@@ -38,20 +38,20 @@ STAGE PLANS:
aggregations: example_max(_col0), example_max(_col1)
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string), _col1 (type: string)
Reduce Operator Tree:
Group By Operator
aggregations: example_max(VALUE._col0), example_max(VALUE._col1)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/contrib/src/test/results/clientpositive/udaf_example_max_n.q.out
----------------------------------------------------------------------
diff --git a/contrib/src/test/results/clientpositive/udaf_example_max_n.q.out b/contrib/src/test/results/clientpositive/udaf_example_max_n.q.out
index 16ae212..47d8f52 100644
--- a/contrib/src/test/results/clientpositive/udaf_example_max_n.q.out
+++ b/contrib/src/test/results/clientpositive/udaf_example_max_n.q.out
@@ -33,20 +33,20 @@ STAGE PLANS:
aggregations: example_max_n(_col0, 10), example_max_n(_col2, 10)
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 424 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 424 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: struct<a:array<double>,n:int>), _col1 (type: struct<a:array<double>,n:int>)
Reduce Operator Tree:
Group By Operator
aggregations: example_max_n(VALUE._col0), example_max_n(VALUE._col1)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 424 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 424 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/contrib/src/test/results/clientpositive/udaf_example_min.q.out
----------------------------------------------------------------------
diff --git a/contrib/src/test/results/clientpositive/udaf_example_min.q.out b/contrib/src/test/results/clientpositive/udaf_example_min.q.out
index b0ffe53..feb2add 100644
--- a/contrib/src/test/results/clientpositive/udaf_example_min.q.out
+++ b/contrib/src/test/results/clientpositive/udaf_example_min.q.out
@@ -38,20 +38,20 @@ STAGE PLANS:
aggregations: example_min(_col0), example_min(_col1)
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string), _col1 (type: string)
Reduce Operator Tree:
Group By Operator
aggregations: example_min(VALUE._col0), example_min(VALUE._col1)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/contrib/src/test/results/clientpositive/udaf_example_min_n.q.out
----------------------------------------------------------------------
diff --git a/contrib/src/test/results/clientpositive/udaf_example_min_n.q.out b/contrib/src/test/results/clientpositive/udaf_example_min_n.q.out
index 7e7dd84..16c4684 100644
--- a/contrib/src/test/results/clientpositive/udaf_example_min_n.q.out
+++ b/contrib/src/test/results/clientpositive/udaf_example_min_n.q.out
@@ -33,20 +33,20 @@ STAGE PLANS:
aggregations: example_min_n(_col0, 10), example_min_n(_col2, 10)
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 424 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 424 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: struct<a:array<double>,n:int>), _col1 (type: struct<a:array<double>,n:int>)
Reduce Operator Tree:
Group By Operator
aggregations: example_min_n(VALUE._col0), example_min_n(VALUE._col1)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 424 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 424 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
index 33778cd..d9f70a7 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
@@ -1236,7 +1236,7 @@ public class StatsRulesProcFactory {
ColStatistics cs = new ColStatistics(colName, colType);
cs.setCountDistint(stats.getNumRows());
cs.setNumNulls(0);
- cs.setAvgColLen(StatsUtils.getAvgColLenOfFixedLengthTypes(colType));
+ cs.setAvgColLen(StatsUtils.getAvgColLenOf(conf, ci.getObjectInspector(), colType));
aggColStats.add(cs);
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
index 0a2e4a5..0f1e3a3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
@@ -432,7 +432,7 @@ public class StatsUtils {
long numPartitions = getNDVPartitionColumn(partList.getPartitions(),
ci.getInternalName());
partCS.setCountDistint(numPartitions);
- partCS.setAvgColLen(StatsUtils.getAvgColLenOfVariableLengthTypes(conf,
+ partCS.setAvgColLen(StatsUtils.getAvgColLenOf(conf,
ci.getObjectInspector(), partCS.getColumnType()));
partCS.setRange(getRangePartitionColumn(partList.getPartitions(), ci.getInternalName(),
ci.getType().getTypeName(), conf.getVar(ConfVars.DEFAULTPARTITIONNAME)));
@@ -551,7 +551,7 @@ public class StatsUtils {
|| colTypeLowerCase.startsWith(serdeConstants.MAP_TYPE_NAME)
|| colTypeLowerCase.startsWith(serdeConstants.STRUCT_TYPE_NAME)
|| colTypeLowerCase.startsWith(serdeConstants.UNION_TYPE_NAME)) {
- avgRowSize += getAvgColLenOfVariableLengthTypes(conf, oi, colTypeLowerCase);
+ avgRowSize += getAvgColLenOf(conf, oi, colTypeLowerCase);
} else {
avgRowSize += getAvgColLenOfFixedLengthTypes(colTypeLowerCase);
}
@@ -841,7 +841,7 @@ public class StatsUtils {
* - column type
* @return raw data size
*/
- public static long getAvgColLenOfVariableLengthTypes(HiveConf conf, ObjectInspector oi,
+ public static long getAvgColLenOf(HiveConf conf, ObjectInspector oi,
String colType) {
long configVarLen = HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVE_STATS_MAX_VARIABLE_LENGTH);
@@ -908,7 +908,7 @@ public class StatsUtils {
return getSizeOfComplexTypes(conf, oi);
}
- return 0;
+ throw new IllegalArgumentException("Size requested for unknown type: " + colType + " OI: " + oi.getTypeName());
}
/**
@@ -931,10 +931,10 @@ public class StatsUtils {
if (colTypeLowerCase.equals(serdeConstants.STRING_TYPE_NAME)
|| colTypeLowerCase.startsWith(serdeConstants.VARCHAR_TYPE_NAME)
|| colTypeLowerCase.startsWith(serdeConstants.CHAR_TYPE_NAME)) {
- int avgColLen = (int) getAvgColLenOfVariableLengthTypes(conf, oi, colTypeLowerCase);
+ int avgColLen = (int) getAvgColLenOf(conf, oi, colTypeLowerCase);
result += JavaDataModel.get().lengthForStringOfLength(avgColLen);
} else if (colTypeLowerCase.equals(serdeConstants.BINARY_TYPE_NAME)) {
- int avgColLen = (int) getAvgColLenOfVariableLengthTypes(conf, oi, colTypeLowerCase);
+ int avgColLen = (int) getAvgColLenOf(conf, oi, colTypeLowerCase);
result += JavaDataModel.get().lengthForByteArrayOfSize(avgColLen);
} else {
result += getAvgColLenOfFixedLengthTypes(colTypeLowerCase);
@@ -1025,11 +1025,13 @@ public class StatsUtils {
if (colTypeLowerCase.equals(serdeConstants.TINYINT_TYPE_NAME)
|| colTypeLowerCase.equals(serdeConstants.SMALLINT_TYPE_NAME)
|| colTypeLowerCase.equals(serdeConstants.INT_TYPE_NAME)
+ || colTypeLowerCase.equals(serdeConstants.VOID_TYPE_NAME)
|| colTypeLowerCase.equals(serdeConstants.BOOLEAN_TYPE_NAME)
|| colTypeLowerCase.equals(serdeConstants.FLOAT_TYPE_NAME)) {
return JavaDataModel.get().primitive1();
} else if (colTypeLowerCase.equals(serdeConstants.DOUBLE_TYPE_NAME)
|| colTypeLowerCase.equals(serdeConstants.BIGINT_TYPE_NAME)
+ || colTypeLowerCase.equals(serdeConstants.INTERVAL_YEAR_MONTH_TYPE_NAME)
|| colTypeLowerCase.equals("long")) {
return JavaDataModel.get().primitive2();
} else if (colTypeLowerCase.equals(serdeConstants.TIMESTAMP_TYPE_NAME)) {
@@ -1038,8 +1040,10 @@ public class StatsUtils {
return JavaDataModel.get().lengthOfDate();
} else if (colTypeLowerCase.startsWith(serdeConstants.DECIMAL_TYPE_NAME)) {
return JavaDataModel.get().lengthOfDecimal();
+ } else if (colTypeLowerCase.equals(serdeConstants.INTERVAL_DAY_TIME_TYPE_NAME)) {
+ return JavaDataModel.JAVA32_META;
} else {
- return 0;
+ throw new IllegalArgumentException("Size requested for unknown type: " + colType);
}
}
@@ -1261,7 +1265,7 @@ public class StatsUtils {
double avgColSize = 0;
long countDistincts = 0;
long numNulls = 0;
- ObjectInspector oi = null;
+ ObjectInspector oi = end.getWritableObjectInspector();
long numRows = parentStats.getNumRows();
if (end instanceof ExprNodeColumnDesc) {
@@ -1284,7 +1288,6 @@ public class StatsUtils {
// virtual columns
colType = encd.getTypeInfo().getTypeName();
countDistincts = numRows;
- oi = encd.getWritableObjectInspector();
} else {
// clone the column stats and return
@@ -1303,16 +1306,13 @@ public class StatsUtils {
// constant projection
ExprNodeConstantDesc encd = (ExprNodeConstantDesc) end;
- // null projection
+ colName = encd.getName();
+ colType = encd.getTypeString();
if (encd.getValue() == null) {
- colName = encd.getName();
- colType = serdeConstants.VOID_TYPE_NAME;
+ // null projection
numNulls = numRows;
} else {
- colName = encd.getName();
- colType = encd.getTypeString();
countDistincts = 1;
- oi = encd.getWritableObjectInspector();
}
} else if (end instanceof ExprNodeGenericFuncDesc) {
ExprNodeGenericFuncDesc engfd = (ExprNodeGenericFuncDesc) end;
@@ -1353,7 +1353,6 @@ public class StatsUtils {
// fallback to default
countDistincts = getNDVFor(engfd, numRows, parentStats);
- oi = engfd.getWritableObjectInspector();
} else if (end instanceof ExprNodeColumnListDesc) {
// column list
@@ -1361,7 +1360,6 @@ public class StatsUtils {
colName = Joiner.on(",").join(encd.getCols());
colType = serdeConstants.LIST_TYPE_NAME;
countDistincts = numRows;
- oi = encd.getWritableObjectInspector();
} else if (end instanceof ExprNodeFieldDesc) {
// field within complex type
@@ -1369,25 +1367,12 @@ public class StatsUtils {
colName = enfd.getFieldName();
colType = enfd.getTypeString();
countDistincts = numRows;
- oi = enfd.getWritableObjectInspector();
} else {
throw new IllegalArgumentException("not supported expr type " + end.getClass());
}
colType = colType.toLowerCase();
- if (colType.equals(serdeConstants.STRING_TYPE_NAME)
- || colType.equals(serdeConstants.BINARY_TYPE_NAME)
- || colType.startsWith(serdeConstants.VARCHAR_TYPE_NAME)
- || colType.startsWith(serdeConstants.CHAR_TYPE_NAME)
- || colType.startsWith(serdeConstants.LIST_TYPE_NAME)
- || colType.startsWith(serdeConstants.MAP_TYPE_NAME)
- || colType.startsWith(serdeConstants.STRUCT_TYPE_NAME)
- || colType.startsWith(serdeConstants.UNION_TYPE_NAME)) {
- avgColSize = getAvgColLenOfVariableLengthTypes(conf, oi, colType);
- } else {
- avgColSize = getAvgColLenOfFixedLengthTypes(colType);
- }
-
+ avgColSize = getAvgColLenOf(conf, oi, colType);
ColStatistics colStats = new ColStatistics(colName, colType);
colStats.setAvgColLen(avgColSize);
colStats.setCountDistint(countDistincts);
@@ -1544,7 +1529,7 @@ public class StatsUtils {
for (ColStatistics cs : colStats) {
if (cs != null) {
String colTypeLowerCase = cs.getColumnType().toLowerCase();
- long nonNullCount = numRows - cs.getNumNulls();
+ long nonNullCount = cs.getNumNulls() > 0 ? numRows - cs.getNumNulls() + 1 : numRows;
double sizeOf = 0;
if (colTypeLowerCase.equals(serdeConstants.TINYINT_TYPE_NAME)
|| colTypeLowerCase.equals(serdeConstants.SMALLINT_TYPE_NAME)
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFAverage.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFAverage.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFAverage.java
index 3c1ce26..1b65df3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFAverage.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFAverage.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hive.ql.udf.generic;
import java.util.ArrayList;
+import java.util.HashSet;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -27,7 +28,9 @@ import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.plan.ptf.WindowFrameDef;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.AbstractAggregationBuffer;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.AggregationBuffer;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.AggregationType;
import org.apache.hadoop.hive.ql.util.JavaDataModel;
import org.apache.hadoop.hive.serde2.io.DoubleWritable;
import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
@@ -332,10 +335,16 @@ public class GenericUDAFAverage extends AbstractGenericUDAFResolver {
}
}
- private static class AverageAggregationBuffer<TYPE> implements AggregationBuffer {
- private Object previousValue;
- private long count;
+ @AggregationType(estimable = true)
+ private static class AverageAggregationBuffer<TYPE> extends AbstractAggregationBuffer {
+ private Object previousValue;
+ private long count;
private TYPE sum;
+
+ @Override
+ public int estimate() {
+ return 2*JavaDataModel.PRIMITIVES2;
+ }
};
@SuppressWarnings("unchecked")
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFMax.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFMax.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFMax.java
index 43b23fa..763bfd5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFMax.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFMax.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.hive.ql.plan.ptf.BoundaryDef;
import org.apache.hadoop.hive.ql.plan.ptf.WindowFrameDef;
import org.apache.hadoop.hive.ql.udf.UDFType;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.AggregationBuffer;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.AggregationType;
import org.apache.hadoop.hive.ql.util.JavaDataModel;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
@@ -79,8 +80,13 @@ public class GenericUDAFMax extends AbstractGenericUDAFResolver {
}
/** class for storing the current max value */
+ @AggregationType(estimable = true)
static class MaxAgg extends AbstractAggregationBuffer {
Object o;
+ @Override
+ public int estimate() {
+ return JavaDataModel.PRIMITIVES2;
+ }
}
@Override
@@ -138,7 +144,7 @@ public class GenericUDAFMax extends AbstractGenericUDAFResolver {
/*
* Based on the Paper by Daniel Lemire: Streaming Max-Min filter using no more
* than 3 comparisons per elem.
- *
+ *
* 1. His algorithm works on fixed size windows up to the current row. For row
* 'i' and window 'w' it computes the min/max for window (i-w, i). 2. The core
* idea is to keep a queue of (max, idx) tuples. A tuple in the queue
@@ -150,7 +156,7 @@ public class GenericUDAFMax extends AbstractGenericUDAFResolver {
* element at the front of the queue has reached its max range of influence;
* i.e. frontTuple.idx + w > i. If yes we can remove it from the queue. - on
* the ith step o/p the front of the queue as the max for the ith entry.
- *
+ *
* Here we modify the algorithm: 1. to handle window's that are of the form
* (i-p, i+f), where p is numPreceding,f = numFollowing - we start outputing
* rows only after receiving f rows. - the formula for 'influence range' of an
@@ -192,6 +198,7 @@ public class GenericUDAFMax extends AbstractGenericUDAFResolver {
+ (3 * JavaDataModel.PRIMITIVES1);
}
+ @Override
protected void reset() {
maxChain.clear();
super.reset();
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFMin.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFMin.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFMin.java
index 70e0db1..132bad6 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFMin.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFMin.java
@@ -26,7 +26,9 @@ import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.plan.ptf.BoundaryDef;
import org.apache.hadoop.hive.ql.plan.ptf.WindowFrameDef;
import org.apache.hadoop.hive.ql.udf.UDFType;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.AggregationType;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFMax.MaxStreamingFixedWindow;
+import org.apache.hadoop.hive.ql.util.JavaDataModel;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.FullMapEqualComparer;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
@@ -76,8 +78,13 @@ public class GenericUDAFMin extends AbstractGenericUDAFResolver {
}
/** class for storing the current max value */
+ @AggregationType(estimable = true)
static class MinAgg extends AbstractAggregationBuffer {
Object o;
+ @Override
+ public int estimate() {
+ return JavaDataModel.PRIMITIVES2;
+ }
}
@Override
@@ -139,14 +146,17 @@ public class GenericUDAFMin extends AbstractGenericUDAFResolver {
super(wrappedEval, wFrmDef);
}
+ @Override
protected ObjectInspector inputOI() {
return ((GenericUDAFMinEvaluator) wrappedEval).inputOI;
}
+ @Override
protected ObjectInspector outputOI() {
return ((GenericUDAFMinEvaluator) wrappedEval).outputOI;
}
+ @Override
protected boolean removeLast(Object in, Object last) {
return isLess(in, last);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/queries/clientpositive/vector_coalesce.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/vector_coalesce.q b/ql/src/test/queries/clientpositive/vector_coalesce.q
index b1a7766..cfba7be 100644
--- a/ql/src/test/queries/clientpositive/vector_coalesce.q
+++ b/ql/src/test/queries/clientpositive/vector_coalesce.q
@@ -1,3 +1,4 @@
+set hive.stats.fetch.column.stats=true;
set hive.explain.user=false;
SET hive.vectorized.execution.enabled=true;
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientnegative/udf_assert_true.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/udf_assert_true.q.out b/ql/src/test/results/clientnegative/udf_assert_true.q.out
index baa9074..7fc50d6 100644
--- a/ql/src/test/results/clientnegative/udf_assert_true.q.out
+++ b/ql/src/test/results/clientnegative/udf_assert_true.q.out
@@ -28,13 +28,13 @@ STAGE PLANS:
Select Operator
expressions: assert_true((_col5 > 0)) (type: void)
outputColumnNames: _col0
- Statistics: Num rows: 1000 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 1000 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE
Limit
Number of rows: 2
- Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -52,13 +52,13 @@ STAGE PLANS:
Select Operator
expressions: assert_true((_col5 > 0)) (type: void)
outputColumnNames: _col0
- Statistics: Num rows: 1000 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 1000 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE
Limit
Number of rows: 2
- Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -105,13 +105,13 @@ STAGE PLANS:
Select Operator
expressions: assert_true((_col5 < 2)) (type: void)
outputColumnNames: _col0
- Statistics: Num rows: 1000 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 1000 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE
Limit
Number of rows: 2
- Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -129,13 +129,13 @@ STAGE PLANS:
Select Operator
expressions: assert_true((_col5 < 2)) (type: void)
outputColumnNames: _col0
- Statistics: Num rows: 1000 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 1000 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE
Limit
Number of rows: 2
- Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/acid_table_stats.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/acid_table_stats.q.out b/ql/src/test/results/clientpositive/acid_table_stats.q.out
index 9a3a255..0177ad1 100644
--- a/ql/src/test/results/clientpositive/acid_table_stats.q.out
+++ b/ql/src/test/results/clientpositive/acid_table_stats.q.out
@@ -537,20 +537,20 @@ STAGE PLANS:
aggregations: max(key)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string)
Reduce Operator Tree:
Group By Operator
aggregations: max(VALUE._col0)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/annotate_stats_deep_filters.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/annotate_stats_deep_filters.q.out b/ql/src/test/results/clientpositive/annotate_stats_deep_filters.q.out
index b7a87fd..32644dc 100644
--- a/ql/src/test/results/clientpositive/annotate_stats_deep_filters.q.out
+++ b/ql/src/test/results/clientpositive/annotate_stats_deep_filters.q.out
@@ -118,12 +118,12 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: over1k
- Statistics: Num rows: 2098 Data size: 16736 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 2098 Data size: 16744 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: (((t = 1) and (si = 2)) or ((t = 2) and (si = 3)) or ((t = 3) and (si = 4)) or ((t = 4) and (si = 5)) or ((t = 5) and (si = 6)) or ((t = 6) and (si = 7)) or ((t = 7) and (si = 8)) or ((t = 9) and (si = 10)) or ((t = 10) and (si = 11)) or ((t = 11) and (si = 12)) or ((t = 12) and (si = 13)) or ((t = 13) and (si = 14)) or ((t = 14) and (si = 15)) or ((t = 15) and (si = 16)) or ((t = 16) and (si = 17)) or ((t = 17) and (si = 18)) or ((t = 27) and (si = 28)) or ((t = 37) and (si = 38)) or ((t = 47) and (si = 48)) or ((t = 52) and (si = 53))) (type: boolean)
- Statistics: Num rows: 300 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 300 Data size: 2400 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
- Statistics: Num rows: 300 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 300 Data size: 2400 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: count()
mode: hash
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/annotate_stats_filter.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/annotate_stats_filter.q.out b/ql/src/test/results/clientpositive/annotate_stats_filter.q.out
index 8e8dcc1..66dd92b 100644
--- a/ql/src/test/results/clientpositive/annotate_stats_filter.q.out
+++ b/ql/src/test/results/clientpositive/annotate_stats_filter.q.out
@@ -131,7 +131,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: loc_orc
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: (state = 'OH') (type: boolean)
Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE
@@ -167,17 +167,17 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: loc_orc
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: (state <> 'OH') (type: boolean)
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: int)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -203,17 +203,17 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: loc_orc
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: (state <> 'OH') (type: boolean)
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: int)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -239,17 +239,17 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: loc_orc
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: zip is null (type: boolean)
Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: state (type: string), locid (type: int), null (type: bigint), year (type: int)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -275,17 +275,17 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: loc_orc
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: zip is null (type: boolean)
Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: state (type: string), locid (type: int), null (type: bigint), year (type: int)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -311,17 +311,17 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: loc_orc
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: zip is not null (type: boolean)
- Statistics: Num rows: 7 Data size: 702 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 7 Data size: 714 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: int)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 7 Data size: 702 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 7 Data size: 714 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 7 Data size: 702 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 7 Data size: 714 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -347,17 +347,17 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: loc_orc
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: zip is not null (type: boolean)
- Statistics: Num rows: 7 Data size: 702 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 7 Data size: 714 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: int)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 7 Data size: 702 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 7 Data size: 714 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 7 Data size: 702 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 7 Data size: 714 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -383,11 +383,11 @@ STAGE PLANS:
Processor Tree:
TableScan
alias: loc_orc
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: int)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
ListSink
PREHOOK: query: explain select * from loc_orc where !true
@@ -404,7 +404,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: loc_orc
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: false (type: boolean)
Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE
@@ -440,11 +440,11 @@ STAGE PLANS:
Processor Tree:
TableScan
alias: loc_orc
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: int)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
ListSink
PREHOOK: query: explain select * from loc_orc where 'foo'
@@ -461,17 +461,17 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: loc_orc
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: 'foo' (type: string)
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: int)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -497,11 +497,11 @@ STAGE PLANS:
Processor Tree:
TableScan
alias: loc_orc
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: int)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
ListSink
PREHOOK: query: explain select * from loc_orc where false = true
@@ -518,7 +518,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: loc_orc
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: false (type: boolean)
Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE
@@ -554,7 +554,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: loc_orc
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: false (type: boolean)
Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE
@@ -590,7 +590,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: loc_orc
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: false (type: boolean)
Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE
@@ -626,7 +626,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: loc_orc
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: ((state = 'OH') or (state = 'CA')) (type: boolean)
Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE
@@ -662,7 +662,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: loc_orc
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: false (type: boolean)
Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE
@@ -698,7 +698,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: loc_orc
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: ((year = 2001) and (state = 'OH') and (state = 'FL')) (type: boolean)
Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE
@@ -734,7 +734,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: loc_orc
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: (((year = 2001) and year is null) or (state = 'CA')) (type: boolean)
Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE
@@ -770,7 +770,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: loc_orc
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: (((year = 2001) or year is null) and (state = 'CA')) (type: boolean)
Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE
@@ -806,17 +806,17 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: loc_orc
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: (locid < 30) (type: boolean)
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: int)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -842,7 +842,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: loc_orc
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: (locid > 30) (type: boolean)
Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE
@@ -878,17 +878,17 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: loc_orc
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: (locid <= 30) (type: boolean)
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: int)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -914,7 +914,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: loc_orc
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: (locid >= 30) (type: boolean)
Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE
@@ -950,7 +950,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: loc_orc
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: (locid < 3) (type: boolean)
Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE
@@ -986,7 +986,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: loc_orc
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: (locid > 3) (type: boolean)
Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE
@@ -1022,7 +1022,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: loc_orc
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: (locid <= 3) (type: boolean)
Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE
@@ -1058,7 +1058,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: loc_orc
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: (locid >= 3) (type: boolean)
Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/annotate_stats_groupby.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/annotate_stats_groupby.q.out b/ql/src/test/results/clientpositive/annotate_stats_groupby.q.out
index 24658f1..a8e4854 100644
--- a/ql/src/test/results/clientpositive/annotate_stats_groupby.q.out
+++ b/ql/src/test/results/clientpositive/annotate_stats_groupby.q.out
@@ -196,21 +196,21 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: loc_orc
- Statistics: Num rows: 8 Data size: 28 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: year (type: int)
outputColumnNames: year
- Statistics: Num rows: 8 Data size: 28 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
keys: year (type: int)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 8 Data size: 28 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 8 Data size: 28 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Operator Tree:
Group By Operator
keys: KEY._col0 (type: int)
@@ -644,11 +644,11 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: loc_orc
- Statistics: Num rows: 8 Data size: 28 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: year (type: int)
outputColumnNames: year
- Statistics: Num rows: 8 Data size: 28 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
keys: year (type: int)
mode: hash
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/annotate_stats_join.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/annotate_stats_join.q.out b/ql/src/test/results/clientpositive/annotate_stats_join.q.out
index eb8c6c5..5d4fe6c 100644
--- a/ql/src/test/results/clientpositive/annotate_stats_join.q.out
+++ b/ql/src/test/results/clientpositive/annotate_stats_join.q.out
@@ -514,19 +514,19 @@ STAGE PLANS:
value expressions: _col1 (type: string)
TableScan
alias: l
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: locid is not null (type: boolean)
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: int)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col1 (type: int)
sort order: +
Map-reduce partition columns: _col1 (type: int)
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col0 (type: string), _col2 (type: bigint), _col3 (type: int)
Reduce Operator Tree:
Join Operator
@@ -598,19 +598,19 @@ STAGE PLANS:
Statistics: Num rows: 6 Data size: 570 Basic stats: COMPLETE Column stats: COMPLETE
TableScan
alias: l
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: (state is not null and locid is not null) (type: boolean)
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: int)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: int)
sort order: ++
Map-reduce partition columns: _col0 (type: string), _col1 (type: int)
- Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 8 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col2 (type: bigint), _col3 (type: int)
Reduce Operator Tree:
Join Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/annotate_stats_join_pkfk.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/annotate_stats_join_pkfk.q.out b/ql/src/test/results/clientpositive/annotate_stats_join_pkfk.q.out
index 5d3d809..71da136 100644
--- a/ql/src/test/results/clientpositive/annotate_stats_join_pkfk.q.out
+++ b/ql/src/test/results/clientpositive/annotate_stats_join_pkfk.q.out
@@ -379,19 +379,19 @@ STAGE PLANS:
Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
TableScan
alias: ss
- Statistics: Num rows: 1000 Data size: 3856 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1000 Data size: 3860 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: ss_store_sk is not null (type: boolean)
- Statistics: Num rows: 964 Data size: 3716 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 964 Data size: 3720 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: ss_store_sk (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 964 Data size: 3716 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 964 Data size: 3720 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 964 Data size: 3716 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 964 Data size: 3720 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Operator Tree:
Join Operator
condition map:
@@ -509,19 +509,19 @@ STAGE PLANS:
Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
TableScan
alias: ss
- Statistics: Num rows: 1000 Data size: 3856 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1000 Data size: 3860 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: (ss_store_sk > 0) (type: boolean)
- Statistics: Num rows: 1000 Data size: 3856 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1000 Data size: 3860 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: ss_store_sk (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 1000 Data size: 3856 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1000 Data size: 3860 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 1000 Data size: 3856 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1000 Data size: 3860 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Operator Tree:
Join Operator
condition map:
@@ -574,19 +574,19 @@ STAGE PLANS:
Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL
TableScan
alias: ss
- Statistics: Num rows: 1000 Data size: 7668 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1000 Data size: 7676 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: ((ss_quantity > 10) and ss_store_sk is not null) (type: boolean)
- Statistics: Num rows: 321 Data size: 2460 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 321 Data size: 2468 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: ss_store_sk (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 321 Data size: 2460 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 321 Data size: 2468 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 321 Data size: 2460 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 321 Data size: 2468 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Operator Tree:
Join Operator
condition map:
@@ -639,19 +639,19 @@ STAGE PLANS:
Statistics: Num rows: 12 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE
TableScan
alias: ss
- Statistics: Num rows: 1000 Data size: 3856 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1000 Data size: 3860 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: ss_store_sk is not null (type: boolean)
- Statistics: Num rows: 964 Data size: 3716 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 964 Data size: 3720 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: ss_store_sk (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 964 Data size: 3716 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 964 Data size: 3720 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 964 Data size: 3716 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 964 Data size: 3720 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Operator Tree:
Join Operator
condition map:
@@ -704,19 +704,19 @@ STAGE PLANS:
Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
TableScan
alias: ss
- Statistics: Num rows: 1000 Data size: 7668 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1000 Data size: 7676 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: ((ss_quantity > 10) and ss_store_sk is not null) (type: boolean)
- Statistics: Num rows: 321 Data size: 2460 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 321 Data size: 2468 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: ss_store_sk (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 321 Data size: 2460 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 321 Data size: 2468 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 321 Data size: 2460 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 321 Data size: 2468 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Operator Tree:
Join Operator
condition map:
@@ -754,19 +754,19 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: ss
- Statistics: Num rows: 1000 Data size: 3856 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1000 Data size: 3860 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: ss_store_sk is not null (type: boolean)
- Statistics: Num rows: 964 Data size: 3716 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 964 Data size: 3720 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: ss_store_sk (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 964 Data size: 3716 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 964 Data size: 3720 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 964 Data size: 3716 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 964 Data size: 3720 Basic stats: COMPLETE Column stats: COMPLETE
TableScan
alias: s
Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
@@ -840,7 +840,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: ss
- Statistics: Num rows: 1000 Data size: 3856 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1000 Data size: 3860 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: (ss_store_sk > 1000) (type: boolean)
Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
@@ -926,19 +926,19 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: ss
- Statistics: Num rows: 1000 Data size: 3856 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1000 Data size: 3860 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: ss_store_sk is not null (type: boolean)
- Statistics: Num rows: 964 Data size: 3716 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 964 Data size: 3720 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: ss_store_sk (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 964 Data size: 3716 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 964 Data size: 3720 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 964 Data size: 3716 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 964 Data size: 3720 Basic stats: COMPLETE Column stats: COMPLETE
TableScan
alias: s
Statistics: Num rows: 12 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE
@@ -1012,19 +1012,19 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: ss
- Statistics: Num rows: 1000 Data size: 7668 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1000 Data size: 7676 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: ((ss_quantity > 10) and ss_store_sk is not null) (type: boolean)
- Statistics: Num rows: 321 Data size: 2460 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 321 Data size: 2468 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: ss_store_sk (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 321 Data size: 2460 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 321 Data size: 2468 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 321 Data size: 2460 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 321 Data size: 2468 Basic stats: COMPLETE Column stats: COMPLETE
TableScan
alias: s
Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
@@ -1099,19 +1099,19 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: ss
- Statistics: Num rows: 1000 Data size: 7656 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1000 Data size: 7664 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: (ss_store_sk is not null and ss_addr_sk is not null) (type: boolean)
- Statistics: Num rows: 916 Data size: 7012 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 916 Data size: 7020 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: ss_addr_sk (type: int), ss_store_sk (type: int)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 916 Data size: 7012 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 916 Data size: 7020 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col1 (type: int)
sort order: +
Map-reduce partition columns: _col1 (type: int)
- Statistics: Num rows: 916 Data size: 7012 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 916 Data size: 7020 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col0 (type: int)
TableScan
alias: s
[38/50] [abbrv] hive git commit: HIVE-16022: BloomFilter check not
showing up in MERGE statement queries (Jason Dere reviewed by Gunt her
Hagleitner)
Posted by om...@apache.org.
HIVE-16022: BloomFilter check not showing up in MERGE statement queries (Jason Dere reviewed by Gunt
her Hagleitner)
Change-Id: Ie773cdb3ac5cac2174b06b7d64f8c72d6a17cb06
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c3d311f4
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c3d311f4
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c3d311f4
Branch: refs/heads/branch-2.2
Commit: c3d311f43eee59adaf6806cb366480e39e7cffb0
Parents: 127deb2
Author: Jason Dere <jd...@hortonworks.com>
Authored: Wed Mar 1 11:40:40 2017 -0800
Committer: Owen O'Malley <om...@apache.org>
Committed: Tue Mar 28 15:27:54 2017 -0700
----------------------------------------------------------------------
.../test/resources/testconfiguration.properties | 1 +
.../hadoop/hive/ql/exec/OperatorUtils.java | 7 +
.../DynamicPartitionPruningOptimization.java | 11 +-
.../dynamic_semijoin_reduction_3.q | 37 +
.../llap/dynamic_partition_pruning.q.out | 4 +-
.../llap/dynamic_semijoin_reduction.q.out | 28 +-
.../llap/dynamic_semijoin_reduction_2.q.out | 4 +-
.../llap/dynamic_semijoin_reduction_3.q.out | 765 +++++++++++++++++++
.../results/clientpositive/llap/orc_llap.q.out | 8 +-
.../vectorized_dynamic_partition_pruning.q.out | 4 +-
.../vectorized_dynamic_semijoin_reduction.q.out | 28 +-
...vectorized_dynamic_semijoin_reduction2.q.out | 32 +-
.../tez/dynamic_partition_pruning.q.out | 4 +-
.../clientpositive/tez/explainuser_3.q.out | 4 +-
.../results/clientpositive/tez/mergejoin.q.out | 48 +-
.../vectorized_dynamic_partition_pruning.q.out | 4 +-
16 files changed, 900 insertions(+), 89 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/c3d311f4/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index 9720f76..891bf11 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -510,6 +510,7 @@ minillap.query.files=cross_prod_1.q,\
cross_prod_3.q,\
dynamic_semijoin_reduction.q,\
dynamic_semijoin_reduction_2.q,\
+ dynamic_semijoin_reduction_3.q,\
except_distinct.q,\
intersect_all.q,\
intersect_distinct.q,\
http://git-wip-us.apache.org/repos/asf/hive/blob/c3d311f4/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorUtils.java
index d294e25..51faf1d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorUtils.java
@@ -346,4 +346,11 @@ public class OperatorUtils {
}
}
}
+
+ public static String getOpNamePretty(Operator<?> op) {
+ if (op instanceof TableScanOperator) {
+ return op.toString() + " (" + ((TableScanOperator) op).getConf().getAlias() + ")";
+ }
+ return op.toString();
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/c3d311f4/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java
index 3085f5d..e6f21e9 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java
@@ -239,7 +239,6 @@ public class DynamicPartitionPruningOptimization implements NodeProcessor {
betweenArgs.add(new ExprNodeDynamicValueDesc(new DynamicValue(keyBaseAlias + "_max", ctx.desc.getTypeInfo())));
ExprNodeDesc betweenNode = ExprNodeGenericFuncDesc.newInstance(
FunctionRegistry.getFunctionInfo("between").getGenericUDF(), betweenArgs);
- replaceExprNode(ctx, desc, betweenNode);
// add column expression for bloom filter
List<ExprNodeDesc> bloomFilterArgs = new ArrayList<ExprNodeDesc>();
bloomFilterArgs.add(ctx.parent.getChildren().get(0));
@@ -249,10 +248,12 @@ public class DynamicPartitionPruningOptimization implements NodeProcessor {
ExprNodeDesc bloomFilterNode = ExprNodeGenericFuncDesc.newInstance(
FunctionRegistry.getFunctionInfo("in_bloom_filter").
getGenericUDF(), bloomFilterArgs);
- // ctx may not have the grandparent but it is set in filterDesc by now.
- ExprNodeDesc grandParent = ctx.grandParent == null ?
- desc.getPredicate() : ctx.grandParent;
- grandParent.getChildren().add(bloomFilterNode);
+ List<ExprNodeDesc> andArgs = new ArrayList<ExprNodeDesc>();
+ andArgs.add(betweenNode);
+ andArgs.add(bloomFilterNode);
+ ExprNodeDesc andExpr = ExprNodeGenericFuncDesc.newInstance(
+ FunctionRegistry.getFunctionInfo("and").getGenericUDF(), andArgs);
+ replaceExprNode(ctx, desc, andExpr);
} else {
ExprNodeDesc replaceNode = new ExprNodeConstantDesc(ctx.parent.getTypeInfo(), true);
replaceExprNode(ctx, desc, replaceNode);
http://git-wip-us.apache.org/repos/asf/hive/blob/c3d311f4/ql/src/test/queries/clientpositive/dynamic_semijoin_reduction_3.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/dynamic_semijoin_reduction_3.q b/ql/src/test/queries/clientpositive/dynamic_semijoin_reduction_3.q
new file mode 100644
index 0000000..adfc7a6
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/dynamic_semijoin_reduction_3.q
@@ -0,0 +1,37 @@
+set hive.mapred.mode=nonstrict;
+set hive.support.concurrency=true;
+set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
+set hive.explain.user=false;
+set hive.merge.cardinality.check=true;
+
+set hive.compute.query.using.stats=false;
+set hive.mapred.mode=nonstrict;
+set hive.optimize.ppd=true;
+set hive.ppd.remove.duplicatefilters=true;
+set hive.tez.dynamic.partition.pruning=true;
+set hive.tez.dynamic.semijoin.reduction=true;
+set hive.optimize.metadataonly=false;
+set hive.optimize.index.filter=true;
+
+-- Try with merge statements
+create table acidTbl(a int, b int) clustered by (a) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true');
+create table nonAcidOrcTbl(a int, b int) clustered by (a) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='false');
+
+--expect a cardinality check because there is update and hive.merge.cardinality.check=true by default
+explain merge into acidTbl as t using nonAcidOrcTbl s ON t.a = s.a
+WHEN MATCHED AND s.a > 8 THEN DELETE
+WHEN MATCHED THEN UPDATE SET b = 7
+WHEN NOT MATCHED THEN INSERT VALUES(s.a, s.b);
+
+--now we expect no cardinality check since only have insert clause
+explain merge into acidTbl as t using nonAcidOrcTbl s ON t.a = s.a
+WHEN NOT MATCHED THEN INSERT VALUES(s.a, s.b);
+
+explain merge into acidTbl as t using (
+ select * from nonAcidOrcTbl where a > 0
+ union all
+ select * from nonAcidOrcTbl where b > 0
+) s ON t.a = s.a
+WHEN MATCHED AND s.a > 8 THEN DELETE
+WHEN MATCHED THEN UPDATE SET b = 7
+WHEN NOT MATCHED THEN INSERT VALUES(s.a, s.b);
http://git-wip-us.apache.org/repos/asf/hive/blob/c3d311f4/ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out b/ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out
index b2955ed..d28bafb 100644
--- a/ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out
+++ b/ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out
@@ -3187,10 +3187,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: srcpart_hour
- filterExpr: ((UDFToDouble(hr) = 13.0) and hr BETWEEN DynamicValue(RS_12_srcpart_hr_min) AND DynamicValue(RS_12_srcpart_hr_max) and in_bloom_filter(hr, DynamicValue(RS_12_srcpart_hr_bloom_filter))) (type: boolean)
+ filterExpr: ((UDFToDouble(hr) = 13.0) and (hr BETWEEN DynamicValue(RS_12_srcpart_hr_min) AND DynamicValue(RS_12_srcpart_hr_max) and in_bloom_filter(hr, DynamicValue(RS_12_srcpart_hr_bloom_filter)))) (type: boolean)
Statistics: Num rows: 2 Data size: 10 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((UDFToDouble(hr) = 13.0) and hr BETWEEN DynamicValue(RS_12_srcpart_hr_min) AND DynamicValue(RS_12_srcpart_hr_max) and in_bloom_filter(hr, DynamicValue(RS_12_srcpart_hr_bloom_filter))) (type: boolean)
+ predicate: ((UDFToDouble(hr) = 13.0) and (hr BETWEEN DynamicValue(RS_12_srcpart_hr_min) AND DynamicValue(RS_12_srcpart_hr_max) and in_bloom_filter(hr, DynamicValue(RS_12_srcpart_hr_bloom_filter)))) (type: boolean)
Statistics: Num rows: 1 Data size: 5 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: hr (type: string)
http://git-wip-us.apache.org/repos/asf/hive/blob/c3d311f4/ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction.q.out b/ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction.q.out
index 385b5cf..c9cf19a 100644
--- a/ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction.q.out
+++ b/ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction.q.out
@@ -242,10 +242,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: srcpart_date
- filterExpr: (key is not null and key BETWEEN DynamicValue(RS_7_srcpart_small_key_min) AND DynamicValue(RS_7_srcpart_small_key_max) and in_bloom_filter(key, DynamicValue(RS_7_srcpart_small_key_bloom_filter))) (type: boolean)
+ filterExpr: (key is not null and (key BETWEEN DynamicValue(RS_7_srcpart_small_key_min) AND DynamicValue(RS_7_srcpart_small_key_max) and in_bloom_filter(key, DynamicValue(RS_7_srcpart_small_key_bloom_filter)))) (type: boolean)
Statistics: Num rows: 2000 Data size: 352000 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (key is not null and key BETWEEN DynamicValue(RS_7_srcpart_small_key_min) AND DynamicValue(RS_7_srcpart_small_key_max) and in_bloom_filter(key, DynamicValue(RS_7_srcpart_small_key_bloom_filter))) (type: boolean)
+ predicate: (key is not null and (key BETWEEN DynamicValue(RS_7_srcpart_small_key_min) AND DynamicValue(RS_7_srcpart_small_key_max) and in_bloom_filter(key, DynamicValue(RS_7_srcpart_small_key_bloom_filter)))) (type: boolean)
Statistics: Num rows: 2000 Data size: 352000 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string)
@@ -659,10 +659,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: srcpart_date
- filterExpr: (key is not null and key BETWEEN DynamicValue(RS_10_srcpart_small_key_min) AND DynamicValue(RS_10_srcpart_small_key_max) and in_bloom_filter(key, DynamicValue(RS_10_srcpart_small_key_bloom_filter))) (type: boolean)
+ filterExpr: (key is not null and (key BETWEEN DynamicValue(RS_10_srcpart_small_key_min) AND DynamicValue(RS_10_srcpart_small_key_max) and in_bloom_filter(key, DynamicValue(RS_10_srcpart_small_key_bloom_filter)))) (type: boolean)
Statistics: Num rows: 2000 Data size: 352000 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (key is not null and key BETWEEN DynamicValue(RS_10_srcpart_small_key_min) AND DynamicValue(RS_10_srcpart_small_key_max) and in_bloom_filter(key, DynamicValue(RS_10_srcpart_small_key_bloom_filter))) (type: boolean)
+ predicate: (key is not null and (key BETWEEN DynamicValue(RS_10_srcpart_small_key_min) AND DynamicValue(RS_10_srcpart_small_key_max) and in_bloom_filter(key, DynamicValue(RS_10_srcpart_small_key_bloom_filter)))) (type: boolean)
Statistics: Num rows: 2000 Data size: 352000 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string)
@@ -738,10 +738,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: alltypesorc_int
- filterExpr: (cstring is not null and cstring BETWEEN DynamicValue(RS_9_srcpart_date_cstring_min) AND DynamicValue(RS_9_srcpart_date_cstring_max) and cstring BETWEEN DynamicValue(RS_10_srcpart_small_cstring_min) AND DynamicValue(RS_10_srcpart_small_cstring_max) and in_bloom_filter(cstring, DynamicValue(RS_9_srcpart_date_cstring_bloom_filter)) and in_bloom_filter(cstring, DynamicValue(RS_10_srcpart_small_cstring_bloom_filter))) (type: boolean)
+ filterExpr: (cstring is not null and (cstring BETWEEN DynamicValue(RS_9_srcpart_date_cstring_min) AND DynamicValue(RS_9_srcpart_date_cstring_max) and in_bloom_filter(cstring, DynamicValue(RS_9_srcpart_date_cstring_bloom_filter))) and (cstring BETWEEN DynamicValue(RS_10_srcpart_small_cstring_min) AND DynamicValue(RS_10_srcpart_small_cstring_max) and in_bloom_filter(cstring, DynamicValue(RS_10_srcpart_small_cstring_bloom_filter)))) (type: boolean)
Statistics: Num rows: 12288 Data size: 926570 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (cstring is not null and cstring BETWEEN DynamicValue(RS_9_srcpart_date_cstring_min) AND DynamicValue(RS_9_srcpart_date_cstring_max) and cstring BETWEEN DynamicValue(RS_10_srcpart_small_cstring_min) AND DynamicValue(RS_10_srcpart_small_cstring_max) and in_bloom_filter(cstring, DynamicValue(RS_9_srcpart_date_cstring_bloom_filter)) and in_bloom_filter(cstring, DynamicValue(RS_10_srcpart_small_cstring_bloom_filter))) (type: boolean)
+ predicate: (cstring is not null and (cstring BETWEEN DynamicValue(RS_9_srcpart_date_cstring_min) AND DynamicValue(RS_9_srcpart_date_cstring_max) and in_bloom_filter(cstring, DynamicValue(RS_9_srcpart_date_cstring_bloom_filter))) and (cstring BETWEEN DynamicValue(RS_10_srcpart_small_cstring_min) AND DynamicValue(RS_10_srcpart_small_cstring_max) and in_bloom_filter(cstring, DynamicValue(RS_10_srcpart_small_cstring_bloom_filter)))) (type: boolean)
Statistics: Num rows: 12288 Data size: 926570 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cstring (type: string)
@@ -1014,10 +1014,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: srcpart_date
- filterExpr: (key is not null and value is not null and key BETWEEN DynamicValue(RS_7_srcpart_small_key_min) AND DynamicValue(RS_7_srcpart_small_key_max) and value BETWEEN DynamicValue(RS_7_srcpart_small_value_min) AND DynamicValue(RS_7_srcpart_small_value_max) and in_bloom_filter(key, DynamicValue(RS_7_srcpart_small_key_bloom_filter)) and in_bloom_filter(value, DynamicValue(RS_7_srcpart_small_value_bloom_filter))) (type: boolean)
+ filterExpr: (key is not null and value is not null and (key BETWEEN DynamicValue(RS_7_srcpart_small_key_min) AND DynamicValue(RS_7_srcpart_small_key_max) and in_bloom_filter(key, DynamicValue(RS_7_srcpart_small_key_bloom_filter))) and (value BETWEEN DynamicValue(RS_7_srcpart_small_value_min) AND DynamicValue(RS_7_srcpart_small_value_max) and in_bloom_filter(value, DynamicValue(RS_7_srcpart_small_value_bloom_filter)))) (type: boolean)
Statistics: Num rows: 2000 Data size: 352000 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (key is not null and value is not null and key BETWEEN DynamicValue(RS_7_srcpart_small_key_min) AND DynamicValue(RS_7_srcpart_small_key_max) and value BETWEEN DynamicValue(RS_7_srcpart_small_value_min) AND DynamicValue(RS_7_srcpart_small_value_max) and in_bloom_filter(key, DynamicValue(RS_7_srcpart_small_key_bloom_filter)) and in_bloom_filter(value, DynamicValue(RS_7_srcpart_small_value_bloom_filter))) (type: boolean)
+ predicate: (key is not null and value is not null and (key BETWEEN DynamicValue(RS_7_srcpart_small_key_min) AND DynamicValue(RS_7_srcpart_small_key_max) and in_bloom_filter(key, DynamicValue(RS_7_srcpart_small_key_bloom_filter))) and (value BETWEEN DynamicValue(RS_7_srcpart_small_value_min) AND DynamicValue(RS_7_srcpart_small_value_max) and in_bloom_filter(value, DynamicValue(RS_7_srcpart_small_value_bloom_filter)))) (type: boolean)
Statistics: Num rows: 2000 Data size: 352000 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
@@ -1324,10 +1324,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: srcpart_date
- filterExpr: (key is not null and value is not null and key BETWEEN DynamicValue(RS_10_srcpart_small_key_min) AND DynamicValue(RS_10_srcpart_small_key_max) and in_bloom_filter(key, DynamicValue(RS_10_srcpart_small_key_bloom_filter))) (type: boolean)
+ filterExpr: (key is not null and value is not null and (key BETWEEN DynamicValue(RS_10_srcpart_small_key_min) AND DynamicValue(RS_10_srcpart_small_key_max) and in_bloom_filter(key, DynamicValue(RS_10_srcpart_small_key_bloom_filter)))) (type: boolean)
Statistics: Num rows: 2000 Data size: 352000 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (key is not null and value is not null and key BETWEEN DynamicValue(RS_10_srcpart_small_key_min) AND DynamicValue(RS_10_srcpart_small_key_max) and in_bloom_filter(key, DynamicValue(RS_10_srcpart_small_key_bloom_filter))) (type: boolean)
+ predicate: (key is not null and value is not null and (key BETWEEN DynamicValue(RS_10_srcpart_small_key_min) AND DynamicValue(RS_10_srcpart_small_key_max) and in_bloom_filter(key, DynamicValue(RS_10_srcpart_small_key_bloom_filter)))) (type: boolean)
Statistics: Num rows: 2000 Data size: 352000 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
@@ -1378,10 +1378,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: alltypesorc_int
- filterExpr: (cstring is not null and cstring BETWEEN DynamicValue(RS_12_srcpart_date_cstring_min) AND DynamicValue(RS_12_srcpart_date_cstring_max) and in_bloom_filter(cstring, DynamicValue(RS_12_srcpart_date_cstring_bloom_filter))) (type: boolean)
+ filterExpr: (cstring is not null and (cstring BETWEEN DynamicValue(RS_12_srcpart_date_cstring_min) AND DynamicValue(RS_12_srcpart_date_cstring_max) and in_bloom_filter(cstring, DynamicValue(RS_12_srcpart_date_cstring_bloom_filter)))) (type: boolean)
Statistics: Num rows: 12288 Data size: 926570 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (cstring is not null and cstring BETWEEN DynamicValue(RS_12_srcpart_date_cstring_min) AND DynamicValue(RS_12_srcpart_date_cstring_max) and in_bloom_filter(cstring, DynamicValue(RS_12_srcpart_date_cstring_bloom_filter))) (type: boolean)
+ predicate: (cstring is not null and (cstring BETWEEN DynamicValue(RS_12_srcpart_date_cstring_min) AND DynamicValue(RS_12_srcpart_date_cstring_max) and in_bloom_filter(cstring, DynamicValue(RS_12_srcpart_date_cstring_bloom_filter)))) (type: boolean)
Statistics: Num rows: 12288 Data size: 926570 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cstring (type: string)
@@ -1642,10 +1642,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: srcpart_date
- filterExpr: (key is not null and key BETWEEN DynamicValue(RS_7_srcpart_small_key_min) AND DynamicValue(RS_7_srcpart_small_key_max) and in_bloom_filter(key, DynamicValue(RS_7_srcpart_small_key_bloom_filter))) (type: boolean)
+ filterExpr: (key is not null and (key BETWEEN DynamicValue(RS_7_srcpart_small_key_min) AND DynamicValue(RS_7_srcpart_small_key_max) and in_bloom_filter(key, DynamicValue(RS_7_srcpart_small_key_bloom_filter)))) (type: boolean)
Statistics: Num rows: 2000 Data size: 352000 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (key is not null and key BETWEEN DynamicValue(RS_7_srcpart_small_key_min) AND DynamicValue(RS_7_srcpart_small_key_max) and in_bloom_filter(key, DynamicValue(RS_7_srcpart_small_key_bloom_filter))) (type: boolean)
+ predicate: (key is not null and (key BETWEEN DynamicValue(RS_7_srcpart_small_key_min) AND DynamicValue(RS_7_srcpart_small_key_max) and in_bloom_filter(key, DynamicValue(RS_7_srcpart_small_key_bloom_filter)))) (type: boolean)
Statistics: Num rows: 2000 Data size: 352000 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string)
http://git-wip-us.apache.org/repos/asf/hive/blob/c3d311f4/ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction_2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction_2.q.out b/ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction_2.q.out
index 401e558..e91795b 100644
--- a/ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction_2.q.out
+++ b/ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction_2.q.out
@@ -105,10 +105,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: tt2
- filterExpr: ((timestamp_col_18 = timestamp_col_18) and decimal1911_col_16 is not null and timestamp_col_18 BETWEEN DynamicValue(RS_23_t1_timestamp_col_18_min) AND DynamicValue(RS_23_t1_timestamp_col_18_max) and in_bloom_filter(timestamp_col_18, DynamicValue(RS_23_t1_timestamp_col_18_bloom_filter))) (type: boolean)
+ filterExpr: ((timestamp_col_18 = timestamp_col_18) and decimal1911_col_16 is not null and (timestamp_col_18 BETWEEN DynamicValue(RS_23_t1_timestamp_col_18_min) AND DynamicValue(RS_23_t1_timestamp_col_18_max) and in_bloom_filter(timestamp_col_18, DynamicValue(RS_23_t1_timestamp_col_18_bloom_filter)))) (type: boolean)
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Filter Operator
- predicate: ((timestamp_col_18 = timestamp_col_18) and decimal1911_col_16 is not null and timestamp_col_18 BETWEEN DynamicValue(RS_23_t1_timestamp_col_18_min) AND DynamicValue(RS_23_t1_timestamp_col_18_max) and in_bloom_filter(timestamp_col_18, DynamicValue(RS_23_t1_timestamp_col_18_bloom_filter))) (type: boolean)
+ predicate: ((timestamp_col_18 = timestamp_col_18) and decimal1911_col_16 is not null and (timestamp_col_18 BETWEEN DynamicValue(RS_23_t1_timestamp_col_18_min) AND DynamicValue(RS_23_t1_timestamp_col_18_max) and in_bloom_filter(timestamp_col_18, DynamicValue(RS_23_t1_timestamp_col_18_bloom_filter)))) (type: boolean)
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: decimal1911_col_16 (type: decimal(19,11)), timestamp_col_18 (type: timestamp)
http://git-wip-us.apache.org/repos/asf/hive/blob/c3d311f4/ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction_3.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction_3.q.out b/ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction_3.q.out
new file mode 100644
index 0000000..7ea25d0
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction_3.q.out
@@ -0,0 +1,765 @@
+PREHOOK: query: create table acidTbl(a int, b int) clustered by (a) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true')
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@acidTbl
+POSTHOOK: query: create table acidTbl(a int, b int) clustered by (a) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@acidTbl
+PREHOOK: query: create table nonAcidOrcTbl(a int, b int) clustered by (a) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='false')
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@nonAcidOrcTbl
+POSTHOOK: query: create table nonAcidOrcTbl(a int, b int) clustered by (a) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='false')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@nonAcidOrcTbl
+PREHOOK: query: explain merge into acidTbl as t using nonAcidOrcTbl s ON t.a = s.a
+WHEN MATCHED AND s.a > 8 THEN DELETE
+WHEN MATCHED THEN UPDATE SET b = 7
+WHEN NOT MATCHED THEN INSERT VALUES(s.a, s.b)
+PREHOOK: type: QUERY
+POSTHOOK: query: explain merge into acidTbl as t using nonAcidOrcTbl s ON t.a = s.a
+WHEN MATCHED AND s.a > 8 THEN DELETE
+WHEN MATCHED THEN UPDATE SET b = 7
+WHEN NOT MATCHED THEN INSERT VALUES(s.a, s.b)
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-4 is a root stage
+ Stage-5 depends on stages: Stage-4
+ Stage-0 depends on stages: Stage-5
+ Stage-6 depends on stages: Stage-0
+ Stage-2 depends on stages: Stage-5
+ Stage-7 depends on stages: Stage-2
+ Stage-3 depends on stages: Stage-5
+ Stage-8 depends on stages: Stage-3
+ Stage-1 depends on stages: Stage-5
+ Stage-9 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-4
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Map 1 <- Reducer 8 (BROADCAST_EDGE)
+ Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE)
+ Reducer 3 <- Reducer 2 (SIMPLE_EDGE)
+ Reducer 4 <- Reducer 2 (SIMPLE_EDGE)
+ Reducer 5 <- Reducer 2 (SIMPLE_EDGE)
+ Reducer 6 <- Reducer 2 (SIMPLE_EDGE)
+ Reducer 8 <- Map 7 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: t
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Filter Operator
+ predicate: (a BETWEEN DynamicValue(RS_3_s_a_min) AND DynamicValue(RS_3_s_a_max) and in_bloom_filter(a, DynamicValue(RS_3_s_a_bloom_filter))) (type: boolean)
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Reduce Output Operator
+ key expressions: a (type: int)
+ sort order: +
+ Map-reduce partition columns: a (type: int)
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ value expressions: ROW__ID (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>)
+ Execution mode: llap
+ LLAP IO: may be used (ACID table)
+ Map 7
+ Map Operator Tree:
+ TableScan
+ alias: s
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Reduce Output Operator
+ key expressions: a (type: int)
+ sort order: +
+ Map-reduce partition columns: a (type: int)
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ value expressions: b (type: int)
+ Select Operator
+ expressions: a (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Group By Operator
+ aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=1)
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: binary)
+ Execution mode: llap
+ LLAP IO: all inputs
+ Reducer 2
+ Execution mode: llap
+ Reduce Operator Tree:
+ Merge Join Operator
+ condition map:
+ Right Outer Join0 to 1
+ keys:
+ 0 a (type: int)
+ 1 a (type: int)
+ outputColumnNames: _col0, _col4, _col5, _col6
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Filter Operator
+ predicate: ((_col5 > 8) and (_col0 = _col5)) (type: boolean)
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Select Operator
+ expressions: _col4 (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>)
+ sort order: +
+ Map-reduce partition columns: UDFToInteger(_col0) (type: int)
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Filter Operator
+ predicate: ((_col5 <= 8) and (_col0 = _col5)) (type: boolean)
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Select Operator
+ expressions: _col4 (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>), _col0 (type: int)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>)
+ sort order: +
+ Map-reduce partition columns: UDFToInteger(_col0) (type: int)
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ value expressions: _col1 (type: int)
+ Filter Operator
+ predicate: (_col0 = _col5) (type: boolean)
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Select Operator
+ expressions: _col4 (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>)
+ outputColumnNames: _col4
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Group By Operator
+ aggregations: count()
+ keys: _col4 (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>)
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>)
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ value expressions: _col1 (type: bigint)
+ Filter Operator
+ predicate: _col0 is null (type: boolean)
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Select Operator
+ expressions: _col5 (type: int), _col6 (type: int)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ value expressions: _col0 (type: int), _col1 (type: int)
+ Reducer 3
+ Execution mode: llap
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.acidtbl
+ Reducer 4
+ Execution mode: llap
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>), VALUE._col0 (type: int), 7 (type: int)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.acidtbl
+ Reducer 5
+ Execution mode: llap
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ keys: KEY._col0 (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Filter Operator
+ predicate: (_col1 > 1) (type: boolean)
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Select Operator
+ expressions: cardinality_violation(_col0) (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.merge_tmp_table
+ Reducer 6
+ Execution mode: llap
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: int), VALUE._col1 (type: int)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.acidtbl
+ Reducer 8
+ Execution mode: llap
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: min(VALUE._col0), max(VALUE._col1), bloom_filter(VALUE._col2, expectedEntries=1)
+ mode: final
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: binary)
+
+ Stage: Stage-5
+ Dependency Collection
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ replace: false
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.acidtbl
+
+ Stage: Stage-6
+ Stats-Aggr Operator
+
+ Stage: Stage-2
+ Move Operator
+ tables:
+ replace: false
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.acidtbl
+
+ Stage: Stage-7
+ Stats-Aggr Operator
+
+ Stage: Stage-3
+ Move Operator
+ tables:
+ replace: false
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.merge_tmp_table
+
+ Stage: Stage-8
+ Stats-Aggr Operator
+
+ Stage: Stage-1
+ Move Operator
+ tables:
+ replace: false
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.acidtbl
+
+ Stage: Stage-9
+ Stats-Aggr Operator
+
+PREHOOK: query: explain merge into acidTbl as t using nonAcidOrcTbl s ON t.a = s.a
+WHEN NOT MATCHED THEN INSERT VALUES(s.a, s.b)
+PREHOOK: type: QUERY
+POSTHOOK: query: explain merge into acidTbl as t using nonAcidOrcTbl s ON t.a = s.a
+WHEN NOT MATCHED THEN INSERT VALUES(s.a, s.b)
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-0 depends on stages: Stage-2
+ Stage-3 depends on stages: Stage-0
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Map 1 <- Reducer 5 (BROADCAST_EDGE)
+ Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE)
+ Reducer 3 <- Reducer 2 (SIMPLE_EDGE)
+ Reducer 5 <- Map 4 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: t
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Filter Operator
+ predicate: (a BETWEEN DynamicValue(RS_3_s_a_min) AND DynamicValue(RS_3_s_a_max) and in_bloom_filter(a, DynamicValue(RS_3_s_a_bloom_filter))) (type: boolean)
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Reduce Output Operator
+ key expressions: a (type: int)
+ sort order: +
+ Map-reduce partition columns: a (type: int)
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Execution mode: llap
+ LLAP IO: may be used (ACID table)
+ Map 4
+ Map Operator Tree:
+ TableScan
+ alias: s
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Reduce Output Operator
+ key expressions: a (type: int)
+ sort order: +
+ Map-reduce partition columns: a (type: int)
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ value expressions: b (type: int)
+ Select Operator
+ expressions: a (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Group By Operator
+ aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=1)
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: binary)
+ Execution mode: llap
+ LLAP IO: all inputs
+ Reducer 2
+ Execution mode: llap
+ Reduce Operator Tree:
+ Merge Join Operator
+ condition map:
+ Right Outer Join0 to 1
+ keys:
+ 0 a (type: int)
+ 1 a (type: int)
+ outputColumnNames: _col0, _col5, _col6
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Filter Operator
+ predicate: _col0 is null (type: boolean)
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Select Operator
+ expressions: _col5 (type: int), _col6 (type: int)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ value expressions: _col0 (type: int), _col1 (type: int)
+ Reducer 3
+ Execution mode: llap
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: int), VALUE._col1 (type: int)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.acidtbl
+ Reducer 5
+ Execution mode: llap
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: min(VALUE._col0), max(VALUE._col1), bloom_filter(VALUE._col2, expectedEntries=1)
+ mode: final
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: binary)
+
+ Stage: Stage-2
+ Dependency Collection
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ replace: false
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.acidtbl
+
+ Stage: Stage-3
+ Stats-Aggr Operator
+
+PREHOOK: query: explain merge into acidTbl as t using (
+ select * from nonAcidOrcTbl where a > 0
+ union all
+ select * from nonAcidOrcTbl where b > 0
+) s ON t.a = s.a
+WHEN MATCHED AND s.a > 8 THEN DELETE
+WHEN MATCHED THEN UPDATE SET b = 7
+WHEN NOT MATCHED THEN INSERT VALUES(s.a, s.b)
+PREHOOK: type: QUERY
+POSTHOOK: query: explain merge into acidTbl as t using (
+ select * from nonAcidOrcTbl where a > 0
+ union all
+ select * from nonAcidOrcTbl where b > 0
+) s ON t.a = s.a
+WHEN MATCHED AND s.a > 8 THEN DELETE
+WHEN MATCHED THEN UPDATE SET b = 7
+WHEN NOT MATCHED THEN INSERT VALUES(s.a, s.b)
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-4 is a root stage
+ Stage-5 depends on stages: Stage-4
+ Stage-0 depends on stages: Stage-5
+ Stage-6 depends on stages: Stage-0
+ Stage-2 depends on stages: Stage-5
+ Stage-7 depends on stages: Stage-2
+ Stage-3 depends on stages: Stage-5
+ Stage-8 depends on stages: Stage-3
+ Stage-1 depends on stages: Stage-5
+ Stage-9 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-4
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Map 1 <- Union 2 (CONTAINS)
+ Map 10 <- Reducer 8 (BROADCAST_EDGE)
+ Map 9 <- Union 2 (CONTAINS)
+ Reducer 3 <- Map 10 (SIMPLE_EDGE), Union 2 (SIMPLE_EDGE)
+ Reducer 4 <- Reducer 3 (SIMPLE_EDGE)
+ Reducer 5 <- Reducer 3 (SIMPLE_EDGE)
+ Reducer 6 <- Reducer 3 (SIMPLE_EDGE)
+ Reducer 7 <- Reducer 3 (SIMPLE_EDGE)
+ Reducer 8 <- Union 2 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: nonacidorctbl
+ filterExpr: (a > 0) (type: boolean)
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Filter Operator
+ predicate: (a > 0) (type: boolean)
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Select Operator
+ expressions: a (type: int), b (type: int)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ value expressions: _col1 (type: int)
+ Select Operator
+ expressions: _col0 (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Group By Operator
+ aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=2)
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: binary)
+ Execution mode: llap
+ LLAP IO: all inputs
+ Map 10
+ Map Operator Tree:
+ TableScan
+ alias: t
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Filter Operator
+ predicate: (a BETWEEN DynamicValue(RS_10_nonacidorctbl_a_min) AND DynamicValue(RS_10_nonacidorctbl_a_max) and in_bloom_filter(a, DynamicValue(RS_10_nonacidorctbl_a_bloom_filter))) (type: boolean)
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Reduce Output Operator
+ key expressions: a (type: int)
+ sort order: +
+ Map-reduce partition columns: a (type: int)
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ value expressions: ROW__ID (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>)
+ Execution mode: llap
+ LLAP IO: may be used (ACID table)
+ Map 9
+ Map Operator Tree:
+ TableScan
+ alias: nonacidorctbl
+ filterExpr: (b > 0) (type: boolean)
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Filter Operator
+ predicate: (b > 0) (type: boolean)
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Select Operator
+ expressions: a (type: int), b (type: int)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ value expressions: _col1 (type: int)
+ Select Operator
+ expressions: _col0 (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Group By Operator
+ aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=2)
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: binary)
+ Execution mode: llap
+ LLAP IO: all inputs
+ Reducer 3
+ Execution mode: llap
+ Reduce Operator Tree:
+ Merge Join Operator
+ condition map:
+ Right Outer Join0 to 1
+ keys:
+ 0 a (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col0, _col4, _col5, _col6
+ Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Filter Operator
+ predicate: ((_col5 > 8) and (_col0 = _col5)) (type: boolean)
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Select Operator
+ expressions: _col4 (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>)
+ sort order: +
+ Map-reduce partition columns: UDFToInteger(_col0) (type: int)
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Filter Operator
+ predicate: ((_col5 <= 8) and (_col0 = _col5)) (type: boolean)
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Select Operator
+ expressions: _col4 (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>), _col0 (type: int)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>)
+ sort order: +
+ Map-reduce partition columns: UDFToInteger(_col0) (type: int)
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ value expressions: _col1 (type: int)
+ Filter Operator
+ predicate: (_col0 = _col5) (type: boolean)
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Select Operator
+ expressions: _col4 (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>)
+ outputColumnNames: _col4
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Group By Operator
+ aggregations: count()
+ keys: _col4 (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>)
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>)
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ value expressions: _col1 (type: bigint)
+ Filter Operator
+ predicate: _col0 is null (type: boolean)
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Select Operator
+ expressions: _col5 (type: int), _col6 (type: int)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ value expressions: _col0 (type: int), _col1 (type: int)
+ Reducer 4
+ Execution mode: llap
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.acidtbl
+ Reducer 5
+ Execution mode: llap
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>), VALUE._col0 (type: int), 7 (type: int)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.acidtbl
+ Reducer 6
+ Execution mode: llap
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ keys: KEY._col0 (type: struct<transactionid:bigint,bucketid:int,rowid:bigint>)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Filter Operator
+ predicate: (_col1 > 1) (type: boolean)
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Select Operator
+ expressions: cardinality_violation(_col0) (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.merge_tmp_table
+ Reducer 7
+ Execution mode: llap
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: int), VALUE._col1 (type: int)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.acidtbl
+ Reducer 8
+ Execution mode: llap
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: min(VALUE._col0), max(VALUE._col1), bloom_filter(VALUE._col2, expectedEntries=2)
+ mode: final
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: binary)
+ Union 2
+ Vertex: Union 2
+
+ Stage: Stage-5
+ Dependency Collection
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ replace: false
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.acidtbl
+
+ Stage: Stage-6
+ Stats-Aggr Operator
+
+ Stage: Stage-2
+ Move Operator
+ tables:
+ replace: false
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.acidtbl
+
+ Stage: Stage-7
+ Stats-Aggr Operator
+
+ Stage: Stage-3
+ Move Operator
+ tables:
+ replace: false
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.merge_tmp_table
+
+ Stage: Stage-8
+ Stats-Aggr Operator
+
+ Stage: Stage-1
+ Move Operator
+ tables:
+ replace: false
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.acidtbl
+
+ Stage: Stage-9
+ Stats-Aggr Operator
+
http://git-wip-us.apache.org/repos/asf/hive/blob/c3d311f4/ql/src/test/results/clientpositive/llap/orc_llap.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/orc_llap.q.out b/ql/src/test/results/clientpositive/llap/orc_llap.q.out
index aa1dd96..0b12725 100644
--- a/ql/src/test/results/clientpositive/llap/orc_llap.q.out
+++ b/ql/src/test/results/clientpositive/llap/orc_llap.q.out
@@ -563,10 +563,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: o2
- filterExpr: (csmallint is not null and cbigint is not null and csmallint BETWEEN DynamicValue(RS_6_o1_csmallint_min) AND DynamicValue(RS_6_o1_csmallint_max) and in_bloom_filter(csmallint, DynamicValue(RS_6_o1_csmallint_bloom_filter))) (type: boolean)
+ filterExpr: (csmallint is not null and cbigint is not null and (csmallint BETWEEN DynamicValue(RS_6_o1_csmallint_min) AND DynamicValue(RS_6_o1_csmallint_max) and in_bloom_filter(csmallint, DynamicValue(RS_6_o1_csmallint_bloom_filter)))) (type: boolean)
Statistics: Num rows: 122880 Data size: 29079940 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (csmallint is not null and cbigint is not null and csmallint BETWEEN DynamicValue(RS_6_o1_csmallint_min) AND DynamicValue(RS_6_o1_csmallint_max) and in_bloom_filter(csmallint, DynamicValue(RS_6_o1_csmallint_bloom_filter))) (type: boolean)
+ predicate: (csmallint is not null and cbigint is not null and (csmallint BETWEEN DynamicValue(RS_6_o1_csmallint_min) AND DynamicValue(RS_6_o1_csmallint_max) and in_bloom_filter(csmallint, DynamicValue(RS_6_o1_csmallint_bloom_filter)))) (type: boolean)
Statistics: Num rows: 122880 Data size: 29079940 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: csmallint (type: smallint), cstring2 (type: string)
@@ -1016,10 +1016,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: o2
- filterExpr: (csmallint is not null and cbigint is not null and csmallint BETWEEN DynamicValue(RS_6_o1_csmallint_min) AND DynamicValue(RS_6_o1_csmallint_max) and in_bloom_filter(csmallint, DynamicValue(RS_6_o1_csmallint_bloom_filter))) (type: boolean)
+ filterExpr: (csmallint is not null and cbigint is not null and (csmallint BETWEEN DynamicValue(RS_6_o1_csmallint_min) AND DynamicValue(RS_6_o1_csmallint_max) and in_bloom_filter(csmallint, DynamicValue(RS_6_o1_csmallint_bloom_filter)))) (type: boolean)
Statistics: Num rows: 245760 Data size: 58159880 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (csmallint is not null and cbigint is not null and csmallint BETWEEN DynamicValue(RS_6_o1_csmallint_min) AND DynamicValue(RS_6_o1_csmallint_max) and in_bloom_filter(csmallint, DynamicValue(RS_6_o1_csmallint_bloom_filter))) (type: boolean)
+ predicate: (csmallint is not null and cbigint is not null and (csmallint BETWEEN DynamicValue(RS_6_o1_csmallint_min) AND DynamicValue(RS_6_o1_csmallint_max) and in_bloom_filter(csmallint, DynamicValue(RS_6_o1_csmallint_bloom_filter)))) (type: boolean)
Statistics: Num rows: 245760 Data size: 58159880 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: csmallint (type: smallint), cstring2 (type: string)
http://git-wip-us.apache.org/repos/asf/hive/blob/c3d311f4/ql/src/test/results/clientpositive/llap/vectorized_dynamic_partition_pruning.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vectorized_dynamic_partition_pruning.q.out b/ql/src/test/results/clientpositive/llap/vectorized_dynamic_partition_pruning.q.out
index 99e5a42..0d73623 100644
--- a/ql/src/test/results/clientpositive/llap/vectorized_dynamic_partition_pruning.q.out
+++ b/ql/src/test/results/clientpositive/llap/vectorized_dynamic_partition_pruning.q.out
@@ -2944,10 +2944,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: srcpart_hour
- filterExpr: ((UDFToDouble(hr) = 13.0) and hr BETWEEN DynamicValue(RS_12_srcpart_hr_min) AND DynamicValue(RS_12_srcpart_hr_max) and in_bloom_filter(hr, DynamicValue(RS_12_srcpart_hr_bloom_filter))) (type: boolean)
+ filterExpr: ((UDFToDouble(hr) = 13.0) and (hr BETWEEN DynamicValue(RS_12_srcpart_hr_min) AND DynamicValue(RS_12_srcpart_hr_max) and in_bloom_filter(hr, DynamicValue(RS_12_srcpart_hr_bloom_filter)))) (type: boolean)
Statistics: Num rows: 2 Data size: 344 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((UDFToDouble(hr) = 13.0) and hr BETWEEN DynamicValue(RS_12_srcpart_hr_min) AND DynamicValue(RS_12_srcpart_hr_max) and in_bloom_filter(hr, DynamicValue(RS_12_srcpart_hr_bloom_filter))) (type: boolean)
+ predicate: ((UDFToDouble(hr) = 13.0) and (hr BETWEEN DynamicValue(RS_12_srcpart_hr_min) AND DynamicValue(RS_12_srcpart_hr_max) and in_bloom_filter(hr, DynamicValue(RS_12_srcpart_hr_bloom_filter)))) (type: boolean)
Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: hr (type: string)
http://git-wip-us.apache.org/repos/asf/hive/blob/c3d311f4/ql/src/test/results/clientpositive/llap/vectorized_dynamic_semijoin_reduction.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vectorized_dynamic_semijoin_reduction.q.out b/ql/src/test/results/clientpositive/llap/vectorized_dynamic_semijoin_reduction.q.out
index 62a6779..0bf5e85 100644
--- a/ql/src/test/results/clientpositive/llap/vectorized_dynamic_semijoin_reduction.q.out
+++ b/ql/src/test/results/clientpositive/llap/vectorized_dynamic_semijoin_reduction.q.out
@@ -47,10 +47,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: a
- filterExpr: (key_int is not null and key_int BETWEEN DynamicValue(RS_7_b_key_int_min) AND DynamicValue(RS_7_b_key_int_max) and in_bloom_filter(key_int, DynamicValue(RS_7_b_key_int_bloom_filter))) (type: boolean)
+ filterExpr: (key_int is not null and (key_int BETWEEN DynamicValue(RS_7_b_key_int_min) AND DynamicValue(RS_7_b_key_int_max) and in_bloom_filter(key_int, DynamicValue(RS_7_b_key_int_bloom_filter)))) (type: boolean)
Statistics: Num rows: 500 Data size: 90000 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (key_int is not null and key_int BETWEEN DynamicValue(RS_7_b_key_int_min) AND DynamicValue(RS_7_b_key_int_max) and in_bloom_filter(key_int, DynamicValue(RS_7_b_key_int_bloom_filter))) (type: boolean)
+ predicate: (key_int is not null and (key_int BETWEEN DynamicValue(RS_7_b_key_int_min) AND DynamicValue(RS_7_b_key_int_max) and in_bloom_filter(key_int, DynamicValue(RS_7_b_key_int_bloom_filter)))) (type: boolean)
Statistics: Num rows: 500 Data size: 90000 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key_int (type: int)
@@ -183,10 +183,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: a
- filterExpr: (key_str is not null and key_str BETWEEN DynamicValue(RS_7_b_key_str_min) AND DynamicValue(RS_7_b_key_str_max) and in_bloom_filter(key_str, DynamicValue(RS_7_b_key_str_bloom_filter))) (type: boolean)
+ filterExpr: (key_str is not null and (key_str BETWEEN DynamicValue(RS_7_b_key_str_min) AND DynamicValue(RS_7_b_key_str_max) and in_bloom_filter(key_str, DynamicValue(RS_7_b_key_str_bloom_filter)))) (type: boolean)
Statistics: Num rows: 500 Data size: 90000 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (key_str is not null and key_str BETWEEN DynamicValue(RS_7_b_key_str_min) AND DynamicValue(RS_7_b_key_str_max) and in_bloom_filter(key_str, DynamicValue(RS_7_b_key_str_bloom_filter))) (type: boolean)
+ predicate: (key_str is not null and (key_str BETWEEN DynamicValue(RS_7_b_key_str_min) AND DynamicValue(RS_7_b_key_str_max) and in_bloom_filter(key_str, DynamicValue(RS_7_b_key_str_bloom_filter)))) (type: boolean)
Statistics: Num rows: 500 Data size: 90000 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key_str (type: string)
@@ -319,10 +319,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: a
- filterExpr: (key_str is not null and key_str BETWEEN DynamicValue(RS_7_b_key_str_min) AND DynamicValue(RS_7_b_key_str_max) and in_bloom_filter(key_str, DynamicValue(RS_7_b_key_str_bloom_filter))) (type: boolean)
+ filterExpr: (key_str is not null and (key_str BETWEEN DynamicValue(RS_7_b_key_str_min) AND DynamicValue(RS_7_b_key_str_max) and in_bloom_filter(key_str, DynamicValue(RS_7_b_key_str_bloom_filter)))) (type: boolean)
Statistics: Num rows: 500 Data size: 90000 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (key_str is not null and key_str BETWEEN DynamicValue(RS_7_b_key_str_min) AND DynamicValue(RS_7_b_key_str_max) and in_bloom_filter(key_str, DynamicValue(RS_7_b_key_str_bloom_filter))) (type: boolean)
+ predicate: (key_str is not null and (key_str BETWEEN DynamicValue(RS_7_b_key_str_min) AND DynamicValue(RS_7_b_key_str_max) and in_bloom_filter(key_str, DynamicValue(RS_7_b_key_str_bloom_filter)))) (type: boolean)
Statistics: Num rows: 500 Data size: 90000 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key_str (type: string)
@@ -456,10 +456,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: a
- filterExpr: (key_int is not null and key_int BETWEEN DynamicValue(RS_10_b_key_int_min) AND DynamicValue(RS_10_b_key_int_max) and key_int BETWEEN DynamicValue(RS_11_c_key_int_min) AND DynamicValue(RS_11_c_key_int_max) and in_bloom_filter(key_int, DynamicValue(RS_10_b_key_int_bloom_filter)) and in_bloom_filter(key_int, DynamicValue(RS_11_c_key_int_bloom_filter))) (type: boolean)
+ filterExpr: (key_int is not null and (key_int BETWEEN DynamicValue(RS_10_b_key_int_min) AND DynamicValue(RS_10_b_key_int_max) and in_bloom_filter(key_int, DynamicValue(RS_10_b_key_int_bloom_filter))) and (key_int BETWEEN DynamicValue(RS_11_c_key_int_min) AND DynamicValue(RS_11_c_key_int_max) and in_bloom_filter(key_int, DynamicValue(RS_11_c_key_int_bloom_filter)))) (type: boolean)
Statistics: Num rows: 500 Data size: 90000 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (key_int is not null and key_int BETWEEN DynamicValue(RS_10_b_key_int_min) AND DynamicValue(RS_10_b_key_int_max) and key_int BETWEEN DynamicValue(RS_11_c_key_int_min) AND DynamicValue(RS_11_c_key_int_max) and in_bloom_filter(key_int, DynamicValue(RS_10_b_key_int_bloom_filter)) and in_bloom_filter(key_int, DynamicValue(RS_11_c_key_int_bloom_filter))) (type: boolean)
+ predicate: (key_int is not null and (key_int BETWEEN DynamicValue(RS_10_b_key_int_min) AND DynamicValue(RS_10_b_key_int_max) and in_bloom_filter(key_int, DynamicValue(RS_10_b_key_int_bloom_filter))) and (key_int BETWEEN DynamicValue(RS_11_c_key_int_min) AND DynamicValue(RS_11_c_key_int_max) and in_bloom_filter(key_int, DynamicValue(RS_11_c_key_int_bloom_filter)))) (type: boolean)
Statistics: Num rows: 500 Data size: 90000 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key_int (type: int)
@@ -640,10 +640,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: a
- filterExpr: (key_str is not null and key_int is not null and key_str BETWEEN DynamicValue(RS_7_b_key_str_min) AND DynamicValue(RS_7_b_key_str_max) and key_int BETWEEN DynamicValue(RS_7_b_key_int_min) AND DynamicValue(RS_7_b_key_int_max) and in_bloom_filter(key_str, DynamicValue(RS_7_b_key_str_bloom_filter)) and in_bloom_filter(key_int, DynamicValue(RS_7_b_key_int_bloom_filter))) (type: boolean)
+ filterExpr: (key_str is not null and key_int is not null and (key_str BETWEEN DynamicValue(RS_7_b_key_str_min) AND DynamicValue(RS_7_b_key_str_max) and in_bloom_filter(key_str, DynamicValue(RS_7_b_key_str_bloom_filter))) and (key_int BETWEEN DynamicValue(RS_7_b_key_int_min) AND DynamicValue(RS_7_b_key_int_max) and in_bloom_filter(key_int, DynamicValue(RS_7_b_key_int_bloom_filter)))) (type: boolean)
Statistics: Num rows: 500 Data size: 90000 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (key_str is not null and key_int is not null and key_str BETWEEN DynamicValue(RS_7_b_key_str_min) AND DynamicValue(RS_7_b_key_str_max) and key_int BETWEEN DynamicValue(RS_7_b_key_int_min) AND DynamicValue(RS_7_b_key_int_max) and in_bloom_filter(key_str, DynamicValue(RS_7_b_key_str_bloom_filter)) and in_bloom_filter(key_int, DynamicValue(RS_7_b_key_int_bloom_filter))) (type: boolean)
+ predicate: (key_str is not null and key_int is not null and (key_str BETWEEN DynamicValue(RS_7_b_key_str_min) AND DynamicValue(RS_7_b_key_str_max) and in_bloom_filter(key_str, DynamicValue(RS_7_b_key_str_bloom_filter))) and (key_int BETWEEN DynamicValue(RS_7_b_key_int_min) AND DynamicValue(RS_7_b_key_int_max) and in_bloom_filter(key_int, DynamicValue(RS_7_b_key_int_bloom_filter)))) (type: boolean)
Statistics: Num rows: 500 Data size: 90000 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key_str (type: string), key_int (type: int)
@@ -801,10 +801,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: a
- filterExpr: (key_int is not null and key_int BETWEEN DynamicValue(RS_7_b_key_int_min) AND DynamicValue(RS_7_b_key_int_max) and in_bloom_filter(key_int, DynamicValue(RS_7_b_key_int_bloom_filter))) (type: boolean)
+ filterExpr: (key_int is not null and (key_int BETWEEN DynamicValue(RS_7_b_key_int_min) AND DynamicValue(RS_7_b_key_int_max) and in_bloom_filter(key_int, DynamicValue(RS_7_b_key_int_bloom_filter)))) (type: boolean)
Statistics: Num rows: 500 Data size: 90000 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (key_int is not null and key_int BETWEEN DynamicValue(RS_7_b_key_int_min) AND DynamicValue(RS_7_b_key_int_max) and in_bloom_filter(key_int, DynamicValue(RS_7_b_key_int_bloom_filter))) (type: boolean)
+ predicate: (key_int is not null and (key_int BETWEEN DynamicValue(RS_7_b_key_int_min) AND DynamicValue(RS_7_b_key_int_max) and in_bloom_filter(key_int, DynamicValue(RS_7_b_key_int_bloom_filter)))) (type: boolean)
Statistics: Num rows: 500 Data size: 90000 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key_int (type: int)
@@ -953,10 +953,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: a
- filterExpr: (key_int is not null and key_int BETWEEN DynamicValue(RS_7_b_key_int_min) AND DynamicValue(RS_7_b_key_int_max) and in_bloom_filter(key_int, DynamicValue(RS_7_b_key_int_bloom_filter))) (type: boolean)
+ filterExpr: (key_int is not null and (key_int BETWEEN DynamicValue(RS_7_b_key_int_min) AND DynamicValue(RS_7_b_key_int_max) and in_bloom_filter(key_int, DynamicValue(RS_7_b_key_int_bloom_filter)))) (type: boolean)
Statistics: Num rows: 500 Data size: 90000 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (key_int is not null and key_int BETWEEN DynamicValue(RS_7_b_key_int_min) AND DynamicValue(RS_7_b_key_int_max) and in_bloom_filter(key_int, DynamicValue(RS_7_b_key_int_bloom_filter))) (type: boolean)
+ predicate: (key_int is not null and (key_int BETWEEN DynamicValue(RS_7_b_key_int_min) AND DynamicValue(RS_7_b_key_int_max) and in_bloom_filter(key_int, DynamicValue(RS_7_b_key_int_bloom_filter)))) (type: boolean)
Statistics: Num rows: 500 Data size: 90000 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key_int (type: int)
[48/50] [abbrv] hive git commit: HIVE-16167 : Remove transitive
dependency on mysql connector jar (Ashutosh Chauhan via Gunther Hagleitner)
Posted by om...@apache.org.
HIVE-16167 : Remove transitive dependency on mysql connector jar (Ashutosh Chauhan via Gunther Hagleitner)
Change-Id: Ic0d6cd277d91cea0a1bc210ee1dd46db40572f87
Signed-off-by: Ashutosh Chauhan <ha...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/9f8e0900
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/9f8e0900
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/9f8e0900
Branch: refs/heads/branch-2.2
Commit: 9f8e0900dfae85023c9acfd211ad434c74c5170e
Parents: 154c58b
Author: Ashutosh Chauhan <ha...@apache.org>
Authored: Thu Mar 9 17:22:23 2017 -0800
Committer: Owen O'Malley <om...@apache.org>
Committed: Tue Mar 28 15:27:58 2017 -0700
----------------------------------------------------------------------
druid-handler/pom.xml | 8 +++++++-
1 file changed, 7 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/9f8e0900/druid-handler/pom.xml
----------------------------------------------------------------------
diff --git a/druid-handler/pom.xml b/druid-handler/pom.xml
index 38c8e98..0d27ac1 100644
--- a/druid-handler/pom.xml
+++ b/druid-handler/pom.xml
@@ -156,7 +156,13 @@
<groupId>io.druid.extensions</groupId>
<artifactId>mysql-metadata-storage</artifactId>
<version>${druid.version}</version>
- </dependency>
+ <exclusions>
+ <exclusion>
+ <groupId>mysql</groupId>
+ <artifactId>mysql-connector-java</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
<dependency>
<groupId>io.druid.extensions</groupId>
<artifactId>postgresql-metadata-storage</artifactId>
[04/50] [abbrv] hive git commit: HIVE-14959: Fix DISTINCT with
windowing when CBO is enabled/disabled (Jesus Camacho Rodriguez,
reviewed by Ashutosh Chauhan)
Posted by om...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/9e7daaf4/ql/src/test/results/clientpositive/llap/cbo_rp_windowing_2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/cbo_rp_windowing_2.q.out b/ql/src/test/results/clientpositive/llap/cbo_rp_windowing_2.q.out
new file mode 100644
index 0000000..834768c
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/cbo_rp_windowing_2.q.out
@@ -0,0 +1,2247 @@
+PREHOOK: query: select p_mfgr, p_name, p_size,
+rank() over(distribute by p_mfgr sort by p_name) as r,
+dense_rank() over(distribute by p_mfgr sort by p_name) as dr,
+sum(p_retailprice) over (distribute by p_mfgr sort by p_name rows between unbounded preceding and current row) as s1
+from part
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part
+#### A masked pattern was here ####
+POSTHOOK: query: select p_mfgr, p_name, p_size,
+rank() over(distribute by p_mfgr sort by p_name) as r,
+dense_rank() over(distribute by p_mfgr sort by p_name) as dr,
+sum(p_retailprice) over (distribute by p_mfgr sort by p_name rows between unbounded preceding and current row) as s1
+from part
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part
+#### A masked pattern was here ####
+Manufacturer#1 almond antique burnished rose metallic 2 1 1 1173.15
+Manufacturer#1 almond antique burnished rose metallic 2 1 1 2346.3
+Manufacturer#1 almond antique chartreuse lavender yellow 34 3 2 4100.06
+Manufacturer#1 almond antique salmon chartreuse burlywood 6 4 3 5702.650000000001
+Manufacturer#1 almond aquamarine burnished black steel 28 5 4 7117.070000000001
+Manufacturer#1 almond aquamarine pink moccasin thistle 42 6 5 8749.730000000001
+Manufacturer#2 almond antique violet chocolate turquoise 14 1 1 1690.68
+Manufacturer#2 almond antique violet turquoise frosted 40 2 2 3491.38
+Manufacturer#2 almond aquamarine midnight light salmon 2 3 3 5523.360000000001
+Manufacturer#2 almond aquamarine rose maroon antique 25 4 4 7222.02
+Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 5 5 8923.62
+Manufacturer#3 almond antique chartreuse khaki white 17 1 1 1671.68
+Manufacturer#3 almond antique forest lavender goldenrod 14 2 2 2861.95
+Manufacturer#3 almond antique metallic orange dim 19 3 3 4272.34
+Manufacturer#3 almond antique misty red olive 1 4 4 6195.32
+Manufacturer#3 almond antique olive coral navajo 45 5 5 7532.61
+Manufacturer#4 almond antique gainsboro frosted violet 10 1 1 1620.67
+Manufacturer#4 almond antique violet mint lemon 39 2 2 2996.09
+Manufacturer#4 almond aquamarine floral ivory bisque 27 3 3 4202.35
+Manufacturer#4 almond aquamarine yellow dodger mint 7 4 4 6047.27
+Manufacturer#4 almond azure aquamarine papaya violet 12 5 5 7337.620000000001
+Manufacturer#5 almond antique blue firebrick mint 31 1 1 1789.69
+Manufacturer#5 almond antique medium spring khaki 6 2 2 3401.3500000000004
+Manufacturer#5 almond antique sky peru orange 2 3 3 5190.08
+Manufacturer#5 almond aquamarine dodger light gainsboro 46 4 4 6208.18
+Manufacturer#5 almond azure blanched chiffon midnight 23 5 5 7672.66
+PREHOOK: query: select p_mfgr, p_name, p_size,
+min(p_retailprice),
+rank() over(distribute by p_mfgr sort by p_name)as r,
+dense_rank() over(distribute by p_mfgr sort by p_name) as dr,
+p_size, p_size - lag(p_size,1,p_size) over(distribute by p_mfgr sort by p_name) as deltaSz
+from part
+group by p_mfgr, p_name, p_size
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part
+#### A masked pattern was here ####
+POSTHOOK: query: select p_mfgr, p_name, p_size,
+min(p_retailprice),
+rank() over(distribute by p_mfgr sort by p_name)as r,
+dense_rank() over(distribute by p_mfgr sort by p_name) as dr,
+p_size, p_size - lag(p_size,1,p_size) over(distribute by p_mfgr sort by p_name) as deltaSz
+from part
+group by p_mfgr, p_name, p_size
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part
+#### A masked pattern was here ####
+Manufacturer#1 almond antique burnished rose metallic 2 1173.15 1 1 2 0
+Manufacturer#1 almond antique chartreuse lavender yellow 34 1753.76 2 2 34 32
+Manufacturer#1 almond antique salmon chartreuse burlywood 6 1602.59 3 3 6 -28
+Manufacturer#1 almond aquamarine burnished black steel 28 1414.42 4 4 28 22
+Manufacturer#1 almond aquamarine pink moccasin thistle 42 1632.66 5 5 42 14
+Manufacturer#2 almond antique violet chocolate turquoise 14 1690.68 1 1 14 0
+Manufacturer#2 almond antique violet turquoise frosted 40 1800.7 2 2 40 26
+Manufacturer#2 almond aquamarine midnight light salmon 2 2031.98 3 3 2 -38
+Manufacturer#2 almond aquamarine rose maroon antique 25 1698.66 4 4 25 23
+Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 1701.6 5 5 18 -7
+Manufacturer#3 almond antique chartreuse khaki white 17 1671.68 1 1 17 0
+Manufacturer#3 almond antique forest lavender goldenrod 14 1190.27 2 2 14 -3
+Manufacturer#3 almond antique metallic orange dim 19 1410.39 3 3 19 5
+Manufacturer#3 almond antique misty red olive 1 1922.98 4 4 1 -18
+Manufacturer#3 almond antique olive coral navajo 45 1337.29 5 5 45 44
+Manufacturer#4 almond antique gainsboro frosted violet 10 1620.67 1 1 10 0
+Manufacturer#4 almond antique violet mint lemon 39 1375.42 2 2 39 29
+Manufacturer#4 almond aquamarine floral ivory bisque 27 1206.26 3 3 27 -12
+Manufacturer#4 almond aquamarine yellow dodger mint 7 1844.92 4 4 7 -20
+Manufacturer#4 almond azure aquamarine papaya violet 12 1290.35 5 5 12 5
+Manufacturer#5 almond antique blue firebrick mint 31 1789.69 1 1 31 0
+Manufacturer#5 almond antique medium spring khaki 6 1611.66 2 2 6 -25
+Manufacturer#5 almond antique sky peru orange 2 1788.73 3 3 2 -4
+Manufacturer#5 almond aquamarine dodger light gainsboro 46 1018.1 4 4 46 44
+Manufacturer#5 almond azure blanched chiffon midnight 23 1464.48 5 5 23 -23
+PREHOOK: query: select p_mfgr, p_name, p_size, min(p_retailprice),
+rank() over(distribute by p_mfgr sort by p_name) as r,
+dense_rank() over(distribute by p_mfgr sort by p_name) as dr,
+p_size, p_size - lag(p_size,1,p_size) over(distribute by p_mfgr sort by p_name) as deltaSz
+from part
+group by p_mfgr, p_name, p_size
+having p_size > 0
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part
+#### A masked pattern was here ####
+POSTHOOK: query: select p_mfgr, p_name, p_size, min(p_retailprice),
+rank() over(distribute by p_mfgr sort by p_name) as r,
+dense_rank() over(distribute by p_mfgr sort by p_name) as dr,
+p_size, p_size - lag(p_size,1,p_size) over(distribute by p_mfgr sort by p_name) as deltaSz
+from part
+group by p_mfgr, p_name, p_size
+having p_size > 0
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part
+#### A masked pattern was here ####
+Manufacturer#1 almond antique burnished rose metallic 2 1173.15 1 1 2 0
+Manufacturer#1 almond antique chartreuse lavender yellow 34 1753.76 2 2 34 32
+Manufacturer#1 almond antique salmon chartreuse burlywood 6 1602.59 3 3 6 -28
+Manufacturer#1 almond aquamarine burnished black steel 28 1414.42 4 4 28 22
+Manufacturer#1 almond aquamarine pink moccasin thistle 42 1632.66 5 5 42 14
+Manufacturer#2 almond antique violet chocolate turquoise 14 1690.68 1 1 14 0
+Manufacturer#2 almond antique violet turquoise frosted 40 1800.7 2 2 40 26
+Manufacturer#2 almond aquamarine midnight light salmon 2 2031.98 3 3 2 -38
+Manufacturer#2 almond aquamarine rose maroon antique 25 1698.66 4 4 25 23
+Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 1701.6 5 5 18 -7
+Manufacturer#3 almond antique chartreuse khaki white 17 1671.68 1 1 17 0
+Manufacturer#3 almond antique forest lavender goldenrod 14 1190.27 2 2 14 -3
+Manufacturer#3 almond antique metallic orange dim 19 1410.39 3 3 19 5
+Manufacturer#3 almond antique misty red olive 1 1922.98 4 4 1 -18
+Manufacturer#3 almond antique olive coral navajo 45 1337.29 5 5 45 44
+Manufacturer#4 almond antique gainsboro frosted violet 10 1620.67 1 1 10 0
+Manufacturer#4 almond antique violet mint lemon 39 1375.42 2 2 39 29
+Manufacturer#4 almond aquamarine floral ivory bisque 27 1206.26 3 3 27 -12
+Manufacturer#4 almond aquamarine yellow dodger mint 7 1844.92 4 4 7 -20
+Manufacturer#4 almond azure aquamarine papaya violet 12 1290.35 5 5 12 5
+Manufacturer#5 almond antique blue firebrick mint 31 1789.69 1 1 31 0
+Manufacturer#5 almond antique medium spring khaki 6 1611.66 2 2 6 -25
+Manufacturer#5 almond antique sky peru orange 2 1788.73 3 3 2 -4
+Manufacturer#5 almond aquamarine dodger light gainsboro 46 1018.1 4 4 46 44
+Manufacturer#5 almond azure blanched chiffon midnight 23 1464.48 5 5 23 -23
+PREHOOK: query: select p_mfgr, p_name,
+count(p_size) over(distribute by p_mfgr sort by p_name) as cd
+from part
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part
+#### A masked pattern was here ####
+POSTHOOK: query: select p_mfgr, p_name,
+count(p_size) over(distribute by p_mfgr sort by p_name) as cd
+from part
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part
+#### A masked pattern was here ####
+Manufacturer#1 almond antique burnished rose metallic 2
+Manufacturer#1 almond antique burnished rose metallic 2
+Manufacturer#1 almond antique chartreuse lavender yellow 3
+Manufacturer#1 almond antique salmon chartreuse burlywood 4
+Manufacturer#1 almond aquamarine burnished black steel 5
+Manufacturer#1 almond aquamarine pink moccasin thistle 6
+Manufacturer#2 almond antique violet chocolate turquoise 1
+Manufacturer#2 almond antique violet turquoise frosted 2
+Manufacturer#2 almond aquamarine midnight light salmon 3
+Manufacturer#2 almond aquamarine rose maroon antique 4
+Manufacturer#2 almond aquamarine sandy cyan gainsboro 5
+Manufacturer#3 almond antique chartreuse khaki white 1
+Manufacturer#3 almond antique forest lavender goldenrod 2
+Manufacturer#3 almond antique metallic orange dim 3
+Manufacturer#3 almond antique misty red olive 4
+Manufacturer#3 almond antique olive coral navajo 5
+Manufacturer#4 almond antique gainsboro frosted violet 1
+Manufacturer#4 almond antique violet mint lemon 2
+Manufacturer#4 almond aquamarine floral ivory bisque 3
+Manufacturer#4 almond aquamarine yellow dodger mint 4
+Manufacturer#4 almond azure aquamarine papaya violet 5
+Manufacturer#5 almond antique blue firebrick mint 1
+Manufacturer#5 almond antique medium spring khaki 2
+Manufacturer#5 almond antique sky peru orange 3
+Manufacturer#5 almond aquamarine dodger light gainsboro 4
+Manufacturer#5 almond azure blanched chiffon midnight 5
+PREHOOK: query: select p_mfgr, p_name,
+rank() over(distribute by p_mfgr sort by p_name) as r,
+dense_rank() over(distribute by p_mfgr sort by p_name) as dr,
+count(p_size) over(distribute by p_mfgr sort by p_name) as cd,
+p_retailprice, sum(p_retailprice) over (distribute by p_mfgr sort by p_name rows between unbounded preceding and current row) as s1,
+p_size, p_size - lag(p_size,1,p_size) over(distribute by p_mfgr sort by p_name) as deltaSz
+from part
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part
+#### A masked pattern was here ####
+POSTHOOK: query: select p_mfgr, p_name,
+rank() over(distribute by p_mfgr sort by p_name) as r,
+dense_rank() over(distribute by p_mfgr sort by p_name) as dr,
+count(p_size) over(distribute by p_mfgr sort by p_name) as cd,
+p_retailprice, sum(p_retailprice) over (distribute by p_mfgr sort by p_name rows between unbounded preceding and current row) as s1,
+p_size, p_size - lag(p_size,1,p_size) over(distribute by p_mfgr sort by p_name) as deltaSz
+from part
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part
+#### A masked pattern was here ####
+Manufacturer#1 almond antique burnished rose metallic 1 1 2 1173.15 1173.15 2 0
+Manufacturer#1 almond antique burnished rose metallic 1 1 2 1173.15 2346.3 2 0
+Manufacturer#1 almond antique chartreuse lavender yellow 3 2 3 1753.76 4100.06 34 32
+Manufacturer#1 almond antique salmon chartreuse burlywood 4 3 4 1602.59 5702.650000000001 6 -28
+Manufacturer#1 almond aquamarine burnished black steel 5 4 5 1414.42 7117.070000000001 28 22
+Manufacturer#1 almond aquamarine pink moccasin thistle 6 5 6 1632.66 8749.730000000001 42 14
+Manufacturer#2 almond antique violet chocolate turquoise 1 1 1 1690.68 1690.68 14 0
+Manufacturer#2 almond antique violet turquoise frosted 2 2 2 1800.7 3491.38 40 26
+Manufacturer#2 almond aquamarine midnight light salmon 3 3 3 2031.98 5523.360000000001 2 -38
+Manufacturer#2 almond aquamarine rose maroon antique 4 4 4 1698.66 7222.02 25 23
+Manufacturer#2 almond aquamarine sandy cyan gainsboro 5 5 5 1701.6 8923.62 18 -7
+Manufacturer#3 almond antique chartreuse khaki white 1 1 1 1671.68 1671.68 17 0
+Manufacturer#3 almond antique forest lavender goldenrod 2 2 2 1190.27 2861.95 14 -3
+Manufacturer#3 almond antique metallic orange dim 3 3 3 1410.39 4272.34 19 5
+Manufacturer#3 almond antique misty red olive 4 4 4 1922.98 6195.32 1 -18
+Manufacturer#3 almond antique olive coral navajo 5 5 5 1337.29 7532.61 45 44
+Manufacturer#4 almond antique gainsboro frosted violet 1 1 1 1620.67 1620.67 10 0
+Manufacturer#4 almond antique violet mint lemon 2 2 2 1375.42 2996.09 39 29
+Manufacturer#4 almond aquamarine floral ivory bisque 3 3 3 1206.26 4202.35 27 -12
+Manufacturer#4 almond aquamarine yellow dodger mint 4 4 4 1844.92 6047.27 7 -20
+Manufacturer#4 almond azure aquamarine papaya violet 5 5 5 1290.35 7337.620000000001 12 5
+Manufacturer#5 almond antique blue firebrick mint 1 1 1 1789.69 1789.69 31 0
+Manufacturer#5 almond antique medium spring khaki 2 2 2 1611.66 3401.3500000000004 6 -25
+Manufacturer#5 almond antique sky peru orange 3 3 3 1788.73 5190.08 2 -4
+Manufacturer#5 almond aquamarine dodger light gainsboro 4 4 4 1018.1 6208.18 46 44
+Manufacturer#5 almond azure blanched chiffon midnight 5 5 5 1464.48 7672.66 23 -23
+PREHOOK: query: select sub1.r, sub1.dr, sub1.cd, sub1.s1, sub1.deltaSz
+from (select p_mfgr, p_name,
+rank() over(distribute by p_mfgr sort by p_name) as r,
+dense_rank() over(distribute by p_mfgr sort by p_name) as dr,
+count(p_size) over(distribute by p_mfgr sort by p_name) as cd,
+p_retailprice, sum(p_retailprice) over (distribute by p_mfgr sort by p_name rows between unbounded preceding and current row) as s1,
+p_size, p_size - lag(p_size,1,p_size) over(distribute by p_mfgr sort by p_name) as deltaSz
+from part
+) sub1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part
+#### A masked pattern was here ####
+POSTHOOK: query: select sub1.r, sub1.dr, sub1.cd, sub1.s1, sub1.deltaSz
+from (select p_mfgr, p_name,
+rank() over(distribute by p_mfgr sort by p_name) as r,
+dense_rank() over(distribute by p_mfgr sort by p_name) as dr,
+count(p_size) over(distribute by p_mfgr sort by p_name) as cd,
+p_retailprice, sum(p_retailprice) over (distribute by p_mfgr sort by p_name rows between unbounded preceding and current row) as s1,
+p_size, p_size - lag(p_size,1,p_size) over(distribute by p_mfgr sort by p_name) as deltaSz
+from part
+) sub1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part
+#### A masked pattern was here ####
+1 1 1 1620.67 0
+1 1 1 1671.68 0
+1 1 1 1690.68 0
+1 1 1 1789.69 0
+1 1 2 1173.15 0
+1 1 2 2346.3 0
+2 2 2 2861.95 -3
+2 2 2 2996.09 29
+2 2 2 3401.3500000000004 -25
+2 2 2 3491.38 26
+3 2 3 4100.06 32
+3 3 3 4202.35 -12
+3 3 3 4272.34 5
+3 3 3 5190.08 -4
+3 3 3 5523.360000000001 -38
+4 3 4 5702.650000000001 -28
+4 4 4 6047.27 -20
+4 4 4 6195.32 -18
+4 4 4 6208.18 44
+4 4 4 7222.02 23
+5 4 5 7117.070000000001 22
+5 5 5 7337.620000000001 5
+5 5 5 7532.61 44
+5 5 5 7672.66 -23
+5 5 5 8923.62 -7
+6 5 6 8749.730000000001 14
+PREHOOK: query: select abc.p_mfgr, abc.p_name,
+rank() over(distribute by abc.p_mfgr sort by abc.p_name) as r,
+dense_rank() over(distribute by abc.p_mfgr sort by abc.p_name) as dr,
+abc.p_retailprice, sum(abc.p_retailprice) over (distribute by abc.p_mfgr sort by abc.p_name rows between unbounded preceding and current row) as s1,
+abc.p_size, abc.p_size - lag(abc.p_size,1,abc.p_size) over(distribute by abc.p_mfgr sort by abc.p_name) as deltaSz
+from noop(on part
+partition by p_mfgr
+order by p_name
+) abc join part p1 on abc.p_partkey = p1.p_partkey
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part
+#### A masked pattern was here ####
+POSTHOOK: query: select abc.p_mfgr, abc.p_name,
+rank() over(distribute by abc.p_mfgr sort by abc.p_name) as r,
+dense_rank() over(distribute by abc.p_mfgr sort by abc.p_name) as dr,
+abc.p_retailprice, sum(abc.p_retailprice) over (distribute by abc.p_mfgr sort by abc.p_name rows between unbounded preceding and current row) as s1,
+abc.p_size, abc.p_size - lag(abc.p_size,1,abc.p_size) over(distribute by abc.p_mfgr sort by abc.p_name) as deltaSz
+from noop(on part
+partition by p_mfgr
+order by p_name
+) abc join part p1 on abc.p_partkey = p1.p_partkey
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part
+#### A masked pattern was here ####
+Manufacturer#1 almond antique burnished rose metallic 1 1 1173.15 1173.15 2 0
+Manufacturer#1 almond antique burnished rose metallic 1 1 1173.15 2346.3 2 0
+Manufacturer#1 almond antique burnished rose metallic 1 1 1173.15 3519.4500000000003 2 0
+Manufacturer#1 almond antique burnished rose metallic 1 1 1173.15 4692.6 2 0
+Manufacturer#1 almond antique chartreuse lavender yellow 5 2 1753.76 6446.360000000001 34 32
+Manufacturer#1 almond antique salmon chartreuse burlywood 6 3 1602.59 8048.950000000001 6 -28
+Manufacturer#1 almond aquamarine burnished black steel 7 4 1414.42 9463.37 28 22
+Manufacturer#1 almond aquamarine pink moccasin thistle 8 5 1632.66 11096.03 42 14
+Manufacturer#2 almond antique violet chocolate turquoise 1 1 1690.68 1690.68 14 0
+Manufacturer#2 almond antique violet turquoise frosted 2 2 1800.7 3491.38 40 26
+Manufacturer#2 almond aquamarine midnight light salmon 3 3 2031.98 5523.360000000001 2 -38
+Manufacturer#2 almond aquamarine rose maroon antique 4 4 1698.66 7222.02 25 23
+Manufacturer#2 almond aquamarine sandy cyan gainsboro 5 5 1701.6 8923.62 18 -7
+Manufacturer#3 almond antique chartreuse khaki white 1 1 1671.68 1671.68 17 0
+Manufacturer#3 almond antique forest lavender goldenrod 2 2 1190.27 2861.95 14 -3
+Manufacturer#3 almond antique metallic orange dim 3 3 1410.39 4272.34 19 5
+Manufacturer#3 almond antique misty red olive 4 4 1922.98 6195.32 1 -18
+Manufacturer#3 almond antique olive coral navajo 5 5 1337.29 7532.61 45 44
+Manufacturer#4 almond antique gainsboro frosted violet 1 1 1620.67 1620.67 10 0
+Manufacturer#4 almond antique violet mint lemon 2 2 1375.42 2996.09 39 29
+Manufacturer#4 almond aquamarine floral ivory bisque 3 3 1206.26 4202.35 27 -12
+Manufacturer#4 almond aquamarine yellow dodger mint 4 4 1844.92 6047.27 7 -20
+Manufacturer#4 almond azure aquamarine papaya violet 5 5 1290.35 7337.620000000001 12 5
+Manufacturer#5 almond antique blue firebrick mint 1 1 1789.69 1789.69 31 0
+Manufacturer#5 almond antique medium spring khaki 2 2 1611.66 3401.3500000000004 6 -25
+Manufacturer#5 almond antique sky peru orange 3 3 1788.73 5190.08 2 -4
+Manufacturer#5 almond aquamarine dodger light gainsboro 4 4 1018.1 6208.18 46 44
+Manufacturer#5 almond azure blanched chiffon midnight 5 5 1464.48 7672.66 23 -23
+PREHOOK: query: select p_mfgr, p_name, p_size,
+rank() over(distribute by p_mfgr sort by p_name, p_size desc) as R
+from part
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part
+#### A masked pattern was here ####
+POSTHOOK: query: select p_mfgr, p_name, p_size,
+rank() over(distribute by p_mfgr sort by p_name, p_size desc) as R
+from part
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part
+#### A masked pattern was here ####
+Manufacturer#1 almond antique burnished rose metallic 2 1
+Manufacturer#1 almond antique burnished rose metallic 2 1
+Manufacturer#1 almond antique chartreuse lavender yellow 34 3
+Manufacturer#1 almond antique salmon chartreuse burlywood 6 4
+Manufacturer#1 almond aquamarine burnished black steel 28 5
+Manufacturer#1 almond aquamarine pink moccasin thistle 42 6
+Manufacturer#2 almond antique violet chocolate turquoise 14 1
+Manufacturer#2 almond antique violet turquoise frosted 40 2
+Manufacturer#2 almond aquamarine midnight light salmon 2 3
+Manufacturer#2 almond aquamarine rose maroon antique 25 4
+Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 5
+Manufacturer#3 almond antique chartreuse khaki white 17 1
+Manufacturer#3 almond antique forest lavender goldenrod 14 2
+Manufacturer#3 almond antique metallic orange dim 19 3
+Manufacturer#3 almond antique misty red olive 1 4
+Manufacturer#3 almond antique olive coral navajo 45 5
+Manufacturer#4 almond antique gainsboro frosted violet 10 1
+Manufacturer#4 almond antique violet mint lemon 39 2
+Manufacturer#4 almond aquamarine floral ivory bisque 27 3
+Manufacturer#4 almond aquamarine yellow dodger mint 7 4
+Manufacturer#4 almond azure aquamarine papaya violet 12 5
+Manufacturer#5 almond antique blue firebrick mint 31 1
+Manufacturer#5 almond antique medium spring khaki 6 2
+Manufacturer#5 almond antique sky peru orange 2 3
+Manufacturer#5 almond aquamarine dodger light gainsboro 46 4
+Manufacturer#5 almond azure blanched chiffon midnight 23 5
+PREHOOK: query: select p_mfgr, p_name, p_size,
+rank() over(distribute by p_mfgr sort by p_name) as r,
+dense_rank() over(distribute by p_mfgr sort by p_name) as dr,
+sum(p_retailprice) over (distribute by p_mfgr sort by p_name rows between unbounded preceding and current row) as s1
+from part
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part
+#### A masked pattern was here ####
+POSTHOOK: query: select p_mfgr, p_name, p_size,
+rank() over(distribute by p_mfgr sort by p_name) as r,
+dense_rank() over(distribute by p_mfgr sort by p_name) as dr,
+sum(p_retailprice) over (distribute by p_mfgr sort by p_name rows between unbounded preceding and current row) as s1
+from part
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part
+#### A masked pattern was here ####
+Manufacturer#1 almond antique burnished rose metallic 2 1 1 1173.15
+Manufacturer#1 almond antique burnished rose metallic 2 1 1 2346.3
+Manufacturer#1 almond antique chartreuse lavender yellow 34 3 2 4100.06
+Manufacturer#1 almond antique salmon chartreuse burlywood 6 4 3 5702.650000000001
+Manufacturer#1 almond aquamarine burnished black steel 28 5 4 7117.070000000001
+Manufacturer#1 almond aquamarine pink moccasin thistle 42 6 5 8749.730000000001
+Manufacturer#2 almond antique violet chocolate turquoise 14 1 1 1690.68
+Manufacturer#2 almond antique violet turquoise frosted 40 2 2 3491.38
+Manufacturer#2 almond aquamarine midnight light salmon 2 3 3 5523.360000000001
+Manufacturer#2 almond aquamarine rose maroon antique 25 4 4 7222.02
+Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 5 5 8923.62
+Manufacturer#3 almond antique chartreuse khaki white 17 1 1 1671.68
+Manufacturer#3 almond antique forest lavender goldenrod 14 2 2 2861.95
+Manufacturer#3 almond antique metallic orange dim 19 3 3 4272.34
+Manufacturer#3 almond antique misty red olive 1 4 4 6195.32
+Manufacturer#3 almond antique olive coral navajo 45 5 5 7532.61
+Manufacturer#4 almond antique gainsboro frosted violet 10 1 1 1620.67
+Manufacturer#4 almond antique violet mint lemon 39 2 2 2996.09
+Manufacturer#4 almond aquamarine floral ivory bisque 27 3 3 4202.35
+Manufacturer#4 almond aquamarine yellow dodger mint 7 4 4 6047.27
+Manufacturer#4 almond azure aquamarine papaya violet 12 5 5 7337.620000000001
+Manufacturer#5 almond antique blue firebrick mint 31 1 1 1789.69
+Manufacturer#5 almond antique medium spring khaki 6 2 2 3401.3500000000004
+Manufacturer#5 almond antique sky peru orange 2 3 3 5190.08
+Manufacturer#5 almond aquamarine dodger light gainsboro 46 4 4 6208.18
+Manufacturer#5 almond azure blanched chiffon midnight 23 5 5 7672.66
+PREHOOK: query: select p_mfgr, p_name, p_size,
+rank() over(distribute by p_mfgr sort by p_name) as r,
+dense_rank() over(distribute by p_mfgr sort by p_name) as dr,
+sum(p_retailprice) over (distribute by p_mfgr sort by p_name rows between unbounded preceding and current row) as s1
+from part
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part
+#### A masked pattern was here ####
+POSTHOOK: query: select p_mfgr, p_name, p_size,
+rank() over(distribute by p_mfgr sort by p_name) as r,
+dense_rank() over(distribute by p_mfgr sort by p_name) as dr,
+sum(p_retailprice) over (distribute by p_mfgr sort by p_name rows between unbounded preceding and current row) as s1
+from part
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part
+#### A masked pattern was here ####
+Manufacturer#1 almond antique burnished rose metallic 2 1 1 1173.15
+Manufacturer#1 almond antique burnished rose metallic 2 1 1 2346.3
+Manufacturer#1 almond antique chartreuse lavender yellow 34 3 2 4100.06
+Manufacturer#1 almond antique salmon chartreuse burlywood 6 4 3 5702.650000000001
+Manufacturer#1 almond aquamarine burnished black steel 28 5 4 7117.070000000001
+Manufacturer#1 almond aquamarine pink moccasin thistle 42 6 5 8749.730000000001
+Manufacturer#2 almond antique violet chocolate turquoise 14 1 1 1690.68
+Manufacturer#2 almond antique violet turquoise frosted 40 2 2 3491.38
+Manufacturer#2 almond aquamarine midnight light salmon 2 3 3 5523.360000000001
+Manufacturer#2 almond aquamarine rose maroon antique 25 4 4 7222.02
+Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 5 5 8923.62
+Manufacturer#3 almond antique chartreuse khaki white 17 1 1 1671.68
+Manufacturer#3 almond antique forest lavender goldenrod 14 2 2 2861.95
+Manufacturer#3 almond antique metallic orange dim 19 3 3 4272.34
+Manufacturer#3 almond antique misty red olive 1 4 4 6195.32
+Manufacturer#3 almond antique olive coral navajo 45 5 5 7532.61
+Manufacturer#4 almond antique gainsboro frosted violet 10 1 1 1620.67
+Manufacturer#4 almond antique violet mint lemon 39 2 2 2996.09
+Manufacturer#4 almond aquamarine floral ivory bisque 27 3 3 4202.35
+Manufacturer#4 almond aquamarine yellow dodger mint 7 4 4 6047.27
+Manufacturer#4 almond azure aquamarine papaya violet 12 5 5 7337.620000000001
+Manufacturer#5 almond antique blue firebrick mint 31 1 1 1789.69
+Manufacturer#5 almond antique medium spring khaki 6 2 2 3401.3500000000004
+Manufacturer#5 almond antique sky peru orange 2 3 3 5190.08
+Manufacturer#5 almond aquamarine dodger light gainsboro 46 4 4 6208.18
+Manufacturer#5 almond azure blanched chiffon midnight 23 5 5 7672.66
+PREHOOK: query: select p_mfgr,p_name, p_size,
+sum(p_size) over (distribute by p_mfgr sort by p_name rows between current row and current row) as s2,
+first_value(p_size) over w1 as f,
+last_value(p_size, false) over w1 as l
+from part
+window w1 as (distribute by p_mfgr sort by p_name rows between 2 preceding and 2 following)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part
+#### A masked pattern was here ####
+POSTHOOK: query: select p_mfgr,p_name, p_size,
+sum(p_size) over (distribute by p_mfgr sort by p_name rows between current row and current row) as s2,
+first_value(p_size) over w1 as f,
+last_value(p_size, false) over w1 as l
+from part
+window w1 as (distribute by p_mfgr sort by p_name rows between 2 preceding and 2 following)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part
+#### A masked pattern was here ####
+Manufacturer#1 almond antique burnished rose metallic 2 2 2 34
+Manufacturer#1 almond antique burnished rose metallic 2 2 2 6
+Manufacturer#1 almond antique chartreuse lavender yellow 34 34 2 28
+Manufacturer#1 almond antique salmon chartreuse burlywood 6 6 2 42
+Manufacturer#1 almond aquamarine burnished black steel 28 28 34 42
+Manufacturer#1 almond aquamarine pink moccasin thistle 42 42 6 42
+Manufacturer#2 almond antique violet chocolate turquoise 14 14 14 2
+Manufacturer#2 almond antique violet turquoise frosted 40 40 14 25
+Manufacturer#2 almond aquamarine midnight light salmon 2 2 14 18
+Manufacturer#2 almond aquamarine rose maroon antique 25 25 40 18
+Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 18 2 18
+Manufacturer#3 almond antique chartreuse khaki white 17 17 17 19
+Manufacturer#3 almond antique forest lavender goldenrod 14 14 17 1
+Manufacturer#3 almond antique metallic orange dim 19 19 17 45
+Manufacturer#3 almond antique misty red olive 1 1 14 45
+Manufacturer#3 almond antique olive coral navajo 45 45 19 45
+Manufacturer#4 almond antique gainsboro frosted violet 10 10 10 27
+Manufacturer#4 almond antique violet mint lemon 39 39 10 7
+Manufacturer#4 almond aquamarine floral ivory bisque 27 27 10 12
+Manufacturer#4 almond aquamarine yellow dodger mint 7 7 39 12
+Manufacturer#4 almond azure aquamarine papaya violet 12 12 27 12
+Manufacturer#5 almond antique blue firebrick mint 31 31 31 2
+Manufacturer#5 almond antique medium spring khaki 6 6 31 46
+Manufacturer#5 almond antique sky peru orange 2 2 31 23
+Manufacturer#5 almond aquamarine dodger light gainsboro 46 46 6 23
+Manufacturer#5 almond azure blanched chiffon midnight 23 23 2 23
+PREHOOK: query: select p_mfgr,p_name, p_size,
+rank() over(distribute by p_mfgr sort by p_name) as r,
+sum(p_size) over (distribute by p_mfgr sort by p_name rows between current row and current row) as s2,
+first_value(p_size) over w1 as f,
+last_value(p_size, false) over w1 as l
+from part
+where p_mfgr = 'Manufacturer#3'
+window w1 as (distribute by p_mfgr sort by p_name rows between 2 preceding and 2 following)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part
+#### A masked pattern was here ####
+POSTHOOK: query: select p_mfgr,p_name, p_size,
+rank() over(distribute by p_mfgr sort by p_name) as r,
+sum(p_size) over (distribute by p_mfgr sort by p_name rows between current row and current row) as s2,
+first_value(p_size) over w1 as f,
+last_value(p_size, false) over w1 as l
+from part
+where p_mfgr = 'Manufacturer#3'
+window w1 as (distribute by p_mfgr sort by p_name rows between 2 preceding and 2 following)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part
+#### A masked pattern was here ####
+Manufacturer#3 almond antique chartreuse khaki white 17 1 17 17 19
+Manufacturer#3 almond antique forest lavender goldenrod 14 2 14 17 1
+Manufacturer#3 almond antique metallic orange dim 19 3 19 17 45
+Manufacturer#3 almond antique misty red olive 1 4 1 14 45
+Manufacturer#3 almond antique olive coral navajo 45 5 45 19 45
+PREHOOK: query: select p_mfgr,p_name, p_size,
+sum(p_size) over w1 as s1,
+sum(p_size) over (distribute by p_mfgr sort by p_name rows between current row and current row) as s2
+from part
+window w1 as (distribute by p_mfgr sort by p_name rows between 2 preceding and 2 following)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part
+#### A masked pattern was here ####
+POSTHOOK: query: select p_mfgr,p_name, p_size,
+sum(p_size) over w1 as s1,
+sum(p_size) over (distribute by p_mfgr sort by p_name rows between current row and current row) as s2
+from part
+window w1 as (distribute by p_mfgr sort by p_name rows between 2 preceding and 2 following)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part
+#### A masked pattern was here ####
+Manufacturer#1 almond antique burnished rose metallic 2 38 2
+Manufacturer#1 almond antique burnished rose metallic 2 44 2
+Manufacturer#1 almond antique chartreuse lavender yellow 34 72 34
+Manufacturer#1 almond antique salmon chartreuse burlywood 6 112 6
+Manufacturer#1 almond aquamarine burnished black steel 28 110 28
+Manufacturer#1 almond aquamarine pink moccasin thistle 42 76 42
+Manufacturer#2 almond antique violet chocolate turquoise 14 56 14
+Manufacturer#2 almond antique violet turquoise frosted 40 81 40
+Manufacturer#2 almond aquamarine midnight light salmon 2 99 2
+Manufacturer#2 almond aquamarine rose maroon antique 25 85 25
+Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 45 18
+Manufacturer#3 almond antique chartreuse khaki white 17 50 17
+Manufacturer#3 almond antique forest lavender goldenrod 14 51 14
+Manufacturer#3 almond antique metallic orange dim 19 96 19
+Manufacturer#3 almond antique misty red olive 1 79 1
+Manufacturer#3 almond antique olive coral navajo 45 65 45
+Manufacturer#4 almond antique gainsboro frosted violet 10 76 10
+Manufacturer#4 almond antique violet mint lemon 39 83 39
+Manufacturer#4 almond aquamarine floral ivory bisque 27 95 27
+Manufacturer#4 almond aquamarine yellow dodger mint 7 85 7
+Manufacturer#4 almond azure aquamarine papaya violet 12 46 12
+Manufacturer#5 almond antique blue firebrick mint 31 39 31
+Manufacturer#5 almond antique medium spring khaki 6 85 6
+Manufacturer#5 almond antique sky peru orange 2 108 2
+Manufacturer#5 almond aquamarine dodger light gainsboro 46 77 46
+Manufacturer#5 almond azure blanched chiffon midnight 23 71 23
+PREHOOK: query: select p_mfgr,p_name, p_size,
+rank() over(distribute by p_mfgr sort by p_name) as r, dense_rank() over(distribute by p_mfgr sort by p_name) as dr
+from part
+window w1 as (distribute by p_mfgr sort by p_name rows between 2 preceding and 2 following)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part
+#### A masked pattern was here ####
+POSTHOOK: query: select p_mfgr,p_name, p_size,
+rank() over(distribute by p_mfgr sort by p_name) as r, dense_rank() over(distribute by p_mfgr sort by p_name) as dr
+from part
+window w1 as (distribute by p_mfgr sort by p_name rows between 2 preceding and 2 following)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part
+#### A masked pattern was here ####
+Manufacturer#1 almond antique burnished rose metallic 2 1 1
+Manufacturer#1 almond antique burnished rose metallic 2 1 1
+Manufacturer#1 almond antique chartreuse lavender yellow 34 3 2
+Manufacturer#1 almond antique salmon chartreuse burlywood 6 4 3
+Manufacturer#1 almond aquamarine burnished black steel 28 5 4
+Manufacturer#1 almond aquamarine pink moccasin thistle 42 6 5
+Manufacturer#2 almond antique violet chocolate turquoise 14 1 1
+Manufacturer#2 almond antique violet turquoise frosted 40 2 2
+Manufacturer#2 almond aquamarine midnight light salmon 2 3 3
+Manufacturer#2 almond aquamarine rose maroon antique 25 4 4
+Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 5 5
+Manufacturer#3 almond antique chartreuse khaki white 17 1 1
+Manufacturer#3 almond antique forest lavender goldenrod 14 2 2
+Manufacturer#3 almond antique metallic orange dim 19 3 3
+Manufacturer#3 almond antique misty red olive 1 4 4
+Manufacturer#3 almond antique olive coral navajo 45 5 5
+Manufacturer#4 almond antique gainsboro frosted violet 10 1 1
+Manufacturer#4 almond antique violet mint lemon 39 2 2
+Manufacturer#4 almond aquamarine floral ivory bisque 27 3 3
+Manufacturer#4 almond aquamarine yellow dodger mint 7 4 4
+Manufacturer#4 almond azure aquamarine papaya violet 12 5 5
+Manufacturer#5 almond antique blue firebrick mint 31 1 1
+Manufacturer#5 almond antique medium spring khaki 6 2 2
+Manufacturer#5 almond antique sky peru orange 2 3 3
+Manufacturer#5 almond aquamarine dodger light gainsboro 46 4 4
+Manufacturer#5 almond azure blanched chiffon midnight 23 5 5
+PREHOOK: query: select p_mfgr,p_name, p_size,
+rank() over(distribute by p_mfgr sort by p_name) as r,
+dense_rank() over(distribute by p_mfgr sort by p_name) as dr,
+cume_dist() over(distribute by p_mfgr sort by p_name) as cud,
+percent_rank() over(distribute by p_mfgr sort by p_name) as pr,
+ntile(3) over(distribute by p_mfgr sort by p_name) as nt,
+count(p_size) over(distribute by p_mfgr sort by p_name) as ca,
+avg(p_size) over(distribute by p_mfgr sort by p_name) as avg,
+stddev(p_size) over(distribute by p_mfgr sort by p_name) as st,
+first_value(p_size % 5) over(distribute by p_mfgr sort by p_name) as fv,
+last_value(p_size) over(distribute by p_mfgr sort by p_name) as lv,
+first_value(p_size) over w1 as fvW1
+from part
+window w1 as (distribute by p_mfgr sort by p_mfgr, p_name rows between 2 preceding and 2 following)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part
+#### A masked pattern was here ####
+POSTHOOK: query: select p_mfgr,p_name, p_size,
+rank() over(distribute by p_mfgr sort by p_name) as r,
+dense_rank() over(distribute by p_mfgr sort by p_name) as dr,
+cume_dist() over(distribute by p_mfgr sort by p_name) as cud,
+percent_rank() over(distribute by p_mfgr sort by p_name) as pr,
+ntile(3) over(distribute by p_mfgr sort by p_name) as nt,
+count(p_size) over(distribute by p_mfgr sort by p_name) as ca,
+avg(p_size) over(distribute by p_mfgr sort by p_name) as avg,
+stddev(p_size) over(distribute by p_mfgr sort by p_name) as st,
+first_value(p_size % 5) over(distribute by p_mfgr sort by p_name) as fv,
+last_value(p_size) over(distribute by p_mfgr sort by p_name) as lv,
+first_value(p_size) over w1 as fvW1
+from part
+window w1 as (distribute by p_mfgr sort by p_mfgr, p_name rows between 2 preceding and 2 following)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part
+#### A masked pattern was here ####
+Manufacturer#1 almond antique burnished rose metallic 2 1 1 0 0.0 1 2 2.0 0.0 2 2 2
+Manufacturer#1 almond antique burnished rose metallic 2 1 1 0 0.0 1 2 2.0 0.0 2 2 2
+Manufacturer#1 almond antique chartreuse lavender yellow 34 3 2 0 0.4 2 3 12.666666666666666 15.084944665313014 2 34 2
+Manufacturer#1 almond antique salmon chartreuse burlywood 6 4 3 0 0.6 2 4 11.0 13.379088160259652 2 6 2
+Manufacturer#1 almond aquamarine burnished black steel 28 5 4 0 0.8 3 5 14.4 13.763720427268202 2 28 34
+Manufacturer#1 almond aquamarine pink moccasin thistle 42 6 5 1 1.0 3 6 19.0 16.237815945091466 2 42 6
+Manufacturer#2 almond antique violet chocolate turquoise 14 1 1 0 0.0 1 1 14.0 0.0 4 14 14
+Manufacturer#2 almond antique violet turquoise frosted 40 2 2 0 0.25 1 2 27.0 13.0 4 40 14
+Manufacturer#2 almond aquamarine midnight light salmon 2 3 3 0 0.5 2 3 18.666666666666668 15.86050300449376 4 2 14
+Manufacturer#2 almond aquamarine rose maroon antique 25 4 4 0 0.75 2 4 20.25 14.00669482783144 4 25 40
+Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 5 5 1 1.0 3 5 19.8 12.560254774486067 4 18 2
+Manufacturer#3 almond antique chartreuse khaki white 17 1 1 0 0.0 1 1 17.0 0.0 2 17 17
+Manufacturer#3 almond antique forest lavender goldenrod 14 2 2 0 0.25 1 2 15.5 1.5 2 14 17
+Manufacturer#3 almond antique metallic orange dim 19 3 3 0 0.5 2 3 16.666666666666668 2.0548046676563256 2 19 17
+Manufacturer#3 almond antique misty red olive 1 4 4 0 0.75 2 4 12.75 7.013380069552769 2 1 14
+Manufacturer#3 almond antique olive coral navajo 45 5 5 1 1.0 3 5 19.2 14.344336861632886 2 45 19
+Manufacturer#4 almond antique gainsboro frosted violet 10 1 1 0 0.0 1 1 10.0 0.0 0 10 10
+Manufacturer#4 almond antique violet mint lemon 39 2 2 0 0.25 1 2 24.5 14.5 0 39 10
+Manufacturer#4 almond aquamarine floral ivory bisque 27 3 3 0 0.5 2 3 25.333333333333332 11.897712198383164 0 27 10
+Manufacturer#4 almond aquamarine yellow dodger mint 7 4 4 0 0.75 2 4 20.75 13.007209539328564 0 7 39
+Manufacturer#4 almond azure aquamarine papaya violet 12 5 5 1 1.0 3 5 19.0 12.149074038789951 0 12 27
+Manufacturer#5 almond antique blue firebrick mint 31 1 1 0 0.0 1 1 31.0 0.0 1 31 31
+Manufacturer#5 almond antique medium spring khaki 6 2 2 0 0.25 1 2 18.5 12.5 1 6 31
+Manufacturer#5 almond antique sky peru orange 2 3 3 0 0.5 2 3 13.0 12.832251036613439 1 2 31
+Manufacturer#5 almond aquamarine dodger light gainsboro 46 4 4 0 0.75 2 4 21.25 18.102140757380052 1 46 6
+Manufacturer#5 almond azure blanched chiffon midnight 23 5 5 1 1.0 3 5 21.6 16.206171663906314 1 23 2
+PREHOOK: query: select p_mfgr,p_name, p_size,
+ rank() over(distribute by p_mfgr sort by p_name) as r,
+ dense_rank() over(distribute by p_mfgr sort by p_name) as dr,
+cume_dist() over(distribute by p_mfgr sort by p_name) as cud,
+sum(p_size) over (distribute by p_mfgr sort by p_name range between unbounded preceding and current row) as s1,
+sum(p_size) over (distribute by p_mfgr sort by p_size range between 5 preceding and current row) as s2,
+first_value(p_size) over w1 as fv1
+from part
+window w1 as (distribute by p_mfgr sort by p_mfgr, p_name rows between 2 preceding and 2 following)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part
+#### A masked pattern was here ####
+POSTHOOK: query: select p_mfgr,p_name, p_size,
+ rank() over(distribute by p_mfgr sort by p_name) as r,
+ dense_rank() over(distribute by p_mfgr sort by p_name) as dr,
+cume_dist() over(distribute by p_mfgr sort by p_name) as cud,
+sum(p_size) over (distribute by p_mfgr sort by p_name range between unbounded preceding and current row) as s1,
+sum(p_size) over (distribute by p_mfgr sort by p_size range between 5 preceding and current row) as s2,
+first_value(p_size) over w1 as fv1
+from part
+window w1 as (distribute by p_mfgr sort by p_mfgr, p_name rows between 2 preceding and 2 following)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part
+#### A masked pattern was here ####
+Manufacturer#1 almond antique burnished rose metallic 2 1 1 0 4 4 2
+Manufacturer#1 almond antique burnished rose metallic 2 1 1 0 4 4 2
+Manufacturer#1 almond antique chartreuse lavender yellow 34 3 2 0 38 34 2
+Manufacturer#1 almond antique salmon chartreuse burlywood 6 4 3 0 44 10 2
+Manufacturer#1 almond aquamarine burnished black steel 28 5 4 0 72 28 34
+Manufacturer#1 almond aquamarine pink moccasin thistle 42 6 5 1 114 42 6
+Manufacturer#2 almond antique violet chocolate turquoise 14 1 1 0 14 14 14
+Manufacturer#2 almond antique violet turquoise frosted 40 2 2 0 54 40 14
+Manufacturer#2 almond aquamarine midnight light salmon 2 3 3 0 56 2 14
+Manufacturer#2 almond aquamarine rose maroon antique 25 4 4 0 81 25 40
+Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 5 5 1 99 32 2
+Manufacturer#3 almond antique chartreuse khaki white 17 1 1 0 17 31 17
+Manufacturer#3 almond antique forest lavender goldenrod 14 2 2 0 31 14 17
+Manufacturer#3 almond antique metallic orange dim 19 3 3 0 50 50 17
+Manufacturer#3 almond antique misty red olive 1 4 4 0 51 1 14
+Manufacturer#3 almond antique olive coral navajo 45 5 5 1 96 45 19
+Manufacturer#4 almond antique gainsboro frosted violet 10 1 1 0 10 17 10
+Manufacturer#4 almond antique violet mint lemon 39 2 2 0 49 39 10
+Manufacturer#4 almond aquamarine floral ivory bisque 27 3 3 0 76 27 10
+Manufacturer#4 almond aquamarine yellow dodger mint 7 4 4 0 83 7 39
+Manufacturer#4 almond azure aquamarine papaya violet 12 5 5 1 95 29 27
+Manufacturer#5 almond antique blue firebrick mint 31 1 1 0 31 31 31
+Manufacturer#5 almond antique medium spring khaki 6 2 2 0 37 8 31
+Manufacturer#5 almond antique sky peru orange 2 3 3 0 39 2 31
+Manufacturer#5 almond aquamarine dodger light gainsboro 46 4 4 0 85 46 6
+Manufacturer#5 almond azure blanched chiffon midnight 23 5 5 1 108 23 2
+PREHOOK: query: select p_mfgr,p_name, p_size,
+count(*) over(distribute by p_mfgr sort by p_name ) as c,
+count(p_size) over(distribute by p_mfgr sort by p_name) as ca,
+first_value(p_size) over w1 as fvW1
+from part
+window w1 as (distribute by p_mfgr sort by p_mfgr, p_name rows between 2 preceding and 2 following)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part
+#### A masked pattern was here ####
+POSTHOOK: query: select p_mfgr,p_name, p_size,
+count(*) over(distribute by p_mfgr sort by p_name ) as c,
+count(p_size) over(distribute by p_mfgr sort by p_name) as ca,
+first_value(p_size) over w1 as fvW1
+from part
+window w1 as (distribute by p_mfgr sort by p_mfgr, p_name rows between 2 preceding and 2 following)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part
+#### A masked pattern was here ####
+Manufacturer#1 almond antique burnished rose metallic 2 2 2 2
+Manufacturer#1 almond antique burnished rose metallic 2 2 2 2
+Manufacturer#1 almond antique chartreuse lavender yellow 34 3 3 2
+Manufacturer#1 almond antique salmon chartreuse burlywood 6 4 4 2
+Manufacturer#1 almond aquamarine burnished black steel 28 5 5 34
+Manufacturer#1 almond aquamarine pink moccasin thistle 42 6 6 6
+Manufacturer#2 almond antique violet chocolate turquoise 14 1 1 14
+Manufacturer#2 almond antique violet turquoise frosted 40 2 2 14
+Manufacturer#2 almond aquamarine midnight light salmon 2 3 3 14
+Manufacturer#2 almond aquamarine rose maroon antique 25 4 4 40
+Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 5 5 2
+Manufacturer#3 almond antique chartreuse khaki white 17 1 1 17
+Manufacturer#3 almond antique forest lavender goldenrod 14 2 2 17
+Manufacturer#3 almond antique metallic orange dim 19 3 3 17
+Manufacturer#3 almond antique misty red olive 1 4 4 14
+Manufacturer#3 almond antique olive coral navajo 45 5 5 19
+Manufacturer#4 almond antique gainsboro frosted violet 10 1 1 10
+Manufacturer#4 almond antique violet mint lemon 39 2 2 10
+Manufacturer#4 almond aquamarine floral ivory bisque 27 3 3 10
+Manufacturer#4 almond aquamarine yellow dodger mint 7 4 4 39
+Manufacturer#4 almond azure aquamarine papaya violet 12 5 5 27
+Manufacturer#5 almond antique blue firebrick mint 31 1 1 31
+Manufacturer#5 almond antique medium spring khaki 6 2 2 31
+Manufacturer#5 almond antique sky peru orange 2 3 3 31
+Manufacturer#5 almond aquamarine dodger light gainsboro 46 4 4 6
+Manufacturer#5 almond azure blanched chiffon midnight 23 5 5 2
+PREHOOK: query: select p_mfgr,p_name, p_size,
+sum(p_retailprice) over w1 as s,
+min(p_retailprice) over w1 as mi,
+max(p_retailprice) over w1 as ma,
+avg(p_retailprice) over w1 as ag
+from part
+window w1 as (distribute by p_mfgr sort by p_mfgr, p_name rows between 2 preceding and 2 following)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part
+#### A masked pattern was here ####
+POSTHOOK: query: select p_mfgr,p_name, p_size,
+sum(p_retailprice) over w1 as s,
+min(p_retailprice) over w1 as mi,
+max(p_retailprice) over w1 as ma,
+avg(p_retailprice) over w1 as ag
+from part
+window w1 as (distribute by p_mfgr sort by p_mfgr, p_name rows between 2 preceding and 2 following)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part
+#### A masked pattern was here ####
+Manufacturer#1 almond antique burnished rose metallic 2 4100.06 1173.15 1753.76 1366.6866666666667
+Manufacturer#1 almond antique burnished rose metallic 2 5702.650000000001 1173.15 1753.76 1425.6625000000001
+Manufacturer#1 almond antique chartreuse lavender yellow 34 7117.070000000001 1173.15 1753.76 1423.4140000000002
+Manufacturer#1 almond antique salmon chartreuse burlywood 6 7576.580000000002 1173.15 1753.76 1515.3160000000003
+Manufacturer#1 almond aquamarine burnished black steel 28 6403.430000000001 1414.42 1753.76 1600.8575000000003
+Manufacturer#1 almond aquamarine pink moccasin thistle 42 4649.670000000001 1414.42 1632.66 1549.8900000000003
+Manufacturer#2 almond antique violet chocolate turquoise 14 5523.360000000001 1690.68 2031.98 1841.1200000000001
+Manufacturer#2 almond antique violet turquoise frosted 40 7222.02 1690.68 2031.98 1805.505
+Manufacturer#2 almond aquamarine midnight light salmon 2 8923.62 1690.68 2031.98 1784.7240000000002
+Manufacturer#2 almond aquamarine rose maroon antique 25 7232.9400000000005 1698.66 2031.98 1808.2350000000001
+Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 5432.240000000001 1698.66 2031.98 1810.746666666667
+Manufacturer#3 almond antique chartreuse khaki white 17 4272.34 1190.27 1671.68 1424.1133333333335
+Manufacturer#3 almond antique forest lavender goldenrod 14 6195.32 1190.27 1922.98 1548.83
+Manufacturer#3 almond antique metallic orange dim 19 7532.61 1190.27 1922.98 1506.522
+Manufacturer#3 almond antique misty red olive 1 5860.929999999999 1190.27 1922.98 1465.2324999999998
+Manufacturer#3 almond antique olive coral navajo 45 4670.66 1337.29 1922.98 1556.8866666666665
+Manufacturer#4 almond antique gainsboro frosted violet 10 4202.35 1206.26 1620.67 1400.7833333333335
+Manufacturer#4 almond antique violet mint lemon 39 6047.27 1206.26 1844.92 1511.8175
+Manufacturer#4 almond aquamarine floral ivory bisque 27 7337.620000000001 1206.26 1844.92 1467.5240000000001
+Manufacturer#4 almond aquamarine yellow dodger mint 7 5716.950000000001 1206.26 1844.92 1429.2375000000002
+Manufacturer#4 almond azure aquamarine papaya violet 12 4341.530000000001 1206.26 1844.92 1447.176666666667
+Manufacturer#5 almond antique blue firebrick mint 31 5190.08 1611.66 1789.69 1730.0266666666666
+Manufacturer#5 almond antique medium spring khaki 6 6208.18 1018.1 1789.69 1552.045
+Manufacturer#5 almond antique sky peru orange 2 7672.66 1018.1 1789.69 1534.532
+Manufacturer#5 almond aquamarine dodger light gainsboro 46 5882.969999999999 1018.1 1788.73 1470.7424999999998
+Manufacturer#5 almond azure blanched chiffon midnight 23 4271.3099999999995 1018.1 1788.73 1423.7699999999998
+PREHOOK: query: select p_mfgr,p_name, p_size, p_retailprice,
+sum(p_retailprice) over w1 as s,
+min(p_retailprice) as mi ,
+max(p_retailprice) as ma ,
+avg(p_retailprice) over w1 as ag
+from part
+group by p_mfgr,p_name, p_size, p_retailprice
+window w1 as (distribute by p_mfgr sort by p_mfgr, p_name rows between 2 preceding and 2 following)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part
+#### A masked pattern was here ####
+POSTHOOK: query: select p_mfgr,p_name, p_size, p_retailprice,
+sum(p_retailprice) over w1 as s,
+min(p_retailprice) as mi ,
+max(p_retailprice) as ma ,
+avg(p_retailprice) over w1 as ag
+from part
+group by p_mfgr,p_name, p_size, p_retailprice
+window w1 as (distribute by p_mfgr sort by p_mfgr, p_name rows between 2 preceding and 2 following)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part
+#### A masked pattern was here ####
+Manufacturer#1 almond antique burnished rose metallic 2 1173.15 4529.5 1173.15 1173.15 1509.8333333333333
+Manufacturer#1 almond antique chartreuse lavender yellow 34 1753.76 5943.92 1753.76 1753.76 1485.98
+Manufacturer#1 almond antique salmon chartreuse burlywood 6 1602.59 7576.58 1602.59 1602.59 1515.316
+Manufacturer#1 almond aquamarine burnished black steel 28 1414.42 6403.43 1414.42 1414.42 1600.8575
+Manufacturer#1 almond aquamarine pink moccasin thistle 42 1632.66 4649.67 1632.66 1632.66 1549.89
+Manufacturer#2 almond antique violet chocolate turquoise 14 1690.68 5523.360000000001 1690.68 1690.68 1841.1200000000001
+Manufacturer#2 almond antique violet turquoise frosted 40 1800.7 7222.02 1800.7 1800.7 1805.505
+Manufacturer#2 almond aquamarine midnight light salmon 2 2031.98 8923.62 2031.98 2031.98 1784.7240000000002
+Manufacturer#2 almond aquamarine rose maroon antique 25 1698.66 7232.9400000000005 1698.66 1698.66 1808.2350000000001
+Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 1701.6 5432.240000000001 1701.6 1701.6 1810.746666666667
+Manufacturer#3 almond antique chartreuse khaki white 17 1671.68 4272.34 1671.68 1671.68 1424.1133333333335
+Manufacturer#3 almond antique forest lavender goldenrod 14 1190.27 6195.32 1190.27 1190.27 1548.83
+Manufacturer#3 almond antique metallic orange dim 19 1410.39 7532.61 1410.39 1410.39 1506.522
+Manufacturer#3 almond antique misty red olive 1 1922.98 5860.929999999999 1922.98 1922.98 1465.2324999999998
+Manufacturer#3 almond antique olive coral navajo 45 1337.29 4670.66 1337.29 1337.29 1556.8866666666665
+Manufacturer#4 almond antique gainsboro frosted violet 10 1620.67 4202.35 1620.67 1620.67 1400.7833333333335
+Manufacturer#4 almond antique violet mint lemon 39 1375.42 6047.27 1375.42 1375.42 1511.8175
+Manufacturer#4 almond aquamarine floral ivory bisque 27 1206.26 7337.620000000001 1206.26 1206.26 1467.5240000000001
+Manufacturer#4 almond aquamarine yellow dodger mint 7 1844.92 5716.950000000001 1844.92 1844.92 1429.2375000000002
+Manufacturer#4 almond azure aquamarine papaya violet 12 1290.35 4341.530000000001 1290.35 1290.35 1447.176666666667
+Manufacturer#5 almond antique blue firebrick mint 31 1789.69 5190.08 1789.69 1789.69 1730.0266666666666
+Manufacturer#5 almond antique medium spring khaki 6 1611.66 6208.18 1611.66 1611.66 1552.045
+Manufacturer#5 almond antique sky peru orange 2 1788.73 7672.66 1788.73 1788.73 1534.532
+Manufacturer#5 almond aquamarine dodger light gainsboro 46 1018.1 5882.969999999999 1018.1 1018.1 1470.7424999999998
+Manufacturer#5 almond azure blanched chiffon midnight 23 1464.48 4271.3099999999995 1464.48 1464.48 1423.7699999999998
+PREHOOK: query: select p_mfgr,p_name, p_size,
+stddev(p_retailprice) over w1 as sdev,
+stddev_pop(p_retailprice) over w1 as sdev_pop,
+collect_set(p_size) over w1 as uniq_size,
+variance(p_retailprice) over w1 as var,
+corr(p_size, p_retailprice) over w1 as cor,
+covar_pop(p_size, p_retailprice) over w1 as covarp
+from part
+window w1 as (distribute by p_mfgr sort by p_mfgr, p_name rows between 2 preceding and 2 following)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part
+#### A masked pattern was here ####
+POSTHOOK: query: select p_mfgr,p_name, p_size,
+stddev(p_retailprice) over w1 as sdev,
+stddev_pop(p_retailprice) over w1 as sdev_pop,
+collect_set(p_size) over w1 as uniq_size,
+variance(p_retailprice) over w1 as var,
+corr(p_size, p_retailprice) over w1 as cor,
+covar_pop(p_size, p_retailprice) over w1 as covarp
+from part
+window w1 as (distribute by p_mfgr sort by p_mfgr, p_name rows between 2 preceding and 2 following)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part
+#### A masked pattern was here ####
+Manufacturer#1 almond antique burnished rose metallic 2 258.10677784349235 258.10677784349235 [2,34,6] 66619.10876874991 0.811328754177887 2801.7074999999995
+Manufacturer#1 almond antique burnished rose metallic 2 273.70217881648074 273.70217881648074 [2,34] 74912.8826888888 1.0 4128.782222222221
+Manufacturer#1 almond antique chartreuse lavender yellow 34 230.90151585470358 230.90151585470358 [2,34,6,28] 53315.51002399992 0.695639377397664 2210.7864
+Manufacturer#1 almond antique salmon chartreuse burlywood 6 202.73109328368946 202.73109328368946 [2,34,6,28,42] 41099.896184 0.630785977101214 2009.9536000000007
+Manufacturer#1 almond aquamarine burnished black steel 28 121.6064517973862 121.6064517973862 [34,6,28,42] 14788.129118750014 0.2036684720435979 331.1337500000004
+Manufacturer#1 almond aquamarine pink moccasin thistle 42 96.5751586416853 96.5751586416853 [6,28,42] 9326.761266666683 -1.4442181184933883E-4 -0.20666666666708502
+Manufacturer#2 almond antique violet chocolate turquoise 14 142.2363169751898 142.2363169751898 [14,40,2] 20231.169866666663 -0.49369526554523185 -1113.7466666666658
+Manufacturer#2 almond antique violet turquoise frosted 40 137.76306498840682 137.76306498840682 [14,40,2,25] 18978.662075 -0.5205630897335946 -1004.4812499999995
+Manufacturer#2 almond aquamarine midnight light salmon 2 130.03972279269132 130.03972279269132 [14,40,2,25,18] 16910.329504000005 -0.46908967495720255 -766.1791999999995
+Manufacturer#2 almond aquamarine rose maroon antique 25 135.55100986344584 135.55100986344584 [40,2,25,18] 18374.07627499999 -0.6091405874714462 -1128.1787499999987
+Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 156.44019460768044 156.44019460768044 [2,25,18] 24473.534488888927 -0.9571686373491608 -1441.4466666666676
+Manufacturer#3 almond antique chartreuse khaki white 17 196.7742266885805 196.7742266885805 [17,14,19] 38720.09628888887 0.5557168646224995 224.6944444444446
+Manufacturer#3 almond antique forest lavender goldenrod 14 275.14144189852607 275.14144189852607 [17,14,19,1] 75702.81305 -0.6720833036576083 -1296.9000000000003
+Manufacturer#3 almond antique metallic orange dim 19 260.23473614412046 260.23473614412046 [17,14,19,1,45] 67722.117896 -0.5703526513979519 -2129.0664
+Manufacturer#3 almond antique misty red olive 1 275.9139962356932 275.9139962356932 [14,19,1,45] 76128.53331875012 -0.577476899644802 -2547.7868749999993
+Manufacturer#3 almond antique olive coral navajo 45 260.5815918713796 260.5815918713796 [19,1,45] 67902.76602222225 -0.8710736366736884 -4099.731111111111
+Manufacturer#4 almond antique gainsboro frosted violet 10 170.13011889596618 170.13011889596618 [10,39,27] 28944.25735555559 -0.6656975320098423 -1347.4777777777779
+Manufacturer#4 almond antique violet mint lemon 39 242.26834609323197 242.26834609323197 [10,39,27,7] 58693.95151875002 -0.8051852719193339 -2537.328125
+Manufacturer#4 almond aquamarine floral ivory bisque 27 234.10001662537326 234.10001662537326 [10,39,27,7,12] 54802.817784000035 -0.6046935574240581 -1719.8079999999995
+Manufacturer#4 almond aquamarine yellow dodger mint 7 247.3342714197732 247.3342714197732 [39,27,7,12] 61174.24181875003 -0.5508665654707869 -1719.0368749999975
+Manufacturer#4 almond azure aquamarine papaya violet 12 283.3344330566893 283.3344330566893 [27,7,12] 80278.40095555557 -0.7755740084632333 -1867.4888888888881
+Manufacturer#5 almond antique blue firebrick mint 31 83.69879024746363 83.69879024746363 [31,6,2] 7005.487488888913 0.39004303087285047 418.9233333333353
+Manufacturer#5 almond antique medium spring khaki 6 316.68049612345885 316.68049612345885 [31,6,2,46] 100286.53662500004 -0.713612911776183 -4090.853749999999
+Manufacturer#5 almond antique sky peru orange 2 285.40506298242155 285.40506298242155 [31,6,2,46,23] 81456.04997600002 -0.712858514567818 -3297.2011999999986
+Manufacturer#5 almond aquamarine dodger light gainsboro 46 285.43749038756283 285.43749038756283 [6,2,46,23] 81474.56091875004 -0.984128787153391 -4871.028125000002
+Manufacturer#5 almond azure blanched chiffon midnight 23 315.9225931564038 315.9225931564038 [2,46,23] 99807.08486666664 -0.9978877469246936 -5664.856666666666
+PREHOOK: query: select p_mfgr,p_name, p_size,
+histogram_numeric(p_retailprice, 5) over w1 as hist,
+percentile(p_partkey, 0.5) over w1 as per,
+row_number() over(distribute by p_mfgr sort by p_mfgr, p_name) as rn
+from part
+window w1 as (distribute by p_mfgr sort by p_mfgr, p_name rows between 2 preceding and 2 following)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part
+#### A masked pattern was here ####
+POSTHOOK: query: select p_mfgr,p_name, p_size,
+histogram_numeric(p_retailprice, 5) over w1 as hist,
+percentile(p_partkey, 0.5) over w1 as per,
+row_number() over(distribute by p_mfgr sort by p_mfgr, p_name) as rn
+from part
+window w1 as (distribute by p_mfgr sort by p_mfgr, p_name rows between 2 preceding and 2 following)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part
+#### A masked pattern was here ####
+Manufacturer#1 almond antique burnished rose metallic 2 [{"x":1173.15,"y":2.0},{"x":1602.59,"y":1.0},{"x":1753.76,"y":1.0}] 115872.0 2
+Manufacturer#1 almond antique burnished rose metallic 2 [{"x":1173.15,"y":2.0},{"x":1753.76,"y":1.0}] 121152.0 1
+Manufacturer#1 almond antique chartreuse lavender yellow 34 [{"x":1173.15,"y":2.0},{"x":1414.42,"y":1.0},{"x":1602.59,"y":1.0},{"x":1753.76,"y":1.0}] 110592.0 3
+Manufacturer#1 almond antique salmon chartreuse burlywood 6 [{"x":1173.15,"y":1.0},{"x":1414.42,"y":1.0},{"x":1602.59,"y":1.0},{"x":1632.66,"y":1.0},{"x":1753.76,"y":1.0}] 86428.0 4
+Manufacturer#1 almond aquamarine burnished black steel 28 [{"x":1414.42,"y":1.0},{"x":1602.59,"y":1.0},{"x":1632.66,"y":1.0},{"x":1753.76,"y":1.0}] 86098.0 5
+Manufacturer#1 almond aquamarine pink moccasin thistle 42 [{"x":1414.42,"y":1.0},{"x":1602.59,"y":1.0},{"x":1632.66,"y":1.0}] 86428.0 6
+Manufacturer#2 almond antique violet chocolate turquoise 14 [{"x":1690.68,"y":1.0},{"x":1800.7,"y":1.0},{"x":2031.98,"y":1.0}] 146985.0 1
+Manufacturer#2 almond antique violet turquoise frosted 40 [{"x":1690.68,"y":1.0},{"x":1698.66,"y":1.0},{"x":1800.7,"y":1.0},{"x":2031.98,"y":1.0}] 139825.5 2
+Manufacturer#2 almond aquamarine midnight light salmon 2 [{"x":1690.68,"y":1.0},{"x":1698.66,"y":1.0},{"x":1701.6,"y":1.0},{"x":1800.7,"y":1.0},{"x":2031.98,"y":1.0}] 146985.0 3
+Manufacturer#2 almond aquamarine rose maroon antique 25 [{"x":1698.66,"y":1.0},{"x":1701.6,"y":1.0},{"x":1800.7,"y":1.0},{"x":2031.98,"y":1.0}] 169347.0 4
+Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 [{"x":1698.66,"y":1.0},{"x":1701.6,"y":1.0},{"x":2031.98,"y":1.0}] 146985.0 5
+Manufacturer#3 almond antique chartreuse khaki white 17 [{"x":1190.27,"y":1.0},{"x":1410.39,"y":1.0},{"x":1671.68,"y":1.0}] 90681.0 1
+Manufacturer#3 almond antique forest lavender goldenrod 14 [{"x":1190.27,"y":1.0},{"x":1410.39,"y":1.0},{"x":1671.68,"y":1.0},{"x":1922.98,"y":1.0}] 65831.5 2
+Manufacturer#3 almond antique metallic orange dim 19 [{"x":1190.27,"y":1.0},{"x":1337.29,"y":1.0},{"x":1410.39,"y":1.0},{"x":1671.68,"y":1.0},{"x":1922.98,"y":1.0}] 90681.0 3
+Manufacturer#3 almond antique misty red olive 1 [{"x":1190.27,"y":1.0},{"x":1337.29,"y":1.0},{"x":1410.39,"y":1.0},{"x":1922.98,"y":1.0}] 76690.0 4
+Manufacturer#3 almond antique olive coral navajo 45 [{"x":1337.29,"y":1.0},{"x":1410.39,"y":1.0},{"x":1922.98,"y":1.0}] 112398.0 5
+Manufacturer#4 almond antique gainsboro frosted violet 10 [{"x":1206.26,"y":1.0},{"x":1375.42,"y":1.0},{"x":1620.67,"y":1.0}] 48427.0 1
+Manufacturer#4 almond antique violet mint lemon 39 [{"x":1206.26,"y":1.0},{"x":1375.42,"y":1.0},{"x":1620.67,"y":1.0},{"x":1844.92,"y":1.0}] 46844.0 2
+Manufacturer#4 almond aquamarine floral ivory bisque 27 [{"x":1206.26,"y":1.0},{"x":1290.35,"y":1.0},{"x":1375.42,"y":1.0},{"x":1620.67,"y":1.0},{"x":1844.92,"y":1.0}] 45261.0 3
+Manufacturer#4 almond aquamarine yellow dodger mint 7 [{"x":1206.26,"y":1.0},{"x":1290.35,"y":1.0},{"x":1375.42,"y":1.0},{"x":1844.92,"y":1.0}] 39309.0 4
+Manufacturer#4 almond azure aquamarine papaya violet 12 [{"x":1206.26,"y":1.0},{"x":1290.35,"y":1.0},{"x":1844.92,"y":1.0}] 33357.0 5
+Manufacturer#5 almond antique blue firebrick mint 31 [{"x":1611.66,"y":1.0},{"x":1788.73,"y":1.0},{"x":1789.69,"y":1.0}] 155733.0 1
+Manufacturer#5 almond antique medium spring khaki 6 [{"x":1018.1,"y":1.0},{"x":1611.66,"y":1.0},{"x":1788.73,"y":1.0},{"x":1789.69,"y":1.0}] 99201.0 2
+Manufacturer#5 almond antique sky peru orange 2 [{"x":1018.1,"y":1.0},{"x":1464.48,"y":1.0},{"x":1611.66,"y":1.0},{"x":1788.73,"y":1.0},{"x":1789.69,"y":1.0}] 78486.0 3
+Manufacturer#5 almond aquamarine dodger light gainsboro 46 [{"x":1018.1,"y":1.0},{"x":1464.48,"y":1.0},{"x":1611.66,"y":1.0},{"x":1788.73,"y":1.0}] 60577.5 4
+Manufacturer#5 almond azure blanched chiffon midnight 23 [{"x":1018.1,"y":1.0},{"x":1464.48,"y":1.0},{"x":1788.73,"y":1.0}] 78486.0 5
+PREHOOK: query: create view IF NOT EXISTS mfgr_price_view as
+select p_mfgr, p_brand,
+round(sum(p_retailprice),2) as s
+from part
+group by p_mfgr, p_brand
+PREHOOK: type: CREATEVIEW
+PREHOOK: Input: default@part
+PREHOOK: Output: database:default
+PREHOOK: Output: default@mfgr_price_view
+POSTHOOK: query: create view IF NOT EXISTS mfgr_price_view as
+select p_mfgr, p_brand,
+round(sum(p_retailprice),2) as s
+from part
+group by p_mfgr, p_brand
+POSTHOOK: type: CREATEVIEW
+POSTHOOK: Input: default@part
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@mfgr_price_view
+POSTHOOK: Lineage: mfgr_price_view.p_brand SIMPLE [(part)part.FieldSchema(name:p_brand, type:string, comment:null), ]
+POSTHOOK: Lineage: mfgr_price_view.p_mfgr SIMPLE [(part)part.FieldSchema(name:p_mfgr, type:string, comment:null), ]
+POSTHOOK: Lineage: mfgr_price_view.s EXPRESSION [(part)part.FieldSchema(name:p_retailprice, type:double, comment:null), ]
+PREHOOK: query: select *
+from (
+select p_mfgr, p_brand, s,
+round(sum(s) over w1 , 2) as s1
+from mfgr_price_view
+window w1 as (distribute by p_mfgr sort by p_mfgr )
+) sq
+order by p_mfgr, p_brand
+PREHOOK: type: QUERY
+PREHOOK: Input: default@mfgr_price_view
+PREHOOK: Input: default@part
+#### A masked pattern was here ####
+POSTHOOK: query: select *
+from (
+select p_mfgr, p_brand, s,
+round(sum(s) over w1 , 2) as s1
+from mfgr_price_view
+window w1 as (distribute by p_mfgr sort by p_mfgr )
+) sq
+order by p_mfgr, p_brand
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@mfgr_price_view
+POSTHOOK: Input: default@part
+#### A masked pattern was here ####
+Manufacturer#1 Brand#12 4800.84 8749.73
+Manufacturer#1 Brand#14 2346.3 8749.73
+Manufacturer#1 Brand#15 1602.59 8749.73
+Manufacturer#2 Brand#22 3491.38 8923.62
+Manufacturer#2 Brand#23 2031.98 8923.62
+Manufacturer#2 Brand#24 1698.66 8923.62
+Manufacturer#2 Brand#25 1701.6 8923.62
+Manufacturer#3 Brand#31 1671.68 7532.61
+Manufacturer#3 Brand#32 3333.37 7532.61
+Manufacturer#3 Brand#34 1337.29 7532.61
+Manufacturer#3 Brand#35 1190.27 7532.61
+Manufacturer#4 Brand#41 4755.94 7337.62
+Manufacturer#4 Brand#42 2581.68 7337.62
+Manufacturer#5 Brand#51 1611.66 7672.66
+Manufacturer#5 Brand#52 3254.17 7672.66
+Manufacturer#5 Brand#53 2806.83 7672.66
+PREHOOK: query: select p_mfgr, p_brand, s,
+round(sum(s) over w1 ,2) as s1
+from mfgr_price_view
+window w1 as (distribute by p_mfgr sort by p_brand rows between 2 preceding and current row)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@mfgr_price_view
+PREHOOK: Input: default@part
+#### A masked pattern was here ####
+POSTHOOK: query: select p_mfgr, p_brand, s,
+round(sum(s) over w1 ,2) as s1
+from mfgr_price_view
+window w1 as (distribute by p_mfgr sort by p_brand rows between 2 preceding and current row)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@mfgr_price_view
+POSTHOOK: Input: default@part
+#### A masked pattern was here ####
+Manufacturer#1 Brand#12 4800.84 4800.84
+Manufacturer#1 Brand#14 2346.3 7147.14
+Manufacturer#1 Brand#15 1602.59 8749.73
+Manufacturer#2 Brand#22 3491.38 3491.38
+Manufacturer#2 Brand#23 2031.98 5523.36
+Manufacturer#2 Brand#24 1698.66 7222.02
+Manufacturer#2 Brand#25 1701.6 5432.24
+Manufacturer#3 Brand#31 1671.68 1671.68
+Manufacturer#3 Brand#32 3333.37 5005.05
+Manufacturer#3 Brand#34 1337.29 6342.34
+Manufacturer#3 Brand#35 1190.27 5860.93
+Manufacturer#4 Brand#41 4755.94 4755.94
+Manufacturer#4 Brand#42 2581.68 7337.62
+Manufacturer#5 Brand#51 1611.66 1611.66
+Manufacturer#5 Brand#52 3254.17 4865.83
+Manufacturer#5 Brand#53 2806.83 7672.66
+PREHOOK: query: create view IF NOT EXISTS mfgr_brand_price_view as
+select p_mfgr, p_brand,
+sum(p_retailprice) over w1 as s
+from part
+window w1 as (distribute by p_mfgr sort by p_name rows between 2 preceding and current row)
+PREHOOK: type: CREATEVIEW
+PREHOOK: Input: default@part
+PREHOOK: Output: database:default
+PREHOOK: Output: default@mfgr_brand_price_view
+POSTHOOK: query: create view IF NOT EXISTS mfgr_brand_price_view as
+select p_mfgr, p_brand,
+sum(p_retailprice) over w1 as s
+from part
+window w1 as (distribute by p_mfgr sort by p_name rows between 2 preceding and current row)
+POSTHOOK: type: CREATEVIEW
+POSTHOOK: Input: default@part
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@mfgr_brand_price_view
+POSTHOOK: Lineage: mfgr_brand_price_view.p_brand SIMPLE [(part)part.FieldSchema(name:p_brand, type:string, comment:null), ]
+POSTHOOK: Lineage: mfgr_brand_price_view.p_mfgr SIMPLE [(part)part.FieldSchema(name:p_mfgr, type:string, comment:null), ]
+POSTHOOK: Lineage: mfgr_brand_price_view.s SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ]
+PREHOOK: query: select * from mfgr_brand_price_view
+PREHOOK: type: QUERY
+PREHOOK: Input: default@mfgr_brand_price_view
+PREHOOK: Input: default@part
+#### A masked pattern was here ####
+POSTHOOK: query: select * from mfgr_brand_price_view
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@mfgr_brand_price_view
+POSTHOOK: Input: default@part
+#### A masked pattern was here ####
+Manufacturer#1 Brand#12 4100.06
+Manufacturer#1 Brand#12 4649.670000000001
+Manufacturer#1 Brand#12 4770.77
+Manufacturer#1 Brand#14 1173.15
+Manufacturer#1 Brand#14 2346.3
+Manufacturer#1 Brand#15 4529.5
+Manufacturer#2 Brand#22 1690.68
+Manufacturer#2 Brand#22 3491.38
+Manufacturer#2 Brand#23 5523.360000000001
+Manufacturer#2 Brand#24 5531.34
+Manufacturer#2 Brand#25 5432.240000000001
+Manufacturer#3 Brand#31 1671.68
+Manufacturer#3 Brand#32 4272.34
+Manufacturer#3 Brand#32 4523.639999999999
+Manufacturer#3 Brand#34 4670.66
+Manufacturer#3 Brand#35 2861.95
+Manufacturer#4 Brand#41 1620.67
+Manufacturer#4 Brand#41 4341.530000000001
+Manufacturer#4 Brand#41 4426.6
+Manufacturer#4 Brand#42 2996.09
+Manufacturer#4 Brand#42 4202.35
+Manufacturer#5 Brand#51 3401.3500000000004
+Manufacturer#5 Brand#52 1789.69
+Manufacturer#5 Brand#52 4271.3099999999995
+Manufacturer#5 Brand#53 4418.49
+Manufacturer#5 Brand#53 5190.08
+PREHOOK: query: select p_mfgr, p_name,
+lv_col, p_size, sum(p_size) over w1 as s
+from (select p_mfgr, p_name, p_size, array(1,2,3) arr from part) p
+lateral view explode(arr) part_lv as lv_col
+window w1 as (distribute by p_mfgr sort by p_size, lv_col rows between 2 preceding and current row)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part
+#### A masked pattern was here ####
+POSTHOOK: query: select p_mfgr, p_name,
+lv_col, p_size, sum(p_size) over w1 as s
+from (select p_mfgr, p_name, p_size, array(1,2,3) arr from part) p
+lateral view explode(arr) part_lv as lv_col
+window w1 as (distribute by p_mfgr sort by p_size, lv_col rows between 2 preceding and current row)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part
+#### A masked pattern was here ####
+Manufacturer#1 almond antique burnished rose metallic 1 2 2
+Manufacturer#1 almond antique burnished rose metallic 1 2 4
+Manufacturer#1 almond antique burnished rose metallic 2 2 6
+Manufacturer#1 almond antique burnished rose metallic 2 2 6
+Manufacturer#1 almond antique burnished rose metallic 3 2 6
+Manufacturer#1 almond antique burnished rose metallic 3 2 6
+Manufacturer#1 almond antique chartreuse lavender yellow 1 34 90
+Manufacturer#1 almond antique chartreuse lavender yellow 2 34 96
+Manufacturer#1 almond antique chartreuse lavender yellow 3 34 102
+Manufacturer#1 almond antique salmon chartreuse burlywood 1 6 10
+Manufacturer#1 almond antique salmon chartreuse burlywood 2 6 14
+Manufacturer#1 almond antique salmon chartreuse burlywood 3 6 18
+Manufacturer#1 almond aquamarine burnished black steel 1 28 40
+Manufacturer#1 almond aquamarine burnished black steel 2 28 62
+Manufacturer#1 almond aquamarine burnished black steel 3 28 84
+Manufacturer#1 almond aquamarine pink moccasin thistle 1 42 110
+Manufacturer#1 almond aquamarine pink moccasin thistle 2 42 118
+Manufacturer#1 almond aquamarine pink moccasin thistle 3 42 126
+Manufacturer#2 almond antique violet chocolate turquoise 1 14 18
+Manufacturer#2 almond antique violet chocolate turquoise 2 14 30
+Manufacturer#2 almond antique violet chocolate turquoise 3 14 42
+Manufacturer#2 almond antique violet turquoise frosted 1 40 90
+Manufacturer#2 almond antique violet turquoise frosted 2 40 105
+Manufacturer#2 almond antique violet turquoise frosted 3 40 120
+Manufacturer#2 almond aquamarine midnight light salmon 1 2 2
+Manufacturer#2 almond aquamarine midnight light salmon 2 2 4
+Manufacturer#2 almond aquamarine midnight light salmon 3 2 6
+Manufacturer#2 almond aquamarine rose maroon antique 1 25 61
+Manufacturer#2 almond aquamarine rose maroon antique 2 25 68
+Manufacturer#2 almond aquamarine rose maroon antique 3 25 75
+Manufacturer#2 almond aquamarine sandy cyan gainsboro 1 18 46
+Manufacturer#2 almond aquamarine sandy cyan gainsboro 2 18 50
+Manufacturer#2 almond aquamarine sandy cyan gainsboro 3 18 54
+Manufacturer#3 almond antique chartreuse khaki white 1 17 45
+Manufacturer#3 almond antique chartreuse khaki white 2 17 48
+Manufacturer#3 almond antique chartreuse khaki white 3 17 51
+Manufacturer#3 almond antique forest lavender goldenrod 1 14 16
+Manufacturer#3 almond antique forest lavender goldenrod 2 14 29
+Manufacturer#3 almond antique forest lavender goldenrod 3 14 42
+Manufacturer#3 almond antique metallic orange dim 1 19 53
+Manufacturer#3 almond antique metallic orange dim 2 19 55
+Manufacturer#3 almond antique metallic orange dim 3 19 57
+Manufacturer#3 almond antique misty red olive 1 1 1
+Manufacturer#3 almond antique misty red olive 2 1 2
+Manufacturer#3 almond antique misty red olive 3 1 3
+Manufacturer#3 almond antique olive coral navajo 1 45 83
+Manufacturer#3 almond antique olive coral navajo 2 45 109
+Manufacturer#3 almond antique olive coral navajo 3 45 135
+Manufacturer#4 almond antique gainsboro frosted violet 1 10 24
+Manufacturer#4 almond antique gainsboro frosted violet 2 10 27
+Manufacturer#4 almond antique gainsboro frosted violet 3 10 30
+Manufacturer#4 almond antique violet mint lemon 1 39 93
+Manufacturer#4 almond antique violet mint lemon 2 39 105
+Manufacturer#4 almond antique violet mint lemon 3 39 117
+Manufacturer#4 almond aquamarine floral ivory bisque 1 27 51
+Manufacturer#4 almond aquamarine floral ivory bisque 2 27 66
+Manufacturer#4 almond aquamarine floral ivory bisque 3 27 81
+Manufacturer#4 almond aquamarine yellow dodger mint 1 7 7
+Manufacturer#4 almond aquamarine yellow dodger mint 2 7 14
+Manufacturer#4 almond aquamarine yellow dodger mint 3 7 21
+Manufacturer#4 almond azure aquamarine papaya violet 1 12 32
+Manufacturer#4 almond azure aquamarine papaya violet 2 12 34
+Manufacturer#4 almond azure aquamarine papaya violet 3 12 36
+Manufacturer#5 almond antique blue firebrick mint 1 31 77
+Manufacturer#5 almond antique blue firebrick mint 2 31 85
+Manufacturer#5 almond antique blue firebrick mint 3 31 93
+Manufacturer#5 almond antique medium spring khaki 1 6 10
+Manufacturer#5 almond antique medium spring khaki 2 6 14
+Manufacturer#5 almond antique medium spring khaki 3 6 18
+Manufacturer#5 almond antique sky peru orange 1 2 2
+Manufacturer#5 almond antique sky peru orange 2 2 4
+Manufacturer#5 almond antique sky peru orange 3 2 6
+Manufacturer#5 almond aquamarine dodger light gainsboro 1 46 108
+Manufacturer#5 almond aquamarine dodger light gainsboro 2 46 123
+Manufacturer#5 almond aquamarine dodger light gainsboro 3 46 138
+Manufacturer#5 almond azure blanched chiffon midnight 1 23 35
+Manufacturer#5 almond azure blanched chiffon midnight 2 23 52
+Manufacturer#5 almond azure blanched chiffon midnight 3 23 69
+PREHOOK: query: CREATE TABLE part_1(
+p_mfgr STRING,
+p_name STRING,
+p_size INT,
+r INT,
+dr INT,
+s DOUBLE)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@part_1
+POSTHOOK: query: CREATE TABLE part_1(
+p_mfgr STRING,
+p_name STRING,
+p_size INT,
+r INT,
+dr INT,
+s DOUBLE)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@part_1
+PREHOOK: query: CREATE TABLE part_2(
+p_mfgr STRING,
+p_name STRING,
+p_size INT,
+r INT,
+dr INT,
+cud INT,
+s2 DOUBLE,
+fv1 INT)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@part_2
+POSTHOOK: query: CREATE TABLE part_2(
+p_mfgr STRING,
+p_name STRING,
+p_size INT,
+r INT,
+dr INT,
+cud INT,
+s2 DOUBLE,
+fv1 INT)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@part_2
+PREHOOK: query: CREATE TABLE part_3(
+p_mfgr STRING,
+p_name STRING,
+p_size INT,
+c INT,
+ca INT,
+fv INT)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@part_3
+POSTHOOK: query: CREATE TABLE part_3(
+p_mfgr STRING,
+p_name STRING,
+p_size INT,
+c INT,
+ca INT,
+fv INT)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@part_3
+PREHOOK: query: from part
+INSERT OVERWRITE TABLE part_1
+select p_mfgr, p_name, p_size,
+rank() over(distribute by p_mfgr sort by p_name ) as r,
+dense_rank() over(distribute by p_mfgr sort by p_name ) as dr,
+sum(p_retailprice) over (distribute by p_mfgr sort by p_name rows between unbounded preceding and current row) as s
+INSERT OVERWRITE TABLE part_2
+select p_mfgr,p_name, p_size,
+rank() over(distribute by p_mfgr sort by p_name) as r,
+dense_rank() over(distribute by p_mfgr sort by p_name) as dr,
+cume_dist() over(distribute by p_mfgr sort by p_name) as cud,
+round(sum(p_size) over (distribute by p_mfgr sort by p_size range between 5 preceding and current row),1) as s2,
+first_value(p_size) over w1 as fv1
+window w1 as (distribute by p_mfgr sort by p_mfgr, p_name rows between 2 preceding and 2 following)
+INSERT OVERWRITE TABLE part_3
+select p_mfgr,p_name, p_size,
+count(*) over(distribute by p_mfgr sort by p_name) as c,
+count(p_size) over(distribute by p_mfgr sort by p_name) as ca,
+first_value(p_size) over w1 as fv
+window w1 as (distribute by p_mfgr sort by p_mfgr, p_name rows between 2 preceding and 2 following)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part
+PREHOOK: Output: default@part_1
+PREHOOK: Output: default@part_2
+PREHOOK: Output: default@part_3
+POSTHOOK: query: from part
+INSERT OVERWRITE TABLE part_1
+select p_mfgr, p_name, p_size,
+rank() over(distribute by p_mfgr sort by p_name ) as r,
+dense_rank() over(distribute by p_mfgr sort by p_name ) as dr,
+sum(p_retailprice) over (distribute by p_mfgr sort by p_name rows between unbounded preceding and current row) as s
+INSERT OVERWRITE TABLE part_2
+select p_mfgr,p_name, p_size,
+rank() over(distribute by p_mfgr sort by p_name) as r,
+dense_rank() over(distribute by p_mfgr sort by p_name) as dr,
+cume_dist() over(distribute by p_mfgr sort by p_name) as cud,
+round(sum(p_size) over (distribute by p_mfgr sort by p_size range between 5 preceding and current row),1) as s2,
+first_value(p_size) over w1 as fv1
+window w1 as (distribute by p_mfgr sort by p_mfgr, p_name rows between 2 preceding and 2 following)
+INSERT OVERWRITE TABLE part_3
+select p_mfgr,p_name, p_size,
+count(*) over(distribute by p_mfgr sort by p_name) as c,
+count(p_size) over(distribute by p_mfgr sort by p_name) as ca,
+first_value(p_size) over w1 as fv
+window w1 as (distribute by p_mfgr sort by p_mfgr, p_name rows between 2 preceding and 2 following)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part
+POSTHOOK: Output: default@part_1
+POSTHOOK: Output: default@part_2
+POSTHOOK: Output: default@part_3
+POSTHOOK: Lineage: part_1.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ]
+POSTHOOK: Lineage: part_1.p_mfgr SIMPLE [(part)part.FieldSchema(name:p_mfgr, type:string, comment:null), ]
+POSTHOOK: Lineage: part_1.p_name SIMPLE [(part)part.FieldSchema(name:p_name, type:string, comment:null), ]
+POSTHOOK: Lineage: part_1.p_size SIMPLE [(part)part.FieldSchema(name:p_size, type:int, comment:null), ]
+POSTHOOK: Lineage: part_1.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ]
+POSTHOOK: Lineage: part_1.s SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ]
+POSTHOOK: Lineage: part_2.cud SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ]
+POSTHOOK: Lineage: part_2.dr SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ]
+POSTHOOK: Lineage: part_2.fv1 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ]
+POSTHOOK: Lineage: part_2.p_mfgr SIMPLE [(part)part.FieldSchema(name:p_mfgr, type:string, comment:null), ]
+POSTHOOK: Lineage: part_2.p_name SIMPLE [(part)part.FieldSchema(name:p_name, type:string, comment:null), ]
+POSTHOOK: Lineage: part_2.p_size SIMPLE [(part)part.FieldSchema(name:p_size, type:int, comment:null), ]
+POSTHOOK: Lineage: part_2.r SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ]
+POSTHOOK: Lineage: part_2.s2 SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ]
+POSTHOOK: Lineage: part_3.c SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ]
+POSTHOOK: Lineage: part_3.ca SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ]
+POSTHOOK: Lineage: part_3.fv SCRIPT [(part)part.FieldSchema(name:p_partkey, type:int, comment:null), (part)part.FieldSchema(name:p_name, type:string, comment:null), (part)part.FieldSchema(name:p_mfgr, type:string, comment:null), (part)part.FieldSchema(name:p_brand, type:string, comment:null), (part)part.FieldSchema(name:p_type, type:string, comment:null), (part)part.FieldSchema(name:p_size, type:int, comment:null), (part)part.FieldSchema(name:p_container, type:string, comment:null), (part)part.FieldSchema(name:p_retailprice, type:double, comment:null), (part)part.FieldSchema(name:p_comment, type:string, comment:null), ]
+POSTHOOK: Lineage: part_3.p_mfgr SIMPLE [(part)part.FieldSchema(name:p_mfgr, type:string, comment:null), ]
+POSTHOOK: Lineage: part_3.p_name SIMPLE [(part)part.FieldSchema(name:p_name, type:string, comment:null), ]
+POSTHOOK: Lineage: part_3.p_size SIMPLE [(part)part.FieldSchema(name:p_size, type:int, comment:null), ]
+PREHOOK: query: select * from part_1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part_1
+#### A masked pattern was here ####
+POSTHOOK: query: select * from part_1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part_1
+#### A masked pattern was here ####
+Manufacturer#1 almond antique burnished rose metallic 2 1 1 1173.15
+Manufacturer#1 almond antique burnished rose metallic 2 1 1 2346.3
+Manufacturer#1 almond antique chartreuse lavender yellow 34 3 2 4100.06
+Manufacturer#1 almond antique salmon chartreuse burlywood 6 4 3 5702.650000000001
+Manufacturer#1 almond aquamarine burnished black steel 28 5 4 7117.070000000001
+Manufacturer#1 almond aquamarine pink moccasin thistle 42 6 5 8749.730000000001
+Manufacturer#2 almond antique violet chocolate turquoise 14 1 1 1690.68
+Manufacturer#2 almond antique violet turquoise frosted 40 2 2 3491.38
+Manufacturer#2 almond aquamarine midnight light salmon 2 3 3 5523.360000000001
+Manufacturer#2 almond aquamarine rose maroon antique 25 4 4 7222.02
+Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 5 5 8923.62
+Manufacturer#3 almond antique chartreuse khaki white 17 1 1 1671.68
+Manufacturer#3 almond antique forest lavender goldenrod 14 2 2 2861.95
+Manufacturer#3 almond antique metallic orange dim 19 3 3 4272.34
+Manufacturer#3 almond antique misty red olive 1 4 4 6195.32
+Manufacturer#3 almond antique olive coral navajo 45 5 5 7532.61
+Manufacturer#4 almond antique gainsboro frosted violet 10 1 1 1620.67
+Manufacturer#4 almond antique violet mint lemon 39 2 2 2996.09
+Manufacturer#4 almond aquamarine floral ivory bisque 27 3 3 4202.35
+Manufacturer#4 almond aquamarine yellow dodger mint 7 4 4 6047.27
+Manufacturer#4 almond azure aquamarine papaya violet 12 5 5 7337.620000000001
+Manufacturer#5 almond antique blue firebrick mint 31 1 1 1789.69
+Manufacturer#5 almond antique medium spring khaki 6 2 2 3401.3500000000004
+Manufacturer#5 almond antique sky peru orange 2 3 3 5190.08
+Manufacturer#5 almond aquamarine dodger light gainsboro 46 4 4 6208.18
+Manufacturer#5 almond azure blanched chiffon midnight 23 5 5 7672.66
+PREHOOK: query: select * from part_2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part_2
+#### A masked pattern was here ####
+POSTHOOK: query: select * from part_2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part_2
+#### A masked pattern was here ####
+Manufacturer#1 almond antique burnished rose metallic 2 1 1 0 4.0 2
+Manufacturer#1 almond antique burnished rose metallic 2 1 1 0 4.0 2
+Manufacturer#1 almond antique chartreuse lavender yellow 34 3 2 0 34.0 2
+Manufacturer#1 almond antique salmon chartreuse burlywood 6 4 3 0 10.0 2
+Manufacturer#1 almond aquamarine burnished black steel 28 5 4 0 28.0 34
+Manufacturer#1 almond aquamarine pink moccasin thistle 42 6 5 1 42.0 6
+Manufacturer#2 almond antique violet chocolate turquoise 14 1 1 0 14.0 14
+Manufacturer#2 almond antique violet turquoise frosted 40 2 2 0 40.0 14
+Manufacturer#2 almond aquamarine midnight light salmon 2 3 3 0 2.0 14
+Manufacturer#2 almond aquamarine rose maroon antique 25 4 4 0 25.0 40
+Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 5 5 1 32.0 2
+Manufacturer#3 almond antique chartreuse khaki white 17 1 1 0 31.0 17
+Manufacturer#3 almond antique forest lavender goldenrod 14 2 2 0 14.0 17
+Manufacturer#3 almond antique metallic orange dim 19 3 3 0 50.0 17
+Manufacturer#3 almond antique misty red olive 1 4 4 0 1.0 14
+Manufacturer#3
<TRUNCATED>
[25/50] [abbrv] hive git commit: HIVE-15951 : Make sure base persist
directory is unique and deleted (Slim Bouguerra via Ashutosh Chauhan)
Posted by om...@apache.org.
HIVE-15951 : Make sure base persist directory is unique and deleted (Slim Bouguerra via Ashutosh Chauhan)
Change-Id: I7bfda5a1d5e2ca8324ab67b73145c50416ffe808
Signed-off-by: Ashutosh Chauhan <ha...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/5be5391b
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/5be5391b
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/5be5391b
Branch: refs/heads/branch-2.2
Commit: 5be5391b8a3103c54e0cb6bb931ba47b37537ef8
Parents: 9f5bea3
Author: Slim Bouguerra <sl...@gmail.com>
Authored: Thu Feb 16 14:27:00 2017 -0800
Committer: Owen O'Malley <om...@apache.org>
Committed: Tue Mar 28 15:27:52 2017 -0700
----------------------------------------------------------------------
.../hadoop/hive/druid/io/DruidRecordWriter.java | 25 ++++++++++++++------
1 file changed, 18 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/5be5391b/druid-handler/src/java/org/apache/hadoop/hive/druid/io/DruidRecordWriter.java
----------------------------------------------------------------------
diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/io/DruidRecordWriter.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/io/DruidRecordWriter.java
index 1601a9a..5f107da 100644
--- a/druid-handler/src/java/org/apache/hadoop/hive/druid/io/DruidRecordWriter.java
+++ b/druid-handler/src/java/org/apache/hadoop/hive/druid/io/DruidRecordWriter.java
@@ -25,6 +25,7 @@ import io.druid.segment.realtime.plumber.Committers;
import io.druid.timeline.DataSegment;
import io.druid.timeline.partition.LinearShardSpec;
import org.apache.calcite.adapter.druid.DruidTable;
+import org.apache.commons.io.FileUtils;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.Constants;
@@ -40,9 +41,11 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.annotation.Nullable;
+import java.io.File;
import java.io.IOException;
import java.util.HashSet;
import java.util.List;
+import java.util.UUID;
import java.util.concurrent.ExecutionException;
public class DruidRecordWriter implements RecordWriter<NullWritable, DruidWritable>,
@@ -73,16 +76,19 @@ public class DruidRecordWriter implements RecordWriter<NullWritable, DruidWritab
final Path segmentsDescriptorsDir,
final FileSystem fileSystem
) {
+ File basePersistDir = new File(realtimeTuningConfig.getBasePersistDirectory(),
+ UUID.randomUUID().toString()
+ );
this.tuningConfig = Preconditions
- .checkNotNull(realtimeTuningConfig, "realtimeTuningConfig is null");
+ .checkNotNull(realtimeTuningConfig.withBasePersistDirectory(basePersistDir),
+ "realtimeTuningConfig is null"
+ );
this.dataSchema = Preconditions.checkNotNull(dataSchema, "data schema is null");
+
appenderator = Appenderators
- .createOffline(this.dataSchema,
- tuningConfig,
- new FireDepartmentMetrics(), dataSegmentPusher,
- DruidStorageHandlerUtils.JSON_MAPPER,
- DruidStorageHandlerUtils.INDEX_IO,
- DruidStorageHandlerUtils.INDEX_MERGER_V9
+ .createOffline(this.dataSchema, tuningConfig, new FireDepartmentMetrics(),
+ dataSegmentPusher, DruidStorageHandlerUtils.JSON_MAPPER,
+ DruidStorageHandlerUtils.INDEX_IO, DruidStorageHandlerUtils.INDEX_MERGER_V9
);
Preconditions.checkArgument(maxPartitionSize > 0, "maxPartitionSize need to be greater than 0");
this.maxPartitionSize = maxPartitionSize;
@@ -243,6 +249,11 @@ public class DruidRecordWriter implements RecordWriter<NullWritable, DruidWritab
} catch (InterruptedException e) {
Throwables.propagate(e);
} finally {
+ try {
+ FileUtils.deleteDirectory(tuningConfig.getBasePersistDirectory());
+ } catch (Exception e){
+ LOG.error("error cleaning of base persist directory", e);
+ }
appenderator.close();
}
}
[34/50] [abbrv] hive git commit: HIVE-15935: ACL is not set in ATS
data (Daniel Dai, reviewed by Siddharth Seth)
Posted by om...@apache.org.
HIVE-15935: ACL is not set in ATS data (Daniel Dai, reviewed by Siddharth Seth)
Change-Id: I95734893c18cefe58731323ad33e6097118c3378
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/44e49982
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/44e49982
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/44e49982
Branch: refs/heads/branch-2.2
Commit: 44e49982a4049990fbfff9b3ad53e104e8499414
Parents: badea3f
Author: Daniel Dai <da...@hortonworks.com>
Authored: Tue Feb 28 15:40:50 2017 -0800
Committer: Owen O'Malley <om...@apache.org>
Committed: Tue Mar 28 15:27:53 2017 -0700
----------------------------------------------------------------------
.../apache/hadoop/hive/ql/hooks/ATSHook.java | 88 +++++++++++++++++++-
.../hadoop/hive/ql/session/SessionState.java | 10 +++
2 files changed, 95 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/44e49982/ql/src/java/org/apache/hadoop/hive/ql/hooks/ATSHook.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/hooks/ATSHook.java b/ql/src/java/org/apache/hadoop/hive/ql/hooks/ATSHook.java
index 2dd03f5..218e0b4 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/hooks/ATSHook.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/hooks/ATSHook.java
@@ -33,6 +33,7 @@ import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.llap.registry.impl.LlapRegistryService;
import org.apache.hadoop.hive.ql.QueryPlan;
import org.apache.hadoop.hive.ql.QueryState;
@@ -43,12 +44,16 @@ import org.apache.hadoop.hive.ql.exec.tez.TezTask;
import org.apache.hadoop.hive.ql.log.PerfLogger;
import org.apache.hadoop.hive.ql.parse.ExplainConfiguration;
import org.apache.hadoop.hive.ql.plan.ExplainWork;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent;
import org.apache.hadoop.yarn.client.api.TimelineClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hive.common.util.ShutdownHookManager;
+import org.apache.tez.dag.api.TezConfiguration;
import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -69,6 +74,9 @@ public class ATSHook implements ExecuteWithHookContext {
private static TimelineClient timelineClient;
private enum EntityTypes { HIVE_QUERY_ID };
private enum EventTypes { QUERY_SUBMITTED, QUERY_COMPLETED };
+ private static final String ATS_DOMAIN_PREFIX = "hive_";
+ private static boolean defaultATSDomainCreated = false;
+ private static final String DEFAULT_ATS_DOMAIN = "hive_default_ats";
private enum OtherInfoTypes {
QUERY, STATUS, TEZ, MAPRED, INVOKER_INFO, SESSION_ID, THREAD_NAME, VERSION,
@@ -136,6 +144,78 @@ public class ATSHook implements ExecuteWithHookContext {
LOG.info("Created ATS Hook");
}
+ private void createTimelineDomain(String domainId, String readers, String writers) throws Exception {
+ TimelineDomain timelineDomain = new TimelineDomain();
+ timelineDomain.setId(domainId);
+ timelineDomain.setReaders(readers);
+ timelineDomain.setWriters(writers);
+ timelineClient.putDomain(timelineDomain);
+ LOG.info("ATS domain created:" + domainId + "(" + readers + "," + writers + ")");
+ }
+
+ private String createOrGetDomain(final HookContext hookContext) throws Exception {
+ final String domainId;
+ String domainReaders = null;
+ String domainWriters = null;
+ boolean create = false;
+ if (SessionState.get() != null) {
+ if (SessionState.get().getATSDomainId() == null) {
+ domainId = ATS_DOMAIN_PREFIX + hookContext.getSessionId();
+ // Create session domain if not present
+ if (SessionState.get().getATSDomainId() == null) {
+ String requestuser = hookContext.getUserName();
+ if (hookContext.getUserName() == null ){
+ requestuser = hookContext.getUgi().getShortUserName() ;
+ }
+ boolean addHs2User =
+ HiveConf.getBoolVar(hookContext.getConf(), ConfVars.HIVETEZHS2USERACCESS);
+
+ UserGroupInformation loginUserUgi = UserGroupInformation.getLoginUser();
+ String loginUser =
+ loginUserUgi == null ? null : loginUserUgi.getShortUserName();
+
+ // In Tez, TEZ_AM_VIEW_ACLS/TEZ_AM_MODIFY_ACLS is used as the base for Tez ATS ACLS,
+ // so if exists, honor it. So we get the same ACLS for Tez ATS entries and
+ // Hive entries
+ domainReaders = Utilities.getAclStringWithHiveModification(hookContext.getConf(),
+ TezConfiguration.TEZ_AM_VIEW_ACLS, addHs2User, requestuser, loginUser);
+
+ domainWriters = Utilities.getAclStringWithHiveModification(hookContext.getConf(),
+ TezConfiguration.TEZ_AM_MODIFY_ACLS, addHs2User, requestuser, loginUser);
+ SessionState.get().setATSDomainId(domainId);
+ create = true;
+ }
+ } else {
+ domainId = SessionState.get().getATSDomainId();
+ }
+ } else {
+ // SessionState is null, this is unlikely to happen, just in case
+ if (!defaultATSDomainCreated) {
+ domainReaders = domainWriters = UserGroupInformation.getCurrentUser().getShortUserName();
+ defaultATSDomainCreated = true;
+ create = true;
+ }
+ domainId = DEFAULT_ATS_DOMAIN;
+ }
+ if (create) {
+ final String readers = domainReaders;
+ final String writers = domainWriters;
+ // executor is single thread, so we can guarantee
+ // domain created before any ATS entries
+ executor.submit(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ createTimelineDomain(domainId, readers, writers);
+ } catch (Exception e) {
+ LOG.warn("Failed to create ATS domain " + domainId, e);
+ }
+ }
+ });
+ }
+ return domainId;
+ }
+
@Override
public void run(final HookContext hookContext) throws Exception {
final long currentTime = System.currentTimeMillis();
@@ -151,6 +231,7 @@ public class ATSHook implements ExecuteWithHookContext {
try {
setupAtsExecutor(conf);
+ final String domainId = createOrGetDomain(hookContext);
executor.submit(new Runnable() {
@Override
public void run() {
@@ -223,12 +304,12 @@ public class ATSHook implements ExecuteWithHookContext {
case POST_EXEC_HOOK:
fireAndForget(createPostHookEvent(queryId,
currentTime, user, requestuser, true, opId,
- durations));
+ durations, domainId));
break;
case ON_FAILURE_HOOK:
fireAndForget(createPostHookEvent(queryId,
currentTime, user, requestuser, false,
- opId, durations));
+ opId, durations, domainId));
break;
default:
// ignore
@@ -342,7 +423,7 @@ public class ATSHook implements ExecuteWithHookContext {
}
TimelineEntity createPostHookEvent(String queryId, long stopTime, String user, String requestuser, boolean success,
- String opId, Map<String, Long> durations) throws Exception {
+ String opId, Map<String, Long> durations, String domainId) throws Exception {
LOG.info("Received post-hook notification for :" + queryId);
TimelineEntity atsEntity = new TimelineEntity();
@@ -367,6 +448,7 @@ public class ATSHook implements ExecuteWithHookContext {
perfObj.put(entry.getKey(), entry.getValue());
}
atsEntity.addOtherInfo(OtherInfoTypes.PERF.name(), perfObj.toString());
+ atsEntity.setDomainId(domainId);
return atsEntity;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/44e49982/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
index 847081b..6fc9055 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
@@ -283,6 +283,8 @@ public class SessionState {
private List<String> forwardedAddresses;
+ private String atsDomainId;
+
/**
* Get the lineage state stored in this session.
*
@@ -1251,6 +1253,14 @@ public class SessionState {
}
}
+ public String getATSDomainId() {
+ return atsDomainId;
+ }
+
+ public void setATSDomainId(String domainId) {
+ this.atsDomainId = domainId;
+ }
+
/**
* ResourceType.
*
[40/50] [abbrv] hive git commit: HIVE-16067. LLAP: send out container
complete messages after a fragment completes. (Siddharth Seth,
reviewed by Sergey Shelukhin)
Posted by om...@apache.org.
HIVE-16067. LLAP: send out container complete messages after a fragment completes. (Siddharth Seth, reviewed by Sergey Shelukhin)
(cherry picked from commit ba5397a7819ca0388c147cd1ab53c4d102b35132)
Change-Id: Ibbcc4cb01a6a6c81cd4845899e31c9ad765225ed
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/4bcb0772
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/4bcb0772
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/4bcb0772
Branch: refs/heads/branch-2.2
Commit: 4bcb077294702e1d5e768ca0fb611ded688cdaad
Parents: c3d311f
Author: Siddharth Seth <ss...@apache.org>
Authored: Wed Mar 1 09:10:47 2017 -0800
Committer: Owen O'Malley <om...@apache.org>
Committed: Tue Mar 28 15:27:55 2017 -0700
----------------------------------------------------------------------
.../hive/llap/tezplugins/LlapContainerLauncher.java | 14 ++++++++++----
.../llap/tezplugins/LlapTaskSchedulerService.java | 9 ++++++++-
2 files changed, 18 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/4bcb0772/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapContainerLauncher.java
----------------------------------------------------------------------
diff --git a/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapContainerLauncher.java b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapContainerLauncher.java
index 07703a2..9e928e6 100644
--- a/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapContainerLauncher.java
+++ b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapContainerLauncher.java
@@ -30,14 +30,20 @@ public class LlapContainerLauncher extends ContainerLauncher {
@Override
public void launchContainer(ContainerLaunchRequest containerLaunchRequest) {
- LOG.info("No-op launch for container: " + containerLaunchRequest.getContainerId() +
- " succeeded on host: " + containerLaunchRequest.getNodeId());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("No-op launch for container: " +
+ containerLaunchRequest.getContainerId() +
+ " succeeded on host: " + containerLaunchRequest.getNodeId());
+ }
getContext().containerLaunched(containerLaunchRequest.getContainerId());
}
@Override
public void stopContainer(ContainerStopRequest containerStopRequest) {
- LOG.info("DEBUG: Ignoring STOP_REQUEST for event: " + containerStopRequest);
- getContext().containerStopRequested(containerStopRequest.getContainerId());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("No-op stopContainer invocation for containerId={}",
+ containerStopRequest.getContainerId());
+ }
+ // Nothing to do here.
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/4bcb0772/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskSchedulerService.java
----------------------------------------------------------------------
diff --git a/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskSchedulerService.java b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskSchedulerService.java
index 9d8840c..f29c1d0 100644
--- a/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskSchedulerService.java
+++ b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskSchedulerService.java
@@ -72,6 +72,8 @@ import org.apache.hadoop.hive.llap.tezplugins.metrics.LlapTaskSchedulerMetrics;
import org.apache.hadoop.util.JvmPauseMonitor;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.NodeState;
@@ -703,12 +705,17 @@ public class LlapTaskSchedulerService extends TaskScheduler {
writeLock.unlock();
}
getContext().containerBeingReleased(taskInfo.containerId);
+ getContext().containerCompleted(taskInfo.task, ContainerStatus.newInstance(taskInfo.containerId,
+ ContainerState.COMPLETE, "", 0));
return true;
}
@Override
public Object deallocateContainer(ContainerId containerId) {
- LOG.debug("Ignoring deallocateContainer for containerId: " + containerId);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Ignoring deallocateContainer for containerId: {}",
+ containerId);
+ }
// Containers are not being tracked for re-use.
// This is safe to ignore since a deallocate task will come in.
return null;
[20/50] [abbrv] hive git commit: HIVE-14362: Support explain analyze
in Hive (Pengcheng Xiong, reviewed by Ashutosh Chauhan,
Gopal V and Gabor Szadovszky)
Posted by om...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/fd72ec4a/ql/src/java/org/apache/hadoop/hive/ql/parse/SubQueryDiagnostic.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SubQueryDiagnostic.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SubQueryDiagnostic.java
index 57f9432..f3c6820 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SubQueryDiagnostic.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SubQueryDiagnostic.java
@@ -29,7 +29,7 @@ public class SubQueryDiagnostic {
static QBSubQueryRewrite getRewrite(QBSubQuery subQuery,
TokenRewriteStream stream,
Context ctx) {
- if (ctx.getExplain()) {
+ if (ctx.isExplainSkipExecution()) {
return new QBSubQueryRewrite(subQuery, stream);
} else {
return new QBSubQueryRewriteNoop(subQuery, stream);
http://git-wip-us.apache.org/repos/asf/hive/blob/fd72ec4a/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java
index b07ab1f..6edab16 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java
@@ -28,6 +28,7 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Queue;
import java.util.Set;
+import java.util.Stack;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -41,6 +42,7 @@ import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.QueryState;
import org.apache.hadoop.hive.ql.exec.ColumnStatsTask;
import org.apache.hadoop.hive.ql.exec.FetchTask;
+import org.apache.hadoop.hive.ql.exec.Operator;
import org.apache.hadoop.hive.ql.exec.StatsTask;
import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.exec.TaskFactory;
@@ -52,6 +54,7 @@ import org.apache.hadoop.hive.ql.hooks.WriteEntity;
import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.optimizer.GenMapRedUtils;
+import org.apache.hadoop.hive.ql.optimizer.physical.AnnotateRunTimeStatsOptimizer;
import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.AnalyzeRewriteContext;
import org.apache.hadoop.hive.ql.plan.ColumnStatsDesc;
import org.apache.hadoop.hive.ql.plan.ColumnStatsWork;
@@ -128,6 +131,7 @@ public abstract class TaskCompiler {
conf.set(SerDeUtils.LIST_SINK_OUTPUT_FORMATTER, DefaultFetchFormatter.class.getName());
}
}
+
return;
}
@@ -506,4 +510,5 @@ public abstract class TaskCompiler {
return clone;
}
+
}
http://git-wip-us.apache.org/repos/asf/hive/blob/fd72ec4a/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java
index 79f925d..c0e56e7 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java
@@ -45,6 +45,7 @@ import org.apache.hadoop.hive.ql.optimizer.ReduceSinkMapJoinProc;
import org.apache.hadoop.hive.ql.optimizer.RemoveDynamicPruningBySize;
import org.apache.hadoop.hive.ql.optimizer.SetReducerParallelism;
import org.apache.hadoop.hive.ql.optimizer.metainfo.annotation.AnnotateWithOpTraits;
+import org.apache.hadoop.hive.ql.optimizer.physical.AnnotateRunTimeStatsOptimizer;
import org.apache.hadoop.hive.ql.optimizer.physical.CrossProductCheck;
import org.apache.hadoop.hive.ql.optimizer.physical.LlapDecider;
import org.apache.hadoop.hive.ql.optimizer.physical.MemoryDecider;
@@ -424,8 +425,9 @@ public class TezCompiler extends TaskCompiler {
}
// we need to clone some operator plans and remove union operators still
+ int indexForTezUnion = 0;
for (BaseWork w: procCtx.workWithUnionOperators) {
- GenTezUtils.removeUnionOperators(procCtx, w);
+ GenTezUtils.removeUnionOperators(procCtx, w, indexForTezUnion++);
}
// then we make sure the file sink operators are set up right
@@ -531,7 +533,8 @@ public class TezCompiler extends TaskCompiler {
LOG.debug("Skipping cross product analysis");
}
- if (conf.getBoolVar(HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED)) {
+ if (conf.getBoolVar(HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED)
+ && ctx.getExplainAnalyze() == null) {
physicalCtx = new Vectorizer().resolve(physicalCtx);
} else {
LOG.debug("Skipping vectorization");
@@ -559,6 +562,11 @@ public class TezCompiler extends TaskCompiler {
// the backend. If you have a physical optimization that changes
// table scans or filters, you have to invoke it before this one.
physicalCtx = new SerializeFilter().resolve(physicalCtx);
+
+ if (physicalCtx.getContext().getExplainAnalyze() != null) {
+ new AnnotateRunTimeStatsOptimizer().resolve(physicalCtx);
+ }
+
perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.TEZ_COMPILER, "optimizeTaskPlan");
return;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/fd72ec4a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkCompiler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkCompiler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkCompiler.java
index 08278de..baf77c7 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkCompiler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkCompiler.java
@@ -61,6 +61,7 @@ import org.apache.hadoop.hive.ql.optimizer.ConstantPropagate;
import org.apache.hadoop.hive.ql.optimizer.DynamicPartitionPruningOptimization;
import org.apache.hadoop.hive.ql.optimizer.SparkRemoveDynamicPruningBySize;
import org.apache.hadoop.hive.ql.optimizer.metainfo.annotation.AnnotateWithOpTraits;
+import org.apache.hadoop.hive.ql.optimizer.physical.AnnotateRunTimeStatsOptimizer;
import org.apache.hadoop.hive.ql.optimizer.physical.MetadataOnlyOptimizer;
import org.apache.hadoop.hive.ql.optimizer.physical.NullScanOptimizer;
import org.apache.hadoop.hive.ql.optimizer.physical.PhysicalContext;
@@ -429,7 +430,8 @@ public class SparkCompiler extends TaskCompiler {
LOG.debug("Skipping cross product analysis");
}
- if (conf.getBoolVar(HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED)) {
+ if (conf.getBoolVar(HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED)
+ && ctx.getExplainAnalyze() == null) {
(new Vectorizer()).resolve(physicalCtx);
} else {
LOG.debug("Skipping vectorization");
@@ -443,6 +445,10 @@ public class SparkCompiler extends TaskCompiler {
new CombineEquivalentWorkResolver().resolve(physicalCtx);
+ if (physicalCtx.getContext().getExplainAnalyze() != null) {
+ new AnnotateRunTimeStatsOptimizer().resolve(physicalCtx);
+ }
+
PERF_LOGGER.PerfLogEnd(CLASS_NAME, PerfLogger.SPARK_OPTIMIZE_TASK_TREE);
return;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/fd72ec4a/ql/src/java/org/apache/hadoop/hive/ql/plan/AbstractOperatorDesc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/AbstractOperatorDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/AbstractOperatorDesc.java
index 936a6ff..7df9d07 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/AbstractOperatorDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/AbstractOperatorDesc.java
@@ -21,8 +21,10 @@ package org.apache.hadoop.hive.ql.plan;
import java.util.Map;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.ql.exec.PTFUtils;
import org.apache.hadoop.hive.ql.plan.Explain.Level;
+import org.apache.hadoop.hive.ql.stats.StatsCollectionContext;
public class AbstractOperatorDesc implements OperatorDesc {
@@ -32,6 +34,7 @@ public class AbstractOperatorDesc implements OperatorDesc {
protected transient Map<String, String> opProps;
protected long memNeeded = 0;
protected long memAvailable = 0;
+ protected String runtimeStatsTmpDir;
@Override
@Explain(skipHeader = true, displayName = "Statistics")
@@ -101,4 +104,12 @@ public class AbstractOperatorDesc implements OperatorDesc {
this.memAvailable = memoryAvailble;
}
+ public String getRuntimeStatsTmpDir() {
+ return runtimeStatsTmpDir;
+ }
+
+ public void setRuntimeStatsTmpDir(String runtimeStatsTmpDir) {
+ this.runtimeStatsTmpDir = runtimeStatsTmpDir;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hive/blob/fd72ec4a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExplainWork.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExplainWork.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExplainWork.java
index a213c83..9f4767c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExplainWork.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExplainWork.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.hooks.ReadEntity;
import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer;
+import org.apache.hadoop.hive.ql.parse.ExplainConfiguration;
import org.apache.hadoop.hive.ql.parse.ParseContext;
/**
@@ -42,15 +43,10 @@ public class ExplainWork implements Serializable {
private HashSet<ReadEntity> inputs;
private ParseContext pCtx;
- boolean extended;
- boolean formatted;
- boolean dependency;
- boolean logical;
+ private ExplainConfiguration config;
boolean appendTaskType;
- boolean authorize;
- boolean userLevelExplain;
String cboInfo;
private transient BaseSemanticAnalyzer analyzer;
@@ -63,12 +59,7 @@ public class ExplainWork implements Serializable {
List<Task<? extends Serializable>> rootTasks,
Task<? extends Serializable> fetchTask,
BaseSemanticAnalyzer analyzer,
- boolean extended,
- boolean formatted,
- boolean dependency,
- boolean logical,
- boolean authorize,
- boolean userLevelExplain,
+ ExplainConfiguration config,
String cboInfo) {
this.resFile = resFile;
this.rootTasks = new ArrayList<Task<? extends Serializable>>(rootTasks);
@@ -77,14 +68,9 @@ public class ExplainWork implements Serializable {
if (analyzer != null) {
this.inputs = analyzer.getInputs();
}
- this.extended = extended;
- this.formatted = formatted;
- this.dependency = dependency;
- this.logical = logical;
this.pCtx = pCtx;
- this.authorize = authorize;
- this.userLevelExplain = userLevelExplain;
this.cboInfo = cboInfo;
+ this.config = config;
}
public Path getResFile() {
@@ -120,27 +106,15 @@ public class ExplainWork implements Serializable {
}
public boolean getExtended() {
- return extended;
- }
-
- public void setExtended(boolean extended) {
- this.extended = extended;
+ return config.isExtended();
}
public boolean getDependency() {
- return dependency;
- }
-
- public void setDependency(boolean dependency) {
- this.dependency = dependency;
+ return config.isDependency();
}
public boolean isFormatted() {
- return formatted;
- }
-
- public void setFormatted(boolean formatted) {
- this.formatted = formatted;
+ return config.isFormatted();
}
public ParseContext getParseContext() {
@@ -152,11 +126,7 @@ public class ExplainWork implements Serializable {
}
public boolean isLogical() {
- return logical;
- }
-
- public void setLogical(boolean logical) {
- this.logical = logical;
+ return config.isLogical();
}
public boolean isAppendTaskType() {
@@ -168,11 +138,7 @@ public class ExplainWork implements Serializable {
}
public boolean isAuthorize() {
- return authorize;
- }
-
- public void setAuthorize(boolean authorize) {
- this.authorize = authorize;
+ return config.isAuthorize();
}
public BaseSemanticAnalyzer getAnalyzer() {
@@ -180,11 +146,7 @@ public class ExplainWork implements Serializable {
}
public boolean isUserLevelExplain() {
- return userLevelExplain;
- }
-
- public void setUserLevelExplain(boolean userLevelExplain) {
- this.userLevelExplain = userLevelExplain;
+ return config.isUserLevelExplain();
}
public String getCboInfo() {
@@ -195,4 +157,12 @@ public class ExplainWork implements Serializable {
this.cboInfo = cboInfo;
}
+ public ExplainConfiguration getConfig() {
+ return config;
+ }
+
+ public void setConfig(ExplainConfiguration config) {
+ this.config = config;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hive/blob/fd72ec4a/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java
index 194f77a..4d9139b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java
@@ -61,7 +61,6 @@ public class FileSinkDesc extends AbstractOperatorDesc {
private DynamicPartitionCtx dpCtx;
private String staticSpec; // static partition spec ends with a '/'
private boolean gatherStats;
- private int indexInTezUnion = -1;
// Consider a query like:
// insert overwrite table T3 select ... from T1 join T2 on T1.key = T2.key;
@@ -489,12 +488,4 @@ public class FileSinkDesc extends AbstractOperatorDesc {
this.statsTmpDir = statsCollectionTempDir;
}
- public int getIndexInTezUnion() {
- return indexInTezUnion;
- }
-
- public void setIndexInTezUnion(int indexInTezUnion) {
- this.indexInTezUnion = indexInTezUnion;
- }
-
}
http://git-wip-us.apache.org/repos/asf/hive/blob/fd72ec4a/ql/src/java/org/apache/hadoop/hive/ql/plan/MergeJoinWork.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/MergeJoinWork.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/MergeJoinWork.java
index a5527dc..20cd56f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/MergeJoinWork.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/MergeJoinWork.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.hive.ql.plan;
import java.util.ArrayList;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@@ -48,7 +49,12 @@ public class MergeJoinWork extends BaseWork {
@Override
public Set<Operator<?>> getAllRootOperators() {
- return getMainWork().getAllRootOperators();
+ Set<Operator<?>> set = new HashSet<>();
+ set.addAll(getMainWork().getAllRootOperators());
+ for (BaseWork w : mergeWorkList) {
+ set.addAll(w.getAllRootOperators());
+ }
+ return set;
}
@Override
http://git-wip-us.apache.org/repos/asf/hive/blob/fd72ec4a/ql/src/java/org/apache/hadoop/hive/ql/plan/OperatorDesc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/OperatorDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/OperatorDesc.java
index b5fb888..850576c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/OperatorDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/OperatorDesc.java
@@ -32,4 +32,6 @@ public interface OperatorDesc extends Serializable, Cloneable {
public void setMemoryNeeded(long memoryNeeded);
public long getMaxMemoryAvailable();
public void setMaxMemoryAvailable(long memoryAvailble);
+ public String getRuntimeStatsTmpDir();
+ public void setRuntimeStatsTmpDir(String runtimeStatsTmpDir);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/fd72ec4a/ql/src/java/org/apache/hadoop/hive/ql/plan/Statistics.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/Statistics.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/Statistics.java
index 029043f..c46ea70 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/Statistics.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/Statistics.java
@@ -40,18 +40,20 @@ public class Statistics implements Serializable {
}
private long numRows;
+ private long runTimeNumRows;
private long dataSize;
private State basicStatsState;
private Map<String, ColStatistics> columnStats;
private State columnStatsState;
public Statistics() {
- this(0, 0);
+ this(0, 0, -1);
}
- public Statistics(long nr, long ds) {
+ public Statistics(long nr, long ds, long rnr) {
this.setNumRows(nr);
this.setDataSize(ds);
+ this.setRunTimeNumRows(rnr);
this.basicStatsState = State.NONE;
this.columnStats = null;
this.columnStatsState = State.NONE;
@@ -107,6 +109,9 @@ public class Statistics implements Serializable {
StringBuilder sb = new StringBuilder();
sb.append("Num rows: ");
sb.append(numRows);
+ if (runTimeNumRows >= 0) {
+ sb.append("/" + runTimeNumRows);
+ }
sb.append(" Data size: ");
sb.append(dataSize);
sb.append(" Basic stats: ");
@@ -121,6 +126,9 @@ public class Statistics implements Serializable {
StringBuilder sb = new StringBuilder();
sb.append("rows=");
sb.append(numRows);
+ if (runTimeNumRows >= 0) {
+ sb.append("/" + runTimeNumRows);
+ }
sb.append(" width=");
// just to be safe about numRows
if (numRows != 0) {
@@ -148,7 +156,7 @@ public class Statistics implements Serializable {
@Override
public Statistics clone() throws CloneNotSupportedException {
- Statistics clone = new Statistics(numRows, dataSize);
+ Statistics clone = new Statistics(numRows, dataSize, runTimeNumRows);
clone.setBasicStatsState(basicStatsState);
clone.setColumnStatsState(columnStatsState);
if (columnStats != null) {
@@ -263,4 +271,12 @@ public class Statistics implements Serializable {
}
return null;
}
+
+ public long getRunTimeNumRows() {
+ return runTimeNumRows;
+ }
+
+ public void setRunTimeNumRows(long runTimeNumRows) {
+ this.runTimeNumRows = runTimeNumRows;
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/fd72ec4a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java
index c6e317b..a573808 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java
@@ -295,8 +295,10 @@ public class TestUpdateDeleteSemanticAnalyzer {
Path tmp = new Path(f.getPath());
fs.create(tmp);
fs.deleteOnExit(tmp);
+ ExplainConfiguration config = new ExplainConfiguration();
+ config.setExtended(true);
ExplainWork work = new ExplainWork(tmp, sem.getParseContext(), sem.getRootTasks(),
- sem.getFetchTask(), sem, true, false, false, false, false, false, null);
+ sem.getFetchTask(), sem, config, null);
ExplainTask task = new ExplainTask();
task.setWork(work);
task.initialize(queryState, plan, null, null);
http://git-wip-us.apache.org/repos/asf/hive/blob/fd72ec4a/ql/src/test/queries/clientpositive/explainanalyze_1.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/explainanalyze_1.q b/ql/src/test/queries/clientpositive/explainanalyze_1.q
new file mode 100644
index 0000000..a4b3dc5
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/explainanalyze_1.q
@@ -0,0 +1,38 @@
+set hive.mapred.mode=nonstrict;
+
+explain analyze select * from src a union all select * from src b limit 10;
+
+explain analyze select key from src;
+
+explain analyze create table t as select key from src;
+
+create table t as select key from src;
+
+explain analyze insert overwrite table t select key from src;
+
+explain analyze select key from src limit 10;
+
+explain analyze select key from src where value < 10;
+
+explain analyze select key from src where key < 10;
+select count(*) from (select key from src where key < 10)subq;
+
+explain analyze select key, count(key) from src group by key;
+select count(*) from (select key, count(key) from src group by key)subq;
+
+explain analyze select count(*) from src a join src b on a.key = b.value where a.key > 0;
+
+explain analyze select count(*) from src a join src b on a.key = b.key where a.key > 0;
+select count(*) from src a join src b on a.key = b.key where a.key > 0;
+
+
+explain analyze select * from src a union all select * from src b;
+select count(*) from (select * from src a union all select * from src b)subq;
+
+set hive.auto.convert.join=true;
+set hive.auto.convert.join.noconditionaltask=true;
+set hive.auto.convert.join.noconditionaltask.size=10000;
+
+EXPLAIN analyze
+SELECT x.key, y.value
+FROM src x JOIN src y ON (x.key = y.key);
http://git-wip-us.apache.org/repos/asf/hive/blob/fd72ec4a/ql/src/test/queries/clientpositive/explainanalyze_2.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/explainanalyze_2.q b/ql/src/test/queries/clientpositive/explainanalyze_2.q
new file mode 100644
index 0000000..dfee826
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/explainanalyze_2.q
@@ -0,0 +1,329 @@
+set hive.explain.user=true;
+set hive.metastore.aggregate.stats.cache.enabled=false;
+
+-- SORT_QUERY_RESULTS
+
+CREATE TABLE dest_j1(key STRING, value STRING, val2 STRING) STORED AS TEXTFILE;
+
+CREATE TABLE ss(k1 STRING,v1 STRING,k2 STRING,v2 STRING,k3 STRING,v3 STRING) STORED AS TEXTFILE;
+
+CREATE TABLE sr(k1 STRING,v1 STRING,k2 STRING,v2 STRING,k3 STRING,v3 STRING) STORED AS TEXTFILE;
+
+CREATE TABLE cs(k1 STRING,v1 STRING,k2 STRING,v2 STRING,k3 STRING,v3 STRING) STORED AS TEXTFILE;
+
+INSERT OVERWRITE TABLE ss
+SELECT x.key,x.value,y.key,y.value,z.key,z.value
+FROM src1 x
+JOIN src y ON (x.key = y.key)
+JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11);
+
+INSERT OVERWRITE TABLE sr
+SELECT x.key,x.value,y.key,y.value,z.key,z.value
+FROM src1 x
+JOIN src y ON (x.key = y.key)
+JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=12);
+
+INSERT OVERWRITE TABLE cs
+SELECT x.key,x.value,y.key,y.value,z.key,z.value
+FROM src1 x
+JOIN src y ON (x.key = y.key)
+JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08');
+
+
+ANALYZE TABLE ss COMPUTE STATISTICS;
+ANALYZE TABLE ss COMPUTE STATISTICS FOR COLUMNS k1,v1,k2,v2,k3,v3;
+
+ANALYZE TABLE sr COMPUTE STATISTICS;
+ANALYZE TABLE sr COMPUTE STATISTICS FOR COLUMNS k1,v1,k2,v2,k3,v3;
+
+ANALYZE TABLE cs COMPUTE STATISTICS;
+ANALYZE TABLE cs COMPUTE STATISTICS FOR COLUMNS k1,v1,k2,v2,k3,v3;
+
+set hive.auto.convert.join=false;
+
+explain analyze
+SELECT x.key, z.value, y.value
+FROM src1 x JOIN src y ON (x.key = y.key)
+JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11);
+
+explain analyze
+select
+ss.k1,sr.k2,cs.k3,count(ss.v1),count(sr.v2),count(cs.v3)
+FROM
+ss,sr,cs,src d1,src d2,src d3,src1,srcpart
+where
+ ss.k1 = d1.key
+and sr.k1 = d2.key
+and cs.k1 = d3.key
+and ss.k2 = sr.k2
+and ss.k3 = sr.k3
+and ss.v1 = src1.value
+and ss.v2 = srcpart.value
+and sr.v2 = cs.v2
+and sr.v3 = cs.v3
+and ss.v3='ssv3'
+and sr.v1='srv1'
+and src1.key = 'src1key'
+and srcpart.key = 'srcpartkey'
+and d1.value = 'd1value'
+and d2.value in ('2000Q1','2000Q2','2000Q3')
+and d3.value in ('2000Q1','2000Q2','2000Q3')
+group by
+ss.k1,sr.k2,cs.k3
+order by
+ss.k1,sr.k2,cs.k3
+limit 100;
+
+explain analyze
+SELECT x.key, z.value, y.value
+FROM src1 x JOIN src y ON (x.key = y.key)
+JOIN (select * from src1 union select * from src)z ON (x.value = z.value)
+union
+SELECT x.key, z.value, y.value
+FROM src1 x JOIN src y ON (x.key = y.key)
+JOIN (select * from src1 union select * from src)z ON (x.value = z.value);
+
+explain analyze
+SELECT x.key, y.value
+FROM src1 x JOIN src y ON (x.key = y.key)
+JOIN (select * from src1 union select * from src)z ON (x.value = z.value)
+union
+SELECT x.key, y.value
+FROM src1 x JOIN src y ON (x.key = y.key)
+JOIN (select key, value from src1 union select key, value from src union select key, value from src)z ON (x.value = z.value)
+union
+SELECT x.key, y.value
+FROM src1 x JOIN src y ON (x.key = y.key)
+JOIN (select key, value from src1 union select key, value from src union select key, value from src union select key, value from src)z ON (x.value = z.value);
+
+
+set hive.auto.convert.join=true;
+set hive.auto.convert.join.noconditionaltask=true;
+set hive.auto.convert.join.noconditionaltask.size=10000;
+set hive.stats.fetch.column.stats=false;
+
+
+explain analyze
+SELECT x.key, z.value, y.value
+FROM src1 x JOIN src y ON (x.key = y.key)
+JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11);
+
+explain analyze
+select
+ss.k1,sr.k2,cs.k3,count(ss.v1),count(sr.v2),count(cs.v3)
+FROM
+ss,sr,cs,src d1,src d2,src d3,src1,srcpart
+where
+ ss.k1 = d1.key
+and sr.k1 = d2.key
+and cs.k1 = d3.key
+and ss.k2 = sr.k2
+and ss.k3 = sr.k3
+and ss.v1 = src1.value
+and ss.v2 = srcpart.value
+and sr.v2 = cs.v2
+and sr.v3 = cs.v3
+and ss.v3='ssv3'
+and sr.v1='srv1'
+and src1.key = 'src1key'
+and srcpart.key = 'srcpartkey'
+and d1.value = 'd1value'
+and d2.value in ('2000Q1','2000Q2','2000Q3')
+and d3.value in ('2000Q1','2000Q2','2000Q3')
+group by
+ss.k1,sr.k2,cs.k3
+order by
+ss.k1,sr.k2,cs.k3
+limit 100;
+
+explain analyze
+SELECT x.key, z.value, y.value
+FROM src1 x JOIN src y ON (x.key = y.key)
+JOIN (select * from src1 union select * from src)z ON (x.value = z.value)
+union
+SELECT x.key, z.value, y.value
+FROM src1 x JOIN src y ON (x.key = y.key)
+JOIN (select * from src1 union select * from src)z ON (x.value = z.value);
+
+explain analyze
+SELECT x.key, y.value
+FROM src1 x JOIN src y ON (x.key = y.key)
+JOIN (select * from src1 union select * from src)z ON (x.value = z.value)
+union
+SELECT x.key, y.value
+FROM src1 x JOIN src y ON (x.key = y.key)
+JOIN (select key, value from src1 union select key, value from src union select key, value from src)z ON (x.value = z.value)
+union
+SELECT x.key, y.value
+FROM src1 x JOIN src y ON (x.key = y.key)
+JOIN (select key, value from src1 union select key, value from src union select key, value from src union select key, value from src)z ON (x.value = z.value);
+
+
+set hive.auto.convert.join=true;
+set hive.auto.convert.join.noconditionaltask=true;
+set hive.auto.convert.join.noconditionaltask.size=10000;
+set hive.auto.convert.sortmerge.join.bigtable.selection.policy = org.apache.hadoop.hive.ql.optimizer.TableSizeBasedBigTableSelectorForAutoSMJ;
+
+CREATE TABLE srcbucket_mapjoin(key int, value string) partitioned by (ds string) CLUSTERED BY (key) INTO 2 BUCKETS STORED AS TEXTFILE;
+CREATE TABLE tab_part (key int, value string) PARTITIONED BY(ds STRING) CLUSTERED BY (key) SORTED BY (key) INTO 4 BUCKETS STORED AS TEXTFILE;
+CREATE TABLE srcbucket_mapjoin_part (key int, value string) partitioned by (ds string) CLUSTERED BY (key) INTO 4 BUCKETS STORED AS TEXTFILE;
+
+load data local inpath '../../data/files/srcbucket20.txt' INTO TABLE srcbucket_mapjoin partition(ds='2008-04-08');
+load data local inpath '../../data/files/srcbucket22.txt' INTO TABLE srcbucket_mapjoin partition(ds='2008-04-08');
+
+load data local inpath '../../data/files/srcbucket20.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08');
+load data local inpath '../../data/files/srcbucket21.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08');
+load data local inpath '../../data/files/srcbucket22.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08');
+load data local inpath '../../data/files/srcbucket23.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08');
+
+
+
+set hive.optimize.bucketingsorting=false;
+insert overwrite table tab_part partition (ds='2008-04-08')
+select key,value from srcbucket_mapjoin_part;
+
+CREATE TABLE tab(key int, value string) PARTITIONED BY(ds STRING) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS STORED AS TEXTFILE;
+insert overwrite table tab partition (ds='2008-04-08')
+select key,value from srcbucket_mapjoin;
+
+CREATE TABLE tab2(key int, value string) PARTITIONED BY(ds STRING) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS STORED AS TEXTFILE;
+insert overwrite table tab2 partition (ds='2008-04-08')
+select key,value from srcbucket_mapjoin;
+
+set hive.convert.join.bucket.mapjoin.tez = false;
+set hive.auto.convert.sortmerge.join = true;
+
+set hive.auto.convert.join.noconditionaltask.size=500;
+
+explain analyze
+select s1.key as key, s1.value as value from tab s1 join tab s3 on s1.key=s3.key;
+
+explain analyze
+select s1.key as key, s1.value as value from tab s1 join tab s3 on s1.key=s3.key join tab s2 on s1.value=s2.value;
+
+explain analyze
+select s1.key as key, s1.value as value from tab s1 join tab2 s3 on s1.key=s3.key;
+
+explain analyze
+select s1.key as key, s1.value as value from tab s1 join tab2 s3 on s1.key=s3.key join tab2 s2 on s1.value=s2.value;
+
+explain analyze
+select count(*) from (select s1.key as key, s1.value as value from tab s1 join tab s3 on s1.key=s3.key
+UNION ALL
+select s2.key as key, s2.value as value from tab s2
+) a join tab_part b on (a.key = b.key);
+
+explain analyze
+select count(*) from (select s1.key as key, s1.value as value from tab s1 join tab s3 on s1.key=s3.key join tab s2 on s1.value=s2.value
+UNION ALL
+select s2.key as key, s2.value as value from tab s2
+) a join tab_part b on (a.key = b.key);
+
+explain analyze
+SELECT x.key, y.value
+FROM src1 x JOIN src y ON (x.key = y.key)
+JOIN (select * from src1 union all select * from src)z ON (x.value = z.value)
+union all
+SELECT x.key, y.value
+FROM src x JOIN src y ON (x.key = y.key)
+JOIN (select key, value from src1 union all select key, value from src union all select key, value from src)z ON (x.value = z.value)
+union all
+SELECT x.key, y.value
+FROM src1 x JOIN src1 y ON (x.key = y.key)
+JOIN (select key, value from src1 union all select key, value from src union all select key, value from src union all select key, value from src)z ON (x.value = z.value);
+
+explain analyze
+SELECT x.key, y.value
+FROM src1 x JOIN src y ON (x.key = y.key)
+JOIN (select * from src1 union select * from src)z ON (x.value = z.value)
+union
+SELECT x.key, y.value
+FROM src x JOIN src y ON (x.key = y.key)
+JOIN (select key, value from src1 union select key, value from src union select key, value from src)z ON (x.value = z.value)
+union
+SELECT x.key, y.value
+FROM src1 x JOIN src1 y ON (x.key = y.key)
+JOIN (select key, value from src1 union select key, value from src union select key, value from src union select key, value from src)z ON (x.value = z.value);
+
+CREATE TABLE a(key STRING, value STRING) STORED AS TEXTFILE;
+CREATE TABLE b(key STRING, value STRING) STORED AS TEXTFILE;
+CREATE TABLE c(key STRING, value STRING) STORED AS TEXTFILE;
+
+explain analyze
+from
+(
+SELECT x.key, y.value
+FROM src1 x JOIN src y ON (x.key = y.key)
+JOIN (select * from src1 union all select * from src)z ON (x.value = z.value)
+union all
+SELECT x.key, y.value
+FROM src x JOIN src y ON (x.key = y.key)
+JOIN (select key, value from src1 union all select key, value from src union all select key, value from src)z ON (x.value = z.value)
+union all
+SELECT x.key, y.value
+FROM src1 x JOIN src1 y ON (x.key = y.key)
+JOIN (select key, value from src1 union all select key, value from src union all select key, value from src union all select key, value from src)z ON (x.value = z.value)
+) tmp
+INSERT OVERWRITE TABLE a SELECT tmp.key, tmp.value
+INSERT OVERWRITE TABLE b SELECT tmp.key, tmp.value
+INSERT OVERWRITE TABLE c SELECT tmp.key, tmp.value;
+
+explain analyze
+FROM
+(
+SELECT x.key as key, y.value as value from src1 x JOIN src y ON (x.key = y.key)
+JOIN (select * from src1 union select * from src)z ON (x.value = z.value)
+union
+SELECT x.key as key, y.value as value from src x JOIN src y ON (x.key = y.key)
+JOIN (select key, value from src1 union select key, value from src union select key, value from src)z ON (x.value = z.value)
+union
+SELECT x.key as key, y.value as value from src1 x JOIN src1 y ON (x.key = y.key)
+JOIN (select key, value from src1 union select key, value from src union select key, value from src union select key, value from src)z ON (x.value = z.value)
+) tmp
+INSERT OVERWRITE TABLE a SELECT tmp.key, tmp.value
+INSERT OVERWRITE TABLE b SELECT tmp.key, tmp.value
+INSERT OVERWRITE TABLE c SELECT tmp.key, tmp.value;
+
+
+CREATE TABLE DEST1(key STRING, value STRING) STORED AS TEXTFILE;
+CREATE TABLE DEST2(key STRING, val1 STRING, val2 STRING) STORED AS TEXTFILE;
+
+explain analyze
+FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1
+ UNION DISTINCT
+ select s2.key as key, s2.value as value from src s2) unionsrc
+INSERT OVERWRITE TABLE DEST1 SELECT unionsrc.key, COUNT(DISTINCT SUBSTR(unionsrc.value,5)) GROUP BY unionsrc.key
+INSERT OVERWRITE TABLE DEST2 SELECT unionsrc.key, unionsrc.value, COUNT(DISTINCT SUBSTR(unionsrc.value,5)) GROUP BY unionsrc.key, unionsrc.value;
+
+explain analyze FROM UNIQUEJOIN PRESERVE src a (a.key), PRESERVE src1 b (b.key), PRESERVE srcpart c (c.key) SELECT a.key, b.key, c.key;
+
+set hive.entity.capture.transform=true;
+
+explain analyze
+SELECT
+TRANSFORM(a.key, a.value) USING 'cat' AS (tkey, tvalue)
+FROM src a join src b
+on a.key = b.key;
+
+explain analyze
+FROM (
+ select key, value from (
+ select 'tst1' as key, cast(count(1) as string) as value, 'tst1' as value2 from src s1
+ UNION all
+ select s2.key as key, s2.value as value, 'tst1' as value2 from src s2) unionsub
+ UNION all
+ select key, value from src s0
+ ) unionsrc
+INSERT OVERWRITE TABLE DEST1 SELECT unionsrc.key, COUNT(DISTINCT SUBSTR(unionsrc.value,5)) GROUP BY unionsrc.key
+INSERT OVERWRITE TABLE DEST2 SELECT unionsrc.key, unionsrc.value, COUNT(DISTINCT SUBSTR(unionsrc.value,5))
+GROUP BY unionsrc.key, unionsrc.value;
+
+explain analyze
+FROM (
+ select 'tst1' as key, cast(count(1) as string) as value, 'tst1' as value2 from src s1
+ UNION all
+ select s2.key as key, s2.value as value, 'tst1' as value2 from src s2
+ ) unionsrc
+INSERT OVERWRITE TABLE DEST1 SELECT unionsrc.key, COUNT(DISTINCT SUBSTR(unionsrc.value,5)) GROUP BY unionsrc.key
+INSERT OVERWRITE TABLE DEST2 SELECT unionsrc.key, unionsrc.value, COUNT(DISTINCT SUBSTR(unionsrc.value,5))
+GROUP BY unionsrc.key, unionsrc.value;
http://git-wip-us.apache.org/repos/asf/hive/blob/fd72ec4a/ql/src/test/queries/clientpositive/explainanalyze_3.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/explainanalyze_3.q b/ql/src/test/queries/clientpositive/explainanalyze_3.q
new file mode 100644
index 0000000..69f82e5
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/explainanalyze_3.q
@@ -0,0 +1,158 @@
+set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.DefaultHiveAuthorizationProvider;
+set hive.metastore.filter.hook=org.apache.hadoop.hive.metastore.DefaultMetaStoreFilterHookImpl;
+set hive.mapred.mode=nonstrict;
+set hive.explain.user=true;
+
+set hive.support.concurrency=true;
+set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
+
+set hive.exec.dynamic.partition.mode=nonstrict;
+set hive.vectorized.execution.enabled=true;
+
+explain analyze select key, value
+FROM srcpart LATERAL VIEW explode(array(1,2,3)) myTable AS myCol;
+
+explain analyze show tables;
+
+explain analyze create database newDB location "/tmp/";
+
+create database newDB location "/tmp/";
+
+explain analyze describe database extended newDB;
+
+describe database extended newDB;
+
+explain analyze use newDB;
+
+use newDB;
+
+create table tab (name string);
+
+explain analyze alter table tab rename to newName;
+
+explain analyze drop table tab;
+
+drop table tab;
+
+explain analyze use default;
+
+use default;
+
+drop database newDB;
+
+explain analyze analyze table src compute statistics;
+
+explain analyze analyze table src compute statistics for columns;
+
+explain analyze
+CREATE TEMPORARY MACRO SIGMOID (x DOUBLE) 1.0 / (1.0 + EXP(-x));
+
+CREATE TEMPORARY MACRO SIGMOID (x DOUBLE) 1.0 / (1.0 + EXP(-x));
+
+explain analyze SELECT SIGMOID(2) FROM src LIMIT 1;
+explain analyze DROP TEMPORARY MACRO SIGMOID;
+DROP TEMPORARY MACRO SIGMOID;
+
+explain analyze create table src_autho_test as select * from src;
+create table src_autho_test as select * from src;
+
+set hive.security.authorization.enabled=true;
+
+explain analyze grant select on table src_autho_test to user hive_test_user;
+grant select on table src_autho_test to user hive_test_user;
+
+explain analyze show grant user hive_test_user on table src_autho_test;
+explain analyze show grant user hive_test_user on table src_autho_test(key);
+
+select key from src_autho_test order by key limit 20;
+
+explain analyze revoke select on table src_autho_test from user hive_test_user;
+
+explain analyze grant select(key) on table src_autho_test to user hive_test_user;
+
+explain analyze revoke select(key) on table src_autho_test from user hive_test_user;
+
+explain analyze
+create role sRc_roLE;
+
+create role sRc_roLE;
+
+explain analyze
+grant role sRc_roLE to user hive_test_user;
+
+grant role sRc_roLE to user hive_test_user;
+
+explain analyze show role grant user hive_test_user;
+
+explain analyze drop role sRc_roLE;
+drop role sRc_roLE;
+
+set hive.security.authorization.enabled=false;
+drop table src_autho_test;
+
+explain analyze drop view v;
+
+explain analyze create view v as with cte as (select * from src order by key limit 5)
+select * from cte;
+
+explain analyze with cte as (select * from src order by key limit 5)
+select * from cte;
+
+create table orc_merge5 (userid bigint, string1 string, subtype double, decimal1 decimal, ts timestamp) stored as orc;
+
+load data local inpath '../../data/files/orc_split_elim.orc' into table orc_merge5;
+
+SET hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat;
+SET mapred.min.split.size=1000;
+SET mapred.max.split.size=50000;
+SET hive.optimize.index.filter=true;
+set hive.merge.orcfile.stripe.level=false;
+set hive.merge.tezfiles=false;
+set hive.merge.mapfiles=false;
+set hive.merge.mapredfiles=false;
+set hive.compute.splits.in.am=true;
+set tez.grouping.min-size=1000;
+set tez.grouping.max-size=50000;
+
+set hive.merge.orcfile.stripe.level=true;
+set hive.merge.tezfiles=true;
+set hive.merge.mapfiles=true;
+set hive.merge.mapredfiles=true;
+
+explain analyze insert overwrite table orc_merge5 select userid,string1,subtype,decimal1,ts from orc_merge5 where userid<=13;
+
+drop table orc_merge5;
+
+set hive.auto.convert.join=true;
+set hive.auto.convert.join.noconditionaltask=true;
+set hive.auto.convert.join.noconditionaltask.size=10000;
+
+CREATE TABLE srcbucket_mapjoin(key int, value string) partitioned by (ds string) CLUSTERED BY (key) INTO 2 BUCKETS STORED AS TEXTFILE;
+CREATE TABLE tab_part (key int, value string) PARTITIONED BY(ds STRING) CLUSTERED BY (key) INTO 4 BUCKETS STORED AS TEXTFILE;
+CREATE TABLE srcbucket_mapjoin_part (key int, value string) partitioned by (ds string) CLUSTERED BY (key) INTO 4 BUCKETS STORED AS TEXTFILE;
+
+load data local inpath '../../data/files/srcbucket20.txt' INTO TABLE srcbucket_mapjoin partition(ds='2008-04-08');
+load data local inpath '../../data/files/srcbucket22.txt' INTO TABLE srcbucket_mapjoin partition(ds='2008-04-08');
+
+load data local inpath '../../data/files/srcbucket20.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08');
+load data local inpath '../../data/files/srcbucket21.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08');
+load data local inpath '../../data/files/srcbucket22.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08');
+load data local inpath '../../data/files/srcbucket23.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08');
+
+
+
+set hive.optimize.bucketingsorting=false;
+insert overwrite table tab_part partition (ds='2008-04-08')
+select key,value from srcbucket_mapjoin_part;
+
+CREATE TABLE tab(key int, value string) PARTITIONED BY(ds STRING) CLUSTERED BY (key) INTO 2 BUCKETS STORED AS TEXTFILE;
+insert overwrite table tab partition (ds='2008-04-08')
+select key,value from srcbucket_mapjoin;
+
+set hive.convert.join.bucket.mapjoin.tez = true;
+explain analyze
+select a.key, a.value, b.value
+from tab a join tab_part b on a.key = b.key;
+
+
+
http://git-wip-us.apache.org/repos/asf/hive/blob/fd72ec4a/ql/src/test/queries/clientpositive/explainanalyze_4.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/explainanalyze_4.q b/ql/src/test/queries/clientpositive/explainanalyze_4.q
new file mode 100644
index 0000000..dad397b
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/explainanalyze_4.q
@@ -0,0 +1,103 @@
+set hive.mapred.mode=nonstrict;
+
+set hive.explain.user=true;
+set hive.auto.convert.join=false;
+set hive.optimize.dynamic.partition.hashjoin=false;
+
+-- First try with regular mergejoin
+explain analyze
+select
+ *
+from alltypesorc a join alltypesorc b on a.cint = b.cint
+where
+ a.cint between 1000000 and 3000000 and b.cbigint is not null
+order by a.cint;
+
+select
+ *
+from alltypesorc a join alltypesorc b on a.cint = b.cint
+where
+ a.cint between 1000000 and 3000000 and b.cbigint is not null
+order by a.cint;
+
+explain analyze
+select
+ count(*)
+from alltypesorc a join alltypesorc b on a.cint = b.cint
+where
+ a.cint between 1000000 and 3000000 and b.cbigint is not null;
+
+select
+ count(*)
+from alltypesorc a join alltypesorc b on a.cint = b.cint
+where
+ a.cint between 1000000 and 3000000 and b.cbigint is not null;
+
+explain analyze
+select
+ a.csmallint, count(*) c1
+from alltypesorc a join alltypesorc b on a.cint = b.cint
+where
+ a.cint between 1000000 and 3000000 and b.cbigint is not null
+group by a.csmallint
+order by c1;
+
+select
+ a.csmallint, count(*) c1
+from alltypesorc a join alltypesorc b on a.cint = b.cint
+where
+ a.cint between 1000000 and 3000000 and b.cbigint is not null
+group by a.csmallint
+order by c1;
+
+set hive.auto.convert.join=true;
+set hive.optimize.dynamic.partition.hashjoin=true;
+set hive.auto.convert.join.noconditionaltask.size=200000;
+set hive.stats.fetch.column.stats=false;
+set hive.exec.reducers.bytes.per.reducer=200000;
+
+-- Try with dynamically partitioned hashjoin
+explain analyze
+select
+ *
+from alltypesorc a join alltypesorc b on a.cint = b.cint
+where
+ a.cint between 1000000 and 3000000 and b.cbigint is not null
+order by a.cint;
+
+select
+ *
+from alltypesorc a join alltypesorc b on a.cint = b.cint
+where
+ a.cint between 1000000 and 3000000 and b.cbigint is not null
+order by a.cint;
+
+explain analyze
+select
+ count(*)
+from alltypesorc a join alltypesorc b on a.cint = b.cint
+where
+ a.cint between 1000000 and 3000000 and b.cbigint is not null;
+
+select
+ count(*)
+from alltypesorc a join alltypesorc b on a.cint = b.cint
+where
+ a.cint between 1000000 and 3000000 and b.cbigint is not null;
+
+explain analyze
+select
+ a.csmallint, count(*) c1
+from alltypesorc a join alltypesorc b on a.cint = b.cint
+where
+ a.cint between 1000000 and 3000000 and b.cbigint is not null
+group by a.csmallint
+order by c1;
+
+select
+ a.csmallint, count(*) c1
+from alltypesorc a join alltypesorc b on a.cint = b.cint
+where
+ a.cint between 1000000 and 3000000 and b.cbigint is not null
+group by a.csmallint
+order by c1;
http://git-wip-us.apache.org/repos/asf/hive/blob/fd72ec4a/ql/src/test/queries/clientpositive/explainanalyze_5.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/explainanalyze_5.q b/ql/src/test/queries/clientpositive/explainanalyze_5.q
new file mode 100644
index 0000000..bb23e45
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/explainanalyze_5.q
@@ -0,0 +1,81 @@
+set hive.stats.column.autogather=true;
+
+explain analyze analyze table src compute statistics;
+
+explain analyze analyze table src compute statistics for columns;
+
+drop table src_multi2;
+
+create table src_multi2 like src;
+
+explain analyze insert overwrite table src_multi2 select subq.key, src.value from (select * from src union select * from src1)subq join src on subq.key=src.key;
+
+select count(*) from (select * from src union select * from src1)subq;
+
+insert overwrite table src_multi2 select subq.key, src.value from (select * from src union select * from src1)subq join src on subq.key=src.key;
+
+describe formatted src_multi2;
+
+
+set hive.mapred.mode=nonstrict;
+set hive.exec.dynamic.partition.mode=nonstrict;
+set hive.support.concurrency=true;
+set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
+
+
+-- SORT_QUERY_RESULTS
+
+create table acid_uami(i int,
+ de decimal(5,2),
+ vc varchar(128)) clustered by (i) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true');
+
+insert into table acid_uami values
+ (1, 109.23, 'mary had a little lamb'),
+ (6553, 923.19, 'its fleece was white as snow');
+
+insert into table acid_uami values
+ (10, 119.23, 'and everywhere that mary went'),
+ (65530, 823.19, 'the lamb was sure to go');
+
+select * from acid_uami order by de;
+
+explain analyze update acid_uami set de = 3.14 where de = 109.23 or de = 119.23;
+
+select * from acid_uami order by de;
+
+update acid_uami set de = 3.14 where de = 109.23 or de = 119.23;
+
+select * from acid_uami order by de;
+
+set hive.support.concurrency=true;
+set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
+set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat;
+
+dfs ${system:test.dfs.mkdir} ${system:test.tmp.dir}/delete_orig_table;
+dfs -copyFromLocal ../../data/files/alltypesorc ${system:test.tmp.dir}/delete_orig_table/00000_0;
+
+create table acid_dot(
+ ctinyint TINYINT,
+ csmallint SMALLINT,
+ cint INT,
+ cbigint BIGINT,
+ cfloat FLOAT,
+ cdouble DOUBLE,
+ cstring1 STRING,
+ cstring2 STRING,
+ ctimestamp1 TIMESTAMP,
+ ctimestamp2 TIMESTAMP,
+ cboolean1 BOOLEAN,
+ cboolean2 BOOLEAN) clustered by (cint) into 1 buckets stored as orc location '${system:test.tmp.dir}/delete_orig_table' TBLPROPERTIES ('transactional'='true');
+
+select count(*) from acid_dot;
+
+explain analyze delete from acid_dot where cint < -1070551679;
+
+select count(*) from acid_dot;
+
+delete from acid_dot where cint < -1070551679;
+
+select count(*) from acid_dot;
+
+dfs -rmr ${system:test.tmp.dir}/delete_orig_table;
http://git-wip-us.apache.org/repos/asf/hive/blob/fd72ec4a/ql/src/test/results/clientpositive/columnstats_partlvl.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/columnstats_partlvl.q.out b/ql/src/test/results/clientpositive/columnstats_partlvl.q.out
index f6f2bfa..7e2edd9 100644
--- a/ql/src/test/results/clientpositive/columnstats_partlvl.q.out
+++ b/ql/src/test/results/clientpositive/columnstats_partlvl.q.out
@@ -46,18 +46,22 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: employee_part
+ Statistics: Num rows: 26 Data size: 105 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: employeeid (type: int)
outputColumnNames: employeeid
+ Statistics: Num rows: 26 Data size: 105 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: compute_stats(employeeid, 16)
keys: 2000.0 (type: double)
mode: hash
outputColumnNames: _col0, _col1
+ Statistics: Num rows: 26 Data size: 105 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: 2000.0 (type: double)
sort order: +
Map-reduce partition columns: 2000.0 (type: double)
+ Statistics: Num rows: 26 Data size: 105 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,bitvector:string,numbitvectors:int>)
Reduce Operator Tree:
Group By Operator
@@ -65,11 +69,14 @@ STAGE PLANS:
keys: 2000.0 (type: double)
mode: mergepartial
outputColumnNames: _col0, _col1
+ Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col1 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,numdistinctvalues:bigint,ndvbitvector:string>), 2000.0 (type: double)
outputColumnNames: _col0, _col1
+ Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
+ Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -98,20 +105,24 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: employee_part
+ Statistics: Num rows: 26 Data size: 105 Basic stats: COMPLETE Column stats: NONE
GatherStats: false
Select Operator
expressions: employeeid (type: int)
outputColumnNames: employeeid
+ Statistics: Num rows: 26 Data size: 105 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: compute_stats(employeeid, 16)
keys: 2000.0 (type: double)
mode: hash
outputColumnNames: _col0, _col1
+ Statistics: Num rows: 26 Data size: 105 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: 2000.0 (type: double)
null sort order: a
sort order: +
Map-reduce partition columns: 2000.0 (type: double)
+ Statistics: Num rows: 26 Data size: 105 Basic stats: COMPLETE Column stats: NONE
tag: -1
value expressions: _col1 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,bitvector:string,numbitvectors:int>)
auto parallelism: false
@@ -173,14 +184,17 @@ STAGE PLANS:
keys: 2000.0 (type: double)
mode: mergepartial
outputColumnNames: _col0, _col1
+ Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col1 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,numdistinctvalues:bigint,ndvbitvector:string>), 2000.0 (type: double)
outputColumnNames: _col0, _col1
+ Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
GlobalTableId: 0
#### A masked pattern was here ####
NumFilesPerFileSink: 1
+ Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE
#### A masked pattern was here ####
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
@@ -232,18 +246,22 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: employee_part
+ Statistics: Num rows: 26 Data size: 105 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: employeeid (type: int)
outputColumnNames: employeeid
+ Statistics: Num rows: 26 Data size: 105 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: compute_stats(employeeid, 16)
keys: 4000.0 (type: double)
mode: hash
outputColumnNames: _col0, _col1
+ Statistics: Num rows: 26 Data size: 105 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: 4000.0 (type: double)
sort order: +
Map-reduce partition columns: 4000.0 (type: double)
+ Statistics: Num rows: 26 Data size: 105 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,bitvector:string,numbitvectors:int>)
Reduce Operator Tree:
Group By Operator
@@ -251,11 +269,14 @@ STAGE PLANS:
keys: 4000.0 (type: double)
mode: mergepartial
outputColumnNames: _col0, _col1
+ Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col1 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,numdistinctvalues:bigint,ndvbitvector:string>), 4000.0 (type: double)
outputColumnNames: _col0, _col1
+ Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
+ Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -284,20 +305,24 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: employee_part
+ Statistics: Num rows: 26 Data size: 105 Basic stats: COMPLETE Column stats: NONE
GatherStats: false
Select Operator
expressions: employeeid (type: int)
outputColumnNames: employeeid
+ Statistics: Num rows: 26 Data size: 105 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: compute_stats(employeeid, 16)
keys: 4000.0 (type: double)
mode: hash
outputColumnNames: _col0, _col1
+ Statistics: Num rows: 26 Data size: 105 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: 4000.0 (type: double)
null sort order: a
sort order: +
Map-reduce partition columns: 4000.0 (type: double)
+ Statistics: Num rows: 26 Data size: 105 Basic stats: COMPLETE Column stats: NONE
tag: -1
value expressions: _col1 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,bitvector:string,numbitvectors:int>)
auto parallelism: false
@@ -359,14 +384,17 @@ STAGE PLANS:
keys: 4000.0 (type: double)
mode: mergepartial
outputColumnNames: _col0, _col1
+ Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col1 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,numdistinctvalues:bigint,ndvbitvector:string>), 4000.0 (type: double)
outputColumnNames: _col0, _col1
+ Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
GlobalTableId: 0
#### A masked pattern was here ####
NumFilesPerFileSink: 1
+ Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE
#### A masked pattern was here ####
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
@@ -418,18 +446,22 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: employee_part
+ Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: employeeid (type: int), employeename (type: string)
outputColumnNames: employeeid, employeename
+ Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: compute_stats(employeeid, 16), compute_stats(employeename, 16)
keys: 2000.0 (type: double)
mode: hash
outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: 2000.0 (type: double)
sort order: +
Map-reduce partition columns: 2000.0 (type: double)
+ Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,bitvector:string,numbitvectors:int>), _col2 (type: struct<columntype:string,maxlength:bigint,sumlength:bigint,count:bigint,countnulls:bigint,bitvector:string,numbitvectors:int>)
Reduce Operator Tree:
Group By Operator
@@ -437,11 +469,14 @@ STAGE PLANS:
keys: 2000.0 (type: double)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col1 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,numdistinctvalues:bigint,ndvbitvector:string>), _col2 (type: struct<columntype:string,maxlength:bigint,avglength:double,countnulls:bigint,numdistinctvalues:bigint,ndvbitvector:string>), 2000.0 (type: double)
outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
+ Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -498,18 +533,22 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: employee_part
+ Statistics: Num rows: 2 Data size: 210 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: employeesalary (type: double), employeeid (type: int), employeename (type: string)
outputColumnNames: employeesalary, employeeid, employeename
+ Statistics: Num rows: 2 Data size: 210 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: compute_stats(employeeid, 16), compute_stats(employeename, 16)
keys: employeesalary (type: double)
mode: hash
outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 2 Data size: 210 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: double)
sort order: +
Map-reduce partition columns: _col0 (type: double)
+ Statistics: Num rows: 2 Data size: 210 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,bitvector:string,numbitvectors:int>), _col2 (type: struct<columntype:string,maxlength:bigint,sumlength:bigint,count:bigint,countnulls:bigint,bitvector:string,numbitvectors:int>)
Reduce Operator Tree:
Group By Operator
@@ -517,11 +556,14 @@ STAGE PLANS:
keys: KEY._col0 (type: double)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col1 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,numdistinctvalues:bigint,ndvbitvector:string>), _col2 (type: struct<columntype:string,maxlength:bigint,avglength:double,countnulls:bigint,numdistinctvalues:bigint,ndvbitvector:string>), _col0 (type: double)
outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
+ Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -580,23 +622,29 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: employee_part
+ Statistics: Num rows: 2 Data size: 210 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: employeeid (type: int), employeename (type: string)
outputColumnNames: employeeid, employeename
+ Statistics: Num rows: 2 Data size: 210 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: compute_stats(employeeid, 16), compute_stats(employeename, 16)
mode: hash
outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 968 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
+ Statistics: Num rows: 1 Data size: 968 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,bitvector:string,numbitvectors:int>), _col1 (type: struct<columntype:string,maxlength:bigint,sumlength:bigint,count:bigint,countnulls:bigint,bitvector:string,numbitvectors:int>)
Reduce Operator Tree:
Group By Operator
aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1)
mode: mergepartial
outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 972 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
+ Statistics: Num rows: 1 Data size: 972 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/fd72ec4a/ql/src/test/results/clientpositive/columnstats_partlvl_dp.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/columnstats_partlvl_dp.q.out b/ql/src/test/results/clientpositive/columnstats_partlvl_dp.q.out
index 46035b5..08fb42e 100644
--- a/ql/src/test/results/clientpositive/columnstats_partlvl_dp.q.out
+++ b/ql/src/test/results/clientpositive/columnstats_partlvl_dp.q.out
@@ -82,18 +82,22 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: employee_part
+ Statistics: Num rows: 1 Data size: 64 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: country (type: string), employeename (type: string), employeeid (type: int)
outputColumnNames: country, employeename, employeeid
+ Statistics: Num rows: 1 Data size: 64 Basic stats: PARTIAL Column stats: NONE
Group By Operator
aggregations: compute_stats(employeename, 16), compute_stats(employeeid, 16)
keys: 4000.0 (type: double), country (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: 4000.0 (type: double), _col1 (type: string)
sort order: ++
Map-reduce partition columns: 4000.0 (type: double), _col1 (type: string)
+ Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE
value expressions: _col2 (type: struct<columntype:string,maxlength:bigint,sumlength:bigint,count:bigint,countnulls:bigint,bitvector:string,numbitvectors:int>), _col3 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,bitvector:string,numbitvectors:int>)
Reduce Operator Tree:
Group By Operator
@@ -101,11 +105,14 @@ STAGE PLANS:
keys: 4000.0 (type: double), KEY._col1 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col2 (type: struct<columntype:string,maxlength:bigint,avglength:double,countnulls:bigint,numdistinctvalues:bigint,ndvbitvector:string>), _col3 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,numdistinctvalues:bigint,ndvbitvector:string>), 4000.0 (type: double), _col1 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
+ Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -153,18 +160,22 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: employee_part
+ Statistics: Num rows: 42 Data size: 169 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: country (type: string), employeeid (type: int)
outputColumnNames: country, employeeid
+ Statistics: Num rows: 42 Data size: 169 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: compute_stats(employeeid, 16)
keys: 2000.0 (type: double), country (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 42 Data size: 169 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: 2000.0 (type: double), _col1 (type: string)
sort order: ++
Map-reduce partition columns: 2000.0 (type: double), _col1 (type: string)
+ Statistics: Num rows: 42 Data size: 169 Basic stats: COMPLETE Column stats: NONE
value expressions: _col2 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,bitvector:string,numbitvectors:int>)
Reduce Operator Tree:
Group By Operator
@@ -172,11 +183,14 @@ STAGE PLANS:
keys: 2000.0 (type: double), KEY._col1 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 21 Data size: 84 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col2 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,numdistinctvalues:bigint,ndvbitvector:string>), 2000.0 (type: double), _col1 (type: string)
outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 21 Data size: 84 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
+ Statistics: Num rows: 21 Data size: 84 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -235,18 +249,22 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: employee_part
+ Statistics: Num rows: 116 Data size: 466 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: employeesalary (type: double), country (type: string), employeeid (type: int)
outputColumnNames: employeesalary, country, employeeid
+ Statistics: Num rows: 116 Data size: 466 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: compute_stats(employeeid, 16)
keys: employeesalary (type: double), country (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 116 Data size: 466 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: double), _col1 (type: string)
sort order: ++
Map-reduce partition columns: _col0 (type: double), _col1 (type: string)
+ Statistics: Num rows: 116 Data size: 466 Basic stats: COMPLETE Column stats: NONE
value expressions: _col2 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,bitvector:string,numbitvectors:int>)
Reduce Operator Tree:
Group By Operator
@@ -254,11 +272,14 @@ STAGE PLANS:
keys: KEY._col0 (type: double), KEY._col1 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 58 Data size: 233 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col2 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,numdistinctvalues:bigint,ndvbitvector:string>), _col0 (type: double), _col1 (type: string)
outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 58 Data size: 233 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
+ Statistics: Num rows: 58 Data size: 233 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -316,18 +337,22 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: employee_part
+ Statistics: Num rows: 2 Data size: 466 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: employeesalary (type: double), country (type: string), employeeid (type: int), employeename (type: string)
outputColumnNames: employeesalary, country, employeeid, employeename
+ Statistics: Num rows: 2 Data size: 466 Basic stats: PARTIAL Column stats: NONE
Group By Operator
aggregations: compute_stats(employeeid, 16), compute_stats(employeename, 16)
keys: employeesalary (type: double), country (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 2 Data size: 466 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: double), _col1 (type: string)
sort order: ++
Map-reduce partition columns: _col0 (type: double), _col1 (type: string)
+ Statistics: Num rows: 2 Data size: 466 Basic stats: COMPLETE Column stats: NONE
value expressions: _col2 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,bitvector:string,numbitvectors:int>), _col3 (type: struct<columntype:string,maxlength:bigint,sumlength:bigint,count:bigint,countnulls:bigint,bitvector:string,numbitvectors:int>)
Reduce Operator Tree:
Group By Operator
@@ -335,11 +360,14 @@ STAGE PLANS:
keys: KEY._col0 (type: double), KEY._col1 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 1 Data size: 233 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col2 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,numdistinctvalues:bigint,ndvbitvector:string>), _col3 (type: struct<columntype:string,maxlength:bigint,avglength:double,countnulls:bigint,numdistinctvalues:bigint,ndvbitvector:string>), _col0 (type: double), _col1 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 1 Data size: 233 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
+ Statistics: Num rows: 1 Data size: 233 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/fd72ec4a/ql/src/test/results/clientpositive/columnstats_quoting.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/columnstats_quoting.q.out b/ql/src/test/results/clientpositive/columnstats_quoting.q.out
index 288e61b..52e3538 100644
--- a/ql/src/test/results/clientpositive/columnstats_quoting.q.out
+++ b/ql/src/test/results/clientpositive/columnstats_quoting.q.out
@@ -24,23 +24,29 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: user_web_events
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: user id (type: bigint), user name (type: string)
outputColumnNames: user id, user name
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Group By Operator
aggregations: compute_stats(user id, 16), compute_stats(user name, 16)
mode: hash
outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 968 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
+ Statistics: Num rows: 1 Data size: 968 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,bitvector:string,numbitvectors:int>), _col1 (type: struct<columntype:string,maxlength:bigint,sumlength:bigint,count:bigint,countnulls:bigint,bitvector:string,numbitvectors:int>)
Reduce Operator Tree:
Group By Operator
aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1)
mode: mergepartial
outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 972 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
+ Statistics: Num rows: 1 Data size: 972 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -75,23 +81,29 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: user_web_events
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: user id (type: bigint)
outputColumnNames: user id
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Group By Operator
aggregations: compute_stats(user id, 16)
mode: hash
outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 476 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
+ Statistics: Num rows: 1 Data size: 476 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,bitvector:string,numbitvectors:int>)
Reduce Operator Tree:
Group By Operator
aggregations: compute_stats(VALUE._col0)
mode: mergepartial
outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 480 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
+ Statistics: Num rows: 1 Data size: 480 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
[27/50] [abbrv] hive git commit: HIVE-15084: Flaky test:
TestMiniTezCliDriver:explainanalyze_1, 2, 3, 4, 5 (Pengcheng Xiong)
Posted by om...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/9f5bea3b/ql/src/test/results/clientpositive/tez/explainanalyze_4.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/explainanalyze_4.q.out b/ql/src/test/results/clientpositive/tez/explainanalyze_4.q.out
index 3426d19..2145a8a 100644
--- a/ql/src/test/results/clientpositive/tez/explainanalyze_4.q.out
+++ b/ql/src/test/results/clientpositive/tez/explainanalyze_4.q.out
@@ -16,8 +16,7 @@ order by a.cint
POSTHOOK: type: QUERY
POSTHOOK: Input: default@alltypesorc
#### A masked pattern was here ####
-PREHOOK: query: -- First try with regular mergejoin
-explain analyze
+PREHOOK: query: explain analyze
select
*
from alltypesorc a join alltypesorc b on a.cint = b.cint
@@ -25,8 +24,7 @@ where
a.cint between 1000000 and 3000000 and b.cbigint is not null
order by a.cint
PREHOOK: type: QUERY
-POSTHOOK: query: -- First try with regular mergejoin
-explain analyze
+POSTHOOK: query: explain analyze
select
*
from alltypesorc a join alltypesorc b on a.cint = b.cint
@@ -46,30 +44,30 @@ Stage-0
Stage-1
Reducer 3
File Output Operator [FS_12]
- Select Operator [SEL_11] (rows=7286/10 width=620)
+ Select Operator [SEL_11] (rows=1501/10 width=215)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23"]
<-Reducer 2 [SIMPLE_EDGE]
SHUFFLE [RS_10]
- Merge Join Operator [MERGEJOIN_17] (rows=7286/10 width=620)
+ Merge Join Operator [MERGEJOIN_17] (rows=1501/10 width=215)
Conds:RS_6._col2=RS_7._col2(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23"]
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_6]
PartitionCols:_col2
- Select Operator [SEL_2] (rows=6144/10 width=251)
+ Select Operator [SEL_2] (rows=1365/10 width=215)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"]
- Filter Operator [FIL_15] (rows=6144/10 width=251)
+ Filter Operator [FIL_15] (rows=1365/10 width=215)
predicate:cint BETWEEN 1000000 AND 3000000
- TableScan [TS_0] (rows=12288/12288 width=251)
- default@alltypesorc,a,Tbl:COMPLETE,Col:COMPLETE,Output:["ctinyint","csmallint","cint","cbigint","cfloat","cdouble","cstring1","cstring2","ctimestamp1","ctimestamp2","cboolean1","cboolean2"]
+ TableScan [TS_0] (rows=12288/12288 width=215)
+ default@alltypesorc,a,Tbl:COMPLETE,Col:NONE,Output:["ctinyint","csmallint","cint","cbigint","cfloat","cdouble","cstring1","cstring2","ctimestamp1","ctimestamp2","cboolean1","cboolean2"]
<-Map 4 [SIMPLE_EDGE]
SHUFFLE [RS_7]
PartitionCols:_col2
- Select Operator [SEL_5] (rows=3424/10 width=251)
+ Select Operator [SEL_5] (rows=1365/10 width=215)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"]
- Filter Operator [FIL_16] (rows=3424/10 width=251)
- predicate:(cint is not null and cbigint is not null and cint BETWEEN 1000000 AND 3000000)
- TableScan [TS_3] (rows=12288/12288 width=251)
- default@alltypesorc,b,Tbl:COMPLETE,Col:COMPLETE,Output:["ctinyint","csmallint","cint","cbigint","cfloat","cdouble","cstring1","cstring2","ctimestamp1","ctimestamp2","cboolean1","cboolean2"]
+ Filter Operator [FIL_16] (rows=1365/10 width=215)
+ predicate:(cint BETWEEN 1000000 AND 3000000 and cbigint is not null)
+ TableScan [TS_3] (rows=12288/12288 width=215)
+ default@alltypesorc,b,Tbl:COMPLETE,Col:NONE,Output:["ctinyint","csmallint","cint","cbigint","cfloat","cdouble","cstring1","cstring2","ctimestamp1","ctimestamp2","cboolean1","cboolean2"]
PREHOOK: query: select
*
@@ -140,33 +138,31 @@ Stage-0
limit:-1
Stage-1
Reducer 3
- File Output Operator [FS_14]
- Group By Operator [GBY_12] (rows=1/1 width=8)
- Output:["_col0"],aggregations:["count(VALUE._col0)"]
+ File Output Operator [FS_13]
+ Group By Operator [GBY_11] (rows=1/1 width=8)
+ Output:["_col0"],aggregations:["count()"]
<-Reducer 2 [SIMPLE_EDGE]
- SHUFFLE [RS_11]
- Group By Operator [GBY_10] (rows=1/1 width=8)
- Output:["_col0"],aggregations:["count()"]
- Merge Join Operator [MERGEJOIN_19] (rows=7286/10 width=8)
- Conds:RS_6._col0=RS_7._col0(Inner)
- <-Map 1 [SIMPLE_EDGE]
- SHUFFLE [RS_6]
- PartitionCols:_col0
- Select Operator [SEL_2] (rows=6144/10 width=2)
- Output:["_col0"]
- Filter Operator [FIL_17] (rows=6144/10 width=2)
- predicate:cint BETWEEN 1000000 AND 3000000
- TableScan [TS_0] (rows=12288/12288 width=2)
- default@alltypesorc,a,Tbl:COMPLETE,Col:COMPLETE,Output:["cint"]
- <-Map 4 [SIMPLE_EDGE]
- SHUFFLE [RS_7]
- PartitionCols:_col0
- Select Operator [SEL_5] (rows=3424/10 width=8)
- Output:["_col0"]
- Filter Operator [FIL_18] (rows=3424/10 width=8)
- predicate:(cint is not null and cbigint is not null and cint BETWEEN 1000000 AND 3000000)
- TableScan [TS_3] (rows=12288/12288 width=8)
- default@alltypesorc,b,Tbl:COMPLETE,Col:COMPLETE,Output:["cint","cbigint"]
+ SHUFFLE [RS_10]
+ Merge Join Operator [MERGEJOIN_18] (rows=1501/10 width=215)
+ Conds:RS_6._col0=RS_7._col0(Inner)
+ <-Map 1 [SIMPLE_EDGE]
+ SHUFFLE [RS_6]
+ PartitionCols:_col0
+ Select Operator [SEL_2] (rows=1365/10 width=215)
+ Output:["_col0"]
+ Filter Operator [FIL_16] (rows=1365/10 width=215)
+ predicate:cint BETWEEN 1000000 AND 3000000
+ TableScan [TS_0] (rows=12288/12288 width=215)
+ default@alltypesorc,a,Tbl:COMPLETE,Col:NONE,Output:["cint"]
+ <-Map 4 [SIMPLE_EDGE]
+ SHUFFLE [RS_7]
+ PartitionCols:_col0
+ Select Operator [SEL_5] (rows=1365/10 width=215)
+ Output:["_col0"]
+ Filter Operator [FIL_17] (rows=1365/10 width=215)
+ predicate:(cint BETWEEN 1000000 AND 3000000 and cbigint is not null)
+ TableScan [TS_3] (rows=12288/12288 width=215)
+ default@alltypesorc,b,Tbl:COMPLETE,Col:NONE,Output:["cint","cbigint"]
PREHOOK: query: select
count(*)
@@ -235,38 +231,36 @@ Stage-0
limit:-1
Stage-1
Reducer 4
- File Output Operator [FS_16]
- Select Operator [SEL_15] (rows=2765/5 width=12)
+ File Output Operator [FS_15]
+ Select Operator [SEL_14] (rows=750/5 width=215)
Output:["_col0","_col1"]
<-Reducer 3 [SIMPLE_EDGE]
- SHUFFLE [RS_14]
- Group By Operator [GBY_12] (rows=2765/5 width=12)
- Output:["_col0","_col1"],aggregations:["count(VALUE._col0)"],keys:KEY._col0
+ SHUFFLE [RS_13]
+ Group By Operator [GBY_11] (rows=750/5 width=215)
+ Output:["_col0","_col1"],aggregations:["count()"],keys:KEY._col0
<-Reducer 2 [SIMPLE_EDGE]
- SHUFFLE [RS_11]
+ SHUFFLE [RS_10]
PartitionCols:_col0
- Group By Operator [GBY_10] (rows=2765/5 width=12)
- Output:["_col0","_col1"],aggregations:["count()"],keys:_col0
- Merge Join Operator [MERGEJOIN_21] (rows=7286/10 width=4)
- Conds:RS_6._col1=RS_7._col0(Inner),Output:["_col0"]
- <-Map 1 [SIMPLE_EDGE]
- SHUFFLE [RS_6]
- PartitionCols:_col1
- Select Operator [SEL_2] (rows=6144/10 width=5)
- Output:["_col0","_col1"]
- Filter Operator [FIL_19] (rows=6144/10 width=5)
- predicate:cint BETWEEN 1000000 AND 3000000
- TableScan [TS_0] (rows=12288/12288 width=5)
- default@alltypesorc,a,Tbl:COMPLETE,Col:COMPLETE,Output:["csmallint","cint"]
- <-Map 5 [SIMPLE_EDGE]
- SHUFFLE [RS_7]
- PartitionCols:_col0
- Select Operator [SEL_5] (rows=3424/10 width=8)
- Output:["_col0"]
- Filter Operator [FIL_20] (rows=3424/10 width=8)
- predicate:(cint is not null and cbigint is not null and cint BETWEEN 1000000 AND 3000000)
- TableScan [TS_3] (rows=12288/12288 width=8)
- default@alltypesorc,b,Tbl:COMPLETE,Col:COMPLETE,Output:["cint","cbigint"]
+ Merge Join Operator [MERGEJOIN_20] (rows=1501/10 width=215)
+ Conds:RS_6._col1=RS_7._col0(Inner),Output:["_col0"]
+ <-Map 1 [SIMPLE_EDGE]
+ SHUFFLE [RS_6]
+ PartitionCols:_col1
+ Select Operator [SEL_2] (rows=1365/10 width=215)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_18] (rows=1365/10 width=215)
+ predicate:cint BETWEEN 1000000 AND 3000000
+ TableScan [TS_0] (rows=12288/12288 width=215)
+ default@alltypesorc,a,Tbl:COMPLETE,Col:NONE,Output:["csmallint","cint"]
+ <-Map 5 [SIMPLE_EDGE]
+ SHUFFLE [RS_7]
+ PartitionCols:_col0
+ Select Operator [SEL_5] (rows=1365/10 width=215)
+ Output:["_col0"]
+ Filter Operator [FIL_19] (rows=1365/10 width=215)
+ predicate:(cint BETWEEN 1000000 AND 3000000 and cbigint is not null)
+ TableScan [TS_3] (rows=12288/12288 width=215)
+ default@alltypesorc,b,Tbl:COMPLETE,Col:NONE,Output:["cint","cbigint"]
PREHOOK: query: select
a.csmallint, count(*) c1
@@ -311,8 +305,7 @@ order by a.cint
POSTHOOK: type: QUERY
POSTHOOK: Input: default@alltypesorc
#### A masked pattern was here ####
-PREHOOK: query: -- Try with dynamically partitioned hashjoin
-explain analyze
+PREHOOK: query: explain analyze
select
*
from alltypesorc a join alltypesorc b on a.cint = b.cint
@@ -320,8 +313,7 @@ where
a.cint between 1000000 and 3000000 and b.cbigint is not null
order by a.cint
PREHOOK: type: QUERY
-POSTHOOK: query: -- Try with dynamically partitioned hashjoin
-explain analyze
+POSTHOOK: query: explain analyze
select
*
from alltypesorc a join alltypesorc b on a.cint = b.cint
@@ -341,27 +333,27 @@ Stage-0
Stage-1
Reducer 3
File Output Operator [FS_12]
- Select Operator [SEL_11] (rows=6758/10 width=215)
+ Select Operator [SEL_11] (rows=1501/10 width=215)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23"]
<-Reducer 2 [SIMPLE_EDGE]
SHUFFLE [RS_10]
- Map Join Operator [MAPJOIN_17] (rows=6758/10 width=215)
+ Map Join Operator [MAPJOIN_17] (rows=1501/10 width=215)
Conds:RS_6.KEY.reducesinkkey0=RS_7.KEY.reducesinkkey0(Inner),HybridGraceHashJoin:true,Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23"]
<-Map 4 [CUSTOM_SIMPLE_EDGE]
PARTITION_ONLY_SHUFFLE [RS_7]
PartitionCols:_col2
- Select Operator [SEL_5] (rows=6144/10 width=215)
+ Select Operator [SEL_5] (rows=1365/10 width=215)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"]
- Filter Operator [FIL_16] (rows=6144/10 width=215)
- predicate:(cint is not null and cbigint is not null and cint BETWEEN 1000000 AND 3000000)
+ Filter Operator [FIL_16] (rows=1365/10 width=215)
+ predicate:(cint BETWEEN 1000000 AND 3000000 and cbigint is not null)
TableScan [TS_3] (rows=12288/12288 width=215)
default@alltypesorc,b,Tbl:COMPLETE,Col:NONE,Output:["ctinyint","csmallint","cint","cbigint","cfloat","cdouble","cstring1","cstring2","ctimestamp1","ctimestamp2","cboolean1","cboolean2"]
<-Map 1 [CUSTOM_SIMPLE_EDGE]
PARTITION_ONLY_SHUFFLE [RS_6]
PartitionCols:_col2
- Select Operator [SEL_2] (rows=6144/10 width=215)
+ Select Operator [SEL_2] (rows=1365/10 width=215)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"]
- Filter Operator [FIL_15] (rows=6144/10 width=215)
+ Filter Operator [FIL_15] (rows=1365/10 width=215)
predicate:cint BETWEEN 1000000 AND 3000000
TableScan [TS_0] (rows=12288/12288 width=215)
default@alltypesorc,a,Tbl:COMPLETE,Col:NONE,Output:["ctinyint","csmallint","cint","cbigint","cfloat","cdouble","cstring1","cstring2","ctimestamp1","ctimestamp2","cboolean1","cboolean2"]
@@ -435,33 +427,31 @@ Stage-0
limit:-1
Stage-1
Reducer 3
- File Output Operator [FS_14]
- Group By Operator [GBY_12] (rows=1/1 width=8)
- Output:["_col0"],aggregations:["count(VALUE._col0)"]
+ File Output Operator [FS_13]
+ Group By Operator [GBY_11] (rows=1/1 width=8)
+ Output:["_col0"],aggregations:["count()"]
<-Reducer 2 [SIMPLE_EDGE]
- SHUFFLE [RS_11]
- Group By Operator [GBY_10] (rows=1/14 width=8)
- Output:["_col0"],aggregations:["count()"]
- Map Join Operator [MAPJOIN_19] (rows=6758/10 width=215)
- Conds:RS_6.KEY.reducesinkkey0=RS_7.KEY.reducesinkkey0(Inner),HybridGraceHashJoin:true
- <-Map 4 [CUSTOM_SIMPLE_EDGE]
- PARTITION_ONLY_SHUFFLE [RS_7]
- PartitionCols:_col0
- Select Operator [SEL_5] (rows=6144/10 width=215)
- Output:["_col0"]
- Filter Operator [FIL_18] (rows=6144/10 width=215)
- predicate:(cint is not null and cbigint is not null and cint BETWEEN 1000000 AND 3000000)
- TableScan [TS_3] (rows=12288/12288 width=215)
- default@alltypesorc,b,Tbl:COMPLETE,Col:NONE,Output:["cint","cbigint"]
- <-Map 1 [CUSTOM_SIMPLE_EDGE]
- PARTITION_ONLY_SHUFFLE [RS_6]
- PartitionCols:_col0
- Select Operator [SEL_2] (rows=6144/10 width=215)
- Output:["_col0"]
- Filter Operator [FIL_17] (rows=6144/10 width=215)
- predicate:cint BETWEEN 1000000 AND 3000000
- TableScan [TS_0] (rows=12288/12288 width=215)
- default@alltypesorc,a,Tbl:COMPLETE,Col:NONE,Output:["cint"]
+ SHUFFLE [RS_10]
+ Map Join Operator [MAPJOIN_18] (rows=1501/10 width=215)
+ Conds:RS_6.KEY.reducesinkkey0=RS_7.KEY.reducesinkkey0(Inner),HybridGraceHashJoin:true
+ <-Map 4 [CUSTOM_SIMPLE_EDGE]
+ PARTITION_ONLY_SHUFFLE [RS_7]
+ PartitionCols:_col0
+ Select Operator [SEL_5] (rows=1365/10 width=215)
+ Output:["_col0"]
+ Filter Operator [FIL_17] (rows=1365/10 width=215)
+ predicate:(cint BETWEEN 1000000 AND 3000000 and cbigint is not null)
+ TableScan [TS_3] (rows=12288/12288 width=215)
+ default@alltypesorc,b,Tbl:COMPLETE,Col:NONE,Output:["cint","cbigint"]
+ <-Map 1 [CUSTOM_SIMPLE_EDGE]
+ PARTITION_ONLY_SHUFFLE [RS_6]
+ PartitionCols:_col0
+ Select Operator [SEL_2] (rows=1365/10 width=215)
+ Output:["_col0"]
+ Filter Operator [FIL_16] (rows=1365/10 width=215)
+ predicate:cint BETWEEN 1000000 AND 3000000
+ TableScan [TS_0] (rows=12288/12288 width=215)
+ default@alltypesorc,a,Tbl:COMPLETE,Col:NONE,Output:["cint"]
PREHOOK: query: select
count(*)
@@ -530,38 +520,36 @@ Stage-0
limit:-1
Stage-1
Reducer 4
- File Output Operator [FS_16]
- Select Operator [SEL_15] (rows=3379/5 width=215)
+ File Output Operator [FS_15]
+ Select Operator [SEL_14] (rows=750/5 width=215)
Output:["_col0","_col1"]
<-Reducer 3 [SIMPLE_EDGE]
- SHUFFLE [RS_14]
- Group By Operator [GBY_12] (rows=3379/5 width=215)
- Output:["_col0","_col1"],aggregations:["count(VALUE._col0)"],keys:KEY._col0
+ SHUFFLE [RS_13]
+ Group By Operator [GBY_11] (rows=750/5 width=215)
+ Output:["_col0","_col1"],aggregations:["count()"],keys:KEY._col0
<-Reducer 2 [SIMPLE_EDGE]
- SHUFFLE [RS_11]
+ SHUFFLE [RS_10]
PartitionCols:_col0
- Group By Operator [GBY_10] (rows=6758/9 width=215)
- Output:["_col0","_col1"],aggregations:["count()"],keys:_col0
- Map Join Operator [MAPJOIN_21] (rows=6758/10 width=215)
- Conds:RS_6.KEY.reducesinkkey0=RS_7.KEY.reducesinkkey0(Inner),HybridGraceHashJoin:true,Output:["_col0"]
- <-Map 5 [CUSTOM_SIMPLE_EDGE]
- PARTITION_ONLY_SHUFFLE [RS_7]
- PartitionCols:_col0
- Select Operator [SEL_5] (rows=6144/10 width=215)
- Output:["_col0"]
- Filter Operator [FIL_20] (rows=6144/10 width=215)
- predicate:(cint is not null and cbigint is not null and cint BETWEEN 1000000 AND 3000000)
- TableScan [TS_3] (rows=12288/12288 width=215)
- default@alltypesorc,b,Tbl:COMPLETE,Col:NONE,Output:["cint","cbigint"]
- <-Map 1 [CUSTOM_SIMPLE_EDGE]
- PARTITION_ONLY_SHUFFLE [RS_6]
- PartitionCols:_col1
- Select Operator [SEL_2] (rows=6144/10 width=215)
- Output:["_col0","_col1"]
- Filter Operator [FIL_19] (rows=6144/10 width=215)
- predicate:cint BETWEEN 1000000 AND 3000000
- TableScan [TS_0] (rows=12288/12288 width=215)
- default@alltypesorc,a,Tbl:COMPLETE,Col:NONE,Output:["csmallint","cint"]
+ Map Join Operator [MAPJOIN_20] (rows=1501/10 width=215)
+ Conds:RS_6.KEY.reducesinkkey0=RS_7.KEY.reducesinkkey0(Inner),HybridGraceHashJoin:true,Output:["_col0"]
+ <-Map 5 [CUSTOM_SIMPLE_EDGE]
+ PARTITION_ONLY_SHUFFLE [RS_7]
+ PartitionCols:_col0
+ Select Operator [SEL_5] (rows=1365/10 width=215)
+ Output:["_col0"]
+ Filter Operator [FIL_19] (rows=1365/10 width=215)
+ predicate:(cint BETWEEN 1000000 AND 3000000 and cbigint is not null)
+ TableScan [TS_3] (rows=12288/12288 width=215)
+ default@alltypesorc,b,Tbl:COMPLETE,Col:NONE,Output:["cint","cbigint"]
+ <-Map 1 [CUSTOM_SIMPLE_EDGE]
+ PARTITION_ONLY_SHUFFLE [RS_6]
+ PartitionCols:_col1
+ Select Operator [SEL_2] (rows=1365/10 width=215)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_18] (rows=1365/10 width=215)
+ predicate:cint BETWEEN 1000000 AND 3000000
+ TableScan [TS_0] (rows=12288/12288 width=215)
+ default@alltypesorc,a,Tbl:COMPLETE,Col:NONE,Output:["csmallint","cint"]
PREHOOK: query: select
a.csmallint, count(*) c1
@@ -583,8 +571,8 @@ order by c1
POSTHOOK: type: QUERY
POSTHOOK: Input: default@alltypesorc
#### A masked pattern was here ####
+-13036 1
-8915 1
-3799 1
10782 1
--13036 1
NULL 6
http://git-wip-us.apache.org/repos/asf/hive/blob/9f5bea3b/ql/src/test/results/clientpositive/tez/explainanalyze_5.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/explainanalyze_5.q.out b/ql/src/test/results/clientpositive/tez/explainanalyze_5.q.out
index 39bc6f4..65f7b06 100644
--- a/ql/src/test/results/clientpositive/tez/explainanalyze_5.q.out
+++ b/ql/src/test/results/clientpositive/tez/explainanalyze_5.q.out
@@ -1,33 +1,49 @@
-PREHOOK: query: analyze table src compute statistics
-PREHOOK: type: QUERY
+PREHOOK: query: drop table src_stats
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table src_stats
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table src_stats as select * from src
+PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: default@src
-PREHOOK: Output: default@src
-FAILED: Hive Internal Error: java.lang.RuntimeException(Cannot overwrite read-only table: src)
-java.lang.RuntimeException: Cannot overwrite read-only table: src
-#### A masked pattern was here ####
-
-PREHOOK: query: explain analyze analyze table src compute statistics
+PREHOOK: Output: database:default
+PREHOOK: Output: default@src_stats
+POSTHOOK: query: create table src_stats as select * from src
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@src
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@src_stats
+POSTHOOK: Lineage: src_stats.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: src_stats.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: analyze table src_stats compute statistics
PREHOOK: type: QUERY
-POSTHOOK: query: explain analyze analyze table src compute statistics
+PREHOOK: Input: default@src_stats
+PREHOOK: Output: default@src_stats
+POSTHOOK: query: analyze table src_stats compute statistics
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_stats
+POSTHOOK: Output: default@src_stats
+PREHOOK: query: explain analyze analyze table src_stats compute statistics
+PREHOOK: type: QUERY
+POSTHOOK: query: explain analyze analyze table src_stats compute statistics
POSTHOOK: type: QUERY
Stage-2
Stats-Aggr Operator
Stage-0
Map 1
- TableScan [TS_0] (rows=500/0 width=10)
- default@src,src,Tbl:COMPLETE,Col:COMPLETE
+ TableScan [TS_0] (rows=500/500 width=10)
+ default@src_stats,src_stats,Tbl:COMPLETE,Col:COMPLETE
-PREHOOK: query: analyze table src compute statistics for columns
+PREHOOK: query: analyze table src_stats compute statistics for columns
PREHOOK: type: QUERY
-PREHOOK: Input: default@src
+PREHOOK: Input: default@src_stats
#### A masked pattern was here ####
-POSTHOOK: query: analyze table src compute statistics for columns
+POSTHOOK: query: analyze table src_stats compute statistics for columns
POSTHOOK: type: QUERY
-POSTHOOK: Input: default@src
+POSTHOOK: Input: default@src_stats
#### A masked pattern was here ####
-PREHOOK: query: explain analyze analyze table src compute statistics for columns
+PREHOOK: query: explain analyze analyze table src_stats compute statistics for columns
PREHOOK: type: QUERY
-POSTHOOK: query: explain analyze analyze table src compute statistics for columns
+POSTHOOK: query: explain analyze analyze table src_stats compute statistics for columns
POSTHOOK: type: QUERY
Vertex dependency in root stage
Reducer 2 <- Map 1 (SIMPLE_EDGE)
@@ -36,17 +52,15 @@ Stage-2
Column Stats Work{}
Stage-0
Reducer 2
- File Output Operator [FS_6]
- Group By Operator [GBY_4] (rows=1/1 width=960)
- Output:["_col0","_col1"],aggregations:["compute_stats(VALUE._col0)","compute_stats(VALUE._col1)"]
+ File Output Operator [FS_5]
+ Group By Operator [GBY_3] (rows=1/1 width=960)
+ Output:["_col0","_col1"],aggregations:["compute_stats(VALUE._col0, 16)","compute_stats(VALUE._col2, 16)"]
<-Map 1 [SIMPLE_EDGE]
- SHUFFLE [RS_3]
- Group By Operator [GBY_2] (rows=1/1 width=984)
- Output:["_col0","_col1"],aggregations:["compute_stats(key, 16)","compute_stats(value, 16)"]
- Select Operator [SEL_1] (rows=500/500 width=178)
- Output:["key","value"]
- TableScan [TS_0] (rows=500/500 width=178)
- default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
+ SHUFFLE [RS_2]
+ Select Operator [SEL_1] (rows=500/500 width=10)
+ Output:["key","value"]
+ TableScan [TS_0] (rows=500/500 width=10)
+ default@src_stats,src_stats,Tbl:COMPLETE,Col:NONE,Output:["key","value"]
PREHOOK: query: drop table src_multi2
PREHOOK: type: DROPTABLE
@@ -94,61 +108,55 @@ Stage-4
Dependency Collection{}
Stage-1
Reducer 5
- File Output Operator [FS_6]
- Group By Operator [GBY_4] (rows=1/1 width=960)
- Output:["_col0","_col1"],aggregations:["compute_stats(VALUE._col0)","compute_stats(VALUE._col1)"]
+ File Output Operator [FS_5]
+ Group By Operator [GBY_3] (rows=1/1 width=960)
+ Output:["_col0","_col1"],aggregations:["compute_stats(VALUE._col0, 16)","compute_stats(VALUE._col2, 16)"]
<-Reducer 4 [SIMPLE_EDGE]
- File Output Operator [FS_20]
+ File Output Operator [FS_19]
table:{"name:":"default.src_multi2"}
- Select Operator [SEL_19] (rows=639/508 width=178)
+ Select Operator [SEL_18] (rows=550/508 width=10)
Output:["_col0","_col1"]
- Merge Join Operator [MERGEJOIN_27] (rows=639/508 width=178)
- Conds:RS_16._col0=RS_17._col0(Inner),Output:["_col0","_col3"]
+ Merge Join Operator [MERGEJOIN_26] (rows=550/508 width=10)
+ Conds:RS_15._col0=RS_16._col0(Inner),Output:["_col0","_col3"]
<-Map 7 [SIMPLE_EDGE]
- SHUFFLE [RS_17]
+ SHUFFLE [RS_16]
PartitionCols:_col0
- Select Operator [SEL_15] (rows=500/500 width=178)
+ Select Operator [SEL_14] (rows=500/500 width=10)
Output:["_col0","_col1"]
- Filter Operator [FIL_26] (rows=500/500 width=178)
+ Filter Operator [FIL_25] (rows=500/500 width=10)
predicate:key is not null
- TableScan [TS_13] (rows=500/500 width=178)
- default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
+ TableScan [TS_12] (rows=500/500 width=10)
+ default@src,src,Tbl:COMPLETE,Col:NONE,Output:["key","value"]
<-Reducer 3 [SIMPLE_EDGE]
- SHUFFLE [RS_16]
+ SHUFFLE [RS_15]
PartitionCols:_col0
- Select Operator [SEL_12] (rows=262/319 width=178)
+ Select Operator [SEL_11] (rows=262/319 width=10)
Output:["_col0"]
- Group By Operator [GBY_11] (rows=262/319 width=178)
+ Group By Operator [GBY_10] (rows=262/319 width=10)
Output:["_col0","_col1"],keys:KEY._col0, KEY._col1
<-Union 2 [SIMPLE_EDGE]
<-Map 1 [CONTAINS]
- Reduce Output Operator [RS_10]
+ Reduce Output Operator [RS_9]
PartitionCols:_col0, _col1
- Group By Operator [GBY_9] (rows=262/331 width=178)
- Output:["_col0","_col1"],keys:_col0, _col1
- Select Operator [SEL_2] (rows=500/500 width=178)
- Output:["_col0","_col1"]
- Filter Operator [FIL_24] (rows=500/500 width=178)
- predicate:key is not null
- TableScan [TS_0] (rows=500/500 width=178)
- Output:["key","value"]
+ Select Operator [SEL_2] (rows=500/500 width=10)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_23] (rows=500/500 width=10)
+ predicate:key is not null
+ TableScan [TS_0] (rows=500/500 width=10)
+ Output:["key","value"]
<-Map 6 [CONTAINS]
- Reduce Output Operator [RS_10]
+ Reduce Output Operator [RS_9]
PartitionCols:_col0, _col1
- Group By Operator [GBY_9] (rows=262/331 width=178)
- Output:["_col0","_col1"],keys:_col0, _col1
- Select Operator [SEL_5] (rows=25/25 width=175)
- Output:["_col0","_col1"]
- Filter Operator [FIL_25] (rows=25/25 width=175)
- predicate:key is not null
- TableScan [TS_3] (rows=25/25 width=175)
- Output:["key","value"]
- SHUFFLE [RS_3]
- Group By Operator [GBY_2] (rows=1/1 width=984)
- Output:["_col0","_col1"],aggregations:["compute_stats(key, 16)","compute_stats(value, 16)"]
- Select Operator [SEL_1] (rows=639/508 width=178)
- Output:["key","value"]
- Please refer to the previous Select Operator [SEL_19]
+ Select Operator [SEL_5] (rows=25/25 width=7)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_24] (rows=25/25 width=7)
+ predicate:key is not null
+ TableScan [TS_3] (rows=25/25 width=7)
+ Output:["key","value"]
+ SHUFFLE [RS_2]
+ Select Operator [SEL_1] (rows=550/508 width=10)
+ Output:["key","value"]
+ Please refer to the previous Select Operator [SEL_18]
PREHOOK: query: select count(*) from (select * from src union select * from src1)subq
PREHOOK: type: QUERY
@@ -208,17 +216,13 @@ Bucket Columns: []
Sort Columns: []
Storage Desc Params:
serialization.format 1
-PREHOOK: query: -- SORT_QUERY_RESULTS
-
-create table acid_uami(i int,
+PREHOOK: query: create table acid_uami(i int,
de decimal(5,2),
vc varchar(128)) clustered by (i) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true')
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@acid_uami
-POSTHOOK: query: -- SORT_QUERY_RESULTS
-
-create table acid_uami(i int,
+POSTHOOK: query: create table acid_uami(i int,
de decimal(5,2),
vc varchar(128)) clustered by (i) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true')
POSTHOOK: type: CREATETABLE
@@ -292,17 +296,17 @@ Stage-3
Reducer 2
File Output Operator [FS_8]
table:{"name:":"default.acid_uami"}
- Select Operator [SEL_4] (rows=6/2 width=302)
+ Select Operator [SEL_4] (rows=8 width=245)
Output:["_col0","_col1","_col2","_col3"]
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_3]
PartitionCols:UDFToInteger(_col0)
- Select Operator [SEL_2] (rows=6/2 width=302)
+ Select Operator [SEL_2] (rows=8 width=245)
Output:["_col0","_col1","_col3"]
- Filter Operator [FIL_9] (rows=6/2 width=226)
+ Filter Operator [FIL_9] (rows=8 width=245)
predicate:((de = 109.23) or (de = 119.23))
- TableScan [TS_0] (rows=8/4 width=226)
- default@acid_uami,acid_uami, ACID table,Tbl:COMPLETE,Col:COMPLETE,Output:["i","de","vc"]
+ TableScan [TS_0] (rows=8 width=245)
+ default@acid_uami,acid_uami, ACID table,Tbl:COMPLETE,Col:NONE,Output:["i","de","vc"]
PREHOOK: query: select * from acid_uami order by de
PREHOOK: type: QUERY
@@ -405,16 +409,16 @@ Stage-3
Reducer 2
File Output Operator [FS_7]
table:{"name:":"default.acid_dot"}
- Select Operator [SEL_4] (rows=31436/8 width=4)
+ Select Operator [SEL_4] (rows=31436 width=4)
Output:["_col0"]
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_3]
PartitionCols:UDFToInteger(_col0)
- Select Operator [SEL_2] (rows=31436/8 width=4)
+ Select Operator [SEL_2] (rows=31436 width=4)
Output:["_col0"]
- Filter Operator [FIL_8] (rows=31436/8 width=4)
+ Filter Operator [FIL_8] (rows=31436 width=4)
predicate:(cint < -1070551679)
- TableScan [TS_0] (rows=94309/12288 width=4)
+ TableScan [TS_0] (rows=94309 width=4)
default@acid_dot,acid_dot, ACID table,Tbl:COMPLETE,Col:NONE,Output:["cint"]
PREHOOK: query: select count(*) from acid_dot
http://git-wip-us.apache.org/repos/asf/hive/blob/9f5bea3b/ql/src/test/results/clientpositive/tez/explainuser_2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/explainuser_2.q.out b/ql/src/test/results/clientpositive/tez/explainuser_2.q.out
index a4d538b..68b5ba9 100644
--- a/ql/src/test/results/clientpositive/tez/explainuser_2.q.out
+++ b/ql/src/test/results/clientpositive/tez/explainuser_2.q.out
@@ -3911,13 +3911,13 @@ Stage-4
PartitionCols:_col0
Group By Operator [GBY_17] (rows=1 width=280)
Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT substr(_col1, 5))"],keys:_col0, substr(_col1, 5)
- Group By Operator [GBY_14] (rows=1 width=172)
+ Group By Operator [GBY_14] (rows=1 width=272)
Output:["_col0","_col1"],keys:KEY._col0, KEY._col1
<-Union 3 [SIMPLE_EDGE]
<-Map 6 [CONTAINS]
Reduce Output Operator [RS_13]
PartitionCols:_col0, _col1
- Group By Operator [GBY_12] (rows=1 width=172)
+ Group By Operator [GBY_12] (rows=1 width=272)
Output:["_col0","_col1"],keys:_col0, _col1
Select Operator [SEL_8] (rows=500 width=10)
Output:["_col0","_col1"]
@@ -3926,9 +3926,9 @@ Stage-4
<-Reducer 2 [CONTAINS]
Reduce Output Operator [RS_13]
PartitionCols:_col0, _col1
- Group By Operator [GBY_12] (rows=1 width=172)
+ Group By Operator [GBY_12] (rows=1 width=272)
Output:["_col0","_col1"],keys:_col0, _col1
- Select Operator [SEL_6] (rows=1 width=172)
+ Select Operator [SEL_6] (rows=1 width=272)
Output:["_col0","_col1"]
Group By Operator [GBY_5] (rows=1 width=8)
Output:["_col0"],aggregations:["count(VALUE._col0)"]
@@ -3941,9 +3941,9 @@ Stage-4
default@src,s1,Tbl:COMPLETE,Col:COMPLETE
File Output Operator [FS_27]
table:{"name:":"default.dest2"}
- Select Operator [SEL_26] (rows=1 width=356)
+ Select Operator [SEL_26] (rows=1 width=456)
Output:["_col0","_col1","_col2"]
- Group By Operator [GBY_25] (rows=1 width=364)
+ Group By Operator [GBY_25] (rows=1 width=464)
Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT substr(_col1, 5))"],keys:_col0, _col1
Please refer to the previous Group By Operator [GBY_14]
Stage-5
@@ -4090,7 +4090,7 @@ Stage-4
Output:["key","value"]
Reduce Output Operator [RS_22]
PartitionCols:_col0, _col1
- Group By Operator [GBY_21] (rows=1 width=364)
+ Group By Operator [GBY_21] (rows=1 width=464)
Output:["_col0","_col1","_col2","_col3"],aggregations:["count(DISTINCT substr(_col1, 5))"],keys:_col0, _col1, substr(_col1, 5)
Please refer to the previous Select Operator [SEL_8]
<-Map 7 [CONTAINS]
@@ -4104,7 +4104,7 @@ Stage-4
Output:["key","value"]
Reduce Output Operator [RS_22]
PartitionCols:_col0, _col1
- Group By Operator [GBY_21] (rows=1 width=364)
+ Group By Operator [GBY_21] (rows=1 width=464)
Output:["_col0","_col1","_col2","_col3"],aggregations:["count(DISTINCT substr(_col1, 5))"],keys:_col0, _col1, substr(_col1, 5)
Please refer to the previous Select Operator [SEL_12]
<-Reducer 2 [CONTAINS]
@@ -4112,7 +4112,7 @@ Stage-4
PartitionCols:_col0
Group By Operator [GBY_15] (rows=1 width=280)
Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT substr(_col1, 5))"],keys:_col0, substr(_col1, 5)
- Select Operator [SEL_6] (rows=1 width=172)
+ Select Operator [SEL_6] (rows=1 width=272)
Output:["_col0","_col1"]
Group By Operator [GBY_5] (rows=1 width=8)
Output:["_col0"],aggregations:["count(VALUE._col0)"]
@@ -4125,13 +4125,13 @@ Stage-4
default@src,s1,Tbl:COMPLETE,Col:COMPLETE
Reduce Output Operator [RS_22]
PartitionCols:_col0, _col1
- Group By Operator [GBY_21] (rows=1 width=364)
+ Group By Operator [GBY_21] (rows=1 width=464)
Output:["_col0","_col1","_col2","_col3"],aggregations:["count(DISTINCT substr(_col1, 5))"],keys:_col0, _col1, substr(_col1, 5)
Please refer to the previous Select Operator [SEL_6]
Reducer 5
File Output Operator [FS_25]
table:{"name:":"default.dest2"}
- Group By Operator [GBY_23] (rows=1 width=180)
+ Group By Operator [GBY_23] (rows=1 width=280)
Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT KEY._col2:0._col0)"],keys:KEY._col0, KEY._col1
<- Please refer to the previous Union 3 [SIMPLE_EDGE]
Stage-5
@@ -4194,7 +4194,7 @@ Stage-4
Output:["key","value"]
Reduce Output Operator [RS_19]
PartitionCols:_col0, _col1
- Group By Operator [GBY_18] (rows=1 width=364)
+ Group By Operator [GBY_18] (rows=1 width=464)
Output:["_col0","_col1","_col2","_col3"],aggregations:["count(DISTINCT substr(_col1, 5))"],keys:_col0, _col1, substr(_col1, 5)
Please refer to the previous Select Operator [SEL_8]
<-Reducer 2 [CONTAINS]
@@ -4202,7 +4202,7 @@ Stage-4
PartitionCols:_col0
Group By Operator [GBY_12] (rows=1 width=280)
Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT substr(_col1, 5))"],keys:_col0, substr(_col1, 5)
- Select Operator [SEL_6] (rows=1 width=172)
+ Select Operator [SEL_6] (rows=1 width=272)
Output:["_col0","_col1"]
Group By Operator [GBY_5] (rows=1 width=8)
Output:["_col0"],aggregations:["count(VALUE._col0)"]
@@ -4215,13 +4215,13 @@ Stage-4
default@src,s1,Tbl:COMPLETE,Col:COMPLETE
Reduce Output Operator [RS_19]
PartitionCols:_col0, _col1
- Group By Operator [GBY_18] (rows=1 width=364)
+ Group By Operator [GBY_18] (rows=1 width=464)
Output:["_col0","_col1","_col2","_col3"],aggregations:["count(DISTINCT substr(_col1, 5))"],keys:_col0, _col1, substr(_col1, 5)
Please refer to the previous Select Operator [SEL_6]
Reducer 5
File Output Operator [FS_22]
table:{"name:":"default.dest2"}
- Group By Operator [GBY_20] (rows=1 width=180)
+ Group By Operator [GBY_20] (rows=1 width=280)
Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT KEY._col2:0._col0)"],keys:KEY._col0, KEY._col1
<- Please refer to the previous Union 3 [SIMPLE_EDGE]
Stage-5
http://git-wip-us.apache.org/repos/asf/hive/blob/9f5bea3b/ql/src/test/results/clientpositive/tez/mergejoin.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/mergejoin.q.out b/ql/src/test/results/clientpositive/tez/mergejoin.q.out
index e6b81ca..5853b2b 100644
--- a/ql/src/test/results/clientpositive/tez/mergejoin.q.out
+++ b/ql/src/test/results/clientpositive/tez/mergejoin.q.out
@@ -64,10 +64,10 @@ STAGE PLANS:
aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=25)
mode: hash
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary)
Reducer 2
Reduce Operator Tree:
@@ -93,10 +93,10 @@ STAGE PLANS:
aggregations: min(VALUE._col0), max(VALUE._col1), bloom_filter(VALUE._col2, expectedEntries=25)
mode: final
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary)
Stage: Stage-0
@@ -1860,10 +1860,10 @@ STAGE PLANS:
aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=25)
mode: hash
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary)
Reducer 2
Reduce Operator Tree:
@@ -1932,10 +1932,10 @@ STAGE PLANS:
aggregations: min(VALUE._col0), max(VALUE._col1), bloom_filter(VALUE._col2, expectedEntries=25)
mode: final
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary)
Stage: Stage-0
@@ -2006,10 +2006,10 @@ STAGE PLANS:
aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=242)
mode: hash
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary)
Execution mode: vectorized
Map 5
@@ -2071,10 +2071,10 @@ STAGE PLANS:
aggregations: min(VALUE._col0), max(VALUE._col1), bloom_filter(VALUE._col2, expectedEntries=242)
mode: final
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary)
Stage: Stage-0
@@ -2343,10 +2343,10 @@ STAGE PLANS:
aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=242)
mode: hash
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary)
Execution mode: vectorized
Map 5
@@ -2408,10 +2408,10 @@ STAGE PLANS:
aggregations: min(VALUE._col0), max(VALUE._col1), bloom_filter(VALUE._col2, expectedEntries=242)
mode: final
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary)
Stage: Stage-0
@@ -2535,10 +2535,10 @@ STAGE PLANS:
aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=25)
mode: hash
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary)
Reducer 2
Reduce Operator Tree:
@@ -2607,10 +2607,10 @@ STAGE PLANS:
aggregations: min(VALUE._col0), max(VALUE._col1), bloom_filter(VALUE._col2, expectedEntries=25)
mode: final
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary)
Stage: Stage-0
http://git-wip-us.apache.org/repos/asf/hive/blob/9f5bea3b/ql/src/test/results/clientpositive/tez/tez_union_multiinsert.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/tez_union_multiinsert.q.out b/ql/src/test/results/clientpositive/tez/tez_union_multiinsert.q.out
index ddc56ce..c69c346 100644
--- a/ql/src/test/results/clientpositive/tez/tez_union_multiinsert.q.out
+++ b/ql/src/test/results/clientpositive/tez/tez_union_multiinsert.q.out
@@ -101,12 +101,12 @@ STAGE PLANS:
keys: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 1 Data size: 364 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 464 Basic stats: COMPLETE Column stats: PARTIAL
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string)
sort order: +++
Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
- Statistics: Num rows: 1 Data size: 364 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 464 Basic stats: COMPLETE Column stats: PARTIAL
Map 7
Map Operator Tree:
TableScan
@@ -132,12 +132,12 @@ STAGE PLANS:
keys: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 1 Data size: 364 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 464 Basic stats: COMPLETE Column stats: PARTIAL
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string)
sort order: +++
Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
- Statistics: Num rows: 1 Data size: 364 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 464 Basic stats: COMPLETE Column stats: PARTIAL
Reducer 2
Reduce Operator Tree:
Group By Operator
@@ -148,7 +148,7 @@ STAGE PLANS:
Select Operator
expressions: 'tst1' (type: string), UDFToString(_col0) (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: count(DISTINCT substr(_col1, 5))
keys: _col0 (type: string), substr(_col1, 5) (type: string)
@@ -165,12 +165,12 @@ STAGE PLANS:
keys: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 1 Data size: 364 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 464 Basic stats: COMPLETE Column stats: PARTIAL
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string)
sort order: +++
Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
- Statistics: Num rows: 1 Data size: 364 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 464 Basic stats: COMPLETE Column stats: PARTIAL
Reducer 4
Reduce Operator Tree:
Group By Operator
@@ -194,10 +194,10 @@ STAGE PLANS:
keys: KEY._col0 (type: string), KEY._col1 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 180 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 280 Basic stats: COMPLETE Column stats: PARTIAL
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 180 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 280 Basic stats: COMPLETE Column stats: PARTIAL
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -975,12 +975,12 @@ STAGE PLANS:
keys: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 1 Data size: 364 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 464 Basic stats: COMPLETE Column stats: PARTIAL
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string)
sort order: +++
Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
- Statistics: Num rows: 1 Data size: 364 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 464 Basic stats: COMPLETE Column stats: PARTIAL
Map 5
Map Operator Tree:
TableScan
@@ -1022,12 +1022,12 @@ STAGE PLANS:
keys: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 1 Data size: 364 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 464 Basic stats: COMPLETE Column stats: PARTIAL
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string)
sort order: +++
Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
- Statistics: Num rows: 1 Data size: 364 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 464 Basic stats: COMPLETE Column stats: PARTIAL
Reducer 3
Reduce Operator Tree:
Group By Operator
@@ -1051,10 +1051,10 @@ STAGE PLANS:
keys: KEY._col0 (type: string), KEY._col1 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 180 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 280 Basic stats: COMPLETE Column stats: PARTIAL
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 180 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 280 Basic stats: COMPLETE Column stats: PARTIAL
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -1070,7 +1070,7 @@ STAGE PLANS:
Select Operator
expressions: 'tst1' (type: string), UDFToString(_col0) (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: count(DISTINCT substr(_col1, 5))
keys: _col0 (type: string), substr(_col1, 5) (type: string)
@@ -1087,12 +1087,12 @@ STAGE PLANS:
keys: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 1 Data size: 364 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 464 Basic stats: COMPLETE Column stats: PARTIAL
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string)
sort order: +++
Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
- Statistics: Num rows: 1 Data size: 364 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 464 Basic stats: COMPLETE Column stats: PARTIAL
Union 2
Vertex: Union 2
@@ -1861,12 +1861,12 @@ STAGE PLANS:
keys: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 1 Data size: 364 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 464 Basic stats: COMPLETE Column stats: PARTIAL
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string)
sort order: +++
Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
- Statistics: Num rows: 1 Data size: 364 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 464 Basic stats: COMPLETE Column stats: PARTIAL
Map 5
Map Operator Tree:
TableScan
@@ -1908,12 +1908,12 @@ STAGE PLANS:
keys: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 1 Data size: 364 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 464 Basic stats: COMPLETE Column stats: PARTIAL
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string)
sort order: +++
Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
- Statistics: Num rows: 1 Data size: 364 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 464 Basic stats: COMPLETE Column stats: PARTIAL
Reducer 3
Reduce Operator Tree:
Group By Operator
@@ -1937,10 +1937,10 @@ STAGE PLANS:
keys: KEY._col0 (type: string), KEY._col1 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 180 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 280 Basic stats: COMPLETE Column stats: PARTIAL
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 180 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 280 Basic stats: COMPLETE Column stats: PARTIAL
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -1956,7 +1956,7 @@ STAGE PLANS:
Select Operator
expressions: 'tst1' (type: string), UDFToString(_col0) (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: count(DISTINCT substr(_col1, 5))
keys: _col0 (type: string), substr(_col1, 5) (type: string)
@@ -1973,12 +1973,12 @@ STAGE PLANS:
keys: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 1 Data size: 364 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 464 Basic stats: COMPLETE Column stats: PARTIAL
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string)
sort order: +++
Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
- Statistics: Num rows: 1 Data size: 364 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 464 Basic stats: COMPLETE Column stats: PARTIAL
Union 2
Vertex: Union 2
@@ -2754,12 +2754,12 @@ STAGE PLANS:
keys: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 1 Data size: 364 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 464 Basic stats: COMPLETE Column stats: PARTIAL
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string)
sort order: +++
Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
- Statistics: Num rows: 1 Data size: 364 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 464 Basic stats: COMPLETE Column stats: PARTIAL
Reducer 2
Reduce Operator Tree:
Group By Operator
@@ -2770,7 +2770,7 @@ STAGE PLANS:
Select Operator
expressions: 'tst1' (type: string), UDFToString(_col0) (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: count(DISTINCT substr(_col1, 5))
keys: _col0 (type: string), substr(_col1, 5) (type: string)
@@ -2787,12 +2787,12 @@ STAGE PLANS:
keys: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 1 Data size: 364 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 464 Basic stats: COMPLETE Column stats: PARTIAL
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string)
sort order: +++
Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
- Statistics: Num rows: 1 Data size: 364 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 464 Basic stats: COMPLETE Column stats: PARTIAL
Reducer 4
Reduce Operator Tree:
Group By Operator
@@ -2816,10 +2816,10 @@ STAGE PLANS:
keys: KEY._col0 (type: string), KEY._col1 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 180 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 280 Basic stats: COMPLETE Column stats: PARTIAL
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 180 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 280 Basic stats: COMPLETE Column stats: PARTIAL
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -3582,12 +3582,12 @@ STAGE PLANS:
keys: _col0 (type: string), _col1 (type: string)
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string)
sort order: +++
Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL
Reducer 2
Reduce Operator Tree:
Group By Operator
@@ -3598,24 +3598,24 @@ STAGE PLANS:
Select Operator
expressions: 'tst1' (type: string), UDFToString(_col0) (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
keys: _col0 (type: string), _col1 (type: string)
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string)
sort order: +++
Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL
Reducer 4
Reduce Operator Tree:
Group By Operator
keys: KEY._col0 (type: string), KEY._col1 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL
Group By Operator
aggregations: count(DISTINCT substr(_col1, 5))
keys: _col0 (type: string), substr(_col1, 5) (type: string)
@@ -3632,14 +3632,14 @@ STAGE PLANS:
keys: _col0 (type: string), _col1 (type: string)
mode: complete
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 364 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 464 Basic stats: COMPLETE Column stats: PARTIAL
Select Operator
expressions: _col0 (type: string), _col1 (type: string), _col2 (type: bigint)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 356 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 456 Basic stats: COMPLETE Column stats: PARTIAL
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 356 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 456 Basic stats: COMPLETE Column stats: PARTIAL
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/9f5bea3b/ql/src/test/results/clientpositive/tez/union6.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/union6.q.out b/ql/src/test/results/clientpositive/tez/union6.q.out
index 3d8a5ca..dc99bde 100644
--- a/ql/src/test/results/clientpositive/tez/union6.q.out
+++ b/ql/src/test/results/clientpositive/tez/union6.q.out
@@ -60,7 +60,7 @@ STAGE PLANS:
Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 26 Data size: 363 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 26 Data size: 463 Basic stats: COMPLETE Column stats: PARTIAL
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -76,10 +76,10 @@ STAGE PLANS:
Select Operator
expressions: 'tst1' (type: string), UDFToString(_col0) (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 26 Data size: 363 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 26 Data size: 463 Basic stats: COMPLETE Column stats: PARTIAL
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/9f5bea3b/ql/src/test/results/clientpositive/union17.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/union17.q.out b/ql/src/test/results/clientpositive/union17.q.out
index 91ff95f..5c2e6ac 100644
--- a/ql/src/test/results/clientpositive/union17.q.out
+++ b/ql/src/test/results/clientpositive/union17.q.out
@@ -64,7 +64,7 @@ STAGE PLANS:
Select Operator
expressions: 'tst1' (type: string), UDFToString(_col0) (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
table:
@@ -77,7 +77,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
Union
- Statistics: Num rows: 501 Data size: 5484 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 501 Data size: 5584 Basic stats: COMPLETE Column stats: PARTIAL
Group By Operator
aggregations: count(DISTINCT substr(_col1, 5))
keys: _col0 (type: string), substr(_col1, 5) (type: string)
@@ -94,7 +94,7 @@ STAGE PLANS:
keys: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 1 Data size: 364 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 464 Basic stats: COMPLETE Column stats: PARTIAL
File Output Operator
compressed: false
table:
@@ -109,7 +109,7 @@ STAGE PLANS:
outputColumnNames: _col0, _col1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Union
- Statistics: Num rows: 501 Data size: 5484 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 501 Data size: 5584 Basic stats: COMPLETE Column stats: PARTIAL
Group By Operator
aggregations: count(DISTINCT substr(_col1, 5))
keys: _col0 (type: string), substr(_col1, 5) (type: string)
@@ -126,7 +126,7 @@ STAGE PLANS:
keys: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 1 Data size: 364 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 464 Basic stats: COMPLETE Column stats: PARTIAL
File Output Operator
compressed: false
table:
@@ -170,17 +170,17 @@ STAGE PLANS:
key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string)
sort order: +++
Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
- Statistics: Num rows: 1 Data size: 364 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 464 Basic stats: COMPLETE Column stats: PARTIAL
Reduce Operator Tree:
Group By Operator
aggregations: count(DISTINCT KEY._col2:0._col0)
keys: KEY._col0 (type: string), KEY._col1 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 180 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 280 Basic stats: COMPLETE Column stats: PARTIAL
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 180 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 280 Basic stats: COMPLETE Column stats: PARTIAL
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/9f5bea3b/ql/src/test/results/clientpositive/union18.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/union18.q.out b/ql/src/test/results/clientpositive/union18.q.out
index 146387f..caa664c 100644
--- a/ql/src/test/results/clientpositive/union18.q.out
+++ b/ql/src/test/results/clientpositive/union18.q.out
@@ -73,7 +73,7 @@ STAGE PLANS:
Select Operator
expressions: 'tst1' (type: string), UDFToString(_col0) (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
table:
@@ -86,10 +86,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
Union
- Statistics: Num rows: 501 Data size: 5484 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 501 Data size: 5584 Basic stats: COMPLETE Column stats: PARTIAL
File Output Operator
compressed: false
- Statistics: Num rows: 501 Data size: 5484 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 501 Data size: 5584 Basic stats: COMPLETE Column stats: PARTIAL
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -98,10 +98,10 @@ STAGE PLANS:
Select Operator
expressions: _col0 (type: string), _col1 (type: string), _col1 (type: string)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 501 Data size: 128256 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 501 Data size: 228456 Basic stats: COMPLETE Column stats: PARTIAL
File Output Operator
compressed: false
- Statistics: Num rows: 501 Data size: 128256 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 501 Data size: 228456 Basic stats: COMPLETE Column stats: PARTIAL
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -115,10 +115,10 @@ STAGE PLANS:
outputColumnNames: _col0, _col1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Union
- Statistics: Num rows: 501 Data size: 5484 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 501 Data size: 5584 Basic stats: COMPLETE Column stats: PARTIAL
File Output Operator
compressed: false
- Statistics: Num rows: 501 Data size: 5484 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 501 Data size: 5584 Basic stats: COMPLETE Column stats: PARTIAL
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -127,10 +127,10 @@ STAGE PLANS:
Select Operator
expressions: _col0 (type: string), _col1 (type: string), _col1 (type: string)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 501 Data size: 128256 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 501 Data size: 228456 Basic stats: COMPLETE Column stats: PARTIAL
File Output Operator
compressed: false
- Statistics: Num rows: 501 Data size: 128256 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 501 Data size: 228456 Basic stats: COMPLETE Column stats: PARTIAL
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
[09/50] [abbrv] hive git commit: HIVE-14367 : Estimated size for
constant nulls is 0 (Ashutosh Chauhan via Jesus Camacho Rodriguez)
Posted by om...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/tez/vectorized_dynamic_partition_pruning.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/vectorized_dynamic_partition_pruning.q.out b/ql/src/test/results/clientpositive/tez/vectorized_dynamic_partition_pruning.q.out
index 3a71688..1e688eb 100644
--- a/ql/src/test/results/clientpositive/tez/vectorized_dynamic_partition_pruning.q.out
+++ b/ql/src/test/results/clientpositive/tez/vectorized_dynamic_partition_pruning.q.out
@@ -2891,10 +2891,10 @@ STAGE PLANS:
aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=1)
mode: hash
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary)
Reducer 3
Reduce Operator Tree:
@@ -2936,10 +2936,10 @@ STAGE PLANS:
aggregations: min(VALUE._col0), max(VALUE._col1), bloom_filter(VALUE._col2, expectedEntries=1)
mode: final
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary)
Stage: Stage-0
@@ -3011,10 +3011,10 @@ STAGE PLANS:
aggregations: max(ds)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string)
Map 8
Map Operator Tree:
@@ -3029,10 +3029,10 @@ STAGE PLANS:
aggregations: min(ds)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string)
Reducer 2
Reduce Operator Tree:
@@ -3074,20 +3074,20 @@ STAGE PLANS:
aggregations: max(VALUE._col0)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: _col0 is not null (type: boolean)
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Reducer 7
Execution mode: vectorized
Reduce Operator Tree:
@@ -3095,26 +3095,26 @@ STAGE PLANS:
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Dynamic Partitioning Event Operator
Target column: ds (string)
Target Input: srcpart
Partition key expr: ds
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Target Vertex: Map 1
Reducer 9
Execution mode: vectorized
@@ -3123,20 +3123,20 @@ STAGE PLANS:
aggregations: min(VALUE._col0)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: _col0 is not null (type: boolean)
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Union 6
Vertex: Union 6
@@ -3211,10 +3211,10 @@ STAGE PLANS:
aggregations: max(ds)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string)
Map 8
Map Operator Tree:
@@ -3229,10 +3229,10 @@ STAGE PLANS:
aggregations: min(ds)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string)
Reducer 2
Reduce Operator Tree:
@@ -3276,20 +3276,20 @@ STAGE PLANS:
aggregations: max(VALUE._col0)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: _col0 is not null (type: boolean)
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Reducer 7
Execution mode: vectorized
Reduce Operator Tree:
@@ -3297,26 +3297,26 @@ STAGE PLANS:
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Dynamic Partitioning Event Operator
Target column: ds (string)
Target Input: srcpart
Partition key expr: ds
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Target Vertex: Map 1
Reducer 9
Execution mode: vectorized
@@ -3325,20 +3325,20 @@ STAGE PLANS:
aggregations: min(VALUE._col0)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: _col0 is not null (type: boolean)
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Union 6
Vertex: Union 6
@@ -3415,10 +3415,10 @@ STAGE PLANS:
aggregations: min(ds)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string)
Map 5
Map Operator Tree:
@@ -3449,10 +3449,10 @@ STAGE PLANS:
aggregations: max(ds)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string)
Reducer 11
Execution mode: vectorized
@@ -3461,49 +3461,49 @@ STAGE PLANS:
aggregations: min(VALUE._col0)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: _col0 is not null (type: boolean)
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Dynamic Partitioning Event Operator
Target column: ds (string)
Target Input: srcpart
Partition key expr: ds
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Target Vertex: Map 1
Select Operator
expressions: _col0 (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Dynamic Partitioning Event Operator
Target column: ds (string)
Target Input: srcpart
Partition key expr: ds
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Target Vertex: Map 5
Reducer 2
Execution mode: vectorized
@@ -3555,49 +3555,49 @@ STAGE PLANS:
aggregations: max(VALUE._col0)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: _col0 is not null (type: boolean)
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Dynamic Partitioning Event Operator
Target column: ds (string)
Target Input: srcpart
Partition key expr: ds
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Target Vertex: Map 1
Select Operator
expressions: _col0 (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Dynamic Partitioning Event Operator
Target column: ds (string)
Target Input: srcpart
Partition key expr: ds
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Target Vertex: Map 5
Union 3
Vertex: Union 3
@@ -5185,10 +5185,10 @@ STAGE PLANS:
aggregations: max(ds)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string)
Map 7
Map Operator Tree:
@@ -5203,10 +5203,10 @@ STAGE PLANS:
aggregations: min(ds)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string)
Reducer 3
Execution mode: vectorized
@@ -5215,20 +5215,20 @@ STAGE PLANS:
aggregations: max(VALUE._col0)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: _col0 is not null (type: boolean)
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Reducer 5
Execution mode: vectorized
Reduce Operator Tree:
@@ -5236,7 +5236,7 @@ STAGE PLANS:
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Inner Join 0 to 1
@@ -5280,20 +5280,20 @@ STAGE PLANS:
aggregations: min(VALUE._col0)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: _col0 is not null (type: boolean)
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 2 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE
Union 4
Vertex: Union 4
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/tez/vectorized_mapjoin.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/vectorized_mapjoin.q.out b/ql/src/test/results/clientpositive/tez/vectorized_mapjoin.q.out
index 6efd39d..846db59 100644
--- a/ql/src/test/results/clientpositive/tez/vectorized_mapjoin.q.out
+++ b/ql/src/test/results/clientpositive/tez/vectorized_mapjoin.q.out
@@ -50,10 +50,10 @@ STAGE PLANS:
aggregations: count(_col0), max(_col1), min(_col0), avg(_col2)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: bigint), _col1 (type: int), _col2 (type: int), _col3 (type: struct<count:bigint,sum:double,input:int>)
Execution mode: vectorized
Map 3
@@ -80,10 +80,10 @@ STAGE PLANS:
aggregations: count(VALUE._col0), max(VALUE._col1), min(VALUE._col2), avg(VALUE._col3)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/tez/vectorized_shufflejoin.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/vectorized_shufflejoin.q.out b/ql/src/test/results/clientpositive/tez/vectorized_shufflejoin.q.out
index ad22b29..cda409f 100644
--- a/ql/src/test/results/clientpositive/tez/vectorized_shufflejoin.q.out
+++ b/ql/src/test/results/clientpositive/tez/vectorized_shufflejoin.q.out
@@ -74,10 +74,10 @@ STAGE PLANS:
aggregations: count(_col0), max(_col1), min(_col0), avg(_col2)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: bigint), _col1 (type: int), _col2 (type: int), _col3 (type: struct<count:bigint,sum:double,input:int>)
Reducer 3
Reduce Operator Tree:
@@ -85,11 +85,11 @@ STAGE PLANS:
aggregations: count(VALUE._col0), max(VALUE._col1), min(VALUE._col2), avg(VALUE._col3)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: bigint)
sort order: +
- Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: double)
Reducer 4
Execution mode: vectorized
@@ -97,10 +97,10 @@ STAGE PLANS:
Select Operator
expressions: KEY.reducesinkkey0 (type: bigint), VALUE._col0 (type: int), VALUE._col1 (type: int), VALUE._col2 (type: double)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/tez/vectorized_timestamp_funcs.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/vectorized_timestamp_funcs.q.out b/ql/src/test/results/clientpositive/tez/vectorized_timestamp_funcs.q.out
index 694f49d..0487321 100644
--- a/ql/src/test/results/clientpositive/tez/vectorized_timestamp_funcs.q.out
+++ b/ql/src/test/results/clientpositive/tez/vectorized_timestamp_funcs.q.out
@@ -812,10 +812,10 @@ STAGE PLANS:
aggregations: avg(ctimestamp1), variance(ctimestamp1), var_pop(ctimestamp1), var_samp(ctimestamp1), std(ctimestamp1), stddev(ctimestamp1), stddev_pop(ctimestamp1), stddev_samp(ctimestamp1)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 672 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 672 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: struct<count:bigint,sum:double,input:timestamp>), _col1 (type: struct<count:bigint,sum:double,variance:double>), _col2 (type: struct<count:bigint,sum:double,variance:double>), _col3 (type: struct<count:bigint,sum:double,variance:double>), _col4 (type: struct<count:bigint,sum:double,variance:double>), _col5 (type: struct<count:bigint,sum:double,variance:double>), _col6 (type: struct<count:bigint,sum:double,variance:double>), _col7 (type: struct<count:bigint,sum:double,variance:double>)
Execution mode: vectorized
Reducer 2
@@ -824,14 +824,14 @@ STAGE PLANS:
aggregations: avg(VALUE._col0), variance(VALUE._col1), var_pop(VALUE._col2), var_samp(VALUE._col3), std(VALUE._col4), stddev(VALUE._col5), stddev_pop(VALUE._col6), stddev_samp(VALUE._col7)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
- Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 672 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: round(_col0, 0) (type: double), _col1 BETWEEN 8.97077295279421E19 AND 8.97077295279422E19 (type: boolean), _col2 BETWEEN 8.97077295279421E19 AND 8.97077295279422E19 (type: boolean), _col3 BETWEEN 9.20684592523616E19 AND 9.20684592523617E19 (type: boolean), round(_col4, 3) (type: double), round(_col5, 3) (type: double), round(_col6, 3) (type: double), round(_col7, 3) (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
- Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 672 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 672 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/udaf_number_format.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/udaf_number_format.q.out b/ql/src/test/results/clientpositive/udaf_number_format.q.out
index 624b42a..14ab23d 100644
--- a/ql/src/test/results/clientpositive/udaf_number_format.q.out
+++ b/ql/src/test/results/clientpositive/udaf_number_format.q.out
@@ -29,10 +29,10 @@ STAGE PLANS:
aggregations: sum('a'), avg('a'), variance('a'), std('a')
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 424 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 424 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col0 (type: double), _col1 (type: struct<count:bigint,sum:double,input:string>), _col2 (type: struct<count:bigint,sum:double,variance:double>), _col3 (type: struct<count:bigint,sum:double,variance:double>)
Reduce Operator Tree:
Group By Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/udaf_percentile_approx_23.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/udaf_percentile_approx_23.q.out b/ql/src/test/results/clientpositive/udaf_percentile_approx_23.q.out
index 05d4df8..ff6942a 100644
--- a/ql/src/test/results/clientpositive/udaf_percentile_approx_23.q.out
+++ b/ql/src/test/results/clientpositive/udaf_percentile_approx_23.q.out
@@ -519,20 +519,20 @@ STAGE PLANS:
aggregations: percentile_approx(_col0, 0.5)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 160 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 160 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: array<double>)
Reduce Operator Tree:
Group By Operator
aggregations: percentile_approx(VALUE._col0)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 160 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 160 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -578,20 +578,20 @@ STAGE PLANS:
aggregations: percentile_approx(_col0, 0.5)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 160 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 160 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: array<double>)
Reduce Operator Tree:
Group By Operator
aggregations: percentile_approx(VALUE._col0)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 160 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 160 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/udf3.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/udf3.q.out b/ql/src/test/results/clientpositive/udf3.q.out
index 8b2ad31..528b8c7 100644
--- a/ql/src/test/results/clientpositive/udf3.q.out
+++ b/ql/src/test/results/clientpositive/udf3.q.out
@@ -32,10 +32,10 @@ STAGE PLANS:
aggregations: count(null), sum(null), avg(null), min(null), max(null)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4
- Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: struct<count:bigint,sum:double,input:int>), _col3 (type: int), _col4 (type: int)
Reduce Operator Tree:
Group By Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/udf4.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/udf4.q.out b/ql/src/test/results/clientpositive/udf4.q.out
index 9751168..6064cf2 100644
--- a/ql/src/test/results/clientpositive/udf4.q.out
+++ b/ql/src/test/results/clientpositive/udf4.q.out
@@ -79,10 +79,10 @@ STAGE PLANS:
Select Operator
expressions: 1.0 (type: double), 2.0 (type: double), -2.0 (type: double), 1 (type: bigint), 1 (type: bigint), -2 (type: bigint), 1.0 (type: double), null (type: double), 0.0 (type: double), 1 (type: bigint), 2 (type: bigint), -1 (type: bigint), 1 (type: bigint), rand(3) (type: double), 3 (type: int), -3 (type: int), 3 (type: int), -1 (type: int), -2 (type: int), -2 (type: tinyint), -2 (type: smallint), -2 (type: bigint), 0 (type: tinyint), 0 (type: smallint), 0 (type: int), 0 (type: bigint), 3 (type: tinyint), 3 (type: smallint), 3 (type: int), 3 (type: bigint), 2 (type: tinyint), 2 (type: smallint), 2 (type: int), 2 (type: bigint)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29, _col30, _col31, _col32, _col33
- Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/udf7.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/udf7.q.out b/ql/src/test/results/clientpositive/udf7.q.out
index b15ccaf..12ebb2d 100644
--- a/ql/src/test/results/clientpositive/udf7.q.out
+++ b/ql/src/test/results/clientpositive/udf7.q.out
@@ -49,7 +49,7 @@ STAGE PLANS:
Select Operator
expressions: 1.098612288668 (type: double), null (type: double), null (type: double), 1.098612288668 (type: double), null (type: double), null (type: double), 1.584962500721 (type: double), null (type: double), null (type: double), 0.47712125472 (type: double), null (type: double), null (type: double), 1.584962500721 (type: double), null (type: double), null (type: double), null (type: double), -1.0 (type: double), 7.389056098931 (type: double), 8.0 (type: double), 8.0 (type: double), 0.125 (type: double), 8.0 (type: double), 2.0 (type: double), NaN (type: double), 1.0 (type: double), 1.0 (type: double), 8.0 (type: double), 8.0 (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26, _col27
- Statistics: Num rows: 1 Data size: 136 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 224 Basic stats: COMPLETE Column stats: COMPLETE
ListSink
PREHOOK: query: SELECT ROUND(LN(3.0),12), LN(0.0), LN(-1), ROUND(LOG(3.0),12), LOG(0.0),
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/udf8.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/udf8.q.out b/ql/src/test/results/clientpositive/udf8.q.out
index 72cd434..d75cd78 100644
--- a/ql/src/test/results/clientpositive/udf8.q.out
+++ b/ql/src/test/results/clientpositive/udf8.q.out
@@ -49,20 +49,20 @@ STAGE PLANS:
aggregations: avg(c1), sum(c1), count(c1)
mode: hash
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: struct<count:bigint,sum:double,input:string>), _col1 (type: double), _col2 (type: bigint)
Reduce Operator Tree:
Group By Operator
aggregations: avg(VALUE._col0), sum(VALUE._col1), count(VALUE._col2)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/udf_case.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/udf_case.q.out b/ql/src/test/results/clientpositive/udf_case.q.out
index 554f05d..a49d2c3 100644
--- a/ql/src/test/results/clientpositive/udf_case.q.out
+++ b/ql/src/test/results/clientpositive/udf_case.q.out
@@ -91,7 +91,7 @@ STAGE PLANS:
Select Operator
expressions: 2 (type: int), 5 (type: int), 15 (type: int), null (type: int), 20 (type: int), 24 (type: int)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
- Statistics: Num rows: 500 Data size: 10000 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 500 Data size: 10004 Basic stats: COMPLETE Column stats: COMPLETE
ListSink
PREHOOK: query: SELECT CASE 1
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/udf_coalesce.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/udf_coalesce.q.out b/ql/src/test/results/clientpositive/udf_coalesce.q.out
index 83024f2..5a5f219 100644
--- a/ql/src/test/results/clientpositive/udf_coalesce.q.out
+++ b/ql/src/test/results/clientpositive/udf_coalesce.q.out
@@ -68,7 +68,7 @@ STAGE PLANS:
Select Operator
expressions: 1 (type: int), 1 (type: int), 2 (type: int), 1 (type: int), 3 (type: int), 4 (type: int), '1' (type: string), '1' (type: string), '2' (type: string), '1' (type: string), '3' (type: string), '4' (type: string), 1.0 (type: double), 1.0 (type: double), 2.0 (type: double), 2.0 (type: double), 2.0 (type: double), null (type: int)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17
- Statistics: Num rows: 500 Data size: 287000 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 500 Data size: 287004 Basic stats: COMPLETE Column stats: COMPLETE
ListSink
PREHOOK: query: SELECT COALESCE(1),
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/udf_elt.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/udf_elt.q.out b/ql/src/test/results/clientpositive/udf_elt.q.out
index 18f3481..8edfa6d 100644
--- a/ql/src/test/results/clientpositive/udf_elt.q.out
+++ b/ql/src/test/results/clientpositive/udf_elt.q.out
@@ -54,7 +54,7 @@ STAGE PLANS:
Select Operator
expressions: 'defg' (type: string), 'cc' (type: string), 'abc' (type: string), '2' (type: string), '12345' (type: string), '123456789012' (type: string), '1.25' (type: string), '16.0' (type: string), null (type: string), null (type: string), null (type: string)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10
- Statistics: Num rows: 500 Data size: 353500 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 500 Data size: 353752 Basic stats: COMPLETE Column stats: COMPLETE
ListSink
PREHOOK: query: SELECT elt(2, 'abc', 'defg'),
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/udf_greatest.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/udf_greatest.q.out b/ql/src/test/results/clientpositive/udf_greatest.q.out
index 47cfb3f..37e532d 100644
--- a/ql/src/test/results/clientpositive/udf_greatest.q.out
+++ b/ql/src/test/results/clientpositive/udf_greatest.q.out
@@ -60,7 +60,7 @@ STAGE PLANS:
Select Operator
expressions: 'c' (type: string), 'a' (type: string), 'AaA' (type: string), 'AAA' (type: string), '13' (type: string), '2' (type: string), '03' (type: string), '1' (type: string), null (type: double), null (type: double), null (type: double), null (type: double), null (type: double), null (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13
- Statistics: Num rows: 500 Data size: 343000 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 500 Data size: 343048 Basic stats: COMPLETE Column stats: COMPLETE
ListSink
PREHOOK: query: SELECT GREATEST('a', 'b', 'c'),
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/udf_if.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/udf_if.q.out b/ql/src/test/results/clientpositive/udf_if.q.out
index d0eeb53..df4c5e9 100644
--- a/ql/src/test/results/clientpositive/udf_if.q.out
+++ b/ql/src/test/results/clientpositive/udf_if.q.out
@@ -41,7 +41,7 @@ STAGE PLANS:
Select Operator
expressions: 1 (type: int), '1' (type: string), 1 (type: int), 1 (type: int), null (type: int), 2 (type: int)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
- Statistics: Num rows: 500 Data size: 50500 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 500 Data size: 50504 Basic stats: COMPLETE Column stats: COMPLETE
ListSink
PREHOOK: query: SELECT IF(TRUE, 1, 2) AS COL1,
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/udf_instr.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/udf_instr.q.out b/ql/src/test/results/clientpositive/udf_instr.q.out
index eafd8d6..50e65a1 100644
--- a/ql/src/test/results/clientpositive/udf_instr.q.out
+++ b/ql/src/test/results/clientpositive/udf_instr.q.out
@@ -58,7 +58,7 @@ STAGE PLANS:
Select Operator
expressions: 1 (type: int), 0 (type: int), 2 (type: int), 2 (type: int), 0 (type: int), 0 (type: int), 2 (type: int), 3 (type: int), 4 (type: int), 2 (type: int), 3 (type: int), null (type: int), null (type: int)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12
- Statistics: Num rows: 500 Data size: 22000 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 500 Data size: 22008 Basic stats: COMPLETE Column stats: COMPLETE
ListSink
PREHOOK: query: SELECT instr('abcd', 'abc'),
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/udf_least.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/udf_least.q.out b/ql/src/test/results/clientpositive/udf_least.q.out
index 2363abe..6f15dc3 100644
--- a/ql/src/test/results/clientpositive/udf_least.q.out
+++ b/ql/src/test/results/clientpositive/udf_least.q.out
@@ -60,7 +60,7 @@ STAGE PLANS:
Select Operator
expressions: 'a' (type: string), 'B' (type: string), 'AAA' (type: string), 'A' (type: string), '11' (type: string), '11' (type: string), '01' (type: string), '01' (type: string), null (type: double), null (type: double), null (type: double), null (type: double), null (type: double), null (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13
- Statistics: Num rows: 500 Data size: 343000 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 500 Data size: 343048 Basic stats: COMPLETE Column stats: COMPLETE
ListSink
PREHOOK: query: SELECT LEAST('a', 'b', 'c'),
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/udf_locate.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/udf_locate.q.out b/ql/src/test/results/clientpositive/udf_locate.q.out
index 1bf2b3a..00136dc 100644
--- a/ql/src/test/results/clientpositive/udf_locate.q.out
+++ b/ql/src/test/results/clientpositive/udf_locate.q.out
@@ -66,7 +66,7 @@ STAGE PLANS:
Select Operator
expressions: 1 (type: int), 0 (type: int), 2 (type: int), 2 (type: int), 4 (type: int), 4 (type: int), 0 (type: int), 0 (type: int), 2 (type: int), 3 (type: int), 4 (type: int), 2 (type: int), 3 (type: int), null (type: int), null (type: int), 0 (type: int), 0 (type: int)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16
- Statistics: Num rows: 500 Data size: 30000 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 500 Data size: 30008 Basic stats: COMPLETE Column stats: COMPLETE
ListSink
PREHOOK: query: SELECT locate('abc', 'abcd'),
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/udf_trunc.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/udf_trunc.q.out b/ql/src/test/results/clientpositive/udf_trunc.q.out
index 4a41238..a4c0b42 100644
--- a/ql/src/test/results/clientpositive/udf_trunc.q.out
+++ b/ql/src/test/results/clientpositive/udf_trunc.q.out
@@ -567,7 +567,7 @@ STAGE PLANS:
Select Operator
expressions: '2014-02-01' (type: string), null (type: string), null (type: string), null (type: string), null (type: string), null (type: string)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
- Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 514 Basic stats: COMPLETE Column stats: COMPLETE
ListSink
PREHOOK: query: SELECT
@@ -624,7 +624,7 @@ STAGE PLANS:
Select Operator
expressions: '2014-01-01' (type: string), null (type: string), null (type: string), null (type: string), null (type: string), null (type: string)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
- Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 514 Basic stats: COMPLETE Column stats: COMPLETE
ListSink
PREHOOK: query: SELECT
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/udf_when.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/udf_when.q.out b/ql/src/test/results/clientpositive/udf_when.q.out
index 7341f81..d3e31a9 100644
--- a/ql/src/test/results/clientpositive/udf_when.q.out
+++ b/ql/src/test/results/clientpositive/udf_when.q.out
@@ -91,7 +91,7 @@ STAGE PLANS:
Select Operator
expressions: 2 (type: int), 9 (type: int), 14 (type: int), null (type: int), 24 (type: int), null (type: int)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
- Statistics: Num rows: 500 Data size: 8000 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 500 Data size: 8008 Basic stats: COMPLETE Column stats: COMPLETE
ListSink
PREHOOK: query: SELECT CASE
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/udtf_stack.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/udtf_stack.q.out b/ql/src/test/results/clientpositive/udtf_stack.q.out
index f5f7dc4..97dcd3a 100644
--- a/ql/src/test/results/clientpositive/udtf_stack.q.out
+++ b/ql/src/test/results/clientpositive/udtf_stack.q.out
@@ -178,13 +178,13 @@ STAGE PLANS:
Select Operator
expressions: 1 (type: int), 'en' (type: string), 'dbpedia' (type: string), null (type: void)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 1 Data size: 181 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 185 Basic stats: COMPLETE Column stats: COMPLETE
UDTF Operator
- Statistics: Num rows: 1 Data size: 181 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 185 Basic stats: COMPLETE Column stats: COMPLETE
function name: stack
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 181 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 185 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/union_remove_6_subq.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/union_remove_6_subq.q.out b/ql/src/test/results/clientpositive/union_remove_6_subq.q.out
index fecb654..f5deb1d 100644
--- a/ql/src/test/results/clientpositive/union_remove_6_subq.q.out
+++ b/ql/src/test/results/clientpositive/union_remove_6_subq.q.out
@@ -325,10 +325,10 @@ STAGE PLANS:
aggregations: avg(_col0)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col0 (type: struct<count:bigint,sum:double,input:bigint>)
TableScan
Union
@@ -337,10 +337,10 @@ STAGE PLANS:
aggregations: avg(_col0)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col0 (type: struct<count:bigint,sum:double,input:bigint>)
Reduce Operator Tree:
Group By Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/vector_aggregate_9.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vector_aggregate_9.q.out b/ql/src/test/results/clientpositive/vector_aggregate_9.q.out
index 2698c68..6dc5479 100644
--- a/ql/src/test/results/clientpositive/vector_aggregate_9.q.out
+++ b/ql/src/test/results/clientpositive/vector_aggregate_9.q.out
@@ -126,10 +126,10 @@ STAGE PLANS:
aggregations: min(dc), max(dc), sum(dc), avg(dc)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 1 Data size: 336 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 624 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 336 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 624 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: decimal(38,18)), _col1 (type: decimal(38,18)), _col2 (type: decimal(38,18)), _col3 (type: struct<count:bigint,sum:decimal(38,18),input:decimal(38,18)>)
Execution mode: vectorized
Reduce Operator Tree:
@@ -137,10 +137,10 @@ STAGE PLANS:
aggregations: min(VALUE._col0), max(VALUE._col1), sum(VALUE._col2), avg(VALUE._col3)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 1 Data size: 448 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 624 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 448 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 624 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/vector_aggregate_without_gby.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vector_aggregate_without_gby.q.out b/ql/src/test/results/clientpositive/vector_aggregate_without_gby.q.out
index 112cb03..3043a6c 100644
--- a/ql/src/test/results/clientpositive/vector_aggregate_without_gby.q.out
+++ b/ql/src/test/results/clientpositive/vector_aggregate_without_gby.q.out
@@ -59,10 +59,10 @@ STAGE PLANS:
aggregations: max(dt), max(greg_dt)
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 88 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 88 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: int), _col1 (type: string)
Execution mode: vectorized
Reduce Operator Tree:
@@ -70,10 +70,10 @@ STAGE PLANS:
aggregations: max(VALUE._col0), max(VALUE._col1)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 88 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 88 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/vector_coalesce.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vector_coalesce.q.out b/ql/src/test/results/clientpositive/vector_coalesce.q.out
index 628466a..6344ddc 100644
--- a/ql/src/test/results/clientpositive/vector_coalesce.q.out
+++ b/ql/src/test/results/clientpositive/vector_coalesce.q.out
@@ -20,31 +20,31 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: alltypesorc
- Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 1045942 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: cdouble is null (type: boolean)
- Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 3114 Data size: 265164 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: cstring1 (type: string), cint (type: int), cfloat (type: float), csmallint (type: smallint), COALESCE(null,cstring1,cint,cfloat,csmallint) (type: string)
outputColumnNames: _col1, _col2, _col3, _col4, _col5
- Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 3114 Data size: 819540 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col1 (type: string), _col2 (type: int), _col3 (type: float), _col4 (type: smallint), _col5 (type: string)
sort order: +++++
- Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 3114 Data size: 819540 Basic stats: COMPLETE Column stats: COMPLETE
TopN Hash Memory Usage: 0.1
Execution mode: vectorized
Reduce Operator Tree:
Select Operator
expressions: null (type: double), KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: int), KEY.reducesinkkey2 (type: float), KEY.reducesinkkey3 (type: smallint), KEY.reducesinkkey4 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
- Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 3114 Data size: 246572 Basic stats: COMPLETE Column stats: COMPLETE
Limit
Number of rows: 10
- Statistics: Num rows: 10 Data size: 2150 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 10 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 10 Data size: 2150 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 10 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -104,31 +104,31 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: alltypesorc
- Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 146792 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: ctinyint is null (type: boolean)
- Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 3115 Data size: 37224 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: cdouble (type: double), cint (type: int), COALESCE(null,(cdouble + log2(cint)),0) (type: double)
outputColumnNames: _col1, _col2, _col3
- Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 3115 Data size: 52844 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col1 (type: double), _col2 (type: int), _col3 (type: double)
sort order: +++
- Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 3115 Data size: 52844 Basic stats: COMPLETE Column stats: COMPLETE
TopN Hash Memory Usage: 0.1
Execution mode: vectorized
Reduce Operator Tree:
Select Operator
expressions: null (type: tinyint), KEY.reducesinkkey0 (type: double), KEY.reducesinkkey1 (type: int), KEY.reducesinkkey2 (type: double)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 3115 Data size: 27928 Basic stats: COMPLETE Column stats: COMPLETE
Limit
Number of rows: 10
- Statistics: Num rows: 10 Data size: 2150 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 10 Data size: 2150 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -188,28 +188,28 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: alltypesorc
- Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 110088 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: (cfloat is null and cbigint is null) (type: boolean)
- Statistics: Num rows: 3072 Data size: 660491 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 790 Data size: 7092 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
- Statistics: Num rows: 3072 Data size: 660491 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 790 Data size: 3172 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
sort order:
- Statistics: Num rows: 3072 Data size: 660491 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 790 Data size: 3172 Basic stats: COMPLETE Column stats: COMPLETE
TopN Hash Memory Usage: 0.1
Execution mode: vectorized
Reduce Operator Tree:
Select Operator
expressions: null (type: float), null (type: bigint), 0.0 (type: float)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 3072 Data size: 660491 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 790 Data size: 3172 Basic stats: COMPLETE Column stats: COMPLETE
Limit
Number of rows: 10
- Statistics: Num rows: 10 Data size: 2150 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 10 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 10 Data size: 2150 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 10 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -269,31 +269,31 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: alltypesorc
- Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 983040 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: (ctimestamp1 is not null or ctimestamp2 is not null) (type: boolean)
- Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 983040 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), COALESCE(ctimestamp1,ctimestamp2) (type: timestamp)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 1474560 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: timestamp), _col1 (type: timestamp), _col2 (type: timestamp)
sort order: +++
- Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 1474560 Basic stats: COMPLETE Column stats: COMPLETE
TopN Hash Memory Usage: 0.1
Execution mode: vectorized
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: timestamp), KEY.reducesinkkey1 (type: timestamp), KEY.reducesinkkey2 (type: timestamp)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 1474560 Basic stats: COMPLETE Column stats: COMPLETE
Limit
Number of rows: 10
- Statistics: Num rows: 10 Data size: 2150 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 10 Data size: 1200 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 10 Data size: 2150 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 10 Data size: 1200 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -353,28 +353,28 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: alltypesorc
- Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 110088 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: (cfloat is null and cbigint is null) (type: boolean)
- Statistics: Num rows: 3072 Data size: 660491 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 790 Data size: 7092 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
- Statistics: Num rows: 3072 Data size: 660491 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 790 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
sort order:
- Statistics: Num rows: 3072 Data size: 660491 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 790 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
TopN Hash Memory Usage: 0.1
Execution mode: vectorized
Reduce Operator Tree:
Select Operator
expressions: null (type: float), null (type: bigint), null (type: float)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 3072 Data size: 660491 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 790 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
Limit
Number of rows: 10
- Statistics: Num rows: 10 Data size: 2150 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 10 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 10 Data size: 2150 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 10 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -432,20 +432,20 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: alltypesorc
- Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 110088 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: cbigint is null (type: boolean)
- Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 3115 Data size: 27912 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: null (type: bigint), ctinyint (type: tinyint), COALESCE(null,ctinyint) (type: tinyint)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 3115 Data size: 21772 Basic stats: COMPLETE Column stats: COMPLETE
Limit
Number of rows: 10
- Statistics: Num rows: 10 Data size: 2150 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 10 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 10 Data size: 2150 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 10 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/vector_decimal_precision.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vector_decimal_precision.q.out b/ql/src/test/results/clientpositive/vector_decimal_precision.q.out
index bd66dc7..94b9657 100644
--- a/ql/src/test/results/clientpositive/vector_decimal_precision.q.out
+++ b/ql/src/test/results/clientpositive/vector_decimal_precision.q.out
@@ -568,10 +568,10 @@ STAGE PLANS:
aggregations: avg(dec), sum(dec)
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 400 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 400 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: struct<count:bigint,sum:decimal(30,10),input:decimal(20,10)>), _col1 (type: decimal(30,10))
Execution mode: vectorized
Reduce Operator Tree:
@@ -579,10 +579,10 @@ STAGE PLANS:
aggregations: avg(VALUE._col0), sum(VALUE._col1)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 224 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 400 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 224 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 400 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
[30/50] [abbrv] hive git commit: HIVE-15084: Flaky test:
TestMiniTezCliDriver:explainanalyze_1, 2, 3, 4, 5 (Pengcheng Xiong)
Posted by om...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/9f5bea3b/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java
index c0e56e7..375516d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java
@@ -45,8 +45,8 @@ import org.apache.hadoop.hive.ql.optimizer.ReduceSinkMapJoinProc;
import org.apache.hadoop.hive.ql.optimizer.RemoveDynamicPruningBySize;
import org.apache.hadoop.hive.ql.optimizer.SetReducerParallelism;
import org.apache.hadoop.hive.ql.optimizer.metainfo.annotation.AnnotateWithOpTraits;
-import org.apache.hadoop.hive.ql.optimizer.physical.AnnotateRunTimeStatsOptimizer;
import org.apache.hadoop.hive.ql.optimizer.physical.CrossProductCheck;
+import org.apache.hadoop.hive.ql.optimizer.physical.AnnotateRunTimeStatsOptimizer;
import org.apache.hadoop.hive.ql.optimizer.physical.LlapDecider;
import org.apache.hadoop.hive.ql.optimizer.physical.MemoryDecider;
import org.apache.hadoop.hive.ql.optimizer.physical.MetadataOnlyOptimizer;
http://git-wip-us.apache.org/repos/asf/hive/blob/9f5bea3b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
index 0f1e3a3..7f3be26 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
@@ -1341,7 +1341,7 @@ public class StatsUtils {
|| colType.startsWith(serdeConstants.MAP_TYPE_NAME)
|| colType.startsWith(serdeConstants.STRUCT_TYPE_NAME)
|| colType.startsWith(serdeConstants.UNION_TYPE_NAME)) {
- avgColSize = getAvgColLenOfVariableLengthTypes(conf, oi, colType);
+ avgColSize = getAvgColLenOf(conf, oi, colType);
} else {
avgColSize = getAvgColLenOfFixedLengthTypes(colType);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/9f5bea3b/ql/src/test/queries/clientpositive/explainanalyze_1.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/explainanalyze_1.q b/ql/src/test/queries/clientpositive/explainanalyze_1.q
index a4b3dc5..0a33a4f 100644
--- a/ql/src/test/queries/clientpositive/explainanalyze_1.q
+++ b/ql/src/test/queries/clientpositive/explainanalyze_1.q
@@ -1,3 +1,4 @@
+set hive.map.aggr=false;
set hive.mapred.mode=nonstrict;
explain analyze select * from src a union all select * from src b limit 10;
http://git-wip-us.apache.org/repos/asf/hive/blob/9f5bea3b/ql/src/test/queries/clientpositive/explainanalyze_2.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/explainanalyze_2.q b/ql/src/test/queries/clientpositive/explainanalyze_2.q
index dfee826..7637101 100644
--- a/ql/src/test/queries/clientpositive/explainanalyze_2.q
+++ b/ql/src/test/queries/clientpositive/explainanalyze_2.q
@@ -1,199 +1,11 @@
-set hive.explain.user=true;
-set hive.metastore.aggregate.stats.cache.enabled=false;
-
--- SORT_QUERY_RESULTS
-
-CREATE TABLE dest_j1(key STRING, value STRING, val2 STRING) STORED AS TEXTFILE;
-
-CREATE TABLE ss(k1 STRING,v1 STRING,k2 STRING,v2 STRING,k3 STRING,v3 STRING) STORED AS TEXTFILE;
-
-CREATE TABLE sr(k1 STRING,v1 STRING,k2 STRING,v2 STRING,k3 STRING,v3 STRING) STORED AS TEXTFILE;
-
-CREATE TABLE cs(k1 STRING,v1 STRING,k2 STRING,v2 STRING,k3 STRING,v3 STRING) STORED AS TEXTFILE;
-
-INSERT OVERWRITE TABLE ss
-SELECT x.key,x.value,y.key,y.value,z.key,z.value
-FROM src1 x
-JOIN src y ON (x.key = y.key)
-JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11);
-
-INSERT OVERWRITE TABLE sr
-SELECT x.key,x.value,y.key,y.value,z.key,z.value
-FROM src1 x
-JOIN src y ON (x.key = y.key)
-JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=12);
-
-INSERT OVERWRITE TABLE cs
-SELECT x.key,x.value,y.key,y.value,z.key,z.value
-FROM src1 x
-JOIN src y ON (x.key = y.key)
-JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08');
-
-
-ANALYZE TABLE ss COMPUTE STATISTICS;
-ANALYZE TABLE ss COMPUTE STATISTICS FOR COLUMNS k1,v1,k2,v2,k3,v3;
-
-ANALYZE TABLE sr COMPUTE STATISTICS;
-ANALYZE TABLE sr COMPUTE STATISTICS FOR COLUMNS k1,v1,k2,v2,k3,v3;
-
-ANALYZE TABLE cs COMPUTE STATISTICS;
-ANALYZE TABLE cs COMPUTE STATISTICS FOR COLUMNS k1,v1,k2,v2,k3,v3;
-
-set hive.auto.convert.join=false;
-
-explain analyze
-SELECT x.key, z.value, y.value
-FROM src1 x JOIN src y ON (x.key = y.key)
-JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11);
-
-explain analyze
-select
-ss.k1,sr.k2,cs.k3,count(ss.v1),count(sr.v2),count(cs.v3)
-FROM
-ss,sr,cs,src d1,src d2,src d3,src1,srcpart
-where
- ss.k1 = d1.key
-and sr.k1 = d2.key
-and cs.k1 = d3.key
-and ss.k2 = sr.k2
-and ss.k3 = sr.k3
-and ss.v1 = src1.value
-and ss.v2 = srcpart.value
-and sr.v2 = cs.v2
-and sr.v3 = cs.v3
-and ss.v3='ssv3'
-and sr.v1='srv1'
-and src1.key = 'src1key'
-and srcpart.key = 'srcpartkey'
-and d1.value = 'd1value'
-and d2.value in ('2000Q1','2000Q2','2000Q3')
-and d3.value in ('2000Q1','2000Q2','2000Q3')
-group by
-ss.k1,sr.k2,cs.k3
-order by
-ss.k1,sr.k2,cs.k3
-limit 100;
-
-explain analyze
-SELECT x.key, z.value, y.value
-FROM src1 x JOIN src y ON (x.key = y.key)
-JOIN (select * from src1 union select * from src)z ON (x.value = z.value)
-union
-SELECT x.key, z.value, y.value
-FROM src1 x JOIN src y ON (x.key = y.key)
-JOIN (select * from src1 union select * from src)z ON (x.value = z.value);
-
-explain analyze
-SELECT x.key, y.value
-FROM src1 x JOIN src y ON (x.key = y.key)
-JOIN (select * from src1 union select * from src)z ON (x.value = z.value)
-union
-SELECT x.key, y.value
-FROM src1 x JOIN src y ON (x.key = y.key)
-JOIN (select key, value from src1 union select key, value from src union select key, value from src)z ON (x.value = z.value)
-union
-SELECT x.key, y.value
-FROM src1 x JOIN src y ON (x.key = y.key)
-JOIN (select key, value from src1 union select key, value from src union select key, value from src union select key, value from src)z ON (x.value = z.value);
-
-
-set hive.auto.convert.join=true;
-set hive.auto.convert.join.noconditionaltask=true;
-set hive.auto.convert.join.noconditionaltask.size=10000;
-set hive.stats.fetch.column.stats=false;
-
-
-explain analyze
-SELECT x.key, z.value, y.value
-FROM src1 x JOIN src y ON (x.key = y.key)
-JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11);
-
-explain analyze
-select
-ss.k1,sr.k2,cs.k3,count(ss.v1),count(sr.v2),count(cs.v3)
-FROM
-ss,sr,cs,src d1,src d2,src d3,src1,srcpart
-where
- ss.k1 = d1.key
-and sr.k1 = d2.key
-and cs.k1 = d3.key
-and ss.k2 = sr.k2
-and ss.k3 = sr.k3
-and ss.v1 = src1.value
-and ss.v2 = srcpart.value
-and sr.v2 = cs.v2
-and sr.v3 = cs.v3
-and ss.v3='ssv3'
-and sr.v1='srv1'
-and src1.key = 'src1key'
-and srcpart.key = 'srcpartkey'
-and d1.value = 'd1value'
-and d2.value in ('2000Q1','2000Q2','2000Q3')
-and d3.value in ('2000Q1','2000Q2','2000Q3')
-group by
-ss.k1,sr.k2,cs.k3
-order by
-ss.k1,sr.k2,cs.k3
-limit 100;
-
-explain analyze
-SELECT x.key, z.value, y.value
-FROM src1 x JOIN src y ON (x.key = y.key)
-JOIN (select * from src1 union select * from src)z ON (x.value = z.value)
-union
-SELECT x.key, z.value, y.value
-FROM src1 x JOIN src y ON (x.key = y.key)
-JOIN (select * from src1 union select * from src)z ON (x.value = z.value);
-
-explain analyze
-SELECT x.key, y.value
-FROM src1 x JOIN src y ON (x.key = y.key)
-JOIN (select * from src1 union select * from src)z ON (x.value = z.value)
-union
-SELECT x.key, y.value
-FROM src1 x JOIN src y ON (x.key = y.key)
-JOIN (select key, value from src1 union select key, value from src union select key, value from src)z ON (x.value = z.value)
-union
-SELECT x.key, y.value
-FROM src1 x JOIN src y ON (x.key = y.key)
-JOIN (select key, value from src1 union select key, value from src union select key, value from src union select key, value from src)z ON (x.value = z.value);
-
-
-set hive.auto.convert.join=true;
-set hive.auto.convert.join.noconditionaltask=true;
-set hive.auto.convert.join.noconditionaltask.size=10000;
-set hive.auto.convert.sortmerge.join.bigtable.selection.policy = org.apache.hadoop.hive.ql.optimizer.TableSizeBasedBigTableSelectorForAutoSMJ;
-
-CREATE TABLE srcbucket_mapjoin(key int, value string) partitioned by (ds string) CLUSTERED BY (key) INTO 2 BUCKETS STORED AS TEXTFILE;
-CREATE TABLE tab_part (key int, value string) PARTITIONED BY(ds STRING) CLUSTERED BY (key) SORTED BY (key) INTO 4 BUCKETS STORED AS TEXTFILE;
-CREATE TABLE srcbucket_mapjoin_part (key int, value string) partitioned by (ds string) CLUSTERED BY (key) INTO 4 BUCKETS STORED AS TEXTFILE;
-
-load data local inpath '../../data/files/srcbucket20.txt' INTO TABLE srcbucket_mapjoin partition(ds='2008-04-08');
-load data local inpath '../../data/files/srcbucket22.txt' INTO TABLE srcbucket_mapjoin partition(ds='2008-04-08');
-
-load data local inpath '../../data/files/srcbucket20.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08');
-load data local inpath '../../data/files/srcbucket21.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08');
-load data local inpath '../../data/files/srcbucket22.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08');
-load data local inpath '../../data/files/srcbucket23.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08');
-
+set hive.map.aggr=false;
+set hive.explain.user=true;
-set hive.optimize.bucketingsorting=false;
-insert overwrite table tab_part partition (ds='2008-04-08')
-select key,value from srcbucket_mapjoin_part;
+CREATE TABLE tab(key int, value string) PARTITIONED BY(ds STRING);
-CREATE TABLE tab(key int, value string) PARTITIONED BY(ds STRING) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS STORED AS TEXTFILE;
insert overwrite table tab partition (ds='2008-04-08')
-select key,value from srcbucket_mapjoin;
-
-CREATE TABLE tab2(key int, value string) PARTITIONED BY(ds STRING) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS STORED AS TEXTFILE;
-insert overwrite table tab2 partition (ds='2008-04-08')
-select key,value from srcbucket_mapjoin;
-
-set hive.convert.join.bucket.mapjoin.tez = false;
-set hive.auto.convert.sortmerge.join = true;
-
-set hive.auto.convert.join.noconditionaltask.size=500;
+select key,value from src;
explain analyze
select s1.key as key, s1.value as value from tab s1 join tab s3 on s1.key=s3.key;
@@ -201,129 +13,3 @@ select s1.key as key, s1.value as value from tab s1 join tab s3 on s1.key=s3.key
explain analyze
select s1.key as key, s1.value as value from tab s1 join tab s3 on s1.key=s3.key join tab s2 on s1.value=s2.value;
-explain analyze
-select s1.key as key, s1.value as value from tab s1 join tab2 s3 on s1.key=s3.key;
-
-explain analyze
-select s1.key as key, s1.value as value from tab s1 join tab2 s3 on s1.key=s3.key join tab2 s2 on s1.value=s2.value;
-
-explain analyze
-select count(*) from (select s1.key as key, s1.value as value from tab s1 join tab s3 on s1.key=s3.key
-UNION ALL
-select s2.key as key, s2.value as value from tab s2
-) a join tab_part b on (a.key = b.key);
-
-explain analyze
-select count(*) from (select s1.key as key, s1.value as value from tab s1 join tab s3 on s1.key=s3.key join tab s2 on s1.value=s2.value
-UNION ALL
-select s2.key as key, s2.value as value from tab s2
-) a join tab_part b on (a.key = b.key);
-
-explain analyze
-SELECT x.key, y.value
-FROM src1 x JOIN src y ON (x.key = y.key)
-JOIN (select * from src1 union all select * from src)z ON (x.value = z.value)
-union all
-SELECT x.key, y.value
-FROM src x JOIN src y ON (x.key = y.key)
-JOIN (select key, value from src1 union all select key, value from src union all select key, value from src)z ON (x.value = z.value)
-union all
-SELECT x.key, y.value
-FROM src1 x JOIN src1 y ON (x.key = y.key)
-JOIN (select key, value from src1 union all select key, value from src union all select key, value from src union all select key, value from src)z ON (x.value = z.value);
-
-explain analyze
-SELECT x.key, y.value
-FROM src1 x JOIN src y ON (x.key = y.key)
-JOIN (select * from src1 union select * from src)z ON (x.value = z.value)
-union
-SELECT x.key, y.value
-FROM src x JOIN src y ON (x.key = y.key)
-JOIN (select key, value from src1 union select key, value from src union select key, value from src)z ON (x.value = z.value)
-union
-SELECT x.key, y.value
-FROM src1 x JOIN src1 y ON (x.key = y.key)
-JOIN (select key, value from src1 union select key, value from src union select key, value from src union select key, value from src)z ON (x.value = z.value);
-
-CREATE TABLE a(key STRING, value STRING) STORED AS TEXTFILE;
-CREATE TABLE b(key STRING, value STRING) STORED AS TEXTFILE;
-CREATE TABLE c(key STRING, value STRING) STORED AS TEXTFILE;
-
-explain analyze
-from
-(
-SELECT x.key, y.value
-FROM src1 x JOIN src y ON (x.key = y.key)
-JOIN (select * from src1 union all select * from src)z ON (x.value = z.value)
-union all
-SELECT x.key, y.value
-FROM src x JOIN src y ON (x.key = y.key)
-JOIN (select key, value from src1 union all select key, value from src union all select key, value from src)z ON (x.value = z.value)
-union all
-SELECT x.key, y.value
-FROM src1 x JOIN src1 y ON (x.key = y.key)
-JOIN (select key, value from src1 union all select key, value from src union all select key, value from src union all select key, value from src)z ON (x.value = z.value)
-) tmp
-INSERT OVERWRITE TABLE a SELECT tmp.key, tmp.value
-INSERT OVERWRITE TABLE b SELECT tmp.key, tmp.value
-INSERT OVERWRITE TABLE c SELECT tmp.key, tmp.value;
-
-explain analyze
-FROM
-(
-SELECT x.key as key, y.value as value from src1 x JOIN src y ON (x.key = y.key)
-JOIN (select * from src1 union select * from src)z ON (x.value = z.value)
-union
-SELECT x.key as key, y.value as value from src x JOIN src y ON (x.key = y.key)
-JOIN (select key, value from src1 union select key, value from src union select key, value from src)z ON (x.value = z.value)
-union
-SELECT x.key as key, y.value as value from src1 x JOIN src1 y ON (x.key = y.key)
-JOIN (select key, value from src1 union select key, value from src union select key, value from src union select key, value from src)z ON (x.value = z.value)
-) tmp
-INSERT OVERWRITE TABLE a SELECT tmp.key, tmp.value
-INSERT OVERWRITE TABLE b SELECT tmp.key, tmp.value
-INSERT OVERWRITE TABLE c SELECT tmp.key, tmp.value;
-
-
-CREATE TABLE DEST1(key STRING, value STRING) STORED AS TEXTFILE;
-CREATE TABLE DEST2(key STRING, val1 STRING, val2 STRING) STORED AS TEXTFILE;
-
-explain analyze
-FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1
- UNION DISTINCT
- select s2.key as key, s2.value as value from src s2) unionsrc
-INSERT OVERWRITE TABLE DEST1 SELECT unionsrc.key, COUNT(DISTINCT SUBSTR(unionsrc.value,5)) GROUP BY unionsrc.key
-INSERT OVERWRITE TABLE DEST2 SELECT unionsrc.key, unionsrc.value, COUNT(DISTINCT SUBSTR(unionsrc.value,5)) GROUP BY unionsrc.key, unionsrc.value;
-
-explain analyze FROM UNIQUEJOIN PRESERVE src a (a.key), PRESERVE src1 b (b.key), PRESERVE srcpart c (c.key) SELECT a.key, b.key, c.key;
-
-set hive.entity.capture.transform=true;
-
-explain analyze
-SELECT
-TRANSFORM(a.key, a.value) USING 'cat' AS (tkey, tvalue)
-FROM src a join src b
-on a.key = b.key;
-
-explain analyze
-FROM (
- select key, value from (
- select 'tst1' as key, cast(count(1) as string) as value, 'tst1' as value2 from src s1
- UNION all
- select s2.key as key, s2.value as value, 'tst1' as value2 from src s2) unionsub
- UNION all
- select key, value from src s0
- ) unionsrc
-INSERT OVERWRITE TABLE DEST1 SELECT unionsrc.key, COUNT(DISTINCT SUBSTR(unionsrc.value,5)) GROUP BY unionsrc.key
-INSERT OVERWRITE TABLE DEST2 SELECT unionsrc.key, unionsrc.value, COUNT(DISTINCT SUBSTR(unionsrc.value,5))
-GROUP BY unionsrc.key, unionsrc.value;
-
-explain analyze
-FROM (
- select 'tst1' as key, cast(count(1) as string) as value, 'tst1' as value2 from src s1
- UNION all
- select s2.key as key, s2.value as value, 'tst1' as value2 from src s2
- ) unionsrc
-INSERT OVERWRITE TABLE DEST1 SELECT unionsrc.key, COUNT(DISTINCT SUBSTR(unionsrc.value,5)) GROUP BY unionsrc.key
-INSERT OVERWRITE TABLE DEST2 SELECT unionsrc.key, unionsrc.value, COUNT(DISTINCT SUBSTR(unionsrc.value,5))
-GROUP BY unionsrc.key, unionsrc.value;
http://git-wip-us.apache.org/repos/asf/hive/blob/9f5bea3b/ql/src/test/queries/clientpositive/explainanalyze_3.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/explainanalyze_3.q b/ql/src/test/queries/clientpositive/explainanalyze_3.q
index 69f82e5..53dfabd 100644
--- a/ql/src/test/queries/clientpositive/explainanalyze_3.q
+++ b/ql/src/test/queries/clientpositive/explainanalyze_3.q
@@ -1,3 +1,5 @@
+set hive.map.aggr=false;
+
set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.DefaultHiveAuthorizationProvider;
set hive.metastore.filter.hook=org.apache.hadoop.hive.metastore.DefaultMetaStoreFilterHookImpl;
set hive.mapred.mode=nonstrict;
@@ -40,9 +42,13 @@ use default;
drop database newDB;
-explain analyze analyze table src compute statistics;
+drop table src_stats;
+
+create table src_stats as select * from src;
+
+explain analyze analyze table src_stats compute statistics;
-explain analyze analyze table src compute statistics for columns;
+explain analyze analyze table src_stats compute statistics for columns;
explain analyze
CREATE TEMPORARY MACRO SIGMOID (x DOUBLE) 1.0 / (1.0 + EXP(-x));
http://git-wip-us.apache.org/repos/asf/hive/blob/9f5bea3b/ql/src/test/queries/clientpositive/explainanalyze_4.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/explainanalyze_4.q b/ql/src/test/queries/clientpositive/explainanalyze_4.q
index dad397b..498a128 100644
--- a/ql/src/test/queries/clientpositive/explainanalyze_4.q
+++ b/ql/src/test/queries/clientpositive/explainanalyze_4.q
@@ -1,3 +1,5 @@
+set hive.map.aggr=false;
+
set hive.mapred.mode=nonstrict;
set hive.explain.user=true;
http://git-wip-us.apache.org/repos/asf/hive/blob/9f5bea3b/ql/src/test/queries/clientpositive/explainanalyze_5.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/explainanalyze_5.q b/ql/src/test/queries/clientpositive/explainanalyze_5.q
index bb23e45..696064c 100644
--- a/ql/src/test/queries/clientpositive/explainanalyze_5.q
+++ b/ql/src/test/queries/clientpositive/explainanalyze_5.q
@@ -1,8 +1,14 @@
+set hive.map.aggr=false;
+
set hive.stats.column.autogather=true;
-explain analyze analyze table src compute statistics;
+drop table src_stats;
+
+create table src_stats as select * from src;
+
+explain analyze analyze table src_stats compute statistics;
-explain analyze analyze table src compute statistics for columns;
+explain analyze analyze table src_stats compute statistics for columns;
drop table src_multi2;
http://git-wip-us.apache.org/repos/asf/hive/blob/9f5bea3b/ql/src/test/results/clientpositive/annotate_stats_join_pkfk.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/annotate_stats_join_pkfk.q.out b/ql/src/test/results/clientpositive/annotate_stats_join_pkfk.q.out
index 71da136..b4d46d2 100644
--- a/ql/src/test/results/clientpositive/annotate_stats_join_pkfk.q.out
+++ b/ql/src/test/results/clientpositive/annotate_stats_join_pkfk.q.out
@@ -444,19 +444,19 @@ STAGE PLANS:
Statistics: Num rows: 12 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE
TableScan
alias: ss
- Statistics: Num rows: 1000 Data size: 3856 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1000 Data size: 3860 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: ss_store_sk is not null (type: boolean)
- Statistics: Num rows: 964 Data size: 3716 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 964 Data size: 3720 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: ss_store_sk (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 964 Data size: 3716 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 964 Data size: 3720 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: UDFToLong(_col0) (type: bigint)
sort order: +
Map-reduce partition columns: UDFToLong(_col0) (type: bigint)
- Statistics: Num rows: 964 Data size: 3716 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 964 Data size: 3720 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Operator Tree:
Join Operator
condition map:
http://git-wip-us.apache.org/repos/asf/hive/blob/9f5bea3b/ql/src/test/results/clientpositive/dynamic_rdd_cache.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/dynamic_rdd_cache.q.out b/ql/src/test/results/clientpositive/dynamic_rdd_cache.q.out
index 3a8da0e..87166a7 100644
--- a/ql/src/test/results/clientpositive/dynamic_rdd_cache.q.out
+++ b/ql/src/test/results/clientpositive/dynamic_rdd_cache.q.out
@@ -679,7 +679,7 @@ STAGE PLANS:
Select Operator
expressions: 'tst1' (type: string), UDFToString(_col0) (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: true
table:
@@ -692,12 +692,12 @@ STAGE PLANS:
Map Operator Tree:
TableScan
Union
- Statistics: Num rows: 167 Data size: 1935 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 167 Data size: 2035 Basic stats: COMPLETE Column stats: PARTIAL
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 167 Data size: 1935 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 167 Data size: 2035 Basic stats: COMPLETE Column stats: PARTIAL
value expressions: _col1 (type: string)
TableScan
alias: s2
@@ -710,21 +710,21 @@ STAGE PLANS:
outputColumnNames: _col0, _col1
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
Union
- Statistics: Num rows: 167 Data size: 1935 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 167 Data size: 2035 Basic stats: COMPLETE Column stats: PARTIAL
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 167 Data size: 1935 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 167 Data size: 2035 Basic stats: COMPLETE Column stats: PARTIAL
value expressions: _col1 (type: string)
TableScan
Union
- Statistics: Num rows: 167 Data size: 1935 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 167 Data size: 2035 Basic stats: COMPLETE Column stats: PARTIAL
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 167 Data size: 1935 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 167 Data size: 2035 Basic stats: COMPLETE Column stats: PARTIAL
value expressions: _col1 (type: string)
TableScan
alias: s4
@@ -737,12 +737,12 @@ STAGE PLANS:
outputColumnNames: _col0, _col1
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
Union
- Statistics: Num rows: 167 Data size: 1935 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 167 Data size: 2035 Basic stats: COMPLETE Column stats: PARTIAL
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 167 Data size: 1935 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 167 Data size: 2035 Basic stats: COMPLETE Column stats: PARTIAL
value expressions: _col1 (type: string)
Reduce Operator Tree:
Join Operator
@@ -752,10 +752,10 @@ STAGE PLANS:
0 _col0 (type: string)
1 _col0 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 27889 Data size: 9593816 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 27889 Data size: 15171616 Basic stats: COMPLETE Column stats: PARTIAL
File Output Operator
compressed: true
- Statistics: Num rows: 27889 Data size: 9593816 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 27889 Data size: 15171616 Basic stats: COMPLETE Column stats: PARTIAL
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -787,7 +787,7 @@ STAGE PLANS:
Select Operator
expressions: 'tst1' (type: string), UDFToString(_col0) (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: true
table:
http://git-wip-us.apache.org/repos/asf/hive/blob/9f5bea3b/ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction.q.out b/ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction.q.out
index 1f25d49..385b5cf 100644
--- a/ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction.q.out
+++ b/ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction.q.out
@@ -284,10 +284,10 @@ STAGE PLANS:
aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=1000)
mode: hash
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary)
Execution mode: llap
LLAP IO: all inputs
@@ -332,10 +332,10 @@ STAGE PLANS:
aggregations: min(VALUE._col0), max(VALUE._col1), bloom_filter(VALUE._col2, expectedEntries=1000)
mode: final
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary)
Stage: Stage-0
@@ -681,10 +681,10 @@ STAGE PLANS:
aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=2000)
mode: hash
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary)
Execution mode: llap
LLAP IO: all inputs
@@ -714,10 +714,10 @@ STAGE PLANS:
aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=1000)
mode: hash
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary)
Select Operator
expressions: _col0 (type: string)
@@ -727,10 +727,10 @@ STAGE PLANS:
aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=1000)
mode: hash
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary)
Execution mode: llap
LLAP IO: all inputs
@@ -797,10 +797,10 @@ STAGE PLANS:
aggregations: min(VALUE._col0), max(VALUE._col1), bloom_filter(VALUE._col2, expectedEntries=2000)
mode: final
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary)
Reducer 6
Execution mode: llap
@@ -809,10 +809,10 @@ STAGE PLANS:
aggregations: min(VALUE._col0), max(VALUE._col1), bloom_filter(VALUE._col2, expectedEntries=1000)
mode: final
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary)
Reducer 7
Execution mode: llap
@@ -821,10 +821,10 @@ STAGE PLANS:
aggregations: min(VALUE._col0), max(VALUE._col1), bloom_filter(VALUE._col2, expectedEntries=1000)
mode: final
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary)
Stage: Stage-0
@@ -1056,10 +1056,10 @@ STAGE PLANS:
aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=1000)
mode: hash
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary)
Select Operator
expressions: _col1 (type: string)
@@ -1069,10 +1069,10 @@ STAGE PLANS:
aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=1000)
mode: hash
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary)
Execution mode: llap
LLAP IO: all inputs
@@ -1117,10 +1117,10 @@ STAGE PLANS:
aggregations: min(VALUE._col0), max(VALUE._col1), bloom_filter(VALUE._col2, expectedEntries=1000)
mode: final
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary)
Reducer 6
Execution mode: llap
@@ -1129,10 +1129,10 @@ STAGE PLANS:
aggregations: min(VALUE._col0), max(VALUE._col1), bloom_filter(VALUE._col2, expectedEntries=1000)
mode: final
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary)
Stage: Stage-0
@@ -1367,10 +1367,10 @@ STAGE PLANS:
aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=1000)
mode: hash
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary)
Execution mode: llap
LLAP IO: all inputs
@@ -1418,10 +1418,10 @@ STAGE PLANS:
aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=2200)
mode: hash
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary)
Reducer 3
Execution mode: llap
@@ -1464,10 +1464,10 @@ STAGE PLANS:
aggregations: min(VALUE._col0), max(VALUE._col1), bloom_filter(VALUE._col2, expectedEntries=2200)
mode: final
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary)
Reducer 7
Execution mode: llap
@@ -1476,10 +1476,10 @@ STAGE PLANS:
aggregations: min(VALUE._col0), max(VALUE._col1), bloom_filter(VALUE._col2, expectedEntries=1000)
mode: final
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary)
Stage: Stage-0
@@ -1697,10 +1697,10 @@ STAGE PLANS:
aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=1000)
mode: hash
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary)
Execution mode: llap
LLAP IO: all inputs
@@ -1726,10 +1726,10 @@ STAGE PLANS:
aggregations: min(VALUE._col0), max(VALUE._col1), bloom_filter(VALUE._col2, expectedEntries=1000)
mode: final
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 252 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 552 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: binary)
Stage: Stage-0
http://git-wip-us.apache.org/repos/asf/hive/blob/9f5bea3b/ql/src/test/results/clientpositive/llap/subquery_multi.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/subquery_multi.q.out b/ql/src/test/results/clientpositive/llap/subquery_multi.q.out
index 854fb78..0cc1781 100644
--- a/ql/src/test/results/clientpositive/llap/subquery_multi.q.out
+++ b/ql/src/test/results/clientpositive/llap/subquery_multi.q.out
@@ -3148,10 +3148,10 @@ STAGE PLANS:
aggregations: avg(l_quantity)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: struct<count:bigint,sum:double,input:double>)
Execution mode: llap
LLAP IO: no inputs
@@ -3206,10 +3206,10 @@ STAGE PLANS:
aggregations: avg(l_quantity)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: struct<count:bigint,sum:double,input:double>)
Execution mode: llap
LLAP IO: no inputs
@@ -3220,15 +3220,15 @@ STAGE PLANS:
aggregations: avg(VALUE._col0)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(), count(_col0)
mode: complete
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 88 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 88 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: bigint), _col1 (type: bigint)
Reducer 12
Execution mode: llap
@@ -3237,17 +3237,17 @@ STAGE PLANS:
aggregations: avg(VALUE._col0)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: double)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: double)
sort order: +
Map-reduce partition columns: _col0 (type: double)
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: NONE
Reducer 13
Execution mode: llap
Reduce Operator Tree:
@@ -3255,16 +3255,16 @@ STAGE PLANS:
keys: KEY._col0 (type: double)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: double), true (type: boolean)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: double)
sort order: +
Map-reduce partition columns: _col0 (type: double)
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: boolean)
Reducer 2
Execution mode: llap
@@ -3328,12 +3328,12 @@ STAGE PLANS:
0
1
outputColumnNames: _col0, _col1, _col2, _col4, _col5
- Statistics: Num rows: 25 Data size: 3424 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 25 Data size: 5224 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col2 (type: double)
sort order: +
Map-reduce partition columns: _col2 (type: double)
- Statistics: Num rows: 25 Data size: 3424 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 25 Data size: 5224 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: int), _col1 (type: int), _col4 (type: bigint), _col5 (type: bigint)
Reducer 8
Execution mode: llap
@@ -3345,24 +3345,24 @@ STAGE PLANS:
0 _col2 (type: double)
1 _col0 (type: double)
outputColumnNames: _col0, _col1, _col2, _col4, _col5, _col7
- Statistics: Num rows: 27 Data size: 3766 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 27 Data size: 5746 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (not CASE WHEN ((_col4 = 0)) THEN (false) WHEN (_col7 is not null) THEN (true) WHEN (_col2 is null) THEN (null) WHEN ((_col5 < _col4)) THEN (true) ELSE (false) END) (type: boolean)
- Statistics: Num rows: 14 Data size: 1952 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 14 Data size: 2979 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: int), _col1 (type: int)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 14 Data size: 1952 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 14 Data size: 2979 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: int), _col1 (type: int)
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 14 Data size: 1952 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 14 Data size: 2979 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int), _col1 (type: int)
sort order: ++
Map-reduce partition columns: _col0 (type: int), _col1 (type: int)
- Statistics: Num rows: 14 Data size: 1952 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 14 Data size: 2979 Basic stats: COMPLETE Column stats: NONE
Stage: Stage-0
Fetch Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/9f5bea3b/ql/src/test/results/clientpositive/llap/subquery_scalar.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/subquery_scalar.q.out b/ql/src/test/results/clientpositive/llap/subquery_scalar.q.out
index 9fa2bd8..e78aadd 100644
--- a/ql/src/test/results/clientpositive/llap/subquery_scalar.q.out
+++ b/ql/src/test/results/clientpositive/llap/subquery_scalar.q.out
@@ -148,10 +148,10 @@ STAGE PLANS:
aggregations: avg(p_size)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 76 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 76 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: struct<count:bigint,sum:double,input:int>)
Execution mode: llap
LLAP IO: no inputs
@@ -180,17 +180,17 @@ STAGE PLANS:
0
1
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10
- Statistics: Num rows: 26 Data size: 3615 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 26 Data size: 5383 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (UDFToDouble(_col5) > _col10) (type: boolean)
- Statistics: Num rows: 8 Data size: 1112 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 8 Data size: 1656 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
- Statistics: Num rows: 8 Data size: 1112 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 8 Data size: 1656 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 8 Data size: 1112 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 8 Data size: 1656 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -225,10 +225,10 @@ STAGE PLANS:
aggregations: avg(VALUE._col0)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 76 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 76 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: double)
Stage: Stage-0
@@ -801,10 +801,10 @@ STAGE PLANS:
aggregations: max(p_name)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string)
Execution mode: llap
LLAP IO: no inputs
@@ -820,10 +820,10 @@ STAGE PLANS:
1
2
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
- Statistics: Num rows: 26 Data size: 5565 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 26 Data size: 8165 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 26 Data size: 5565 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 26 Data size: 8165 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -858,15 +858,15 @@ STAGE PLANS:
aggregations: max(VALUE._col0)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: _col0 is not null (type: boolean)
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Select Operator
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Stage: Stage-0
Fetch Operator
@@ -986,10 +986,10 @@ STAGE PLANS:
aggregations: avg(p_size)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 76 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 76 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: struct<count:bigint,sum:double,input:int>)
Execution mode: llap
LLAP IO: no inputs
@@ -1063,10 +1063,10 @@ STAGE PLANS:
aggregations: avg(VALUE._col0)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 76 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 76 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: double)
Reducer 2
Execution mode: llap
@@ -1123,17 +1123,17 @@ STAGE PLANS:
0
1
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10, _col12
- Statistics: Num rows: 26 Data size: 3979 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 26 Data size: 5747 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: _col5 BETWEEN _col10 AND _col12 (type: boolean)
- Statistics: Num rows: 2 Data size: 306 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 442 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
- Statistics: Num rows: 2 Data size: 306 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 442 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 2 Data size: 306 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 442 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -2501,10 +2501,10 @@ STAGE PLANS:
aggregations: min(p_brand)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string)
Execution mode: llap
LLAP IO: no inputs
@@ -2540,15 +2540,15 @@ STAGE PLANS:
1
2
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10
- Statistics: Num rows: 26 Data size: 5565 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 26 Data size: 8165 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (_col3 <> _col10) (type: boolean)
- Statistics: Num rows: 26 Data size: 5565 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 26 Data size: 8165 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col4 (type: string), _col5 (type: int)
sort order: ++
Map-reduce partition columns: _col4 (type: string), _col5 (type: int)
- Statistics: Num rows: 26 Data size: 5565 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 26 Data size: 8165 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col6 (type: string), _col7 (type: double), _col8 (type: string)
Reducer 3
Execution mode: llap
@@ -2560,10 +2560,10 @@ STAGE PLANS:
0 _col4 (type: string), _col5 (type: int)
1 _col1 (type: string), _col0 (type: int)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
- Statistics: Num rows: 28 Data size: 6121 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 28 Data size: 8981 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 28 Data size: 6121 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 28 Data size: 8981 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -2598,10 +2598,10 @@ STAGE PLANS:
aggregations: min(VALUE._col0)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string)
Reducer 9
Execution mode: llap
@@ -2924,10 +2924,10 @@ STAGE PLANS:
aggregations: max(value)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string)
Execution mode: llap
LLAP IO: no inputs
@@ -2938,10 +2938,10 @@ STAGE PLANS:
aggregations: max(VALUE._col0)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string)
Reducer 12
Execution mode: llap
@@ -3020,25 +3020,25 @@ STAGE PLANS:
0
1
outputColumnNames: _col0, _col1, _col3
- Statistics: Num rows: 500 Data size: 52312 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 500 Data size: 102312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (_col1 <> _col3) (type: boolean)
- Statistics: Num rows: 500 Data size: 52312 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 500 Data size: 102312 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 500 Data size: 52312 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 500 Data size: 102312 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count()
keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 500 Data size: 52312 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 500 Data size: 102312 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 500 Data size: 52312 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 500 Data size: 102312 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: bigint)
Reducer 4
Execution mode: llap
@@ -3048,10 +3048,10 @@ STAGE PLANS:
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 250 Data size: 26156 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 250 Data size: 51156 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 250 Data size: 26156 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 250 Data size: 51156 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string), _col1 (type: bigint)
Reducer 5
Execution mode: llap
@@ -3063,10 +3063,10 @@ STAGE PLANS:
0
1
outputColumnNames: _col0, _col1
- Statistics: Num rows: 250 Data size: 28406 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 250 Data size: 53406 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 250 Data size: 28406 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 250 Data size: 53406 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string), _col1 (type: bigint)
Reducer 6
Execution mode: llap
@@ -3078,17 +3078,17 @@ STAGE PLANS:
0
1
outputColumnNames: _col0, _col1, _col3
- Statistics: Num rows: 31250 Data size: 3914000 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 31250 Data size: 7039000 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (_col1 > _col3) (type: boolean)
- Statistics: Num rows: 10416 Data size: 1304583 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 10416 Data size: 2346183 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: string), _col1 (type: bigint)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 10416 Data size: 1304583 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 10416 Data size: 2346183 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 10416 Data size: 1304583 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 10416 Data size: 2346183 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -4055,10 +4055,10 @@ STAGE PLANS:
aggregations: min(p_name)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string)
Execution mode: llap
LLAP IO: no inputs
@@ -4117,17 +4117,17 @@ STAGE PLANS:
0
1
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col10
- Statistics: Num rows: 5 Data size: 3726 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 5 Data size: 4226 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (not (_col1 like _col10)) (type: boolean)
- Statistics: Num rows: 3 Data size: 2235 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 3 Data size: 2535 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
- Statistics: Num rows: 3 Data size: 2235 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 3 Data size: 2535 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 3 Data size: 2235 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 3 Data size: 2535 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string)
Reducer 4
Execution mode: llap
@@ -4139,12 +4139,12 @@ STAGE PLANS:
0
1
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col11, _col12
- Statistics: Num rows: 3 Data size: 2286 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 3 Data size: 2586 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col3 (type: string)
sort order: +
Map-reduce partition columns: _col3 (type: string)
- Statistics: Num rows: 3 Data size: 2286 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 3 Data size: 2586 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col11 (type: bigint), _col12 (type: bigint)
Reducer 5
Execution mode: llap
@@ -4201,10 +4201,10 @@ STAGE PLANS:
aggregations: min(VALUE._col0)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string)
Stage: Stage-0
[17/50] [abbrv] hive git commit: HIVE-14362: Support explain analyze
in Hive (Pengcheng Xiong, reviewed by Ashutosh Chauhan,
Gopal V and Gabor Szadovszky)
Posted by om...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/fd72ec4a/ql/src/test/results/clientpositive/tez/explainanalyze_3.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/explainanalyze_3.q.out b/ql/src/test/results/clientpositive/tez/explainanalyze_3.q.out
index dd8849d..dfe163e 100644
--- a/ql/src/test/results/clientpositive/tez/explainanalyze_3.q.out
+++ b/ql/src/test/results/clientpositive/tez/explainanalyze_3.q.out
@@ -244,7 +244,7 @@ PREHOOK: type: QUERY
POSTHOOK: query: explain analyze analyze table src_stats compute statistics for columns
POSTHOOK: type: QUERY
Vertex dependency in root stage
-Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
+Reducer 2 <- Map 1 (SIMPLE_EDGE)
Stage-2
Column Stats Work{}
@@ -253,8 +253,8 @@ Stage-2
File Output Operator [FS_5]
Group By Operator [GBY_3] (rows=1/1 width=960)
Output:["_col0","_col1"],aggregations:["compute_stats(VALUE._col0, 16)","compute_stats(VALUE._col2, 16)"]
- <-Map 1 [CUSTOM_SIMPLE_EDGE]
- PARTITION_ONLY_SHUFFLE [RS_2]
+ <-Map 1 [SIMPLE_EDGE]
+ SHUFFLE [RS_2]
Select Operator [SEL_1] (rows=500/500 width=10)
Output:["key","value"]
TableScan [TS_0] (rows=500/500 width=10)
@@ -348,10 +348,10 @@ Stage-3
Map 1
File Output Operator [FS_2]
table:{"name:":"default.src_autho_test"}
- Select Operator [SEL_1] (rows=500/500 width=178)
+ Select Operator [SEL_1] (rows=500/500 width=10)
Output:["_col0","_col1"]
- TableScan [TS_0] (rows=500/500 width=178)
- default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
+ TableScan [TS_0] (rows=500/500 width=10)
+ default@src,src,Tbl:COMPLETE,Col:NONE,Output:["key","value"]
Stage-0
Move Operator
Please refer to the previous Stage-1
@@ -607,16 +607,16 @@ Stage-0
Stage-1
Reducer 2
File Output Operator [FS_5]
- Limit [LIM_4] (rows=5/5 width=178)
+ Limit [LIM_4] (rows=5/5 width=10)
Number of rows:5
- Select Operator [SEL_3] (rows=500/5 width=178)
+ Select Operator [SEL_3] (rows=500/5 width=10)
Output:["_col0","_col1"]
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_2]
- Select Operator [SEL_1] (rows=500/500 width=178)
+ Select Operator [SEL_1] (rows=500/500 width=10)
Output:["_col0","_col1"]
- TableScan [TS_0] (rows=500/500 width=178)
- default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
+ TableScan [TS_0] (rows=500/500 width=10)
+ default@src,src,Tbl:COMPLETE,Col:NONE,Output:["key","value"]
PREHOOK: query: create table orc_merge5 (userid bigint, string1 string, subtype double, decimal1 decimal, ts timestamp) stored as orc
PREHOOK: type: CREATETABLE
@@ -830,7 +830,7 @@ STAGE PLANS:
#### A masked pattern was here ####
Edges:
Map 3 <- Map 1 (CUSTOM_EDGE), Reducer 2 (BROADCAST_EDGE)
- Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -838,24 +838,24 @@ STAGE PLANS:
TableScan
alias: a
filterExpr: key is not null (type: boolean)
- Statistics: Num rows: 242/242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 242/242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: key is not null (type: boolean)
- Statistics: Num rows: 242/242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 242/242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: int), value (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 242/242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 242/242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 242/242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 242/242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: string)
Select Operator
expressions: _col0 (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 242/242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 242/242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=242)
mode: hash
@@ -870,14 +870,14 @@ STAGE PLANS:
TableScan
alias: b
filterExpr: (key is not null and key BETWEEN DynamicValue(RS_6_a_key_min) AND DynamicValue(RS_6_a_key_max) and in_bloom_filter(key, DynamicValue(RS_6_a_key_bloom_filter))) (type: boolean)
- Statistics: Num rows: 500/500 Data size: 9312 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 500/500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (key is not null and key BETWEEN DynamicValue(RS_6_a_key_min) AND DynamicValue(RS_6_a_key_max) and in_bloom_filter(key, DynamicValue(RS_6_a_key_bloom_filter))) (type: boolean)
- Statistics: Num rows: 500/244 Data size: 9312 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 500/244 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: int), value (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 500/244 Data size: 9312 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 500/244 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Inner Join 0 to 1
@@ -887,15 +887,15 @@ STAGE PLANS:
outputColumnNames: _col0, _col1, _col3
input vertices:
0 Map 1
- Statistics: Num rows: 550/480 Data size: 10243 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 550/480 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
HybridGraceHashJoin: true
Select Operator
expressions: _col0 (type: int), _col1 (type: string), _col3 (type: string)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 550/480 Data size: 10243 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 550/480 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 550/480 Data size: 10243 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 550/480 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/fd72ec4a/ql/src/test/results/clientpositive/tez/explainanalyze_4.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/explainanalyze_4.q.out b/ql/src/test/results/clientpositive/tez/explainanalyze_4.q.out
new file mode 100644
index 0000000..3426d19
--- /dev/null
+++ b/ql/src/test/results/clientpositive/tez/explainanalyze_4.q.out
@@ -0,0 +1,590 @@
+PREHOOK: query: select
+ *
+from alltypesorc a join alltypesorc b on a.cint = b.cint
+where
+ a.cint between 1000000 and 3000000 and b.cbigint is not null
+order by a.cint
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+POSTHOOK: query: select
+ *
+from alltypesorc a join alltypesorc b on a.cint = b.cint
+where
+ a.cint between 1000000 and 3000000 and b.cbigint is not null
+order by a.cint
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+PREHOOK: query: -- First try with regular mergejoin
+explain analyze
+select
+ *
+from alltypesorc a join alltypesorc b on a.cint = b.cint
+where
+ a.cint between 1000000 and 3000000 and b.cbigint is not null
+order by a.cint
+PREHOOK: type: QUERY
+POSTHOOK: query: -- First try with regular mergejoin
+explain analyze
+select
+ *
+from alltypesorc a join alltypesorc b on a.cint = b.cint
+where
+ a.cint between 1000000 and 3000000 and b.cbigint is not null
+order by a.cint
+POSTHOOK: type: QUERY
+Plan optimized by CBO.
+
+Vertex dependency in root stage
+Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE)
+Reducer 3 <- Reducer 2 (SIMPLE_EDGE)
+
+Stage-0
+ Fetch Operator
+ limit:-1
+ Stage-1
+ Reducer 3
+ File Output Operator [FS_12]
+ Select Operator [SEL_11] (rows=7286/10 width=620)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23"]
+ <-Reducer 2 [SIMPLE_EDGE]
+ SHUFFLE [RS_10]
+ Merge Join Operator [MERGEJOIN_17] (rows=7286/10 width=620)
+ Conds:RS_6._col2=RS_7._col2(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23"]
+ <-Map 1 [SIMPLE_EDGE]
+ SHUFFLE [RS_6]
+ PartitionCols:_col2
+ Select Operator [SEL_2] (rows=6144/10 width=251)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"]
+ Filter Operator [FIL_15] (rows=6144/10 width=251)
+ predicate:cint BETWEEN 1000000 AND 3000000
+ TableScan [TS_0] (rows=12288/12288 width=251)
+ default@alltypesorc,a,Tbl:COMPLETE,Col:COMPLETE,Output:["ctinyint","csmallint","cint","cbigint","cfloat","cdouble","cstring1","cstring2","ctimestamp1","ctimestamp2","cboolean1","cboolean2"]
+ <-Map 4 [SIMPLE_EDGE]
+ SHUFFLE [RS_7]
+ PartitionCols:_col2
+ Select Operator [SEL_5] (rows=3424/10 width=251)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"]
+ Filter Operator [FIL_16] (rows=3424/10 width=251)
+ predicate:(cint is not null and cbigint is not null and cint BETWEEN 1000000 AND 3000000)
+ TableScan [TS_3] (rows=12288/12288 width=251)
+ default@alltypesorc,b,Tbl:COMPLETE,Col:COMPLETE,Output:["ctinyint","csmallint","cint","cbigint","cfloat","cdouble","cstring1","cstring2","ctimestamp1","ctimestamp2","cboolean1","cboolean2"]
+
+PREHOOK: query: select
+ *
+from alltypesorc a join alltypesorc b on a.cint = b.cint
+where
+ a.cint between 1000000 and 3000000 and b.cbigint is not null
+order by a.cint
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+POSTHOOK: query: select
+ *
+from alltypesorc a join alltypesorc b on a.cint = b.cint
+where
+ a.cint between 1000000 and 3000000 and b.cbigint is not null
+order by a.cint
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+11 NULL 1000828 1531084669 11.0 NULL wM316f6NqGIkoP388j3F6 poWQQo3Upvt3Wh 1969-12-31 16:00:02.351 NULL false true 11 NULL 1000828 1531084669 11.0 NULL wM316f6NqGIkoP388j3F6 poWQQo3Upvt3Wh 1969-12-31 16:00:02.351 NULL false true
+NULL -3799 1248059 1864027286 NULL -3799.0 Uhps6mMh3IfHB3j7yH62K 4KWs6gw7lv2WYd66P NULL 1969-12-31 15:59:54.622 false true NULL -3799 1248059 1864027286 NULL -3799.0 Uhps6mMh3IfHB3j7yH62K 4KWs6gw7lv2WYd66P NULL 1969-12-31 15:59:54.622 false true
+NULL 10782 1286921 1864027286 NULL 10782.0 ODLrXI8882q8LS8 4KWs6gw7lv2WYd66P NULL 1969-12-31 15:59:52.138 true true NULL 10782 1286921 1864027286 NULL 10782.0 ODLrXI8882q8LS8 4KWs6gw7lv2WYd66P NULL 1969-12-31 15:59:52.138 true true
+NULL -13036 1288927 -1645852809 NULL -13036.0 yinBY725P7V2 xH7445Rals48VOulSyR5F NULL 1969-12-31 16:00:00.763 true false NULL -13036 1288927 -1645852809 NULL -13036.0 yinBY725P7V2 xH7445Rals48VOulSyR5F NULL 1969-12-31 16:00:00.763 true false
+11 NULL 1310786 -413875656 11.0 NULL W0rvA4H1xn0xMG4uk0 8yVVjG 1969-12-31 16:00:02.351 NULL false true 11 NULL 1310786 -413875656 11.0 NULL W0rvA4H1xn0xMG4uk0 8yVVjG 1969-12-31 16:00:02.351 NULL false true
+-51 NULL 2089466 -240556350 -51.0 NULL cXX24dH7tblSj46j2g C31eea0wrHHqvj 1969-12-31 16:00:08.451 NULL true true -51 NULL 2089466 -240556350 -51.0 NULL cXX24dH7tblSj46j2g C31eea0wrHHqvj 1969-12-31 16:00:08.451 NULL true true
+NULL -8915 2101183 1864027286 NULL -8915.0 x7By66525 4KWs6gw7lv2WYd66P NULL 1969-12-31 16:00:05.831 false true NULL -8915 2101183 1864027286 NULL -8915.0 x7By66525 4KWs6gw7lv2WYd66P NULL 1969-12-31 16:00:05.831 false true
+8 NULL 2229621 -381406148 8.0 NULL q7onkS7QRPh5ghOK oKb0bi 1969-12-31 16:00:15.892 NULL true false 8 NULL 2229621 -381406148 8.0 NULL q7onkS7QRPh5ghOK oKb0bi 1969-12-31 16:00:15.892 NULL true false
+8 NULL 2433892 -1611863517 8.0 NULL 674ILv3V2TxFqXP6wSbL VLprkK2XfX 1969-12-31 16:00:15.892 NULL false true 8 NULL 2433892 -1611863517 8.0 NULL 674ILv3V2TxFqXP6wSbL VLprkK2XfX 1969-12-31 16:00:15.892 NULL false true
+-51 NULL 2949963 -1580871111 -51.0 NULL 0K68k3bdl7jO7 TPPAu 1969-12-31 16:00:08.451 NULL true false -51 NULL 2949963 -1580871111 -51.0 NULL 0K68k3bdl7jO7 TPPAu 1969-12-31 16:00:08.451 NULL true false
+PREHOOK: query: select
+ count(*)
+from alltypesorc a join alltypesorc b on a.cint = b.cint
+where
+ a.cint between 1000000 and 3000000 and b.cbigint is not null
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+POSTHOOK: query: select
+ count(*)
+from alltypesorc a join alltypesorc b on a.cint = b.cint
+where
+ a.cint between 1000000 and 3000000 and b.cbigint is not null
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+PREHOOK: query: explain analyze
+select
+ count(*)
+from alltypesorc a join alltypesorc b on a.cint = b.cint
+where
+ a.cint between 1000000 and 3000000 and b.cbigint is not null
+PREHOOK: type: QUERY
+POSTHOOK: query: explain analyze
+select
+ count(*)
+from alltypesorc a join alltypesorc b on a.cint = b.cint
+where
+ a.cint between 1000000 and 3000000 and b.cbigint is not null
+POSTHOOK: type: QUERY
+Plan optimized by CBO.
+
+Vertex dependency in root stage
+Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE)
+Reducer 3 <- Reducer 2 (SIMPLE_EDGE)
+
+Stage-0
+ Fetch Operator
+ limit:-1
+ Stage-1
+ Reducer 3
+ File Output Operator [FS_14]
+ Group By Operator [GBY_12] (rows=1/1 width=8)
+ Output:["_col0"],aggregations:["count(VALUE._col0)"]
+ <-Reducer 2 [SIMPLE_EDGE]
+ SHUFFLE [RS_11]
+ Group By Operator [GBY_10] (rows=1/1 width=8)
+ Output:["_col0"],aggregations:["count()"]
+ Merge Join Operator [MERGEJOIN_19] (rows=7286/10 width=8)
+ Conds:RS_6._col0=RS_7._col0(Inner)
+ <-Map 1 [SIMPLE_EDGE]
+ SHUFFLE [RS_6]
+ PartitionCols:_col0
+ Select Operator [SEL_2] (rows=6144/10 width=2)
+ Output:["_col0"]
+ Filter Operator [FIL_17] (rows=6144/10 width=2)
+ predicate:cint BETWEEN 1000000 AND 3000000
+ TableScan [TS_0] (rows=12288/12288 width=2)
+ default@alltypesorc,a,Tbl:COMPLETE,Col:COMPLETE,Output:["cint"]
+ <-Map 4 [SIMPLE_EDGE]
+ SHUFFLE [RS_7]
+ PartitionCols:_col0
+ Select Operator [SEL_5] (rows=3424/10 width=8)
+ Output:["_col0"]
+ Filter Operator [FIL_18] (rows=3424/10 width=8)
+ predicate:(cint is not null and cbigint is not null and cint BETWEEN 1000000 AND 3000000)
+ TableScan [TS_3] (rows=12288/12288 width=8)
+ default@alltypesorc,b,Tbl:COMPLETE,Col:COMPLETE,Output:["cint","cbigint"]
+
+PREHOOK: query: select
+ count(*)
+from alltypesorc a join alltypesorc b on a.cint = b.cint
+where
+ a.cint between 1000000 and 3000000 and b.cbigint is not null
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+POSTHOOK: query: select
+ count(*)
+from alltypesorc a join alltypesorc b on a.cint = b.cint
+where
+ a.cint between 1000000 and 3000000 and b.cbigint is not null
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+10
+PREHOOK: query: select
+ a.csmallint, count(*) c1
+from alltypesorc a join alltypesorc b on a.cint = b.cint
+where
+ a.cint between 1000000 and 3000000 and b.cbigint is not null
+group by a.csmallint
+order by c1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+POSTHOOK: query: select
+ a.csmallint, count(*) c1
+from alltypesorc a join alltypesorc b on a.cint = b.cint
+where
+ a.cint between 1000000 and 3000000 and b.cbigint is not null
+group by a.csmallint
+order by c1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+PREHOOK: query: explain analyze
+select
+ a.csmallint, count(*) c1
+from alltypesorc a join alltypesorc b on a.cint = b.cint
+where
+ a.cint between 1000000 and 3000000 and b.cbigint is not null
+group by a.csmallint
+order by c1
+PREHOOK: type: QUERY
+POSTHOOK: query: explain analyze
+select
+ a.csmallint, count(*) c1
+from alltypesorc a join alltypesorc b on a.cint = b.cint
+where
+ a.cint between 1000000 and 3000000 and b.cbigint is not null
+group by a.csmallint
+order by c1
+POSTHOOK: type: QUERY
+Plan optimized by CBO.
+
+Vertex dependency in root stage
+Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE)
+Reducer 3 <- Reducer 2 (SIMPLE_EDGE)
+Reducer 4 <- Reducer 3 (SIMPLE_EDGE)
+
+Stage-0
+ Fetch Operator
+ limit:-1
+ Stage-1
+ Reducer 4
+ File Output Operator [FS_16]
+ Select Operator [SEL_15] (rows=2765/5 width=12)
+ Output:["_col0","_col1"]
+ <-Reducer 3 [SIMPLE_EDGE]
+ SHUFFLE [RS_14]
+ Group By Operator [GBY_12] (rows=2765/5 width=12)
+ Output:["_col0","_col1"],aggregations:["count(VALUE._col0)"],keys:KEY._col0
+ <-Reducer 2 [SIMPLE_EDGE]
+ SHUFFLE [RS_11]
+ PartitionCols:_col0
+ Group By Operator [GBY_10] (rows=2765/5 width=12)
+ Output:["_col0","_col1"],aggregations:["count()"],keys:_col0
+ Merge Join Operator [MERGEJOIN_21] (rows=7286/10 width=4)
+ Conds:RS_6._col1=RS_7._col0(Inner),Output:["_col0"]
+ <-Map 1 [SIMPLE_EDGE]
+ SHUFFLE [RS_6]
+ PartitionCols:_col1
+ Select Operator [SEL_2] (rows=6144/10 width=5)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_19] (rows=6144/10 width=5)
+ predicate:cint BETWEEN 1000000 AND 3000000
+ TableScan [TS_0] (rows=12288/12288 width=5)
+ default@alltypesorc,a,Tbl:COMPLETE,Col:COMPLETE,Output:["csmallint","cint"]
+ <-Map 5 [SIMPLE_EDGE]
+ SHUFFLE [RS_7]
+ PartitionCols:_col0
+ Select Operator [SEL_5] (rows=3424/10 width=8)
+ Output:["_col0"]
+ Filter Operator [FIL_20] (rows=3424/10 width=8)
+ predicate:(cint is not null and cbigint is not null and cint BETWEEN 1000000 AND 3000000)
+ TableScan [TS_3] (rows=12288/12288 width=8)
+ default@alltypesorc,b,Tbl:COMPLETE,Col:COMPLETE,Output:["cint","cbigint"]
+
+PREHOOK: query: select
+ a.csmallint, count(*) c1
+from alltypesorc a join alltypesorc b on a.cint = b.cint
+where
+ a.cint between 1000000 and 3000000 and b.cbigint is not null
+group by a.csmallint
+order by c1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+POSTHOOK: query: select
+ a.csmallint, count(*) c1
+from alltypesorc a join alltypesorc b on a.cint = b.cint
+where
+ a.cint between 1000000 and 3000000 and b.cbigint is not null
+group by a.csmallint
+order by c1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+-13036 1
+-8915 1
+-3799 1
+10782 1
+NULL 6
+PREHOOK: query: select
+ *
+from alltypesorc a join alltypesorc b on a.cint = b.cint
+where
+ a.cint between 1000000 and 3000000 and b.cbigint is not null
+order by a.cint
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+POSTHOOK: query: select
+ *
+from alltypesorc a join alltypesorc b on a.cint = b.cint
+where
+ a.cint between 1000000 and 3000000 and b.cbigint is not null
+order by a.cint
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+PREHOOK: query: -- Try with dynamically partitioned hashjoin
+explain analyze
+select
+ *
+from alltypesorc a join alltypesorc b on a.cint = b.cint
+where
+ a.cint between 1000000 and 3000000 and b.cbigint is not null
+order by a.cint
+PREHOOK: type: QUERY
+POSTHOOK: query: -- Try with dynamically partitioned hashjoin
+explain analyze
+select
+ *
+from alltypesorc a join alltypesorc b on a.cint = b.cint
+where
+ a.cint between 1000000 and 3000000 and b.cbigint is not null
+order by a.cint
+POSTHOOK: type: QUERY
+Plan optimized by CBO.
+
+Vertex dependency in root stage
+Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 4 (CUSTOM_SIMPLE_EDGE)
+Reducer 3 <- Reducer 2 (SIMPLE_EDGE)
+
+Stage-0
+ Fetch Operator
+ limit:-1
+ Stage-1
+ Reducer 3
+ File Output Operator [FS_12]
+ Select Operator [SEL_11] (rows=6758/10 width=215)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23"]
+ <-Reducer 2 [SIMPLE_EDGE]
+ SHUFFLE [RS_10]
+ Map Join Operator [MAPJOIN_17] (rows=6758/10 width=215)
+ Conds:RS_6.KEY.reducesinkkey0=RS_7.KEY.reducesinkkey0(Inner),HybridGraceHashJoin:true,Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23"]
+ <-Map 4 [CUSTOM_SIMPLE_EDGE]
+ PARTITION_ONLY_SHUFFLE [RS_7]
+ PartitionCols:_col2
+ Select Operator [SEL_5] (rows=6144/10 width=215)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"]
+ Filter Operator [FIL_16] (rows=6144/10 width=215)
+ predicate:(cint is not null and cbigint is not null and cint BETWEEN 1000000 AND 3000000)
+ TableScan [TS_3] (rows=12288/12288 width=215)
+ default@alltypesorc,b,Tbl:COMPLETE,Col:NONE,Output:["ctinyint","csmallint","cint","cbigint","cfloat","cdouble","cstring1","cstring2","ctimestamp1","ctimestamp2","cboolean1","cboolean2"]
+ <-Map 1 [CUSTOM_SIMPLE_EDGE]
+ PARTITION_ONLY_SHUFFLE [RS_6]
+ PartitionCols:_col2
+ Select Operator [SEL_2] (rows=6144/10 width=215)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"]
+ Filter Operator [FIL_15] (rows=6144/10 width=215)
+ predicate:cint BETWEEN 1000000 AND 3000000
+ TableScan [TS_0] (rows=12288/12288 width=215)
+ default@alltypesorc,a,Tbl:COMPLETE,Col:NONE,Output:["ctinyint","csmallint","cint","cbigint","cfloat","cdouble","cstring1","cstring2","ctimestamp1","ctimestamp2","cboolean1","cboolean2"]
+
+PREHOOK: query: select
+ *
+from alltypesorc a join alltypesorc b on a.cint = b.cint
+where
+ a.cint between 1000000 and 3000000 and b.cbigint is not null
+order by a.cint
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+POSTHOOK: query: select
+ *
+from alltypesorc a join alltypesorc b on a.cint = b.cint
+where
+ a.cint between 1000000 and 3000000 and b.cbigint is not null
+order by a.cint
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+11 NULL 1000828 1531084669 11.0 NULL wM316f6NqGIkoP388j3F6 poWQQo3Upvt3Wh 1969-12-31 16:00:02.351 NULL false true 11 NULL 1000828 1531084669 11.0 NULL wM316f6NqGIkoP388j3F6 poWQQo3Upvt3Wh 1969-12-31 16:00:02.351 NULL false true
+NULL -3799 1248059 1864027286 NULL -3799.0 Uhps6mMh3IfHB3j7yH62K 4KWs6gw7lv2WYd66P NULL 1969-12-31 15:59:54.622 false true NULL -3799 1248059 1864027286 NULL -3799.0 Uhps6mMh3IfHB3j7yH62K 4KWs6gw7lv2WYd66P NULL 1969-12-31 15:59:54.622 false true
+NULL 10782 1286921 1864027286 NULL 10782.0 ODLrXI8882q8LS8 4KWs6gw7lv2WYd66P NULL 1969-12-31 15:59:52.138 true true NULL 10782 1286921 1864027286 NULL 10782.0 ODLrXI8882q8LS8 4KWs6gw7lv2WYd66P NULL 1969-12-31 15:59:52.138 true true
+NULL -13036 1288927 -1645852809 NULL -13036.0 yinBY725P7V2 xH7445Rals48VOulSyR5F NULL 1969-12-31 16:00:00.763 true false NULL -13036 1288927 -1645852809 NULL -13036.0 yinBY725P7V2 xH7445Rals48VOulSyR5F NULL 1969-12-31 16:00:00.763 true false
+11 NULL 1310786 -413875656 11.0 NULL W0rvA4H1xn0xMG4uk0 8yVVjG 1969-12-31 16:00:02.351 NULL false true 11 NULL 1310786 -413875656 11.0 NULL W0rvA4H1xn0xMG4uk0 8yVVjG 1969-12-31 16:00:02.351 NULL false true
+-51 NULL 2089466 -240556350 -51.0 NULL cXX24dH7tblSj46j2g C31eea0wrHHqvj 1969-12-31 16:00:08.451 NULL true true -51 NULL 2089466 -240556350 -51.0 NULL cXX24dH7tblSj46j2g C31eea0wrHHqvj 1969-12-31 16:00:08.451 NULL true true
+NULL -8915 2101183 1864027286 NULL -8915.0 x7By66525 4KWs6gw7lv2WYd66P NULL 1969-12-31 16:00:05.831 false true NULL -8915 2101183 1864027286 NULL -8915.0 x7By66525 4KWs6gw7lv2WYd66P NULL 1969-12-31 16:00:05.831 false true
+8 NULL 2229621 -381406148 8.0 NULL q7onkS7QRPh5ghOK oKb0bi 1969-12-31 16:00:15.892 NULL true false 8 NULL 2229621 -381406148 8.0 NULL q7onkS7QRPh5ghOK oKb0bi 1969-12-31 16:00:15.892 NULL true false
+8 NULL 2433892 -1611863517 8.0 NULL 674ILv3V2TxFqXP6wSbL VLprkK2XfX 1969-12-31 16:00:15.892 NULL false true 8 NULL 2433892 -1611863517 8.0 NULL 674ILv3V2TxFqXP6wSbL VLprkK2XfX 1969-12-31 16:00:15.892 NULL false true
+-51 NULL 2949963 -1580871111 -51.0 NULL 0K68k3bdl7jO7 TPPAu 1969-12-31 16:00:08.451 NULL true false -51 NULL 2949963 -1580871111 -51.0 NULL 0K68k3bdl7jO7 TPPAu 1969-12-31 16:00:08.451 NULL true false
+PREHOOK: query: select
+ count(*)
+from alltypesorc a join alltypesorc b on a.cint = b.cint
+where
+ a.cint between 1000000 and 3000000 and b.cbigint is not null
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+POSTHOOK: query: select
+ count(*)
+from alltypesorc a join alltypesorc b on a.cint = b.cint
+where
+ a.cint between 1000000 and 3000000 and b.cbigint is not null
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+PREHOOK: query: explain analyze
+select
+ count(*)
+from alltypesorc a join alltypesorc b on a.cint = b.cint
+where
+ a.cint between 1000000 and 3000000 and b.cbigint is not null
+PREHOOK: type: QUERY
+POSTHOOK: query: explain analyze
+select
+ count(*)
+from alltypesorc a join alltypesorc b on a.cint = b.cint
+where
+ a.cint between 1000000 and 3000000 and b.cbigint is not null
+POSTHOOK: type: QUERY
+Plan optimized by CBO.
+
+Vertex dependency in root stage
+Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 4 (CUSTOM_SIMPLE_EDGE)
+Reducer 3 <- Reducer 2 (SIMPLE_EDGE)
+
+Stage-0
+ Fetch Operator
+ limit:-1
+ Stage-1
+ Reducer 3
+ File Output Operator [FS_14]
+ Group By Operator [GBY_12] (rows=1/1 width=8)
+ Output:["_col0"],aggregations:["count(VALUE._col0)"]
+ <-Reducer 2 [SIMPLE_EDGE]
+ SHUFFLE [RS_11]
+ Group By Operator [GBY_10] (rows=1/14 width=8)
+ Output:["_col0"],aggregations:["count()"]
+ Map Join Operator [MAPJOIN_19] (rows=6758/10 width=215)
+ Conds:RS_6.KEY.reducesinkkey0=RS_7.KEY.reducesinkkey0(Inner),HybridGraceHashJoin:true
+ <-Map 4 [CUSTOM_SIMPLE_EDGE]
+ PARTITION_ONLY_SHUFFLE [RS_7]
+ PartitionCols:_col0
+ Select Operator [SEL_5] (rows=6144/10 width=215)
+ Output:["_col0"]
+ Filter Operator [FIL_18] (rows=6144/10 width=215)
+ predicate:(cint is not null and cbigint is not null and cint BETWEEN 1000000 AND 3000000)
+ TableScan [TS_3] (rows=12288/12288 width=215)
+ default@alltypesorc,b,Tbl:COMPLETE,Col:NONE,Output:["cint","cbigint"]
+ <-Map 1 [CUSTOM_SIMPLE_EDGE]
+ PARTITION_ONLY_SHUFFLE [RS_6]
+ PartitionCols:_col0
+ Select Operator [SEL_2] (rows=6144/10 width=215)
+ Output:["_col0"]
+ Filter Operator [FIL_17] (rows=6144/10 width=215)
+ predicate:cint BETWEEN 1000000 AND 3000000
+ TableScan [TS_0] (rows=12288/12288 width=215)
+ default@alltypesorc,a,Tbl:COMPLETE,Col:NONE,Output:["cint"]
+
+PREHOOK: query: select
+ count(*)
+from alltypesorc a join alltypesorc b on a.cint = b.cint
+where
+ a.cint between 1000000 and 3000000 and b.cbigint is not null
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+POSTHOOK: query: select
+ count(*)
+from alltypesorc a join alltypesorc b on a.cint = b.cint
+where
+ a.cint between 1000000 and 3000000 and b.cbigint is not null
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+10
+PREHOOK: query: select
+ a.csmallint, count(*) c1
+from alltypesorc a join alltypesorc b on a.cint = b.cint
+where
+ a.cint between 1000000 and 3000000 and b.cbigint is not null
+group by a.csmallint
+order by c1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+POSTHOOK: query: select
+ a.csmallint, count(*) c1
+from alltypesorc a join alltypesorc b on a.cint = b.cint
+where
+ a.cint between 1000000 and 3000000 and b.cbigint is not null
+group by a.csmallint
+order by c1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+PREHOOK: query: explain analyze
+select
+ a.csmallint, count(*) c1
+from alltypesorc a join alltypesorc b on a.cint = b.cint
+where
+ a.cint between 1000000 and 3000000 and b.cbigint is not null
+group by a.csmallint
+order by c1
+PREHOOK: type: QUERY
+POSTHOOK: query: explain analyze
+select
+ a.csmallint, count(*) c1
+from alltypesorc a join alltypesorc b on a.cint = b.cint
+where
+ a.cint between 1000000 and 3000000 and b.cbigint is not null
+group by a.csmallint
+order by c1
+POSTHOOK: type: QUERY
+Plan optimized by CBO.
+
+Vertex dependency in root stage
+Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 5 (CUSTOM_SIMPLE_EDGE)
+Reducer 3 <- Reducer 2 (SIMPLE_EDGE)
+Reducer 4 <- Reducer 3 (SIMPLE_EDGE)
+
+Stage-0
+ Fetch Operator
+ limit:-1
+ Stage-1
+ Reducer 4
+ File Output Operator [FS_16]
+ Select Operator [SEL_15] (rows=3379/5 width=215)
+ Output:["_col0","_col1"]
+ <-Reducer 3 [SIMPLE_EDGE]
+ SHUFFLE [RS_14]
+ Group By Operator [GBY_12] (rows=3379/5 width=215)
+ Output:["_col0","_col1"],aggregations:["count(VALUE._col0)"],keys:KEY._col0
+ <-Reducer 2 [SIMPLE_EDGE]
+ SHUFFLE [RS_11]
+ PartitionCols:_col0
+ Group By Operator [GBY_10] (rows=6758/9 width=215)
+ Output:["_col0","_col1"],aggregations:["count()"],keys:_col0
+ Map Join Operator [MAPJOIN_21] (rows=6758/10 width=215)
+ Conds:RS_6.KEY.reducesinkkey0=RS_7.KEY.reducesinkkey0(Inner),HybridGraceHashJoin:true,Output:["_col0"]
+ <-Map 5 [CUSTOM_SIMPLE_EDGE]
+ PARTITION_ONLY_SHUFFLE [RS_7]
+ PartitionCols:_col0
+ Select Operator [SEL_5] (rows=6144/10 width=215)
+ Output:["_col0"]
+ Filter Operator [FIL_20] (rows=6144/10 width=215)
+ predicate:(cint is not null and cbigint is not null and cint BETWEEN 1000000 AND 3000000)
+ TableScan [TS_3] (rows=12288/12288 width=215)
+ default@alltypesorc,b,Tbl:COMPLETE,Col:NONE,Output:["cint","cbigint"]
+ <-Map 1 [CUSTOM_SIMPLE_EDGE]
+ PARTITION_ONLY_SHUFFLE [RS_6]
+ PartitionCols:_col1
+ Select Operator [SEL_2] (rows=6144/10 width=215)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_19] (rows=6144/10 width=215)
+ predicate:cint BETWEEN 1000000 AND 3000000
+ TableScan [TS_0] (rows=12288/12288 width=215)
+ default@alltypesorc,a,Tbl:COMPLETE,Col:NONE,Output:["csmallint","cint"]
+
+PREHOOK: query: select
+ a.csmallint, count(*) c1
+from alltypesorc a join alltypesorc b on a.cint = b.cint
+where
+ a.cint between 1000000 and 3000000 and b.cbigint is not null
+group by a.csmallint
+order by c1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+POSTHOOK: query: select
+ a.csmallint, count(*) c1
+from alltypesorc a join alltypesorc b on a.cint = b.cint
+where
+ a.cint between 1000000 and 3000000 and b.cbigint is not null
+group by a.csmallint
+order by c1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+-8915 1
+-3799 1
+10782 1
+-13036 1
+NULL 6
http://git-wip-us.apache.org/repos/asf/hive/blob/fd72ec4a/ql/src/test/results/clientpositive/tez/explainanalyze_5.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/explainanalyze_5.q.out b/ql/src/test/results/clientpositive/tez/explainanalyze_5.q.out
new file mode 100644
index 0000000..39bc6f4
--- /dev/null
+++ b/ql/src/test/results/clientpositive/tez/explainanalyze_5.q.out
@@ -0,0 +1,445 @@
+PREHOOK: query: analyze table src compute statistics
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@src
+FAILED: Hive Internal Error: java.lang.RuntimeException(Cannot overwrite read-only table: src)
+java.lang.RuntimeException: Cannot overwrite read-only table: src
+#### A masked pattern was here ####
+
+PREHOOK: query: explain analyze analyze table src compute statistics
+PREHOOK: type: QUERY
+POSTHOOK: query: explain analyze analyze table src compute statistics
+POSTHOOK: type: QUERY
+Stage-2
+ Stats-Aggr Operator
+ Stage-0
+ Map 1
+ TableScan [TS_0] (rows=500/0 width=10)
+ default@src,src,Tbl:COMPLETE,Col:COMPLETE
+
+PREHOOK: query: analyze table src compute statistics for columns
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: analyze table src compute statistics for columns
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+PREHOOK: query: explain analyze analyze table src compute statistics for columns
+PREHOOK: type: QUERY
+POSTHOOK: query: explain analyze analyze table src compute statistics for columns
+POSTHOOK: type: QUERY
+Vertex dependency in root stage
+Reducer 2 <- Map 1 (SIMPLE_EDGE)
+
+Stage-2
+ Column Stats Work{}
+ Stage-0
+ Reducer 2
+ File Output Operator [FS_6]
+ Group By Operator [GBY_4] (rows=1/1 width=960)
+ Output:["_col0","_col1"],aggregations:["compute_stats(VALUE._col0)","compute_stats(VALUE._col1)"]
+ <-Map 1 [SIMPLE_EDGE]
+ SHUFFLE [RS_3]
+ Group By Operator [GBY_2] (rows=1/1 width=984)
+ Output:["_col0","_col1"],aggregations:["compute_stats(key, 16)","compute_stats(value, 16)"]
+ Select Operator [SEL_1] (rows=500/500 width=178)
+ Output:["key","value"]
+ TableScan [TS_0] (rows=500/500 width=178)
+ default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
+
+PREHOOK: query: drop table src_multi2
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table src_multi2
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table src_multi2 like src
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@src_multi2
+POSTHOOK: query: create table src_multi2 like src
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@src_multi2
+PREHOOK: query: insert overwrite table src_multi2 select subq.key, src.value from (select * from src union select * from src1)subq join src on subq.key=src.key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Input: default@src1
+PREHOOK: Output: default@src_multi2
+POSTHOOK: query: insert overwrite table src_multi2 select subq.key, src.value from (select * from src union select * from src1)subq join src on subq.key=src.key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Input: default@src1
+POSTHOOK: Output: default@src_multi2
+PREHOOK: query: explain analyze insert overwrite table src_multi2 select subq.key, src.value from (select * from src union select * from src1)subq join src on subq.key=src.key
+PREHOOK: type: QUERY
+POSTHOOK: query: explain analyze insert overwrite table src_multi2 select subq.key, src.value from (select * from src union select * from src1)subq join src on subq.key=src.key
+POSTHOOK: type: QUERY
+Plan optimized by CBO.
+
+Vertex dependency in root stage
+Map 1 <- Union 2 (CONTAINS)
+Map 6 <- Union 2 (CONTAINS)
+Reducer 3 <- Union 2 (SIMPLE_EDGE)
+Reducer 4 <- Map 7 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE)
+Reducer 5 <- Reducer 4 (SIMPLE_EDGE)
+
+Stage-4
+ Column Stats Work{}
+ Stage-3
+ Stats-Aggr Operator
+ Stage-0
+ Move Operator
+ table:{"name:":"default.src_multi2"}
+ Stage-2
+ Dependency Collection{}
+ Stage-1
+ Reducer 5
+ File Output Operator [FS_6]
+ Group By Operator [GBY_4] (rows=1/1 width=960)
+ Output:["_col0","_col1"],aggregations:["compute_stats(VALUE._col0)","compute_stats(VALUE._col1)"]
+ <-Reducer 4 [SIMPLE_EDGE]
+ File Output Operator [FS_20]
+ table:{"name:":"default.src_multi2"}
+ Select Operator [SEL_19] (rows=639/508 width=178)
+ Output:["_col0","_col1"]
+ Merge Join Operator [MERGEJOIN_27] (rows=639/508 width=178)
+ Conds:RS_16._col0=RS_17._col0(Inner),Output:["_col0","_col3"]
+ <-Map 7 [SIMPLE_EDGE]
+ SHUFFLE [RS_17]
+ PartitionCols:_col0
+ Select Operator [SEL_15] (rows=500/500 width=178)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_26] (rows=500/500 width=178)
+ predicate:key is not null
+ TableScan [TS_13] (rows=500/500 width=178)
+ default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
+ <-Reducer 3 [SIMPLE_EDGE]
+ SHUFFLE [RS_16]
+ PartitionCols:_col0
+ Select Operator [SEL_12] (rows=262/319 width=178)
+ Output:["_col0"]
+ Group By Operator [GBY_11] (rows=262/319 width=178)
+ Output:["_col0","_col1"],keys:KEY._col0, KEY._col1
+ <-Union 2 [SIMPLE_EDGE]
+ <-Map 1 [CONTAINS]
+ Reduce Output Operator [RS_10]
+ PartitionCols:_col0, _col1
+ Group By Operator [GBY_9] (rows=262/331 width=178)
+ Output:["_col0","_col1"],keys:_col0, _col1
+ Select Operator [SEL_2] (rows=500/500 width=178)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_24] (rows=500/500 width=178)
+ predicate:key is not null
+ TableScan [TS_0] (rows=500/500 width=178)
+ Output:["key","value"]
+ <-Map 6 [CONTAINS]
+ Reduce Output Operator [RS_10]
+ PartitionCols:_col0, _col1
+ Group By Operator [GBY_9] (rows=262/331 width=178)
+ Output:["_col0","_col1"],keys:_col0, _col1
+ Select Operator [SEL_5] (rows=25/25 width=175)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_25] (rows=25/25 width=175)
+ predicate:key is not null
+ TableScan [TS_3] (rows=25/25 width=175)
+ Output:["key","value"]
+ SHUFFLE [RS_3]
+ Group By Operator [GBY_2] (rows=1/1 width=984)
+ Output:["_col0","_col1"],aggregations:["compute_stats(key, 16)","compute_stats(value, 16)"]
+ Select Operator [SEL_1] (rows=639/508 width=178)
+ Output:["key","value"]
+ Please refer to the previous Select Operator [SEL_19]
+
+PREHOOK: query: select count(*) from (select * from src union select * from src1)subq
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Input: default@src1
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from (select * from src union select * from src1)subq
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Input: default@src1
+#### A masked pattern was here ####
+319
+PREHOOK: query: insert overwrite table src_multi2 select subq.key, src.value from (select * from src union select * from src1)subq join src on subq.key=src.key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Input: default@src1
+PREHOOK: Output: default@src_multi2
+POSTHOOK: query: insert overwrite table src_multi2 select subq.key, src.value from (select * from src union select * from src1)subq join src on subq.key=src.key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Input: default@src1
+POSTHOOK: Output: default@src_multi2
+POSTHOOK: Lineage: src_multi2.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src1)src1.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: src_multi2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: describe formatted src_multi2
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@src_multi2
+POSTHOOK: query: describe formatted src_multi2
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@src_multi2
+# col_name data_type comment
+
+key string default
+value string default
+
+# Detailed Table Information
+Database: default
+#### A masked pattern was here ####
+Retention: 0
+#### A masked pattern was here ####
+Table Type: MANAGED_TABLE
+Table Parameters:
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}}
+ numFiles 1
+ numRows 508
+ rawDataSize 5400
+ totalSize 5908
+#### A masked pattern was here ####
+
+# Storage Information
+SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+InputFormat: org.apache.hadoop.mapred.TextInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+Storage Desc Params:
+ serialization.format 1
+PREHOOK: query: -- SORT_QUERY_RESULTS
+
+create table acid_uami(i int,
+ de decimal(5,2),
+ vc varchar(128)) clustered by (i) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true')
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@acid_uami
+POSTHOOK: query: -- SORT_QUERY_RESULTS
+
+create table acid_uami(i int,
+ de decimal(5,2),
+ vc varchar(128)) clustered by (i) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@acid_uami
+PREHOOK: query: insert into table acid_uami values
+ (1, 109.23, 'mary had a little lamb'),
+ (6553, 923.19, 'its fleece was white as snow')
+PREHOOK: type: QUERY
+PREHOOK: Input: default@values__tmp__table__1
+PREHOOK: Output: default@acid_uami
+POSTHOOK: query: insert into table acid_uami values
+ (1, 109.23, 'mary had a little lamb'),
+ (6553, 923.19, 'its fleece was white as snow')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@values__tmp__table__1
+POSTHOOK: Output: default@acid_uami
+POSTHOOK: Lineage: acid_uami.de EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ]
+POSTHOOK: Lineage: acid_uami.i EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+POSTHOOK: Lineage: acid_uami.vc EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col3, type:string, comment:), ]
+PREHOOK: query: insert into table acid_uami values
+ (10, 119.23, 'and everywhere that mary went'),
+ (65530, 823.19, 'the lamb was sure to go')
+PREHOOK: type: QUERY
+PREHOOK: Input: default@values__tmp__table__2
+PREHOOK: Output: default@acid_uami
+POSTHOOK: query: insert into table acid_uami values
+ (10, 119.23, 'and everywhere that mary went'),
+ (65530, 823.19, 'the lamb was sure to go')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@values__tmp__table__2
+POSTHOOK: Output: default@acid_uami
+POSTHOOK: Lineage: acid_uami.de EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col2, type:string, comment:), ]
+POSTHOOK: Lineage: acid_uami.i EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+POSTHOOK: Lineage: acid_uami.vc EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col3, type:string, comment:), ]
+PREHOOK: query: select * from acid_uami order by de
+PREHOOK: type: QUERY
+PREHOOK: Input: default@acid_uami
+#### A masked pattern was here ####
+POSTHOOK: query: select * from acid_uami order by de
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@acid_uami
+#### A masked pattern was here ####
+1 109.23 mary had a little lamb
+10 119.23 and everywhere that mary went
+6553 923.19 its fleece was white as snow
+65530 823.19 the lamb was sure to go
+PREHOOK: query: update acid_uami set de = 3.14 where de = 109.23 or de = 119.23
+PREHOOK: type: QUERY
+PREHOOK: Input: default@acid_uami
+PREHOOK: Output: default@acid_uami
+POSTHOOK: query: update acid_uami set de = 3.14 where de = 109.23 or de = 119.23
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@acid_uami
+POSTHOOK: Output: default@acid_uami
+PREHOOK: query: explain analyze update acid_uami set de = 3.14 where de = 109.23 or de = 119.23
+PREHOOK: type: QUERY
+POSTHOOK: query: explain analyze update acid_uami set de = 3.14 where de = 109.23 or de = 119.23
+POSTHOOK: type: QUERY
+Vertex dependency in root stage
+Reducer 2 <- Map 1 (SIMPLE_EDGE)
+
+Stage-3
+ Stats-Aggr Operator
+ Stage-0
+ Move Operator
+ table:{"name:":"default.acid_uami"}
+ Stage-2
+ Dependency Collection{}
+ Stage-1
+ Reducer 2
+ File Output Operator [FS_8]
+ table:{"name:":"default.acid_uami"}
+ Select Operator [SEL_4] (rows=6/2 width=302)
+ Output:["_col0","_col1","_col2","_col3"]
+ <-Map 1 [SIMPLE_EDGE]
+ SHUFFLE [RS_3]
+ PartitionCols:UDFToInteger(_col0)
+ Select Operator [SEL_2] (rows=6/2 width=302)
+ Output:["_col0","_col1","_col3"]
+ Filter Operator [FIL_9] (rows=6/2 width=226)
+ predicate:((de = 109.23) or (de = 119.23))
+ TableScan [TS_0] (rows=8/4 width=226)
+ default@acid_uami,acid_uami, ACID table,Tbl:COMPLETE,Col:COMPLETE,Output:["i","de","vc"]
+
+PREHOOK: query: select * from acid_uami order by de
+PREHOOK: type: QUERY
+PREHOOK: Input: default@acid_uami
+#### A masked pattern was here ####
+POSTHOOK: query: select * from acid_uami order by de
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@acid_uami
+#### A masked pattern was here ####
+1 109.23 mary had a little lamb
+10 119.23 and everywhere that mary went
+6553 923.19 its fleece was white as snow
+65530 823.19 the lamb was sure to go
+PREHOOK: query: update acid_uami set de = 3.14 where de = 109.23 or de = 119.23
+PREHOOK: type: QUERY
+PREHOOK: Input: default@acid_uami
+PREHOOK: Output: default@acid_uami
+POSTHOOK: query: update acid_uami set de = 3.14 where de = 109.23 or de = 119.23
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@acid_uami
+POSTHOOK: Output: default@acid_uami
+PREHOOK: query: select * from acid_uami order by de
+PREHOOK: type: QUERY
+PREHOOK: Input: default@acid_uami
+#### A masked pattern was here ####
+POSTHOOK: query: select * from acid_uami order by de
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@acid_uami
+#### A masked pattern was here ####
+1 3.14 mary had a little lamb
+10 3.14 and everywhere that mary went
+6553 923.19 its fleece was white as snow
+65530 823.19 the lamb was sure to go
+PREHOOK: query: create table acid_dot(
+ ctinyint TINYINT,
+ csmallint SMALLINT,
+ cint INT,
+ cbigint BIGINT,
+ cfloat FLOAT,
+ cdouble DOUBLE,
+ cstring1 STRING,
+ cstring2 STRING,
+ ctimestamp1 TIMESTAMP,
+ ctimestamp2 TIMESTAMP,
+ cboolean1 BOOLEAN,
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+PREHOOK: Output: database:default
+PREHOOK: Output: default@acid_dot
+POSTHOOK: query: create table acid_dot(
+ ctinyint TINYINT,
+ csmallint SMALLINT,
+ cint INT,
+ cbigint BIGINT,
+ cfloat FLOAT,
+ cdouble DOUBLE,
+ cstring1 STRING,
+ cstring2 STRING,
+ ctimestamp1 TIMESTAMP,
+ ctimestamp2 TIMESTAMP,
+ cboolean1 BOOLEAN,
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@acid_dot
+PREHOOK: query: select count(*) from acid_dot
+PREHOOK: type: QUERY
+PREHOOK: Input: default@acid_dot
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from acid_dot
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@acid_dot
+#### A masked pattern was here ####
+12288
+PREHOOK: query: delete from acid_dot where cint < -1070551679
+PREHOOK: type: QUERY
+PREHOOK: Input: default@acid_dot
+PREHOOK: Output: default@acid_dot
+POSTHOOK: query: delete from acid_dot where cint < -1070551679
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@acid_dot
+POSTHOOK: Output: default@acid_dot
+PREHOOK: query: explain analyze delete from acid_dot where cint < -1070551679
+PREHOOK: type: QUERY
+POSTHOOK: query: explain analyze delete from acid_dot where cint < -1070551679
+POSTHOOK: type: QUERY
+Vertex dependency in root stage
+Reducer 2 <- Map 1 (SIMPLE_EDGE)
+
+Stage-3
+ Stats-Aggr Operator
+ Stage-0
+ Move Operator
+ table:{"name:":"default.acid_dot"}
+ Stage-2
+ Dependency Collection{}
+ Stage-1
+ Reducer 2
+ File Output Operator [FS_7]
+ table:{"name:":"default.acid_dot"}
+ Select Operator [SEL_4] (rows=31436/8 width=4)
+ Output:["_col0"]
+ <-Map 1 [SIMPLE_EDGE]
+ SHUFFLE [RS_3]
+ PartitionCols:UDFToInteger(_col0)
+ Select Operator [SEL_2] (rows=31436/8 width=4)
+ Output:["_col0"]
+ Filter Operator [FIL_8] (rows=31436/8 width=4)
+ predicate:(cint < -1070551679)
+ TableScan [TS_0] (rows=94309/12288 width=4)
+ default@acid_dot,acid_dot, ACID table,Tbl:COMPLETE,Col:NONE,Output:["cint"]
+
+PREHOOK: query: select count(*) from acid_dot
+PREHOOK: type: QUERY
+PREHOOK: Input: default@acid_dot
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from acid_dot
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@acid_dot
+#### A masked pattern was here ####
+12288
+PREHOOK: query: delete from acid_dot where cint < -1070551679
+PREHOOK: type: QUERY
+PREHOOK: Input: default@acid_dot
+PREHOOK: Output: default@acid_dot
+POSTHOOK: query: delete from acid_dot where cint < -1070551679
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@acid_dot
+POSTHOOK: Output: default@acid_dot
+PREHOOK: query: select count(*) from acid_dot
+PREHOOK: type: QUERY
+PREHOOK: Input: default@acid_dot
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from acid_dot
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@acid_dot
+#### A masked pattern was here ####
+12280
[11/50] [abbrv] hive git commit: HIVE-14367 : Estimated size for
constant nulls is 0 (Ashutosh Chauhan via Jesus Camacho Rodriguez)
Posted by om...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/tez/explainuser_1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/explainuser_1.q.out b/ql/src/test/results/clientpositive/tez/explainuser_1.q.out
index e7f8733..812315e 100644
--- a/ql/src/test/results/clientpositive/tez/explainuser_1.q.out
+++ b/ql/src/test/results/clientpositive/tez/explainuser_1.q.out
@@ -300,18 +300,18 @@ Stage-0
Stage-1
Reducer 2
File Output Operator [FS_7]
- Select Operator [SEL_5] (rows=10 width=88)
+ Select Operator [SEL_5] (rows=10 width=97)
Output:["_col0","_col1","_col2"]
- Group By Operator [GBY_4] (rows=10 width=91)
+ Group By Operator [GBY_4] (rows=10 width=101)
Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_3]
PartitionCols:_col0, _col1, _col2
- Group By Operator [GBY_2] (rows=10 width=91)
+ Group By Operator [GBY_2] (rows=10 width=101)
Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float
- Select Operator [SEL_1] (rows=20 width=83)
+ Select Operator [SEL_1] (rows=20 width=88)
Output:["key","c_int","c_float"]
- TableScan [TS_0] (rows=20 width=83)
+ TableScan [TS_0] (rows=20 width=88)
default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"]
PREHOOK: query: explain select x, y, count(*) from (select key, (c_int+c_float+1+2) as x, sum(c_int) as y from cbo_t1 group by c_float, cbo_t1.c_int, key) R group by y, x
@@ -339,18 +339,18 @@ Stage-0
PartitionCols:_col0, _col1
Group By Operator [GBY_8] (rows=5 width=20)
Output:["_col0","_col1","_col2"],aggregations:["count()"],keys:_col0, _col1
- Select Operator [SEL_5] (rows=10 width=91)
+ Select Operator [SEL_5] (rows=10 width=101)
Output:["_col0","_col1"]
- Group By Operator [GBY_4] (rows=10 width=91)
+ Group By Operator [GBY_4] (rows=10 width=101)
Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_3]
PartitionCols:_col0, _col1, _col2
- Group By Operator [GBY_2] (rows=10 width=91)
+ Group By Operator [GBY_2] (rows=10 width=101)
Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float
- Select Operator [SEL_1] (rows=20 width=83)
+ Select Operator [SEL_1] (rows=20 width=88)
Output:["key","c_int","c_float"]
- TableScan [TS_0] (rows=20 width=83)
+ TableScan [TS_0] (rows=20 width=88)
default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"]
PREHOOK: query: explain select cbo_t3.c_int, c, count(*) from (select key as a, c_int+1 as b, sum(c_int) as c from cbo_t1 where (cbo_t1.c_int + 1 >= 0) and (cbo_t1.c_int > 0 or cbo_t1.c_float >= 0) group by c_float, cbo_t1.c_int, key order by a) cbo_t1 join (select key as p, c_int+1 as q, sum(c_int) as r from cbo_t2 where (cbo_t2.c_int + 1 >= 0) and (cbo_t2.c_int > 0 or cbo_t2.c_float >= 0) group by c_float, cbo_t2.c_int, key order by q/10 desc, r asc) cbo_t2 on cbo_t1.a=p join cbo_t3 on cbo_t1.a=key where (b + cbo_t2.q >= 0) and (b > 0 or c_int >= 0) group by cbo_t3.c_int, c order by cbo_t3.c_int+c desc, c
@@ -397,11 +397,11 @@ Stage-0
<-Map 11 [SIMPLE_EDGE]
SHUFFLE [RS_31]
PartitionCols:_col0
- Select Operator [SEL_29] (rows=18 width=79)
+ Select Operator [SEL_29] (rows=18 width=84)
Output:["_col0","_col1"]
- Filter Operator [FIL_50] (rows=18 width=79)
+ Filter Operator [FIL_50] (rows=18 width=84)
predicate:key is not null
- TableScan [TS_27] (rows=20 width=80)
+ TableScan [TS_27] (rows=20 width=84)
default@cbo_t3,cbo_t3,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"]
<-Reducer 4 [SIMPLE_EDGE]
SHUFFLE [RS_30]
@@ -428,9 +428,9 @@ Stage-0
PartitionCols:_col0, _col1, _col2
Group By Operator [GBY_14] (rows=2 width=101)
Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float
- Filter Operator [FIL_49] (rows=5 width=74)
+ Filter Operator [FIL_49] (rows=5 width=93)
predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0.0)) and key is not null)
- TableScan [TS_11] (rows=20 width=83)
+ TableScan [TS_11] (rows=20 width=88)
default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"]
<-Reducer 3 [SIMPLE_EDGE]
SHUFFLE [RS_22]
@@ -448,9 +448,9 @@ Stage-0
PartitionCols:_col0, _col1, _col2
Group By Operator [GBY_3] (rows=2 width=101)
Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float
- Filter Operator [FIL_48] (rows=5 width=74)
+ Filter Operator [FIL_48] (rows=5 width=93)
predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0.0)) and key is not null)
- TableScan [TS_0] (rows=20 width=83)
+ TableScan [TS_0] (rows=20 width=88)
default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"]
PREHOOK: query: explain select cbo_t3.c_int, c, count(*) from (select key as a, c_int+1 as b, sum(c_int) as c from cbo_t1 where (cbo_t1.c_int + 1 >= 0) and (cbo_t1.c_int > 0 or cbo_t1.c_float >= 0) group by c_float, cbo_t1.c_int, key having cbo_t1.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by b % c asc, b desc) cbo_t1 left outer join (select key as p, c_int+1 as q, sum(c_int) as r from cbo_t2 where (cbo_t2.c_int + 1 >= 0) and (cbo_t2.c_int > 0 or cbo_t2.c_float >= 0) group by c_float, cbo_t2.c_int, key having cbo_t2.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0) cbo_t2 on cbo_t1.a=p left outer join cbo_t3 on cbo_t1.a=key where (b + cbo_t2.q >= 0) and (b > 0 or c_int >= 0) group by cbo_t3.c_int, c having cbo_t3.c_int > 0 and (c_int >=1 or c >= 1) and (c_int + c) >= 0 order by cbo_t3.c_int % c asc, cbo_t3.c_int desc
@@ -494,11 +494,11 @@ Stage-0
<-Map 10 [SIMPLE_EDGE]
SHUFFLE [RS_28]
PartitionCols:_col0
- Select Operator [SEL_26] (rows=18 width=79)
+ Select Operator [SEL_26] (rows=18 width=84)
Output:["_col0","_col1"]
- Filter Operator [FIL_46] (rows=18 width=79)
+ Filter Operator [FIL_46] (rows=18 width=84)
predicate:((c_int > 0) and key is not null)
- TableScan [TS_24] (rows=20 width=80)
+ TableScan [TS_24] (rows=20 width=84)
default@cbo_t3,cbo_t3,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"]
<-Reducer 4 [SIMPLE_EDGE]
SHUFFLE [RS_27]
@@ -527,7 +527,7 @@ Stage-0
Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float
Filter Operator [FIL_44] (rows=1 width=93)
predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0.0)) and (c_float > 0.0) and ((c_int >= 1) or (c_float >= 1.0)) and ((UDFToFloat(c_int) + c_float) >= 0.0) and key is not null)
- TableScan [TS_0] (rows=20 width=83)
+ TableScan [TS_0] (rows=20 width=88)
default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"]
<-Reducer 9 [SIMPLE_EDGE]
SHUFFLE [RS_20]
@@ -543,7 +543,7 @@ Stage-0
Output:["_col0","_col1","_col2"],keys:key, c_int, c_float
Filter Operator [FIL_45] (rows=1 width=93)
predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0.0)) and (c_float > 0.0) and ((c_int >= 1) or (c_float >= 1.0)) and ((UDFToFloat(c_int) + c_float) >= 0.0) and key is not null)
- TableScan [TS_11] (rows=20 width=83)
+ TableScan [TS_11] (rows=20 width=88)
default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"]
PREHOOK: query: explain select cbo_t3.c_int, c, count(*) from (select key as a, c_int+1 as b, sum(c_int) as c from cbo_t1 where (cbo_t1.c_int + 1 >= 0) and (cbo_t1.c_int > 0 or cbo_t1.c_float >= 0) group by c_float, cbo_t1.c_int, key having cbo_t1.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by b+c, a desc) cbo_t1 right outer join (select key as p, c_int+1 as q, sum(c_int) as r from cbo_t2 where (cbo_t2.c_int + 1 >= 0) and (cbo_t2.c_int > 0 or cbo_t2.c_float >= 0) group by c_float, cbo_t2.c_int, key having cbo_t2.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0) cbo_t2 on cbo_t1.a=p right outer join cbo_t3 on cbo_t1.a=key where (b + cbo_t2.q >= 2) and (b > 0 or c_int >= 0) group by cbo_t3.c_int, c
@@ -583,9 +583,9 @@ Stage-0
<-Map 8 [SIMPLE_EDGE]
SHUFFLE [RS_23]
PartitionCols:_col0
- Select Operator [SEL_20] (rows=20 width=80)
+ Select Operator [SEL_20] (rows=20 width=84)
Output:["_col0","_col1"]
- TableScan [TS_19] (rows=20 width=80)
+ TableScan [TS_19] (rows=20 width=84)
default@cbo_t3,cbo_t3,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"]
<-Reducer 3 [SIMPLE_EDGE]
SHUFFLE [RS_21]
@@ -605,7 +605,7 @@ Stage-0
Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float
Filter Operator [FIL_34] (rows=1 width=93)
predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0.0)) and (c_float > 0.0) and ((c_int >= 1) or (c_float >= 1.0)) and ((UDFToFloat(c_int) + c_float) >= 0.0))
- TableScan [TS_0] (rows=20 width=83)
+ TableScan [TS_0] (rows=20 width=88)
default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"]
<-Reducer 7 [SIMPLE_EDGE]
SHUFFLE [RS_22]
@@ -621,7 +621,7 @@ Stage-0
Output:["_col0","_col1","_col2"],keys:key, c_int, c_float
Filter Operator [FIL_35] (rows=1 width=93)
predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0.0)) and (c_float > 0.0) and ((c_int >= 1) or (c_float >= 1.0)) and ((UDFToFloat(c_int) + c_float) >= 0.0))
- TableScan [TS_11] (rows=20 width=83)
+ TableScan [TS_11] (rows=20 width=88)
default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"]
PREHOOK: query: explain select cbo_t3.c_int, c, count(*) from (select key as a, c_int+1 as b, sum(c_int) as c from cbo_t1 where (cbo_t1.c_int + 1 >= 0) and (cbo_t1.c_int > 0 or cbo_t1.c_float >= 0) group by c_float, cbo_t1.c_int, key having cbo_t1.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by c+a desc) cbo_t1 full outer join (select key as p, c_int+1 as q, sum(c_int) as r from cbo_t2 where (cbo_t2.c_int + 1 >= 0) and (cbo_t2.c_int > 0 or cbo_t2.c_float >= 0) group by c_float, cbo_t2.c_int, key having cbo_t2.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by p+q desc, r asc) cbo_t2 on cbo_t1.a=p full outer join cbo_t3 on cbo_t1.a=key where (b + cbo_t2.q >= 0) and (b > 0 or c_int >= 0) group by cbo_t3.c_int, c having cbo_t3.c_int > 0 and (c_int >=1 or c >= 1) and (c_int + c) >= 0 order by cbo_t3.c_int
@@ -665,11 +665,11 @@ Stage-0
<-Map 10 [SIMPLE_EDGE]
SHUFFLE [RS_27]
PartitionCols:_col0
- Select Operator [SEL_24] (rows=20 width=80)
+ Select Operator [SEL_24] (rows=20 width=84)
Output:["_col0","_col1"]
- Filter Operator [FIL_41] (rows=20 width=80)
+ Filter Operator [FIL_41] (rows=20 width=84)
predicate:(c_int > 0)
- TableScan [TS_22] (rows=20 width=80)
+ TableScan [TS_22] (rows=20 width=84)
default@cbo_t3,cbo_t3,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"]
<-Reducer 3 [SIMPLE_EDGE]
SHUFFLE [RS_25]
@@ -689,7 +689,7 @@ Stage-0
Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float
Filter Operator [FIL_39] (rows=1 width=93)
predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0.0)) and (c_float > 0.0) and ((c_int >= 1) or (c_float >= 1.0)) and ((UDFToFloat(c_int) + c_float) >= 0.0))
- TableScan [TS_0] (rows=20 width=83)
+ TableScan [TS_0] (rows=20 width=88)
default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"]
<-Reducer 9 [SIMPLE_EDGE]
SHUFFLE [RS_26]
@@ -709,7 +709,7 @@ Stage-0
Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float
Filter Operator [FIL_40] (rows=1 width=93)
predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0.0)) and (c_float > 0.0) and ((c_int >= 1) or (c_float >= 1.0)) and ((UDFToFloat(c_int) + c_float) >= 0.0))
- TableScan [TS_11] (rows=20 width=83)
+ TableScan [TS_11] (rows=20 width=88)
default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"]
PREHOOK: query: explain select cbo_t3.c_int, c, count(*) from (select key as a, c_int+1 as b, sum(c_int) as c from cbo_t1 where (cbo_t1.c_int + 1 >= 0) and (cbo_t1.c_int > 0 or cbo_t1.c_float >= 0) group by c_float, cbo_t1.c_int, key having cbo_t1.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0) cbo_t1 join (select key as p, c_int+1 as q, sum(c_int) as r from cbo_t2 where (cbo_t2.c_int + 1 >= 0) and (cbo_t2.c_int > 0 or cbo_t2.c_float >= 0) group by c_float, cbo_t2.c_int, key having cbo_t2.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0) cbo_t2 on cbo_t1.a=p join cbo_t3 on cbo_t1.a=key where (b + cbo_t2.q >= 0) and (b > 0 or c_int >= 0) group by cbo_t3.c_int, c
@@ -749,11 +749,11 @@ Stage-0
<-Map 8 [SIMPLE_EDGE]
SHUFFLE [RS_25]
PartitionCols:_col0
- Select Operator [SEL_23] (rows=18 width=79)
+ Select Operator [SEL_23] (rows=18 width=84)
Output:["_col0","_col1"]
- Filter Operator [FIL_41] (rows=18 width=79)
+ Filter Operator [FIL_41] (rows=18 width=84)
predicate:key is not null
- TableScan [TS_21] (rows=20 width=80)
+ TableScan [TS_21] (rows=20 width=84)
default@cbo_t3,cbo_t3,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"]
<-Reducer 3 [SIMPLE_EDGE]
SHUFFLE [RS_24]
@@ -778,7 +778,7 @@ Stage-0
Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float
Filter Operator [FIL_39] (rows=1 width=93)
predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0.0)) and (c_float > 0.0) and ((c_int >= 1) or (c_float >= 1.0)) and ((UDFToFloat(c_int) + c_float) >= 0.0) and key is not null)
- TableScan [TS_0] (rows=20 width=83)
+ TableScan [TS_0] (rows=20 width=88)
default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"]
<-Reducer 7 [SIMPLE_EDGE]
SHUFFLE [RS_17]
@@ -794,7 +794,7 @@ Stage-0
Output:["_col0","_col1","_col2"],keys:key, c_int, c_float
Filter Operator [FIL_40] (rows=1 width=93)
predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0.0)) and (c_float > 0.0) and ((c_int >= 1) or (c_float >= 1.0)) and ((UDFToFloat(c_int) + c_float) >= 0.0) and key is not null)
- TableScan [TS_8] (rows=20 width=83)
+ TableScan [TS_8] (rows=20 width=88)
default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"]
PREHOOK: query: explain select unionsrc.key FROM (select 'tst1' as key, count(1) as value from src) unionsrc
@@ -864,9 +864,9 @@ Stage-0
SHUFFLE [RS_3]
Group By Operator [GBY_2] (rows=1 width=8)
Output:["_col0"],aggregations:["count(key)"]
- Select Operator [SEL_1] (rows=20 width=76)
+ Select Operator [SEL_1] (rows=20 width=80)
Output:["key"]
- TableScan [TS_0] (rows=20 width=76)
+ TableScan [TS_0] (rows=20 width=80)
default@cbo_t3,s1,Tbl:COMPLETE,Col:COMPLETE,Output:["key"]
<-Reducer 6 [CONTAINS]
Reduce Output Operator [RS_24]
@@ -878,9 +878,9 @@ Stage-0
SHUFFLE [RS_10]
Group By Operator [GBY_9] (rows=1 width=8)
Output:["_col0"],aggregations:["count(key)"]
- Select Operator [SEL_8] (rows=20 width=76)
+ Select Operator [SEL_8] (rows=20 width=80)
Output:["key"]
- TableScan [TS_7] (rows=20 width=76)
+ TableScan [TS_7] (rows=20 width=80)
default@cbo_t3,s2,Tbl:COMPLETE,Col:COMPLETE,Output:["key"]
<-Reducer 8 [CONTAINS]
Reduce Output Operator [RS_24]
@@ -892,9 +892,9 @@ Stage-0
SHUFFLE [RS_19]
Group By Operator [GBY_18] (rows=1 width=8)
Output:["_col0"],aggregations:["count(key)"]
- Select Operator [SEL_17] (rows=20 width=76)
+ Select Operator [SEL_17] (rows=20 width=80)
Output:["key"]
- TableScan [TS_16] (rows=20 width=76)
+ TableScan [TS_16] (rows=20 width=80)
default@cbo_t3,s3,Tbl:COMPLETE,Col:COMPLETE,Output:["key"]
PREHOOK: query: explain select unionsrc.key, count(1) FROM (select 'max' as key, max(c_int) as value from cbo_t3 s1
@@ -944,9 +944,9 @@ Stage-0
SHUFFLE [RS_3]
Group By Operator [GBY_2] (rows=1 width=8)
Output:["_col0"],aggregations:["count(key)"]
- Select Operator [SEL_1] (rows=20 width=76)
+ Select Operator [SEL_1] (rows=20 width=80)
Output:["key"]
- TableScan [TS_0] (rows=20 width=76)
+ TableScan [TS_0] (rows=20 width=80)
default@cbo_t3,s1,Tbl:COMPLETE,Col:COMPLETE,Output:["key"]
<-Reducer 7 [CONTAINS]
Reduce Output Operator [RS_26]
@@ -961,9 +961,9 @@ Stage-0
SHUFFLE [RS_10]
Group By Operator [GBY_9] (rows=1 width=8)
Output:["_col0"],aggregations:["count(key)"]
- Select Operator [SEL_8] (rows=20 width=76)
+ Select Operator [SEL_8] (rows=20 width=80)
Output:["key"]
- TableScan [TS_7] (rows=20 width=76)
+ TableScan [TS_7] (rows=20 width=80)
default@cbo_t3,s2,Tbl:COMPLETE,Col:COMPLETE,Output:["key"]
<-Reducer 9 [CONTAINS]
Reduce Output Operator [RS_26]
@@ -978,9 +978,9 @@ Stage-0
SHUFFLE [RS_19]
Group By Operator [GBY_18] (rows=1 width=8)
Output:["_col0"],aggregations:["count(key)"]
- Select Operator [SEL_17] (rows=20 width=76)
+ Select Operator [SEL_17] (rows=20 width=80)
Output:["key"]
- TableScan [TS_16] (rows=20 width=76)
+ TableScan [TS_16] (rows=20 width=80)
default@cbo_t3,s3,Tbl:COMPLETE,Col:COMPLETE,Output:["key"]
PREHOOK: query: explain select cbo_t1.key from cbo_t1 join cbo_t3 where cbo_t1.key=cbo_t3.key and cbo_t1.key >= 1
@@ -1003,20 +1003,20 @@ Stage-0
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_6]
PartitionCols:_col0
- Select Operator [SEL_2] (rows=6 width=70)
+ Select Operator [SEL_2] (rows=6 width=85)
Output:["_col0"]
- Filter Operator [FIL_13] (rows=6 width=70)
+ Filter Operator [FIL_13] (rows=6 width=85)
predicate:(UDFToDouble(key) >= 1.0)
- TableScan [TS_0] (rows=20 width=76)
+ TableScan [TS_0] (rows=20 width=80)
default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key"]
<-Map 3 [SIMPLE_EDGE]
SHUFFLE [RS_7]
PartitionCols:_col0
- Select Operator [SEL_5] (rows=6 width=70)
+ Select Operator [SEL_5] (rows=6 width=85)
Output:["_col0"]
- Filter Operator [FIL_14] (rows=6 width=70)
+ Filter Operator [FIL_14] (rows=6 width=85)
predicate:(UDFToDouble(key) >= 1.0)
- TableScan [TS_3] (rows=20 width=76)
+ TableScan [TS_3] (rows=20 width=80)
default@cbo_t3,cbo_t3,Tbl:COMPLETE,Col:COMPLETE,Output:["key"]
PREHOOK: query: explain select cbo_t1.c_int, cbo_t2.c_int from cbo_t1 left outer join cbo_t2 on cbo_t1.key=cbo_t2.key
@@ -1041,16 +1041,16 @@ Stage-0
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_4]
PartitionCols:_col0
- Select Operator [SEL_1] (rows=20 width=80)
+ Select Operator [SEL_1] (rows=20 width=84)
Output:["_col0","_col1"]
- TableScan [TS_0] (rows=20 width=80)
+ TableScan [TS_0] (rows=20 width=84)
default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"]
<-Map 3 [SIMPLE_EDGE]
SHUFFLE [RS_5]
PartitionCols:_col0
- Select Operator [SEL_3] (rows=20 width=80)
+ Select Operator [SEL_3] (rows=20 width=84)
Output:["_col0","_col1"]
- TableScan [TS_2] (rows=20 width=80)
+ TableScan [TS_2] (rows=20 width=84)
default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"]
PREHOOK: query: explain select cbo_t1.c_int, cbo_t2.c_int from cbo_t1 full outer join cbo_t2 on cbo_t1.key=cbo_t2.key
@@ -1075,16 +1075,16 @@ Stage-0
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_4]
PartitionCols:_col0
- Select Operator [SEL_1] (rows=20 width=80)
+ Select Operator [SEL_1] (rows=20 width=84)
Output:["_col0","_col1"]
- TableScan [TS_0] (rows=20 width=80)
+ TableScan [TS_0] (rows=20 width=84)
default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"]
<-Map 3 [SIMPLE_EDGE]
SHUFFLE [RS_5]
PartitionCols:_col0
- Select Operator [SEL_3] (rows=20 width=80)
+ Select Operator [SEL_3] (rows=20 width=84)
Output:["_col0","_col1"]
- TableScan [TS_2] (rows=20 width=80)
+ TableScan [TS_2] (rows=20 width=84)
default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"]
PREHOOK: query: explain select b, cbo_t1.c, cbo_t2.p, q, cbo_t3.c_int from (select key as a, c_int as b, cbo_t1.c_float as c from cbo_t1) cbo_t1 join (select cbo_t2.key as p, cbo_t2.c_int as q, c_float as r from cbo_t2) cbo_t2 on cbo_t1.a=p join cbo_t3 on cbo_t1.a=key
@@ -1109,29 +1109,29 @@ Stage-0
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_9]
PartitionCols:_col0
- Select Operator [SEL_2] (rows=18 width=82)
+ Select Operator [SEL_2] (rows=18 width=87)
Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_21] (rows=18 width=82)
+ Filter Operator [FIL_21] (rows=18 width=87)
predicate:key is not null
- TableScan [TS_0] (rows=20 width=83)
+ TableScan [TS_0] (rows=20 width=88)
default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"]
<-Map 3 [SIMPLE_EDGE]
SHUFFLE [RS_10]
PartitionCols:_col0
- Select Operator [SEL_5] (rows=18 width=79)
+ Select Operator [SEL_5] (rows=18 width=84)
Output:["_col0","_col1"]
- Filter Operator [FIL_22] (rows=18 width=79)
+ Filter Operator [FIL_22] (rows=18 width=84)
predicate:key is not null
- TableScan [TS_3] (rows=20 width=80)
+ TableScan [TS_3] (rows=20 width=84)
default@cbo_t3,cbo_t3,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"]
<-Map 4 [SIMPLE_EDGE]
SHUFFLE [RS_11]
PartitionCols:_col0
- Select Operator [SEL_8] (rows=18 width=79)
+ Select Operator [SEL_8] (rows=18 width=84)
Output:["_col0","_col1"]
- Filter Operator [FIL_23] (rows=18 width=79)
+ Filter Operator [FIL_23] (rows=18 width=84)
predicate:key is not null
- TableScan [TS_6] (rows=20 width=80)
+ TableScan [TS_6] (rows=20 width=84)
default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"]
PREHOOK: query: explain select key, cbo_t1.c_int, cbo_t2.p, q from cbo_t1 join (select cbo_t2.key as p, cbo_t2.c_int as q, c_float as r from cbo_t2) cbo_t2 on cbo_t1.key=p join (select key as a, c_int as b, cbo_t3.c_float as c from cbo_t3)cbo_t3 on cbo_t1.key=a
@@ -1156,29 +1156,29 @@ Stage-0
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_9]
PartitionCols:_col0
- Select Operator [SEL_2] (rows=18 width=79)
+ Select Operator [SEL_2] (rows=18 width=84)
Output:["_col0","_col1"]
- Filter Operator [FIL_21] (rows=18 width=79)
+ Filter Operator [FIL_21] (rows=18 width=84)
predicate:key is not null
- TableScan [TS_0] (rows=20 width=80)
+ TableScan [TS_0] (rows=20 width=84)
default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"]
<-Map 3 [SIMPLE_EDGE]
SHUFFLE [RS_10]
PartitionCols:_col0
- Select Operator [SEL_5] (rows=18 width=75)
+ Select Operator [SEL_5] (rows=18 width=80)
Output:["_col0"]
- Filter Operator [FIL_22] (rows=18 width=75)
+ Filter Operator [FIL_22] (rows=18 width=80)
predicate:key is not null
- TableScan [TS_3] (rows=20 width=76)
+ TableScan [TS_3] (rows=20 width=80)
default@cbo_t3,cbo_t3,Tbl:COMPLETE,Col:COMPLETE,Output:["key"]
<-Map 4 [SIMPLE_EDGE]
SHUFFLE [RS_11]
PartitionCols:_col0
- Select Operator [SEL_8] (rows=18 width=79)
+ Select Operator [SEL_8] (rows=18 width=84)
Output:["_col0","_col1"]
- Filter Operator [FIL_23] (rows=18 width=79)
+ Filter Operator [FIL_23] (rows=18 width=84)
predicate:key is not null
- TableScan [TS_6] (rows=20 width=80)
+ TableScan [TS_6] (rows=20 width=84)
default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"]
PREHOOK: query: explain select * from (select q, b, cbo_t2.p, cbo_t1.c, cbo_t3.c_int from (select key as a, c_int as b, cbo_t1.c_float as c from cbo_t1 where (cbo_t1.c_int + 1 == 2) and (cbo_t1.c_int > 0 or cbo_t1.c_float >= 0)) cbo_t1 full outer join (select cbo_t2.key as p, cbo_t2.c_int as q, c_float as r from cbo_t2 where (cbo_t2.c_int + 1 == 2) and (cbo_t2.c_int > 0 or cbo_t2.c_float >= 0)) cbo_t2 on cbo_t1.a=p join cbo_t3 on cbo_t1.a=key where (b + cbo_t2.q == 2) and (b > 0 or c_int >= 0)) R where (q + 1 = 2) and (R.b > 0 or c_int >= 0)
@@ -1206,11 +1206,11 @@ Stage-0
<-Map 5 [SIMPLE_EDGE]
SHUFFLE [RS_15]
PartitionCols:_col0
- Select Operator [SEL_13] (rows=18 width=79)
+ Select Operator [SEL_13] (rows=18 width=84)
Output:["_col0","_col1"]
- Filter Operator [FIL_26] (rows=18 width=79)
+ Filter Operator [FIL_26] (rows=18 width=84)
predicate:key is not null
- TableScan [TS_11] (rows=20 width=80)
+ TableScan [TS_11] (rows=20 width=84)
default@cbo_t3,cbo_t3,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"]
<-Reducer 2 [SIMPLE_EDGE]
SHUFFLE [RS_14]
@@ -1222,20 +1222,20 @@ Stage-0
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_6]
PartitionCols:_col0
- Select Operator [SEL_2] (rows=9 width=82)
+ Select Operator [SEL_2] (rows=9 width=93)
Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_24] (rows=9 width=82)
+ Filter Operator [FIL_24] (rows=9 width=93)
predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0.0)) and key is not null)
- TableScan [TS_0] (rows=20 width=83)
+ TableScan [TS_0] (rows=20 width=88)
default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"]
<-Map 4 [SIMPLE_EDGE]
SHUFFLE [RS_7]
PartitionCols:_col0
- Select Operator [SEL_5] (rows=9 width=79)
+ Select Operator [SEL_5] (rows=9 width=89)
Output:["_col0","_col1"]
- Filter Operator [FIL_25] (rows=9 width=82)
+ Filter Operator [FIL_25] (rows=9 width=93)
predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0.0)) and key is not null)
- TableScan [TS_3] (rows=20 width=83)
+ TableScan [TS_3] (rows=20 width=88)
default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"]
PREHOOK: query: explain select * from (select q, b, cbo_t2.p, cbo_t1.c, cbo_t3.c_int from (select key as a, c_int as b, cbo_t1.c_float as c from cbo_t1 where (cbo_t1.c_int + 1 == 2) and (cbo_t1.c_int > 0 or cbo_t1.c_float >= 0)) cbo_t1 right outer join (select cbo_t2.key as p, cbo_t2.c_int as q, c_float as r from cbo_t2 where (cbo_t2.c_int + 1 == 2) and (cbo_t2.c_int > 0 or cbo_t2.c_float >= 0)) cbo_t2 on cbo_t1.a=p right outer join cbo_t3 on cbo_t1.a=key where (b + cbo_t2.q == 2) and (b > 0 or c_int >= 0)) R where (q + 1 = 2) and (R.b > 0 or c_int >= 0)
@@ -1262,27 +1262,27 @@ Stage-0
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_8]
PartitionCols:_col0
- Select Operator [SEL_2] (rows=10 width=83)
+ Select Operator [SEL_2] (rows=10 width=93)
Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_17] (rows=10 width=83)
+ Filter Operator [FIL_17] (rows=10 width=93)
predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0.0)))
- TableScan [TS_0] (rows=20 width=83)
+ TableScan [TS_0] (rows=20 width=88)
default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"]
<-Map 3 [SIMPLE_EDGE]
SHUFFLE [RS_9]
PartitionCols:_col0
- Select Operator [SEL_5] (rows=10 width=80)
+ Select Operator [SEL_5] (rows=10 width=89)
Output:["_col0","_col1"]
- Filter Operator [FIL_18] (rows=10 width=83)
+ Filter Operator [FIL_18] (rows=10 width=93)
predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0.0)))
- TableScan [TS_3] (rows=20 width=83)
+ TableScan [TS_3] (rows=20 width=88)
default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"]
<-Map 4 [SIMPLE_EDGE]
SHUFFLE [RS_10]
PartitionCols:_col0
- Select Operator [SEL_7] (rows=20 width=80)
+ Select Operator [SEL_7] (rows=20 width=84)
Output:["_col0","_col1"]
- TableScan [TS_6] (rows=20 width=80)
+ TableScan [TS_6] (rows=20 width=84)
default@cbo_t3,cbo_t3,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"]
PREHOOK: query: explain select key, (c_int+1)+2 as x, sum(c_int) from cbo_t1 group by c_float, cbo_t1.c_int, key order by x limit 1
@@ -1303,22 +1303,22 @@ Stage-0
File Output Operator [FS_10]
Limit [LIM_9] (rows=1 width=97)
Number of rows:1
- Select Operator [SEL_8] (rows=10 width=88)
+ Select Operator [SEL_8] (rows=10 width=97)
Output:["_col0","_col1","_col2"]
<-Reducer 2 [SIMPLE_EDGE]
SHUFFLE [RS_7]
- Select Operator [SEL_5] (rows=10 width=88)
+ Select Operator [SEL_5] (rows=10 width=97)
Output:["_col0","_col1","_col2"]
- Group By Operator [GBY_4] (rows=10 width=91)
+ Group By Operator [GBY_4] (rows=10 width=101)
Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_3]
PartitionCols:_col0, _col1, _col2
- Group By Operator [GBY_2] (rows=10 width=91)
+ Group By Operator [GBY_2] (rows=10 width=101)
Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float
- Select Operator [SEL_1] (rows=20 width=83)
+ Select Operator [SEL_1] (rows=20 width=88)
Output:["key","c_int","c_float"]
- TableScan [TS_0] (rows=20 width=83)
+ TableScan [TS_0] (rows=20 width=88)
default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"]
PREHOOK: query: explain select x, y, count(*) from (select key, (c_int+c_float+1+2) as x, sum(c_int) as y from cbo_t1 group by c_float, cbo_t1.c_int, key) R group by y, x order by x,y limit 1
@@ -1351,18 +1351,18 @@ Stage-0
PartitionCols:_col0, _col1
Group By Operator [GBY_8] (rows=5 width=20)
Output:["_col0","_col1","_col2"],aggregations:["count()"],keys:_col1, _col0
- Select Operator [SEL_5] (rows=10 width=91)
+ Select Operator [SEL_5] (rows=10 width=101)
Output:["_col0","_col1"]
- Group By Operator [GBY_4] (rows=10 width=91)
+ Group By Operator [GBY_4] (rows=10 width=101)
Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_3]
PartitionCols:_col0, _col1, _col2
- Group By Operator [GBY_2] (rows=10 width=91)
+ Group By Operator [GBY_2] (rows=10 width=101)
Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float
- Select Operator [SEL_1] (rows=20 width=83)
+ Select Operator [SEL_1] (rows=20 width=88)
Output:["key","c_int","c_float"]
- TableScan [TS_0] (rows=20 width=83)
+ TableScan [TS_0] (rows=20 width=88)
default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"]
PREHOOK: query: explain select key from(select key from (select key from cbo_t1 limit 5)cbo_t2 limit 5)cbo_t3 limit 5
@@ -1381,27 +1381,27 @@ Stage-0
Stage-1
Reducer 3
File Output Operator [FS_13]
- Limit [LIM_12] (rows=5 width=68)
+ Limit [LIM_12] (rows=5 width=85)
Number of rows:5
- Limit [LIM_10] (rows=5 width=68)
+ Limit [LIM_10] (rows=5 width=85)
Number of rows:5
- Select Operator [SEL_9] (rows=5 width=68)
+ Select Operator [SEL_9] (rows=5 width=85)
Output:["_col0"]
<-Reducer 2 [SIMPLE_EDGE]
SHUFFLE [RS_8]
- Limit [LIM_7] (rows=5 width=68)
+ Limit [LIM_7] (rows=5 width=85)
Number of rows:5
- Limit [LIM_5] (rows=5 width=68)
+ Limit [LIM_5] (rows=5 width=85)
Number of rows:5
- Select Operator [SEL_4] (rows=5 width=68)
+ Select Operator [SEL_4] (rows=5 width=85)
Output:["_col0"]
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_3]
- Limit [LIM_2] (rows=5 width=68)
+ Limit [LIM_2] (rows=5 width=85)
Number of rows:5
- Select Operator [SEL_1] (rows=20 width=76)
+ Select Operator [SEL_1] (rows=20 width=80)
Output:["_col0"]
- TableScan [TS_0] (rows=20 width=76)
+ TableScan [TS_0] (rows=20 width=80)
default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key"]
PREHOOK: query: explain select key, c_int from(select key, c_int from (select key, c_int from cbo_t1 order by c_int limit 5)cbo_t1 order by c_int limit 5)cbo_t2 order by c_int limit 5
@@ -1421,27 +1421,27 @@ Stage-0
Stage-1
Reducer 4
File Output Operator [FS_13]
- Limit [LIM_12] (rows=5 width=71)
+ Limit [LIM_12] (rows=5 width=89)
Number of rows:5
- Select Operator [SEL_11] (rows=5 width=71)
+ Select Operator [SEL_11] (rows=5 width=89)
Output:["_col0","_col1"]
<-Reducer 3 [SIMPLE_EDGE]
SHUFFLE [RS_10]
- Limit [LIM_8] (rows=5 width=71)
+ Limit [LIM_8] (rows=5 width=89)
Number of rows:5
- Select Operator [SEL_7] (rows=5 width=71)
+ Select Operator [SEL_7] (rows=5 width=89)
Output:["_col0","_col1"]
<-Reducer 2 [SIMPLE_EDGE]
SHUFFLE [RS_6]
- Limit [LIM_4] (rows=5 width=71)
+ Limit [LIM_4] (rows=5 width=89)
Number of rows:5
- Select Operator [SEL_3] (rows=20 width=80)
+ Select Operator [SEL_3] (rows=20 width=84)
Output:["_col0","_col1"]
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_2]
- Select Operator [SEL_1] (rows=20 width=80)
+ Select Operator [SEL_1] (rows=20 width=84)
Output:["_col0","_col1"]
- TableScan [TS_0] (rows=20 width=80)
+ TableScan [TS_0] (rows=20 width=84)
default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"]
PREHOOK: query: explain select cbo_t3.c_int, c, count(*) from (select key as a, c_int+1 as b, sum(c_int) as c from cbo_t1 where (cbo_t1.c_int + 1 >= 0) and (cbo_t1.c_int > 0 or cbo_t1.c_float >= 0) group by c_float, cbo_t1.c_int, key order by a limit 5) cbo_t1 join (select key as p, c_int+1 as q, sum(c_int) as r from cbo_t2 where (cbo_t2.c_int + 1 >= 0) and (cbo_t2.c_int > 0 or cbo_t2.c_float >= 0) group by c_float, cbo_t2.c_int, key order by q/10 desc, r asc limit 5) cbo_t2 on cbo_t1.a=p join cbo_t3 on cbo_t1.a=key where (b + cbo_t2.q >= 0) and (b > 0 or c_int >= 0) group by cbo_t3.c_int, c order by cbo_t3.c_int+c desc, c limit 5
@@ -1490,11 +1490,11 @@ Stage-0
<-Map 11 [SIMPLE_EDGE]
SHUFFLE [RS_37]
PartitionCols:_col0
- Select Operator [SEL_35] (rows=18 width=79)
+ Select Operator [SEL_35] (rows=18 width=84)
Output:["_col0","_col1"]
- Filter Operator [FIL_59] (rows=18 width=79)
+ Filter Operator [FIL_59] (rows=18 width=84)
predicate:key is not null
- TableScan [TS_33] (rows=20 width=80)
+ TableScan [TS_33] (rows=20 width=84)
default@cbo_t3,cbo_t3,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"]
<-Reducer 4 [SIMPLE_EDGE]
SHUFFLE [RS_36]
@@ -1508,50 +1508,50 @@ Stage-0
<-Reducer 10 [SIMPLE_EDGE]
SHUFFLE [RS_29]
PartitionCols:_col0
- Filter Operator [FIL_26] (rows=2 width=62)
+ Filter Operator [FIL_26] (rows=2 width=105)
predicate:_col0 is not null
- Limit [LIM_24] (rows=3 width=76)
+ Limit [LIM_24] (rows=3 width=105)
Number of rows:5
- Select Operator [SEL_23] (rows=3 width=76)
+ Select Operator [SEL_23] (rows=3 width=105)
Output:["_col0","_col1"]
<-Reducer 9 [SIMPLE_EDGE]
SHUFFLE [RS_22]
- Select Operator [SEL_20] (rows=3 width=76)
+ Select Operator [SEL_20] (rows=3 width=105)
Output:["_col0","_col1","_col2","_col3"]
- Group By Operator [GBY_19] (rows=3 width=70)
+ Group By Operator [GBY_19] (rows=3 width=101)
Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2
<-Map 8 [SIMPLE_EDGE]
SHUFFLE [RS_18]
PartitionCols:_col0, _col1, _col2
- Group By Operator [GBY_17] (rows=3 width=70)
+ Group By Operator [GBY_17] (rows=3 width=101)
Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float
- Filter Operator [FIL_58] (rows=6 width=77)
+ Filter Operator [FIL_58] (rows=6 width=93)
predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0.0)))
- TableScan [TS_14] (rows=20 width=83)
+ TableScan [TS_14] (rows=20 width=88)
default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"]
<-Reducer 3 [SIMPLE_EDGE]
SHUFFLE [RS_28]
PartitionCols:_col0
- Filter Operator [FIL_12] (rows=2 width=54)
+ Filter Operator [FIL_12] (rows=2 width=97)
predicate:_col0 is not null
- Limit [LIM_10] (rows=3 width=68)
+ Limit [LIM_10] (rows=3 width=97)
Number of rows:5
- Select Operator [SEL_9] (rows=3 width=68)
+ Select Operator [SEL_9] (rows=3 width=97)
Output:["_col0","_col1","_col2"]
<-Reducer 2 [SIMPLE_EDGE]
SHUFFLE [RS_8]
- Select Operator [SEL_6] (rows=3 width=68)
+ Select Operator [SEL_6] (rows=3 width=97)
Output:["_col0","_col1","_col2"]
- Group By Operator [GBY_5] (rows=3 width=70)
+ Group By Operator [GBY_5] (rows=3 width=101)
Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_4]
PartitionCols:_col0, _col1, _col2
- Group By Operator [GBY_3] (rows=3 width=70)
+ Group By Operator [GBY_3] (rows=3 width=101)
Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float
- Filter Operator [FIL_56] (rows=6 width=77)
+ Filter Operator [FIL_56] (rows=6 width=93)
predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0.0)))
- TableScan [TS_0] (rows=20 width=83)
+ TableScan [TS_0] (rows=20 width=88)
default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"]
PREHOOK: query: explain select cbo_t1.c_int from cbo_t1 left semi join cbo_t2 on cbo_t1.key=cbo_t2.key where (cbo_t1.c_int + 1 == 2) and (cbo_t1.c_int > 0 or cbo_t1.c_float >= 0)
@@ -1576,22 +1576,22 @@ Stage-0
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_8]
PartitionCols:_col0
- Select Operator [SEL_2] (rows=9 width=82)
+ Select Operator [SEL_2] (rows=9 width=93)
Output:["_col0","_col1"]
- Filter Operator [FIL_15] (rows=9 width=82)
+ Filter Operator [FIL_15] (rows=9 width=93)
predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0.0)) and key is not null)
- TableScan [TS_0] (rows=20 width=83)
+ TableScan [TS_0] (rows=20 width=88)
default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"]
<-Map 3 [SIMPLE_EDGE]
SHUFFLE [RS_9]
PartitionCols:_col0
- Group By Operator [GBY_7] (rows=5 width=68)
+ Group By Operator [GBY_7] (rows=5 width=85)
Output:["_col0"],keys:_col0
- Select Operator [SEL_5] (rows=18 width=75)
+ Select Operator [SEL_5] (rows=18 width=80)
Output:["_col0"]
- Filter Operator [FIL_16] (rows=18 width=75)
+ Filter Operator [FIL_16] (rows=18 width=80)
predicate:key is not null
- TableScan [TS_3] (rows=20 width=76)
+ TableScan [TS_3] (rows=20 width=80)
default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key"]
PREHOOK: query: explain select * from (select c, b, a from (select key as a, c_int as b, cbo_t1.c_float as c from cbo_t1 where (cbo_t1.c_int + 1 == 2) and (cbo_t1.c_int > 0 or cbo_t1.c_float >= 0)) cbo_t1 left semi join (select cbo_t2.key as p, cbo_t2.c_int as q, c_float as r from cbo_t2 where (cbo_t2.c_int + 1 == 2) and (cbo_t2.c_int > 0 or cbo_t2.c_float >= 0)) cbo_t2 on cbo_t1.a=p left semi join cbo_t3 on cbo_t1.a=key where (b + 1 == 2) and (b > 0 or c >= 0)) R where (b + 1 = 2) and (R.b > 0 or c >= 0)
@@ -1616,33 +1616,33 @@ Stage-0
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_13]
PartitionCols:_col0
- Select Operator [SEL_2] (rows=9 width=82)
+ Select Operator [SEL_2] (rows=9 width=93)
Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_25] (rows=9 width=82)
+ Filter Operator [FIL_25] (rows=9 width=93)
predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0.0)) and key is not null)
- TableScan [TS_0] (rows=20 width=83)
+ TableScan [TS_0] (rows=20 width=88)
default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"]
<-Map 3 [SIMPLE_EDGE]
SHUFFLE [RS_14]
PartitionCols:_col0
Group By Operator [GBY_10] (rows=3 width=85)
Output:["_col0"],keys:_col0
- Select Operator [SEL_5] (rows=9 width=75)
+ Select Operator [SEL_5] (rows=9 width=85)
Output:["_col0"]
- Filter Operator [FIL_26] (rows=9 width=82)
+ Filter Operator [FIL_26] (rows=9 width=93)
predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0.0)) and key is not null)
- TableScan [TS_3] (rows=20 width=83)
+ TableScan [TS_3] (rows=20 width=88)
default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"]
<-Map 4 [SIMPLE_EDGE]
SHUFFLE [RS_15]
PartitionCols:_col0
- Group By Operator [GBY_12] (rows=6 width=70)
+ Group By Operator [GBY_12] (rows=6 width=85)
Output:["_col0"],keys:_col0
- Select Operator [SEL_8] (rows=18 width=75)
+ Select Operator [SEL_8] (rows=18 width=80)
Output:["_col0"]
- Filter Operator [FIL_27] (rows=18 width=75)
+ Filter Operator [FIL_27] (rows=18 width=80)
predicate:key is not null
- TableScan [TS_6] (rows=20 width=76)
+ TableScan [TS_6] (rows=20 width=80)
default@cbo_t3,cbo_t3,Tbl:COMPLETE,Col:COMPLETE,Output:["key"]
PREHOOK: query: explain select a, c, count(*) from (select key as a, c_int+1 as b, sum(c_int) as c from cbo_t1 where (cbo_t1.c_int + 1 >= 0) and (cbo_t1.c_int > 0 or cbo_t1.c_float >= 0) group by c_float, cbo_t1.c_int, key having cbo_t1.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by a+b desc, c asc) cbo_t1 left semi join (select key as p, c_int+1 as q, sum(c_int) as r from cbo_t2 where (cbo_t2.c_int + 1 >= 0) and (cbo_t2.c_int > 0 or cbo_t2.c_float >= 0) group by c_float, cbo_t2.c_int, key having cbo_t2.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by q+r/10 desc, p) cbo_t2 on cbo_t1.a=p left semi join cbo_t3 on cbo_t1.a=key where (b + 1 >= 0) and (b > 0 or a >= 0) group by a, c having a > 0 and (a >=1 or c >= 1) and (a + c) >= 0 order by c, a
@@ -1684,13 +1684,13 @@ Stage-0
<-Map 10 [SIMPLE_EDGE]
SHUFFLE [RS_32]
PartitionCols:_col0
- Group By Operator [GBY_29] (rows=3 width=56)
+ Group By Operator [GBY_29] (rows=3 width=85)
Output:["_col0"],keys:_col0
- Select Operator [SEL_25] (rows=6 width=70)
+ Select Operator [SEL_25] (rows=6 width=85)
Output:["_col0"]
- Filter Operator [FIL_50] (rows=6 width=70)
+ Filter Operator [FIL_50] (rows=6 width=85)
predicate:(UDFToDouble(key) > 0.0)
- TableScan [TS_23] (rows=20 width=76)
+ TableScan [TS_23] (rows=20 width=80)
default@cbo_t3,cbo_t3,Tbl:COMPLETE,Col:COMPLETE,Output:["key"]
<-Reducer 3 [SIMPLE_EDGE]
SHUFFLE [RS_30]
@@ -1714,7 +1714,7 @@ Stage-0
Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float
Filter Operator [FIL_48] (rows=1 width=93)
predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0.0)) and (c_float > 0.0) and ((c_int >= 1) or (c_float >= 1.0)) and ((UDFToFloat(c_int) + c_float) >= 0.0) and (((c_int + 1) + 1) >= 0) and (((c_int + 1) > 0) or (UDFToDouble(key) >= 0.0)) and (UDFToDouble(key) > 0.0))
- TableScan [TS_0] (rows=20 width=83)
+ TableScan [TS_0] (rows=20 width=88)
default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"]
<-Reducer 9 [SIMPLE_EDGE]
SHUFFLE [RS_31]
@@ -1736,7 +1736,7 @@ Stage-0
Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float
Filter Operator [FIL_49] (rows=1 width=93)
predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0.0)) and (c_float > 0.0) and ((c_int >= 1) or (c_float >= 1.0)) and ((UDFToFloat(c_int) + c_float) >= 0.0) and (UDFToDouble(key) > 0.0))
- TableScan [TS_12] (rows=20 width=83)
+ TableScan [TS_12] (rows=20 width=88)
default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"]
PREHOOK: query: explain select cbo_t1.key as x, c_int as c_int, (((c_int+c_float)*10)+5) as y from cbo_t1
@@ -2294,7 +2294,7 @@ Stage-0
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_4]
PartitionCols:_col0, _col1
- Group By Operator [GBY_3] (rows=13 width=219)
+ Group By Operator [GBY_3] (rows=13 width=295)
Output:["_col0","_col1","_col2"],aggregations:["avg(p_size)"],keys:p_name, p_mfgr
Filter Operator [FIL_24] (rows=26 width=223)
predicate:p_name is not null
@@ -2544,7 +2544,7 @@ Stage-0
Output:["_col0"],aggregations:["avg(VALUE._col0)"]
<-Map 5 [SIMPLE_EDGE]
SHUFFLE [RS_6]
- Group By Operator [GBY_5] (rows=1 width=0)
+ Group By Operator [GBY_5] (rows=1 width=76)
Output:["_col0"],aggregations:["avg(p_size)"]
Filter Operator [FIL_38] (rows=8 width=4)
predicate:(p_size < 10)
@@ -2566,7 +2566,7 @@ Stage-0
Output:["_col0"],aggregations:["avg(VALUE._col0)"]
<-Map 7 [SIMPLE_EDGE]
SHUFFLE [RS_18]
- Group By Operator [GBY_17] (rows=1 width=0)
+ Group By Operator [GBY_17] (rows=1 width=76)
Output:["_col0"],aggregations:["avg(p_size)"]
Filter Operator [FIL_40] (rows=8 width=4)
predicate:(p_size < 10)
@@ -2764,14 +2764,14 @@ Stage-0
File Output Operator [FS_6]
Select Operator [SEL_4] (rows=20 width=64)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10"]
- PTF Operator [PTF_3] (rows=20 width=612)
+ PTF Operator [PTF_3] (rows=20 width=621)
Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 ASC NULLS FIRST","partition by:":"_col0"}]
- Select Operator [SEL_2] (rows=20 width=612)
+ Select Operator [SEL_2] (rows=20 width=621)
Output:["_col0","_col1","_col2","_col3"]
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_1]
PartitionCols:key
- TableScan [TS_0] (rows=20 width=160)
+ TableScan [TS_0] (rows=20 width=169)
default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["c_float","c_int","key","value"]
PREHOOK: query: explain select *, rank() over(partition by key order by value) as rr from src1
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/tez/explainuser_3.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/explainuser_3.q.out b/ql/src/test/results/clientpositive/tez/explainuser_3.q.out
index e2aea13..1fa21fe 100644
--- a/ql/src/test/results/clientpositive/tez/explainuser_3.q.out
+++ b/ql/src/test/results/clientpositive/tez/explainuser_3.q.out
@@ -205,11 +205,11 @@ Stage-2
Stage-0
Reducer 2
File Output Operator [FS_6]
- Group By Operator [GBY_4] (rows=1 width=0)
+ Group By Operator [GBY_4] (rows=1 width=984)
Output:["_col0","_col1"],aggregations:["compute_stats(VALUE._col0)","compute_stats(VALUE._col1)"]
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_3]
- Group By Operator [GBY_2] (rows=1 width=0)
+ Group By Operator [GBY_2] (rows=1 width=984)
Output:["_col0","_col1"],aggregations:["compute_stats(key, 16)","compute_stats(value, 16)"]
Select Operator [SEL_1] (rows=500 width=10)
Output:["key","value"]
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/tez/groupby3.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/groupby3.q.out b/ql/src/test/results/clientpositive/tez/groupby3.q.out
index c46d1df..771a0ed 100644
--- a/ql/src/test/results/clientpositive/tez/groupby3.q.out
+++ b/ql/src/test/results/clientpositive/tez/groupby3.q.out
@@ -67,10 +67,10 @@ STAGE PLANS:
aggregations: sum(KEY._col0:0._col0), avg(KEY._col0:0._col0), avg(DISTINCT KEY._col0:0._col0), max(KEY._col0:0._col0), min(KEY._col0:0._col0), std(KEY._col0:0._col0), stddev_samp(KEY._col0:0._col0), variance(KEY._col0:0._col0), var_samp(KEY._col0:0._col0)
mode: partial1
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
- Statistics: Num rows: 1 Data size: 176 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 1208 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 176 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 1208 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: double), _col1 (type: struct<count:bigint,sum:double,input:string>), _col2 (type: struct<count:bigint,sum:double,input:string>), _col3 (type: string), _col4 (type: string), _col5 (type: struct<count:bigint,sum:double,variance:double>), _col6 (type: struct<count:bigint,sum:double,variance:double>), _col7 (type: struct<count:bigint,sum:double,variance:double>), _col8 (type: struct<count:bigint,sum:double,variance:double>)
Reducer 3
Reduce Operator Tree:
@@ -78,14 +78,14 @@ STAGE PLANS:
aggregations: sum(VALUE._col0), avg(VALUE._col1), avg(VALUE._col2), max(VALUE._col3), min(VALUE._col4), std(VALUE._col5), stddev_samp(VALUE._col6), variance(VALUE._col7), var_samp(VALUE._col8)
mode: final
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
- Statistics: Num rows: 1 Data size: 224 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 1208 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: double), _col1 (type: double), _col2 (type: double), UDFToDouble(_col3) (type: double), UDFToDouble(_col4) (type: double), _col5 (type: double), _col6 (type: double), _col7 (type: double), _col8 (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
- Statistics: Num rows: 1 Data size: 224 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 1208 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 224 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 1208 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/tez/metadataonly1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/metadataonly1.q.out b/ql/src/test/results/clientpositive/tez/metadataonly1.q.out
index 8e1842a..07d4bf7 100644
--- a/ql/src/test/results/clientpositive/tez/metadataonly1.q.out
+++ b/ql/src/test/results/clientpositive/tez/metadataonly1.q.out
@@ -36,11 +36,11 @@ STAGE PLANS:
aggregations: max(ds)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
null sort order:
sort order:
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
tag: -1
value expressions: _col0 (type: string)
auto parallelism: false
@@ -51,13 +51,13 @@ STAGE PLANS:
aggregations: max(VALUE._col0)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
GlobalTableId: 0
#### A masked pattern was here ####
NumFilesPerFileSink: 1
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
#### A masked pattern was here ####
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
@@ -127,11 +127,11 @@ STAGE PLANS:
aggregations: max(ds)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
null sort order:
sort order:
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
tag: -1
value expressions: _col0 (type: string)
auto parallelism: false
@@ -191,13 +191,13 @@ STAGE PLANS:
aggregations: max(VALUE._col0)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
GlobalTableId: 0
#### A masked pattern was here ####
NumFilesPerFileSink: 1
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
#### A masked pattern was here ####
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
@@ -659,11 +659,11 @@ STAGE PLANS:
aggregations: max(ds)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
null sort order:
sort order:
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
tag: -1
value expressions: _col0 (type: string)
auto parallelism: false
@@ -823,33 +823,33 @@ STAGE PLANS:
aggregations: max(VALUE._col0)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Filter Operator
isSamplingPred: false
predicate: _col0 is not null (type: boolean)
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
null sort order: a
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
tag: 1
auto parallelism: true
Select Operator
expressions: _col0 (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Dynamic Partitioning Event Operator
Target column: ds (string)
Target Input: a2
Partition key expr: ds
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Target Vertex: Map 1
Stage: Stage-0
@@ -1405,11 +1405,11 @@ STAGE PLANS:
aggregations: max(ds)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
null sort order:
sort order:
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
tag: -1
value expressions: _col0 (type: string)
auto parallelism: false
@@ -1515,13 +1515,13 @@ STAGE PLANS:
aggregations: max(VALUE._col0)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
GlobalTableId: 0
#### A masked pattern was here ####
NumFilesPerFileSink: 1
- Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
#### A masked pattern was here ####
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/tez/subquery_in.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/subquery_in.q.out b/ql/src/test/results/clientpositive/tez/subquery_in.q.out
index b71a67f..c6bf433 100644
--- a/ql/src/test/results/clientpositive/tez/subquery_in.q.out
+++ b/ql/src/test/results/clientpositive/tez/subquery_in.q.out
@@ -344,10 +344,10 @@ STAGE PLANS:
aggregations: avg(_col0)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 76 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 1 Data size: 76 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: struct<count:bigint,sum:double,input:int>)
Reducer 5
Reduce Operator Tree:
@@ -355,20 +355,20 @@ STAGE PLANS:
aggregations: avg(VALUE._col0)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 76 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: _col0 is not null (type: boolean)
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 76 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: double)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 76 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: double)
sort order: +
Map-reduce partition columns: _col0 (type: double)
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 76 Basic stats: COMPLETE Column stats: NONE
Stage: Stage-0
Fetch Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/tez/vector_aggregate_9.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/vector_aggregate_9.q.out b/ql/src/test/results/clientpositive/tez/vector_aggregate_9.q.out
index 32981c1..6ab92be 100644
--- a/ql/src/test/results/clientpositive/tez/vector_aggregate_9.q.out
+++ b/ql/src/test/results/clientpositive/tez/vector_aggregate_9.q.out
@@ -132,10 +132,10 @@ STAGE PLANS:
aggregations: min(dc), max(dc), sum(dc), avg(dc)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 1 Data size: 336 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 624 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 336 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 624 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: decimal(38,18)), _col1 (type: decimal(38,18)), _col2 (type: decimal(38,18)), _col3 (type: struct<count:bigint,sum:decimal(38,18),input:decimal(38,18)>)
Execution mode: vectorized
Reducer 2
@@ -144,10 +144,10 @@ STAGE PLANS:
aggregations: min(VALUE._col0), max(VALUE._col1), sum(VALUE._col2), avg(VALUE._col3)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 1 Data size: 448 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 624 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 448 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 624 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/47dbc005/ql/src/test/results/clientpositive/tez/vector_aggregate_without_gby.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/vector_aggregate_without_gby.q.out b/ql/src/test/results/clientpositive/tez/vector_aggregate_without_gby.q.out
index 3bbbb46..dfac90b 100644
--- a/ql/src/test/results/clientpositive/tez/vector_aggregate_without_gby.q.out
+++ b/ql/src/test/results/clientpositive/tez/vector_aggregate_without_gby.q.out
@@ -48,11 +48,11 @@ Stage-0
Stage-1
Reducer 2 vectorized
File Output Operator [FS_14]
- Group By Operator [GBY_13] (rows=1 width=88)
+ Group By Operator [GBY_13] (rows=1 width=188)
Output:["_col0","_col1"],aggregations:["max(VALUE._col0)","max(VALUE._col1)"]
<-Map 1 [SIMPLE_EDGE] vectorized
SHUFFLE [RS_12]
- Group By Operator [GBY_11] (rows=1 width=88)
+ Group By Operator [GBY_11] (rows=1 width=188)
Output:["_col0","_col1"],aggregations:["max(dt)","max(greg_dt)"]
Select Operator [SEL_10] (rows=3 width=102)
Output:["dt","greg_dt"]
[46/50] [abbrv] hive git commit: HIVE-16082. Allow user to change
number of listener thread in LlapTaskCommunicator. (Zhiyuan Yang,
reviewed by Rajesh Balamohan)
Posted by om...@apache.org.
HIVE-16082. Allow user to change number of listener thread in
LlapTaskCommunicator. (Zhiyuan Yang, reviewed by Rajesh Balamohan)
Change-Id: Iefbd8f38addf2e3dfb6b60ad1f691e5b19102a21
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/29db8089
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/29db8089
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/29db8089
Branch: refs/heads/branch-2.2
Commit: 29db8089a90c0b736c4e4ac17bcad22836e2c0e1
Parents: 43906f9
Author: Siddharth Seth <ss...@apache.org>
Authored: Fri Mar 3 14:18:58 2017 -0800
Committer: Owen O'Malley <om...@apache.org>
Committed: Tue Mar 28 15:27:57 2017 -0700
----------------------------------------------------------------------
common/src/java/org/apache/hadoop/hive/conf/HiveConf.java | 3 +++
.../apache/hadoop/hive/llap/tezplugins/LlapTaskCommunicator.java | 4 ++--
2 files changed, 5 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/29db8089/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index c6c5284..de2e9d6 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -3153,6 +3153,9 @@ public class HiveConf extends Configuration {
new TimeValidator(TimeUnit.MILLISECONDS),
"Connection timeout (in milliseconds) before a failure to an LLAP daemon from Tez AM.",
"llap.task.communicator.connection.timeout-millis"),
+ LLAP_TASK_COMMUNICATOR_LISTENER_THREAD_COUNT(
+ "hive.llap.task.communicator.listener.thread-count", 30,
+ "The number of task communicator listener threads."),
LLAP_TASK_COMMUNICATOR_CONNECTION_SLEEP_BETWEEN_RETRIES_MS(
"hive.llap.task.communicator.connection.sleep.between.retries.ms", "2000ms",
new TimeValidator(TimeUnit.MILLISECONDS),
http://git-wip-us.apache.org/repos/asf/hive/blob/29db8089/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskCommunicator.java
----------------------------------------------------------------------
diff --git a/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskCommunicator.java b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskCommunicator.java
index b2feb4a..c716c5e 100644
--- a/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskCommunicator.java
+++ b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskCommunicator.java
@@ -216,8 +216,8 @@ public class LlapTaskCommunicator extends TezTaskCommunicatorImpl {
new JobTokenSecretManager();
jobTokenSecretManager.addTokenForJob(tokenIdentifier, sessionToken);
- int numHandlers = conf.getInt(TezConfiguration.TEZ_AM_TASK_LISTENER_THREAD_COUNT,
- TezConfiguration.TEZ_AM_TASK_LISTENER_THREAD_COUNT_DEFAULT);
+ int numHandlers =
+ HiveConf.getIntVar(conf, ConfVars.LLAP_TASK_COMMUNICATOR_LISTENER_THREAD_COUNT);
server = new RPC.Builder(conf)
.setProtocol(LlapTaskUmbilicalProtocol.class)
.setBindAddress("0.0.0.0")
[26/50] [abbrv] hive git commit: HIVE-15084: Flaky test:
TestMiniTezCliDriver:explainanalyze_1, 2, 3, 4, 5 (Pengcheng Xiong)
Posted by om...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/9f5bea3b/ql/src/test/results/clientpositive/union19.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/union19.q.out b/ql/src/test/results/clientpositive/union19.q.out
index 8ff7913..5781ae5 100644
--- a/ql/src/test/results/clientpositive/union19.q.out
+++ b/ql/src/test/results/clientpositive/union19.q.out
@@ -63,7 +63,7 @@ STAGE PLANS:
Select Operator
expressions: 'tst1' (type: string), UDFToString(_col0) (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
table:
@@ -76,7 +76,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
Union
- Statistics: Num rows: 501 Data size: 5484 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 501 Data size: 5584 Basic stats: COMPLETE Column stats: PARTIAL
Group By Operator
aggregations: count(_col1)
keys: _col0 (type: string)
@@ -92,10 +92,10 @@ STAGE PLANS:
Select Operator
expressions: _col0 (type: string), _col1 (type: string), _col1 (type: string)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 501 Data size: 128256 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 501 Data size: 228456 Basic stats: COMPLETE Column stats: PARTIAL
File Output Operator
compressed: false
- Statistics: Num rows: 501 Data size: 128256 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 501 Data size: 228456 Basic stats: COMPLETE Column stats: PARTIAL
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -109,7 +109,7 @@ STAGE PLANS:
outputColumnNames: _col0, _col1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Union
- Statistics: Num rows: 501 Data size: 5484 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 501 Data size: 5584 Basic stats: COMPLETE Column stats: PARTIAL
Group By Operator
aggregations: count(_col1)
keys: _col0 (type: string)
@@ -125,10 +125,10 @@ STAGE PLANS:
Select Operator
expressions: _col0 (type: string), _col1 (type: string), _col1 (type: string)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 501 Data size: 128256 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 501 Data size: 228456 Basic stats: COMPLETE Column stats: PARTIAL
File Output Operator
compressed: false
- Statistics: Num rows: 501 Data size: 128256 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 501 Data size: 228456 Basic stats: COMPLETE Column stats: PARTIAL
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/9f5bea3b/ql/src/test/results/clientpositive/union20.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/union20.q.out b/ql/src/test/results/clientpositive/union20.q.out
index 8dc6a80..ae55a8b 100644
--- a/ql/src/test/results/clientpositive/union20.q.out
+++ b/ql/src/test/results/clientpositive/union20.q.out
@@ -53,7 +53,7 @@ STAGE PLANS:
Select Operator
expressions: 'tst1' (type: string), UDFToString(_col0) (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
table:
@@ -66,12 +66,12 @@ STAGE PLANS:
Map Operator Tree:
TableScan
Union
- Statistics: Num rows: 167 Data size: 1935 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 167 Data size: 2035 Basic stats: COMPLETE Column stats: PARTIAL
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 167 Data size: 1935 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 167 Data size: 2035 Basic stats: COMPLETE Column stats: PARTIAL
value expressions: _col1 (type: string)
TableScan
alias: s2
@@ -84,21 +84,21 @@ STAGE PLANS:
outputColumnNames: _col0, _col1
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
Union
- Statistics: Num rows: 167 Data size: 1935 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 167 Data size: 2035 Basic stats: COMPLETE Column stats: PARTIAL
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 167 Data size: 1935 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 167 Data size: 2035 Basic stats: COMPLETE Column stats: PARTIAL
value expressions: _col1 (type: string)
TableScan
Union
- Statistics: Num rows: 167 Data size: 1935 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 167 Data size: 2035 Basic stats: COMPLETE Column stats: PARTIAL
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 167 Data size: 1935 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 167 Data size: 2035 Basic stats: COMPLETE Column stats: PARTIAL
value expressions: _col1 (type: string)
TableScan
alias: s4
@@ -111,12 +111,12 @@ STAGE PLANS:
outputColumnNames: _col0, _col1
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
Union
- Statistics: Num rows: 167 Data size: 1935 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 167 Data size: 2035 Basic stats: COMPLETE Column stats: PARTIAL
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 167 Data size: 1935 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 167 Data size: 2035 Basic stats: COMPLETE Column stats: PARTIAL
value expressions: _col1 (type: string)
Reduce Operator Tree:
Join Operator
@@ -126,10 +126,10 @@ STAGE PLANS:
0 _col0 (type: string)
1 _col0 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 27889 Data size: 9593816 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 27889 Data size: 15171616 Basic stats: COMPLETE Column stats: PARTIAL
File Output Operator
compressed: false
- Statistics: Num rows: 27889 Data size: 9593816 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 27889 Data size: 15171616 Basic stats: COMPLETE Column stats: PARTIAL
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -161,7 +161,7 @@ STAGE PLANS:
Select Operator
expressions: 'tst1' (type: string), UDFToString(_col0) (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
table:
http://git-wip-us.apache.org/repos/asf/hive/blob/9f5bea3b/ql/src/test/results/clientpositive/union6.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/union6.q.out b/ql/src/test/results/clientpositive/union6.q.out
index 531550e..8448d86 100644
--- a/ql/src/test/results/clientpositive/union6.q.out
+++ b/ql/src/test/results/clientpositive/union6.q.out
@@ -56,7 +56,7 @@ STAGE PLANS:
Select Operator
expressions: 'tst1' (type: string), UDFToString(_col0) (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
table:
@@ -69,10 +69,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
Union
- Statistics: Num rows: 26 Data size: 363 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 26 Data size: 463 Basic stats: COMPLETE Column stats: PARTIAL
File Output Operator
compressed: false
- Statistics: Num rows: 26 Data size: 363 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 26 Data size: 463 Basic stats: COMPLETE Column stats: PARTIAL
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -86,10 +86,10 @@ STAGE PLANS:
outputColumnNames: _col0, _col1
Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE
Union
- Statistics: Num rows: 26 Data size: 363 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 26 Data size: 463 Basic stats: COMPLETE Column stats: PARTIAL
File Output Operator
compressed: false
- Statistics: Num rows: 26 Data size: 363 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 26 Data size: 463 Basic stats: COMPLETE Column stats: PARTIAL
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/9f5bea3b/ql/src/test/results/clientpositive/unionDistinct_1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/unionDistinct_1.q.out b/ql/src/test/results/clientpositive/unionDistinct_1.q.out
index 9200ba3..81b565a 100644
--- a/ql/src/test/results/clientpositive/unionDistinct_1.q.out
+++ b/ql/src/test/results/clientpositive/unionDistinct_1.q.out
@@ -1224,7 +1224,7 @@ STAGE PLANS:
Select Operator
expressions: 'tst1' (type: string), UDFToString(_col0) (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
table:
@@ -1243,40 +1243,40 @@ STAGE PLANS:
outputColumnNames: _col0, _col1
Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE
Union
- Statistics: Num rows: 26 Data size: 363 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 26 Data size: 463 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
keys: _col0 (type: string), _col1 (type: string)
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: string)
sort order: ++
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE
TableScan
Union
- Statistics: Num rows: 26 Data size: 363 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 26 Data size: 463 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
keys: _col0 (type: string), _col1 (type: string)
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: string)
sort order: ++
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Operator Tree:
Group By Operator
keys: KEY._col0 (type: string), KEY._col1 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: _col0 (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: count(1)
keys: _col0 (type: string)
@@ -1377,7 +1377,7 @@ STAGE PLANS:
Select Operator
expressions: 'tst1' (type: string), UDFToString(_col0) (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
table:
@@ -1390,17 +1390,17 @@ STAGE PLANS:
Map Operator Tree:
TableScan
Union
- Statistics: Num rows: 26 Data size: 363 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 26 Data size: 463 Basic stats: COMPLETE Column stats: PARTIAL
Group By Operator
keys: _col0 (type: string), _col1 (type: string)
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: string)
sort order: ++
Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL
TableScan
alias: s2
Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE
@@ -1409,23 +1409,23 @@ STAGE PLANS:
outputColumnNames: _col0, _col1
Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE
Union
- Statistics: Num rows: 26 Data size: 363 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 26 Data size: 463 Basic stats: COMPLETE Column stats: PARTIAL
Group By Operator
keys: _col0 (type: string), _col1 (type: string)
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: string)
sort order: ++
Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL
Reduce Operator Tree:
Group By Operator
keys: KEY._col0 (type: string), KEY._col1 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL
File Output Operator
compressed: false
table:
@@ -1438,17 +1438,17 @@ STAGE PLANS:
Map Operator Tree:
TableScan
Union
- Statistics: Num rows: 26 Data size: 363 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 26 Data size: 463 Basic stats: COMPLETE Column stats: PARTIAL
Group By Operator
keys: _col0 (type: string), _col1 (type: string)
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: string)
sort order: ++
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL
TableScan
alias: s3
Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE
@@ -1457,27 +1457,27 @@ STAGE PLANS:
outputColumnNames: _col0, _col1
Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE
Union
- Statistics: Num rows: 26 Data size: 363 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 26 Data size: 463 Basic stats: COMPLETE Column stats: PARTIAL
Group By Operator
keys: _col0 (type: string), _col1 (type: string)
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: string)
sort order: ++
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL
Reduce Operator Tree:
Group By Operator
keys: KEY._col0 (type: string), KEY._col1 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL
Select Operator
expressions: _col0 (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL
Group By Operator
aggregations: count(1)
keys: _col0 (type: string)
@@ -2952,7 +2952,7 @@ STAGE PLANS:
Select Operator
expressions: 'tst1' (type: string), UDFToString(_col0) (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
table:
@@ -2965,17 +2965,17 @@ STAGE PLANS:
Map Operator Tree:
TableScan
Union
- Statistics: Num rows: 501 Data size: 5484 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 501 Data size: 5584 Basic stats: COMPLETE Column stats: PARTIAL
Group By Operator
keys: _col0 (type: string), _col1 (type: string)
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string)
sort order: +++
Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL
TableScan
alias: s2
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
@@ -2984,23 +2984,23 @@ STAGE PLANS:
outputColumnNames: _col0, _col1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Union
- Statistics: Num rows: 501 Data size: 5484 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 501 Data size: 5584 Basic stats: COMPLETE Column stats: PARTIAL
Group By Operator
keys: _col0 (type: string), _col1 (type: string)
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: string), substr(_col1, 5) (type: string)
sort order: +++
Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL
Reduce Operator Tree:
Group By Operator
keys: KEY._col0 (type: string), KEY._col1 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL
Group By Operator
aggregations: count(DISTINCT substr(_col1, 5))
keys: _col0 (type: string), substr(_col1, 5) (type: string)
@@ -3018,14 +3018,14 @@ STAGE PLANS:
keys: _col0 (type: string), _col1 (type: string)
mode: complete
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 364 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 464 Basic stats: COMPLETE Column stats: PARTIAL
Select Operator
expressions: _col0 (type: string), _col1 (type: string), _col2 (type: bigint)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 356 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 456 Basic stats: COMPLETE Column stats: PARTIAL
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 356 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 456 Basic stats: COMPLETE Column stats: PARTIAL
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -3807,7 +3807,7 @@ STAGE PLANS:
Select Operator
expressions: 'tst1' (type: string), UDFToString(_col0) (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
table:
@@ -3820,17 +3820,17 @@ STAGE PLANS:
Map Operator Tree:
TableScan
Union
- Statistics: Num rows: 501 Data size: 5484 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 501 Data size: 5584 Basic stats: COMPLETE Column stats: PARTIAL
Group By Operator
keys: _col0 (type: string), _col1 (type: string)
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: string)
sort order: ++
Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL
TableScan
alias: s2
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
@@ -3839,26 +3839,26 @@ STAGE PLANS:
outputColumnNames: _col0, _col1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Union
- Statistics: Num rows: 501 Data size: 5484 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 501 Data size: 5584 Basic stats: COMPLETE Column stats: PARTIAL
Group By Operator
keys: _col0 (type: string), _col1 (type: string)
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: string)
sort order: ++
Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL
Reduce Operator Tree:
Group By Operator
keys: KEY._col0 (type: string), KEY._col1 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -3867,10 +3867,10 @@ STAGE PLANS:
Select Operator
expressions: _col0 (type: string), _col1 (type: string), _col1 (type: string)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 256 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 456 Basic stats: COMPLETE Column stats: PARTIAL
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 256 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 456 Basic stats: COMPLETE Column stats: PARTIAL
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -4627,7 +4627,7 @@ STAGE PLANS:
Select Operator
expressions: 'tst1' (type: string), UDFToString(_col0) (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
table:
@@ -4640,17 +4640,17 @@ STAGE PLANS:
Map Operator Tree:
TableScan
Union
- Statistics: Num rows: 501 Data size: 5484 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 501 Data size: 5584 Basic stats: COMPLETE Column stats: PARTIAL
Group By Operator
keys: _col0 (type: string), _col1 (type: string)
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: string)
sort order: ++
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL
TableScan
alias: s2
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
@@ -4659,23 +4659,23 @@ STAGE PLANS:
outputColumnNames: _col0, _col1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Union
- Statistics: Num rows: 501 Data size: 5484 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 501 Data size: 5584 Basic stats: COMPLETE Column stats: PARTIAL
Group By Operator
keys: _col0 (type: string), _col1 (type: string)
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: string)
sort order: ++
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL
Reduce Operator Tree:
Group By Operator
keys: KEY._col0 (type: string), KEY._col1 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL
Group By Operator
aggregations: count(_col1)
keys: _col0 (type: string)
@@ -4693,10 +4693,10 @@ STAGE PLANS:
Select Operator
expressions: _col0 (type: string), _col1 (type: string), _col1 (type: string)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 256 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 456 Basic stats: COMPLETE Column stats: PARTIAL
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 256 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 456 Basic stats: COMPLETE Column stats: PARTIAL
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -5559,7 +5559,7 @@ STAGE PLANS:
Select Operator
expressions: 'tst1' (type: string), UDFToString(_col0) (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
table:
@@ -5572,17 +5572,17 @@ STAGE PLANS:
Map Operator Tree:
TableScan
Union
- Statistics: Num rows: 167 Data size: 1935 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 167 Data size: 2035 Basic stats: COMPLETE Column stats: PARTIAL
Group By Operator
keys: _col0 (type: string), _col1 (type: string)
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: string)
sort order: ++
Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL
TableScan
alias: s2
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
@@ -5594,23 +5594,23 @@ STAGE PLANS:
outputColumnNames: _col0, _col1
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
Union
- Statistics: Num rows: 167 Data size: 1935 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 167 Data size: 2035 Basic stats: COMPLETE Column stats: PARTIAL
Group By Operator
keys: _col0 (type: string), _col1 (type: string)
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: string)
sort order: ++
Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL
Reduce Operator Tree:
Group By Operator
keys: KEY._col0 (type: string), KEY._col1 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL
File Output Operator
compressed: false
table:
@@ -5626,14 +5626,14 @@ STAGE PLANS:
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL
value expressions: _col1 (type: string)
TableScan
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL
value expressions: _col1 (type: string)
Reduce Operator Tree:
Join Operator
@@ -5643,10 +5643,10 @@ STAGE PLANS:
0 _col0 (type: string)
1 _col0 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 1 Data size: 344 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 544 Basic stats: COMPLETE Column stats: PARTIAL
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 344 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 544 Basic stats: COMPLETE Column stats: PARTIAL
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -5678,7 +5678,7 @@ STAGE PLANS:
Select Operator
expressions: 'tst1' (type: string), UDFToString(_col0) (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
table:
@@ -5691,17 +5691,17 @@ STAGE PLANS:
Map Operator Tree:
TableScan
Union
- Statistics: Num rows: 167 Data size: 1935 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 167 Data size: 2035 Basic stats: COMPLETE Column stats: PARTIAL
Group By Operator
keys: _col0 (type: string), _col1 (type: string)
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: string)
sort order: ++
Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL
TableScan
alias: s4
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
@@ -5713,23 +5713,23 @@ STAGE PLANS:
outputColumnNames: _col0, _col1
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
Union
- Statistics: Num rows: 167 Data size: 1935 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 167 Data size: 2035 Basic stats: COMPLETE Column stats: PARTIAL
Group By Operator
keys: _col0 (type: string), _col1 (type: string)
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: string)
sort order: ++
Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL
Reduce Operator Tree:
Group By Operator
keys: KEY._col0 (type: string), KEY._col1 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL
File Output Operator
compressed: false
table:
@@ -15445,7 +15445,7 @@ STAGE PLANS:
Select Operator
expressions: 'tst1' (type: string), UDFToString(_col0) (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
table:
@@ -15458,17 +15458,17 @@ STAGE PLANS:
Map Operator Tree:
TableScan
Union
- Statistics: Num rows: 26 Data size: 363 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 26 Data size: 463 Basic stats: COMPLETE Column stats: PARTIAL
Group By Operator
keys: _col0 (type: string), _col1 (type: string)
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: string)
sort order: ++
Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL
TableScan
alias: s2
Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE
@@ -15477,26 +15477,26 @@ STAGE PLANS:
outputColumnNames: _col0, _col1
Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE
Union
- Statistics: Num rows: 26 Data size: 363 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 26 Data size: 463 Basic stats: COMPLETE Column stats: PARTIAL
Group By Operator
keys: _col0 (type: string), _col1 (type: string)
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: string)
sort order: ++
Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL
Reduce Operator Tree:
Group By Operator
keys: KEY._col0 (type: string), KEY._col1 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -15607,7 +15607,7 @@ STAGE PLANS:
Select Operator
expressions: 'tst1' (type: string), UDFToString(_col0) (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
table:
@@ -15620,17 +15620,17 @@ STAGE PLANS:
Map Operator Tree:
TableScan
Union
- Statistics: Num rows: 26 Data size: 363 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 26 Data size: 463 Basic stats: COMPLETE Column stats: PARTIAL
Group By Operator
keys: _col0 (type: string), _col1 (type: string)
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: string)
sort order: ++
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL
TableScan
alias: s2
Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE
@@ -15639,27 +15639,27 @@ STAGE PLANS:
outputColumnNames: _col0, _col1
Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE
Union
- Statistics: Num rows: 26 Data size: 363 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 26 Data size: 463 Basic stats: COMPLETE Column stats: PARTIAL
Group By Operator
keys: _col0 (type: string), _col1 (type: string)
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: string)
sort order: ++
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL
Reduce Operator Tree:
Group By Operator
keys: KEY._col0 (type: string), KEY._col1 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL
Select Operator
expressions: _col0 (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 172 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: PARTIAL
Group By Operator
aggregations: count(1)
keys: _col0 (type: string)