You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ok...@apache.org on 2026/03/16 03:22:53 UTC
(hive) branch master updated: HIVE-29438: properly handle unknown (negative) numNulls/numTrues/numFalses from ColStats (#6294)
This is an automated email from the ASF dual-hosted git repository.
okumin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 2abffc720e1 HIVE-29438: properly handle unknown (negative) numNulls/numTrues/numFalses from ColStats (#6294)
2abffc720e1 is described below
commit 2abffc720e1d24190d1f1cce4eeece59ce0c1d05
Author: konstantinb <ko...@users.noreply.github.com>
AuthorDate: Mon Mar 16 12:22:45 2026 +0900
HIVE-29438: properly handle unknown (negative) numNulls/numTrues/numFalses from ColStats (#6294)
* HIVE-29438: properly handle unknown (negative) numNulls/numTrues/numFalses from ColStats
* HIVE-29438: proper out file instead of the stale one
* HIVE-29438: still different masking locally vs the pipeline?
* HIVE-29438: _ptoperly_ regenerated the new .q.out file
* fork/HIVE-29438: whitespace formatting fixes in test code
* HIVE-29438: fine-tuned the NDV calculations based on numTrues/numFalses, properly handling "const NULL" cases
* HIVE-29438: attempt a rebuild
* HIVE-29438: trigger a rebuild
* fork/HIVE-29438: addressing SonarQube feedback
* fork/HIVE-29438: more PR & SonarQube feedback items
---
.../stats/annotation/StatsRulesProcFactory.java | 24 +-
.../org/apache/hadoop/hive/ql/plan/Statistics.java | 33 +-
.../apache/hadoop/hive/ql/stats/StatsUtils.java | 38 +-
.../stats/estimator/PessimisticStatCombiner.java | 45 +-
.../annotation/TestStatsRulesProcFactory.java | 121 +
.../hadoop/hive/ql/stats/TestStatsUtils.java | 255 ++
.../estimator/TestPessimisticStatCombiner.java | 165 ++
ql/src/test/queries/clientpositive/unknown_stats.q | 62 +
.../clientpositive/llap/unknown_stats.q.out | 2577 ++++++++++++++++++++
9 files changed, 3264 insertions(+), 56 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
index 42d62e0a64e..abfe6170217 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
@@ -425,10 +425,10 @@ protected long evaluateExpression(Statistics stats, ExprNodeDesc pred,
String colType = encd.getTypeString();
if (colType.equalsIgnoreCase(serdeConstants.BOOLEAN_TYPE_NAME)) {
ColStatistics cs = stats.getColumnStatisticsFromColName(colName);
- if (cs != null) {
+ if (cs != null && cs.getNumTrues() >= 0) {
newNumRows = cs.getNumTrues();
} else {
- // default
+ // default (no stats or numTrues unknown i.e. negative)
newNumRows = stats.getNumRows() / 2;
}
} else {
@@ -924,11 +924,11 @@ private long evaluateNotExpr(Statistics stats, ExprNodeDesc pred, long currNumRo
String colType = encd.getTypeString();
if (colType.equalsIgnoreCase(serdeConstants.BOOLEAN_TYPE_NAME)) {
ColStatistics cs = stats.getColumnStatisticsFromColName(colName);
- if (cs != null) {
+ if (cs != null && cs.getNumFalses() >= 0) {
return cs.getNumFalses();
}
}
- // if not boolean column return half the number of rows
+ // if not boolean column, or numFalses unknown (negative), return half the number of rows
return numRows / 2;
}
}
@@ -953,7 +953,7 @@ private long evaluateColEqualsNullExpr(Statistics stats, AnnotateStatsProcCtx as
aspCtx.addAffectedColumn(colDesc);
String colName = colDesc.getColumn();
ColStatistics cs = stats.getColumnStatisticsFromColName(colName);
- if (cs != null) {
+ if (cs != null && cs.getNumNulls() >= 0) {
return cs.getNumNulls();
}
}
@@ -1707,16 +1707,19 @@ public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx,
* If possible, sets the min / max value for the column based on the aggregate function
* being calculated and its input.
*/
- private static void computeAggregateColumnMinMax(ColStatistics cs, HiveConf conf, AggregationDesc agg, String aggType,
+ @VisibleForTesting
+ static void computeAggregateColumnMinMax(ColStatistics cs, HiveConf conf, AggregationDesc agg, String aggType,
Statistics parentStats) throws SemanticException {
if (agg.getParameters() != null && agg.getParameters().size() == 1) {
ColStatistics parentCS = StatsUtils.getColStatisticsFromExpression(
conf, parentStats, agg.getParameters().get(0));
if (parentCS != null && parentCS.getRange() != null &&
parentCS.getRange().minValue != null && parentCS.getRange().maxValue != null) {
+ // numNulls < 0 means "unknown" - treat as 0 for conservative COUNT estimate
+ long numNulls = parentCS.getNumNulls() < 0 ? 0 : parentCS.getNumNulls();
long valuesCount = agg.getDistinct() ?
parentCS.getCountDistint() :
- parentStats.getNumRows() - parentCS.getNumNulls();
+ parentStats.getNumRows() - numNulls;
Range range = parentCS.getRange();
// Get the aggregate function matching the name in the query.
GenericUDAFResolver udaf =
@@ -2548,7 +2551,8 @@ private boolean isJoinKey(final String columnName,
return false;
}
- private void updateNumNulls(ColStatistics colStats, long leftUnmatchedRows, long rightUnmatchedRows,
+ @VisibleForTesting
+ void updateNumNulls(ColStatistics colStats, long leftUnmatchedRows, long rightUnmatchedRows,
long newNumRows, long pos, CommonJoinOperator<? extends JoinDesc> jop) {
if (!(jop.getConf().getConds().length == 1)) {
@@ -2557,6 +2561,10 @@ private void updateNumNulls(ColStatistics colStats, long leftUnmatchedRows, long
}
long oldNumNulls = colStats.getNumNulls();
+ // numNulls < 0 means "unknown" - preserve the sentinel value
+ if (oldNumNulls < 0) {
+ return;
+ }
long newNumNulls = Math.min(newNumRows, oldNumNulls);
JoinCondDesc joinCond = jop.getConf().getConds()[0];
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/Statistics.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/Statistics.java
index 78ca89d9fd0..d672b7acfc2 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/Statistics.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/Statistics.java
@@ -240,28 +240,27 @@ public void setColumnStats(List<ColStatistics> colStats) {
}
public void addToColumnStats(List<ColStatistics> colStats) {
-
+ if (colStats == null) {
+ return;
+ }
if (columnStats == null) {
columnStats = Maps.newHashMap();
}
- if (colStats != null) {
- for (ColStatistics cs : colStats) {
- ColStatistics updatedCS = null;
- if (cs != null) {
-
- String key = cs.getColumnName();
- // if column statistics for a column is already found then merge the statistics
- if (columnStats.containsKey(key) && columnStats.get(key) != null) {
- updatedCS = columnStats.get(key);
- updatedCS.setAvgColLen(Math.max(updatedCS.getAvgColLen(), cs.getAvgColLen()));
- updatedCS.setNumNulls(StatsUtils.safeAdd(updatedCS.getNumNulls(), cs.getNumNulls()));
- updatedCS.setCountDistint(Math.max(updatedCS.getCountDistint(), cs.getCountDistint()));
- columnStats.put(key, updatedCS);
- } else {
- columnStats.put(key, cs);
- }
+ for (ColStatistics cs : colStats) {
+ if (cs == null) {
+ continue;
+ }
+ String key = cs.getColumnName();
+ ColStatistics existing = columnStats.computeIfAbsent(key, k -> cs);
+ if (existing != cs) {
+ existing.setAvgColLen(Math.max(existing.getAvgColLen(), cs.getAvgColLen()));
+ if (cs.getNumNulls() < 0 || existing.getNumNulls() < 0) {
+ existing.setNumNulls(-1);
+ } else {
+ existing.setNumNulls(StatsUtils.safeAdd(existing.getNumNulls(), cs.getNumNulls()));
}
+ existing.setCountDistint(Math.max(existing.getCountDistint(), cs.getCountDistint()));
}
}
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
index 3f333886965..65586278e2e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
@@ -818,13 +818,20 @@ public static ColStatistics getColStatistics(ColumnStatisticsObj cso, String col
cs.setAvgColLen(csd.getStringStats().getAvgColLen());
cs.setBitVectors(csd.getStringStats().getBitVectors());
} else if (colTypeLowerCase.equals(serdeConstants.BOOLEAN_TYPE_NAME)) {
- if (csd.getBooleanStats().getNumFalses() > 0 && csd.getBooleanStats().getNumTrues() > 0) {
- cs.setCountDistint(2);
- } else {
+ long numTrues = csd.getBooleanStats().getNumTrues();
+ long numFalses = csd.getBooleanStats().getNumFalses();
+ if (numTrues == 0 && numFalses == 0) {
+ // All NULL column - no non-null distinct values
+ cs.setCountDistint(0);
+ } else if (numTrues == 0 || numFalses == 0) {
+ // One value type confirmed absent (=0), other is present (>0) or unknown (<0)
cs.setCountDistint(1);
+ } else {
+ // Both != 0: either both present (>0), both unknown (<0), or one present + one unknown
+ cs.setCountDistint(2);
}
- cs.setNumTrues(csd.getBooleanStats().getNumTrues());
- cs.setNumFalses(csd.getBooleanStats().getNumFalses());
+ cs.setNumTrues(numTrues);
+ cs.setNumFalses(numFalses);
cs.setNumNulls(csd.getBooleanStats().getNumNulls());
cs.setAvgColLen(JavaDataModel.get().primitive1());
} else if (colTypeLowerCase.equals(serdeConstants.BINARY_TYPE_NAME)) {
@@ -2042,8 +2049,11 @@ public static void updateStats(Statistics stats, long newNumRows,
if (oldDV > newNumRows) {
cs.setCountDistint(newNumRows);
}
- long newNumNulls = Math.round(ratio * cs.getNumNulls());
- cs.setNumNulls(Math.min(newNumNulls, newNumRows));
+ // numNulls < 0 means "unknown" - preserve the sentinel value
+ if (cs.getNumNulls() >= 0) {
+ long newNumNulls = Math.round(ratio * cs.getNumNulls());
+ cs.setNumNulls(Math.min(newNumNulls, newNumRows));
+ }
}
stats.setColumnStats(colStats);
long newDataSize = StatsUtils.getDataSizeFromColumnStats(newNumRows, colStats);
@@ -2056,9 +2066,17 @@ public static void updateStats(Statistics stats, long newNumRows,
public static void scaleColStatistics(List<ColStatistics> colStats, double factor) {
for (ColStatistics cs : colStats) {
- cs.setNumFalses(StatsUtils.safeMult(cs.getNumFalses(), factor));
- cs.setNumTrues(StatsUtils.safeMult(cs.getNumTrues(), factor));
- cs.setNumNulls(StatsUtils.safeMult(cs.getNumNulls(), factor));
+ // numTrues/numFalses < 0 means "unknown" - preserve the sentinel value
+ if (cs.getNumFalses() >= 0) {
+ cs.setNumFalses(StatsUtils.safeMult(cs.getNumFalses(), factor));
+ }
+ if (cs.getNumTrues() >= 0) {
+ cs.setNumTrues(StatsUtils.safeMult(cs.getNumTrues(), factor));
+ }
+ // numNulls < 0 means "unknown" - preserve the sentinel value
+ if (cs.getNumNulls() >= 0) {
+ cs.setNumNulls(StatsUtils.safeMult(cs.getNumNulls(), factor));
+ }
if (factor < 1.0) {
final double newNDV = Math.ceil(cs.getCountDistint() * factor);
cs.setCountDistint(newNDV > Long.MAX_VALUE ? Long.MAX_VALUE : (long) newNDV);
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/estimator/PessimisticStatCombiner.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/estimator/PessimisticStatCombiner.java
index dde2019eadf..4de2867de7c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/stats/estimator/PessimisticStatCombiner.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/estimator/PessimisticStatCombiner.java
@@ -37,28 +37,31 @@ public void add(ColStatistics stat) {
result.setRange(null);
result.setIsEstimated(true);
return;
- } else {
- if (stat.getAvgColLen() > result.getAvgColLen()) {
- result.setAvgColLen(stat.getAvgColLen());
- }
- if (stat.getCountDistint() > result.getCountDistint()) {
- result.setCountDistint(stat.getCountDistint());
- }
- if (stat.getNumNulls() > result.getNumNulls()) {
- result.setNumNulls(stat.getNumNulls());
- }
- if (stat.getNumTrues() > result.getNumTrues()) {
- result.setNumTrues(stat.getNumTrues());
- }
- if (stat.getNumFalses() > result.getNumFalses()) {
- result.setNumFalses(stat.getNumFalses());
- }
- if (stat.isFilteredColumn()) {
- result.setFilterColumn();
- }
-
}
-
+ if (stat.getAvgColLen() > result.getAvgColLen()) {
+ result.setAvgColLen(stat.getAvgColLen());
+ }
+ if (stat.getCountDistint() > result.getCountDistint()) {
+ result.setCountDistint(stat.getCountDistint());
+ }
+ if (stat.getNumNulls() < 0 || result.getNumNulls() < 0) {
+ result.setNumNulls(-1);
+ } else if (stat.getNumNulls() > result.getNumNulls()) {
+ result.setNumNulls(stat.getNumNulls());
+ }
+ if (stat.getNumTrues() < 0 || result.getNumTrues() < 0) {
+ result.setNumTrues(-1);
+ } else if (stat.getNumTrues() > result.getNumTrues()) {
+ result.setNumTrues(stat.getNumTrues());
+ }
+ if (stat.getNumFalses() < 0 || result.getNumFalses() < 0) {
+ result.setNumFalses(-1);
+ } else if (stat.getNumFalses() > result.getNumFalses()) {
+ result.setNumFalses(stat.getNumFalses());
+ }
+ if (stat.isFilteredColumn()) {
+ result.setFilterColumn();
+ }
}
public Optional<ColStatistics> getResult() {
return Optional.of(result);
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/optimizer/stats/annotation/TestStatsRulesProcFactory.java b/ql/src/test/org/apache/hadoop/hive/ql/optimizer/stats/annotation/TestStatsRulesProcFactory.java
index 1290a54c1f3..4d9d351af8f 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/optimizer/stats/annotation/TestStatsRulesProcFactory.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/optimizer/stats/annotation/TestStatsRulesProcFactory.java
@@ -35,14 +35,24 @@
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqualOrLessThan;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPGreaterThan;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPLessThan;
+import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+import org.apache.hadoop.hive.ql.plan.AggregationDesc;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator;
import org.junit.Test;
import java.util.Arrays;
import java.util.Collections;
+import org.apache.hadoop.hive.ql.exec.CommonJoinOperator;
+import org.apache.hadoop.hive.ql.plan.JoinCondDesc;
+import org.apache.hadoop.hive.ql.plan.JoinDesc;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
import static org.apache.hadoop.hive.ql.optimizer.stats.annotation.StatsRulesProcFactory.FilterStatsRule.extractFloatFromLiteralValue;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertThrows;
public class TestStatsRulesProcFactory {
@@ -631,4 +641,115 @@ private static ColStatistics createColStatistics(
return colStatistics;
}
+
+ /**
+ * Test that computeAggregateColumnMinMax properly handles numNulls=-1 (unknown).
+ * With the fix, numNulls=-1 should be treated as 0, giving valuesCount = numRows.
+ * Without the fix, valuesCount = numRows - (-1) = numRows + 1 (wrong).
+ */
+ @Test
+ public void testComputeAggregateColumnMinMaxWithUnknownNumNulls() throws SemanticException {
+ ColStatistics cs = new ColStatistics("_col0", "bigint");
+ HiveConf conf = new HiveConf();
+
+ // Create parent column stats with numNulls=-1 (unknown) and Range(1, 100)
+ ColStatistics parentColStats = new ColStatistics("val", "int");
+ parentColStats.setNumNulls(-1); // unknown numNulls - this is what we're testing
+ parentColStats.setCountDistint(100);
+ parentColStats.setRange(1, 100);
+
+ Statistics parentStats = new Statistics(100, 400, 400, 400);
+ parentStats.addToColumnStats(Collections.singletonList(parentColStats));
+
+ ExprNodeColumnDesc colExpr = new ExprNodeColumnDesc(
+ TypeInfoFactory.intTypeInfo, "val", "t", false);
+
+ AggregationDesc agg = new AggregationDesc();
+ agg.setGenericUDAFName("count");
+ agg.setParameters(Collections.singletonList(colExpr));
+ agg.setDistinct(false);
+ agg.setMode(GenericUDAFEvaluator.Mode.COMPLETE);
+
+ StatsRulesProcFactory.GroupByStatsRule.computeAggregateColumnMinMax(
+ cs, conf, agg, "bigint", parentStats);
+
+ // Verify: With the fix, COUNT Range should be (0, 100)
+ // numNulls=-1 is treated as 0, so valuesCount = 100 - 0 = 100
+ // Without the fix, valuesCount = 100 - (-1) = 101 (WRONG)
+ assertNotNull("Range should be set on COUNT column", cs.getRange());
+ assertEquals("COUNT min should be 0", 0L, ((Number) cs.getRange().minValue).longValue());
+ assertEquals("COUNT max should be 100 (numRows), not 101",
+ 100L, ((Number) cs.getRange().maxValue).longValue());
+ }
+
+ @Test
+ public void testComputeAggregateColumnMinMaxWithKnownNumNulls() throws SemanticException {
+ ColStatistics cs = new ColStatistics("_col0", "bigint");
+ HiveConf conf = new HiveConf();
+
+ // Create parent column stats with numNulls=20 (known) and Range
+ ColStatistics parentColStats = new ColStatistics("val", "int");
+ parentColStats.setNumNulls(20); // known numNulls
+ parentColStats.setCountDistint(80);
+ parentColStats.setRange(1, 100);
+
+ Statistics parentStats = new Statistics(100, 400, 400, 400);
+ parentStats.addToColumnStats(Collections.singletonList(parentColStats));
+
+ ExprNodeColumnDesc colExpr = new ExprNodeColumnDesc(
+ TypeInfoFactory.intTypeInfo, "val", "t", false);
+ AggregationDesc agg = new AggregationDesc();
+ agg.setGenericUDAFName("count");
+ agg.setParameters(Collections.singletonList(colExpr));
+ agg.setDistinct(false);
+ agg.setMode(GenericUDAFEvaluator.Mode.COMPLETE);
+
+ StatsRulesProcFactory.GroupByStatsRule.computeAggregateColumnMinMax(
+ cs, conf, agg, "bigint", parentStats);
+
+ // With known numNulls=20, valuesCount = 100 - 20 = 80
+ assertNotNull("Range should be set", cs.getRange());
+ assertEquals(0L, ((Number) cs.getRange().minValue).longValue());
+ assertEquals("COUNT max should be 80 (numRows - numNulls)",
+ 80L, ((Number) cs.getRange().maxValue).longValue());
+ }
+
+ /**
+ * Test that JoinStatsRule.updateNumNulls preserves unknown numNulls (-1).
+ * With the fix, when numNulls is -1 (unknown), the method returns early without modification.
+ * Without the fix, LEFT_OUTER_JOIN would calculate: newNumNulls = oldNumNulls + leftUnmatchedRows = -1 + 100 = 99
+ */
+ @Test
+ public void testUpdateNumNullsPreservesUnknownNumNulls() {
+ StatsRulesProcFactory.JoinStatsRule joinStatsRule = new StatsRulesProcFactory.JoinStatsRule();
+
+ // Create ColStatistics with numNulls = -1 (unknown)
+ ColStatistics colStats = new ColStatistics("non_join_col", "int");
+ colStats.setNumNulls(-1);
+ colStats.setCountDistint(100);
+
+ // Create a mock JoinOperator with LEFT_OUTER_JOIN
+ JoinCondDesc joinCond = mock(JoinCondDesc.class);
+ when(joinCond.getType()).thenReturn(JoinDesc.LEFT_OUTER_JOIN);
+ when(joinCond.getRight()).thenReturn(0); // pos=0 will match getRight()
+
+ JoinDesc joinDesc = mock(JoinDesc.class);
+ when(joinDesc.getConds()).thenReturn(new JoinCondDesc[] {joinCond});
+ when(joinDesc.getJoinKeys()).thenReturn(new ExprNodeDesc[][] {});
+
+ @SuppressWarnings("unchecked")
+ CommonJoinOperator<JoinDesc> mockJop = mock(CommonJoinOperator.class);
+ when(mockJop.getConf()).thenReturn(joinDesc);
+
+ // Call updateNumNulls with:
+ // - leftUnmatchedRows=100 (without fix, this would be added to -1, giving 99)
+ // - pos=0 (matches joinCond.getRight())
+ // With the fix: should return early because numNulls is -1
+ // Without fix: numNulls would become Math.min(1000, -1 + 100) = 99
+ joinStatsRule.updateNumNulls(colStats, 100L, 100L, 1000L, 0L, mockJop);
+
+ // Assert that numNulls is still -1 (unchanged)
+ assertEquals("Unknown numNulls (-1) should be preserved after updateNumNulls",
+ -1L, colStats.getNumNulls());
+ }
}
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/stats/TestStatsUtils.java b/ql/src/test/org/apache/hadoop/hive/ql/stats/TestStatsUtils.java
index 5701fc40581..3f76c554d44 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/stats/TestStatsUtils.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/stats/TestStatsUtils.java
@@ -25,17 +25,21 @@
import java.lang.reflect.Field;
import java.lang.reflect.Modifier;
+import java.util.Collections;
+import java.util.List;
import java.util.Set;
import java.util.stream.Stream;
import org.apache.commons.lang3.reflect.FieldUtils;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData;
import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
import org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData;
import org.apache.hadoop.hive.metastore.api.LongColumnStatsData;
import org.apache.hadoop.hive.ql.plan.ColStatistics;
import org.apache.hadoop.hive.ql.plan.ColStatistics.Range;
+import org.apache.hadoop.hive.ql.plan.Statistics;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
@@ -244,4 +248,255 @@ static Stream<Arguments> floatingPointStatisticsTestData() {
);
}
+ private ColStatistics createColStats(String name, long ndv, long numNulls) {
+ ColStatistics cs = new ColStatistics(name, "string");
+ cs.setCountDistint(ndv);
+ cs.setNumNulls(numNulls);
+ return cs;
+ }
+
+ @Test
+ void testStatisticsAddToColumnStatsPropagatesUnknownNumNulls() {
+ Statistics stats = new Statistics(1000, 8000, 0, 0);
+ ColStatistics cs1 = createColStats("col1", 100, 50);
+ stats.setColumnStats(Collections.singletonList(cs1));
+
+ ColStatistics cs2 = createColStats("col1", 150, -1); // unknown numNulls
+ stats.addToColumnStats(Collections.singletonList(cs2));
+
+ ColStatistics merged = stats.getColumnStatisticsFromColName("col1");
+ assertEquals(-1, merged.getNumNulls(), "Unknown numNulls (-1) should be propagated when merging");
+ }
+
+ @Test
+ void testStatisticsAddToColumnStatsPropagatesUnknownFromExisting() {
+ Statistics stats = new Statistics(1000, 8000, 0, 0);
+ ColStatistics cs1 = createColStats("col1", 100, -1); // unknown numNulls
+ stats.setColumnStats(Collections.singletonList(cs1));
+
+ ColStatistics cs2 = createColStats("col1", 150, 50);
+ stats.addToColumnStats(Collections.singletonList(cs2));
+
+ ColStatistics merged = stats.getColumnStatisticsFromColName("col1");
+ assertEquals(-1, merged.getNumNulls(), "Unknown numNulls (-1) should be propagated when existing is unknown");
+ }
+
+ @Test
+ void testGetColStatisticsBooleanWithUnknownNumTrues() {
+ ColumnStatisticsObj cso = new ColumnStatisticsObj();
+ cso.setColName("bool_col");
+ cso.setColType(serdeConstants.BOOLEAN_TYPE_NAME);
+
+ BooleanColumnStatsData boolStats = new BooleanColumnStatsData();
+ boolStats.setNumTrues(-1); // unknown
+ boolStats.setNumFalses(100);
+ boolStats.setNumNulls(10);
+
+ ColumnStatisticsData data = new ColumnStatisticsData();
+ data.setBooleanStats(boolStats);
+ cso.setStatsData(data);
+
+ ColStatistics cs = StatsUtils.getColStatistics(cso, "bool_col");
+
+ assertNotNull(cs);
+ assertEquals(2, cs.getCountDistint(), "Boolean NDV should be 2 when numTrues is unknown (-1)");
+ }
+
+ @Test
+ void testGetColStatisticsBooleanWithUnknownNumFalses() {
+ ColumnStatisticsObj cso = new ColumnStatisticsObj();
+ cso.setColName("bool_col");
+ cso.setColType(serdeConstants.BOOLEAN_TYPE_NAME);
+
+ BooleanColumnStatsData boolStats = new BooleanColumnStatsData();
+ boolStats.setNumTrues(100);
+ boolStats.setNumFalses(-1); // unknown
+ boolStats.setNumNulls(10);
+
+ ColumnStatisticsData data = new ColumnStatisticsData();
+ data.setBooleanStats(boolStats);
+ cso.setStatsData(data);
+
+ ColStatistics cs = StatsUtils.getColStatistics(cso, "bool_col");
+
+ assertNotNull(cs);
+ assertEquals(2, cs.getCountDistint(), "Boolean NDV should be 2 when numFalses is unknown (-1)");
+ }
+
+ @Test
+ void testGetColStatisticsBooleanWithBothUnknown() {
+ ColumnStatisticsObj cso = new ColumnStatisticsObj();
+ cso.setColName("bool_col");
+ cso.setColType(serdeConstants.BOOLEAN_TYPE_NAME);
+
+ BooleanColumnStatsData boolStats = new BooleanColumnStatsData();
+ boolStats.setNumTrues(-1); // unknown
+ boolStats.setNumFalses(-1); // unknown
+ boolStats.setNumNulls(10);
+
+ ColumnStatisticsData data = new ColumnStatisticsData();
+ data.setBooleanStats(boolStats);
+ cso.setStatsData(data);
+
+ ColStatistics cs = StatsUtils.getColStatistics(cso, "bool_col");
+
+ assertNotNull(cs);
+ assertEquals(2, cs.getCountDistint(), "Boolean NDV should be 2 when both numTrues and numFalses are unknown");
+ }
+
+ @Test
+ void testGetColStatisticsBooleanAllNull() {
+ ColumnStatisticsObj cso = new ColumnStatisticsObj();
+ cso.setColName("bool_col");
+ cso.setColType(serdeConstants.BOOLEAN_TYPE_NAME);
+
+ BooleanColumnStatsData boolStats = new BooleanColumnStatsData();
+ boolStats.setNumTrues(0);
+ boolStats.setNumFalses(0);
+ boolStats.setNumNulls(100); // all NULL
+
+ ColumnStatisticsData data = new ColumnStatisticsData();
+ data.setBooleanStats(boolStats);
+ cso.setStatsData(data);
+
+ ColStatistics cs = StatsUtils.getColStatistics(cso, "bool_col");
+
+ assertNotNull(cs);
+ assertEquals(0, cs.getCountDistint(), "Boolean NDV should be 0 for all-NULL column");
+ }
+
+ @Test
+ void testGetColStatisticsBooleanOnlyTrueValues() {
+ ColumnStatisticsObj cso = new ColumnStatisticsObj();
+ cso.setColName("bool_col");
+ cso.setColType(serdeConstants.BOOLEAN_TYPE_NAME);
+
+ BooleanColumnStatsData boolStats = new BooleanColumnStatsData();
+ boolStats.setNumTrues(100);
+ boolStats.setNumFalses(0); // no FALSE values
+ boolStats.setNumNulls(10);
+
+ ColumnStatisticsData data = new ColumnStatisticsData();
+ data.setBooleanStats(boolStats);
+ cso.setStatsData(data);
+
+ ColStatistics cs = StatsUtils.getColStatistics(cso, "bool_col");
+
+ assertNotNull(cs);
+ assertEquals(1, cs.getCountDistint(), "Boolean NDV should be 1 when only TRUE values present");
+ }
+
+ @Test
+ void testGetColStatisticsBooleanOnlyFalseValues() {
+ ColumnStatisticsObj cso = new ColumnStatisticsObj();
+ cso.setColName("bool_col");
+ cso.setColType(serdeConstants.BOOLEAN_TYPE_NAME);
+
+ BooleanColumnStatsData boolStats = new BooleanColumnStatsData();
+ boolStats.setNumTrues(0); // no TRUE values
+ boolStats.setNumFalses(100);
+ boolStats.setNumNulls(10);
+
+ ColumnStatisticsData data = new ColumnStatisticsData();
+ data.setBooleanStats(boolStats);
+ cso.setStatsData(data);
+
+ ColStatistics cs = StatsUtils.getColStatistics(cso, "bool_col");
+
+ assertNotNull(cs);
+ assertEquals(1, cs.getCountDistint(), "Boolean NDV should be 1 when only FALSE values present");
+ }
+
+ @Test
+ void testGetColStatisticsBooleanNoTrueUnknownFalse() {
+ ColumnStatisticsObj cso = new ColumnStatisticsObj();
+ cso.setColName("bool_col");
+ cso.setColType(serdeConstants.BOOLEAN_TYPE_NAME);
+
+ BooleanColumnStatsData boolStats = new BooleanColumnStatsData();
+ boolStats.setNumTrues(0); // confirmed no TRUE values
+ boolStats.setNumFalses(-1); // unknown FALSE count
+ boolStats.setNumNulls(10);
+
+ ColumnStatisticsData data = new ColumnStatisticsData();
+ data.setBooleanStats(boolStats);
+ cso.setStatsData(data);
+
+ ColStatistics cs = StatsUtils.getColStatistics(cso, "bool_col");
+
+ assertNotNull(cs);
+ assertEquals(1, cs.getCountDistint(), "Boolean NDV should be 1 when TRUE is 0 and FALSE is unknown");
+ }
+
+ @Test
+ void testGetColStatisticsBooleanUnknownTrueNoFalse() {
+ ColumnStatisticsObj cso = new ColumnStatisticsObj();
+ cso.setColName("bool_col");
+ cso.setColType(serdeConstants.BOOLEAN_TYPE_NAME);
+
+ BooleanColumnStatsData boolStats = new BooleanColumnStatsData();
+ boolStats.setNumTrues(-1); // unknown TRUE count
+ boolStats.setNumFalses(0); // confirmed no FALSE values
+ boolStats.setNumNulls(10);
+
+ ColumnStatisticsData data = new ColumnStatisticsData();
+ data.setBooleanStats(boolStats);
+ cso.setStatsData(data);
+
+ ColStatistics cs = StatsUtils.getColStatistics(cso, "bool_col");
+
+ assertNotNull(cs);
+ assertEquals(1, cs.getCountDistint(), "Boolean NDV should be 1 when TRUE is unknown and FALSE is 0");
+ }
+
+ @Test
+ void testUpdateStatsPreservesUnknownNumNulls() {
+ Statistics stats = new Statistics(1000, 8000, 0, 0);
+ ColStatistics cs = createColStats("col1", 100, -1); // unknown numNulls
+ stats.setColumnStats(Collections.singletonList(cs));
+
+ StatsUtils.updateStats(stats, 500, true, null); // scale down to 500 rows
+
+ ColStatistics updated = stats.getColumnStats().get(0);
+ assertEquals(-1, updated.getNumNulls(), "Unknown numNulls (-1) should be preserved after scaling");
+ }
+
+ @Test
+ void testScaleColStatisticsPreservesUnknownNumNulls() {
+ ColStatistics cs = createColStats("col1", 100, -1); // unknown numNulls
+ List<ColStatistics> colStats = Collections.singletonList(cs);
+
+ StatsUtils.scaleColStatistics(colStats, 2.0);
+
+ assertEquals(-1, colStats.get(0).getNumNulls(), "Unknown numNulls (-1) should be preserved after scaling");
+ }
+
+ @Test
+ void testScaleColStatisticsPreservesUnknownNumTrues() {
+ ColStatistics cs = new ColStatistics("bool_col", "boolean");
+ cs.setNumTrues(-1); // unknown
+ cs.setNumFalses(100);
+ cs.setNumNulls(10);
+ List<ColStatistics> colStats = Collections.singletonList(cs);
+
+ StatsUtils.scaleColStatistics(colStats, 2.0);
+
+ assertEquals(-1, colStats.get(0).getNumTrues(), "Unknown numTrues (-1) should be preserved after scaling");
+ assertEquals(200, colStats.get(0).getNumFalses(), "Known numFalses should be scaled");
+ }
+
+ @Test
+ void testScaleColStatisticsPreservesUnknownNumFalses() {
+ ColStatistics cs = new ColStatistics("bool_col", "boolean");
+ cs.setNumTrues(100);
+ cs.setNumFalses(-1); // unknown
+ cs.setNumNulls(10);
+ List<ColStatistics> colStats = Collections.singletonList(cs);
+
+ StatsUtils.scaleColStatistics(colStats, 2.0);
+
+ assertEquals(200, colStats.get(0).getNumTrues(), "Known numTrues should be scaled");
+ assertEquals(-1, colStats.get(0).getNumFalses(), "Unknown numFalses (-1) should be preserved after scaling");
+ }
+
}
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/stats/estimator/TestPessimisticStatCombiner.java b/ql/src/test/org/apache/hadoop/hive/ql/stats/estimator/TestPessimisticStatCombiner.java
new file mode 100644
index 00000000000..98bc589e40d
--- /dev/null
+++ b/ql/src/test/org/apache/hadoop/hive/ql/stats/estimator/TestPessimisticStatCombiner.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.stats.estimator;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import org.apache.hadoop.hive.ql.plan.ColStatistics;
+import org.junit.jupiter.api.Test;
+
+class TestPessimisticStatCombiner {
+
+ @Test
+ void testCombinePropagatesUnknownNumNullsFromFirst() {
+ ColStatistics stat1 = createStat("col1", "int", 50, -1, 4.0); // unknown numNulls
+ ColStatistics stat2 = createStat("col2", "int", 30, 100, 4.0);
+
+ PessimisticStatCombiner combiner = new PessimisticStatCombiner();
+ combiner.add(stat1);
+ combiner.add(stat2);
+
+ ColStatistics combined = combiner.getResult().get();
+ assertEquals(-1, combined.getNumNulls(), "Unknown numNulls (-1) should be propagated");
+ }
+
+ @Test
+ void testCombinePropagatesUnknownNumNullsFromSecond() {
+ ColStatistics stat1 = createStat("col1", "int", 50, 100, 4.0);
+ ColStatistics stat2 = createStat("col2", "int", 30, -1, 4.0); // unknown numNulls
+
+ PessimisticStatCombiner combiner = new PessimisticStatCombiner();
+ combiner.add(stat1);
+ combiner.add(stat2);
+
+ ColStatistics combined = combiner.getResult().get();
+ assertEquals(-1, combined.getNumNulls(), "Unknown numNulls (-1) should be propagated");
+ }
+
+ @Test
+ void testCombinePropagatesUnknownNumTruesFromFirst() {
+ ColStatistics stat1 = createStat("col1", "boolean", 2, 5, 1.0);
+ stat1.setNumTrues(-1); // unknown
+ stat1.setNumFalses(50);
+
+ ColStatistics stat2 = createStat("col2", "boolean", 2, 10, 1.0);
+ stat2.setNumTrues(100);
+ stat2.setNumFalses(150);
+
+ PessimisticStatCombiner combiner = new PessimisticStatCombiner();
+ combiner.add(stat1);
+ combiner.add(stat2);
+
+ ColStatistics combined = combiner.getResult().get();
+ assertEquals(-1, combined.getNumTrues(), "Unknown numTrues (-1) should be propagated");
+ }
+
+ @Test
+ void testCombinePropagatesUnknownNumTruesFromSecond() {
+ ColStatistics stat1 = createStat("col1", "boolean", 2, 5, 1.0);
+ stat1.setNumTrues(100);
+ stat1.setNumFalses(50);
+
+ ColStatistics stat2 = createStat("col2", "boolean", 2, 10, 1.0);
+ stat2.setNumTrues(-1); // unknown
+ stat2.setNumFalses(150);
+
+ PessimisticStatCombiner combiner = new PessimisticStatCombiner();
+ combiner.add(stat1);
+ combiner.add(stat2);
+
+ ColStatistics combined = combiner.getResult().get();
+ assertEquals(-1, combined.getNumTrues(), "Unknown numTrues (-1) should be propagated");
+ }
+
+ @Test
+ void testCombinePropagatesUnknownNumFalsesFromFirst() {
+ ColStatistics stat1 = createStat("col1", "boolean", 2, 5, 1.0);
+ stat1.setNumTrues(100);
+ stat1.setNumFalses(-1); // unknown
+
+ ColStatistics stat2 = createStat("col2", "boolean", 2, 10, 1.0);
+ stat2.setNumTrues(50);
+ stat2.setNumFalses(150);
+
+ PessimisticStatCombiner combiner = new PessimisticStatCombiner();
+ combiner.add(stat1);
+ combiner.add(stat2);
+
+ ColStatistics combined = combiner.getResult().get();
+ assertEquals(-1, combined.getNumFalses(), "Unknown numFalses (-1) should be propagated");
+ }
+
+ @Test
+ void testCombinePropagatesUnknownNumFalsesFromSecond() {
+ ColStatistics stat1 = createStat("col1", "boolean", 2, 5, 1.0);
+ stat1.setNumTrues(100);
+ stat1.setNumFalses(50);
+
+ ColStatistics stat2 = createStat("col2", "boolean", 2, 10, 1.0);
+ stat2.setNumTrues(50);
+ stat2.setNumFalses(-1); // unknown
+
+ PessimisticStatCombiner combiner = new PessimisticStatCombiner();
+ combiner.add(stat1);
+ combiner.add(stat2);
+
+ ColStatistics combined = combiner.getResult().get();
+ assertEquals(-1, combined.getNumFalses(), "Unknown numFalses (-1) should be propagated");
+ }
+
+ @Test
+ void testCombineBothUnknownNumNulls() {
+ ColStatistics stat1 = createStat("col1", "int", 50, -1, 4.0);
+ ColStatistics stat2 = createStat("col2", "int", 30, -1, 4.0);
+
+ PessimisticStatCombiner combiner = new PessimisticStatCombiner();
+ combiner.add(stat1);
+ combiner.add(stat2);
+
+ ColStatistics combined = combiner.getResult().get();
+ assertEquals(-1, combined.getNumNulls(), "Both unknown should result in unknown (-1)");
+ }
+
+ @Test
+ void testCombineBothUnknownNumTruesAndNumFalses() {
+ ColStatistics stat1 = createStat("col1", "boolean", 2, 5, 1.0);
+ stat1.setNumTrues(-1);
+ stat1.setNumFalses(-1);
+
+ ColStatistics stat2 = createStat("col2", "boolean", 2, 10, 1.0);
+ stat2.setNumTrues(-1);
+ stat2.setNumFalses(-1);
+
+ PessimisticStatCombiner combiner = new PessimisticStatCombiner();
+ combiner.add(stat1);
+ combiner.add(stat2);
+
+ ColStatistics combined = combiner.getResult().get();
+ assertEquals(-1, combined.getNumTrues(), "Both unknown should result in unknown (-1)");
+ assertEquals(-1, combined.getNumFalses(), "Both unknown should result in unknown (-1)");
+ }
+
+ private ColStatistics createStat(String name, String type, long ndv, long numNulls, double avgColLen) {
+ ColStatistics stat = new ColStatistics(name, type);
+ stat.setCountDistint(ndv);
+ stat.setNumNulls(numNulls);
+ stat.setAvgColLen(avgColLen);
+ return stat;
+ }
+}
diff --git a/ql/src/test/queries/clientpositive/unknown_stats.q b/ql/src/test/queries/clientpositive/unknown_stats.q
new file mode 100644
index 00000000000..a062a3fb68b
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/unknown_stats.q
@@ -0,0 +1,62 @@
+CREATE TABLE t (flag BOOLEAN);
+ALTER TABLE t UPDATE STATISTICS SET('numRows'='1000');
+ALTER TABLE t UPDATE STATISTICS for column flag SET ('numNulls'='-1','numTrues'='-1','numFalses'='-1');
+
+EXPLAIN EXTENDED SELECT COUNT(*) FROM t WHERE flag;
+EXPLAIN EXTENDED SELECT COUNT(*) FROM t WHERE NOT flag;
+EXPLAIN EXTENDED SELECT COUNT(*) FROM t WHERE flag IS NULL;
+
+ALTER TABLE t UPDATE STATISTICS for column flag SET ('numNulls'='0','numTrues'='-1','numFalses'='-1');
+
+EXPLAIN EXTENDED SELECT COUNT(*) FROM t WHERE flag;
+EXPLAIN EXTENDED SELECT COUNT(*) FROM t WHERE NOT flag;
+
+ALTER TABLE t UPDATE STATISTICS for column flag SET ('numNulls'='100','numTrues'='-1','numFalses'='-1');
+
+EXPLAIN EXTENDED SELECT COUNT(*) FROM t WHERE flag;
+EXPLAIN EXTENDED SELECT COUNT(*) FROM t WHERE NOT flag;
+
+ALTER TABLE t UPDATE STATISTICS for column flag SET ('numNulls'='-1','numTrues'='500','numFalses'='-1');
+
+EXPLAIN EXTENDED SELECT COUNT(*) FROM t WHERE NOT flag;
+EXPLAIN EXTENDED SELECT COUNT(*) FROM t WHERE flag IS NULL;
+
+ALTER TABLE t UPDATE STATISTICS for column flag SET ('numNulls'='-1','numTrues'='-1','numFalses'='300');
+
+EXPLAIN EXTENDED SELECT COUNT(*) FROM t WHERE flag;
+EXPLAIN EXTENDED SELECT COUNT(*) FROM t WHERE flag IS NULL;
+
+-- Test boolean NDV calculation for GROUP BY
+-- Case 1: All NULL column (numTrues=0, numFalses=0) -> NDV should be 0
+ALTER TABLE t UPDATE STATISTICS for column flag SET ('numNulls'='1000','numTrues'='0','numFalses'='0');
+EXPLAIN EXTENDED SELECT flag, COUNT(*) FROM t GROUP BY flag;
+
+-- Case 2: Only TRUE values (numTrues>0, numFalses=0) -> NDV should be 1
+ALTER TABLE t UPDATE STATISTICS for column flag SET ('numNulls'='100','numTrues'='900','numFalses'='0');
+EXPLAIN EXTENDED SELECT flag, COUNT(*) FROM t GROUP BY flag;
+
+-- Case 3: Only FALSE values (numTrues=0, numFalses>0) -> NDV should be 1
+ALTER TABLE t UPDATE STATISTICS for column flag SET ('numNulls'='100','numTrues'='0','numFalses'='900');
+EXPLAIN EXTENDED SELECT flag, COUNT(*) FROM t GROUP BY flag;
+
+-- Case 4: No TRUE, unknown FALSE (numTrues=0, numFalses=-1) -> NDV should be 1
+ALTER TABLE t UPDATE STATISTICS for column flag SET ('numNulls'='100','numTrues'='0','numFalses'='-1');
+EXPLAIN EXTENDED SELECT flag, COUNT(*) FROM t GROUP BY flag;
+
+-- Case 5: Unknown TRUE, no FALSE (numTrues=-1, numFalses=0) -> NDV should be 1
+ALTER TABLE t UPDATE STATISTICS for column flag SET ('numNulls'='100','numTrues'='-1','numFalses'='0');
+EXPLAIN EXTENDED SELECT flag, COUNT(*) FROM t GROUP BY flag;
+
+-- Case 6: Both values present (numTrues>0, numFalses>0) -> NDV should be 2
+ALTER TABLE t UPDATE STATISTICS for column flag SET ('numNulls'='100','numTrues'='450','numFalses'='450');
+EXPLAIN EXTENDED SELECT flag, COUNT(*) FROM t GROUP BY flag;
+
+CREATE TABLE t3 (id INT);
+CREATE TABLE t4 (id INT, data INT);
+ALTER TABLE t3 UPDATE STATISTICS SET('numRows'='10000');
+ALTER TABLE t3 UPDATE STATISTICS for column id SET ('numDVs'='10000','numNulls'='0');
+ALTER TABLE t4 UPDATE STATISTICS SET('numRows'='5000');
+ALTER TABLE t4 UPDATE STATISTICS for column id SET ('numDVs'='5000','numNulls'='0');
+ALTER TABLE t4 UPDATE STATISTICS for column data SET ('numDVs'='100','numNulls'='-1');
+
+EXPLAIN EXTENDED SELECT * FROM (SELECT t3.id, t4.data FROM t3 LEFT OUTER JOIN t4 ON t3.id = t4.id) sub WHERE sub.data IS NULL;
diff --git a/ql/src/test/results/clientpositive/llap/unknown_stats.q.out b/ql/src/test/results/clientpositive/llap/unknown_stats.q.out
new file mode 100644
index 00000000000..0b876c139e4
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/unknown_stats.q.out
@@ -0,0 +1,2577 @@
+PREHOOK: query: CREATE TABLE t (flag BOOLEAN)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@t
+POSTHOOK: query: CREATE TABLE t (flag BOOLEAN)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@t
+PREHOOK: query: ALTER TABLE t UPDATE STATISTICS SET('numRows'='1000')
+PREHOOK: type: ALTERTABLE_UPDATETABLESTATS
+PREHOOK: Input: default@t
+PREHOOK: Output: default@t
+POSTHOOK: query: ALTER TABLE t UPDATE STATISTICS SET('numRows'='1000')
+POSTHOOK: type: ALTERTABLE_UPDATETABLESTATS
+POSTHOOK: Input: default@t
+POSTHOOK: Output: default@t
+PREHOOK: query: ALTER TABLE t UPDATE STATISTICS for column flag SET ('numNulls'='-1','numTrues'='-1','numFalses'='-1')
+PREHOOK: type: ALTERTABLE_UPDATETABLESTATS
+PREHOOK: Input: default@t
+PREHOOK: Output: default@t
+POSTHOOK: query: ALTER TABLE t UPDATE STATISTICS for column flag SET ('numNulls'='-1','numTrues'='-1','numFalses'='-1')
+POSTHOOK: type: ALTERTABLE_UPDATETABLESTATS
+POSTHOOK: Input: default@t
+POSTHOOK: Output: default@t
+PREHOOK: query: EXPLAIN EXTENDED SELECT COUNT(*) FROM t WHERE flag
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN EXTENDED SELECT COUNT(*) FROM t WHERE flag
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t
+#### A masked pattern was here ####
+OPTIMIZED SQL: SELECT COUNT(*) AS `_c0`
+FROM `default`.`t`
+WHERE `flag`
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: t
+ Statistics: Num rows: 1000 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE
+ GatherStats: false
+ Filter Operator
+ isSamplingPred: false
+ predicate: flag (type: boolean)
+ Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ aggregations: count()
+ minReductionHashAggr: 0.99
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ bucketingVersion: 2
+ null sort order:
+ numBuckets: -1
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ tag: -1
+ value expressions: _col0 (type: bigint)
+ auto parallelism: false
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Path -> Alias:
+#### A masked pattern was here ####
+ Path -> Partition:
+#### A masked pattern was here ####
+ Partition
+ base file name: t
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count -1
+ bucketing_version 2
+ column.name.delimiter ,
+ columns flag
+ columns.types boolean
+#### A masked pattern was here ####
+ name default.t
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucketing_version 2
+ column.name.delimiter ,
+ columns flag
+ columns.comments
+ columns.types boolean
+#### A masked pattern was here ####
+ name default.t
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.t
+ name: default.t
+ Truncated Path -> Alias:
+ /t [t]
+ Reducer 2
+ Execution mode: vectorized, llap
+ Needs Tagging: false
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ bucketingVersion: 2
+ compressed: false
+ GlobalTableId: 0
+#### A masked pattern was here ####
+ NumFilesPerFileSink: 1
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+#### A masked pattern was here ####
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ properties:
+ bucketing_version -1
+ columns _col0
+ columns.types bigint
+ escape.delim \
+ hive.serialization.extend.additional.nesting.levels true
+ serialization.escape.crlf true
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ TotalFiles: 1
+ GatherStats: false
+ MultiFileSpray: false
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: EXPLAIN EXTENDED SELECT COUNT(*) FROM t WHERE NOT flag
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN EXTENDED SELECT COUNT(*) FROM t WHERE NOT flag
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t
+#### A masked pattern was here ####
+OPTIMIZED SQL: SELECT COUNT(*) AS `_c0`
+FROM `default`.`t`
+WHERE NOT `flag`
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: t
+ filterExpr: (not flag) (type: boolean)
+ Statistics: Num rows: 1000 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE
+ GatherStats: false
+ Filter Operator
+ isSamplingPred: false
+ predicate: (not flag) (type: boolean)
+ Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ aggregations: count()
+ minReductionHashAggr: 0.99
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ bucketingVersion: 2
+ null sort order:
+ numBuckets: -1
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ tag: -1
+ value expressions: _col0 (type: bigint)
+ auto parallelism: false
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Path -> Alias:
+#### A masked pattern was here ####
+ Path -> Partition:
+#### A masked pattern was here ####
+ Partition
+ base file name: t
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count -1
+ bucketing_version 2
+ column.name.delimiter ,
+ columns flag
+ columns.types boolean
+#### A masked pattern was here ####
+ name default.t
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucketing_version 2
+ column.name.delimiter ,
+ columns flag
+ columns.comments
+ columns.types boolean
+#### A masked pattern was here ####
+ name default.t
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.t
+ name: default.t
+ Truncated Path -> Alias:
+ /t [t]
+ Reducer 2
+ Execution mode: vectorized, llap
+ Needs Tagging: false
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ bucketingVersion: 2
+ compressed: false
+ GlobalTableId: 0
+#### A masked pattern was here ####
+ NumFilesPerFileSink: 1
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+#### A masked pattern was here ####
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ properties:
+ bucketing_version -1
+ columns _col0
+ columns.types bigint
+ escape.delim \
+ hive.serialization.extend.additional.nesting.levels true
+ serialization.escape.crlf true
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ TotalFiles: 1
+ GatherStats: false
+ MultiFileSpray: false
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: EXPLAIN EXTENDED SELECT COUNT(*) FROM t WHERE flag IS NULL
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN EXTENDED SELECT COUNT(*) FROM t WHERE flag IS NULL
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t
+#### A masked pattern was here ####
+OPTIMIZED SQL: SELECT COUNT(*) AS `_c0`
+FROM `default`.`t`
+WHERE `flag` IS NULL
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: t
+ filterExpr: flag is null (type: boolean)
+ Statistics: Num rows: 1000 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE
+ GatherStats: false
+ Filter Operator
+ isSamplingPred: false
+ predicate: flag is null (type: boolean)
+ Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ aggregations: count()
+ minReductionHashAggr: 0.99
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ bucketingVersion: 2
+ null sort order:
+ numBuckets: -1
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ tag: -1
+ value expressions: _col0 (type: bigint)
+ auto parallelism: false
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Path -> Alias:
+#### A masked pattern was here ####
+ Path -> Partition:
+#### A masked pattern was here ####
+ Partition
+ base file name: t
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count -1
+ bucketing_version 2
+ column.name.delimiter ,
+ columns flag
+ columns.types boolean
+#### A masked pattern was here ####
+ name default.t
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucketing_version 2
+ column.name.delimiter ,
+ columns flag
+ columns.comments
+ columns.types boolean
+#### A masked pattern was here ####
+ name default.t
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.t
+ name: default.t
+ Truncated Path -> Alias:
+ /t [t]
+ Reducer 2
+ Execution mode: vectorized, llap
+ Needs Tagging: false
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ bucketingVersion: 2
+ compressed: false
+ GlobalTableId: 0
+#### A masked pattern was here ####
+ NumFilesPerFileSink: 1
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+#### A masked pattern was here ####
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ properties:
+ bucketing_version -1
+ columns _col0
+ columns.types bigint
+ escape.delim \
+ hive.serialization.extend.additional.nesting.levels true
+ serialization.escape.crlf true
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ TotalFiles: 1
+ GatherStats: false
+ MultiFileSpray: false
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: ALTER TABLE t UPDATE STATISTICS for column flag SET ('numNulls'='0','numTrues'='-1','numFalses'='-1')
+PREHOOK: type: ALTERTABLE_UPDATETABLESTATS
+PREHOOK: Input: default@t
+PREHOOK: Output: default@t
+POSTHOOK: query: ALTER TABLE t UPDATE STATISTICS for column flag SET ('numNulls'='0','numTrues'='-1','numFalses'='-1')
+POSTHOOK: type: ALTERTABLE_UPDATETABLESTATS
+POSTHOOK: Input: default@t
+POSTHOOK: Output: default@t
+PREHOOK: query: EXPLAIN EXTENDED SELECT COUNT(*) FROM t WHERE flag
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN EXTENDED SELECT COUNT(*) FROM t WHERE flag
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t
+#### A masked pattern was here ####
+OPTIMIZED SQL: SELECT COUNT(*) AS `_c0`
+FROM `default`.`t`
+WHERE `flag`
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: t
+ Statistics: Num rows: 1000 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE
+ GatherStats: false
+ Filter Operator
+ isSamplingPred: false
+ predicate: flag (type: boolean)
+ Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ aggregations: count()
+ minReductionHashAggr: 0.99
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ bucketingVersion: 2
+ null sort order:
+ numBuckets: -1
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ tag: -1
+ value expressions: _col0 (type: bigint)
+ auto parallelism: false
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Path -> Alias:
+#### A masked pattern was here ####
+ Path -> Partition:
+#### A masked pattern was here ####
+ Partition
+ base file name: t
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count -1
+ bucketing_version 2
+ column.name.delimiter ,
+ columns flag
+ columns.types boolean
+#### A masked pattern was here ####
+ name default.t
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucketing_version 2
+ column.name.delimiter ,
+ columns flag
+ columns.comments
+ columns.types boolean
+#### A masked pattern was here ####
+ name default.t
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.t
+ name: default.t
+ Truncated Path -> Alias:
+ /t [t]
+ Reducer 2
+ Execution mode: vectorized, llap
+ Needs Tagging: false
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ bucketingVersion: 2
+ compressed: false
+ GlobalTableId: 0
+#### A masked pattern was here ####
+ NumFilesPerFileSink: 1
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+#### A masked pattern was here ####
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ properties:
+ bucketing_version -1
+ columns _col0
+ columns.types bigint
+ escape.delim \
+ hive.serialization.extend.additional.nesting.levels true
+ serialization.escape.crlf true
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ TotalFiles: 1
+ GatherStats: false
+ MultiFileSpray: false
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: EXPLAIN EXTENDED SELECT COUNT(*) FROM t WHERE NOT flag
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN EXTENDED SELECT COUNT(*) FROM t WHERE NOT flag
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t
+#### A masked pattern was here ####
+OPTIMIZED SQL: SELECT COUNT(*) AS `_c0`
+FROM `default`.`t`
+WHERE NOT `flag`
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: t
+ filterExpr: (not flag) (type: boolean)
+ Statistics: Num rows: 1000 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE
+ GatherStats: false
+ Filter Operator
+ isSamplingPred: false
+ predicate: (not flag) (type: boolean)
+ Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ aggregations: count()
+ minReductionHashAggr: 0.99
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ bucketingVersion: 2
+ null sort order:
+ numBuckets: -1
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ tag: -1
+ value expressions: _col0 (type: bigint)
+ auto parallelism: false
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Path -> Alias:
+#### A masked pattern was here ####
+ Path -> Partition:
+#### A masked pattern was here ####
+ Partition
+ base file name: t
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count -1
+ bucketing_version 2
+ column.name.delimiter ,
+ columns flag
+ columns.types boolean
+#### A masked pattern was here ####
+ name default.t
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucketing_version 2
+ column.name.delimiter ,
+ columns flag
+ columns.comments
+ columns.types boolean
+#### A masked pattern was here ####
+ name default.t
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.t
+ name: default.t
+ Truncated Path -> Alias:
+ /t [t]
+ Reducer 2
+ Execution mode: vectorized, llap
+ Needs Tagging: false
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ bucketingVersion: 2
+ compressed: false
+ GlobalTableId: 0
+#### A masked pattern was here ####
+ NumFilesPerFileSink: 1
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+#### A masked pattern was here ####
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ properties:
+ bucketing_version -1
+ columns _col0
+ columns.types bigint
+ escape.delim \
+ hive.serialization.extend.additional.nesting.levels true
+ serialization.escape.crlf true
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ TotalFiles: 1
+ GatherStats: false
+ MultiFileSpray: false
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: ALTER TABLE t UPDATE STATISTICS for column flag SET ('numNulls'='100','numTrues'='-1','numFalses'='-1')
+PREHOOK: type: ALTERTABLE_UPDATETABLESTATS
+PREHOOK: Input: default@t
+PREHOOK: Output: default@t
+POSTHOOK: query: ALTER TABLE t UPDATE STATISTICS for column flag SET ('numNulls'='100','numTrues'='-1','numFalses'='-1')
+POSTHOOK: type: ALTERTABLE_UPDATETABLESTATS
+POSTHOOK: Input: default@t
+POSTHOOK: Output: default@t
+PREHOOK: query: EXPLAIN EXTENDED SELECT COUNT(*) FROM t WHERE flag
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN EXTENDED SELECT COUNT(*) FROM t WHERE flag
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t
+#### A masked pattern was here ####
+OPTIMIZED SQL: SELECT COUNT(*) AS `_c0`
+FROM `default`.`t`
+WHERE `flag`
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: t
+ Statistics: Num rows: 1000 Data size: 3604 Basic stats: COMPLETE Column stats: COMPLETE
+ GatherStats: false
+ Filter Operator
+ isSamplingPred: false
+ predicate: flag (type: boolean)
+ Statistics: Num rows: 500 Data size: 1804 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ Statistics: Num rows: 500 Data size: 1804 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ aggregations: count()
+ minReductionHashAggr: 0.99
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ bucketingVersion: 2
+ null sort order:
+ numBuckets: -1
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ tag: -1
+ value expressions: _col0 (type: bigint)
+ auto parallelism: false
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Path -> Alias:
+#### A masked pattern was here ####
+ Path -> Partition:
+#### A masked pattern was here ####
+ Partition
+ base file name: t
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count -1
+ bucketing_version 2
+ column.name.delimiter ,
+ columns flag
+ columns.types boolean
+#### A masked pattern was here ####
+ name default.t
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucketing_version 2
+ column.name.delimiter ,
+ columns flag
+ columns.comments
+ columns.types boolean
+#### A masked pattern was here ####
+ name default.t
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.t
+ name: default.t
+ Truncated Path -> Alias:
+ /t [t]
+ Reducer 2
+ Execution mode: vectorized, llap
+ Needs Tagging: false
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ bucketingVersion: 2
+ compressed: false
+ GlobalTableId: 0
+#### A masked pattern was here ####
+ NumFilesPerFileSink: 1
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+#### A masked pattern was here ####
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ properties:
+ bucketing_version -1
+ columns _col0
+ columns.types bigint
+ escape.delim \
+ hive.serialization.extend.additional.nesting.levels true
+ serialization.escape.crlf true
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ TotalFiles: 1
+ GatherStats: false
+ MultiFileSpray: false
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: EXPLAIN EXTENDED SELECT COUNT(*) FROM t WHERE NOT flag
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN EXTENDED SELECT COUNT(*) FROM t WHERE NOT flag
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t
+#### A masked pattern was here ####
+OPTIMIZED SQL: SELECT COUNT(*) AS `_c0`
+FROM `default`.`t`
+WHERE NOT `flag`
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: t
+ filterExpr: (not flag) (type: boolean)
+ Statistics: Num rows: 1000 Data size: 3604 Basic stats: COMPLETE Column stats: COMPLETE
+ GatherStats: false
+ Filter Operator
+ isSamplingPred: false
+ predicate: (not flag) (type: boolean)
+ Statistics: Num rows: 500 Data size: 1804 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ Statistics: Num rows: 500 Data size: 1804 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ aggregations: count()
+ minReductionHashAggr: 0.99
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ bucketingVersion: 2
+ null sort order:
+ numBuckets: -1
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ tag: -1
+ value expressions: _col0 (type: bigint)
+ auto parallelism: false
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Path -> Alias:
+#### A masked pattern was here ####
+ Path -> Partition:
+#### A masked pattern was here ####
+ Partition
+ base file name: t
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count -1
+ bucketing_version 2
+ column.name.delimiter ,
+ columns flag
+ columns.types boolean
+#### A masked pattern was here ####
+ name default.t
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucketing_version 2
+ column.name.delimiter ,
+ columns flag
+ columns.comments
+ columns.types boolean
+#### A masked pattern was here ####
+ name default.t
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.t
+ name: default.t
+ Truncated Path -> Alias:
+ /t [t]
+ Reducer 2
+ Execution mode: vectorized, llap
+ Needs Tagging: false
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ bucketingVersion: 2
+ compressed: false
+ GlobalTableId: 0
+#### A masked pattern was here ####
+ NumFilesPerFileSink: 1
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+#### A masked pattern was here ####
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ properties:
+ bucketing_version -1
+ columns _col0
+ columns.types bigint
+ escape.delim \
+ hive.serialization.extend.additional.nesting.levels true
+ serialization.escape.crlf true
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ TotalFiles: 1
+ GatherStats: false
+ MultiFileSpray: false
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: ALTER TABLE t UPDATE STATISTICS for column flag SET ('numNulls'='-1','numTrues'='500','numFalses'='-1')
+PREHOOK: type: ALTERTABLE_UPDATETABLESTATS
+PREHOOK: Input: default@t
+PREHOOK: Output: default@t
+POSTHOOK: query: ALTER TABLE t UPDATE STATISTICS for column flag SET ('numNulls'='-1','numTrues'='500','numFalses'='-1')
+POSTHOOK: type: ALTERTABLE_UPDATETABLESTATS
+POSTHOOK: Input: default@t
+POSTHOOK: Output: default@t
+PREHOOK: query: EXPLAIN EXTENDED SELECT COUNT(*) FROM t WHERE NOT flag
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN EXTENDED SELECT COUNT(*) FROM t WHERE NOT flag
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t
+#### A masked pattern was here ####
+OPTIMIZED SQL: SELECT COUNT(*) AS `_c0`
+FROM `default`.`t`
+WHERE NOT `flag`
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: t
+ filterExpr: (not flag) (type: boolean)
+ Statistics: Num rows: 1000 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE
+ GatherStats: false
+ Filter Operator
+ isSamplingPred: false
+ predicate: (not flag) (type: boolean)
+ Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ aggregations: count()
+ minReductionHashAggr: 0.99
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ bucketingVersion: 2
+ null sort order:
+ numBuckets: -1
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ tag: -1
+ value expressions: _col0 (type: bigint)
+ auto parallelism: false
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Path -> Alias:
+#### A masked pattern was here ####
+ Path -> Partition:
+#### A masked pattern was here ####
+ Partition
+ base file name: t
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count -1
+ bucketing_version 2
+ column.name.delimiter ,
+ columns flag
+ columns.types boolean
+#### A masked pattern was here ####
+ name default.t
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucketing_version 2
+ column.name.delimiter ,
+ columns flag
+ columns.comments
+ columns.types boolean
+#### A masked pattern was here ####
+ name default.t
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.t
+ name: default.t
+ Truncated Path -> Alias:
+ /t [t]
+ Reducer 2
+ Execution mode: vectorized, llap
+ Needs Tagging: false
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ bucketingVersion: 2
+ compressed: false
+ GlobalTableId: 0
+#### A masked pattern was here ####
+ NumFilesPerFileSink: 1
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+#### A masked pattern was here ####
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ properties:
+ bucketing_version -1
+ columns _col0
+ columns.types bigint
+ escape.delim \
+ hive.serialization.extend.additional.nesting.levels true
+ serialization.escape.crlf true
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ TotalFiles: 1
+ GatherStats: false
+ MultiFileSpray: false
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: EXPLAIN EXTENDED SELECT COUNT(*) FROM t WHERE flag IS NULL
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN EXTENDED SELECT COUNT(*) FROM t WHERE flag IS NULL
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t
+#### A masked pattern was here ####
+OPTIMIZED SQL: SELECT COUNT(*) AS `_c0`
+FROM `default`.`t`
+WHERE `flag` IS NULL
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: t
+ filterExpr: flag is null (type: boolean)
+ Statistics: Num rows: 1000 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE
+ GatherStats: false
+ Filter Operator
+ isSamplingPred: false
+ predicate: flag is null (type: boolean)
+ Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ aggregations: count()
+ minReductionHashAggr: 0.99
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ bucketingVersion: 2
+ null sort order:
+ numBuckets: -1
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ tag: -1
+ value expressions: _col0 (type: bigint)
+ auto parallelism: false
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Path -> Alias:
+#### A masked pattern was here ####
+ Path -> Partition:
+#### A masked pattern was here ####
+ Partition
+ base file name: t
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count -1
+ bucketing_version 2
+ column.name.delimiter ,
+ columns flag
+ columns.types boolean
+#### A masked pattern was here ####
+ name default.t
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucketing_version 2
+ column.name.delimiter ,
+ columns flag
+ columns.comments
+ columns.types boolean
+#### A masked pattern was here ####
+ name default.t
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.t
+ name: default.t
+ Truncated Path -> Alias:
+ /t [t]
+ Reducer 2
+ Execution mode: vectorized, llap
+ Needs Tagging: false
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ bucketingVersion: 2
+ compressed: false
+ GlobalTableId: 0
+#### A masked pattern was here ####
+ NumFilesPerFileSink: 1
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+#### A masked pattern was here ####
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ properties:
+ bucketing_version -1
+ columns _col0
+ columns.types bigint
+ escape.delim \
+ hive.serialization.extend.additional.nesting.levels true
+ serialization.escape.crlf true
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ TotalFiles: 1
+ GatherStats: false
+ MultiFileSpray: false
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: ALTER TABLE t UPDATE STATISTICS for column flag SET ('numNulls'='-1','numTrues'='-1','numFalses'='300')
+PREHOOK: type: ALTERTABLE_UPDATETABLESTATS
+PREHOOK: Input: default@t
+PREHOOK: Output: default@t
+POSTHOOK: query: ALTER TABLE t UPDATE STATISTICS for column flag SET ('numNulls'='-1','numTrues'='-1','numFalses'='300')
+POSTHOOK: type: ALTERTABLE_UPDATETABLESTATS
+POSTHOOK: Input: default@t
+POSTHOOK: Output: default@t
+PREHOOK: query: EXPLAIN EXTENDED SELECT COUNT(*) FROM t WHERE flag
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN EXTENDED SELECT COUNT(*) FROM t WHERE flag
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t
+#### A masked pattern was here ####
+OPTIMIZED SQL: SELECT COUNT(*) AS `_c0`
+FROM `default`.`t`
+WHERE `flag`
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: t
+ Statistics: Num rows: 1000 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE
+ GatherStats: false
+ Filter Operator
+ isSamplingPred: false
+ predicate: flag (type: boolean)
+ Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ aggregations: count()
+ minReductionHashAggr: 0.99
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ bucketingVersion: 2
+ null sort order:
+ numBuckets: -1
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ tag: -1
+ value expressions: _col0 (type: bigint)
+ auto parallelism: false
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Path -> Alias:
+#### A masked pattern was here ####
+ Path -> Partition:
+#### A masked pattern was here ####
+ Partition
+ base file name: t
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count -1
+ bucketing_version 2
+ column.name.delimiter ,
+ columns flag
+ columns.types boolean
+#### A masked pattern was here ####
+ name default.t
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucketing_version 2
+ column.name.delimiter ,
+ columns flag
+ columns.comments
+ columns.types boolean
+#### A masked pattern was here ####
+ name default.t
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.t
+ name: default.t
+ Truncated Path -> Alias:
+ /t [t]
+ Reducer 2
+ Execution mode: vectorized, llap
+ Needs Tagging: false
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ bucketingVersion: 2
+ compressed: false
+ GlobalTableId: 0
+#### A masked pattern was here ####
+ NumFilesPerFileSink: 1
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+#### A masked pattern was here ####
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ properties:
+ bucketing_version -1
+ columns _col0
+ columns.types bigint
+ escape.delim \
+ hive.serialization.extend.additional.nesting.levels true
+ serialization.escape.crlf true
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ TotalFiles: 1
+ GatherStats: false
+ MultiFileSpray: false
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: EXPLAIN EXTENDED SELECT COUNT(*) FROM t WHERE flag IS NULL
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN EXTENDED SELECT COUNT(*) FROM t WHERE flag IS NULL
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t
+#### A masked pattern was here ####
+OPTIMIZED SQL: SELECT COUNT(*) AS `_c0`
+FROM `default`.`t`
+WHERE `flag` IS NULL
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: t
+ filterExpr: flag is null (type: boolean)
+ Statistics: Num rows: 1000 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE
+ GatherStats: false
+ Filter Operator
+ isSamplingPred: false
+ predicate: flag is null (type: boolean)
+ Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ Statistics: Num rows: 500 Data size: 2000 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ aggregations: count()
+ minReductionHashAggr: 0.99
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ bucketingVersion: 2
+ null sort order:
+ numBuckets: -1
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ tag: -1
+ value expressions: _col0 (type: bigint)
+ auto parallelism: false
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Path -> Alias:
+#### A masked pattern was here ####
+ Path -> Partition:
+#### A masked pattern was here ####
+ Partition
+ base file name: t
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count -1
+ bucketing_version 2
+ column.name.delimiter ,
+ columns flag
+ columns.types boolean
+#### A masked pattern was here ####
+ name default.t
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucketing_version 2
+ column.name.delimiter ,
+ columns flag
+ columns.comments
+ columns.types boolean
+#### A masked pattern was here ####
+ name default.t
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.t
+ name: default.t
+ Truncated Path -> Alias:
+ /t [t]
+ Reducer 2
+ Execution mode: vectorized, llap
+ Needs Tagging: false
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ bucketingVersion: 2
+ compressed: false
+ GlobalTableId: 0
+#### A masked pattern was here ####
+ NumFilesPerFileSink: 1
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+#### A masked pattern was here ####
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ properties:
+ bucketing_version -1
+ columns _col0
+ columns.types bigint
+ escape.delim \
+ hive.serialization.extend.additional.nesting.levels true
+ serialization.escape.crlf true
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ TotalFiles: 1
+ GatherStats: false
+ MultiFileSpray: false
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: ALTER TABLE t UPDATE STATISTICS for column flag SET ('numNulls'='1000','numTrues'='0','numFalses'='0')
+PREHOOK: type: ALTERTABLE_UPDATETABLESTATS
+PREHOOK: Input: default@t
+PREHOOK: Output: default@t
+POSTHOOK: query: ALTER TABLE t UPDATE STATISTICS for column flag SET ('numNulls'='1000','numTrues'='0','numFalses'='0')
+POSTHOOK: type: ALTERTABLE_UPDATETABLESTATS
+POSTHOOK: Input: default@t
+POSTHOOK: Output: default@t
+PREHOOK: query: EXPLAIN EXTENDED SELECT flag, COUNT(*) FROM t GROUP BY flag
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN EXTENDED SELECT flag, COUNT(*) FROM t GROUP BY flag
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t
+#### A masked pattern was here ####
+OPTIMIZED SQL: SELECT `flag`, COUNT(*) AS `_c1`
+FROM `default`.`t`
+GROUP BY `flag`
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: t
+ Statistics: Num rows: 1000 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
+ GatherStats: false
+ Select Operator
+ expressions: flag (type: boolean)
+ outputColumnNames: flag
+ Statistics: Num rows: 1000 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ aggregations: count()
+ keys: flag (type: boolean)
+ minReductionHashAggr: 0.99
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ bucketingVersion: 2
+ key expressions: _col0 (type: boolean)
+ null sort order: z
+ numBuckets: -1
+ sort order: +
+ Map-reduce partition columns: _col0 (type: boolean)
+ Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE
+ tag: -1
+ value expressions: _col1 (type: bigint)
+ auto parallelism: true
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Path -> Alias:
+#### A masked pattern was here ####
+ Path -> Partition:
+#### A masked pattern was here ####
+ Partition
+ base file name: t
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count -1
+ bucketing_version 2
+ column.name.delimiter ,
+ columns flag
+ columns.types boolean
+#### A masked pattern was here ####
+ name default.t
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucketing_version 2
+ column.name.delimiter ,
+ columns flag
+ columns.comments
+ columns.types boolean
+#### A masked pattern was here ####
+ name default.t
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.t
+ name: default.t
+ Truncated Path -> Alias:
+ /t [t]
+ Reducer 2
+ Execution mode: vectorized, llap
+ Needs Tagging: false
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ keys: KEY._col0 (type: boolean)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ bucketingVersion: 2
+ compressed: false
+ GlobalTableId: 0
+#### A masked pattern was here ####
+ NumFilesPerFileSink: 1
+ Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE
+#### A masked pattern was here ####
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ properties:
+ bucketing_version -1
+ columns _col0,_col1
+ columns.types boolean:bigint
+ escape.delim \
+ hive.serialization.extend.additional.nesting.levels true
+ serialization.escape.crlf true
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ TotalFiles: 1
+ GatherStats: false
+ MultiFileSpray: false
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: ALTER TABLE t UPDATE STATISTICS for column flag SET ('numNulls'='100','numTrues'='900','numFalses'='0')
+PREHOOK: type: ALTERTABLE_UPDATETABLESTATS
+PREHOOK: Input: default@t
+PREHOOK: Output: default@t
+POSTHOOK: query: ALTER TABLE t UPDATE STATISTICS for column flag SET ('numNulls'='100','numTrues'='900','numFalses'='0')
+POSTHOOK: type: ALTERTABLE_UPDATETABLESTATS
+POSTHOOK: Input: default@t
+POSTHOOK: Output: default@t
+PREHOOK: query: EXPLAIN EXTENDED SELECT flag, COUNT(*) FROM t GROUP BY flag
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN EXTENDED SELECT flag, COUNT(*) FROM t GROUP BY flag
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t
+#### A masked pattern was here ####
+OPTIMIZED SQL: SELECT `flag`, COUNT(*) AS `_c1`
+FROM `default`.`t`
+GROUP BY `flag`
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: t
+ Statistics: Num rows: 1000 Data size: 3604 Basic stats: COMPLETE Column stats: COMPLETE
+ GatherStats: false
+ Select Operator
+ expressions: flag (type: boolean)
+ outputColumnNames: flag
+ Statistics: Num rows: 1000 Data size: 3604 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ aggregations: count()
+ keys: flag (type: boolean)
+ minReductionHashAggr: 0.99
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ bucketingVersion: 2
+ key expressions: _col0 (type: boolean)
+ null sort order: z
+ numBuckets: -1
+ sort order: +
+ Map-reduce partition columns: _col0 (type: boolean)
+ Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE
+ tag: -1
+ value expressions: _col1 (type: bigint)
+ auto parallelism: true
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Path -> Alias:
+#### A masked pattern was here ####
+ Path -> Partition:
+#### A masked pattern was here ####
+ Partition
+ base file name: t
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count -1
+ bucketing_version 2
+ column.name.delimiter ,
+ columns flag
+ columns.types boolean
+#### A masked pattern was here ####
+ name default.t
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucketing_version 2
+ column.name.delimiter ,
+ columns flag
+ columns.comments
+ columns.types boolean
+#### A masked pattern was here ####
+ name default.t
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.t
+ name: default.t
+ Truncated Path -> Alias:
+ /t [t]
+ Reducer 2
+ Execution mode: vectorized, llap
+ Needs Tagging: false
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ keys: KEY._col0 (type: boolean)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ bucketingVersion: 2
+ compressed: false
+ GlobalTableId: 0
+#### A masked pattern was here ####
+ NumFilesPerFileSink: 1
+ Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE
+#### A masked pattern was here ####
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ properties:
+ bucketing_version -1
+ columns _col0,_col1
+ columns.types boolean:bigint
+ escape.delim \
+ hive.serialization.extend.additional.nesting.levels true
+ serialization.escape.crlf true
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ TotalFiles: 1
+ GatherStats: false
+ MultiFileSpray: false
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: ALTER TABLE t UPDATE STATISTICS for column flag SET ('numNulls'='100','numTrues'='0','numFalses'='900')
+PREHOOK: type: ALTERTABLE_UPDATETABLESTATS
+PREHOOK: Input: default@t
+PREHOOK: Output: default@t
+POSTHOOK: query: ALTER TABLE t UPDATE STATISTICS for column flag SET ('numNulls'='100','numTrues'='0','numFalses'='900')
+POSTHOOK: type: ALTERTABLE_UPDATETABLESTATS
+POSTHOOK: Input: default@t
+POSTHOOK: Output: default@t
+PREHOOK: query: EXPLAIN EXTENDED SELECT flag, COUNT(*) FROM t GROUP BY flag
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN EXTENDED SELECT flag, COUNT(*) FROM t GROUP BY flag
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t
+#### A masked pattern was here ####
+OPTIMIZED SQL: SELECT `flag`, COUNT(*) AS `_c1`
+FROM `default`.`t`
+GROUP BY `flag`
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: t
+ Statistics: Num rows: 1000 Data size: 3604 Basic stats: COMPLETE Column stats: COMPLETE
+ GatherStats: false
+ Select Operator
+ expressions: flag (type: boolean)
+ outputColumnNames: flag
+ Statistics: Num rows: 1000 Data size: 3604 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ aggregations: count()
+ keys: flag (type: boolean)
+ minReductionHashAggr: 0.99
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ bucketingVersion: 2
+ key expressions: _col0 (type: boolean)
+ null sort order: z
+ numBuckets: -1
+ sort order: +
+ Map-reduce partition columns: _col0 (type: boolean)
+ Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE
+ tag: -1
+ value expressions: _col1 (type: bigint)
+ auto parallelism: true
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Path -> Alias:
+#### A masked pattern was here ####
+ Path -> Partition:
+#### A masked pattern was here ####
+ Partition
+ base file name: t
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count -1
+ bucketing_version 2
+ column.name.delimiter ,
+ columns flag
+ columns.types boolean
+#### A masked pattern was here ####
+ name default.t
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucketing_version 2
+ column.name.delimiter ,
+ columns flag
+ columns.comments
+ columns.types boolean
+#### A masked pattern was here ####
+ name default.t
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.t
+ name: default.t
+ Truncated Path -> Alias:
+ /t [t]
+ Reducer 2
+ Execution mode: vectorized, llap
+ Needs Tagging: false
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ keys: KEY._col0 (type: boolean)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ bucketingVersion: 2
+ compressed: false
+ GlobalTableId: 0
+#### A masked pattern was here ####
+ NumFilesPerFileSink: 1
+ Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE
+#### A masked pattern was here ####
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ properties:
+ bucketing_version -1
+ columns _col0,_col1
+ columns.types boolean:bigint
+ escape.delim \
+ hive.serialization.extend.additional.nesting.levels true
+ serialization.escape.crlf true
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ TotalFiles: 1
+ GatherStats: false
+ MultiFileSpray: false
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: ALTER TABLE t UPDATE STATISTICS for column flag SET ('numNulls'='100','numTrues'='0','numFalses'='-1')
+PREHOOK: type: ALTERTABLE_UPDATETABLESTATS
+PREHOOK: Input: default@t
+PREHOOK: Output: default@t
+POSTHOOK: query: ALTER TABLE t UPDATE STATISTICS for column flag SET ('numNulls'='100','numTrues'='0','numFalses'='-1')
+POSTHOOK: type: ALTERTABLE_UPDATETABLESTATS
+POSTHOOK: Input: default@t
+POSTHOOK: Output: default@t
+PREHOOK: query: EXPLAIN EXTENDED SELECT flag, COUNT(*) FROM t GROUP BY flag
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN EXTENDED SELECT flag, COUNT(*) FROM t GROUP BY flag
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t
+#### A masked pattern was here ####
+OPTIMIZED SQL: SELECT `flag`, COUNT(*) AS `_c1`
+FROM `default`.`t`
+GROUP BY `flag`
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: t
+ Statistics: Num rows: 1000 Data size: 3604 Basic stats: COMPLETE Column stats: COMPLETE
+ GatherStats: false
+ Select Operator
+ expressions: flag (type: boolean)
+ outputColumnNames: flag
+ Statistics: Num rows: 1000 Data size: 3604 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ aggregations: count()
+ keys: flag (type: boolean)
+ minReductionHashAggr: 0.99
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ bucketingVersion: 2
+ key expressions: _col0 (type: boolean)
+ null sort order: z
+ numBuckets: -1
+ sort order: +
+ Map-reduce partition columns: _col0 (type: boolean)
+ Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE
+ tag: -1
+ value expressions: _col1 (type: bigint)
+ auto parallelism: true
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Path -> Alias:
+#### A masked pattern was here ####
+ Path -> Partition:
+#### A masked pattern was here ####
+ Partition
+ base file name: t
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count -1
+ bucketing_version 2
+ column.name.delimiter ,
+ columns flag
+ columns.types boolean
+#### A masked pattern was here ####
+ name default.t
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucketing_version 2
+ column.name.delimiter ,
+ columns flag
+ columns.comments
+ columns.types boolean
+#### A masked pattern was here ####
+ name default.t
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.t
+ name: default.t
+ Truncated Path -> Alias:
+ /t [t]
+ Reducer 2
+ Execution mode: vectorized, llap
+ Needs Tagging: false
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ keys: KEY._col0 (type: boolean)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ bucketingVersion: 2
+ compressed: false
+ GlobalTableId: 0
+#### A masked pattern was here ####
+ NumFilesPerFileSink: 1
+ Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE
+#### A masked pattern was here ####
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ properties:
+ bucketing_version -1
+ columns _col0,_col1
+ columns.types boolean:bigint
+ escape.delim \
+ hive.serialization.extend.additional.nesting.levels true
+ serialization.escape.crlf true
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ TotalFiles: 1
+ GatherStats: false
+ MultiFileSpray: false
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: ALTER TABLE t UPDATE STATISTICS for column flag SET ('numNulls'='100','numTrues'='-1','numFalses'='0')
+PREHOOK: type: ALTERTABLE_UPDATETABLESTATS
+PREHOOK: Input: default@t
+PREHOOK: Output: default@t
+POSTHOOK: query: ALTER TABLE t UPDATE STATISTICS for column flag SET ('numNulls'='100','numTrues'='-1','numFalses'='0')
+POSTHOOK: type: ALTERTABLE_UPDATETABLESTATS
+POSTHOOK: Input: default@t
+POSTHOOK: Output: default@t
+PREHOOK: query: EXPLAIN EXTENDED SELECT flag, COUNT(*) FROM t GROUP BY flag
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN EXTENDED SELECT flag, COUNT(*) FROM t GROUP BY flag
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t
+#### A masked pattern was here ####
+OPTIMIZED SQL: SELECT `flag`, COUNT(*) AS `_c1`
+FROM `default`.`t`
+GROUP BY `flag`
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: t
+ Statistics: Num rows: 1000 Data size: 3604 Basic stats: COMPLETE Column stats: COMPLETE
+ GatherStats: false
+ Select Operator
+ expressions: flag (type: boolean)
+ outputColumnNames: flag
+ Statistics: Num rows: 1000 Data size: 3604 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ aggregations: count()
+ keys: flag (type: boolean)
+ minReductionHashAggr: 0.99
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ bucketingVersion: 2
+ key expressions: _col0 (type: boolean)
+ null sort order: z
+ numBuckets: -1
+ sort order: +
+ Map-reduce partition columns: _col0 (type: boolean)
+ Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE
+ tag: -1
+ value expressions: _col1 (type: bigint)
+ auto parallelism: true
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Path -> Alias:
+#### A masked pattern was here ####
+ Path -> Partition:
+#### A masked pattern was here ####
+ Partition
+ base file name: t
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count -1
+ bucketing_version 2
+ column.name.delimiter ,
+ columns flag
+ columns.types boolean
+#### A masked pattern was here ####
+ name default.t
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucketing_version 2
+ column.name.delimiter ,
+ columns flag
+ columns.comments
+ columns.types boolean
+#### A masked pattern was here ####
+ name default.t
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.t
+ name: default.t
+ Truncated Path -> Alias:
+ /t [t]
+ Reducer 2
+ Execution mode: vectorized, llap
+ Needs Tagging: false
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ keys: KEY._col0 (type: boolean)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ bucketingVersion: 2
+ compressed: false
+ GlobalTableId: 0
+#### A masked pattern was here ####
+ NumFilesPerFileSink: 1
+ Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE
+#### A masked pattern was here ####
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ properties:
+ bucketing_version -1
+ columns _col0,_col1
+ columns.types boolean:bigint
+ escape.delim \
+ hive.serialization.extend.additional.nesting.levels true
+ serialization.escape.crlf true
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ TotalFiles: 1
+ GatherStats: false
+ MultiFileSpray: false
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: ALTER TABLE t UPDATE STATISTICS for column flag SET ('numNulls'='100','numTrues'='450','numFalses'='450')
+PREHOOK: type: ALTERTABLE_UPDATETABLESTATS
+PREHOOK: Input: default@t
+PREHOOK: Output: default@t
+POSTHOOK: query: ALTER TABLE t UPDATE STATISTICS for column flag SET ('numNulls'='100','numTrues'='450','numFalses'='450')
+POSTHOOK: type: ALTERTABLE_UPDATETABLESTATS
+POSTHOOK: Input: default@t
+POSTHOOK: Output: default@t
+PREHOOK: query: EXPLAIN EXTENDED SELECT flag, COUNT(*) FROM t GROUP BY flag
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN EXTENDED SELECT flag, COUNT(*) FROM t GROUP BY flag
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t
+#### A masked pattern was here ####
+OPTIMIZED SQL: SELECT `flag`, COUNT(*) AS `_c1`
+FROM `default`.`t`
+GROUP BY `flag`
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: t
+ Statistics: Num rows: 1000 Data size: 3604 Basic stats: COMPLETE Column stats: COMPLETE
+ GatherStats: false
+ Select Operator
+ expressions: flag (type: boolean)
+ outputColumnNames: flag
+ Statistics: Num rows: 1000 Data size: 3604 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ aggregations: count()
+ keys: flag (type: boolean)
+ minReductionHashAggr: 0.99
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 3 Data size: 36 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ bucketingVersion: 2
+ key expressions: _col0 (type: boolean)
+ null sort order: z
+ numBuckets: -1
+ sort order: +
+ Map-reduce partition columns: _col0 (type: boolean)
+ Statistics: Num rows: 3 Data size: 36 Basic stats: COMPLETE Column stats: COMPLETE
+ tag: -1
+ value expressions: _col1 (type: bigint)
+ auto parallelism: true
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Path -> Alias:
+#### A masked pattern was here ####
+ Path -> Partition:
+#### A masked pattern was here ####
+ Partition
+ base file name: t
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count -1
+ bucketing_version 2
+ column.name.delimiter ,
+ columns flag
+ columns.types boolean
+#### A masked pattern was here ####
+ name default.t
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucketing_version 2
+ column.name.delimiter ,
+ columns flag
+ columns.comments
+ columns.types boolean
+#### A masked pattern was here ####
+ name default.t
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.t
+ name: default.t
+ Truncated Path -> Alias:
+ /t [t]
+ Reducer 2
+ Execution mode: vectorized, llap
+ Needs Tagging: false
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ keys: KEY._col0 (type: boolean)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ bucketingVersion: 2
+ compressed: false
+ GlobalTableId: 0
+#### A masked pattern was here ####
+ NumFilesPerFileSink: 1
+ Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE
+#### A masked pattern was here ####
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ properties:
+ bucketing_version -1
+ columns _col0,_col1
+ columns.types boolean:bigint
+ escape.delim \
+ hive.serialization.extend.additional.nesting.levels true
+ serialization.escape.crlf true
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ TotalFiles: 1
+ GatherStats: false
+ MultiFileSpray: false
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: CREATE TABLE t3 (id INT)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@t3
+POSTHOOK: query: CREATE TABLE t3 (id INT)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@t3
+PREHOOK: query: CREATE TABLE t4 (id INT, data INT)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@t4
+POSTHOOK: query: CREATE TABLE t4 (id INT, data INT)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@t4
+PREHOOK: query: ALTER TABLE t3 UPDATE STATISTICS SET('numRows'='10000')
+PREHOOK: type: ALTERTABLE_UPDATETABLESTATS
+PREHOOK: Input: default@t3
+PREHOOK: Output: default@t3
+POSTHOOK: query: ALTER TABLE t3 UPDATE STATISTICS SET('numRows'='10000')
+POSTHOOK: type: ALTERTABLE_UPDATETABLESTATS
+POSTHOOK: Input: default@t3
+POSTHOOK: Output: default@t3
+PREHOOK: query: ALTER TABLE t3 UPDATE STATISTICS for column id SET ('numDVs'='10000','numNulls'='0')
+PREHOOK: type: ALTERTABLE_UPDATETABLESTATS
+PREHOOK: Input: default@t3
+PREHOOK: Output: default@t3
+POSTHOOK: query: ALTER TABLE t3 UPDATE STATISTICS for column id SET ('numDVs'='10000','numNulls'='0')
+POSTHOOK: type: ALTERTABLE_UPDATETABLESTATS
+POSTHOOK: Input: default@t3
+POSTHOOK: Output: default@t3
+PREHOOK: query: ALTER TABLE t4 UPDATE STATISTICS SET('numRows'='5000')
+PREHOOK: type: ALTERTABLE_UPDATETABLESTATS
+PREHOOK: Input: default@t4
+PREHOOK: Output: default@t4
+POSTHOOK: query: ALTER TABLE t4 UPDATE STATISTICS SET('numRows'='5000')
+POSTHOOK: type: ALTERTABLE_UPDATETABLESTATS
+POSTHOOK: Input: default@t4
+POSTHOOK: Output: default@t4
+PREHOOK: query: ALTER TABLE t4 UPDATE STATISTICS for column id SET ('numDVs'='5000','numNulls'='0')
+PREHOOK: type: ALTERTABLE_UPDATETABLESTATS
+PREHOOK: Input: default@t4
+PREHOOK: Output: default@t4
+POSTHOOK: query: ALTER TABLE t4 UPDATE STATISTICS for column id SET ('numDVs'='5000','numNulls'='0')
+POSTHOOK: type: ALTERTABLE_UPDATETABLESTATS
+POSTHOOK: Input: default@t4
+POSTHOOK: Output: default@t4
+PREHOOK: query: ALTER TABLE t4 UPDATE STATISTICS for column data SET ('numDVs'='100','numNulls'='-1')
+PREHOOK: type: ALTERTABLE_UPDATETABLESTATS
+PREHOOK: Input: default@t4
+PREHOOK: Output: default@t4
+POSTHOOK: query: ALTER TABLE t4 UPDATE STATISTICS for column data SET ('numDVs'='100','numNulls'='-1')
+POSTHOOK: type: ALTERTABLE_UPDATETABLESTATS
+POSTHOOK: Input: default@t4
+POSTHOOK: Output: default@t4
+PREHOOK: query: EXPLAIN EXTENDED SELECT * FROM (SELECT t3.id, t4.data FROM t3 LEFT OUTER JOIN t4 ON t3.id = t4.id) sub WHERE sub.data IS NULL
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t3
+PREHOOK: Input: default@t4
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN EXTENDED SELECT * FROM (SELECT t3.id, t4.data FROM t3 LEFT OUTER JOIN t4 ON t3.id = t4.id) sub WHERE sub.data IS NULL
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t3
+POSTHOOK: Input: default@t4
+#### A masked pattern was here ####
+OPTIMIZED SQL: SELECT `t`.`id` AS `sub.id`, CAST(NULL AS INT) AS `sub.data`
+FROM (SELECT `id`
+FROM `default`.`t3`) AS `t`
+LEFT JOIN (SELECT `id`, `data`
+FROM `default`.`t4`
+WHERE `id` IS NOT NULL) AS `t1` ON `t`.`id` = `t1`.`id`
+WHERE `t1`.`data` IS NULL
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: t3
+ Statistics: Num rows: 10000 Data size: 40000 Basic stats: COMPLETE Column stats: COMPLETE
+ GatherStats: false
+ Select Operator
+ expressions: id (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 10000 Data size: 40000 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ bucketingVersion: 2
+ key expressions: _col0 (type: int)
+ null sort order: z
+ numBuckets: -1
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 10000 Data size: 40000 Basic stats: COMPLETE Column stats: COMPLETE
+ tag: 0
+ auto parallelism: true
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Path -> Alias:
+#### A masked pattern was here ####
+ Path -> Partition:
+#### A masked pattern was here ####
+ Partition
+ base file name: t3
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count -1
+ bucketing_version 2
+ column.name.delimiter ,
+ columns id
+ columns.types int
+#### A masked pattern was here ####
+ name default.t3
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucketing_version 2
+ column.name.delimiter ,
+ columns id
+ columns.comments
+ columns.types int
+#### A masked pattern was here ####
+ name default.t3
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.t3
+ name: default.t3
+ Truncated Path -> Alias:
+ /t3 [t3]
+ Map 3
+ Map Operator Tree:
+ TableScan
+ alias: t4
+ filterExpr: id is not null (type: boolean)
+ Statistics: Num rows: 5000 Data size: 40000 Basic stats: COMPLETE Column stats: COMPLETE
+ GatherStats: false
+ Filter Operator
+ isSamplingPred: false
+ predicate: id is not null (type: boolean)
+ Statistics: Num rows: 5000 Data size: 40000 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: id (type: int), data (type: int)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 5000 Data size: 40000 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ bucketingVersion: 2
+ key expressions: _col0 (type: int)
+ null sort order: z
+ numBuckets: -1
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 5000 Data size: 40000 Basic stats: COMPLETE Column stats: COMPLETE
+ tag: 1
+ value expressions: _col1 (type: int)
+ auto parallelism: true
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Path -> Alias:
+#### A masked pattern was here ####
+ Path -> Partition:
+#### A masked pattern was here ####
+ Partition
+ base file name: t4
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count -1
+ bucketing_version 2
+ column.name.delimiter ,
+ columns id,data
+ columns.types int:int
+#### A masked pattern was here ####
+ name default.t4
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucketing_version 2
+ column.name.delimiter ,
+ columns id,data
+ columns.comments
+ columns.types int:int
+#### A masked pattern was here ####
+ name default.t4
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.t4
+ name: default.t4
+ Truncated Path -> Alias:
+ /t4 [t4]
+ Reducer 2
+ Execution mode: llap
+ Needs Tagging: false
+ Reduce Operator Tree:
+ Merge Join Operator
+ condition map:
+ Left Outer Join 0 to 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col0, _col2
+ Position of Big Table: 0
+ Statistics: Num rows: 15000 Data size: 120000 Basic stats: COMPLETE Column stats: COMPLETE
+ Filter Operator
+ isSamplingPred: false
+ predicate: _col2 is null (type: boolean)
+ Statistics: Num rows: 7500 Data size: 60000 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: _col0 (type: int), null (type: int)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 7500 Data size: 30004 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ bucketingVersion: 2
+ compressed: false
+ GlobalTableId: 0
+#### A masked pattern was here ####
+ NumFilesPerFileSink: 1
+ Statistics: Num rows: 7500 Data size: 30004 Basic stats: COMPLETE Column stats: COMPLETE
+#### A masked pattern was here ####
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ properties:
+ bucketing_version -1
+ columns _col0,_col1
+ columns.types int:int
+ escape.delim \
+ hive.serialization.extend.additional.nesting.levels true
+ serialization.escape.crlf true
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ TotalFiles: 1
+ GatherStats: false
+ MultiFileSpray: false
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+