You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2018/07/14 01:53:41 UTC
[19/20] hive git commit: HIVE-19820 : add ACID stats support to
background stats updater and fix bunch of edge cases found in SU tests
(Sergey Shelukhin)
http://git-wip-us.apache.org/repos/asf/hive/blob/1c9947f3/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUpdaterThread.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUpdaterThread.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUpdaterThread.java
index bb181a1..838d277 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUpdaterThread.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUpdaterThread.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.hive.common.TableName;
import org.apache.hadoop.hive.common.ValidReaderWriteIdList;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.MetaStoreThread;
+import org.apache.hadoop.hive.metastore.ObjectStore;
import org.apache.hadoop.hive.metastore.RawStore;
import org.apache.hadoop.hive.metastore.RawStoreProxy;
import org.apache.hadoop.hive.metastore.Warehouse;
@@ -214,11 +215,17 @@ public class StatsUpdaterThread extends Thread implements MetaStoreThread {
String skipParam = table.getParameters().get(SKIP_STATS_AUTOUPDATE_PROPERTY);
if ("true".equalsIgnoreCase(skipParam)) return null;
- // TODO: when txn stats are implemented, use writeIds to determine stats accuracy
- @SuppressWarnings("unused")
- ValidReaderWriteIdList writeIds = null;
- if (AcidUtils.isTransactionalTable(table)) {
- writeIds = getWriteIds(fullTableName);
+ // Note: ideally we should take a lock here to pretend to be a real reader.
+ // For now, this check is going to have race potential; it may run a spurious analyze.
+ String writeIdString = null;
+ boolean isTxn = AcidUtils.isTransactionalTable(table);
+ if (isTxn) {
+ ValidReaderWriteIdList writeIds = getWriteIds(fullTableName);
+ if (writeIds == null) {
+ LOG.error("Cannot get writeIds for transactional table " + fullTableName + "; skipping");
+ return null;
+ }
+ writeIdString = writeIds.writeToString();
}
List<String> allCols = new ArrayList<>(table.getSd().getColsSize());
for (FieldSchema fs : table.getSd().getCols()) {
@@ -227,9 +234,16 @@ public class StatsUpdaterThread extends Thread implements MetaStoreThread {
Collections.sort(allCols);
if (table.getPartitionKeysSize() == 0) {
Map<String, String> params = table.getParameters();
- List<String> colsToUpdate = isExistingOnly
- ? getExistingNonPartTableStatsToUpdate(fullTableName, cat, db, tbl, params, allCols)
- : getAnyStatsToUpdate(allCols, params);
+ List<String> colsToUpdate = null;
+ long writeId = isTxn ? table.getWriteId() : -1;
+ if (isExistingOnly) {
+ // Get the existing stats, including the txn state if any, to see if we need to update.
+ colsToUpdate = getExistingNonPartTableStatsToUpdate(
+ fullTableName, cat, db, tbl, params, writeId, allCols, writeIdString);
+ } else {
+ colsToUpdate = getAnyStatsToUpdate(db, tbl, allCols, params, writeId, writeIdString);
+ }
+
LOG.debug("Columns to update are {}; existing only: {}, out of: {} based on {}",
colsToUpdate, isExistingOnly, allCols, params);
@@ -241,7 +255,7 @@ public class StatsUpdaterThread extends Thread implements MetaStoreThread {
} else {
Map<String, List<String>> partsToAnalyze = new HashMap<>();
List<String> colsForAllParts = findPartitionsToAnalyze(
- fullTableName, cat, db, tbl, allCols, partsToAnalyze);
+ fullTableName, cat, db, tbl, allCols, partsToAnalyze, writeIdString);
LOG.debug("Columns to update are {} for all partitions; {} individual partitions."
+ " Existing only: {}, out of: {}", colsForAllParts, partsToAnalyze.size(),
isExistingOnly, allCols);
@@ -263,18 +277,30 @@ public class StatsUpdaterThread extends Thread implements MetaStoreThread {
}
private List<String> findPartitionsToAnalyze(TableName fullTableName, String cat, String db,
- String tbl, List<String> allCols, Map<String, List<String>> partsToAnalyze)
- throws MetaException, NoSuchObjectException {
+ String tbl, List<String> allCols, Map<String, List<String>> partsToAnalyze,
+ String writeIdString) throws MetaException, NoSuchObjectException {
// TODO: ideally when col-stats-accurate stuff is stored in some sane structure, this should
- // to retrieve partsToUpdate in a single query; no checking partition params in java.
+ // retrieve partsToUpdate in a single query; no checking partition params in java.
List<String> partNames = null;
Map<String, List<String>> colsPerPartition = null;
boolean isAllParts = true;
if (isExistingOnly) {
- colsPerPartition = rs.getPartitionColsWithStats(cat, db, tbl);
- partNames = Lists.newArrayList(colsPerPartition.keySet());
- int partitionCount = rs.getNumPartitionsByFilter(cat, db, tbl, "");
- isAllParts = partitionCount == partNames.size();
+ // Make sure the number of partitions we get, and the number of stats objects, is consistent.
+ rs.openTransaction();
+ boolean isOk = false;
+ try {
+ colsPerPartition = rs.getPartitionColsWithStats(cat, db, tbl);
+ partNames = Lists.newArrayList(colsPerPartition.keySet());
+ int partitionCount = rs.getNumPartitionsByFilter(cat, db, tbl, "");
+ isAllParts = partitionCount == partNames.size();
+ isOk = true;
+ } finally {
+ if (isOk) {
+ rs.commitTransaction();
+ } else {
+ rs.rollbackTransaction();
+ }
+ }
} else {
partNames = rs.listPartitionNames(cat, db, tbl, (short) -1);
isAllParts = true;
@@ -326,9 +352,10 @@ public class StatsUpdaterThread extends Thread implements MetaStoreThread {
colsToMaybeUpdate = colsPerPartition.get(partName);
Collections.sort(colsToMaybeUpdate);
}
- List<String> colsToUpdate = getAnyStatsToUpdate(colsToMaybeUpdate, params);
- LOG.debug("Updating {} based on {} and {}", colsToUpdate, colsToMaybeUpdate, params);
+ List<String> colsToUpdate = getAnyStatsToUpdate(db, tbl, colsToMaybeUpdate, params,
+ writeIdString == null ? -1 : part.getWriteId(), writeIdString);
+ LOG.debug("Updating {} based on {} and {}", colsToUpdate, colsToMaybeUpdate, params);
if (colsToUpdate == null || colsToUpdate.isEmpty()) {
if (isAllParts) {
@@ -405,8 +432,8 @@ public class StatsUpdaterThread extends Thread implements MetaStoreThread {
}
private List<String> getExistingNonPartTableStatsToUpdate(TableName fullTableName,
- String cat, String db, String tbl, Map<String, String> params,
- List<String> allCols) throws MetaException {
+ String cat, String db, String tbl, Map<String, String> params, long statsWriteId,
+ List<String> allCols, String writeIdString) throws MetaException {
ColumnStatistics existingStats = null;
try {
// Note: this should NOT do txn verification - we want to get outdated stats, to
@@ -416,12 +443,15 @@ public class StatsUpdaterThread extends Thread implements MetaStoreThread {
LOG.error("Cannot retrieve existing stats, skipping " + fullTableName, e);
return null;
}
- return getExistingStatsToUpdate(existingStats, params);
+ // TODO: we should probably skip updating if writeId is from an active txn
+ boolean isTxnValid = (writeIdString == null) || ObjectStore.isCurrentStatsValidForTheQuery(
+ conf, db, tbl, params, statsWriteId , 0, writeIdString, false);
+ return getExistingStatsToUpdate(existingStats, params, isTxnValid);
}
private List<String> getExistingStatsToUpdate(
- ColumnStatistics existingStats, Map<String, String> params) {
- boolean hasAnyAccurate = StatsSetupConst.areBasicStatsUptoDate(params);
+ ColumnStatistics existingStats, Map<String, String> params, boolean isTxnValid) {
+ boolean hasAnyAccurate = isTxnValid && StatsSetupConst.areBasicStatsUptoDate(params);
List<String> colsToUpdate = new ArrayList<>();
for (ColumnStatisticsObj obj : existingStats.getStatsObj()) {
String col = obj.getColName();
@@ -432,12 +462,17 @@ public class StatsUpdaterThread extends Thread implements MetaStoreThread {
return colsToUpdate;
}
- private List<String> getAnyStatsToUpdate(
- List<String> allCols, Map<String, String> params) {
+ private List<String> getAnyStatsToUpdate(String db, String tbl, List<String> allCols,
+ Map<String, String> params, long statsWriteId, String writeIdString) throws MetaException {
// Note: we only run "for columns" command and assume no basic stats means no col stats.
if (!StatsSetupConst.areBasicStatsUptoDate(params)) {
return allCols;
}
+ // TODO: we should probably skip updating if writeId is from an active txn
+ if (writeIdString != null && !ObjectStore.isCurrentStatsValidForTheQuery(
+ conf, db, tbl, params, statsWriteId, 0, writeIdString, false)) {
+ return allCols;
+ }
List<String> colsToUpdate = new ArrayList<>();
for (String col : allCols) {
if (!StatsSetupConst.areColumnStatsUptoDate(params, col)) {
@@ -460,8 +495,9 @@ public class StatsUpdaterThread extends Thread implements MetaStoreThread {
private ValidReaderWriteIdList getWriteIds(
TableName fullTableName) throws NoSuchTxnException, MetaException {
- GetValidWriteIdsRequest req = new GetValidWriteIdsRequest();
- req.setFullTableNames(Lists.newArrayList(fullTableName.toString()));
+ // TODO: acid utils don't support catalogs
+ GetValidWriteIdsRequest req = new GetValidWriteIdsRequest(
+ Lists.newArrayList(fullTableName.getDbTable()), null);
return TxnUtils.createValidReaderWriteIdList(
txnHandler.getValidWriteIds(req).getTblValidWriteIds().get(0));
}
http://git-wip-us.apache.org/repos/asf/hive/blob/1c9947f3/ql/src/test/org/apache/hadoop/hive/ql/TestTxnConcatenate.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnConcatenate.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnConcatenate.java
index 0e436e1..5b8ff15 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnConcatenate.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnConcatenate.java
@@ -219,7 +219,7 @@ public class TestTxnConcatenate extends TxnCommandsBaseForTests {
"select count(*) from COMPACTION_QUEUE where CQ_TABLE='s'"));
Assert.assertEquals(1, TxnDbUtil.countQueryAgent(hiveConf,
"select count(*) from WRITE_SET where WS_TABLE='s'"));
- Assert.assertEquals(2, TxnDbUtil.countQueryAgent(hiveConf,
+ Assert.assertEquals(3, TxnDbUtil.countQueryAgent(hiveConf,
"select count(*) from TXN_TO_WRITE_ID where T2W_TABLE='s'"));
Assert.assertEquals(1, TxnDbUtil.countQueryAgent(hiveConf,
"select count(*) from NEXT_WRITE_ID where NWI_TABLE='s'"));
@@ -234,7 +234,7 @@ public class TestTxnConcatenate extends TxnCommandsBaseForTests {
"select count(*) from COMPACTION_QUEUE where CQ_TABLE='bar'"));
Assert.assertEquals(1, TxnDbUtil.countQueryAgent(hiveConf,
"select count(*) from WRITE_SET where WS_TABLE='bar'"));
- Assert.assertEquals(2, TxnDbUtil.countQueryAgent(hiveConf,
+ Assert.assertEquals(4, TxnDbUtil.countQueryAgent(hiveConf,
"select count(*) from TXN_TO_WRITE_ID where T2W_TABLE='bar'"));
Assert.assertEquals(1, TxnDbUtil.countQueryAgent(hiveConf,
"select count(*) from NEXT_WRITE_ID where NWI_TABLE='bar'"));
http://git-wip-us.apache.org/repos/asf/hive/blob/1c9947f3/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java
index bbe9d5a..2c98e3c 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java
@@ -448,7 +448,7 @@ ekoifman:apache-hive-3.0.0-SNAPSHOT-bin ekoifman$ tree /Users/ekoifman/dev/hiver
{"{\"writeid\":0,\"bucketid\":536870912,\"rowid\":4}\t20\t40", "warehouse/t/HIVE_UNION_SUBDIR_15/000000_0"},
{"{\"writeid\":0,\"bucketid\":536870912,\"rowid\":5}\t50\t60", "warehouse/t/HIVE_UNION_SUBDIR_16/000000_0"},
// update for "{\"writeid\":0,\"bucketid\":536936448,\"rowid\":1}\t60\t80"
- {"{\"writeid\":10000001,\"bucketid\":536936448,\"rowid\":0}\t60\t88", "warehouse/t/delta_10000001_10000001_0000/bucket_00001"},
+ {"{\"writeid\":10000001,\"bucketid\":536936448,\"rowid\":0}\t60\t88", "warehouse/t/delta_10000001_10000001_0000/bucket_00001"},
};
rs = runStatementOnDriver("select ROW__ID, a, b, INPUT__FILE__NAME from T order by a, b, INPUT__FILE__NAME");
checkExpected(rs, expected3,"after converting to acid (no compaction with updates)");
@@ -783,12 +783,12 @@ ekoifman:apache-hive-3.0.0-SNAPSHOT-bin ekoifman$ tree /Users/ekoifman/dev/hiver
String[][] expected = {
{"{\"writeid\":1,\"bucketid\":536870912,\"rowid\":0}\t1\t1\t4\t1", "t/p=1/q=1/delta_0000001_0000001_0000/bucket_00000"},
{"{\"writeid\":1,\"bucketid\":536870912,\"rowid\":1}\t1\t1\t4\t3", "t/p=1/q=1/delta_0000001_0000001_0000/bucket_00000"},
- {"{\"writeid\":2,\"bucketid\":536870912,\"rowid\":0}\t1\t1\t5\t1", "t/p=1/q=1/delta_0000002_0000002_0000/bucket_00000"},
- {"{\"writeid\":2,\"bucketid\":536870912,\"rowid\":1}\t1\t1\t5\t3", "t/p=1/q=1/delta_0000002_0000002_0000/bucket_00000"},
+ {"{\"writeid\":3,\"bucketid\":536870912,\"rowid\":0}\t1\t1\t5\t1", "t/p=1/q=1/delta_0000003_0000003_0000/bucket_00000"},
+ {"{\"writeid\":3,\"bucketid\":536870912,\"rowid\":1}\t1\t1\t5\t3", "t/p=1/q=1/delta_0000003_0000003_0000/bucket_00000"},
{"{\"writeid\":1,\"bucketid\":536870912,\"rowid\":0}\t1\t2\t4\t2", "t/p=1/q=2/delta_0000001_0000001_0000/bucket_00000"},
{"{\"writeid\":1,\"bucketid\":536870912,\"rowid\":1}\t1\t2\t4\t4", "t/p=1/q=2/delta_0000001_0000001_0000/bucket_00000"},
- {"{\"writeid\":2,\"bucketid\":536870912,\"rowid\":0}\t1\t2\t5\t2", "t/p=1/q=2/delta_0000002_0000002_0000/bucket_00000"},
- {"{\"writeid\":2,\"bucketid\":536870912,\"rowid\":1}\t1\t2\t5\t4", "t/p=1/q=2/delta_0000002_0000002_0000/bucket_00000"}
+ {"{\"writeid\":3,\"bucketid\":536870912,\"rowid\":0}\t1\t2\t5\t2", "t/p=1/q=2/delta_0000003_0000003_0000/bucket_00000"},
+ {"{\"writeid\":3,\"bucketid\":536870912,\"rowid\":1}\t1\t2\t5\t4", "t/p=1/q=2/delta_0000003_0000003_0000/bucket_00000"}
};
checkExpected(rs, expected, "insert data");
@@ -801,12 +801,12 @@ ekoifman:apache-hive-3.0.0-SNAPSHOT-bin ekoifman$ tree /Users/ekoifman/dev/hiver
String[][] expected2 = {
{"{\"writeid\":1,\"bucketid\":536870912,\"rowid\":0}\t1\t1\t4\t1", "t/p=1/q=1/delta_0000001_0000001_0000/bucket_00000"},
{"{\"writeid\":1,\"bucketid\":536870912,\"rowid\":1}\t1\t1\t4\t3", "t/p=1/q=1/delta_0000001_0000001_0000/bucket_00000"},
- {"{\"writeid\":2,\"bucketid\":536870912,\"rowid\":0}\t1\t1\t5\t1", "t/p=1/q=1/delta_0000002_0000002_0000/bucket_00000"},
- {"{\"writeid\":2,\"bucketid\":536870912,\"rowid\":1}\t1\t1\t5\t3", "t/p=1/q=1/delta_0000002_0000002_0000/bucket_00000"},
- {"{\"writeid\":1,\"bucketid\":536870912,\"rowid\":0}\t1\t2\t4\t2", "t/p=1/q=2/base_0000002/bucket_00000"},
- {"{\"writeid\":1,\"bucketid\":536870912,\"rowid\":1}\t1\t2\t4\t4", "t/p=1/q=2/base_0000002/bucket_00000"},
- {"{\"writeid\":2,\"bucketid\":536870912,\"rowid\":0}\t1\t2\t5\t2", "t/p=1/q=2/base_0000002/bucket_00000"},
- {"{\"writeid\":2,\"bucketid\":536870912,\"rowid\":1}\t1\t2\t5\t4", "t/p=1/q=2/base_0000002/bucket_00000"}
+ {"{\"writeid\":3,\"bucketid\":536870912,\"rowid\":0}\t1\t1\t5\t1", "t/p=1/q=1/delta_0000003_0000003_0000/bucket_00000"},
+ {"{\"writeid\":3,\"bucketid\":536870912,\"rowid\":1}\t1\t1\t5\t3", "t/p=1/q=1/delta_0000003_0000003_0000/bucket_00000"},
+ {"{\"writeid\":1,\"bucketid\":536870912,\"rowid\":0}\t1\t2\t4\t2", "t/p=1/q=2/base_0000003/bucket_00000"},
+ {"{\"writeid\":1,\"bucketid\":536870912,\"rowid\":1}\t1\t2\t4\t4", "t/p=1/q=2/base_0000003/bucket_00000"},
+ {"{\"writeid\":3,\"bucketid\":536870912,\"rowid\":0}\t1\t2\t5\t2", "t/p=1/q=2/base_0000003/bucket_00000"},
+ {"{\"writeid\":3,\"bucketid\":536870912,\"rowid\":1}\t1\t2\t5\t4", "t/p=1/q=2/base_0000003/bucket_00000"}
};
checkExpected(rs, expected2, "after major compaction");
http://git-wip-us.apache.org/repos/asf/hive/blob/1c9947f3/ql/src/test/org/apache/hadoop/hive/ql/TxnCommandsBaseForTests.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TxnCommandsBaseForTests.java b/ql/src/test/org/apache/hadoop/hive/ql/TxnCommandsBaseForTests.java
index 7319ba0..a2fafca 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TxnCommandsBaseForTests.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TxnCommandsBaseForTests.java
@@ -225,7 +225,7 @@ public abstract class TxnCommandsBaseForTests {
expected.length, rs.size());
//verify data and layout
for(int i = 0; i < expected.length; i++) {
- Assert.assertTrue("Actual line (data) " + i + " data: " + rs.get(i), rs.get(i).startsWith(expected[i][0]));
+ Assert.assertTrue("Actual line (data) " + i + " data: " + rs.get(i) + "; expected " + expected[i][0], rs.get(i).startsWith(expected[i][0]));
if(checkFileName) {
Assert.assertTrue("Actual line(file) " + i + " file: " + rs.get(i), rs.get(i).endsWith(expected[i][1]));
}
http://git-wip-us.apache.org/repos/asf/hive/blob/1c9947f3/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java
index d30bbde..ca4d36f 100755
--- a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java
@@ -325,8 +325,10 @@ public class TestHive extends TestCase {
tbl.getParameters().put(hive_metastoreConstants.DDL_TIME,
ft.getParameters().get(hive_metastoreConstants.DDL_TIME));
// Txn stuff set by metastore
- if (tbl.getTTable().isSetWriteId()) {
- ft.getTTable().setWriteId(tbl.getTTable().getWriteId());
+ if (tbl.getTTable().isSetWriteId() != ft.getTTable().isSetWriteId()) {
+ // No need to compare this field.
+ ft.getTTable().setWriteId(0);
+ tbl.getTTable().setWriteId(0);
}
assertTrue("Tables doesn't match: " + tableName + " (" + ft.getTTable()
+ "; " + tbl.getTTable() + ")", ft.getTTable().equals(tbl.getTTable()));
http://git-wip-us.apache.org/repos/asf/hive/blob/1c9947f3/ql/src/test/org/apache/hadoop/hive/ql/stats/TestStatsUpdaterThread.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/stats/TestStatsUpdaterThread.java b/ql/src/test/org/apache/hadoop/hive/ql/stats/TestStatsUpdaterThread.java
index 14f86ea..6c768c0 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/stats/TestStatsUpdaterThread.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/stats/TestStatsUpdaterThread.java
@@ -29,9 +29,11 @@ import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.curator.shaded.com.google.common.collect.Lists;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hive.common.StatsSetupConst;
+import org.apache.hadoop.hive.common.ValidWriteIdList;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
import org.apache.hadoop.hive.metastore.IMetaStoreClient;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
import org.apache.hadoop.hive.metastore.api.EnvironmentContext;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.Partition;
@@ -140,6 +142,148 @@ public class TestStatsUpdaterThread {
msClient.close();
}
+ @Test(timeout=80000)
+ public void testTxnTable() throws Exception {
+ StatsUpdaterThread su = createUpdater();
+ IMetaStoreClient msClient = new HiveMetaStoreClient(hiveConf);
+
+ executeQuery("create table simple_stats (s string) TBLPROPERTIES "
+ + "(\"transactional\"=\"true\", \"transactional_properties\"=\"insert_only\")");
+ executeQuery("insert into simple_stats (s) values ('test')");
+ List<String> cols = Lists.newArrayList("s");
+ String dbName = ss.getCurrentDatabase(), tblName = "simple_stats", fqName = dbName + "." + tblName;
+ ValidWriteIdList initialWriteIds = msClient.getValidWriteIds(fqName);
+ verifyStatsUpToDate(tblName, cols, msClient, 0, initialWriteIds.toString(), true);
+ assertFalse(su.runOneIteration());
+ drainWorkQueue(su, 0);
+
+ executeQuery("insert overwrite table simple_stats values ('test2')");
+ ValidWriteIdList nextWriteIds = msClient.getValidWriteIds(fqName);
+ verifyStatsUpToDate(tblName, cols, msClient, 0, nextWriteIds.toString(), true);
+ assertFalse(su.runOneIteration());
+ drainWorkQueue(su, 0);
+ String currentWriteIds = msClient.getValidWriteIds(fqName).toString();
+
+ // Overwrite the txn state to refer to an open txn.
+ long badTxnId = msClient.openTxn("moo");
+ long badWriteId = msClient.allocateTableWriteId(badTxnId, dbName, tblName);
+
+ Table tbl = msClient.getTable(dbName, tblName);
+ tbl.setWriteId(badWriteId);
+ msClient.alter_table(
+ null, dbName, tblName, tbl, new EnvironmentContext(), -1, initialWriteIds.toString());
+
+ // Stats should not be valid.
+ verifyStatsUpToDate(tblName, cols, msClient, 0, currentWriteIds, false);
+
+ // Analyze should not be able to set valid stats for a running txn.
+ assertTrue(su.runOneIteration());
+ drainWorkQueue(su);
+
+ currentWriteIds = msClient.getValidWriteIds(fqName).toString();
+ verifyStatsUpToDate(tblName, cols, msClient, 0, currentWriteIds, false);
+
+ msClient.abortTxns(Lists.newArrayList(badTxnId));
+
+ // Analyze should be able to override stats of an aborted txn.
+ assertTrue(su.runOneIteration());
+ drainWorkQueue(su);
+
+ // Stats will now be valid.
+ currentWriteIds = msClient.getValidWriteIds(fqName).toString();
+ verifyStatsUpToDate(tblName, cols, msClient, 0, currentWriteIds, true);
+
+ // Verify that incorrect stats from a valid write ID are also handled.
+ badTxnId = msClient.openTxn("moo");
+ badWriteId = msClient.allocateTableWriteId(badTxnId, dbName, tblName);
+ tbl = msClient.getTable(dbName, tblName);
+ tbl.setWriteId(badWriteId);
+ StatsSetupConst.setBasicStatsState(tbl.getParameters(), StatsSetupConst.FALSE);
+ msClient.alter_table(null, dbName, tblName, tbl, new EnvironmentContext(), -1, initialWriteIds.toString());
+
+ // Stats should not be valid.
+ verifyStatsUpToDate(tblName, cols, msClient, 0, currentWriteIds, false);
+
+ // Analyze should not be able to set valid stats for a running txn.
+ assertTrue(su.runOneIteration());
+ drainWorkQueue(su);
+
+ currentWriteIds = msClient.getValidWriteIds(fqName).toString();
+ verifyStatsUpToDate(tblName, cols, msClient, 0, currentWriteIds, false);
+
+ msClient.commitTxn(badTxnId);
+
+ // Analyze should be able to override stats of an committed txn.
+ assertTrue(su.runOneIteration());
+ drainWorkQueue(su);
+
+ // Stats will now be valid.
+ currentWriteIds = msClient.getValidWriteIds(fqName).toString();
+ verifyStatsUpToDate(tblName, cols, msClient, 0, currentWriteIds, true);
+
+ msClient.close();
+ }
+
+
+ @Test
+ public void testTxnPartitions() throws Exception {
+ StatsUpdaterThread su = createUpdater();
+ IMetaStoreClient msClient = new HiveMetaStoreClient(hiveConf);
+
+ executeQuery("create table simple_stats (s string) partitioned by (p int) TBLPROPERTIES "
+ + "(\"transactional\"=\"true\", \"transactional_properties\"=\"insert_only\")");
+ executeQuery("insert into simple_stats partition(p=1) values ('test')");
+ executeQuery("insert into simple_stats partition(p=2) values ('test2')");
+ executeQuery("insert into simple_stats partition(p=3) values ('test3')");
+ assertFalse(su.runOneIteration());
+ drainWorkQueue(su, 0);
+
+ executeQuery("insert overwrite table simple_stats partition(p=1) values ('test2')");
+ executeQuery("insert overwrite table simple_stats partition(p=2) values ('test3')");
+ assertFalse(su.runOneIteration());
+ drainWorkQueue(su, 0);
+
+ // Overwrite the txn state to refer to an aborted txn on some partitions.
+ String dbName = ss.getCurrentDatabase(), tblName = "simple_stats", fqName = dbName + "." + tblName;
+ long badTxnId = msClient.openTxn("moo");
+ long badWriteId = msClient.allocateTableWriteId(badTxnId, dbName, tblName);
+ msClient.abortTxns(Lists.newArrayList(badTxnId));
+
+ Partition part1 = msClient.getPartition(dbName, tblName, "p=1");
+ Partition part2 = msClient.getPartition(dbName, tblName, "p=2");
+ part1.setWriteId(badWriteId);
+ part2.setWriteId(badWriteId);
+ String currentWriteIds = msClient.getValidWriteIds(fqName).toString();
+ // To update write ID we need to specify the write ID list to validate concurrent writes.
+ msClient.alter_partitions(dbName, tblName,
+ Lists.newArrayList(part1), null, -1, currentWriteIds, badWriteId);
+ msClient.alter_partitions(dbName, tblName,
+ Lists.newArrayList(part2), null, -1, currentWriteIds, badWriteId);
+
+ // We expect two partitions to be updated.
+ Map<String, List<ColumnStatisticsObj>> stats = msClient.getPartitionColumnStatistics(
+ dbName, tblName, Lists.newArrayList("p=1", "p=2", "p=3"),
+ Lists.newArrayList("s"), 0, currentWriteIds);
+ assertEquals(1, stats.size());
+
+ assertTrue(su.runOneIteration());
+ drainWorkQueue(su, 2);
+ // Analyze treats stats like data (new write ID), so stats still should not be valid.
+ stats = msClient.getPartitionColumnStatistics(
+ dbName, tblName, Lists.newArrayList("p=1", "p=2", "p=3"),
+ Lists.newArrayList("s"), 0, currentWriteIds);
+ assertEquals(1, stats.size());
+
+ // New reader.
+ currentWriteIds = msClient.getValidWriteIds(fqName).toString();
+ stats = msClient.getPartitionColumnStatistics(
+ dbName, tblName, Lists.newArrayList("p=1", "p=2", "p=3"),
+ Lists.newArrayList("s"), 0, currentWriteIds);
+ assertEquals(3, stats.size());
+
+ msClient.close();
+ }
+
@Test(timeout=40000)
public void testExistingOnly() throws Exception {
hiveConf.set(MetastoreConf.ConfVars.STATS_AUTO_UPDATE.getVarname(), "existing");
@@ -437,13 +581,19 @@ public class TestStatsUpdaterThread {
}
}
- private void verifyStatsUpToDate(String tbl, ArrayList<String> cols, IMetaStoreClient msClient,
+ private void verifyStatsUpToDate(String tbl, List<String> cols, IMetaStoreClient msClient,
boolean isUpToDate) throws Exception {
Table table = msClient.getTable(ss.getCurrentDatabase(), tbl);
verifyStatsUpToDate(table.getParameters(), cols, isUpToDate);
}
- private void verifyStatsUpToDate(Map<String, String> params, ArrayList<String> cols,
+ private void verifyStatsUpToDate(String tbl, List<String> cols, IMetaStoreClient msClient,
+ long txnId, String validWriteIds, boolean isUpToDate) throws Exception {
+ Table table = msClient.getTable(ss.getCurrentDatabase(), tbl, txnId, validWriteIds);
+ verifyStatsUpToDate(table.getParameters(), cols, isUpToDate);
+ }
+
+ private void verifyStatsUpToDate(Map<String, String> params, List<String> cols,
boolean isUpToDate) {
if (isUpToDate) {
assertTrue(StatsSetupConst.areBasicStatsUptoDate(params));
http://git-wip-us.apache.org/repos/asf/hive/blob/1c9947f3/ql/src/test/queries/clientpositive/acid_stats.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/acid_stats.q b/ql/src/test/queries/clientpositive/acid_stats.q
index 1e1c9b0..15eb930 100644
--- a/ql/src/test/queries/clientpositive/acid_stats.q
+++ b/ql/src/test/queries/clientpositive/acid_stats.q
@@ -35,8 +35,9 @@ drop table stats_part;
create table stats2(key int,value string) tblproperties ("transactional"="true", "transactional_properties"="insert_only");
insert into table stats2 values (1, "foo");
explain select count(*) from stats2;
-insert into table stats2 values (1, "bar");
+insert into table stats2 values (2, "bar");
explain select count(*) from stats2;
+desc formatted stats2 key;
set hive.stats.autogather=false;
set hive.stats.column.autogather=false;
http://git-wip-us.apache.org/repos/asf/hive/blob/1c9947f3/ql/src/test/queries/clientpositive/acid_stats3.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/acid_stats3.q b/ql/src/test/queries/clientpositive/acid_stats3.q
new file mode 100644
index 0000000..7a7bf19
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/acid_stats3.q
@@ -0,0 +1,56 @@
+set hive.stats.dbclass=fs;
+set hive.stats.fetch.column.stats=true;
+set datanucleus.cache.collections=false;
+
+set hive.merge.mapfiles=false;
+set hive.merge.mapredfiles=false;
+
+set hive.stats.autogather=true;
+set hive.stats.column.autogather=true;
+set hive.compute.query.using.stats=true;
+set hive.mapred.mode=nonstrict;
+set hive.explain.user=false;
+
+set hive.fetch.task.conversion=none;
+set hive.support.concurrency=true;
+set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
+set hive.query.results.cache.enabled=false;
+
+-- test truncate
+
+create table stats_nonpart(key int,value string) tblproperties ("transactional"="true", "transactional_properties"="insert_only");
+insert into table stats_nonpart values (1, "foo");
+explain select count(key) from stats_nonpart;
+
+truncate table stats_nonpart;
+explain select count(key) from stats_nonpart;
+
+analyze table stats_nonpart compute statistics for columns;
+explain select count(key) from stats_nonpart;
+drop table stats_nonpart;
+
+
+
+create table stats_part(key int,value string) partitioned by (p int) tblproperties ("transactional"="true", "transactional_properties"="insert_only");
+insert into table stats_part partition(p=101) values (1, "foo");
+insert into table stats_part partition(p=102) values (2, "bar");
+insert into table stats_part partition(p=103) values (3, "baz");
+explain select count(key) from stats_part where p = 101;
+
+truncate table stats_part partition(p=101);
+explain select count(key) from stats_part where p = 102;
+explain select count(key) from stats_part;
+
+alter table stats_part drop partition (p=101);
+explain select count(key) from stats_part;
+
+truncate table stats_part partition(p=102);
+analyze table stats_part partition(p) compute statistics for columns;
+
+-- Note: this currently doesn't work from stats - for ACID tables or otherwise.
+
+explain select count(key) from stats_part;
+drop table stats_part;
+
+
+
http://git-wip-us.apache.org/repos/asf/hive/blob/1c9947f3/ql/src/test/queries/clientpositive/acid_stats4.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/acid_stats4.q b/ql/src/test/queries/clientpositive/acid_stats4.q
new file mode 100644
index 0000000..20d1159
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/acid_stats4.q
@@ -0,0 +1,70 @@
+set hive.stats.dbclass=fs;
+set hive.stats.fetch.column.stats=true;
+set datanucleus.cache.collections=false;
+
+set hive.merge.mapfiles=false;
+set hive.merge.mapredfiles=false;
+
+set hive.stats.autogather=true;
+set hive.stats.column.autogather=true;
+set hive.compute.query.using.stats=true;
+set hive.mapred.mode=nonstrict;
+set hive.explain.user=false;
+
+set hive.fetch.task.conversion=none;
+set hive.support.concurrency=true;
+set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
+set hive.query.results.cache.enabled=false;
+
+-- test various alter commands
+
+create table stats_nonpart(key int,value string) tblproperties ("transactional"="true", "transactional_properties"="insert_only");
+insert into table stats_nonpart values (1, "foo");
+explain select count(key) from stats_nonpart;
+
+ALTER TABLE stats_nonpart CHANGE COLUMN key key2 int;
+explain select count(key2) from stats_nonpart;
+explain select count(value) from stats_nonpart;
+
+analyze table stats_nonpart compute statistics for columns;
+explain select count(key2) from stats_nonpart;
+
+alter table stats_nonpart rename to stats_nonpart2;
+explain select count(key2) from stats_nonpart2;
+
+analyze table stats_nonpart2 compute statistics for columns;
+explain select count(key2) from stats_nonpart2;
+
+drop table stats_nonpart;
+
+
+create table stats_part(key int,value string) partitioned by (p int) tblproperties ("transactional"="true", "transactional_properties"="insert_only");
+insert into table stats_part partition(p=101) values (1, "foo");
+insert into table stats_part partition(p=102) values (2, "bar");
+insert into table stats_part partition(p=103) values (3, "baz");
+
+alter table stats_part partition column (p decimal(10,0));
+explain select count(key) from stats_part;
+
+analyze table stats_part partition(p) compute statistics for columns;
+explain select count(key) from stats_part;
+
+alter table stats_part partition(p=102) rename to partition (p=104);
+explain select count(key) from stats_part where p = 101;
+explain select count(key) from stats_part;
+
+analyze table stats_part partition(p) compute statistics for columns;
+explain select count(key) from stats_part;
+
+ALTER TABLE stats_part CHANGE COLUMN key key2 int;
+explain select count(key2) from stats_part;
+explain select count(value) from stats_part;
+
+analyze table stats_part partition(p) compute statistics for columns;
+explain select count(key2) from stats_part;
+
+
+drop table stats_part;
+
+
+
http://git-wip-us.apache.org/repos/asf/hive/blob/1c9947f3/ql/src/test/results/clientpositive/acid_stats.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/acid_stats.q.out b/ql/src/test/results/clientpositive/acid_stats.q.out
index fd4ebe7..8dcfdfb 100644
--- a/ql/src/test/results/clientpositive/acid_stats.q.out
+++ b/ql/src/test/results/clientpositive/acid_stats.q.out
@@ -116,11 +116,11 @@ STAGE PLANS:
Processor Tree:
ListSink
-PREHOOK: query: insert into table stats2 values (1, "bar")
+PREHOOK: query: insert into table stats2 values (2, "bar")
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: default@stats2
-POSTHOOK: query: insert into table stats2 values (1, "bar")
+POSTHOOK: query: insert into table stats2 values (2, "bar")
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@stats2
@@ -140,6 +140,25 @@ STAGE PLANS:
Processor Tree:
ListSink
+PREHOOK: query: desc formatted stats2 key
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@stats2
+POSTHOOK: query: desc formatted stats2 key
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@stats2
+col_name key
+data_type int
+min 1
+max 2
+num_nulls 0
+distinct_count 2
+avg_col_len
+max_col_len
+num_trues
+num_falses
+bitVector HL
+comment from deserializer
+COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}}
PREHOOK: query: insert into table stats2 values (1, "baz")
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
http://git-wip-us.apache.org/repos/asf/hive/blob/1c9947f3/ql/src/test/results/clientpositive/acid_stats3.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/acid_stats3.q.out b/ql/src/test/results/clientpositive/acid_stats3.q.out
new file mode 100644
index 0000000..ef4d1c8
--- /dev/null
+++ b/ql/src/test/results/clientpositive/acid_stats3.q.out
@@ -0,0 +1,343 @@
+PREHOOK: query: create table stats_nonpart(key int,value string) tblproperties ("transactional"="true", "transactional_properties"="insert_only")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@stats_nonpart
+POSTHOOK: query: create table stats_nonpart(key int,value string) tblproperties ("transactional"="true", "transactional_properties"="insert_only")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@stats_nonpart
+PREHOOK: query: insert into table stats_nonpart values (1, "foo")
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@stats_nonpart
+POSTHOOK: query: insert into table stats_nonpart values (1, "foo")
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@stats_nonpart
+POSTHOOK: Lineage: stats_nonpart.key SCRIPT []
+POSTHOOK: Lineage: stats_nonpart.value SCRIPT []
+PREHOOK: query: explain select count(key) from stats_nonpart
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select count(key) from stats_nonpart
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-0
+ Fetch Operator
+ limit: 1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: truncate table stats_nonpart
+PREHOOK: type: TRUNCATETABLE
+PREHOOK: Output: default@stats_nonpart
+POSTHOOK: query: truncate table stats_nonpart
+POSTHOOK: type: TRUNCATETABLE
+POSTHOOK: Output: default@stats_nonpart
+PREHOOK: query: explain select count(key) from stats_nonpart
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select count(key) from stats_nonpart
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: stats_nonpart
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: key (type: int)
+ outputColumnNames: key
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ aggregations: count(key)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col0 (type: bigint)
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: analyze table stats_nonpart compute statistics for columns
+PREHOOK: type: ANALYZE_TABLE
+PREHOOK: Input: default@stats_nonpart
+PREHOOK: Output: default@stats_nonpart
+#### A masked pattern was here ####
+POSTHOOK: query: analyze table stats_nonpart compute statistics for columns
+POSTHOOK: type: ANALYZE_TABLE
+POSTHOOK: Input: default@stats_nonpart
+POSTHOOK: Output: default@stats_nonpart
+#### A masked pattern was here ####
+PREHOOK: query: explain select count(key) from stats_nonpart
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select count(key) from stats_nonpart
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-0
+ Fetch Operator
+ limit: 1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: drop table stats_nonpart
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@stats_nonpart
+PREHOOK: Output: default@stats_nonpart
+POSTHOOK: query: drop table stats_nonpart
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@stats_nonpart
+POSTHOOK: Output: default@stats_nonpart
+PREHOOK: query: create table stats_part(key int,value string) partitioned by (p int) tblproperties ("transactional"="true", "transactional_properties"="insert_only")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@stats_part
+POSTHOOK: query: create table stats_part(key int,value string) partitioned by (p int) tblproperties ("transactional"="true", "transactional_properties"="insert_only")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@stats_part
+PREHOOK: query: insert into table stats_part partition(p=101) values (1, "foo")
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@stats_part@p=101
+POSTHOOK: query: insert into table stats_part partition(p=101) values (1, "foo")
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@stats_part@p=101
+POSTHOOK: Lineage: stats_part PARTITION(p=101).key SCRIPT []
+POSTHOOK: Lineage: stats_part PARTITION(p=101).value SCRIPT []
+PREHOOK: query: insert into table stats_part partition(p=102) values (2, "bar")
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@stats_part@p=102
+POSTHOOK: query: insert into table stats_part partition(p=102) values (2, "bar")
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@stats_part@p=102
+POSTHOOK: Lineage: stats_part PARTITION(p=102).key SCRIPT []
+POSTHOOK: Lineage: stats_part PARTITION(p=102).value SCRIPT []
+PREHOOK: query: insert into table stats_part partition(p=103) values (3, "baz")
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@stats_part@p=103
+POSTHOOK: query: insert into table stats_part partition(p=103) values (3, "baz")
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@stats_part@p=103
+POSTHOOK: Lineage: stats_part PARTITION(p=103).key SCRIPT []
+POSTHOOK: Lineage: stats_part PARTITION(p=103).value SCRIPT []
+PREHOOK: query: explain select count(key) from stats_part where p = 101
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select count(key) from stats_part where p = 101
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-0
+ Fetch Operator
+ limit: 1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: truncate table stats_part partition(p=101)
+PREHOOK: type: TRUNCATETABLE
+PREHOOK: Output: default@stats_part@p=101
+POSTHOOK: query: truncate table stats_part partition(p=101)
+POSTHOOK: type: TRUNCATETABLE
+POSTHOOK: Output: default@stats_part@p=101
+PREHOOK: query: explain select count(key) from stats_part where p = 102
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select count(key) from stats_part where p = 102
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-0
+ Fetch Operator
+ limit: 1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: explain select count(key) from stats_part
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select count(key) from stats_part
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: stats_part
+ Statistics: Num rows: 2 Data size: 8 Basic stats: PARTIAL Column stats: PARTIAL
+ Select Operator
+ expressions: key (type: int)
+ outputColumnNames: key
+ Statistics: Num rows: 2 Data size: 8 Basic stats: PARTIAL Column stats: PARTIAL
+ Group By Operator
+ aggregations: count(key)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 12 Basic stats: PARTIAL Column stats: PARTIAL
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 12 Basic stats: PARTIAL Column stats: PARTIAL
+ value expressions: _col0 (type: bigint)
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 12 Basic stats: PARTIAL Column stats: PARTIAL
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 12 Basic stats: PARTIAL Column stats: PARTIAL
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: alter table stats_part drop partition (p=101)
+PREHOOK: type: ALTERTABLE_DROPPARTS
+PREHOOK: Input: default@stats_part
+PREHOOK: Output: default@stats_part@p=101
+POSTHOOK: query: alter table stats_part drop partition (p=101)
+POSTHOOK: type: ALTERTABLE_DROPPARTS
+POSTHOOK: Input: default@stats_part
+POSTHOOK: Output: default@stats_part@p=101
+PREHOOK: query: explain select count(key) from stats_part
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select count(key) from stats_part
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-0
+ Fetch Operator
+ limit: 1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: truncate table stats_part partition(p=102)
+PREHOOK: type: TRUNCATETABLE
+PREHOOK: Output: default@stats_part@p=102
+POSTHOOK: query: truncate table stats_part partition(p=102)
+POSTHOOK: type: TRUNCATETABLE
+POSTHOOK: Output: default@stats_part@p=102
+PREHOOK: query: analyze table stats_part partition(p) compute statistics for columns
+PREHOOK: type: ANALYZE_TABLE
+PREHOOK: Input: default@stats_part
+PREHOOK: Input: default@stats_part@p=102
+PREHOOK: Input: default@stats_part@p=103
+PREHOOK: Output: default@stats_part
+PREHOOK: Output: default@stats_part@p=102
+PREHOOK: Output: default@stats_part@p=103
+#### A masked pattern was here ####
+POSTHOOK: query: analyze table stats_part partition(p) compute statistics for columns
+POSTHOOK: type: ANALYZE_TABLE
+POSTHOOK: Input: default@stats_part
+POSTHOOK: Input: default@stats_part@p=102
+POSTHOOK: Input: default@stats_part@p=103
+POSTHOOK: Output: default@stats_part
+POSTHOOK: Output: default@stats_part@p=102
+POSTHOOK: Output: default@stats_part@p=103
+#### A masked pattern was here ####
+PREHOOK: query: explain select count(key) from stats_part
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select count(key) from stats_part
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: stats_part
+ Statistics: Num rows: 1 Data size: 4 Basic stats: PARTIAL Column stats: PARTIAL
+ Select Operator
+ expressions: key (type: int)
+ outputColumnNames: key
+ Statistics: Num rows: 1 Data size: 4 Basic stats: PARTIAL Column stats: PARTIAL
+ Group By Operator
+ aggregations: count(key)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 12 Basic stats: PARTIAL Column stats: PARTIAL
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 12 Basic stats: PARTIAL Column stats: PARTIAL
+ value expressions: _col0 (type: bigint)
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 12 Basic stats: PARTIAL Column stats: PARTIAL
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 12 Basic stats: PARTIAL Column stats: PARTIAL
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: drop table stats_part
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@stats_part
+PREHOOK: Output: default@stats_part
+POSTHOOK: query: drop table stats_part
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@stats_part
+POSTHOOK: Output: default@stats_part
http://git-wip-us.apache.org/repos/asf/hive/blob/1c9947f3/ql/src/test/results/clientpositive/acid_stats4.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/acid_stats4.q.out b/ql/src/test/results/clientpositive/acid_stats4.q.out
new file mode 100644
index 0000000..bfb8898
--- /dev/null
+++ b/ql/src/test/results/clientpositive/acid_stats4.q.out
@@ -0,0 +1,496 @@
+PREHOOK: query: create table stats_nonpart(key int,value string) tblproperties ("transactional"="true", "transactional_properties"="insert_only")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@stats_nonpart
+POSTHOOK: query: create table stats_nonpart(key int,value string) tblproperties ("transactional"="true", "transactional_properties"="insert_only")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@stats_nonpart
+PREHOOK: query: insert into table stats_nonpart values (1, "foo")
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@stats_nonpart
+POSTHOOK: query: insert into table stats_nonpart values (1, "foo")
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@stats_nonpart
+POSTHOOK: Lineage: stats_nonpart.key SCRIPT []
+POSTHOOK: Lineage: stats_nonpart.value SCRIPT []
+PREHOOK: query: explain select count(key) from stats_nonpart
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select count(key) from stats_nonpart
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-0
+ Fetch Operator
+ limit: 1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: ALTER TABLE stats_nonpart CHANGE COLUMN key key2 int
+PREHOOK: type: ALTERTABLE_RENAMECOL
+PREHOOK: Input: default@stats_nonpart
+PREHOOK: Output: default@stats_nonpart
+POSTHOOK: query: ALTER TABLE stats_nonpart CHANGE COLUMN key key2 int
+POSTHOOK: type: ALTERTABLE_RENAMECOL
+POSTHOOK: Input: default@stats_nonpart
+POSTHOOK: Output: default@stats_nonpart
+PREHOOK: query: explain select count(key2) from stats_nonpart
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select count(key2) from stats_nonpart
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: stats_nonpart
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key2 (type: int)
+ outputColumnNames: key2
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count(key2)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: explain select count(value) from stats_nonpart
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select count(value) from stats_nonpart
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-0
+ Fetch Operator
+ limit: 1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: analyze table stats_nonpart compute statistics for columns
+PREHOOK: type: ANALYZE_TABLE
+PREHOOK: Input: default@stats_nonpart
+PREHOOK: Output: default@stats_nonpart
+#### A masked pattern was here ####
+POSTHOOK: query: analyze table stats_nonpart compute statistics for columns
+POSTHOOK: type: ANALYZE_TABLE
+POSTHOOK: Input: default@stats_nonpart
+POSTHOOK: Output: default@stats_nonpart
+#### A masked pattern was here ####
+PREHOOK: query: explain select count(key2) from stats_nonpart
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select count(key2) from stats_nonpart
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-0
+ Fetch Operator
+ limit: 1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: alter table stats_nonpart rename to stats_nonpart2
+PREHOOK: type: ALTERTABLE_RENAME
+PREHOOK: Input: default@stats_nonpart
+PREHOOK: Output: default@stats_nonpart
+POSTHOOK: query: alter table stats_nonpart rename to stats_nonpart2
+POSTHOOK: type: ALTERTABLE_RENAME
+POSTHOOK: Input: default@stats_nonpart
+POSTHOOK: Output: default@stats_nonpart
+POSTHOOK: Output: default@stats_nonpart2
+PREHOOK: query: explain select count(key2) from stats_nonpart2
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select count(key2) from stats_nonpart2
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-0
+ Fetch Operator
+ limit: 1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: analyze table stats_nonpart2 compute statistics for columns
+PREHOOK: type: ANALYZE_TABLE
+PREHOOK: Input: default@stats_nonpart2
+PREHOOK: Output: default@stats_nonpart2
+#### A masked pattern was here ####
+POSTHOOK: query: analyze table stats_nonpart2 compute statistics for columns
+POSTHOOK: type: ANALYZE_TABLE
+POSTHOOK: Input: default@stats_nonpart2
+POSTHOOK: Output: default@stats_nonpart2
+#### A masked pattern was here ####
+PREHOOK: query: explain select count(key2) from stats_nonpart2
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select count(key2) from stats_nonpart2
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-0
+ Fetch Operator
+ limit: 1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: drop table stats_nonpart
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table stats_nonpart
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table stats_part(key int,value string) partitioned by (p int) tblproperties ("transactional"="true", "transactional_properties"="insert_only")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@stats_part
+POSTHOOK: query: create table stats_part(key int,value string) partitioned by (p int) tblproperties ("transactional"="true", "transactional_properties"="insert_only")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@stats_part
+PREHOOK: query: insert into table stats_part partition(p=101) values (1, "foo")
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@stats_part@p=101
+POSTHOOK: query: insert into table stats_part partition(p=101) values (1, "foo")
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@stats_part@p=101
+POSTHOOK: Lineage: stats_part PARTITION(p=101).key SCRIPT []
+POSTHOOK: Lineage: stats_part PARTITION(p=101).value SCRIPT []
+PREHOOK: query: insert into table stats_part partition(p=102) values (2, "bar")
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@stats_part@p=102
+POSTHOOK: query: insert into table stats_part partition(p=102) values (2, "bar")
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@stats_part@p=102
+POSTHOOK: Lineage: stats_part PARTITION(p=102).key SCRIPT []
+POSTHOOK: Lineage: stats_part PARTITION(p=102).value SCRIPT []
+PREHOOK: query: insert into table stats_part partition(p=103) values (3, "baz")
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@stats_part@p=103
+POSTHOOK: query: insert into table stats_part partition(p=103) values (3, "baz")
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@stats_part@p=103
+POSTHOOK: Lineage: stats_part PARTITION(p=103).key SCRIPT []
+POSTHOOK: Lineage: stats_part PARTITION(p=103).value SCRIPT []
+PREHOOK: query: alter table stats_part partition column (p decimal(10,0))
+PREHOOK: type: ALTERTABLE_PARTCOLTYPE
+PREHOOK: Input: default@stats_part
+POSTHOOK: query: alter table stats_part partition column (p decimal(10,0))
+POSTHOOK: type: ALTERTABLE_PARTCOLTYPE
+POSTHOOK: Input: default@stats_part
+POSTHOOK: Output: default@stats_part
+PREHOOK: query: explain select count(key) from stats_part
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select count(key) from stats_part
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-0
+ Fetch Operator
+ limit: 1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: analyze table stats_part partition(p) compute statistics for columns
+PREHOOK: type: ANALYZE_TABLE
+PREHOOK: Input: default@stats_part
+PREHOOK: Input: default@stats_part@p=101
+PREHOOK: Input: default@stats_part@p=102
+PREHOOK: Input: default@stats_part@p=103
+PREHOOK: Output: default@stats_part
+PREHOOK: Output: default@stats_part@p=101
+PREHOOK: Output: default@stats_part@p=102
+PREHOOK: Output: default@stats_part@p=103
+#### A masked pattern was here ####
+POSTHOOK: query: analyze table stats_part partition(p) compute statistics for columns
+POSTHOOK: type: ANALYZE_TABLE
+POSTHOOK: Input: default@stats_part
+POSTHOOK: Input: default@stats_part@p=101
+POSTHOOK: Input: default@stats_part@p=102
+POSTHOOK: Input: default@stats_part@p=103
+POSTHOOK: Output: default@stats_part
+POSTHOOK: Output: default@stats_part@p=101
+POSTHOOK: Output: default@stats_part@p=102
+POSTHOOK: Output: default@stats_part@p=103
+#### A masked pattern was here ####
+PREHOOK: query: explain select count(key) from stats_part
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select count(key) from stats_part
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-0
+ Fetch Operator
+ limit: 1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: alter table stats_part partition(p=102) rename to partition (p=104)
+PREHOOK: type: ALTERTABLE_RENAMEPART
+PREHOOK: Input: default@stats_part
+PREHOOK: Output: default@stats_part@p=102
+POSTHOOK: query: alter table stats_part partition(p=102) rename to partition (p=104)
+POSTHOOK: type: ALTERTABLE_RENAMEPART
+POSTHOOK: Input: default@stats_part
+POSTHOOK: Input: default@stats_part@p=102
+POSTHOOK: Output: default@stats_part@p=102
+POSTHOOK: Output: default@stats_part@p=104
+PREHOOK: query: explain select count(key) from stats_part where p = 101
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select count(key) from stats_part where p = 101
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-0
+ Fetch Operator
+ limit: 1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: explain select count(key) from stats_part
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select count(key) from stats_part
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: stats_part
+ Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: key (type: int)
+ outputColumnNames: key
+ Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ aggregations: count(key)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col0 (type: bigint)
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: analyze table stats_part partition(p) compute statistics for columns
+PREHOOK: type: ANALYZE_TABLE
+PREHOOK: Input: default@stats_part
+PREHOOK: Input: default@stats_part@p=101
+PREHOOK: Input: default@stats_part@p=103
+PREHOOK: Input: default@stats_part@p=104
+PREHOOK: Output: default@stats_part
+PREHOOK: Output: default@stats_part@p=101
+PREHOOK: Output: default@stats_part@p=103
+PREHOOK: Output: default@stats_part@p=104
+#### A masked pattern was here ####
+POSTHOOK: query: analyze table stats_part partition(p) compute statistics for columns
+POSTHOOK: type: ANALYZE_TABLE
+POSTHOOK: Input: default@stats_part
+POSTHOOK: Input: default@stats_part@p=101
+POSTHOOK: Input: default@stats_part@p=103
+POSTHOOK: Input: default@stats_part@p=104
+POSTHOOK: Output: default@stats_part
+POSTHOOK: Output: default@stats_part@p=101
+POSTHOOK: Output: default@stats_part@p=103
+POSTHOOK: Output: default@stats_part@p=104
+#### A masked pattern was here ####
+PREHOOK: query: explain select count(key) from stats_part
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select count(key) from stats_part
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-0
+ Fetch Operator
+ limit: 1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: ALTER TABLE stats_part CHANGE COLUMN key key2 int
+PREHOOK: type: ALTERTABLE_RENAMECOL
+PREHOOK: Input: default@stats_part
+PREHOOK: Output: default@stats_part
+POSTHOOK: query: ALTER TABLE stats_part CHANGE COLUMN key key2 int
+POSTHOOK: type: ALTERTABLE_RENAMECOL
+POSTHOOK: Input: default@stats_part
+POSTHOOK: Output: default@stats_part
+PREHOOK: query: explain select count(key2) from stats_part
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select count(key2) from stats_part
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: stats_part
+ Statistics: Num rows: 3 Data size: 27 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key2 (type: int)
+ outputColumnNames: key2
+ Statistics: Num rows: 3 Data size: 27 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count(key2)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: explain select count(value) from stats_part
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select count(value) from stats_part
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-0
+ Fetch Operator
+ limit: 1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: analyze table stats_part partition(p) compute statistics for columns
+PREHOOK: type: ANALYZE_TABLE
+PREHOOK: Input: default@stats_part
+PREHOOK: Input: default@stats_part@p=101
+PREHOOK: Input: default@stats_part@p=103
+PREHOOK: Input: default@stats_part@p=104
+PREHOOK: Output: default@stats_part
+PREHOOK: Output: default@stats_part@p=101
+PREHOOK: Output: default@stats_part@p=103
+PREHOOK: Output: default@stats_part@p=104
+#### A masked pattern was here ####
+POSTHOOK: query: analyze table stats_part partition(p) compute statistics for columns
+POSTHOOK: type: ANALYZE_TABLE
+POSTHOOK: Input: default@stats_part
+POSTHOOK: Input: default@stats_part@p=101
+POSTHOOK: Input: default@stats_part@p=103
+POSTHOOK: Input: default@stats_part@p=104
+POSTHOOK: Output: default@stats_part
+POSTHOOK: Output: default@stats_part@p=101
+POSTHOOK: Output: default@stats_part@p=103
+POSTHOOK: Output: default@stats_part@p=104
+#### A masked pattern was here ####
+PREHOOK: query: explain select count(key2) from stats_part
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select count(key2) from stats_part
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-0
+ Fetch Operator
+ limit: 1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: drop table stats_part
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@stats_part
+PREHOOK: Output: default@stats_part
+POSTHOOK: query: drop table stats_part
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@stats_part
+POSTHOOK: Output: default@stats_part
http://git-wip-us.apache.org/repos/asf/hive/blob/1c9947f3/ql/src/test/results/clientpositive/acid_table_stats.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/acid_table_stats.q.out b/ql/src/test/results/clientpositive/acid_table_stats.q.out
index fb064f8..f3ada34 100644
--- a/ql/src/test/results/clientpositive/acid_table_stats.q.out
+++ b/ql/src/test/results/clientpositive/acid_table_stats.q.out
@@ -321,7 +321,7 @@ Partition Parameters:
numFiles 4
numRows 2000
rawDataSize 208000
- totalSize 8118
+ totalSize 8126
#### A masked pattern was here ####
# Storage Information
@@ -368,7 +368,7 @@ Partition Parameters:
numFiles 4
numRows 2000
rawDataSize 416000
- totalSize 8118
+ totalSize 8126
#### A masked pattern was here ####
# Storage Information
http://git-wip-us.apache.org/repos/asf/hive/blob/1c9947f3/ql/src/test/results/clientpositive/llap/change_allowincompatible_vectorization_false_date.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/change_allowincompatible_vectorization_false_date.q.out b/ql/src/test/results/clientpositive/llap/change_allowincompatible_vectorization_false_date.q.out
index 1bba3f3..4661a03 100644
--- a/ql/src/test/results/clientpositive/llap/change_allowincompatible_vectorization_false_date.q.out
+++ b/ql/src/test/results/clientpositive/llap/change_allowincompatible_vectorization_false_date.q.out
@@ -41,14 +41,10 @@ POSTHOOK: Lineage: change_allowincompatible_vectorization_false_date PARTITION(s
PREHOOK: query: select count(*) from change_allowincompatible_vectorization_false_date
PREHOOK: type: QUERY
PREHOOK: Input: default@change_allowincompatible_vectorization_false_date
-PREHOOK: Input: default@change_allowincompatible_vectorization_false_date@s=aaa
-PREHOOK: Input: default@change_allowincompatible_vectorization_false_date@s=bbb
#### A masked pattern was here ####
POSTHOOK: query: select count(*) from change_allowincompatible_vectorization_false_date
POSTHOOK: type: QUERY
POSTHOOK: Input: default@change_allowincompatible_vectorization_false_date
-POSTHOOK: Input: default@change_allowincompatible_vectorization_false_date@s=aaa
-POSTHOOK: Input: default@change_allowincompatible_vectorization_false_date@s=bbb
#### A masked pattern was here ####
50
PREHOOK: query: alter table change_allowincompatible_vectorization_false_date change column ts ts timestamp
@@ -62,14 +58,10 @@ POSTHOOK: Output: default@change_allowincompatible_vectorization_false_date
PREHOOK: query: select count(*) from change_allowincompatible_vectorization_false_date
PREHOOK: type: QUERY
PREHOOK: Input: default@change_allowincompatible_vectorization_false_date
-PREHOOK: Input: default@change_allowincompatible_vectorization_false_date@s=aaa
-PREHOOK: Input: default@change_allowincompatible_vectorization_false_date@s=bbb
#### A masked pattern was here ####
POSTHOOK: query: select count(*) from change_allowincompatible_vectorization_false_date
POSTHOOK: type: QUERY
POSTHOOK: Input: default@change_allowincompatible_vectorization_false_date
-POSTHOOK: Input: default@change_allowincompatible_vectorization_false_date@s=aaa
-POSTHOOK: Input: default@change_allowincompatible_vectorization_false_date@s=bbb
#### A masked pattern was here ####
50
PREHOOK: query: insert into table change_allowincompatible_vectorization_false_date partition (s='aaa') values ('2038-03-22 07:26:48.0')
http://git-wip-us.apache.org/repos/asf/hive/blob/1c9947f3/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization_acid.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization_acid.q.out b/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization_acid.q.out
index 2ad9a4d..5018a4c 100644
--- a/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization_acid.q.out
+++ b/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization_acid.q.out
@@ -1327,7 +1327,7 @@ STAGE PLANS:
TableScan
alias: acid_2l_part_sdpo
filterExpr: (value = 'bar') (type: boolean)
- Statistics: Num rows: 4952 Data size: 1456618 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 4200 Data size: 1253037 Basic stats: COMPLETE Column stats: PARTIAL
Filter Operator
predicate: (value = 'bar') (type: boolean)
Statistics: Num rows: 5 Data size: 1375 Basic stats: COMPLETE Column stats: PARTIAL
http://git-wip-us.apache.org/repos/asf/hive/blob/1c9947f3/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_3.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_3.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_3.q.out
index 0628ad8..2538ac1 100644
--- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_3.q.out
+++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_3.q.out
@@ -488,10 +488,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: cmv_basetable_2
- filterExpr: ((c > 10) and (ROW__ID.writeid > 1) and a is not null) (type: boolean)
+ filterExpr: ((c > 10) and (ROW__ID.writeid > 2) and a is not null) (type: boolean)
Statistics: Num rows: 3 Data size: 348 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
- predicate: ((ROW__ID.writeid > 1) and (c > 10) and a is not null) (type: boolean)
+ predicate: ((ROW__ID.writeid > 2) and (c > 10) and a is not null) (type: boolean)
Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: a (type: int), c (type: decimal(10,2))
http://git-wip-us.apache.org/repos/asf/hive/blob/1c9947f3/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_4.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_4.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_4.q.out
index a804637..fe46bfd 100644
--- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_4.q.out
+++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_4.q.out
@@ -730,10 +730,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: cmv_basetable_2_n2
- filterExpr: ((c > 10) and (ROW__ID.writeid > 1) and a is not null) (type: boolean)
+ filterExpr: ((c > 10) and (ROW__ID.writeid > 2) and a is not null) (type: boolean)
Statistics: Num rows: 3 Data size: 360 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
- predicate: ((ROW__ID.writeid > 1) and (c > 10) and a is not null) (type: boolean)
+ predicate: ((ROW__ID.writeid > 2) and (c > 10) and a is not null) (type: boolean)
Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: a (type: int), c (type: decimal(10,2)), d (type: int)
@@ -950,7 +950,7 @@ Table Parameters:
numFiles 3
numRows 3
rawDataSize 248
- totalSize 1508
+ totalSize 1500
transactional true
transactional_properties default
#### A masked pattern was here ####
@@ -1554,10 +1554,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: cmv_basetable_2_n2
- filterExpr: ((c > 10) and (ROW__ID.writeid > 4) and a is not null) (type: boolean)
+ filterExpr: ((c > 10) and (ROW__ID.writeid > 6) and a is not null) (type: boolean)
Statistics: Num rows: 3 Data size: 360 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
- predicate: ((ROW__ID.writeid > 4) and (c > 10) and a is not null) (type: boolean)
+ predicate: ((ROW__ID.writeid > 6) and (c > 10) and a is not null) (type: boolean)
Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: a (type: int), c (type: decimal(10,2)), d (type: int)
http://git-wip-us.apache.org/repos/asf/hive/blob/1c9947f3/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_5.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_5.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_5.q.out
index fae4757..6402db8 100644
--- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_5.q.out
+++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_5.q.out
@@ -278,10 +278,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: cmv_basetable_2_n3
- filterExpr: ((c > 10) and (ROW__ID.writeid > 1) and a is not null) (type: boolean)
+ filterExpr: ((c > 10) and (ROW__ID.writeid > 2) and a is not null) (type: boolean)
Statistics: Num rows: 3 Data size: 348 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
- predicate: ((ROW__ID.writeid > 1) and (c > 10) and a is not null) (type: boolean)
+ predicate: ((ROW__ID.writeid > 2) and (c > 10) and a is not null) (type: boolean)
Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: a (type: int), c (type: decimal(10,2))
@@ -958,10 +958,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: cmv_basetable_2_n3
- filterExpr: ((c > 10) and (ROW__ID.writeid > 4) and a is not null) (type: boolean)
+ filterExpr: ((c > 10) and (ROW__ID.writeid > 6) and a is not null) (type: boolean)
Statistics: Num rows: 3 Data size: 348 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
- predicate: ((ROW__ID.writeid > 4) and (c > 10) and a is not null) (type: boolean)
+ predicate: ((ROW__ID.writeid > 6) and (c > 10) and a is not null) (type: boolean)
Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: a (type: int), c (type: decimal(10,2))
http://git-wip-us.apache.org/repos/asf/hive/blob/1c9947f3/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_rebuild_dummy.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_rebuild_dummy.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_rebuild_dummy.q.out
index fe54771..193e959 100644
--- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_rebuild_dummy.q.out
+++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_rebuild_dummy.q.out
@@ -488,10 +488,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: cmv_basetable_2_n0
- filterExpr: ((c > 10) and (ROW__ID.writeid > 1) and a is not null) (type: boolean)
+ filterExpr: ((c > 10) and (ROW__ID.writeid > 2) and a is not null) (type: boolean)
Statistics: Num rows: 3 Data size: 348 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
- predicate: ((ROW__ID.writeid > 1) and (c > 10) and a is not null) (type: boolean)
+ predicate: ((ROW__ID.writeid > 2) and (c > 10) and a is not null) (type: boolean)
Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: a (type: int), c (type: decimal(10,2))
http://git-wip-us.apache.org/repos/asf/hive/blob/1c9947f3/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_time_window.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_time_window.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_time_window.q.out
index 44a866d..b5183ba 100644
--- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_time_window.q.out
+++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_time_window.q.out
@@ -602,10 +602,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: cmv_basetable_2_n1
- filterExpr: ((c > 10) and (ROW__ID.writeid > 1) and a is not null) (type: boolean)
+ filterExpr: ((c > 10) and (ROW__ID.writeid > 2) and a is not null) (type: boolean)
Statistics: Num rows: 3 Data size: 348 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
- predicate: ((ROW__ID.writeid > 1) and (c > 10) and a is not null) (type: boolean)
+ predicate: ((ROW__ID.writeid > 2) and (c > 10) and a is not null) (type: boolean)
Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: a (type: int), c (type: decimal(10,2))