You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2018/06/21 17:53:11 UTC
[01/12] hive git commit: HIVE-19777 : NPE in TezSessionState (Sergey
Shelukhin, reviewed by Jason Dere)
Repository: hive
Updated Branches:
refs/heads/master-txnstats 1d46608e8 -> ac3f5186e
HIVE-19777 : NPE in TezSessionState (Sergey Shelukhin, reviewed by Jason Dere)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/cf2e1853
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/cf2e1853
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/cf2e1853
Branch: refs/heads/master-txnstats
Commit: cf2e185341cf517ae7c5d36047d345d086f4ecb5
Parents: 1a610cc
Author: sergey <se...@apache.org>
Authored: Mon Jun 18 15:15:38 2018 -0700
Committer: sergey <se...@apache.org>
Committed: Mon Jun 18 15:15:38 2018 -0700
----------------------------------------------------------------------
.../hive/ql/exec/tez/TezSessionState.java | 33 +++++++++++++++++---
1 file changed, 28 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/cf2e1853/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
index fe139c9..08e65a4 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
@@ -180,7 +180,11 @@ public class TezSessionState {
return false;
}
try {
- session = sessionFuture.get(0, TimeUnit.NANOSECONDS);
+ TezClient session = sessionFuture.get(0, TimeUnit.NANOSECONDS);
+ if (session == null) {
+ return false;
+ }
+ this.session = session;
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
return false;
@@ -202,7 +206,11 @@ public class TezSessionState {
return false;
}
try {
- session = sessionFuture.get(0, TimeUnit.NANOSECONDS);
+ TezClient session = sessionFuture.get(0, TimeUnit.NANOSECONDS);
+ if (session == null) {
+ return false;
+ }
+ this.session = session;
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
return false;
@@ -363,12 +371,23 @@ public class TezSessionState {
FutureTask<TezClient> sessionFuture = new FutureTask<>(new Callable<TezClient>() {
@Override
public TezClient call() throws Exception {
+ TezClient result = null;
try {
- return startSessionAndContainers(session, conf, commonLocalResources, tezConfig, true);
+ result = startSessionAndContainers(
+ session, conf, commonLocalResources, tezConfig, true);
} catch (Throwable t) {
+ // The caller has already stopped the session.
LOG.error("Failed to start Tez session", t);
throw (t instanceof Exception) ? (Exception)t : new Exception(t);
}
+ // Check interrupt at the last moment in case we get cancelled quickly.
+ // This is not bulletproof but should allow us to close session in most cases.
+ if (Thread.interrupted()) {
+ LOG.info("Interrupted while starting Tez session");
+ closeAndIgnoreExceptions(result);
+ return null;
+ }
+ return result;
}
});
new Thread(sessionFuture, "Tez session start thread").start();
@@ -471,7 +490,11 @@ public class TezSessionState {
return;
}
try {
- this.session = this.sessionFuture.get();
+ TezClient session = this.sessionFuture.get();
+ if (session == null) {
+ throw new RuntimeException("Initialization was interrupted");
+ }
+ this.session = session;
} catch (ExecutionException e) {
throw new RuntimeException(e);
}
@@ -645,7 +668,7 @@ public class TezSessionState {
appJarLr = null;
try {
- if (getSession() != null) {
+ if (session != null) {
LOG.info("Closing Tez Session");
closeClient(session);
session = null;
[04/12] hive git commit: HIVE-19920: Schematool fails in embedded
mode when auth is on (Daniel Dai, reviewed by Thejas Nair)
Posted by se...@apache.org.
HIVE-19920: Schematool fails in embedded mode when auth is on (Daniel Dai, reviewed by Thejas Nair)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/52f190e1
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/52f190e1
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/52f190e1
Branch: refs/heads/master-txnstats
Commit: 52f190e157a8413897529f1b7eb71b8613786d76
Parents: 2394e40
Author: Daniel Dai <da...@gmail.com>
Authored: Tue Jun 19 22:30:35 2018 -0700
Committer: Daniel Dai <da...@gmail.com>
Committed: Tue Jun 19 22:30:35 2018 -0700
----------------------------------------------------------------------
.../apache/hadoop/hive/metastore/tools/HiveSchemaHelper.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/52f190e1/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/HiveSchemaHelper.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/HiveSchemaHelper.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/HiveSchemaHelper.java
index 1dd97f2..f868982 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/HiveSchemaHelper.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/HiveSchemaHelper.java
@@ -45,9 +45,9 @@ public class HiveSchemaHelper {
public static final String DB_POSTGRACE = "postgres";
public static final String DB_ORACLE = "oracle";
public static final String EMBEDDED_HS2_URL =
- "jdbc:hive2://?hive.conf.restricted.list=;hive.security.authorization.sqlstd.confwhitelist=*;"
- + "hive.security.authorization.sqlstd.confwhitelist.append=*;hive.security.authorization.enabled=false;"
- + "hive.metastore.uris=";
+ "jdbc:hive2://?hive.conf.restricted.list=;hive.security.authorization.sqlstd.confwhitelist=.*;"
+ + "hive.security.authorization.sqlstd.confwhitelist.append=.*;hive.security.authorization.enabled=false;"
+ + "hive.metastore.uris=;hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdConfOnlyAuthorizerFactory";
public static final String HIVE_JDBC_DRIVER = "org.apache.hive.jdbc.HiveDriver";
/***
[10/12] hive git commit: HIVE-19892: Disable query results cache for
for HiveServer2 doAs=true (Jason Dere, reviewed by Sergey Shelukhin)
Posted by se...@apache.org.
HIVE-19892: Disable query results cache for for HiveServer2 doAs=true (Jason Dere, reviewed by Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/9cd3960d
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/9cd3960d
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/9cd3960d
Branch: refs/heads/master-txnstats
Commit: 9cd3960d0aebccae05682ef2f52a81affaf8c201
Parents: 596edd1
Author: Jason Dere <jd...@hortonworks.com>
Authored: Thu Jun 21 10:21:54 2018 -0700
Committer: Jason Dere <jd...@hortonworks.com>
Committed: Thu Jun 21 10:21:54 2018 -0700
----------------------------------------------------------------------
.../hadoop/hive/ql/cache/results/QueryResultsCache.java | 6 ++++++
.../org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java | 7 ++++++-
2 files changed, 12 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/9cd3960d/ql/src/java/org/apache/hadoop/hive/ql/cache/results/QueryResultsCache.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/cache/results/QueryResultsCache.java b/ql/src/java/org/apache/hadoop/hive/ql/cache/results/QueryResultsCache.java
index 11b9c27..d29c4da 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/cache/results/QueryResultsCache.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/cache/results/QueryResultsCache.java
@@ -540,6 +540,12 @@ public final class QueryResultsCache {
} else {
// No actual result directory, no need to move anything.
cachedResultsPath = zeroRowsPath;
+ // Even if there are no results to move, at least check that we have permission
+ // to check the existence of zeroRowsPath, or the read using the cache will fail.
+ // A failure here will cause this query to not be added to the cache.
+ FileSystem cacheFs = cachedResultsPath.getFileSystem(conf);
+ boolean fakePathExists = cacheFs.exists(zeroRowsPath);
+
resultSize = 0;
requiresMove = false;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/9cd3960d/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index c2bcedd..867ffe4 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -12130,7 +12130,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
// If no masking/filtering required, then we can check the cache now, before
// generating the operator tree and going through CBO.
// Otherwise we have to wait until after the masking/filtering step.
- boolean isCacheEnabled = conf.getBoolVar(HiveConf.ConfVars.HIVE_QUERY_RESULTS_CACHE_ENABLED);
+ boolean isCacheEnabled = isResultsCacheEnabled();
QueryResultsCache.LookupInfo lookupInfo = null;
boolean needsTransform = needsTransform();
if (isCacheEnabled && !needsTransform && queryTypeCanUseCache()) {
@@ -14658,6 +14658,11 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
return lookupInfo;
}
+ private boolean isResultsCacheEnabled() {
+ return conf.getBoolVar(HiveConf.ConfVars.HIVE_QUERY_RESULTS_CACHE_ENABLED) &&
+ !(SessionState.get().isHiveServerQuery() && conf.getBoolVar(HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS));
+ }
+
/**
* Set the query plan to use cache entry passed in to return the query results.
* @param cacheEntry The results cache entry that will be used to resolve the query.
[05/12] hive git commit: HIVE-19908 Block Insert Overwrite with Union
All on full CRUD ACID tables using HIVE_UNION_SUBDIR_ (Eugene Koifman,
reviewed by Prasanth Jayachandran)
Posted by se...@apache.org.
HIVE-19908 Block Insert Overwrite with Union All on full CRUD ACID tables using HIVE_UNION_SUBDIR_ (Eugene Koifman, reviewed by Prasanth Jayachandran)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/6c4d7a9b
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/6c4d7a9b
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/6c4d7a9b
Branch: refs/heads/master-txnstats
Commit: 6c4d7a9b945f1dad27ef1a6193a689040fee7b66
Parents: 52f190e
Author: Eugene Koifman <ek...@apache.org>
Authored: Wed Jun 20 09:01:52 2018 -0700
Committer: Eugene Koifman <ek...@apache.org>
Committed: Wed Jun 20 09:01:52 2018 -0700
----------------------------------------------------------------------
ql/src/java/org/apache/hadoop/hive/ql/Driver.java | 15 +++++++++++++--
.../apache/hadoop/hive/ql/plan/FileSinkDesc.java | 9 +++++++++
.../apache/hadoop/hive/ql/TestTxnNoBuckets.java | 18 ++++++++++++++++++
3 files changed, 40 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/6c4d7a9b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
index 43a78ca..762e57c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
@@ -71,6 +71,7 @@ import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
import org.apache.hadoop.hive.ql.cache.results.CacheUsage;
import org.apache.hadoop.hive.ql.cache.results.QueryResultsCache;
import org.apache.hadoop.hive.ql.cache.results.QueryResultsCache.CacheEntry;
+import org.apache.hadoop.hive.ql.exec.AbstractFileMergeOperator;
import org.apache.hadoop.hive.ql.exec.ConditionalTask;
import org.apache.hadoop.hive.ql.exec.DagUtils;
import org.apache.hadoop.hive.ql.exec.ExplainTask;
@@ -1581,9 +1582,19 @@ public class Driver implements IDriver {
Utilities.getTableName(tableInfo.getTableName()));
desc.setTableWriteId(writeId);
- //it's possible to have > 1 FileSink writing to the same table/partition
- //e.g. Merge stmt, multi-insert stmt when mixing DP and SP writes
+ /**
+ * it's possible to have > 1 FileSink writing to the same table/partition
+ * e.g. Merge stmt, multi-insert stmt when mixing DP and SP writes
+ * Insert ... Select ... Union All Select ... using
+ * {@link org.apache.hadoop.hive.ql.exec.AbstractFileMergeOperator#UNION_SUDBIR_PREFIX}
+ */
desc.setStatementId(queryTxnMgr.getStmtIdAndIncrement());
+ String unionAllSubdir = "/" + AbstractFileMergeOperator.UNION_SUDBIR_PREFIX;
+ if(desc.getInsertOverwrite() && desc.getDirName().toString().contains(unionAllSubdir) &&
+ desc.isFullAcidTable()) {
+ throw new UnsupportedOperationException("QueryId=" + plan.getQueryId() +
+ " is not supported due to OVERWRITE and UNION ALL. Please use truncate + insert");
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/6c4d7a9b/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java
index 1d05468..42b8f40 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java
@@ -300,6 +300,15 @@ public class FileSinkDesc extends AbstractOperatorDesc implements IStatsGatherDe
return AcidUtils.isInsertOnlyTable(getTableInfo().getProperties());
}
}
+ public boolean isFullAcidTable() {
+ if(getTable() != null) {
+ return AcidUtils.isFullAcidTable(table);
+ }
+ else {
+ return AcidUtils.isTablePropertyTransactional(getTableInfo().getProperties()) &&
+ !AcidUtils.isInsertOnlyTable(getTableInfo().getProperties());
+ }
+ }
public boolean isMaterialization() {
return materialization;
http://git-wip-us.apache.org/repos/asf/hive/blob/6c4d7a9b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java
index f071531..7ab76b3 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java
@@ -275,6 +275,24 @@ ekoifman:apache-hive-3.0.0-SNAPSHOT-bin ekoifman$ tree /Users/ekoifman/dev/hiver
};
checkExpected(rs, expected, "Unexpected row count after ctas");
}
+ @Test
+ public void testInsertOverwriteToAcidWithUnionRemove() throws Exception {
+ hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_OPTIMIZE_UNION_REMOVE, true);
+ hiveConf.setVar(HiveConf.ConfVars.HIVEFETCHTASKCONVERSION, "none");
+ d.close();
+ d = new Driver(hiveConf);
+ int[][] values = {{1, 2}, {3, 4}, {5, 6}, {7, 8}, {9, 10}};
+ runStatementOnDriver("drop table if exists T");
+ runStatementOnDriver("create table T (a int, b int) stored as ORC TBLPROPERTIES ('transactional'='true')");
+
+ CommandProcessorResponse cpr = runStatementOnDriverNegative(
+ "insert overwrite table T select a, b from " + TxnCommandsBaseForTests.Table.ACIDTBL +
+ " where a between 1 and 3 group by a, b union all select a, b from " +
+ TxnCommandsBaseForTests.Table.ACIDTBL +
+ " where a between 5 and 7 union all select a, b from " +
+ TxnCommandsBaseForTests.Table.ACIDTBL + " where a >= 9");
+ Assert.assertTrue("", cpr.getErrorMessage().contains("not supported due to OVERWRITE and UNION ALL"));
+ }
/**
* The idea here is to create a non acid table that was written by multiple writers, i.e.
* unbucketed table that has 000000_0 & 000001_0, for example.
[02/12] hive git commit: HIVE-19530: Vectorization: Fix JDBCSerde and
re-enable vectorization (Matt McCline, reviewed by Teddy Choi)
Posted by se...@apache.org.
HIVE-19530: Vectorization: Fix JDBCSerde and re-enable vectorization (Matt McCline, reviewed by Teddy Choi)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/d5cb88d5
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/d5cb88d5
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/d5cb88d5
Branch: refs/heads/master-txnstats
Commit: d5cb88d52ef1cc8ca315bbf703fa090ef197ba33
Parents: cf2e185
Author: Matt McCline <mm...@hortonworks.com>
Authored: Tue Jun 19 00:13:48 2018 -0500
Committer: Matt McCline <mm...@hortonworks.com>
Committed: Tue Jun 19 00:13:48 2018 -0500
----------------------------------------------------------------------
common/src/java/org/apache/hadoop/hive/conf/HiveConf.java | 2 +-
ql/src/test/results/clientpositive/llap/jdbc_handler.q.out | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/d5cb88d5/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 933bda4..e7f5fc0 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -3563,7 +3563,7 @@ public class HiveConf extends Configuration {
"The default value is false."),
HIVE_VECTORIZATION_ROW_DESERIALIZE_INPUTFORMAT_EXCLUDES(
"hive.vectorized.row.serde.inputformat.excludes",
- "org.apache.parquet.hadoop.ParquetInputFormat,org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat,org.apache.hive.storage.jdbc.JdbcInputFormat",
+ "org.apache.parquet.hadoop.ParquetInputFormat,org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat",
"The input formats not supported by row deserialize vectorization."),
HIVE_VECTOR_ADAPTOR_USAGE_MODE("hive.vectorized.adaptor.usage.mode", "all", new StringSet("none", "chosen", "all"),
"Specifies the extent to which the VectorUDFAdaptor will be used for UDFs that do not have a corresponding vectorized class.\n" +
http://git-wip-us.apache.org/repos/asf/hive/blob/d5cb88d5/ql/src/test/results/clientpositive/llap/jdbc_handler.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/jdbc_handler.q.out b/ql/src/test/results/clientpositive/llap/jdbc_handler.q.out
index ffeafdb..e7e3bc3 100644
--- a/ql/src/test/results/clientpositive/llap/jdbc_handler.q.out
+++ b/ql/src/test/results/clientpositive/llap/jdbc_handler.q.out
@@ -237,7 +237,7 @@ STAGE PLANS:
sort order:
Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string)
- Execution mode: llap
+ Execution mode: vectorized, llap
LLAP IO: no inputs
Map 4
Map Operator Tree:
[12/12] hive git commit: Merge branch 'master' into master-txnstats
Posted by se...@apache.org.
Merge branch 'master' into master-txnstats
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ac3f5186
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ac3f5186
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ac3f5186
Branch: refs/heads/master-txnstats
Commit: ac3f5186e8101274e6617ff891a110beda86a707
Parents: 0f09453 9cd3960
Author: sergey <se...@apache.org>
Authored: Thu Jun 21 10:47:49 2018 -0700
Committer: sergey <se...@apache.org>
Committed: Thu Jun 21 10:47:49 2018 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hive/conf/HiveConf.java | 2 +-
data/scripts/q_test_init.sql | 52 ----
data/scripts/q_test_init_parse.sql | 10 +
.../mapreduce/FileOutputCommitterContainer.java | 60 ++---
.../hadoop/hive/ql/TestLocationQueries.java | 3 +-
.../apache/hadoop/hive/ql/TestMTQueries.java | 1 +
.../hive/ql/txn/compactor/TestCompactor.java | 2 +-
.../apache/hive/jdbc/TestJdbcWithMiniHS2.java | 67 ++++-
.../test/resources/testconfiguration.properties | 1 -
.../hadoop/hive/accumulo/AccumuloQTestUtil.java | 1 +
.../control/AbstractCoreBlobstoreCliDriver.java | 6 +-
.../hadoop/hive/cli/control/CliConfigs.java | 2 +-
.../hive/cli/control/CoreAccumuloCliDriver.java | 32 ++-
.../hadoop/hive/cli/control/CoreCliDriver.java | 8 +-
.../hive/cli/control/CoreCompareCliDriver.java | 9 +-
.../hive/cli/control/CoreHBaseCliDriver.java | 33 +--
.../cli/control/CoreHBaseNegativeCliDriver.java | 36 ++-
.../hive/cli/control/CoreNegativeCliDriver.java | 6 +-
.../hive/cli/control/CorePerfCliDriver.java | 5 +-
.../org/apache/hadoop/hive/ql/QTestUtil.java | 262 ++++++-------------
.../hadoop/hive/ql/parse/CoreParseNegative.java | 3 +-
.../java/org/apache/hadoop/hive/ql/Driver.java | 15 +-
.../ql/cache/results/QueryResultsCache.java | 6 +
.../hive/ql/exec/tez/TezSessionState.java | 33 ++-
.../apache/hadoop/hive/ql/io/orc/OrcFile.java | 6 +-
.../hive/ql/io/orc/OrcRawRecordMerger.java | 12 +-
.../hadoop/hive/ql/io/orc/OrcRecordUpdater.java | 52 +++-
.../io/orc/VectorizedOrcAcidRowBatchReader.java | 2 +-
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 7 +-
.../hadoop/hive/ql/plan/FileSinkDesc.java | 9 +
.../apache/hadoop/hive/ql/TestTxnCommands2.java | 60 +++++
.../apache/hadoop/hive/ql/TestTxnNoBuckets.java | 18 ++
.../clientpositive/druidkafkamini_basic.q | 2 +-
.../queries/positive/input_testsequencefile.q | 11 +
.../results/clientpositive/acid_nullscan.q.out | 8 +-
.../clientpositive/acid_table_stats.q.out | 14 +-
.../clientpositive/autoColumnStats_4.q.out | 4 +-
.../llap/acid_bucket_pruning.q.out | 6 +-
.../llap/acid_vectorization_original.q.out | 4 +-
.../llap/default_constraint.q.out | 14 +-
.../llap/dynpart_sort_optimization_acid.q.out | 14 +-
.../insert_values_orig_table_use_metadata.q.out | 24 +-
.../clientpositive/llap/jdbc_handler.q.out | 2 +-
.../materialized_view_create_rewrite_3.q.out | 20 +-
.../materialized_view_create_rewrite_4.q.out | 22 +-
.../materialized_view_create_rewrite_5.q.out | 34 +--
...ized_view_create_rewrite_rebuild_dummy.q.out | 20 +-
.../test/results/clientpositive/row__id.q.out | 18 +-
.../tez/acid_vectorization_original_tez.q.out | 4 +-
.../clientpositive/tez/explainanalyze_5.q.out | 2 +-
.../client/builder/HiveObjectRefBuilder.java | 8 +-
.../hive/metastore/tools/HiveSchemaHelper.java | 6 +-
.../hadoop/hive/metastore/TestObjectStore.java | 107 +++++++-
53 files changed, 691 insertions(+), 474 deletions(-)
----------------------------------------------------------------------
[08/12] hive git commit: HIVE-19782: Flash out
TestObjectStore.testDirectSQLDropParitionsCleanup (Peter Vary,
reviewed by Vihang Karajgaonkar)
Posted by se...@apache.org.
HIVE-19782: Flash out TestObjectStore.testDirectSQLDropParitionsCleanup (Peter Vary, reviewed by Vihang Karajgaonkar)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/8f0973b2
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/8f0973b2
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/8f0973b2
Branch: refs/heads/master-txnstats
Commit: 8f0973b28eabbebf1563e1bfeac3609359d05688
Parents: e6577a0
Author: Peter Vary <pv...@cloudera.com>
Authored: Thu Jun 21 09:29:54 2018 +0200
Committer: Peter Vary <pv...@cloudera.com>
Committed: Thu Jun 21 09:29:54 2018 +0200
----------------------------------------------------------------------
.../client/builder/HiveObjectRefBuilder.java | 8 +-
.../hadoop/hive/metastore/TestObjectStore.java | 107 +++++++++++++++++--
2 files changed, 106 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/8f0973b2/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/HiveObjectRefBuilder.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/HiveObjectRefBuilder.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/HiveObjectRefBuilder.java
index 62a227a..666b8ab 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/HiveObjectRefBuilder.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/HiveObjectRefBuilder.java
@@ -57,7 +57,13 @@ public class HiveObjectRefBuilder {
}
public HiveObjectRef buildColumnReference(Table table, String columnName) {
- return new HiveObjectRef(HiveObjectType.TABLE, table.getDbName(), table.getTableName(),
+ return new HiveObjectRef(HiveObjectType.COLUMN, table.getDbName(), table.getTableName(),
Collections.emptyList(), columnName);
}
+
+ public HiveObjectRef buildPartitionColumnReference(Table table, String columnName,
+ List<String> partValues) {
+ return new HiveObjectRef(HiveObjectType.COLUMN, table.getDbName(), table.getTableName(),
+ partValues, columnName);
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/8f0973b2/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java
index 7984af6..5d5bc76 100644
--- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java
+++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java
@@ -23,11 +23,18 @@ import com.google.common.collect.ImmutableList;
import org.apache.hadoop.hive.metastore.ObjectStore.RetryingExecutor;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.metastore.annotation.MetastoreUnitTest;
+import org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData;
import org.apache.hadoop.hive.metastore.api.Catalog;
+import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
import org.apache.hadoop.hive.metastore.api.CurrentNotificationEventId;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.Function;
+import org.apache.hadoop.hive.metastore.api.HiveObjectPrivilege;
+import org.apache.hadoop.hive.metastore.api.HiveObjectRef;
import org.apache.hadoop.hive.metastore.api.InvalidInputException;
import org.apache.hadoop.hive.metastore.api.InvalidObjectException;
import org.apache.hadoop.hive.metastore.api.MetaException;
@@ -37,6 +44,8 @@ import org.apache.hadoop.hive.metastore.api.NotificationEventRequest;
import org.apache.hadoop.hive.metastore.api.NotificationEventResponse;
import org.apache.hadoop.hive.metastore.api.Partition;
import org.apache.hadoop.hive.metastore.api.PrincipalType;
+import org.apache.hadoop.hive.metastore.api.PrivilegeBag;
+import org.apache.hadoop.hive.metastore.api.PrivilegeGrantInfo;
import org.apache.hadoop.hive.metastore.api.Role;
import org.apache.hadoop.hive.metastore.api.SQLForeignKey;
import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey;
@@ -46,6 +55,9 @@ import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.metastore.client.builder.CatalogBuilder;
import org.apache.hadoop.hive.metastore.client.builder.DatabaseBuilder;
import org.apache.hadoop.hive.metastore.client.builder.PartitionBuilder;
+import org.apache.hadoop.hive.metastore.client.builder.HiveObjectPrivilegeBuilder;
+import org.apache.hadoop.hive.metastore.client.builder.HiveObjectRefBuilder;
+import org.apache.hadoop.hive.metastore.client.builder.PrivilegeGrantInfoBuilder;
import org.apache.hadoop.hive.metastore.client.builder.TableBuilder;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.hive.metastore.messaging.EventMessage;
@@ -69,6 +81,7 @@ import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
@@ -356,8 +369,8 @@ public class TestObjectStore {
*/
@Test
public void testDirectSQLDropPartitionsCacheInSession()
- throws MetaException, InvalidObjectException, NoSuchObjectException {
- createPartitionedTable();
+ throws MetaException, InvalidObjectException, NoSuchObjectException, InvalidInputException {
+ createPartitionedTable(false, false);
// query the partitions with JDO
Deadline.startTimer("getPartition");
List<Partition> partitions = objectStore.getPartitionsInternal(DEFAULT_CATALOG_NAME, DB1, TABLE1,
@@ -383,11 +396,11 @@ public class TestObjectStore {
*/
@Test
public void testDirectSQLDropPartitionsCacheCrossSession()
- throws MetaException, InvalidObjectException, NoSuchObjectException {
+ throws MetaException, InvalidObjectException, NoSuchObjectException, InvalidInputException {
ObjectStore objectStore2 = new ObjectStore();
objectStore2.setConf(conf);
- createPartitionedTable();
+ createPartitionedTable(false, false);
// query the partitions with JDO in the 1st session
Deadline.startTimer("getPartition");
List<Partition> partitions = objectStore.getPartitionsInternal(DEFAULT_CATALOG_NAME, DB1, TABLE1,
@@ -419,9 +432,24 @@ public class TestObjectStore {
*/
@Test
public void testDirectSQLDropParitionsCleanup() throws MetaException, InvalidObjectException,
- NoSuchObjectException, SQLException {
-
- createPartitionedTable();
+ NoSuchObjectException, SQLException, InvalidInputException {
+
+ createPartitionedTable(true, true);
+
+ // Check, that every table in the expected state before the drop
+ checkBackendTableSize("PARTITIONS", 3);
+ checkBackendTableSize("PART_PRIVS", 3);
+ checkBackendTableSize("PART_COL_PRIVS", 3);
+ checkBackendTableSize("PART_COL_STATS", 3);
+ checkBackendTableSize("PARTITION_PARAMS", 3);
+ checkBackendTableSize("PARTITION_KEY_VALS", 3);
+ checkBackendTableSize("SD_PARAMS", 3);
+ checkBackendTableSize("BUCKETING_COLS", 3);
+ checkBackendTableSize("SKEWED_COL_NAMES", 3);
+ checkBackendTableSize("SDS", 4); // Table has an SDS
+ checkBackendTableSize("SORT_COLS", 3);
+ checkBackendTableSize("SERDE_PARAMS", 3);
+ checkBackendTableSize("SERDES", 4); // Table has a serde
// drop the partitions
Deadline.startTimer("dropPartitions");
@@ -446,10 +474,13 @@ public class TestObjectStore {
/**
* Creates DB1 database, TABLE1 table with 3 partitions.
+ * @param withPrivileges Should we create privileges as well
+ * @param withStatistics Should we create statitics as well
* @throws MetaException
* @throws InvalidObjectException
*/
- private void createPartitionedTable() throws MetaException, InvalidObjectException {
+ private void createPartitionedTable(boolean withPrivileges, boolean withStatistics)
+ throws MetaException, InvalidObjectException, NoSuchObjectException, InvalidInputException {
Database db1 = new DatabaseBuilder()
.setName(DB1)
.setDescription("description")
@@ -463,16 +494,76 @@ public class TestObjectStore {
.addCol("test_col1", "int")
.addCol("test_col2", "int")
.addPartCol("test_part_col", "int")
+ .addCol("test_bucket_col", "int", "test bucket col comment")
+ .addCol("test_skewed_col", "int", "test skewed col comment")
+ .addCol("test_sort_col", "int", "test sort col comment")
.build(conf);
objectStore.createTable(tbl1);
+ PrivilegeBag privilegeBag = new PrivilegeBag();
// Create partitions for the partitioned table
for(int i=0; i < 3; i++) {
Partition part = new PartitionBuilder()
.inTable(tbl1)
.addValue("a" + i)
+ .addSerdeParam("serdeParam", "serdeParamValue")
+ .addStorageDescriptorParam("sdParam", "sdParamValue")
+ .addBucketCol("test_bucket_col")
+ .addSkewedColName("test_skewed_col")
+ .addSortCol("test_sort_col", 1)
.build(conf);
objectStore.addPartition(part);
+
+ if (withPrivileges) {
+ HiveObjectRef partitionReference = new HiveObjectRefBuilder().buildPartitionReference(part);
+ HiveObjectRef partitionColumnReference = new HiveObjectRefBuilder()
+ .buildPartitionColumnReference(tbl1, "test_part_col", part.getValues());
+ PrivilegeGrantInfo privilegeGrantInfo = new PrivilegeGrantInfoBuilder()
+ .setPrivilege("a")
+ .build();
+ HiveObjectPrivilege partitionPriv = new HiveObjectPrivilegeBuilder()
+ .setHiveObjectRef(partitionReference)
+ .setPrincipleName("a")
+ .setPrincipalType(PrincipalType.USER)
+ .setGrantInfo(privilegeGrantInfo)
+ .build();
+ privilegeBag.addToPrivileges(partitionPriv);
+ HiveObjectPrivilege partitionColPriv = new HiveObjectPrivilegeBuilder()
+ .setHiveObjectRef(partitionColumnReference)
+ .setPrincipleName("a")
+ .setPrincipalType(PrincipalType.USER)
+ .setGrantInfo(privilegeGrantInfo)
+ .build();
+ privilegeBag.addToPrivileges(partitionColPriv);
+ }
+
+ if (withStatistics) {
+ ColumnStatistics stats = new ColumnStatistics();
+ ColumnStatisticsDesc desc = new ColumnStatisticsDesc();
+ desc.setCatName(tbl1.getCatName());
+ desc.setDbName(tbl1.getDbName());
+ desc.setTableName(tbl1.getTableName());
+ desc.setPartName("test_part_col=a" + i);
+ stats.setStatsDesc(desc);
+
+ List<ColumnStatisticsObj> statsObjList = new ArrayList<>(1);
+ stats.setStatsObj(statsObjList);
+
+ ColumnStatisticsData data = new ColumnStatisticsData();
+ BooleanColumnStatsData boolStats = new BooleanColumnStatsData();
+ boolStats.setNumTrues(0);
+ boolStats.setNumFalses(0);
+ boolStats.setNumNulls(0);
+ data.setBooleanStats(boolStats);
+
+ ColumnStatisticsObj partStats = new ColumnStatisticsObj("test_part_col", "int", data);
+ statsObjList.add(partStats);
+
+ objectStore.updatePartitionColumnStatistics(stats, part.getValues());
+ }
+ }
+ if (withPrivileges) {
+ objectStore.grantPrivileges(privilegeBag);
}
}
[09/12] hive git commit: HIVE-19897: Add more tests for parallel
compilation (Yongzhi Chen, reviewed by Aihua Xu)
Posted by se...@apache.org.
HIVE-19897: Add more tests for parallel compilation (Yongzhi Chen, reviewed by Aihua Xu)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/596edd18
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/596edd18
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/596edd18
Branch: refs/heads/master-txnstats
Commit: 596edd181624afb5175fb98eea2c672775c2f378
Parents: 8f0973b
Author: Yongzhi Chen <yc...@apache.org>
Authored: Thu Jun 21 10:08:01 2018 -0400
Committer: Yongzhi Chen <yc...@apache.org>
Committed: Thu Jun 21 10:27:11 2018 -0400
----------------------------------------------------------------------
.../apache/hive/jdbc/TestJdbcWithMiniHS2.java | 67 ++++++++++++++++++--
1 file changed, 62 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/596edd18/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java
index d7d7097..dba0721 100644
--- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java
+++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java
@@ -281,21 +281,75 @@ public class TestJdbcWithMiniHS2 {
}
@Test
+ public void testParallelCompilation3() throws Exception {
+ Statement stmt = conTestDb.createStatement();
+ stmt.execute("set hive.driver.parallel.compilation=true");
+ stmt.execute("set hive.server2.async.exec.async.compile=true");
+ stmt.close();
+ Connection conn = getConnection(testDbName);
+ stmt = conn.createStatement();
+ stmt.execute("set hive.driver.parallel.compilation=true");
+ stmt.execute("set hive.server2.async.exec.async.compile=true");
+ stmt.close();
+ int poolSize = 100;
+ SynchronousQueue<Runnable> executorQueue1 = new SynchronousQueue<Runnable>();
+ ExecutorService workers1 =
+ new ThreadPoolExecutor(1, poolSize, 20, TimeUnit.SECONDS, executorQueue1);
+ SynchronousQueue<Runnable> executorQueue2 = new SynchronousQueue<Runnable>();
+ ExecutorService workers2 =
+ new ThreadPoolExecutor(1, poolSize, 20, TimeUnit.SECONDS, executorQueue2);
+ List<Future<Boolean>> list1 = startTasks(workers1, conTestDb, tableName, 10);
+ List<Future<Boolean>> list2 = startTasks(workers2, conn, tableName, 10);
+ finishTasks(list1, workers1);
+ finishTasks(list2, workers2);
+ conn.close();
+ }
+
+ @Test
+ public void testParallelCompilation4() throws Exception {
+ Statement stmt = conTestDb.createStatement();
+ stmt.execute("set hive.driver.parallel.compilation=true");
+ stmt.execute("set hive.server2.async.exec.async.compile=false");
+ stmt.close();
+ Connection conn = getConnection(testDbName);
+ stmt = conn.createStatement();
+ stmt.execute("set hive.driver.parallel.compilation=true");
+ stmt.execute("set hive.server2.async.exec.async.compile=false");
+ stmt.close();
+ int poolSize = 100;
+ SynchronousQueue<Runnable> executorQueue1 = new SynchronousQueue<Runnable>();
+ ExecutorService workers1 =
+ new ThreadPoolExecutor(1, poolSize, 20, TimeUnit.SECONDS, executorQueue1);
+ SynchronousQueue<Runnable> executorQueue2 = new SynchronousQueue<Runnable>();
+ ExecutorService workers2 =
+ new ThreadPoolExecutor(1, poolSize, 20, TimeUnit.SECONDS, executorQueue2);
+ List<Future<Boolean>> list1 = startTasks(workers1, conTestDb, tableName, 10);
+ List<Future<Boolean>> list2 = startTasks(workers2, conn, tableName, 10);
+ finishTasks(list1, workers1);
+ finishTasks(list2, workers2);
+ conn.close();
+ }
+
+ @Test
public void testConcurrentStatements() throws Exception {
startConcurrencyTest(conTestDb, tableName, 50);
}
private static void startConcurrencyTest(Connection conn, String tableName, int numTasks) {
// Start concurrent testing
- int POOL_SIZE = 100;
- int TASK_COUNT = numTasks;
-
+ int poolSize = 100;
SynchronousQueue<Runnable> executorQueue = new SynchronousQueue<Runnable>();
ExecutorService workers =
- new ThreadPoolExecutor(1, POOL_SIZE, 20, TimeUnit.SECONDS, executorQueue);
+ new ThreadPoolExecutor(1, poolSize, 20, TimeUnit.SECONDS, executorQueue);
+ List<Future<Boolean>> list = startTasks(workers, conn, tableName, numTasks);
+ finishTasks(list, workers);
+ }
+
+ private static List<Future<Boolean>> startTasks(ExecutorService workers, Connection conn,
+ String tableName, int numTasks) {
List<Future<Boolean>> list = new ArrayList<Future<Boolean>>();
int i = 0;
- while (i < TASK_COUNT) {
+ while (i < numTasks) {
try {
Future<Boolean> future = workers.submit(new JDBCTask(conn, i, tableName));
list.add(future);
@@ -308,7 +362,10 @@ public class TestJdbcWithMiniHS2 {
}
}
}
+ return list;
+ }
+ private static void finishTasks(List<Future<Boolean>> list, ExecutorService workers) {
for (Future<Boolean> future : list) {
try {
Boolean result = future.get(30, TimeUnit.SECONDS);
[07/12] hive git commit: HIVE-19882: Fix QTestUtil session lifecycle
(Zoltan Haindrich reviewed by Jason Dere)
Posted by se...@apache.org.
HIVE-19882: Fix QTestUtil session lifecycle (Zoltan Haindrich reviewed by Jason Dere)
Signed-off-by: Zoltan Haindrich <ki...@rxd.hu>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e6577a0d
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e6577a0d
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e6577a0d
Branch: refs/heads/master-txnstats
Commit: e6577a0d910ce7fc982f497848229e0195859dc1
Parents: aea5908
Author: Zoltan Haindrich <ki...@rxd.hu>
Authored: Thu Jun 21 06:15:49 2018 +0200
Committer: Zoltan Haindrich <ki...@rxd.hu>
Committed: Thu Jun 21 06:15:49 2018 +0200
----------------------------------------------------------------------
data/scripts/q_test_init.sql | 52 ----
data/scripts/q_test_init_parse.sql | 10 +
.../hadoop/hive/ql/TestLocationQueries.java | 3 +-
.../apache/hadoop/hive/ql/TestMTQueries.java | 1 +
.../test/resources/testconfiguration.properties | 1 -
.../hadoop/hive/accumulo/AccumuloQTestUtil.java | 1 +
.../control/AbstractCoreBlobstoreCliDriver.java | 6 +-
.../hadoop/hive/cli/control/CliConfigs.java | 2 +-
.../hive/cli/control/CoreAccumuloCliDriver.java | 32 ++-
.../hadoop/hive/cli/control/CoreCliDriver.java | 8 +-
.../hive/cli/control/CoreCompareCliDriver.java | 9 +-
.../hive/cli/control/CoreHBaseCliDriver.java | 33 +--
.../cli/control/CoreHBaseNegativeCliDriver.java | 36 ++-
.../hive/cli/control/CoreNegativeCliDriver.java | 6 +-
.../hive/cli/control/CorePerfCliDriver.java | 5 +-
.../org/apache/hadoop/hive/ql/QTestUtil.java | 262 ++++++-------------
.../hadoop/hive/ql/parse/CoreParseNegative.java | 3 +-
.../clientpositive/druidkafkamini_basic.q | 2 +-
.../queries/positive/input_testsequencefile.q | 11 +
19 files changed, 192 insertions(+), 291 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/e6577a0d/data/scripts/q_test_init.sql
----------------------------------------------------------------------
diff --git a/data/scripts/q_test_init.sql b/data/scripts/q_test_init.sql
index a269c55..df05828 100644
--- a/data/scripts/q_test_init.sql
+++ b/data/scripts/q_test_init.sql
@@ -6,57 +6,5 @@ set hive.stats.dbclass=fs;
DROP FUNCTION IF EXISTS qtest_get_java_boolean;
CREATE FUNCTION qtest_get_java_boolean AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFTestGetJavaBoolean';
---
--- Table dest1
---
-DROP TABLE IF EXISTS dest1;
-
-CREATE TABLE dest1 (key STRING COMMENT 'default', value STRING COMMENT 'default')
-STORED AS
-INPUTFORMAT 'org.apache.hadoop.mapred.TextInputFormat'
-OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat';
-
---
--- Table dest2
---
-DROP TABLE IF EXISTS dest2;
-
-CREATE TABLE dest2 (key STRING COMMENT 'default', value STRING COMMENT 'default')
-STORED AS
-INPUTFORMAT 'org.apache.hadoop.mapred.TextInputFormat'
-OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat';
-
---
--- Table dest3
---
-DROP TABLE IF EXISTS dest3;
-
-CREATE TABLE dest3 (key STRING COMMENT 'default', value STRING COMMENT 'default')
-PARTITIONED BY (ds STRING, hr STRING)
-STORED AS
-INPUTFORMAT 'org.apache.hadoop.mapred.TextInputFormat'
-OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat';
-ALTER TABLE dest3 ADD PARTITION (ds='2008-04-08',hr='12');
-
---
--- Table dest4
---
-DROP TABLE IF EXISTS dest4;
-
-CREATE TABLE dest4 (key STRING COMMENT 'default', value STRING COMMENT 'default')
-STORED AS
-INPUTFORMAT 'org.apache.hadoop.mapred.TextInputFormat'
-OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat';
-
---
--- Table dest4_sequencefile
---
-DROP TABLE IF EXISTS dest4_sequencefile;
-
-CREATE TABLE dest4_sequencefile (key STRING COMMENT 'default', value STRING COMMENT 'default')
-STORED AS
-INPUTFORMAT 'org.apache.hadoop.mapred.SequenceFileInputFormat'
-OUTPUTFORMAT 'org.apache.hadoop.mapred.SequenceFileOutputFormat';
-
reset;
set hive.stats.dbclass=fs;
http://git-wip-us.apache.org/repos/asf/hive/blob/e6577a0d/data/scripts/q_test_init_parse.sql
----------------------------------------------------------------------
diff --git a/data/scripts/q_test_init_parse.sql b/data/scripts/q_test_init_parse.sql
new file mode 100644
index 0000000..f84c847
--- /dev/null
+++ b/data/scripts/q_test_init_parse.sql
@@ -0,0 +1,10 @@
+--
+-- Table dest1
+--
+DROP TABLE IF EXISTS dest1;
+
+CREATE TABLE dest1 (key STRING COMMENT 'default', value STRING COMMENT 'default')
+STORED AS
+INPUTFORMAT 'org.apache.hadoop.mapred.TextInputFormat'
+OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat';
+
http://git-wip-us.apache.org/repos/asf/hive/blob/e6577a0d/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java
index bd33325..31195c4 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java
@@ -109,7 +109,8 @@ public class TestLocationQueries extends BaseTestQueries {
for (int i = 0; i < qfiles.length; i++) {
qt[i] = new CheckResults(resDir, logDir, MiniClusterType.none, "0.20", "parta");
- qt[i].addFile(qfiles[i]);
+ qt[i].newSession();
+ qt[i].addFile(qfiles[i], false);
qt[i].clearTestSideEffects();
}
http://git-wip-us.apache.org/repos/asf/hive/blob/e6577a0d/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestMTQueries.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestMTQueries.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestMTQueries.java
index 6ed872d..3d8eb83 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestMTQueries.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestMTQueries.java
@@ -46,6 +46,7 @@ public class TestMTQueries extends BaseTestQueries {
util.getConf().set("hive.stats.dbclass", "fs");
util.getConf().set("hive.mapred.mode", "nonstrict");
util.getConf().set("hive.stats.column.autogather", "false");
+ util.newSession(true);
}
boolean success = QTestUtil.queryListRunnerMultiThreaded(qfiles, qts);
if (!success) {
http://git-wip-us.apache.org/repos/asf/hive/blob/e6577a0d/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index aeb6211..a3ddbda 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -17,7 +17,6 @@ disabled.query.files=ql_rewrite_gbtoidx.q,\
union_stats.q,\
sample2.q,\
sample4.q,\
- sample6.q,\
root_dir_external_table.q,\
input31.q
http://git-wip-us.apache.org/repos/asf/hive/blob/e6577a0d/itests/util/src/main/java/org/apache/hadoop/hive/accumulo/AccumuloQTestUtil.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/accumulo/AccumuloQTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/accumulo/AccumuloQTestUtil.java
index 0f9528f..956478d 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/accumulo/AccumuloQTestUtil.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/accumulo/AccumuloQTestUtil.java
@@ -32,6 +32,7 @@ public class AccumuloQTestUtil extends QTestUtil {
super(outDir, logDir, miniMr, null, "0.20", initScript, cleanupScript, false);
setup.setupWithHiveConf(conf);
this.setup = setup;
+ this.savedConf = new HiveConf(conf);
}
@Override
http://git-wip-us.apache.org/repos/asf/hive/blob/e6577a0d/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java
index 764a4d8..3cf5ebb 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java
@@ -68,6 +68,7 @@ public abstract class AbstractCoreBlobstoreCliDriver extends CliAdapter {
// do a one time initialization
setupUniqueTestPath();
+ qt.newSession();
qt.cleanUp();
qt.createSources();
} catch (Exception e) {
@@ -82,7 +83,7 @@ public abstract class AbstractCoreBlobstoreCliDriver extends CliAdapter {
@Before
public void setUp() {
try {
- qt.clearTestSideEffects();
+ qt.newSession();
} catch (Exception e) {
System.err.println("Exception: " + e.getMessage());
e.printStackTrace();
@@ -95,6 +96,7 @@ public abstract class AbstractCoreBlobstoreCliDriver extends CliAdapter {
@After
public void tearDown() {
try {
+ qt.clearTestSideEffects();
qt.clearPostTestEffects();
} catch (Exception e) {
System.err.println("Exception: " + e.getMessage());
@@ -131,7 +133,7 @@ public abstract class AbstractCoreBlobstoreCliDriver extends CliAdapter {
System.err.println("Begin query: " + fname);
qt.addFile(fpath);
- qt.cliInit(new File(fpath), false);
+ qt.cliInit(new File(fpath));
int ecode = qt.executeClient(fname);
if ((ecode == 0) ^ expectSuccess) {
qt.failed(ecode, fname, debugHint);
http://git-wip-us.apache.org/repos/asf/hive/blob/e6577a0d/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java
index 59a78d9..d07599e 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java
@@ -84,7 +84,7 @@ public class CliConfigs {
setResultsDir("ql/src/test/results/compiler/errors");
setLogDir("itests/qtest/target/qfile-results/negative");
- setInitScript("q_test_init.sql");
+ setInitScript("q_test_init_parse.sql");
setCleanupScript("q_test_cleanup.sql");
setHiveConfDir("data/conf/perf-reg/");
http://git-wip-us.apache.org/repos/asf/hive/blob/e6577a0d/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreAccumuloCliDriver.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreAccumuloCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreAccumuloCliDriver.java
index 648a05d..0d64cfa 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreAccumuloCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreAccumuloCliDriver.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hive.cli.control;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
import java.io.File;
@@ -43,7 +44,7 @@ public class CoreAccumuloCliDriver extends CliAdapter {
@BeforeClass
public void beforeClass() {
setup = new AccumuloTestSetup();
-
+
MiniClusterType miniMR = cliConfig.getClusterType();
String initScript = cliConfig.getInitScript();
String cleanupScript = cliConfig.getCleanupScript();
@@ -51,20 +52,21 @@ public class CoreAccumuloCliDriver extends CliAdapter {
try {
qt = new AccumuloQTestUtil(cliConfig.getResultsDir(), cliConfig.getLogDir(), miniMR,
setup, initScript, cleanupScript);
-
+
// do a one time initialization
+ qt.newSession();
qt.cleanUp();
qt.createSources();
} catch (Exception e) {
throw new RuntimeException("Unexpected exception in setUp",e);
}
}
-
+
@Override
@AfterClass
public void shutdown() throws Exception {
setup.tearDown();
-
+
try {
qt.shutdown();
}
@@ -72,16 +74,33 @@ public class CoreAccumuloCliDriver extends CliAdapter {
throw new RuntimeException("Unexpected exception in tearDown",e);
}
}
+
@Override
@Before
public void setUp() {
+ try {
+ qt.newSession();
+ } catch (Exception e) {
+ System.err.println("Exception: " + e.getMessage());
+ e.printStackTrace();
+ System.err.flush();
+ fail("Unexpected exception in setup");
+ }
}
@Override
@After
public void tearDown() {
+ try {
+ qt.clearPostTestEffects();
+ qt.clearTestSideEffects();
+ } catch (Exception e) {
+ System.err.println("Exception: " + e.getMessage());
+ e.printStackTrace();
+ System.err.flush();
+ fail("Unexpected exception in tearDown");
+ }
}
-
@Override
public void runTest(String tname, String fname, String fpath) throws Exception {
long startTime = System.currentTimeMillis();
@@ -90,8 +109,7 @@ public class CoreAccumuloCliDriver extends CliAdapter {
qt.addFile(fpath);
- qt.cliInit(new File(fpath), false);
- qt.clearTestSideEffects();
+ qt.cliInit(new File(fpath));
int ecode = qt.executeClient(fname);
if (ecode != 0) {
qt.failed(ecode, fname, null);
http://git-wip-us.apache.org/repos/asf/hive/blob/e6577a0d/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java
index e588592..1ead144 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java
@@ -72,7 +72,8 @@ public class CoreCliDriver extends CliAdapter {
new ElapsedTimeLoggingWrapper<Void>() {
@Override
public Void invokeInternal() throws Exception {
- qt.cleanUp();
+ qt.newSession();
+ qt.cleanUp(); // I don't think this is neccessary...
return null;
}
}.invoke("Initialization cleanup done.", LOG, true);
@@ -100,7 +101,7 @@ public class CoreCliDriver extends CliAdapter {
new ElapsedTimeLoggingWrapper<Void>() {
@Override
public Void invokeInternal() throws Exception {
- qt.clearTestSideEffects();
+ qt.newSession();
return null;
}
}.invoke("PerTestSetup done.", LOG, false);
@@ -120,6 +121,7 @@ public class CoreCliDriver extends CliAdapter {
@Override
public Void invokeInternal() throws Exception {
qt.clearPostTestEffects();
+ qt.clearTestSideEffects();
return null;
}
}.invoke("PerTestTearDown done.", LOG, false);
@@ -165,7 +167,7 @@ public class CoreCliDriver extends CliAdapter {
System.err.println("Begin query: " + fname);
qt.addFile(fpath);
- qt.cliInit(new File(fpath), false);
+ qt.cliInit(new File(fpath));
int ecode = qt.executeClient(fname);
if (ecode != 0) {
failed = true;
http://git-wip-us.apache.org/repos/asf/hive/blob/e6577a0d/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCompareCliDriver.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCompareCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCompareCliDriver.java
index 1ad76f9..6b4c6c6 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCompareCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCompareCliDriver.java
@@ -56,6 +56,7 @@ public class CoreCompareCliDriver extends CliAdapter{
hiveConfDir, hadoopVer, initScript, cleanupScript, false);
// do a one time initialization
+ qt.newSession();
qt.cleanUp();
qt.createSources();
@@ -130,14 +131,14 @@ public class CoreCompareCliDriver extends CliAdapter{
}
int ecode = 0;
-
- qt.cliInit(new File(fpath), false);
-
+
+ qt.cliInit(new File(fpath));
+
List<String> outputs = new ArrayList<>(versionFiles.size());
for (String versionFile : versionFiles) {
// 1 for "_" after tname; 3 for ".qv" at the end. Version is in between.
String versionStr = versionFile.substring(tname.length() + 1, versionFile.length() - 3);
- outputs.add(qt.cliInit(new File(queryDirectory, tname + "." + versionStr), false));
+ outputs.add(qt.cliInit(new File(queryDirectory, tname + "." + versionStr)));
// TODO: will this work?
ecode = qt.executeClient(versionFile, fname);
if (ecode != 0) {
http://git-wip-us.apache.org/repos/asf/hive/blob/e6577a0d/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseCliDriver.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseCliDriver.java
index fc5f75d..70cbf04 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseCliDriver.java
@@ -43,22 +43,23 @@ public class CoreHBaseCliDriver extends CliAdapter {
@Override
@BeforeClass
public void beforeClass() {
- MiniClusterType miniMR = cliConfig.getClusterType();
- String initScript = cliConfig.getInitScript();
- String cleanupScript =cliConfig.getCleanupScript();
+ MiniClusterType miniMR = cliConfig.getClusterType();
+ String initScript = cliConfig.getInitScript();
+ String cleanupScript = cliConfig.getCleanupScript();
- try {
- qt = new HBaseQTestUtil(cliConfig.getResultsDir(), cliConfig.getLogDir(), miniMR,
+ try {
+ qt = new HBaseQTestUtil(cliConfig.getResultsDir(), cliConfig.getLogDir(), miniMR,
setup, initScript, cleanupScript);
- qt.cleanUp(null);
- qt.createSources(null);
+ qt.newSession();
+ qt.cleanUp(null);
+ qt.createSources(null);
- } catch (Exception e) {
- System.err.println("Exception: " + e.getMessage());
- e.printStackTrace();
- System.err.flush();
- fail("Unexpected exception in static initialization: "+e.getMessage());
- }
+ } catch (Exception e) {
+ System.err.println("Exception: " + e.getMessage());
+ e.printStackTrace();
+ System.err.flush();
+ throw new RuntimeException(e);
+ }
}
@@ -66,7 +67,7 @@ public class CoreHBaseCliDriver extends CliAdapter {
@Before
public void setUp() {
try {
- qt.clearTestSideEffects();
+ qt.newSession();
} catch (Exception e) {
System.err.println("Exception: " + e.getMessage());
e.printStackTrace();
@@ -79,6 +80,7 @@ public class CoreHBaseCliDriver extends CliAdapter {
public void tearDown() {
try {
qt.clearPostTestEffects();
+ qt.clearTestSideEffects();
} catch (Exception e) {
System.err.println("Exception: " + e.getMessage());
e.printStackTrace();
@@ -91,7 +93,6 @@ public class CoreHBaseCliDriver extends CliAdapter {
@AfterClass
public void shutdown() throws Exception {
try {
- // FIXME: there were 2 afterclass methods...i guess this is the right order...maybe not
qt.shutdown();
setup.tearDown();
} catch (Exception e) {
@@ -110,7 +111,7 @@ public class CoreHBaseCliDriver extends CliAdapter {
qt.addFile(fpath);
- qt.cliInit(new File(fpath), false);
+ qt.cliInit(new File(fpath));
int ecode = qt.executeClient(fname);
if (ecode != 0) {
http://git-wip-us.apache.org/repos/asf/hive/blob/e6577a0d/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseNegativeCliDriver.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseNegativeCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseNegativeCliDriver.java
index 8fb88d0..c76a70e 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseNegativeCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseNegativeCliDriver.java
@@ -42,13 +42,6 @@ public class CoreHBaseNegativeCliDriver extends CliAdapter {
@Override
public void beforeClass() throws Exception {
- }
-
- // hmm..this looks a bit wierd...setup boots qtestutil...this part used to be in beforeclass
- @Override
- @Before
- public void setUp() {
-
MiniClusterType miniMR = cliConfig.getClusterType();
String initScript = cliConfig.getInitScript();
String cleanupScript = cliConfig.getCleanupScript();
@@ -64,11 +57,26 @@ public class CoreHBaseNegativeCliDriver extends CliAdapter {
}
}
+ // hmm..this looks a bit wierd...setup boots qtestutil...this part used to be in beforeclass
+ @Override
+ @Before
+ public void setUp() {
+ try {
+ qt.newSession();
+ } catch (Exception e) {
+ System.err.println("Exception: " + e.getMessage());
+ e.printStackTrace();
+ System.err.flush();
+ fail("Unexpected exception in setup");
+ }
+ }
+
@Override
@After
public void tearDown() {
try {
- qt.shutdown();
+ qt.clearPostTestEffects();
+ qt.clearTestSideEffects();
} catch (Exception e) {
System.err.println("Exception: " + e.getMessage());
e.printStackTrace();
@@ -80,6 +88,14 @@ public class CoreHBaseNegativeCliDriver extends CliAdapter {
@Override
@AfterClass
public void shutdown() throws Exception {
+ try {
+ qt.shutdown();
+ } catch (Exception e) {
+ System.err.println("Exception: " + e.getMessage());
+ e.printStackTrace();
+ System.err.flush();
+ fail("Unexpected exception in tearDown");
+ }
// closeHBaseConnections
setup.tearDown();
}
@@ -89,11 +105,8 @@ public class CoreHBaseNegativeCliDriver extends CliAdapter {
long startTime = System.currentTimeMillis();
try {
System.err.println("Begin query: " + fname);
-
qt.addFile(fpath);
-
qt.cliInit(new File(fpath));
- qt.clearTestSideEffects();
int ecode = qt.executeClient(fname);
if (ecode == 0) {
qt.failed(fname, null);
@@ -103,7 +116,6 @@ public class CoreHBaseNegativeCliDriver extends CliAdapter {
if (result.getReturnCode() != 0) {
qt.failedDiff(result.getReturnCode(), fname, result.getCapturedOutput());
}
- qt.clearPostTestEffects();
} catch (Exception e) {
qt.failed(e, fname, null);
http://git-wip-us.apache.org/repos/asf/hive/blob/e6577a0d/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java
index 3be6f66..07ae6ac 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java
@@ -50,6 +50,7 @@ public class CoreNegativeCliDriver extends CliAdapter{
qt = new QTestUtil((cliConfig.getResultsDir()), (cliConfig.getLogDir()), miniMR,
hiveConfDir, hadoopVer, initScript, cleanupScript, false);
// do a one time initialization
+ qt.newSession();
qt.cleanUp();
qt.createSources();
} catch (Exception e) {
@@ -64,7 +65,7 @@ public class CoreNegativeCliDriver extends CliAdapter{
@Before
public void setUp() {
try {
- qt.clearTestSideEffects();
+ qt.newSession();
} catch (Throwable e) {
e.printStackTrace();
System.err.flush();
@@ -76,6 +77,7 @@ public class CoreNegativeCliDriver extends CliAdapter{
@After
public void tearDown() {
try {
+ qt.clearTestSideEffects();
qt.clearPostTestEffects();
} catch (Exception e) {
System.err.println("Exception: " + e.getMessage());
@@ -118,7 +120,7 @@ public class CoreNegativeCliDriver extends CliAdapter{
qt.addFile(fpath);
- qt.cliInit(new File(fpath), false);
+ qt.cliInit(new File(fpath));
int ecode = qt.executeClient(fname);
if (ecode == 0) {
qt.failed(fname, debugHint);
http://git-wip-us.apache.org/repos/asf/hive/blob/e6577a0d/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java
index af91866..55e744e 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java
@@ -68,6 +68,7 @@ public class CorePerfCliDriver extends CliAdapter{
cleanupScript, false, null);
// do a one time initialization
+ qt.newSession();
qt.cleanUp();
qt.createSources();
// Manually modify the underlying metastore db to reflect statistics corresponding to
@@ -92,7 +93,7 @@ public class CorePerfCliDriver extends CliAdapter{
@Override
public void setUp() {
try {
- qt.clearPostTestEffects();
+ qt.newSession();
} catch (Exception e) {
System.err.println("Exception: " + e.getMessage());
e.printStackTrace();
@@ -126,7 +127,7 @@ public class CorePerfCliDriver extends CliAdapter{
System.err.println("Begin query: " + fname);
qt.addFile(fpath);
- qt.cliInit(new File(fpath), false);
+ qt.cliInit(new File(fpath));
int ecode = qt.executeClient(fname);
if (ecode != 0) {
http://git-wip-us.apache.org/repos/asf/hive/blob/e6577a0d/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
index 2106fec..0bbd751 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
@@ -22,12 +22,10 @@ import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_DATABASE_NAME;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
-import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
-import java.io.FileReader;
import java.io.FileWriter;
import java.io.FilenameFilter;
import java.io.IOException;
@@ -158,6 +156,7 @@ public class QTestUtil {
private static final String DEFAULT_TEST_EC_POLICY = "RS-3-2-1024k";
private String testWarehouse;
+ @Deprecated
private final String testFiles;
private final File datasetDir;
protected final String outDir;
@@ -181,6 +180,7 @@ public class QTestUtil {
protected Hive db;
protected QueryState queryState;
protected HiveConf conf;
+ protected HiveConf savedConf;
private IDriver drv;
private BaseSemanticAnalyzer sem;
protected final boolean overWrite;
@@ -245,7 +245,7 @@ public class QTestUtil {
private CliDriver getCliDriver() {
if(cliDriver == null){
- cliDriver = new CliDriver();
+ throw new RuntimeException("no clidriver");
}
return cliDriver;
}
@@ -277,80 +277,6 @@ public class QTestUtil {
return conf;
}
- public boolean deleteDirectory(File path) {
- if (path.exists()) {
- File[] files = path.listFiles();
- for (File file : files) {
- if (file.isDirectory()) {
- deleteDirectory(file);
- } else {
- file.delete();
- }
- }
- }
- return (path.delete());
- }
-
- public void copyDirectoryToLocal(Path src, Path dest) throws Exception {
-
- FileSystem srcFs = src.getFileSystem(conf);
- FileSystem destFs = dest.getFileSystem(conf);
- if (srcFs.exists(src)) {
- FileStatus[] files = srcFs.listStatus(src);
- for (FileStatus file : files) {
- String name = file.getPath().getName();
- Path dfs_path = file.getPath();
- Path local_path = new Path(dest, name);
-
- // If this is a source table we do not copy it out
- if (getSrcTables().contains(name)) {
- continue;
- }
-
- if (file.isDirectory()) {
- if (!destFs.exists(local_path)) {
- destFs.mkdirs(local_path);
- }
- copyDirectoryToLocal(dfs_path, local_path);
- } else {
- srcFs.copyToLocalFile(dfs_path, local_path);
- }
- }
- }
- }
-
- static Pattern mapTok = Pattern.compile("(\\.?)(.*)_map_(.*)");
- static Pattern reduceTok = Pattern.compile("(.*)(reduce_[^\\.]*)((\\..*)?)");
-
- public void normalizeNames(File path) throws Exception {
- if (path.isDirectory()) {
- File[] files = path.listFiles();
- for (File file : files) {
- normalizeNames(file);
- }
- } else {
- Matcher m = reduceTok.matcher(path.getName());
- if (m.matches()) {
- String name = m.group(1) + "reduce" + m.group(3);
- path.renameTo(new File(path.getParent(), name));
- } else {
- m = mapTok.matcher(path.getName());
- if (m.matches()) {
- String name = m.group(1) + "map_" + m.group(3);
- path.renameTo(new File(path.getParent(), name));
- }
- }
- }
- }
-
- public String getOutputDirectory() {
- return outDir;
- }
-
- public String getLogDirectory() {
- return logDir;
- }
-
private String getHadoopMainVersion(String input) {
if (input == null) {
return null;
@@ -625,6 +551,7 @@ public class QTestUtil {
dataDir = new File(".").getAbsolutePath() + "/data/files";
}
testFiles = dataDir;
+ conf.set("test.data.dir", dataDir);
// Use path relative to dataDir directory if it is not specified
datasetDir = conf.get("test.data.set.files") == null
@@ -639,6 +566,7 @@ public class QTestUtil {
overWrite = "true".equalsIgnoreCase(System.getProperty("test.output.overwrite"));
init();
+ savedConf = new HiveConf(conf);
}
private String getScriptsDir() {
// Use the current directory if it is not specified
@@ -708,7 +636,7 @@ public class QTestUtil {
if (clusterType == MiniClusterType.druid || clusterType == MiniClusterType.druidKafka) {
final String tempDir = System.getProperty("test.tmp.dir");
druidCluster = new MiniDruidCluster("mini-druid",
- getLogDirectory(),
+ logDir,
tempDir,
setup.zkPort,
Utilities.jarFinderGetJar(MiniDruidCluster.class)
@@ -728,7 +656,7 @@ public class QTestUtil {
if(clusterType == MiniClusterType.kafka || clusterType == MiniClusterType.druidKafka) {
kafkaCluster = new SingleNodeKafkaCluster("kafka",
- getLogDirectory() + "/kafka-cluster",
+ logDir + "/kafka-cluster",
setup.zkPort
);
kafkaCluster.init(conf);
@@ -817,15 +745,7 @@ public class QTestUtil {
}
public void addFile(String queryFile) throws IOException {
- addFile(queryFile, false);
- }
-
- public void addFile(String queryFile, boolean partial) throws IOException {
- addFile(new File(queryFile));
- }
-
- public void addFile(File qf) throws IOException {
- addFile(qf, false);
+ addFile(new File(queryFile), false);
}
public void addFile(File qf, boolean partial) throws IOException {
@@ -1007,6 +927,40 @@ public class QTestUtil {
}
}
+ public void newSession() throws Exception {
+ newSession(true);
+ }
+
+ public void newSession(boolean canReuseSession) throws Exception {
+ // allocate and initialize a new conf since a test can
+ // modify conf by using 'set' commands
+ conf = new HiveConf(savedConf);
+ initConf();
+ initConfFromSetup();
+
+ // renew the metastore since the cluster type is unencrypted
+ db = Hive.get(conf); // propagate new conf to meta store
+
+ HiveConf.setVar(conf, HiveConf.ConfVars.HIVE_AUTHENTICATOR_MANAGER,
+ "org.apache.hadoop.hive.ql.security.DummyAuthenticator");
+ CliSessionState ss = new CliSessionState(conf);
+ assert ss != null;
+ ss.in = System.in;
+
+ SessionState oldSs = SessionState.get();
+
+ restartSessions(canReuseSession, ss, oldSs);
+
+ closeSession(oldSs);
+
+ SessionState.start(ss);
+
+ cliDriver = new CliDriver();
+
+ File outf = new File(logDir, "initialize.log");
+ setSessionOutputs("that_shouldnt_happen_there", ss, outf);
+
+ }
/**
* Clear out any side effects of running tests
*/
@@ -1014,36 +968,19 @@ public class QTestUtil {
if (System.getenv(QTEST_LEAVE_FILES) != null) {
return;
}
+ // the test might have configured security/etc; open a new session to get rid of that
+ newSession();
// Remove any cached results from the previous test.
+ Utilities.clearWorkMap(conf);
NotificationEventPoll.shutdown();
QueryResultsCache.cleanupInstance();
-
- // allocate and initialize a new conf since a test can
- // modify conf by using 'set' commands
- conf = new HiveConf(IDriver.class);
- initConf();
- initConfFromSetup();
-
- // renew the metastore since the cluster type is unencrypted
- db = Hive.get(conf); // propagate new conf to meta store
-
clearTablesCreatedDuringTests();
clearUDFsCreatedDuringTests();
clearKeysCreatedInTests();
StatsSources.clearGlobalStats();
}
- protected void clearSettingsCreatedInTests() throws IOException {
- getCliDriver().processLine(String.format("set hive.security.authorization.enabled=false;"));
- getCliDriver().processLine(String.format("set user.name=%s;",
- System.getProperty(TEST_HIVE_USER_PROPERTY, "hive_test_user")));
-
- getCliDriver().processLine("set hive.metastore.partition.name.whitelist.pattern=;");
- getCliDriver().processLine("set hive.test.mode=false;");
- getCliDriver().processLine("set hive.mapred.mode=nonstrict;");
- }
-
protected void initConfFromSetup() throws Exception {
setup.preTest(conf);
}
@@ -1060,6 +997,7 @@ public class QTestUtil {
if (System.getenv(QTEST_LEAVE_FILES) != null) {
return;
}
+ conf.setBoolean("hive.test.shutdown.phase", true);
clearTablesCreatedDuringTests();
clearUDFsCreatedDuringTests();
@@ -1160,24 +1098,27 @@ public class QTestUtil {
}
}
- private void initDataSetForTest(File file){
- getCliDriver().processLine("set test.data.dir=" + testFiles + ";");
+ private void initDataSetForTest(File file) throws Exception {
+ synchronized (QTestUtil.class) {
+ DatasetParser parser = new DatasetParser();
+ parser.parse(file);
- DatasetParser parser = new DatasetParser();
- parser.parse(file);
+ DatasetCollection datasets = parser.getDatasets();
- DatasetCollection datasets = parser.getDatasets();
- for (String table : datasets.getTables()){
- synchronized (QTestUtil.class){
+ Set<String> missingDatasets = datasets.getTables();
+ missingDatasets.removeAll(getSrcTables());
+ if (missingDatasets.isEmpty()) {
+ return;
+ }
+ newSession(true);
+ for (String table : missingDatasets) {
initDataset(table);
}
+ newSession(true);
}
}
- protected void initDataset(String table) {
- if (getSrcTables().contains(table)){
- return;
- }
+ protected void initDataset(String table) throws Exception {
File tableFile = new File(new File(datasetDir, table), Dataset.INIT_FILE_NAME);
String commands = null;
@@ -1225,30 +1166,21 @@ public class QTestUtil {
cliDriver.processCmd("set hive.cli.print.header=true;");
}
- public void cliInit(File file) throws Exception {
- cliInit(file, true);
- }
-
- public String cliInit(File file, boolean recreate) throws Exception {
+ public String cliInit(File file) throws Exception {
String fileName = file.getName();
- if (recreate) {
- cleanUp(fileName);
- createSources(fileName);
+ initDataSetForTest(file);
+
+ if (qNoSessionReuseQuerySet.contains(fileName)) {
+ newSession(false);
}
- clearSettingsCreatedInTests();
- initDataSetForTest(file);
+ CliSessionState ss = (CliSessionState) SessionState.get();
- HiveConf.setVar(conf, HiveConf.ConfVars.HIVE_AUTHENTICATOR_MANAGER,
- "org.apache.hadoop.hive.ql.security.DummyAuthenticator");
- Utilities.clearWorkMap(conf);
- CliSessionState ss = new CliSessionState(conf);
- assert ss != null;
- ss.in = System.in;
String outFileExtension = getOutFileExtension(fileName);
String stdoutName = null;
+
if (outDir != null) {
// TODO: why is this needed?
File qf = new File(outDir, fileName);
@@ -1256,22 +1188,9 @@ public class QTestUtil {
} else {
stdoutName = fileName + outFileExtension;
}
-
File outf = new File(logDir, stdoutName);
-
setSessionOutputs(fileName, ss, outf);
- SessionState oldSs = SessionState.get();
-
- boolean canReuseSession = !qNoSessionReuseQuerySet.contains(fileName);
- restartSessions(canReuseSession, ss, oldSs);
-
- closeSession(oldSs);
-
- SessionState.start(ss);
-
- cliDriver = new CliDriver();
-
if (fileName.equals("init_file.q")) {
ss.initFiles.add(AbstractCliConfig.HIVE_ROOT + "/data/scripts/test_init_file.sql");
}
@@ -1283,6 +1202,12 @@ public class QTestUtil {
private void setSessionOutputs(String fileName, CliSessionState ss, File outf)
throws FileNotFoundException, Exception, UnsupportedEncodingException {
OutputStream fo = new BufferedOutputStream(new FileOutputStream(outf));
+ if (ss.out != null) {
+ ss.out.flush();
+ }
+ if (ss.err != null) {
+ ss.out.flush();
+ }
if (qSortQuerySet.contains(fileName)) {
ss.out = new SortPrintStream(fo, "UTF-8");
} else if (qHashQuerySet.contains(fileName)) {
@@ -1955,7 +1880,7 @@ public class QTestUtil {
qt.startSessionState(false);
// assumption is that environment has already been cleaned once globally
// hence each thread does not call cleanUp() and createSources() again
- qt.cliInit(file, false);
+ qt.cliInit(file);
qt.executeClient(file.getName());
} catch (Throwable e) {
System.err.println("Query file " + file.getName() + " failed with exception "
@@ -1986,7 +1911,7 @@ public class QTestUtil {
qt[i] = new QTestUtil(resDir, logDir, MiniClusterType.none, null, "0.20",
initScript == null ? defaultInitScript : initScript,
cleanupScript == null ? defaultCleanupScript : cleanupScript, false);
- qt[i].addFile(qfiles[i]);
+ qt[i].addFile(qfiles[i], false);
qt[i].clearTestSideEffects();
}
@@ -2011,7 +1936,7 @@ public class QTestUtil {
qt[0].createSources();
for (int i = 0; i < qfiles.length && !failed; i++) {
qt[i].clearTestSideEffects();
- qt[i].cliInit(qfiles[i], false);
+ qt[i].cliInit(qfiles[i]);
qt[i].executeClient(qfiles[i].getName());
QTestProcessExecResult result = qt[i].checkCliDriverResults(qfiles[i].getName());
if (result.getReturnCode() != 0) {
@@ -2190,41 +2115,6 @@ public class QTestUtil {
(debugHint != null ? debugHint : ""));
}
- public static void addTestsToSuiteFromQfileNames(
- String qFileNamesFile,
- Set<String> qFilesToExecute,
- TestSuite suite,
- Object setup,
- SuiteAddTestFunctor suiteAddTestCallback) {
- try {
- File qFileNames = new File(qFileNamesFile);
- FileReader fr = new FileReader(qFileNames.getCanonicalFile());
- BufferedReader br = new BufferedReader(fr);
- String fName = null;
-
- while ((fName = br.readLine()) != null) {
- if (fName.isEmpty() || fName.trim().equals("")) {
- continue;
- }
-
- int eIdx = fName.indexOf('.');
-
- if (eIdx == -1) {
- continue;
- }
-
- String tName = fName.substring(0, eIdx);
-
- if (qFilesToExecute.isEmpty() || qFilesToExecute.contains(fName)) {
- suiteAddTestCallback.addTestToSuite(suite, setup, tName);
- }
- }
- br.close();
- } catch (Exception e) {
- Assert.fail("Unexpected exception " + org.apache.hadoop.util.StringUtils.stringifyException(e));
- }
- }
-
public QOutProcessor getQOutProcessor() {
return qOutProcessor;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/e6577a0d/itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java
index a071689..8f5744d 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java
@@ -57,6 +57,7 @@ public class CoreParseNegative extends CliAdapter{
String hadoopVer = cliConfig.getHadoopVersion();
qt = new QTestUtil((cliConfig.getResultsDir()), (cliConfig.getLogDir()), miniMR, null,
hadoopVer, initScript, cleanupScript, false);
+ qt.newSession();
} catch (Exception e) {
System.err.println("Exception: " + e.getMessage());
e.printStackTrace();
@@ -106,7 +107,7 @@ public class CoreParseNegative extends CliAdapter{
firstRun = false;
}
- qt.cliInit(new File(fpath), false);
+ qt.cliInit(new File(fpath));
ASTNode tree = qt.parseQuery(fname);
List<Task<? extends Serializable>> tasks = qt.analyzeAST(tree);
http://git-wip-us.apache.org/repos/asf/hive/blob/e6577a0d/ql/src/test/queries/clientpositive/druidkafkamini_basic.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/druidkafkamini_basic.q b/ql/src/test/queries/clientpositive/druidkafkamini_basic.q
index 814890a..2eb9dbc 100644
--- a/ql/src/test/queries/clientpositive/druidkafkamini_basic.q
+++ b/ql/src/test/queries/clientpositive/druidkafkamini_basic.q
@@ -16,7 +16,7 @@ CREATE TABLE druid_kafka_test(`__time` timestamp, page string, `user` string, la
ALTER TABLE druid_kafka_test SET TBLPROPERTIES('druid.kafka.ingestion' = 'START');
-!curl -ss http://localhost:8081/druid/indexer/v1/supervisor;
+!curl --noproxy * -ss http://localhost:8081/druid/indexer/v1/supervisor;
-- Sleep for some time for ingestion tasks to ingest events
!sleep 60;
http://git-wip-us.apache.org/repos/asf/hive/blob/e6577a0d/ql/src/test/queries/positive/input_testsequencefile.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/positive/input_testsequencefile.q b/ql/src/test/queries/positive/input_testsequencefile.q
index 2daa53a..979bb9a 100755
--- a/ql/src/test/queries/positive/input_testsequencefile.q
+++ b/ql/src/test/queries/positive/input_testsequencefile.q
@@ -1,3 +1,14 @@
--! qt:dataset:src
+
+--
+-- Table dest4_sequencefile
+--
+DROP TABLE IF EXISTS dest4_sequencefile;
+
+CREATE TABLE dest4_sequencefile (key STRING COMMENT 'default', value STRING COMMENT 'default')
+STORED AS
+INPUTFORMAT 'org.apache.hadoop.mapred.SequenceFileInputFormat'
+OUTPUTFORMAT 'org.apache.hadoop.mapred.SequenceFileOutputFormat';
+
FROM src
INSERT OVERWRITE TABLE dest4_sequencefile SELECT src.key, src.value
[06/12] hive git commit: HIVE-15190: Field names are not preserved in
ORC files written with ACID (Anthony Hsu reviewed by Prasanth Jayachandran)
Posted by se...@apache.org.
HIVE-15190: Field names are not preserved in ORC files written with ACID
(Anthony Hsu reviewed by Prasanth Jayachandran)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/aea59083
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/aea59083
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/aea59083
Branch: refs/heads/master-txnstats
Commit: aea5908341c7b7084551216efc6c89d64b1cb06a
Parents: 6c4d7a9
Author: Prasanth Jayachandran <pr...@apache.org>
Authored: Wed Jun 20 10:24:36 2018 -0700
Committer: Prasanth Jayachandran <pr...@apache.org>
Committed: Wed Jun 20 10:24:41 2018 -0700
----------------------------------------------------------------------
.../hive/ql/txn/compactor/TestCompactor.java | 2 +-
.../apache/hadoop/hive/ql/io/orc/OrcFile.java | 6 +-
.../hive/ql/io/orc/OrcRawRecordMerger.java | 12 +++-
.../hadoop/hive/ql/io/orc/OrcRecordUpdater.java | 52 ++++++++++++++++-
.../io/orc/VectorizedOrcAcidRowBatchReader.java | 2 +-
.../apache/hadoop/hive/ql/TestTxnCommands2.java | 60 ++++++++++++++++++++
.../results/clientpositive/acid_nullscan.q.out | 8 +--
.../clientpositive/acid_table_stats.q.out | 14 ++---
.../clientpositive/autoColumnStats_4.q.out | 4 +-
.../llap/acid_bucket_pruning.q.out | 6 +-
.../llap/acid_vectorization_original.q.out | 4 +-
.../llap/default_constraint.q.out | 14 ++---
.../llap/dynpart_sort_optimization_acid.q.out | 14 ++---
.../insert_values_orig_table_use_metadata.q.out | 24 ++++----
.../materialized_view_create_rewrite_3.q.out | 20 +++----
.../materialized_view_create_rewrite_4.q.out | 22 +++----
.../materialized_view_create_rewrite_5.q.out | 34 +++++------
...ized_view_create_rewrite_rebuild_dummy.q.out | 20 +++----
.../test/results/clientpositive/row__id.q.out | 18 +++---
.../tez/acid_vectorization_original_tez.q.out | 4 +-
.../clientpositive/tez/explainanalyze_5.q.out | 2 +-
21 files changed, 228 insertions(+), 114 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/aea59083/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
index 4a0e834..cffa21a 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
@@ -1596,7 +1596,7 @@ public class TestCompactor {
"'transactional'='true'," +
"'compactor.mapreduce.map.memory.mb'='2048'," + // 2048 MB memory for compaction map job
"'compactorthreshold.hive.compactor.delta.num.threshold'='4'," + // minor compaction if more than 4 delta dirs
- "'compactorthreshold.hive.compactor.delta.pct.threshold'='0.49'" + // major compaction if more than 49%
+ "'compactorthreshold.hive.compactor.delta.pct.threshold'='0.47'" + // major compaction if more than 47%
")", driver);
// Insert 5 rows to both tables
http://git-wip-us.apache.org/repos/asf/hive/blob/aea59083/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFile.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFile.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFile.java
index 68e88cf..56d590c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFile.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFile.java
@@ -125,8 +125,10 @@ public final class OrcFile extends org.apache.orc.OrcFile {
* @return this
*/
public WriterOptions setSchema(TypeDescription schema) {
- this.explicitSchema = true;
- super.setSchema(schema);
+ if (schema != null) {
+ this.explicitSchema = true;
+ super.setSchema(schema);
+ }
return this;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/aea59083/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRawRecordMerger.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRawRecordMerger.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRawRecordMerger.java
index 6571a24..929ea9b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRawRecordMerger.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRawRecordMerger.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.hive.shims.HadoopShims;
import org.apache.orc.OrcUtils;
import org.apache.orc.StripeInformation;
import org.apache.orc.TypeDescription;
+import org.apache.orc.impl.SchemaEvolution;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
@@ -773,9 +774,10 @@ public class OrcRawRecordMerger implements AcidInputFormat.RawReader<OrcStruct>{
* Convert from the row include/sarg/columnNames to the event equivalent
* for the underlying file.
* @param options options for the row reader
+ * @param rowSchema schema of the row, excluding ACID columns
* @return a cloned options object that is modified for the event reader
*/
- static Reader.Options createEventOptions(Reader.Options options) {
+ static Reader.Options createEventOptions(Reader.Options options, TypeDescription rowSchema) {
Reader.Options result = options.clone();
result.include(options.getInclude());
@@ -788,6 +790,10 @@ public class OrcRawRecordMerger implements AcidInputFormat.RawReader<OrcStruct>{
}
result.searchArgument(options.getSearchArgument(), cols);
}
+
+ // schema evolution will insert the acid columns to row schema for ACID read
+ result.schema(rowSchema);
+
return result;
}
@@ -970,12 +976,12 @@ public class OrcRawRecordMerger implements AcidInputFormat.RawReader<OrcStruct>{
TypeDescription typeDescr =
OrcInputFormat.getDesiredRowTypeDescr(conf, true, Integer.MAX_VALUE);
- objectInspector = OrcRecordUpdater.createEventSchema
+ objectInspector = OrcRecordUpdater.createEventObjectInspector
(OrcStruct.createObjectInspector(0, OrcUtils.getOrcTypes(typeDescr)));
assert !(mergerOptions.isCompacting() && reader != null) : "don't need a reader for compaction";
// modify the options to reflect the event instead of the base row
- Reader.Options eventOptions = createEventOptions(options);
+ Reader.Options eventOptions = createEventOptions(options, typeDescr);
//suppose it's the first Major compaction so we only have deltas
boolean isMajorNoBase = mergerOptions.isCompacting() && mergerOptions.isMajorCompaction()
&& mergerOptions.getBaseDir() == null;
http://git-wip-us.apache.org/repos/asf/hive/blob/aea59083/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java
index 1359dc3..5590470 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java
@@ -23,7 +23,9 @@ import java.nio.charset.CharacterCodingException;
import java.nio.charset.Charset;
import java.nio.charset.CharsetDecoder;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
+import java.util.Properties;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
@@ -33,6 +35,7 @@ import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.io.AcidOutputFormat;
import org.apache.hadoop.hive.ql.io.AcidUtils;
import org.apache.hadoop.hive.ql.io.BucketCodec;
+import org.apache.hadoop.hive.ql.io.IOConstants;
import org.apache.hadoop.hive.ql.io.RecordIdentifier;
import org.apache.hadoop.hive.ql.io.RecordUpdater;
import org.apache.hadoop.hive.serde2.SerDeStats;
@@ -42,15 +45,21 @@ import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.orc.OrcConf;
+import org.apache.orc.TypeDescription;
import org.apache.orc.impl.AcidStats;
import org.apache.orc.impl.OrcAcidUtils;
+import org.apache.orc.impl.SchemaEvolution;
import org.apache.orc.impl.WriterImpl;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.base.Strings;
+
/**
* A RecordUpdater where the files are stored as ORC.
* A note on various record structures: the {@code row} coming in (as in {@link #insert(long, Object)}
@@ -179,7 +188,7 @@ public class OrcRecordUpdater implements RecordUpdater {
* @param rowInspector the row's object inspector
* @return an object inspector for the event stream
*/
- static StructObjectInspector createEventSchema(ObjectInspector rowInspector) {
+ static StructObjectInspector createEventObjectInspector(ObjectInspector rowInspector) {
List<StructField> fields = new ArrayList<StructField>();
fields.add(new OrcStruct.Field("operation",
PrimitiveObjectInspectorFactory.writableIntObjectInspector, OPERATION));
@@ -194,6 +203,41 @@ public class OrcRecordUpdater implements RecordUpdater {
fields.add(new OrcStruct.Field("row", rowInspector, ROW));
return new OrcStruct.OrcStructInspector(fields);
}
+
+ private static TypeDescription createEventSchemaFromTableProperties(Properties tableProps) {
+ TypeDescription rowSchema = getTypeDescriptionFromTableProperties(tableProps);
+ if (rowSchema == null) {
+ return null;
+ }
+
+ return SchemaEvolution.createEventSchema(rowSchema);
+ }
+
+ private static TypeDescription getTypeDescriptionFromTableProperties(Properties tableProperties) {
+ TypeDescription schema = null;
+ if (tableProperties != null) {
+ final String columnNameProperty = tableProperties.getProperty(IOConstants.COLUMNS);
+ final String columnTypeProperty = tableProperties.getProperty(IOConstants.COLUMNS_TYPES);
+ if (!Strings.isNullOrEmpty(columnNameProperty) && !Strings.isNullOrEmpty(columnTypeProperty)) {
+ List<String> columnNames =
+ columnNameProperty.length() == 0 ? new ArrayList<String>() : Arrays.asList(columnNameProperty.split(","));
+ List<TypeInfo> columnTypes = columnTypeProperty.length() == 0 ? new ArrayList<TypeInfo>() : TypeInfoUtils
+ .getTypeInfosFromTypeString(columnTypeProperty);
+
+ schema = TypeDescription.createStruct();
+ for (int i = 0; i < columnNames.size(); i++) {
+ schema.addField(columnNames.get(i), OrcInputFormat.convertTypeInfo(columnTypes.get(i)));
+ }
+ }
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("ORC schema = " + schema);
+ }
+
+ return schema;
+ }
+
/**
* @param partitionRoot - partition root (or table root if not partitioned)
*/
@@ -282,8 +326,9 @@ public class OrcRecordUpdater implements RecordUpdater {
*/
this.deleteWriterOptions = OrcFile.writerOptions(optionsCloneForDelta.getTableProperties(),
optionsCloneForDelta.getConfiguration());
- this.deleteWriterOptions.inspector(createEventSchema(findRecId(options.getInspector(),
+ this.deleteWriterOptions.inspector(createEventObjectInspector(findRecId(options.getInspector(),
options.getRecordIdColumn())));
+ this.deleteWriterOptions.setSchema(createEventSchemaFromTableProperties(options.getTableProperties()));
}
// get buffer size and stripe size for base writer
@@ -305,8 +350,9 @@ public class OrcRecordUpdater implements RecordUpdater {
}
writerOptions.fileSystem(fs).callback(indexBuilder);
rowInspector = (StructObjectInspector)options.getInspector();
- writerOptions.inspector(createEventSchema(findRecId(options.getInspector(),
+ writerOptions.inspector(createEventObjectInspector(findRecId(options.getInspector(),
options.getRecordIdColumn())));
+ writerOptions.setSchema(createEventSchemaFromTableProperties(options.getTableProperties()));
item = new OrcStruct(FIELDS);
item.setFieldValue(OPERATION, operation);
item.setFieldValue(CURRENT_WRITEID, currentWriteId);
http://git-wip-us.apache.org/repos/asf/hive/blob/aea59083/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java
index 1a91d4a..e181d8c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java
@@ -182,7 +182,7 @@ public class VectorizedOrcAcidRowBatchReader
}
reporter.setStatus(orcSplit.toString());
- readerOptions = OrcRawRecordMerger.createEventOptions(OrcInputFormat.createOptionsForReader(conf));
+ readerOptions = OrcInputFormat.createOptionsForReader(conf);
this.offset = orcSplit.getStart();
this.length = orcSplit.getLength();
http://git-wip-us.apache.org/repos/asf/hive/blob/aea59083/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
index 7c201b6..3e98a09 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
@@ -50,6 +50,7 @@ import org.apache.hadoop.hive.metastore.txn.TxnDbUtil;
import org.apache.hadoop.hive.metastore.txn.TxnStore;
import org.apache.hadoop.hive.metastore.txn.TxnUtils;
import org.apache.hadoop.hive.ql.io.AcidOutputFormat;
+import org.apache.hadoop.hive.ql.io.AcidUtils;
import org.apache.hadoop.hive.ql.io.BucketCodec;
import org.apache.hadoop.hive.ql.io.HiveInputFormat;
import org.apache.hadoop.hive.ql.lockmgr.HiveTxnManager;
@@ -60,6 +61,9 @@ import org.apache.hadoop.hive.metastore.txn.AcidOpenTxnsCounterService;
import org.apache.hadoop.hive.ql.txn.compactor.Cleaner;
import org.apache.hadoop.hive.ql.txn.compactor.Initiator;
import org.apache.hadoop.hive.ql.txn.compactor.Worker;
+import org.apache.orc.OrcFile;
+import org.apache.orc.Reader;
+import org.apache.orc.TypeDescription;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -2168,6 +2172,62 @@ public class TestTxnCommands2 {
Assert.assertEquals(stringifyValues(resultData), rs);
}
+ @Test
+ public void testAcidOrcWritePreservesFieldNames() throws Exception {
+ // with vectorization
+ String tableName = "acidorcwritefieldnames";
+ hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED, true);
+ runStatementOnDriver("DROP TABLE IF EXISTS " + tableName);
+ runStatementOnDriver("CREATE TABLE " + tableName + " (a INT, b STRING) CLUSTERED BY (a) INTO " + BUCKET_COUNT + " BUCKETS STORED AS ORC TBLPROPERTIES ('transactional'='true')");
+ runStatementOnDriver("INSERT INTO " + tableName + " VALUES (1, 'foo'), (2, 'bar')");
+
+ tableName = "acidorcwritefieldnames_complex";
+ runStatementOnDriver("DROP TABLE IF EXISTS " + tableName);
+ runStatementOnDriver("CREATE TABLE " + tableName + " (a INT, b STRING, s STRUCT<c:int, si:STRUCT<d:double," +
+ "e:float>>) CLUSTERED BY (a) INTO " + BUCKET_COUNT +
+ " BUCKETS STORED AS ORC TBLPROPERTIES ('transactional'='true')");
+ runStatementOnDriver("INSERT INTO " + tableName + " select a, b, named_struct('c',10,'si'," +
+ "named_struct('d',cast(1.0 as double),'e',cast(2.0 as float))) from acidorcwritefieldnames");
+
+ FileSystem fs = FileSystem.get(hiveConf);
+ FileStatus[] fileStatuses = fs.globStatus(new Path(TEST_WAREHOUSE_DIR + "/" + tableName + "/" + AcidUtils.DELTA_PREFIX + "*/" + AcidUtils.BUCKET_PREFIX + "*"));
+ Assert.assertEquals(BUCKET_COUNT, fileStatuses.length);
+
+ OrcFile.ReaderOptions readerOptions = OrcFile.readerOptions(hiveConf);
+ for (FileStatus fileStatus : fileStatuses) {
+ Reader r = OrcFile.createReader(fileStatus.getPath(), readerOptions);
+ TypeDescription rowSchema = r.getSchema().getChildren().get(5);
+ Assert.assertEquals("struct<a:int,b:string,s:struct<c:int,si:struct<d:double,e:float>>>", rowSchema.toString());
+ }
+
+ // without vectorization
+ tableName = "acidorcwritefieldnames";
+ hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED, false);
+ runStatementOnDriver("DROP TABLE IF EXISTS " + tableName);
+ runStatementOnDriver("CREATE TABLE " + tableName + " (a INT, b STRING) CLUSTERED BY (a) INTO " + BUCKET_COUNT + " BUCKETS STORED AS ORC TBLPROPERTIES ('transactional'='true')");
+ runStatementOnDriver("INSERT INTO " + tableName + " VALUES (1, 'foo'), (2, 'bar')");
+
+ tableName = "acidorcwritefieldnames_complex";
+ runStatementOnDriver("DROP TABLE IF EXISTS " + tableName);
+ runStatementOnDriver("CREATE TABLE " + tableName + " (a INT, b STRING, s STRUCT<c:int, si:STRUCT<d:double," +
+ "e:float>>) CLUSTERED BY (a) INTO " + BUCKET_COUNT +
+ " BUCKETS STORED AS ORC TBLPROPERTIES ('transactional'='true')");
+ runStatementOnDriver("INSERT INTO " + tableName + " select a, b, named_struct('c',10,'si'," +
+ "named_struct('d',cast(1.0 as double),'e',cast(2.0 as float))) from acidorcwritefieldnames");
+
+ fs = FileSystem.get(hiveConf);
+ fileStatuses = fs.globStatus(new Path(TEST_WAREHOUSE_DIR + "/" + tableName + "/" + AcidUtils.DELTA_PREFIX + "*/" + AcidUtils.BUCKET_PREFIX + "*"));
+ Assert.assertEquals(BUCKET_COUNT, fileStatuses.length);
+
+ readerOptions = OrcFile.readerOptions(hiveConf);
+ for (FileStatus fileStatus : fileStatuses) {
+ Reader r = OrcFile.createReader(fileStatus.getPath(), readerOptions);
+ TypeDescription rowSchema = r.getSchema().getChildren().get(5);
+ Assert.assertEquals("struct<a:int,b:string,s:struct<c:int,si:struct<d:double,e:float>>>", rowSchema.toString());
+ }
+ hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED, true);
+ }
+
/**
* takes raw data and turns it into a string as if from Driver.getResults()
* sorts rows in dictionary order
http://git-wip-us.apache.org/repos/asf/hive/blob/aea59083/ql/src/test/results/clientpositive/acid_nullscan.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/acid_nullscan.q.out b/ql/src/test/results/clientpositive/acid_nullscan.q.out
index ccd33ae..c9684dd 100644
--- a/ql/src/test/results/clientpositive/acid_nullscan.q.out
+++ b/ql/src/test/results/clientpositive/acid_nullscan.q.out
@@ -42,12 +42,12 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: acid_vectorized_n1
- Statistics: Num rows: 90 Data size: 26090 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 90 Data size: 25960 Basic stats: COMPLETE Column stats: NONE
GatherStats: false
Filter Operator
isSamplingPred: false
predicate: false (type: boolean)
- Statistics: Num rows: 1 Data size: 289 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 288 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: sum(a)
mode: hash
@@ -82,7 +82,7 @@ STAGE PLANS:
serialization.ddl struct acid_vectorized_n1 { i32 a, string b}
serialization.format 1
serialization.lib org.apache.hadoop.hive.serde2.NullStructSerDe
- totalSize 2609
+ totalSize 2596
transactional true
transactional_properties default
#### A masked pattern was here ####
@@ -104,7 +104,7 @@ STAGE PLANS:
serialization.ddl struct acid_vectorized_n1 { i32 a, string b}
serialization.format 1
serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde
- totalSize 2609
+ totalSize 2596
transactional true
transactional_properties default
#### A masked pattern was here ####
http://git-wip-us.apache.org/repos/asf/hive/blob/aea59083/ql/src/test/results/clientpositive/acid_table_stats.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/acid_table_stats.q.out b/ql/src/test/results/clientpositive/acid_table_stats.q.out
index b266794..c028842 100644
--- a/ql/src/test/results/clientpositive/acid_table_stats.q.out
+++ b/ql/src/test/results/clientpositive/acid_table_stats.q.out
@@ -94,7 +94,7 @@ Table: acid
#### A masked pattern was here ####
Partition Parameters:
numFiles 2
- totalSize 4064
+ totalSize 4063
#### A masked pattern was here ####
# Storage Information
@@ -132,9 +132,9 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: acid
- Statistics: Num rows: 83 Data size: 40640 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 83 Data size: 40630 Basic stats: COMPLETE Column stats: NONE
Select Operator
- Statistics: Num rows: 83 Data size: 40640 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 83 Data size: 40630 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count()
mode: hash
@@ -210,7 +210,7 @@ Partition Parameters:
numFiles 2
numRows 1000
rawDataSize 208000
- totalSize 4064
+ totalSize 4063
#### A masked pattern was here ####
# Storage Information
@@ -261,7 +261,7 @@ Partition Parameters:
numFiles 2
numRows 1000
rawDataSize 208000
- totalSize 4064
+ totalSize 4063
#### A masked pattern was here ####
# Storage Information
@@ -385,7 +385,7 @@ Table: acid
#### A masked pattern was here ####
Partition Parameters:
numFiles 4
- totalSize 8119
+ totalSize 8118
#### A masked pattern was here ####
# Storage Information
@@ -432,7 +432,7 @@ Partition Parameters:
numFiles 4
numRows 2000
rawDataSize 416000
- totalSize 8119
+ totalSize 8118
#### A masked pattern was here ####
# Storage Information
http://git-wip-us.apache.org/repos/asf/hive/blob/aea59083/ql/src/test/results/clientpositive/autoColumnStats_4.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/autoColumnStats_4.q.out b/ql/src/test/results/clientpositive/autoColumnStats_4.q.out
index d651887..def65ff 100644
--- a/ql/src/test/results/clientpositive/autoColumnStats_4.q.out
+++ b/ql/src/test/results/clientpositive/autoColumnStats_4.q.out
@@ -200,7 +200,7 @@ Table Type: MANAGED_TABLE
Table Parameters:
bucketing_version 2
numFiles 2
- totalSize 1907
+ totalSize 1899
transactional true
transactional_properties default
#### A masked pattern was here ####
@@ -243,7 +243,7 @@ Table Parameters:
COLUMN_STATS_ACCURATE {}
bucketing_version 2
numFiles 4
- totalSize 3091
+ totalSize 3275
transactional true
transactional_properties default
#### A masked pattern was here ####
http://git-wip-us.apache.org/repos/asf/hive/blob/aea59083/ql/src/test/results/clientpositive/llap/acid_bucket_pruning.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/acid_bucket_pruning.q.out b/ql/src/test/results/clientpositive/llap/acid_bucket_pruning.q.out
index e9fef82..b856b99 100644
--- a/ql/src/test/results/clientpositive/llap/acid_bucket_pruning.q.out
+++ b/ql/src/test/results/clientpositive/llap/acid_bucket_pruning.q.out
@@ -45,7 +45,7 @@ STAGE PLANS:
alias: acidtbldefault
filterExpr: (a = 1) (type: boolean)
buckets included: [13,] of 16
- Statistics: Num rows: 1854 Data size: 7052 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1850 Data size: 7036 Basic stats: COMPLETE Column stats: NONE
GatherStats: false
Filter Operator
isSamplingPred: false
@@ -102,7 +102,7 @@ STAGE PLANS:
serialization.ddl struct acidtbldefault { i32 a}
serialization.format 1
serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde
- totalSize 34863
+ totalSize 34795
transactional true
transactional_properties default
#### A masked pattern was here ####
@@ -125,7 +125,7 @@ STAGE PLANS:
serialization.ddl struct acidtbldefault { i32 a}
serialization.format 1
serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde
- totalSize 34863
+ totalSize 34795
transactional true
transactional_properties default
#### A masked pattern was here ####
http://git-wip-us.apache.org/repos/asf/hive/blob/aea59083/ql/src/test/results/clientpositive/llap/acid_vectorization_original.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/acid_vectorization_original.q.out b/ql/src/test/results/clientpositive/llap/acid_vectorization_original.q.out
index 66bb2db..be1b4c6 100644
--- a/ql/src/test/results/clientpositive/llap/acid_vectorization_original.q.out
+++ b/ql/src/test/results/clientpositive/llap/acid_vectorization_original.q.out
@@ -665,11 +665,11 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: over10k_orc_bucketed
- Statistics: Num rows: 1237 Data size: 707880 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1237 Data size: 707670 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: ROW__ID (type: struct<writeid:bigint,bucketid:int,rowid:bigint>)
outputColumnNames: ROW__ID
- Statistics: Num rows: 1237 Data size: 707880 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1237 Data size: 707670 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: count()
keys: ROW__ID (type: struct<writeid:bigint,bucketid:int,rowid:bigint>)
http://git-wip-us.apache.org/repos/asf/hive/blob/aea59083/ql/src/test/results/clientpositive/llap/default_constraint.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/default_constraint.q.out b/ql/src/test/results/clientpositive/llap/default_constraint.q.out
index 3365569..775cba8 100644
--- a/ql/src/test/results/clientpositive/llap/default_constraint.q.out
+++ b/ql/src/test/results/clientpositive/llap/default_constraint.q.out
@@ -1498,7 +1498,7 @@ Table Type: MANAGED_TABLE
Table Parameters:
bucketing_version 2
numFiles 1
- totalSize 1102
+ totalSize 1096
transactional true
transactional_properties default
#### A masked pattern was here ####
@@ -1668,7 +1668,7 @@ Table Parameters:
bucketing_version 2
#### A masked pattern was here ####
numFiles 2
- totalSize 2204
+ totalSize 2193
transactional true
transactional_properties default
#### A masked pattern was here ####
@@ -1748,7 +1748,7 @@ Table Parameters:
bucketing_version 2
#### A masked pattern was here ####
numFiles 2
- totalSize 2204
+ totalSize 2193
transactional true
transactional_properties default
#### A masked pattern was here ####
@@ -1925,7 +1925,7 @@ Table Parameters:
bucketing_version 2
#### A masked pattern was here ####
numFiles 3
- totalSize 3296
+ totalSize 3278
transactional true
transactional_properties default
#### A masked pattern was here ####
@@ -2004,7 +2004,7 @@ Table Parameters:
bucketing_version 2
#### A masked pattern was here ####
numFiles 3
- totalSize 3296
+ totalSize 3278
transactional true
transactional_properties default
#### A masked pattern was here ####
@@ -2084,7 +2084,7 @@ Table Parameters:
bucketing_version 2
#### A masked pattern was here ####
numFiles 3
- totalSize 3296
+ totalSize 3278
transactional true
transactional_properties default
#### A masked pattern was here ####
@@ -2660,7 +2660,7 @@ Table Type: MANAGED_TABLE
Table Parameters:
bucketing_version 2
numFiles 1
- totalSize 1102
+ totalSize 1096
transactional true
transactional_properties default
#### A masked pattern was here ####
http://git-wip-us.apache.org/repos/asf/hive/blob/aea59083/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization_acid.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization_acid.q.out b/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization_acid.q.out
index 8dff107..49e5e25 100644
--- a/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization_acid.q.out
+++ b/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization_acid.q.out
@@ -94,7 +94,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: acid_part
- Statistics: Num rows: 160 Data size: 61011 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 160 Data size: 61001 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (key = 'foo') (type: boolean)
Statistics: Num rows: 5 Data size: 1906 Basic stats: COMPLETE Column stats: NONE
@@ -189,7 +189,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: acid_part
- Statistics: Num rows: 159 Data size: 104357 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 159 Data size: 104317 Basic stats: COMPLETE Column stats: PARTIAL
Filter Operator
predicate: (key = 'foo') (type: boolean)
Statistics: Num rows: 5 Data size: 1355 Basic stats: COMPLETE Column stats: PARTIAL
@@ -380,7 +380,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: acid_part_sdpo
- Statistics: Num rows: 176 Data size: 67083 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 176 Data size: 67063 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (key = 'foo') (type: boolean)
Statistics: Num rows: 5 Data size: 1905 Basic stats: COMPLETE Column stats: NONE
@@ -475,7 +475,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: acid_part_sdpo
- Statistics: Num rows: 171 Data size: 112202 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 171 Data size: 112152 Basic stats: COMPLETE Column stats: PARTIAL
Filter Operator
predicate: (key = 'foo') (type: boolean)
Statistics: Num rows: 5 Data size: 1355 Basic stats: COMPLETE Column stats: PARTIAL
@@ -675,7 +675,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: acid_2l_part
- Statistics: Num rows: 157 Data size: 60537 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 157 Data size: 60527 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (key = 'foo') (type: boolean)
Statistics: Num rows: 5 Data size: 1927 Basic stats: COMPLETE Column stats: NONE
@@ -1095,7 +1095,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: acid_2l_part_sdpo
- Statistics: Num rows: 157 Data size: 60537 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 157 Data size: 60527 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (key = 'foo') (type: boolean)
Statistics: Num rows: 5 Data size: 1927 Basic stats: COMPLETE Column stats: NONE
@@ -1515,7 +1515,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: acid_2l_part_sdpo_no_cp
- Statistics: Num rows: 97 Data size: 82932 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 97 Data size: 82922 Basic stats: COMPLETE Column stats: PARTIAL
Filter Operator
predicate: (key = 'foo') (type: boolean)
Statistics: Num rows: 5 Data size: 1860 Basic stats: COMPLETE Column stats: PARTIAL
http://git-wip-us.apache.org/repos/asf/hive/blob/aea59083/ql/src/test/results/clientpositive/llap/insert_values_orig_table_use_metadata.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/insert_values_orig_table_use_metadata.q.out b/ql/src/test/results/clientpositive/llap/insert_values_orig_table_use_metadata.q.out
index d61917d..1c0841c 100644
--- a/ql/src/test/results/clientpositive/llap/insert_values_orig_table_use_metadata.q.out
+++ b/ql/src/test/results/clientpositive/llap/insert_values_orig_table_use_metadata.q.out
@@ -170,7 +170,7 @@ Table Type: MANAGED_TABLE
Table Parameters:
bucketing_version 2
numFiles 1
- totalSize 295544
+ totalSize 295583
transactional true
transactional_properties default
#### A masked pattern was here ####
@@ -205,9 +205,9 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: acid_ivot
- Statistics: Num rows: 5863 Data size: 2955440 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 5864 Data size: 2955830 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
- Statistics: Num rows: 5863 Data size: 2955440 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 5864 Data size: 2955830 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: count()
mode: hash
@@ -376,7 +376,7 @@ Table Type: MANAGED_TABLE
Table Parameters:
bucketing_version 2
numFiles 1
- totalSize 1625
+ totalSize 1663
transactional true
transactional_properties default
#### A masked pattern was here ####
@@ -411,9 +411,9 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: acid_ivot
- Statistics: Num rows: 32 Data size: 16250 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 32 Data size: 16630 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
- Statistics: Num rows: 32 Data size: 16250 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 32 Data size: 16630 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: count()
mode: hash
@@ -509,7 +509,7 @@ Table Type: MANAGED_TABLE
Table Parameters:
bucketing_version 2
numFiles 2
- totalSize 3250
+ totalSize 3326
transactional true
transactional_properties default
#### A masked pattern was here ####
@@ -544,9 +544,9 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: acid_ivot
- Statistics: Num rows: 64 Data size: 32500 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 65 Data size: 33260 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
- Statistics: Num rows: 64 Data size: 32500 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 65 Data size: 33260 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: count()
mode: hash
@@ -638,7 +638,7 @@ Table Type: MANAGED_TABLE
Table Parameters:
bucketing_version 2
numFiles 3
- totalSize 298795
+ totalSize 298909
transactional true
transactional_properties default
#### A masked pattern was here ####
@@ -673,9 +673,9 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: acid_ivot
- Statistics: Num rows: 5928 Data size: 2987950 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 5930 Data size: 2989090 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
- Statistics: Num rows: 5928 Data size: 2987950 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 5930 Data size: 2989090 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: count()
mode: hash
http://git-wip-us.apache.org/repos/asf/hive/blob/aea59083/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_3.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_3.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_3.q.out
index d7a8a64..237ff00 100644
--- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_3.q.out
+++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_3.q.out
@@ -749,19 +749,19 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: cmv_basetable_2
- Statistics: Num rows: 43 Data size: 4988 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 46 Data size: 5336 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: ((c > 10.1) and a is not null) (type: boolean)
- Statistics: Num rows: 14 Data size: 1624 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 15 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: a (type: int), c (type: decimal(10,2))
outputColumnNames: _col0, _col1
- Statistics: Num rows: 14 Data size: 1624 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 15 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 14 Data size: 1624 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 15 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: decimal(10,2))
Execution mode: llap
LLAP IO: may be used (ACID table)
@@ -775,7 +775,7 @@ STAGE PLANS:
0 _col0 (type: int)
1 _col0 (type: int)
outputColumnNames: _col0, _col2
- Statistics: Num rows: 23 Data size: 2668 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 25 Data size: 2900 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
keys: _col0 (type: int), _col2 (type: decimal(10,2))
mode: hash
@@ -1027,19 +1027,19 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: cmv_basetable_2
- Statistics: Num rows: 43 Data size: 4988 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 46 Data size: 5336 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: ((c > 10.1) and a is not null) (type: boolean)
- Statistics: Num rows: 14 Data size: 1624 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 15 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: a (type: int), c (type: decimal(10,2))
outputColumnNames: _col0, _col1
- Statistics: Num rows: 14 Data size: 1624 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 15 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 14 Data size: 1624 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 15 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: decimal(10,2))
Execution mode: llap
LLAP IO: may be used (ACID table)
@@ -1053,7 +1053,7 @@ STAGE PLANS:
0 _col0 (type: int)
1 _col0 (type: int)
outputColumnNames: _col0, _col2
- Statistics: Num rows: 23 Data size: 2668 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 25 Data size: 2900 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
keys: _col0 (type: int), _col2 (type: decimal(10,2))
mode: hash
http://git-wip-us.apache.org/repos/asf/hive/blob/aea59083/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_4.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_4.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_4.q.out
index 2f34aae..2b26eba 100644
--- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_4.q.out
+++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_4.q.out
@@ -937,7 +937,7 @@ Table Type: MATERIALIZED_VIEW
Table Parameters:
bucketing_version 2
numFiles 3
- totalSize 1512
+ totalSize 1508
transactional true
transactional_properties default
#### A masked pattern was here ####
@@ -1283,19 +1283,19 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: cmv_basetable_2_n2
- Statistics: Num rows: 72 Data size: 8640 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 75 Data size: 9000 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: ((c > 10) and a is not null) (type: boolean)
- Statistics: Num rows: 24 Data size: 2880 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 25 Data size: 3000 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: a (type: int), c (type: decimal(10,2)), d (type: int)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 24 Data size: 2880 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 25 Data size: 3000 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 24 Data size: 2880 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 25 Data size: 3000 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: decimal(10,2)), _col2 (type: int)
Execution mode: llap
LLAP IO: may be used (ACID table)
@@ -1309,7 +1309,7 @@ STAGE PLANS:
0 _col0 (type: int)
1 _col0 (type: int)
outputColumnNames: _col0, _col2, _col3
- Statistics: Num rows: 40 Data size: 4800 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 41 Data size: 4920 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: sum(_col3)
keys: _col0 (type: int), _col2 (type: decimal(10,2))
@@ -1532,19 +1532,19 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: cmv_basetable_2_n2
- Statistics: Num rows: 88 Data size: 10560 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 91 Data size: 10920 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: ((ROW__ID.writeid > 4) and (c > 10) and a is not null) (type: boolean)
- Statistics: Num rows: 9 Data size: 1080 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 10 Data size: 1200 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: a (type: int), c (type: decimal(10,2)), d (type: int)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 9 Data size: 1764 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 10 Data size: 1960 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 9 Data size: 1764 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 10 Data size: 1960 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: decimal(10,2)), _col2 (type: int)
Execution mode: llap
LLAP IO: may be used (ACID table)
@@ -1641,7 +1641,7 @@ STAGE PLANS:
0 _col0 (type: int)
1 _col0 (type: int)
outputColumnNames: _col0, _col2, _col3
- Statistics: Num rows: 15 Data size: 1800 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 16 Data size: 1920 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: sum(_col3)
keys: _col0 (type: int), _col2 (type: decimal(10,2))
http://git-wip-us.apache.org/repos/asf/hive/blob/aea59083/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_5.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_5.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_5.q.out
index 3d5acca..20146f8 100644
--- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_5.q.out
+++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_5.q.out
@@ -400,7 +400,7 @@ Table Type: MATERIALIZED_VIEW
Table Parameters:
bucketing_version 2
numFiles 2
- totalSize 1076
+ totalSize 1071
transactional true
transactional_properties default
#### A masked pattern was here ####
@@ -731,19 +731,19 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: cmv_basetable_2_n3
- Statistics: Num rows: 72 Data size: 8352 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 75 Data size: 8700 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: ((c > 10) and a is not null) (type: boolean)
- Statistics: Num rows: 24 Data size: 2784 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 25 Data size: 2900 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: a (type: int), c (type: decimal(10,2))
outputColumnNames: _col0, _col1
- Statistics: Num rows: 24 Data size: 2784 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 25 Data size: 2900 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 24 Data size: 2784 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 25 Data size: 2900 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: decimal(10,2))
Execution mode: llap
LLAP IO: may be used (ACID table)
@@ -757,14 +757,14 @@ STAGE PLANS:
0 _col0 (type: int)
1 _col0 (type: int)
outputColumnNames: _col0, _col2
- Statistics: Num rows: 40 Data size: 4640 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 41 Data size: 4756 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: _col0 (type: int), _col2 (type: decimal(10,2))
outputColumnNames: _col0, _col1
- Statistics: Num rows: 40 Data size: 4640 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 41 Data size: 4756 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 40 Data size: 4640 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 41 Data size: 4756 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
@@ -774,7 +774,7 @@ STAGE PLANS:
Select Operator
expressions: _col0 (type: int), _col1 (type: decimal(10,2))
outputColumnNames: a, c
- Statistics: Num rows: 40 Data size: 4640 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 41 Data size: 4756 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: compute_stats(a, 'hll'), compute_stats(c, 'hll')
mode: hash
@@ -943,19 +943,19 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: cmv_basetable_2_n3
- Statistics: Num rows: 88 Data size: 10208 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 91 Data size: 10556 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: ((ROW__ID.writeid > 4) and (c > 10) and a is not null) (type: boolean)
- Statistics: Num rows: 9 Data size: 1044 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 10 Data size: 1160 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: a (type: int), c (type: decimal(10,2))
outputColumnNames: _col0, _col1
- Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 10 Data size: 1920 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 9 Data size: 1728 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 10 Data size: 1920 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: decimal(10,2))
Execution mode: llap
LLAP IO: may be used (ACID table)
@@ -969,14 +969,14 @@ STAGE PLANS:
0 _col0 (type: int)
1 _col0 (type: int)
outputColumnNames: _col0, _col2
- Statistics: Num rows: 15 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 16 Data size: 1856 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: _col0 (type: int), _col2 (type: decimal(10,2))
outputColumnNames: _col0, _col1
- Statistics: Num rows: 15 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 16 Data size: 1856 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 15 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 16 Data size: 1856 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
@@ -986,7 +986,7 @@ STAGE PLANS:
Select Operator
expressions: _col0 (type: int), _col1 (type: decimal(10,2))
outputColumnNames: a, c
- Statistics: Num rows: 15 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 16 Data size: 1856 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: compute_stats(a, 'hll'), compute_stats(c, 'hll')
mode: hash
http://git-wip-us.apache.org/repos/asf/hive/blob/aea59083/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_rebuild_dummy.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_rebuild_dummy.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_rebuild_dummy.q.out
index c85be4a..3b7fb57 100644
--- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_rebuild_dummy.q.out
+++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_rebuild_dummy.q.out
@@ -749,19 +749,19 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: cmv_basetable_2_n0
- Statistics: Num rows: 43 Data size: 4988 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 46 Data size: 5336 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: ((c > 10.1) and a is not null) (type: boolean)
- Statistics: Num rows: 14 Data size: 1624 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 15 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: a (type: int), c (type: decimal(10,2))
outputColumnNames: _col0, _col1
- Statistics: Num rows: 14 Data size: 1624 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 15 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 14 Data size: 1624 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 15 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: decimal(10,2))
Execution mode: llap
LLAP IO: may be used (ACID table)
@@ -775,7 +775,7 @@ STAGE PLANS:
0 _col0 (type: int)
1 _col0 (type: int)
outputColumnNames: _col0, _col2
- Statistics: Num rows: 23 Data size: 2668 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 25 Data size: 2900 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
keys: _col0 (type: int), _col2 (type: decimal(10,2))
mode: hash
@@ -1027,19 +1027,19 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: cmv_basetable_2_n0
- Statistics: Num rows: 43 Data size: 4988 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 46 Data size: 5336 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: ((c > 10.1) and a is not null) (type: boolean)
- Statistics: Num rows: 14 Data size: 1624 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 15 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: a (type: int), c (type: decimal(10,2))
outputColumnNames: _col0, _col1
- Statistics: Num rows: 14 Data size: 1624 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 15 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 14 Data size: 1624 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 15 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: decimal(10,2))
Execution mode: llap
LLAP IO: may be used (ACID table)
@@ -1053,7 +1053,7 @@ STAGE PLANS:
0 _col0 (type: int)
1 _col0 (type: int)
outputColumnNames: _col0, _col2
- Statistics: Num rows: 23 Data size: 2668 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 25 Data size: 2900 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
keys: _col0 (type: int), _col2 (type: decimal(10,2))
mode: hash
http://git-wip-us.apache.org/repos/asf/hive/blob/aea59083/ql/src/test/results/clientpositive/row__id.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/row__id.q.out b/ql/src/test/results/clientpositive/row__id.q.out
index b4f3b8b..bad2d78 100644
--- a/ql/src/test/results/clientpositive/row__id.q.out
+++ b/ql/src/test/results/clientpositive/row__id.q.out
@@ -62,24 +62,24 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: hello_acid
- Statistics: Num rows: 78 Data size: 19830 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 78 Data size: 19860 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ROW__ID.writeid (type: bigint)
outputColumnNames: _col0
- Statistics: Num rows: 78 Data size: 19830 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 78 Data size: 19860 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: bigint)
sort order: +
- Statistics: Num rows: 78 Data size: 19830 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 78 Data size: 19860 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: bigint)
outputColumnNames: _col0
- Statistics: Num rows: 78 Data size: 19830 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 78 Data size: 19860 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 78 Data size: 19830 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 78 Data size: 19860 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -124,17 +124,17 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: hello_acid
- Statistics: Num rows: 78 Data size: 19830 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 78 Data size: 19860 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (ROW__ID.writeid = 3) (type: boolean)
- Statistics: Num rows: 39 Data size: 9915 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 39 Data size: 9930 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ROW__ID.writeid (type: bigint)
outputColumnNames: _col0
- Statistics: Num rows: 39 Data size: 9915 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 39 Data size: 9930 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 39 Data size: 9915 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 39 Data size: 9930 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/aea59083/ql/src/test/results/clientpositive/tez/acid_vectorization_original_tez.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/acid_vectorization_original_tez.q.out b/ql/src/test/results/clientpositive/tez/acid_vectorization_original_tez.q.out
index d10faeb..88499fd 100644
--- a/ql/src/test/results/clientpositive/tez/acid_vectorization_original_tez.q.out
+++ b/ql/src/test/results/clientpositive/tez/acid_vectorization_original_tez.q.out
@@ -680,11 +680,11 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: over10k_orc_bucketed_n0
- Statistics: Num rows: 1237 Data size: 707880 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1237 Data size: 707670 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: ROW__ID (type: struct<writeid:bigint,bucketid:int,rowid:bigint>)
outputColumnNames: ROW__ID
- Statistics: Num rows: 1237 Data size: 707880 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1237 Data size: 707670 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: count()
keys: ROW__ID (type: struct<writeid:bigint,bucketid:int,rowid:bigint>)
http://git-wip-us.apache.org/repos/asf/hive/blob/aea59083/ql/src/test/results/clientpositive/tez/explainanalyze_5.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/explainanalyze_5.q.out b/ql/src/test/results/clientpositive/tez/explainanalyze_5.q.out
index 7df06ba..9369df8 100644
--- a/ql/src/test/results/clientpositive/tez/explainanalyze_5.q.out
+++ b/ql/src/test/results/clientpositive/tez/explainanalyze_5.q.out
@@ -304,7 +304,7 @@ Stage-3
Output:["_col0","_col1","_col3"]
Filter Operator [FIL_9] (rows=10/2 width=316)
predicate:((de = 109.23) or (de = 119.23))
- TableScan [TS_0] (rows=86/4 width=316)
+ TableScan [TS_0] (rows=85/4 width=316)
default@acid_uami_n2,acid_uami_n2, ACID table,Tbl:COMPLETE,Col:NONE,Output:["i","de","vc"]
PREHOOK: query: select * from acid_uami_n2 order by de
[11/12] hive git commit: HIVE-19931 : stats_nonpart.q test run shows
possibly wrong results and other bugfixes (Steve Yeom,
reviewed by Sergey Shelukhin)
Posted by se...@apache.org.
HIVE-19931 : stats_nonpart.q test run shows possibly wrong results and other bugfixes (Steve Yeom, reviewed by Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/0f094530
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/0f094530
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/0f094530
Branch: refs/heads/master-txnstats
Commit: 0f094530abbca36be5fc80aaf40d7c890d909633
Parents: 1d46608
Author: sergey <se...@apache.org>
Authored: Thu Jun 21 10:47:43 2018 -0700
Committer: sergey <se...@apache.org>
Committed: Thu Jun 21 10:47:43 2018 -0700
----------------------------------------------------------------------
.../results/clientpositive/stats_nonpart.q.out | 189 +------------------
standalone-metastore/pom.xml | 2 +-
.../hadoop/hive/metastore/ObjectStore.java | 7 +-
.../model/MPartitionColumnStatistics.java | 9 -
.../metastore/model/MTableColumnStatistics.java | 9 -
.../metastore/txn/CompactionTxnHandler.java | 87 ++++-----
.../src/main/resources/package.jdo | 6 -
.../main/sql/derby/hive-schema-3.0.0.derby.sql | 11 +-
.../main/sql/derby/hive-schema-4.0.0.derby.sql | 6 +-
.../sql/derby/upgrade-3.1.0-to-4.0.0.derby.sql | 2 -
.../main/sql/mssql/hive-schema-3.0.0.mssql.sql | 14 +-
.../main/sql/mssql/hive-schema-4.0.0.mssql.sql | 6 +-
.../sql/mssql/upgrade-3.1.0-to-4.0.0.mssql.sql | 2 -
.../main/sql/mysql/hive-schema-3.0.0.mysql.sql | 6 -
.../main/sql/mysql/hive-schema-4.0.0.mysql.sql | 2 -
.../sql/mysql/upgrade-3.1.0-to-4.0.0.mysql.sql | 2 -
.../sql/oracle/hive-schema-3.0.0.oracle.sql | 15 +-
.../sql/oracle/hive-schema-4.0.0.oracle.sql | 6 +-
.../oracle/upgrade-3.1.0-to-4.0.0.oracle.sql | 2 -
.../sql/postgres/hive-schema-3.0.0.postgres.sql | 19 +-
.../sql/postgres/hive-schema-4.0.0.postgres.sql | 6 +-
.../upgrade-3.1.0-to-4.0.0.postgres.sql | 2 -
.../DummyRawStoreControlledCommit.java | 58 +++++-
.../DummyRawStoreForJdoConnection.java | 58 +++++-
24 files changed, 193 insertions(+), 333 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/0f094530/ql/src/test/results/clientpositive/stats_nonpart.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/stats_nonpart.q.out b/ql/src/test/results/clientpositive/stats_nonpart.q.out
index cded846..7bf9943 100644
--- a/ql/src/test/results/clientpositive/stats_nonpart.q.out
+++ b/ql/src/test/results/clientpositive/stats_nonpart.q.out
@@ -47,45 +47,12 @@ PREHOOK: type: QUERY
POSTHOOK: query: explain select count(*) from stats_nonpartitioned
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
- Stage-1 is a root stage
- Stage-0 depends on stages: Stage-1
+ Stage-0 is a root stage
STAGE PLANS:
- Stage: Stage-1
- Map Reduce
- Map Operator Tree:
- TableScan
- alias: stats_nonpartitioned
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
- Select Operator
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
- Group By Operator
- aggregations: count()
- mode: hash
- outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: COMPLETE
- Reduce Output Operator
- sort order:
- Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: COMPLETE
- value expressions: _col0 (type: bigint)
- Execution mode: vectorized
- Reduce Operator Tree:
- Group By Operator
- aggregations: count(VALUE._col0)
- mode: mergepartial
- outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: COMPLETE
- File Output Operator
- compressed: false
- Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: COMPLETE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
Stage: Stage-0
Fetch Operator
- limit: -1
+ limit: 1
Processor Tree:
ListSink
@@ -265,45 +232,12 @@ PREHOOK: type: QUERY
POSTHOOK: query: explain select count(*) from stats_nonpartitioned
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
- Stage-1 is a root stage
- Stage-0 depends on stages: Stage-1
+ Stage-0 is a root stage
STAGE PLANS:
- Stage: Stage-1
- Map Reduce
- Map Operator Tree:
- TableScan
- alias: stats_nonpartitioned
- Statistics: Num rows: 2 Data size: 7200 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- Statistics: Num rows: 2 Data size: 7200 Basic stats: COMPLETE Column stats: COMPLETE
- Group By Operator
- aggregations: count()
- mode: hash
- outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- sort order:
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: _col0 (type: bigint)
- Execution mode: vectorized
- Reduce Operator Tree:
- Group By Operator
- aggregations: count(VALUE._col0)
- mode: mergepartial
- outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
- File Output Operator
- compressed: false
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
Stage: Stage-0
Fetch Operator
- limit: -1
+ limit: 1
Processor Tree:
ListSink
@@ -321,47 +255,12 @@ PREHOOK: type: QUERY
POSTHOOK: query: explain select count(key) from stats_nonpartitioned
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
- Stage-1 is a root stage
- Stage-0 depends on stages: Stage-1
+ Stage-0 is a root stage
STAGE PLANS:
- Stage: Stage-1
- Map Reduce
- Map Operator Tree:
- TableScan
- alias: stats_nonpartitioned
- Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: key (type: int)
- outputColumnNames: key
- Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
- Group By Operator
- aggregations: count(key)
- mode: hash
- outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- sort order:
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: _col0 (type: bigint)
- Execution mode: vectorized
- Reduce Operator Tree:
- Group By Operator
- aggregations: count(VALUE._col0)
- mode: mergepartial
- outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
- File Output Operator
- compressed: false
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
Stage: Stage-0
Fetch Operator
- limit: -1
+ limit: 1
Processor Tree:
ListSink
@@ -389,45 +288,12 @@ PREHOOK: type: QUERY
POSTHOOK: query: explain select count(*) from stats_nonpartitioned
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
- Stage-1 is a root stage
- Stage-0 depends on stages: Stage-1
+ Stage-0 is a root stage
STAGE PLANS:
- Stage: Stage-1
- Map Reduce
- Map Operator Tree:
- TableScan
- alias: stats_nonpartitioned
- Statistics: Num rows: 2 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- Statistics: Num rows: 2 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE
- Group By Operator
- aggregations: count()
- mode: hash
- outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- sort order:
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: _col0 (type: bigint)
- Execution mode: vectorized
- Reduce Operator Tree:
- Group By Operator
- aggregations: count(VALUE._col0)
- mode: mergepartial
- outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
- File Output Operator
- compressed: false
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
Stage: Stage-0
Fetch Operator
- limit: -1
+ limit: 1
Processor Tree:
ListSink
@@ -445,47 +311,12 @@ PREHOOK: type: QUERY
POSTHOOK: query: explain select count(key) from stats_nonpartitioned
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
- Stage-1 is a root stage
- Stage-0 depends on stages: Stage-1
+ Stage-0 is a root stage
STAGE PLANS:
- Stage: Stage-1
- Map Reduce
- Map Operator Tree:
- TableScan
- alias: stats_nonpartitioned
- Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: key (type: int)
- outputColumnNames: key
- Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
- Group By Operator
- aggregations: count(key)
- mode: hash
- outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- sort order:
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: _col0 (type: bigint)
- Execution mode: vectorized
- Reduce Operator Tree:
- Group By Operator
- aggregations: count(VALUE._col0)
- mode: mergepartial
- outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
- File Output Operator
- compressed: false
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
Stage: Stage-0
Fetch Operator
- limit: -1
+ limit: 1
Processor Tree:
ListSink
http://git-wip-us.apache.org/repos/asf/hive/blob/0f094530/standalone-metastore/pom.xml
----------------------------------------------------------------------
diff --git a/standalone-metastore/pom.xml b/standalone-metastore/pom.xml
index 1e77285..9bdece7 100644
--- a/standalone-metastore/pom.xml
+++ b/standalone-metastore/pom.xml
@@ -30,7 +30,7 @@
<name>Hive Standalone Metastore</name>
<properties>
- <hive.version.shortname>3.1.0</hive.version.shortname>
+ <hive.version.shortname>4.0.0</hive.version.shortname>
<!-- Build properties -->
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
http://git-wip-us.apache.org/repos/asf/hive/blob/0f094530/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
index 28426b2..85d9b1d 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
@@ -4094,7 +4094,7 @@ public class ObjectStore implements RawStore, Configurable {
if (newTable.getValidWriteIdList() != null &&
TxnUtils.isTransactionalTable(newTable)) {
// Check concurrent INSERT case and set false to the flag.
- if (isCurrentStatsValidForTheQuery(oldt, newt.getTxnId(), newt.getWriteIdList(),
+ if (!isCurrentStatsValidForTheQuery(oldt, newt.getTxnId(), newt.getWriteIdList(),
-1, true)) {
StatsSetupConst.setBasicStatsState(oldt.getParameters(), StatsSetupConst.FALSE);
LOG.info("Removed COLUMN_STATS_ACCURATE from the parameters of the table " +
@@ -12235,6 +12235,11 @@ public class ObjectStore implements RawStore, Configurable {
long queryTxnId, String queryValidWriteIdList,
long statsWriteId, boolean checkConcurrentWrites)
throws MetaException {
+ // if statsWriteIdList is null,
+ // return true since the stats does not seem to be transactional.
+ if (statsWriteIdList == null) {
+ return true;
+ }
// If the current query is a stats updater, then we can return true
// to avoid implementing a logic inside TxnIdUtils.checkEquivalentWriteIds().
if (statsTxnId == queryTxnId) {
http://git-wip-us.apache.org/repos/asf/hive/blob/0f094530/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/model/MPartitionColumnStatistics.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/model/MPartitionColumnStatistics.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/model/MPartitionColumnStatistics.java
index ff68eba..50d9c5b 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/model/MPartitionColumnStatistics.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/model/MPartitionColumnStatistics.java
@@ -55,7 +55,6 @@ public class MPartitionColumnStatistics {
private Long numTrues;
private Long numFalses;
private long lastAnalyzed;
- private long txnId;
public MPartitionColumnStatistics() {}
@@ -279,12 +278,4 @@ public class MPartitionColumnStatistics {
public void setBitVector(byte[] bitVector) {
this.bitVector = bitVector;
}
-
- public long getTxnId() {
- return txnId;
- }
-
- public void setTxnId(long txnId) {
- this.txnId = txnId;
- }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/0f094530/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/model/MTableColumnStatistics.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/model/MTableColumnStatistics.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/model/MTableColumnStatistics.java
index 9d687e4..731cd6f 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/model/MTableColumnStatistics.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/model/MTableColumnStatistics.java
@@ -53,7 +53,6 @@ public class MTableColumnStatistics {
private Long numTrues;
private Long numFalses;
private long lastAnalyzed;
- private long txnId;
public MTableColumnStatistics() {}
@@ -270,12 +269,4 @@ public class MTableColumnStatistics {
public void setBitVector(byte[] bitVector) {
this.bitVector = bitVector;
}
-
- public long getTxnId() {
- return txnId;
- }
-
- public void setTxnId(long txnId) {
- this.txnId = txnId;
- }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/0f094530/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java
index f2642cf..7d8f164 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java
@@ -594,64 +594,47 @@ class CompactionTxnHandler extends TxnHandler {
StringBuilder suffix = new StringBuilder();
// Turn off COLUMN_STATS_ACCURATE for txnids' components in TBLS and PARTITIONS
- for (Long txnId : txnids) {
- // Get table ids for the current txnId.
- s = "select tbl_id from TBLS where txn_id = " + txnId;
- LOG.debug("Going to execute query <" + s + ">");
- rs = stmt.executeQuery(s);
- List<Long> tblIds = new ArrayList<>();
- while (rs.next()) {
- tblIds.add(rs.getLong(1));
- }
- close(rs);
- if(tblIds.size() <= 0) {
- continue;
- }
-
- // Update COLUMN_STATS_AcCURATE.BASIC_STATS to false for each tableId.
- prefix.append("delete from TABLE_PARAMS " +
- " where param_key = '" + "COLUMN_STATS_ACCURATE" + "' and ");
- suffix.append("");
- TxnUtils.buildQueryWithINClause(conf, queries, prefix, suffix, tblIds, "tbl_id", true, false);
-
- for (String query : queries) {
- LOG.debug("Going to execute update <" + query + ">");
- int rc = stmt.executeUpdate(query);
- LOG.info("Turned off " + rc + " COLUMN_STATE_ACCURATE.BASIC_STATS states from TBLS");
- }
-
- queries.clear();
- prefix.setLength(0);
- suffix.setLength(0);
+ prefix.append("select tbl_id from TBLS where ");
+ suffix.append("");
+ TxnUtils.buildQueryWithINClause(conf, queries, prefix, suffix, txnids, "txn_id", true, false);
- // Get partition ids for the current txnId.
- s = "select part_id from PARTITIONS where txn_id = " + txnId;
- LOG.debug("Going to execute query <" + s + ">");
- rs = stmt.executeQuery(s);
- List<Long> ptnIds = new ArrayList<>();
- while (rs.next()) ptnIds.add(rs.getLong(1));
- close(rs);
- if(ptnIds.size() <= 0) {
- continue;
- }
+ // Delete COLUMN_STATS_ACCURATE.BASIC_STATS rows from TABLE_PARAMS for the txnids.
+ List<StringBuilder> finalCommands = new ArrayList<>(queries.size());
+ for (int i = 0; i < queries.size(); i++) {
+ String query = queries.get(i);
+ finalCommands.add(i, new StringBuilder("delete from TABLE_PARAMS " +
+ " where param_key = '" + "COLUMN_STATS_ACCURATE" + "' and tbl_id in ("));
+ finalCommands.get(i).append(query + ")");
+ LOG.debug("Going to execute update <" + finalCommands.get(i) + ">");
+ int rc = stmt.executeUpdate(finalCommands.get(i).toString());
+ LOG.info("Turned off " + rc + " COLUMN_STATE_ACCURATE.BASIC_STATS states from TBLS");
+ }
- // Update COLUMN_STATS_AcCURATE.BASIC_STATS to false for each ptnId.
- prefix.append("delete from PARTITION_PARAMS " +
- " where param_key = '" + "COLUMN_STATS_ACCURATE" + "' and ");
- suffix.append("");
- TxnUtils.buildQueryWithINClause(conf, queries, prefix, suffix, ptnIds, "part_id", true, false);
+ queries.clear();
+ prefix.setLength(0);
+ suffix.setLength(0);
+ finalCommands.clear();
- for (String query : queries) {
- LOG.debug("Going to execute update <" + query + ">");
- int rc = stmt.executeUpdate(query);
- LOG.info("Turned off " + rc + " COLUMN_STATE_ACCURATE.BASIC_STATS states from PARTITIONS");
- }
+ // Delete COLUMN_STATS_ACCURATE.BASIC_STATS rows from PARTITIONS_PARAMS for the txnids.
+ prefix.append("select part_id from PARTITIONS where ");
+ suffix.append("");
+ TxnUtils.buildQueryWithINClause(conf, queries, prefix, suffix, txnids, "txn_id", true, false);
- queries.clear();
- prefix.setLength(0);
- suffix.setLength(0);
+ for (int i = 0; i < queries.size(); i++) {
+ String query = queries.get(i);
+ finalCommands.add(i, new StringBuilder("delete from PARTITION_PARAMS " +
+ " where param_key = '" + "COLUMN_STATS_ACCURATE" + "' and part_id in ("));
+ finalCommands.get(i).append(query + ")");
+ LOG.debug("Going to execute update <" + finalCommands.get(i) + ">");
+ int rc = stmt.executeUpdate(finalCommands.get(i).toString());
+ LOG.info("Turned off " + rc + " COLUMN_STATE_ACCURATE.BASIC_STATS states from PARTITIONS");
}
+ queries.clear();
+ prefix.setLength(0);
+ suffix.setLength(0);
+ finalCommands.clear();
+
// Delete from TXNS.
prefix.append("delete from TXNS where ");
suffix.append("");
http://git-wip-us.apache.org/repos/asf/hive/blob/0f094530/standalone-metastore/src/main/resources/package.jdo
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/resources/package.jdo b/standalone-metastore/src/main/resources/package.jdo
index 3997f53..92b803f 100644
--- a/standalone-metastore/src/main/resources/package.jdo
+++ b/standalone-metastore/src/main/resources/package.jdo
@@ -1001,9 +1001,6 @@
<field name="lastAnalyzed">
<column name="LAST_ANALYZED" jdbc-type="BIGINT" allows-null="false"/>
</field>
- <field name="txnId">
- <column name="TXN_ID"/>
- </field>
</class>
<class name="MPartitionColumnStatistics" table="PART_COL_STATS" identity-type="datastore" detachable="true">
@@ -1074,9 +1071,6 @@
<field name="lastAnalyzed">
<column name="LAST_ANALYZED" jdbc-type="BIGINT" allows-null="false"/>
</field>
- <field name="txnId">
- <column name="TXN_ID"/>
- </field>
</class>
<class name="MVersionTable" table="VERSION" identity-type="datastore" detachable="true">
<datastore-identity>
http://git-wip-us.apache.org/repos/asf/hive/blob/0f094530/standalone-metastore/src/main/sql/derby/hive-schema-3.0.0.derby.sql
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/sql/derby/hive-schema-3.0.0.derby.sql b/standalone-metastore/src/main/sql/derby/hive-schema-3.0.0.derby.sql
index 280fd4a..e818e1b 100644
--- a/standalone-metastore/src/main/sql/derby/hive-schema-3.0.0.derby.sql
+++ b/standalone-metastore/src/main/sql/derby/hive-schema-3.0.0.derby.sql
@@ -47,7 +47,7 @@ CREATE TABLE "APP"."IDXS" ("INDEX_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT
CREATE TABLE "APP"."INDEX_PARAMS" ("INDEX_ID" BIGINT NOT NULL, "PARAM_KEY" VARCHAR(256) NOT NULL, "PARAM_VALUE" VARCHAR(4000));
-CREATE TABLE "APP"."PARTITIONS" ("PART_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT NULL, "LAST_ACCESS_TIME" INTEGER NOT NULL, "PART_NAME" VARCHAR(767), "SD_ID" BIGINT, "TBL_ID" BIGINT, "TXN_ID" BIGINT DEFAULT 0, "WRITEID_LIST" CLOB);
+CREATE TABLE "APP"."PARTITIONS" ("PART_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT NULL, "LAST_ACCESS_TIME" INTEGER NOT NULL, "PART_NAME" VARCHAR(767), "SD_ID" BIGINT, "TBL_ID" BIGINT);
CREATE TABLE "APP"."SERDES" ("SERDE_ID" BIGINT NOT NULL, "NAME" VARCHAR(128), "SLIB" VARCHAR(4000), "DESCRIPTION" VARCHAR(4000), "SERIALIZER_CLASS" VARCHAR(4000), "DESERIALIZER_CLASS" VARCHAR(4000), SERDE_TYPE INTEGER);
@@ -75,7 +75,7 @@ CREATE TABLE "APP"."COLUMNS" ("SD_ID" BIGINT NOT NULL, "COMMENT" VARCHAR(256), "
CREATE TABLE "APP"."ROLES" ("ROLE_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT NULL, "OWNER_NAME" VARCHAR(128), "ROLE_NAME" VARCHAR(128));
-CREATE TABLE "APP"."TBLS" ("TBL_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT NULL, "DB_ID" BIGINT, "LAST_ACCESS_TIME" INTEGER NOT NULL, "OWNER" VARCHAR(767), "OWNER_TYPE" VARCHAR(10), "RETENTION" INTEGER NOT NULL, "SD_ID" BIGINT, "TBL_NAME" VARCHAR(256), "TBL_TYPE" VARCHAR(128), "VIEW_EXPANDED_TEXT" LONG VARCHAR, "VIEW_ORIGINAL_TEXT" LONG VARCHAR, "IS_REWRITE_ENABLED" CHAR(1) NOT NULL DEFAULT 'N', "TXN_ID" BIGINT DEFAULT 0, "WRITEID_LIST" CLOB);
+CREATE TABLE "APP"."TBLS" ("TBL_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT NULL, "DB_ID" BIGINT, "LAST_ACCESS_TIME" INTEGER NOT NULL, "OWNER" VARCHAR(767), "OWNER_TYPE" VARCHAR(10), "RETENTION" INTEGER NOT NULL, "SD_ID" BIGINT, "TBL_NAME" VARCHAR(256), "TBL_TYPE" VARCHAR(128), "VIEW_EXPANDED_TEXT" LONG VARCHAR, "VIEW_ORIGINAL_TEXT" LONG VARCHAR, "IS_REWRITE_ENABLED" CHAR(1) NOT NULL DEFAULT 'N');
CREATE TABLE "APP"."PARTITION_KEYS" ("TBL_ID" BIGINT NOT NULL, "PKEY_COMMENT" VARCHAR(4000), "PKEY_NAME" VARCHAR(128) NOT NULL, "PKEY_TYPE" VARCHAR(767) NOT NULL, "INTEGER_IDX" INTEGER NOT NULL);
@@ -106,8 +106,7 @@ CREATE TABLE "APP"."TAB_COL_STATS"(
"LAST_ANALYZED" BIGINT,
"CS_ID" BIGINT NOT NULL,
"TBL_ID" BIGINT NOT NULL,
- "BIT_VECTOR" BLOB,
- "TXN_ID" BIGINT DEFAULT 0
+ "BIT_VECTOR" BLOB
);
CREATE TABLE "APP"."TABLE_PARAMS" ("TBL_ID" BIGINT NOT NULL, "PARAM_KEY" VARCHAR(256) NOT NULL, "PARAM_VALUE" CLOB);
@@ -156,8 +155,7 @@ CREATE TABLE "APP"."PART_COL_STATS"(
"NUM_FALSES" BIGINT,
"LAST_ANALYZED" BIGINT,
"CS_ID" BIGINT NOT NULL,
- "PART_ID" BIGINT NOT NULL,
- "TXN_ID" BIGINT DEFAULT 0
+ "PART_ID" BIGINT NOT NULL
);
CREATE TABLE "APP"."VERSION" ("VER_ID" BIGINT NOT NULL, "SCHEMA_VERSION" VARCHAR(127) NOT NULL, "VERSION_COMMENT" VARCHAR(255));
@@ -375,6 +373,7 @@ ALTER TABLE "APP"."MV_CREATION_METADATA" ADD CONSTRAINT "MV_CREATION_METADATA_PK
ALTER TABLE "APP"."CTLGS" ADD CONSTRAINT "CTLG_PK" PRIMARY KEY ("CTLG_ID");
+
-- foreign
ALTER TABLE "APP"."IDXS" ADD CONSTRAINT "IDXS_FK1" FOREIGN KEY ("ORIG_TBL_ID") REFERENCES "APP"."TBLS" ("TBL_ID") ON DELETE NO ACTION ON UPDATE NO ACTION;
http://git-wip-us.apache.org/repos/asf/hive/blob/0f094530/standalone-metastore/src/main/sql/derby/hive-schema-4.0.0.derby.sql
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/sql/derby/hive-schema-4.0.0.derby.sql b/standalone-metastore/src/main/sql/derby/hive-schema-4.0.0.derby.sql
index f92f13c..062e374 100644
--- a/standalone-metastore/src/main/sql/derby/hive-schema-4.0.0.derby.sql
+++ b/standalone-metastore/src/main/sql/derby/hive-schema-4.0.0.derby.sql
@@ -106,8 +106,7 @@ CREATE TABLE "APP"."TAB_COL_STATS"(
"LAST_ANALYZED" BIGINT,
"CS_ID" BIGINT NOT NULL,
"TBL_ID" BIGINT NOT NULL,
- "BIT_VECTOR" BLOB,
- "TXN_ID" BIGINT DEFAULT 0
+ "BIT_VECTOR" BLOB
);
CREATE TABLE "APP"."TABLE_PARAMS" ("TBL_ID" BIGINT NOT NULL, "PARAM_KEY" VARCHAR(256) NOT NULL, "PARAM_VALUE" CLOB);
@@ -156,8 +155,7 @@ CREATE TABLE "APP"."PART_COL_STATS"(
"NUM_FALSES" BIGINT,
"LAST_ANALYZED" BIGINT,
"CS_ID" BIGINT NOT NULL,
- "PART_ID" BIGINT NOT NULL,
- "TXN_ID" BIGINT DEFAULT 0
+ "PART_ID" BIGINT NOT NULL
);
CREATE TABLE "APP"."VERSION" ("VER_ID" BIGINT NOT NULL, "SCHEMA_VERSION" VARCHAR(127) NOT NULL, "VERSION_COMMENT" VARCHAR(255));
http://git-wip-us.apache.org/repos/asf/hive/blob/0f094530/standalone-metastore/src/main/sql/derby/upgrade-3.1.0-to-4.0.0.derby.sql
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/sql/derby/upgrade-3.1.0-to-4.0.0.derby.sql b/standalone-metastore/src/main/sql/derby/upgrade-3.1.0-to-4.0.0.derby.sql
index 94f8192..38eecd9 100644
--- a/standalone-metastore/src/main/sql/derby/upgrade-3.1.0-to-4.0.0.derby.sql
+++ b/standalone-metastore/src/main/sql/derby/upgrade-3.1.0-to-4.0.0.derby.sql
@@ -4,8 +4,6 @@ ALTER TABLE "APP"."TBLS" ADD WRITEID_LIST CLOB;
ALTER TABLE "APP"."TBLS" ADD TXN_ID bigint DEFAULT 0;
ALTER TABLE "APP"."PARTITIONS" ADD WRITEID_LIST CLOB;
ALTER TABLE "APP"."PARTITIONS" ADD TXN_ID bigint DEFAULT 0;
-ALTER TABLE "APP"."TAB_COL_STATS" ADD TXN_ID bigint DEFAULT 0;
-ALTER TABLE "APP"."PART_COL_STATS" ADD TXN_ID bigint DEFAULT 0;
-- This needs to be the last thing done. Insert any changes above this line.
UPDATE "APP".VERSION SET SCHEMA_VERSION='4.0.0', VERSION_COMMENT='Hive release version 4.0.0' where VER_ID=1;
http://git-wip-us.apache.org/repos/asf/hive/blob/0f094530/standalone-metastore/src/main/sql/mssql/hive-schema-3.0.0.mssql.sql
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/sql/mssql/hive-schema-3.0.0.mssql.sql b/standalone-metastore/src/main/sql/mssql/hive-schema-3.0.0.mssql.sql
index f20f910..c88fb18 100644
--- a/standalone-metastore/src/main/sql/mssql/hive-schema-3.0.0.mssql.sql
+++ b/standalone-metastore/src/main/sql/mssql/hive-schema-3.0.0.mssql.sql
@@ -94,8 +94,7 @@ CREATE TABLE PART_COL_STATS
PART_ID bigint NULL,
PARTITION_NAME nvarchar(767) NOT NULL,
"TABLE_NAME" nvarchar(256) NOT NULL,
- "CAT_NAME" nvarchar(256) NOT NULL,
- TXN_ID bigint NULL
+ "CAT_NAME" nvarchar(256) NOT NULL
);
ALTER TABLE PART_COL_STATS ADD CONSTRAINT PART_COL_STATS_PK PRIMARY KEY (CS_ID);
@@ -145,9 +144,7 @@ CREATE TABLE PARTITIONS
LAST_ACCESS_TIME int NOT NULL,
PART_NAME nvarchar(767) NULL,
SD_ID bigint NULL,
- TBL_ID bigint NULL,
- TXN_ID bigint NULL,
- WRITEID_LIST text NULL
+ TBL_ID bigint NULL
);
ALTER TABLE PARTITIONS ADD CONSTRAINT PARTITIONS_PK PRIMARY KEY (PART_ID);
@@ -241,8 +238,7 @@ CREATE TABLE TAB_COL_STATS
NUM_TRUES bigint NULL,
TBL_ID bigint NULL,
"TABLE_NAME" nvarchar(256) NOT NULL,
- "CAT_NAME" nvarchar(256) NOT NULL,
- TXN_ID bigint NULL
+ "CAT_NAME" nvarchar(256) NOT NULL
);
ALTER TABLE TAB_COL_STATS ADD CONSTRAINT TAB_COL_STATS_PK PRIMARY KEY (CS_ID);
@@ -373,9 +369,7 @@ CREATE TABLE TBLS
TBL_TYPE nvarchar(128) NULL,
VIEW_EXPANDED_TEXT text NULL,
VIEW_ORIGINAL_TEXT text NULL,
- IS_REWRITE_ENABLED bit NOT NULL DEFAULT 0,
- TXN_ID bigint NULL,
- WRITEID_LIST text NULL
+ IS_REWRITE_ENABLED bit NOT NULL DEFAULT 0
);
ALTER TABLE TBLS ADD CONSTRAINT TBLS_PK PRIMARY KEY (TBL_ID);
http://git-wip-us.apache.org/repos/asf/hive/blob/0f094530/standalone-metastore/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql b/standalone-metastore/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql
index 22637c5..1f31341 100644
--- a/standalone-metastore/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql
+++ b/standalone-metastore/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql
@@ -94,8 +94,7 @@ CREATE TABLE PART_COL_STATS
PART_ID bigint NULL,
PARTITION_NAME nvarchar(767) NOT NULL,
"TABLE_NAME" nvarchar(256) NOT NULL,
- "CAT_NAME" nvarchar(256) NOT NULL,
- TXN_ID bigint NULL
+ "CAT_NAME" nvarchar(256) NOT NULL
);
ALTER TABLE PART_COL_STATS ADD CONSTRAINT PART_COL_STATS_PK PRIMARY KEY (CS_ID);
@@ -245,8 +244,7 @@ CREATE TABLE TAB_COL_STATS
NUM_TRUES bigint NULL,
TBL_ID bigint NULL,
"TABLE_NAME" nvarchar(256) NOT NULL,
- "CAT_NAME" nvarchar(256) NOT NULL,
- TXN_ID bigint NULL
+ "CAT_NAME" nvarchar(256) NOT NULL
);
ALTER TABLE TAB_COL_STATS ADD CONSTRAINT TAB_COL_STATS_PK PRIMARY KEY (CS_ID);
http://git-wip-us.apache.org/repos/asf/hive/blob/0f094530/standalone-metastore/src/main/sql/mssql/upgrade-3.1.0-to-4.0.0.mssql.sql
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/sql/mssql/upgrade-3.1.0-to-4.0.0.mssql.sql b/standalone-metastore/src/main/sql/mssql/upgrade-3.1.0-to-4.0.0.mssql.sql
index f0cbf6c..594d165 100644
--- a/standalone-metastore/src/main/sql/mssql/upgrade-3.1.0-to-4.0.0.mssql.sql
+++ b/standalone-metastore/src/main/sql/mssql/upgrade-3.1.0-to-4.0.0.mssql.sql
@@ -5,8 +5,6 @@ ALTER TABLE TBLS ADD WRITEID_LIST text NULL;
ALTER TABLE TBLS ADD TXN_ID bigint NULL;
ALTER TABLE PARTITIONS ADD WRITEID_LIST text NULL;
ALTER TABLE PARTITIONS ADD TXN_ID bigint NULL;
-ALTER TABLE TAB_COL_STATS ADD TXN_ID bigint NULL;
-ALTER TABLE PART_COL_STATS ADD TXN_ID bigint NULL;
-- These lines need to be last. Insert any changes above.
UPDATE VERSION SET SCHEMA_VERSION='4.0.0', VERSION_COMMENT='Hive release version 4.0.0' where VER_ID=1;
http://git-wip-us.apache.org/repos/asf/hive/blob/0f094530/standalone-metastore/src/main/sql/mysql/hive-schema-3.0.0.mysql.sql
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/sql/mysql/hive-schema-3.0.0.mysql.sql b/standalone-metastore/src/main/sql/mysql/hive-schema-3.0.0.mysql.sql
index 6e34ab5..c54df55 100644
--- a/standalone-metastore/src/main/sql/mysql/hive-schema-3.0.0.mysql.sql
+++ b/standalone-metastore/src/main/sql/mysql/hive-schema-3.0.0.mysql.sql
@@ -222,8 +222,6 @@ CREATE TABLE IF NOT EXISTS `PARTITIONS` (
`PART_NAME` varchar(767) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL,
`SD_ID` bigint(20) DEFAULT NULL,
`TBL_ID` bigint(20) DEFAULT NULL,
- `TXN_ID` bigint(20) DEFAULT 0,
- `WRITEID_LIST` text DEFAULT NULL,
PRIMARY KEY (`PART_ID`),
UNIQUE KEY `UNIQUEPARTITION` (`PART_NAME`,`TBL_ID`),
KEY `PARTITIONS_N49` (`TBL_ID`),
@@ -627,8 +625,6 @@ CREATE TABLE IF NOT EXISTS `TBLS` (
`VIEW_EXPANDED_TEXT` mediumtext,
`VIEW_ORIGINAL_TEXT` mediumtext,
`IS_REWRITE_ENABLED` bit(1) NOT NULL DEFAULT 0,
- `TXN_ID` bigint(20) DEFAULT 0,
- `WRITEID_LIST` text DEFAULT NULL,
PRIMARY KEY (`TBL_ID`),
UNIQUE KEY `UNIQUETABLE` (`TBL_NAME`,`DB_ID`),
KEY `TBLS_N50` (`SD_ID`),
@@ -724,7 +720,6 @@ CREATE TABLE IF NOT EXISTS `TAB_COL_STATS` (
`NUM_TRUES` bigint(20),
`NUM_FALSES` bigint(20),
`LAST_ANALYZED` bigint(20) NOT NULL,
- `TXN_ID` bigint(20) DEFAULT 0,
PRIMARY KEY (`CS_ID`),
CONSTRAINT `TAB_COL_STATS_FK` FOREIGN KEY (`TBL_ID`) REFERENCES `TBLS` (`TBL_ID`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
@@ -755,7 +750,6 @@ CREATE TABLE IF NOT EXISTS `PART_COL_STATS` (
`NUM_TRUES` bigint(20),
`NUM_FALSES` bigint(20),
`LAST_ANALYZED` bigint(20) NOT NULL,
- `TXN_ID` bigint(20) DEFAULT 0,
PRIMARY KEY (`CS_ID`),
CONSTRAINT `PART_COL_STATS_FK` FOREIGN KEY (`PART_ID`) REFERENCES `PARTITIONS` (`PART_ID`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
http://git-wip-us.apache.org/repos/asf/hive/blob/0f094530/standalone-metastore/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql b/standalone-metastore/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql
index f8f229d..90f45ac 100644
--- a/standalone-metastore/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql
+++ b/standalone-metastore/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql
@@ -730,7 +730,6 @@ CREATE TABLE IF NOT EXISTS `TAB_COL_STATS` (
`NUM_TRUES` bigint(20),
`NUM_FALSES` bigint(20),
`LAST_ANALYZED` bigint(20) NOT NULL,
- `TXN_ID` bigint(20) DEFAULT 0,
PRIMARY KEY (`CS_ID`),
CONSTRAINT `TAB_COL_STATS_FK` FOREIGN KEY (`TBL_ID`) REFERENCES `TBLS` (`TBL_ID`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
@@ -762,7 +761,6 @@ CREATE TABLE IF NOT EXISTS `PART_COL_STATS` (
`NUM_TRUES` bigint(20),
`NUM_FALSES` bigint(20),
`LAST_ANALYZED` bigint(20) NOT NULL,
- `TXN_ID` bigint(20) DEFAULT 0,
PRIMARY KEY (`CS_ID`),
CONSTRAINT `PART_COL_STATS_FK` FOREIGN KEY (`PART_ID`) REFERENCES `PARTITIONS` (`PART_ID`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
http://git-wip-us.apache.org/repos/asf/hive/blob/0f094530/standalone-metastore/src/main/sql/mysql/upgrade-3.1.0-to-4.0.0.mysql.sql
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/sql/mysql/upgrade-3.1.0-to-4.0.0.mysql.sql b/standalone-metastore/src/main/sql/mysql/upgrade-3.1.0-to-4.0.0.mysql.sql
index 4ca584c..dc011c2 100644
--- a/standalone-metastore/src/main/sql/mysql/upgrade-3.1.0-to-4.0.0.mysql.sql
+++ b/standalone-metastore/src/main/sql/mysql/upgrade-3.1.0-to-4.0.0.mysql.sql
@@ -5,8 +5,6 @@ ALTER TABLE TBLS ADD TXN_ID bigint;
ALTER TABLE TBLS ADD WRITEID_LIST CLOB;
ALTER TABLE PARTITIONS ADD TXN_ID bigint;
ALTER TABLE PARTITIONS ADD WRITEID_LIST CLOB;
-ALTER TABLE TAB_COL_STATS ADD TXN_ID bigint;
-ALTER TABLE PART_COL_STATS ADD TXN_ID bigint;
-- These lines need to be last. Insert any changes above.
UPDATE VERSION SET SCHEMA_VERSION='4.0.0', VERSION_COMMENT='Hive release version 4.0.0' where VER_ID=1;
http://git-wip-us.apache.org/repos/asf/hive/blob/0f094530/standalone-metastore/src/main/sql/oracle/hive-schema-3.0.0.oracle.sql
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/sql/oracle/hive-schema-3.0.0.oracle.sql b/standalone-metastore/src/main/sql/oracle/hive-schema-3.0.0.oracle.sql
index abdb984..63cc1f7 100644
--- a/standalone-metastore/src/main/sql/oracle/hive-schema-3.0.0.oracle.sql
+++ b/standalone-metastore/src/main/sql/oracle/hive-schema-3.0.0.oracle.sql
@@ -162,9 +162,7 @@ CREATE TABLE PARTITIONS
LAST_ACCESS_TIME NUMBER (10) NOT NULL,
PART_NAME VARCHAR2(767) NULL,
SD_ID NUMBER NULL,
- TBL_ID NUMBER NULL,
- TXN_ID NUMBER NULL,
- WRITEID_LIST CLOB NULL
+ TBL_ID NUMBER NULL
);
ALTER TABLE PARTITIONS ADD CONSTRAINT PARTITIONS_PK PRIMARY KEY (PART_ID);
@@ -394,9 +392,7 @@ CREATE TABLE TBLS
TBL_TYPE VARCHAR2(128) NULL,
VIEW_EXPANDED_TEXT CLOB NULL,
VIEW_ORIGINAL_TEXT CLOB NULL,
- IS_REWRITE_ENABLED NUMBER(1) DEFAULT 0 NOT NULL CHECK (IS_REWRITE_ENABLED IN (1,0)),
- TXN_ID NUMBER NULL,
- WRITEID_LIST CLOB NULL
+ IS_REWRITE_ENABLED NUMBER(1) DEFAULT 0 NOT NULL CHECK (IS_REWRITE_ENABLED IN (1,0))
);
ALTER TABLE TBLS ADD CONSTRAINT TBLS_PK PRIMARY KEY (TBL_ID);
@@ -529,8 +525,7 @@ CREATE TABLE TAB_COL_STATS (
MAX_COL_LEN NUMBER,
NUM_TRUES NUMBER,
NUM_FALSES NUMBER,
- LAST_ANALYZED NUMBER NOT NULL,
- TXN_ID NUMBER NULL
+ LAST_ANALYZED NUMBER NOT NULL
);
CREATE TABLE VERSION (
@@ -568,8 +563,7 @@ CREATE TABLE PART_COL_STATS (
MAX_COL_LEN NUMBER,
NUM_TRUES NUMBER,
NUM_FALSES NUMBER,
- LAST_ANALYZED NUMBER NOT NULL,
- TXN_ID NUMBER NULL
+ LAST_ANALYZED NUMBER NOT NULL
);
ALTER TABLE PART_COL_STATS ADD CONSTRAINT PART_COL_STATS_PKEY PRIMARY KEY (CS_ID);
@@ -1140,6 +1134,7 @@ CREATE TABLE RUNTIME_STATS (
CREATE INDEX IDX_RUNTIME_STATS_CREATE_TIME ON RUNTIME_STATS(CREATE_TIME);
+
-- -----------------------------------------------------------------
-- Record schema version. Should be the last step in the init script
-- -----------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/0f094530/standalone-metastore/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql b/standalone-metastore/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql
index a143fd2..cc08dc1 100644
--- a/standalone-metastore/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql
+++ b/standalone-metastore/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql
@@ -535,8 +535,7 @@ CREATE TABLE TAB_COL_STATS (
MAX_COL_LEN NUMBER,
NUM_TRUES NUMBER,
NUM_FALSES NUMBER,
- LAST_ANALYZED NUMBER NOT NULL,
- TXN_ID NUMBER NULL
+ LAST_ANALYZED NUMBER NOT NULL
);
ALTER TABLE TAB_COL_STATS ADD CONSTRAINT TAB_COL_STATS_PKEY PRIMARY KEY (CS_ID);
@@ -576,8 +575,7 @@ CREATE TABLE PART_COL_STATS (
MAX_COL_LEN NUMBER,
NUM_TRUES NUMBER,
NUM_FALSES NUMBER,
- LAST_ANALYZED NUMBER NOT NULL,
- TXN_ID NUMBER NULL
+ LAST_ANALYZED NUMBER NOT NULL
);
ALTER TABLE PART_COL_STATS ADD CONSTRAINT PART_COL_STATS_PKEY PRIMARY KEY (CS_ID);
http://git-wip-us.apache.org/repos/asf/hive/blob/0f094530/standalone-metastore/src/main/sql/oracle/upgrade-3.1.0-to-4.0.0.oracle.sql
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/sql/oracle/upgrade-3.1.0-to-4.0.0.oracle.sql b/standalone-metastore/src/main/sql/oracle/upgrade-3.1.0-to-4.0.0.oracle.sql
index 7ac4d40..9e1e6cb 100644
--- a/standalone-metastore/src/main/sql/oracle/upgrade-3.1.0-to-4.0.0.oracle.sql
+++ b/standalone-metastore/src/main/sql/oracle/upgrade-3.1.0-to-4.0.0.oracle.sql
@@ -4,8 +4,6 @@ ALTER TABLE TBLS ADD TXN_ID number NULL;
ALTER TABLE TBLS ADD WRITEID_LIST CLOB NULL;
ALTER TABLE PARTITIONS ADD TXN_ID number NULL;
ALTER TABLE PARTITIONS ADD WRITEID_LIST CLOB NULL;
-ALTER TABLE TAB_COL_STATS ADD TXN_ID number NULL;
-ALTER TABLE PART_COL_STATS ADD TXN_ID number NULL;
-- These lines need to be last. Insert any changes above.
UPDATE VERSION SET SCHEMA_VERSION='4.0.0', VERSION_COMMENT='Hive release version 4.0.0' where VER_ID=1;
http://git-wip-us.apache.org/repos/asf/hive/blob/0f094530/standalone-metastore/src/main/sql/postgres/hive-schema-3.0.0.postgres.sql
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/sql/postgres/hive-schema-3.0.0.postgres.sql b/standalone-metastore/src/main/sql/postgres/hive-schema-3.0.0.postgres.sql
index 449f295..97697f8 100644
--- a/standalone-metastore/src/main/sql/postgres/hive-schema-3.0.0.postgres.sql
+++ b/standalone-metastore/src/main/sql/postgres/hive-schema-3.0.0.postgres.sql
@@ -166,9 +166,7 @@ CREATE TABLE "PARTITIONS" (
"LAST_ACCESS_TIME" bigint NOT NULL,
"PART_NAME" character varying(767) DEFAULT NULL::character varying,
"SD_ID" bigint,
- "TBL_ID" bigint,
- "TXN_ID" bigint,
- "WRITEID_LIST" text
+ "TBL_ID" bigint
);
@@ -390,9 +388,7 @@ CREATE TABLE "TBLS" (
"TBL_TYPE" character varying(128) DEFAULT NULL::character varying,
"VIEW_EXPANDED_TEXT" text,
"VIEW_ORIGINAL_TEXT" text,
- "IS_REWRITE_ENABLED" boolean NOT NULL DEFAULT false,
- "TXN_ID" bigint,
- "WRITEID_LIST" text
+ "IS_REWRITE_ENABLED" boolean NOT NULL DEFAULT false
);
--
@@ -543,8 +539,7 @@ CREATE TABLE "TAB_COL_STATS" (
"MAX_COL_LEN" bigint,
"NUM_TRUES" bigint,
"NUM_FALSES" bigint,
- "LAST_ANALYZED" bigint NOT NULL,
- "TXN_ID" bigint
+ "LAST_ANALYZED" bigint NOT NULL
);
--
@@ -582,8 +577,7 @@ CREATE TABLE "PART_COL_STATS" (
"MAX_COL_LEN" bigint,
"NUM_TRUES" bigint,
"NUM_FALSES" bigint,
- "LAST_ANALYZED" bigint NOT NULL,
- "TXN_ID" bigint
+ "LAST_ANALYZED" bigint NOT NULL
);
--
@@ -1080,8 +1074,6 @@ ALTER TABLE ONLY "WM_MAPPING"
ALTER TABLE ONLY "WM_MAPPING"
ADD CONSTRAINT "UNIQUE_WM_MAPPING" UNIQUE ("RP_ID", "ENTITY_TYPE", "ENTITY_NAME");
--- Transactional table stats PK constraints
-
--
-- Name: BUCKETING_COLS_N49; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace:
--
@@ -1626,8 +1618,6 @@ ALTER TABLE ONLY "MV_TABLES_USED"
ALTER TABLE ONLY "MV_TABLES_USED"
ADD CONSTRAINT "MV_TABLES_USED_FK2" FOREIGN KEY ("TBL_ID") REFERENCES "TBLS" ("TBL_ID") DEFERRABLE;
--- Transactional table stats FK constraints
-
--
-- Name: public; Type: ACL; Schema: -; Owner: hiveuser
--
@@ -1832,6 +1822,7 @@ CREATE TABLE RUNTIME_STATS (
CREATE INDEX IDX_RUNTIME_STATS_CREATE_TIME ON RUNTIME_STATS(CREATE_TIME);
+
-- -----------------------------------------------------------------
-- Record schema version. Should be the last step in the init script
-- -----------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/0f094530/standalone-metastore/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql b/standalone-metastore/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql
index 0ead590..c7add63 100644
--- a/standalone-metastore/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql
+++ b/standalone-metastore/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql
@@ -549,8 +549,7 @@ CREATE TABLE "TAB_COL_STATS" (
"MAX_COL_LEN" bigint,
"NUM_TRUES" bigint,
"NUM_FALSES" bigint,
- "LAST_ANALYZED" bigint NOT NULL,
- "TXN_ID" bigint
+ "LAST_ANALYZED" bigint NOT NULL
);
--
@@ -588,8 +587,7 @@ CREATE TABLE "PART_COL_STATS" (
"MAX_COL_LEN" bigint,
"NUM_TRUES" bigint,
"NUM_FALSES" bigint,
- "LAST_ANALYZED" bigint NOT NULL,
- "TXN_ID" bigint
+ "LAST_ANALYZED" bigint NOT NULL
);
--
http://git-wip-us.apache.org/repos/asf/hive/blob/0f094530/standalone-metastore/src/main/sql/postgres/upgrade-3.1.0-to-4.0.0.postgres.sql
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/sql/postgres/upgrade-3.1.0-to-4.0.0.postgres.sql b/standalone-metastore/src/main/sql/postgres/upgrade-3.1.0-to-4.0.0.postgres.sql
index f2bae02..0692db1 100644
--- a/standalone-metastore/src/main/sql/postgres/upgrade-3.1.0-to-4.0.0.postgres.sql
+++ b/standalone-metastore/src/main/sql/postgres/upgrade-3.1.0-to-4.0.0.postgres.sql
@@ -5,8 +5,6 @@ ALTER TABLE "TBLS" ADD "TXN_ID" bigint;
ALTER TABLE "TBLS" ADD "WRITEID_LIST" text;
ALTER TABLE "PARTITIONS" ADD "TXN_ID" bigint;
ALTER TABLE "PARTITIONS" ADD "WRITEID_LIST" text;
-ALTER TABLE "TAB_COL_STATS" ADD "TXN_ID" bigint;
-ALTER TABLE "PART_COL_STATS" ADD "TXN_ID" bigint;
-- These lines need to be last. Insert any changes above.
UPDATE "VERSION" SET "SCHEMA_VERSION"='4.0.0', "VERSION_COMMENT"='Hive release version 4.0.0' where "VER_ID"=1;
http://git-wip-us.apache.org/repos/asf/hive/blob/0f094530/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java
index 6985736..7683c00 100644
--- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java
+++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java
@@ -19,7 +19,63 @@
package org.apache.hadoop.hive.metastore;
import org.apache.hadoop.hive.common.TableName;
-import org.apache.hadoop.hive.metastore.api.*;
+import org.apache.hadoop.hive.metastore.api.AggrStats;
+import org.apache.hadoop.hive.metastore.api.AlreadyExistsException;
+import org.apache.hadoop.hive.metastore.api.Catalog;
+import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
+import org.apache.hadoop.hive.metastore.api.CurrentNotificationEventId;
+import org.apache.hadoop.hive.metastore.api.CreationMetadata;
+import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.FileMetadataExprType;
+import org.apache.hadoop.hive.metastore.api.Function;
+import org.apache.hadoop.hive.metastore.api.HiveObjectPrivilege;
+import org.apache.hadoop.hive.metastore.api.HiveObjectRef;
+import org.apache.hadoop.hive.metastore.api.ISchema;
+import org.apache.hadoop.hive.metastore.api.InvalidInputException;
+import org.apache.hadoop.hive.metastore.api.InvalidObjectException;
+import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
+import org.apache.hadoop.hive.metastore.api.InvalidPartitionException;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
+import org.apache.hadoop.hive.metastore.api.NotificationEvent;
+import org.apache.hadoop.hive.metastore.api.NotificationEventRequest;
+import org.apache.hadoop.hive.metastore.api.NotificationEventResponse;
+import org.apache.hadoop.hive.metastore.api.NotificationEventsCountRequest;
+import org.apache.hadoop.hive.metastore.api.NotificationEventsCountResponse;
+import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.PartitionEventType;
+import org.apache.hadoop.hive.metastore.api.PartitionValuesResponse;
+import org.apache.hadoop.hive.metastore.api.PrincipalPrivilegeSet;
+import org.apache.hadoop.hive.metastore.api.PrincipalType;
+import org.apache.hadoop.hive.metastore.api.PrivilegeBag;
+import org.apache.hadoop.hive.metastore.api.WMNullablePool;
+import org.apache.hadoop.hive.metastore.api.WMNullableResourcePlan;
+import org.apache.hadoop.hive.metastore.api.WMResourcePlan;
+import org.apache.hadoop.hive.metastore.api.WMTrigger;
+import org.apache.hadoop.hive.metastore.api.WMValidateResourcePlanResponse;
+import org.apache.hadoop.hive.metastore.api.Role;
+import org.apache.hadoop.hive.metastore.api.RolePrincipalGrant;
+import org.apache.hadoop.hive.metastore.api.RuntimeStat;
+import org.apache.hadoop.hive.metastore.api.SQLCheckConstraint;
+import org.apache.hadoop.hive.metastore.api.SQLDefaultConstraint;
+import org.apache.hadoop.hive.metastore.api.SQLForeignKey;
+import org.apache.hadoop.hive.metastore.api.SQLNotNullConstraint;
+import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey;
+import org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint;
+import org.apache.hadoop.hive.metastore.api.SchemaVersion;
+import org.apache.hadoop.hive.metastore.api.SerDeInfo;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.metastore.api.TableMeta;
+import org.apache.hadoop.hive.metastore.api.Type;
+import org.apache.hadoop.hive.metastore.api.UnknownDBException;
+import org.apache.hadoop.hive.metastore.api.UnknownPartitionException;
+import org.apache.hadoop.hive.metastore.api.UnknownTableException;
+import org.apache.hadoop.hive.metastore.api.WMMapping;
+import org.apache.hadoop.hive.metastore.api.WMPool;
+import org.apache.hadoop.hive.metastore.api.WMFullResourcePlan;
+import org.apache.hadoop.hive.metastore.api.ISchemaName;
+import org.apache.hadoop.hive.metastore.api.SchemaVersionDescriptor;
import java.nio.ByteBuffer;
import java.util.ArrayList;
http://git-wip-us.apache.org/repos/asf/hive/blob/0f094530/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
index 37e9920..97f9464 100644
--- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
+++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
@@ -19,7 +19,11 @@
package org.apache.hadoop.hive.metastore;
import org.apache.hadoop.hive.common.TableName;
-import org.apache.hadoop.hive.metastore.api.*;
+import org.apache.hadoop.hive.metastore.api.CreationMetadata;
+import org.apache.hadoop.hive.metastore.api.ISchemaName;
+import org.apache.hadoop.hive.metastore.api.SchemaVersionDescriptor;
+import org.apache.hadoop.hive.metastore.api.Catalog;
+import org.apache.hadoop.hive.metastore.api.WMFullResourcePlan;
import java.nio.ByteBuffer;
import java.util.Collections;
@@ -27,6 +31,58 @@ import java.util.List;
import java.util.Map;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.api.AggrStats;
+import org.apache.hadoop.hive.metastore.api.AlreadyExistsException;
+import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
+import org.apache.hadoop.hive.metastore.api.CurrentNotificationEventId;
+import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.FileMetadataExprType;
+import org.apache.hadoop.hive.metastore.api.Function;
+import org.apache.hadoop.hive.metastore.api.HiveObjectPrivilege;
+import org.apache.hadoop.hive.metastore.api.HiveObjectRef;
+import org.apache.hadoop.hive.metastore.api.ISchema;
+import org.apache.hadoop.hive.metastore.api.InvalidInputException;
+import org.apache.hadoop.hive.metastore.api.InvalidObjectException;
+import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
+import org.apache.hadoop.hive.metastore.api.InvalidPartitionException;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
+import org.apache.hadoop.hive.metastore.api.NotificationEvent;
+import org.apache.hadoop.hive.metastore.api.NotificationEventRequest;
+import org.apache.hadoop.hive.metastore.api.NotificationEventResponse;
+import org.apache.hadoop.hive.metastore.api.NotificationEventsCountRequest;
+import org.apache.hadoop.hive.metastore.api.NotificationEventsCountResponse;
+import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.PartitionEventType;
+import org.apache.hadoop.hive.metastore.api.PartitionValuesResponse;
+import org.apache.hadoop.hive.metastore.api.PrincipalPrivilegeSet;
+import org.apache.hadoop.hive.metastore.api.PrincipalType;
+import org.apache.hadoop.hive.metastore.api.PrivilegeBag;
+import org.apache.hadoop.hive.metastore.api.WMNullablePool;
+import org.apache.hadoop.hive.metastore.api.WMNullableResourcePlan;
+import org.apache.hadoop.hive.metastore.api.WMResourcePlan;
+import org.apache.hadoop.hive.metastore.api.WMTrigger;
+import org.apache.hadoop.hive.metastore.api.WMValidateResourcePlanResponse;
+import org.apache.hadoop.hive.metastore.api.Role;
+import org.apache.hadoop.hive.metastore.api.RolePrincipalGrant;
+import org.apache.hadoop.hive.metastore.api.RuntimeStat;
+import org.apache.hadoop.hive.metastore.api.SQLCheckConstraint;
+import org.apache.hadoop.hive.metastore.api.SQLDefaultConstraint;
+import org.apache.hadoop.hive.metastore.api.SQLForeignKey;
+import org.apache.hadoop.hive.metastore.api.SQLNotNullConstraint;
+import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey;
+import org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint;
+import org.apache.hadoop.hive.metastore.api.SchemaVersion;
+import org.apache.hadoop.hive.metastore.api.SerDeInfo;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.metastore.api.TableMeta;
+import org.apache.hadoop.hive.metastore.api.Type;
+import org.apache.hadoop.hive.metastore.api.UnknownDBException;
+import org.apache.hadoop.hive.metastore.api.UnknownPartitionException;
+import org.apache.hadoop.hive.metastore.api.UnknownTableException;
+import org.apache.hadoop.hive.metastore.api.WMMapping;
+import org.apache.hadoop.hive.metastore.api.WMPool;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy;
import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
[03/12] hive git commit: HIVE-19870: HCatalog dynamic partition query
can fail,
if the table path is managed by Sentry (Peter Vary via Marta Kuczora)
Posted by se...@apache.org.
HIVE-19870: HCatalog dynamic partition query can fail, if the table path is managed by Sentry (Peter Vary via Marta Kuczora)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/2394e409
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/2394e409
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/2394e409
Branch: refs/heads/master-txnstats
Commit: 2394e409f87b3a857cc00d9041b354cd47c9a923
Parents: d5cb88d
Author: Peter Vary <pv...@cloudera.com>
Authored: Tue Jun 19 09:54:19 2018 +0200
Committer: Marta Kuczora <ku...@cloudera.com>
Committed: Tue Jun 19 09:54:19 2018 +0200
----------------------------------------------------------------------
.../mapreduce/FileOutputCommitterContainer.java | 60 ++++++--------------
1 file changed, 18 insertions(+), 42 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/2394e409/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputCommitterContainer.java
----------------------------------------------------------------------
diff --git a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputCommitterContainer.java b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputCommitterContainer.java
index 3f5deaf..4a76010 100644
--- a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputCommitterContainer.java
+++ b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputCommitterContainer.java
@@ -33,10 +33,10 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hive.common.FileUtils;
import org.apache.hadoop.hive.common.StatsSetupConst;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.io.HdfsUtils;
import org.apache.hadoop.hive.metastore.IMetaStoreClient;
import org.apache.hadoop.hive.metastore.Warehouse;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
@@ -323,18 +323,18 @@ class FileOutputCommitterContainer extends OutputCommitterContainer {
* @param params The parameters to store inside the partition
* @param table The Table metadata object under which this Partition will reside
* @param fs FileSystem object to operate on the underlying filesystem
- * @param grpName Group name that owns the table dir
- * @param perms FsPermission that's the default permission of the table dir.
+ * @param conf HiveConf used to access FS
+ * @param status Permission that's the default permission of the table dir.
* @return Constructed Partition metadata object
* @throws java.io.IOException
*/
private Partition constructPartition(
- JobContext context, OutputJobInfo jobInfo,
- String partLocnRoot, String dynPartPath, Map<String, String> partKVs,
- HCatSchema outputSchema, Map<String, String> params,
- Table table, FileSystem fs,
- String grpName, FsPermission perms) throws IOException {
+ JobContext context, OutputJobInfo jobInfo,
+ String partLocnRoot, String dynPartPath, Map<String, String> partKVs,
+ HCatSchema outputSchema, Map<String, String> params,
+ Table table, FileSystem fs, HiveConf conf,
+ HdfsUtils.HadoopFileStatus status) throws IOException {
Partition partition = new Partition();
partition.setDbName(table.getDbName());
@@ -371,18 +371,16 @@ class FileOutputCommitterContainer extends OutputCommitterContainer {
for (FieldSchema partKey : table.getPartitionKeys()) {
if (i++ != 0) {
fs.mkdirs(partPath); // Attempt to make the path in case it does not exist before we check
- applyGroupAndPerms(fs, partPath, perms, grpName, false);
+ HdfsUtils.setFullFileStatus(conf, status, status.getFileStatus().getGroup(), fs,
+ partPath, false);
}
partPath = constructPartialPartPath(partPath, partKey.getName().toLowerCase(), partKVs);
}
}
- // Apply the group and permissions to the leaf partition and files.
- // Need not bother in case of HDFS as permission is taken care of by setting UMask
- fs.mkdirs(partPath); // Attempt to make the path in case it does not exist before we check
- if (!ShimLoader.getHadoopShims().getHCatShim().isFileInHDFS(fs, partPath)) {
- applyGroupAndPerms(fs, partPath, perms, grpName, true);
- }
+ // Do not need to set the status on the partition directory. We will do it later recursively.
+ // See: end of the registerPartitions method
+ fs.mkdirs(partPath);
// Set the location in the StorageDescriptor
if (dynamicPartitioningUsed) {
@@ -400,26 +398,6 @@ class FileOutputCommitterContainer extends OutputCommitterContainer {
return partition;
}
- private void applyGroupAndPerms(FileSystem fs, Path dir, FsPermission permission,
- String group, boolean recursive)
- throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("applyGroupAndPerms : " + dir +
- " perms: " + permission +
- " group: " + group + " recursive: " + recursive);
- }
- fs.setPermission(dir, permission);
- if (recursive) {
- for (FileStatus fileStatus : fs.listStatus(dir)) {
- if (fileStatus.isDir()) {
- applyGroupAndPerms(fs, fileStatus.getPath(), permission, group, true);
- } else {
- fs.setPermission(fileStatus.getPath(), permission);
- }
- }
- }
- }
-
private String getFinalDynamicPartitionDestination(Table table, Map<String, String> partKVs,
OutputJobInfo jobInfo) {
Path partPath = new Path(table.getTTable().getSd().getLocation());
@@ -794,21 +772,19 @@ class FileOutputCommitterContainer extends OutputCommitterContainer {
StorerInfo storer = InternalUtil.extractStorerInfo(table.getTTable().getSd(),
table.getParameters());
- FileStatus tblStat = fs.getFileStatus(tblPath);
- String grpName = tblStat.getGroup();
- FsPermission perms = tblStat.getPermission();
+ HdfsUtils.HadoopFileStatus status = new HdfsUtils.HadoopFileStatus(conf, fs, tblPath);
List<Partition> partitionsToAdd = new ArrayList<Partition>();
if (!dynamicPartitioningUsed) {
partitionsToAdd.add(constructPartition(context, jobInfo, tblPath.toString(), null,
jobInfo.getPartitionValues(), jobInfo.getOutputSchema(), getStorerParameterMap(storer),
- table, fs, grpName, perms));
+ table, fs, hiveConf, status));
} else {
for (Entry<String, Map<String, String>> entry : partitionsDiscoveredByPath.entrySet()) {
partitionsToAdd.add(constructPartition(context, jobInfo,
getPartitionRootLocation(entry.getKey(), entry.getValue().size()), entry.getKey(),
entry.getValue(), jobInfo.getOutputSchema(), getStorerParameterMap(storer), table,
- fs, grpName, perms));
+ fs, hiveConf, status));
}
}
@@ -950,9 +926,9 @@ class FileOutputCommitterContainer extends OutputCommitterContainer {
// Set permissions appropriately for each of the partitions we just created
// so as to have their permissions mimic the table permissions
for (Partition p : partitionsAdded){
- applyGroupAndPerms(fs,new Path(p.getSd().getLocation()),tblStat.getPermission(),tblStat.getGroup(),true);
+ HdfsUtils.setFullFileStatus(conf, status, status.getFileStatus().getGroup(), fs,
+ new Path(p.getSd().getLocation()), true);
}
-
}
} catch (Exception e) {
if (partitionsAdded.size() > 0) {