You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ai...@apache.org on 2018/08/22 17:27:07 UTC
[1/2] hive git commit: HIVE-20237: Do Not Print StackTraces to STDERR
in HiveMetaStore (Alice Fan, reviewed Aihua Xu)
Repository: hive
Updated Branches:
refs/heads/master 4408661c0 -> ff2554328
HIVE-20237: Do Not Print StackTraces to STDERR in HiveMetaStore (Alice Fan, reviewed Aihua Xu)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/1f9c70e5
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/1f9c70e5
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/1f9c70e5
Branch: refs/heads/master
Commit: 1f9c70e5148add278a1cb9b2186fb9a65d7ed1c5
Parents: 4408661
Author: Aihua Xu <ai...@apache.org>
Authored: Wed Aug 22 10:02:08 2018 -0700
Committer: Aihua Xu <ai...@apache.org>
Committed: Wed Aug 22 10:02:08 2018 -0700
----------------------------------------------------------------------
.../hadoop/hive/metastore/HiveMetaStore.java | 280 +++++++++----------
1 file changed, 137 insertions(+), 143 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/1f9c70e5/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
index 324035a..067eb5a 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
@@ -9219,166 +9219,160 @@ public class HiveMetaStore extends ThriftHiveMetastore {
public static void startMetaStore(int port, HadoopThriftAuthBridge bridge,
Configuration conf, Lock startLock, Condition startCondition,
AtomicBoolean startedServing) throws Throwable {
- try {
- isMetaStoreRemote = true;
- // Server will create new threads up to max as necessary. After an idle
- // period, it will destroy threads to keep the number of threads in the
- // pool to min.
- long maxMessageSize = MetastoreConf.getLongVar(conf, ConfVars.SERVER_MAX_MESSAGE_SIZE);
- int minWorkerThreads = MetastoreConf.getIntVar(conf, ConfVars.SERVER_MIN_THREADS);
- int maxWorkerThreads = MetastoreConf.getIntVar(conf, ConfVars.SERVER_MAX_THREADS);
- boolean tcpKeepAlive = MetastoreConf.getBoolVar(conf, ConfVars.TCP_KEEP_ALIVE);
- boolean useFramedTransport = MetastoreConf.getBoolVar(conf, ConfVars.USE_THRIFT_FRAMED_TRANSPORT);
- boolean useCompactProtocol = MetastoreConf.getBoolVar(conf, ConfVars.USE_THRIFT_COMPACT_PROTOCOL);
- boolean useSSL = MetastoreConf.getBoolVar(conf, ConfVars.USE_SSL);
- useSasl = MetastoreConf.getBoolVar(conf, ConfVars.USE_THRIFT_SASL);
-
- if (useSasl) {
- // we are in secure mode. Login using keytab
- String kerberosName = SecurityUtil
- .getServerPrincipal(MetastoreConf.getVar(conf, ConfVars.KERBEROS_PRINCIPAL), "0.0.0.0");
- String keyTabFile = MetastoreConf.getVar(conf, ConfVars.KERBEROS_KEYTAB_FILE);
- UserGroupInformation.loginUserFromKeytab(kerberosName, keyTabFile);
- }
-
- TProcessor processor;
- TTransportFactory transFactory;
- final TProtocolFactory protocolFactory;
- final TProtocolFactory inputProtoFactory;
- if (useCompactProtocol) {
- protocolFactory = new TCompactProtocol.Factory();
- inputProtoFactory = new TCompactProtocol.Factory(maxMessageSize, maxMessageSize);
+ isMetaStoreRemote = true;
+ // Server will create new threads up to max as necessary. After an idle
+ // period, it will destroy threads to keep the number of threads in the
+ // pool to min.
+ long maxMessageSize = MetastoreConf.getLongVar(conf, ConfVars.SERVER_MAX_MESSAGE_SIZE);
+ int minWorkerThreads = MetastoreConf.getIntVar(conf, ConfVars.SERVER_MIN_THREADS);
+ int maxWorkerThreads = MetastoreConf.getIntVar(conf, ConfVars.SERVER_MAX_THREADS);
+ boolean tcpKeepAlive = MetastoreConf.getBoolVar(conf, ConfVars.TCP_KEEP_ALIVE);
+ boolean useFramedTransport = MetastoreConf.getBoolVar(conf, ConfVars.USE_THRIFT_FRAMED_TRANSPORT);
+ boolean useCompactProtocol = MetastoreConf.getBoolVar(conf, ConfVars.USE_THRIFT_COMPACT_PROTOCOL);
+ boolean useSSL = MetastoreConf.getBoolVar(conf, ConfVars.USE_SSL);
+ useSasl = MetastoreConf.getBoolVar(conf, ConfVars.USE_THRIFT_SASL);
+
+ if (useSasl) {
+ // we are in secure mode. Login using keytab
+ String kerberosName = SecurityUtil
+ .getServerPrincipal(MetastoreConf.getVar(conf, ConfVars.KERBEROS_PRINCIPAL), "0.0.0.0");
+ String keyTabFile = MetastoreConf.getVar(conf, ConfVars.KERBEROS_KEYTAB_FILE);
+ UserGroupInformation.loginUserFromKeytab(kerberosName, keyTabFile);
+ }
+
+ TProcessor processor;
+ TTransportFactory transFactory;
+ final TProtocolFactory protocolFactory;
+ final TProtocolFactory inputProtoFactory;
+ if (useCompactProtocol) {
+ protocolFactory = new TCompactProtocol.Factory();
+ inputProtoFactory = new TCompactProtocol.Factory(maxMessageSize, maxMessageSize);
+ } else {
+ protocolFactory = new TBinaryProtocol.Factory();
+ inputProtoFactory = new TBinaryProtocol.Factory(true, true, maxMessageSize, maxMessageSize);
+ }
+ HMSHandler baseHandler = new HiveMetaStore.HMSHandler("new db based metaserver", conf,
+ false);
+ IHMSHandler handler = newRetryingHMSHandler(baseHandler, conf);
+
+ TServerSocket serverSocket;
+
+ if (useSasl) {
+ // we are in secure mode.
+ if (useFramedTransport) {
+ throw new HiveMetaException("Framed transport is not supported with SASL enabled.");
+ }
+ saslServer = bridge.createServer(
+ MetastoreConf.getVar(conf, ConfVars.KERBEROS_KEYTAB_FILE),
+ MetastoreConf.getVar(conf, ConfVars.KERBEROS_PRINCIPAL),
+ MetastoreConf.getVar(conf, ConfVars.CLIENT_KERBEROS_PRINCIPAL));
+ // Start delegation token manager
+ delegationTokenManager = new MetastoreDelegationTokenManager();
+ delegationTokenManager.startDelegationTokenSecretManager(conf, baseHandler, HadoopThriftAuthBridge.Server.ServerMode.METASTORE);
+ saslServer.setSecretManager(delegationTokenManager.getSecretManager());
+ transFactory = saslServer.createTransportFactory(
+ MetaStoreUtils.getMetaStoreSaslProperties(conf, useSSL));
+ processor = saslServer.wrapProcessor(
+ new ThriftHiveMetastore.Processor<>(handler));
+
+ LOG.info("Starting DB backed MetaStore Server in Secure Mode");
+ } else {
+ // we are in unsecure mode.
+ if (MetastoreConf.getBoolVar(conf, ConfVars.EXECUTE_SET_UGI)) {
+ transFactory = useFramedTransport ?
+ new ChainedTTransportFactory(new TFramedTransport.Factory(),
+ new TUGIContainingTransport.Factory())
+ : new TUGIContainingTransport.Factory();
+
+ processor = new TUGIBasedProcessor<>(handler);
+ LOG.info("Starting DB backed MetaStore Server with SetUGI enabled");
} else {
- protocolFactory = new TBinaryProtocol.Factory();
- inputProtoFactory = new TBinaryProtocol.Factory(true, true, maxMessageSize, maxMessageSize);
+ transFactory = useFramedTransport ?
+ new TFramedTransport.Factory() : new TTransportFactory();
+ processor = new TSetIpAddressProcessor<>(handler);
+ LOG.info("Starting DB backed MetaStore Server");
}
- HMSHandler baseHandler = new HiveMetaStore.HMSHandler("new db based metaserver", conf,
- false);
- IHMSHandler handler = newRetryingHMSHandler(baseHandler, conf);
-
- TServerSocket serverSocket;
+ }
- if (useSasl) {
- // we are in secure mode.
- if (useFramedTransport) {
- throw new HiveMetaException("Framed transport is not supported with SASL enabled.");
- }
- saslServer = bridge.createServer(
- MetastoreConf.getVar(conf, ConfVars.KERBEROS_KEYTAB_FILE),
- MetastoreConf.getVar(conf, ConfVars.KERBEROS_PRINCIPAL),
- MetastoreConf.getVar(conf, ConfVars.CLIENT_KERBEROS_PRINCIPAL));
- // Start delegation token manager
- delegationTokenManager = new MetastoreDelegationTokenManager();
- delegationTokenManager.startDelegationTokenSecretManager(conf, baseHandler, HadoopThriftAuthBridge.Server.ServerMode.METASTORE);
- saslServer.setSecretManager(delegationTokenManager.getSecretManager());
- transFactory = saslServer.createTransportFactory(
- MetaStoreUtils.getMetaStoreSaslProperties(conf, useSSL));
- processor = saslServer.wrapProcessor(
- new ThriftHiveMetastore.Processor<>(handler));
-
- LOG.info("Starting DB backed MetaStore Server in Secure Mode");
- } else {
- // we are in unsecure mode.
- if (MetastoreConf.getBoolVar(conf, ConfVars.EXECUTE_SET_UGI)) {
- transFactory = useFramedTransport ?
- new ChainedTTransportFactory(new TFramedTransport.Factory(),
- new TUGIContainingTransport.Factory())
- : new TUGIContainingTransport.Factory();
-
- processor = new TUGIBasedProcessor<>(handler);
- LOG.info("Starting DB backed MetaStore Server with SetUGI enabled");
- } else {
- transFactory = useFramedTransport ?
- new TFramedTransport.Factory() : new TTransportFactory();
- processor = new TSetIpAddressProcessor<>(handler);
- LOG.info("Starting DB backed MetaStore Server");
- }
+ if (!useSSL) {
+ serverSocket = SecurityUtils.getServerSocket(null, port);
+ } else {
+ String keyStorePath = MetastoreConf.getVar(conf, ConfVars.SSL_KEYSTORE_PATH).trim();
+ if (keyStorePath.isEmpty()) {
+ throw new IllegalArgumentException(ConfVars.SSL_KEYSTORE_PATH.toString()
+ + " Not configured for SSL connection");
}
+ String keyStorePassword =
+ MetastoreConf.getPassword(conf, MetastoreConf.ConfVars.SSL_KEYSTORE_PASSWORD);
- if (!useSSL) {
- serverSocket = SecurityUtils.getServerSocket(null, port);
- } else {
- String keyStorePath = MetastoreConf.getVar(conf, ConfVars.SSL_KEYSTORE_PATH).trim();
- if (keyStorePath.isEmpty()) {
- throw new IllegalArgumentException(ConfVars.SSL_KEYSTORE_PATH.toString()
- + " Not configured for SSL connection");
- }
- String keyStorePassword =
- MetastoreConf.getPassword(conf, MetastoreConf.ConfVars.SSL_KEYSTORE_PASSWORD);
-
- // enable SSL support for HMS
- List<String> sslVersionBlacklist = new ArrayList<>();
- for (String sslVersion : MetastoreConf.getVar(conf, ConfVars.SSL_PROTOCOL_BLACKLIST).split(",")) {
- sslVersionBlacklist.add(sslVersion);
- }
-
- serverSocket = SecurityUtils.getServerSSLSocket(null, port, keyStorePath,
- keyStorePassword, sslVersionBlacklist);
+ // enable SSL support for HMS
+ List<String> sslVersionBlacklist = new ArrayList<>();
+ for (String sslVersion : MetastoreConf.getVar(conf, ConfVars.SSL_PROTOCOL_BLACKLIST).split(",")) {
+ sslVersionBlacklist.add(sslVersion);
}
- if (tcpKeepAlive) {
- serverSocket = new TServerSocketKeepAlive(serverSocket);
- }
+ serverSocket = SecurityUtils.getServerSSLSocket(null, port, keyStorePath,
+ keyStorePassword, sslVersionBlacklist);
+ }
- // Metrics will have already been initialized if we're using them since HMSHandler
- // initializes them.
- openConnections = Metrics.getOrCreateGauge(MetricsConstants.OPEN_CONNECTIONS);
+ if (tcpKeepAlive) {
+ serverSocket = new TServerSocketKeepAlive(serverSocket);
+ }
- TThreadPoolServer.Args args = new TThreadPoolServer.Args(serverSocket)
- .processor(processor)
- .transportFactory(transFactory)
- .protocolFactory(protocolFactory)
- .inputProtocolFactory(inputProtoFactory)
- .minWorkerThreads(minWorkerThreads)
- .maxWorkerThreads(maxWorkerThreads);
+ // Metrics will have already been initialized if we're using them since HMSHandler
+ // initializes them.
+ openConnections = Metrics.getOrCreateGauge(MetricsConstants.OPEN_CONNECTIONS);
- TServer tServer = new TThreadPoolServer(args);
- TServerEventHandler tServerEventHandler = new TServerEventHandler() {
- @Override
- public void preServe() {
- }
+ TThreadPoolServer.Args args = new TThreadPoolServer.Args(serverSocket)
+ .processor(processor)
+ .transportFactory(transFactory)
+ .protocolFactory(protocolFactory)
+ .inputProtocolFactory(inputProtoFactory)
+ .minWorkerThreads(minWorkerThreads)
+ .maxWorkerThreads(maxWorkerThreads);
- @Override
- public ServerContext createContext(TProtocol tProtocol, TProtocol tProtocol1) {
- openConnections.incrementAndGet();
- return null;
- }
+ TServer tServer = new TThreadPoolServer(args);
+ TServerEventHandler tServerEventHandler = new TServerEventHandler() {
+ @Override
+ public void preServe() {
+ }
- @Override
- public void deleteContext(ServerContext serverContext, TProtocol tProtocol, TProtocol tProtocol1) {
- openConnections.decrementAndGet();
- // If the IMetaStoreClient#close was called, HMSHandler#shutdown would have already
- // cleaned up thread local RawStore. Otherwise, do it now.
- cleanupRawStore();
- }
+ @Override
+ public ServerContext createContext(TProtocol tProtocol, TProtocol tProtocol1) {
+ openConnections.incrementAndGet();
+ return null;
+ }
- @Override
- public void processContext(ServerContext serverContext, TTransport tTransport, TTransport tTransport1) {
- }
- };
+ @Override
+ public void deleteContext(ServerContext serverContext, TProtocol tProtocol, TProtocol tProtocol1) {
+ openConnections.decrementAndGet();
+ // If the IMetaStoreClient#close was called, HMSHandler#shutdown would have already
+ // cleaned up thread local RawStore. Otherwise, do it now.
+ cleanupRawStore();
+ }
- tServer.setServerEventHandler(tServerEventHandler);
- HMSHandler.LOG.info("Started the new metaserver on port [" + port
- + "]...");
- HMSHandler.LOG.info("Options.minWorkerThreads = "
- + minWorkerThreads);
- HMSHandler.LOG.info("Options.maxWorkerThreads = "
- + maxWorkerThreads);
- HMSHandler.LOG.info("TCP keepalive = " + tcpKeepAlive);
- HMSHandler.LOG.info("Enable SSL = " + useSSL);
+ @Override
+ public void processContext(ServerContext serverContext, TTransport tTransport, TTransport tTransport1) {
+ }
+ };
- boolean directSqlEnabled = MetastoreConf.getBoolVar(conf, ConfVars.TRY_DIRECT_SQL);
- HMSHandler.LOG.info("Direct SQL optimization = {}", directSqlEnabled);
+ tServer.setServerEventHandler(tServerEventHandler);
+ HMSHandler.LOG.info("Started the new metaserver on port [" + port
+ + "]...");
+ HMSHandler.LOG.info("Options.minWorkerThreads = "
+ + minWorkerThreads);
+ HMSHandler.LOG.info("Options.maxWorkerThreads = "
+ + maxWorkerThreads);
+ HMSHandler.LOG.info("TCP keepalive = " + tcpKeepAlive);
+ HMSHandler.LOG.info("Enable SSL = " + useSSL);
- if (startLock != null) {
- signalOtherThreadsToStart(tServer, startLock, startCondition, startedServing);
- }
- tServer.serve();
- } catch (Throwable x) {
- x.printStackTrace();
- HMSHandler.LOG.error(StringUtils.stringifyException(x));
- throw x;
+ boolean directSqlEnabled = MetastoreConf.getBoolVar(conf, ConfVars.TRY_DIRECT_SQL);
+ HMSHandler.LOG.info("Direct SQL optimization = {}", directSqlEnabled);
+
+ if (startLock != null) {
+ signalOtherThreadsToStart(tServer, startLock, startCondition, startedServing);
}
+ tServer.serve();
}
private static void cleanupRawStore() {
[2/2] hive git commit: HIVE-20246: Configurable collecting stats by
using DO_NOT_UPDATE_STATS table property (Alice Fan, reviewed by Aihua Xu)
Posted by ai...@apache.org.
HIVE-20246: Configurable collecting stats by using DO_NOT_UPDATE_STATS table property (Alice Fan, reviewed by Aihua Xu)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ff255432
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ff255432
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ff255432
Branch: refs/heads/master
Commit: ff25543282532613c9cfa768a8bf8ee728b3734e
Parents: 1f9c70e
Author: Aihua Xu <ai...@apache.org>
Authored: Wed Aug 22 10:05:12 2018 -0700
Committer: Aihua Xu <ai...@apache.org>
Committed: Wed Aug 22 10:05:12 2018 -0700
----------------------------------------------------------------------
.../hadoop/hive/metastore/HiveMetaStore.java | 27 ++++++++--
.../hive/metastore/TestHiveMetaStore.java | 57 +++++++++++++++++++-
2 files changed, 79 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/ff255432/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
index 067eb5a..e971d0f 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
@@ -3174,8 +3174,7 @@ public class HiveMetaStore extends ThriftHiveMetastore {
part.setCreateTime((int) time);
part.putToParameters(hive_metastoreConstants.DDL_TIME, Long.toString(time));
- if (MetastoreConf.getBoolVar(conf, ConfVars.STATS_AUTO_GATHER) &&
- !MetaStoreServerUtils.isView(tbl)) {
+ if (canUpdateStats(tbl)) {
MetaStoreServerUtils.updatePartitionStatsFast(part, tbl, wh, madeDir, false, envContext, true);
}
@@ -3791,6 +3790,27 @@ public class HiveMetaStore extends ThriftHiveMetastore {
return result;
}
+ /**
+ * Verify if update stats while altering partition(s)
+ * For the following three cases HMS will not update partition stats
+ * 1) Table property 'DO_NOT_UPDATE_STATS' = True
+ * 2) HMS configuration property 'STATS_AUTO_GATHER' = False
+ * 3) Is View
+ */
+ private boolean canUpdateStats(Table tbl) {
+ Map<String,String> tblParams = tbl.getParameters();
+ boolean updateStatsTbl = true;
+ if ((tblParams != null) && tblParams.containsKey(StatsSetupConst.DO_NOT_UPDATE_STATS)) {
+ updateStatsTbl = !Boolean.valueOf(tblParams.get(StatsSetupConst.DO_NOT_UPDATE_STATS));
+ }
+ if (!MetastoreConf.getBoolVar(conf, ConfVars.STATS_AUTO_GATHER) ||
+ MetaStoreServerUtils.isView(tbl) ||
+ !updateStatsTbl) {
+ return false;
+ }
+ return true;
+ }
+
private void initializeAddedPartition(
final Table tbl, final Partition part, boolean madeDir) throws MetaException {
initializeAddedPartition(tbl, new PartitionSpecProxy.SimplePartitionWrapperIterator(part), madeDir);
@@ -3798,8 +3818,7 @@ public class HiveMetaStore extends ThriftHiveMetastore {
private void initializeAddedPartition(
final Table tbl, final PartitionSpecProxy.PartitionIterator part, boolean madeDir) throws MetaException {
- if (MetastoreConf.getBoolVar(conf, ConfVars.STATS_AUTO_GATHER) &&
- !MetaStoreServerUtils.isView(tbl)) {
+ if (canUpdateStats(tbl)) {
MetaStoreServerUtils.updatePartitionStatsFast(part, tbl, wh, madeDir, false, null, true);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/ff255432/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
index 60beab6..4937d9d 100644
--- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
+++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.hive.metastore;
-import java.lang.reflect.Field;
import java.io.IOException;
import java.sql.Connection;
import java.sql.DriverManager;
@@ -38,6 +37,8 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
+import java.lang.reflect.*;
+import static org.mockito.Mockito.mock;
import com.google.common.collect.Sets;
import org.apache.hadoop.hive.metastore.client.builder.DatabaseBuilder;
@@ -57,6 +58,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hive.common.StatsSetupConst;
import org.apache.hadoop.hive.metastore.api.AggrStats;
import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
@@ -95,6 +97,8 @@ import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
public abstract class TestHiveMetaStore {
private static final Logger LOG = LoggerFactory.getLogger(TestHiveMetaStore.class);
@@ -3098,4 +3102,55 @@ public abstract class TestHiveMetaStore {
int size = allUuids.size();
assertEquals(numAPICallsPerThread * parallelCalls, size);
}
+
+ /**
+ * While altering partition(s), verify DO NOT calculate partition statistics if
+ * <ol>
+ * <li>table property DO_NOT_UPDATE_STATS is true</li>
+ * <li>STATS_AUTO_GATHER is false</li>
+ * <li>Is View</li>
+ * </ol>
+ */
+ @Test
+ public void testUpdatePartitionStat_doesNotUpdateStats() throws Exception {
+ final String DB_NAME = "db1";
+ final String TABLE_NAME = "tbl1";
+ Table tbl = new TableBuilder()
+ .setDbName(DB_NAME)
+ .setTableName(TABLE_NAME)
+ .addCol("id", "int")
+ .addTableParam(StatsSetupConst.DO_NOT_UPDATE_STATS, "true")
+ .build(null);
+ List<String> vals = new ArrayList<>(2);
+ vals.add("col1");
+ vals.add("col2");
+ Partition part = new Partition();
+ part.setDbName(DB_NAME);
+ part.setTableName(TABLE_NAME);
+ part.setValues(vals);
+ part.setParameters(new HashMap<>());
+ part.setSd(tbl.getSd().deepCopy());
+ part.getSd().setSerdeInfo(tbl.getSd().getSerdeInfo());
+ part.getSd().setLocation(tbl.getSd().getLocation() + "/partCol=1");
+ Warehouse wh = mock(Warehouse.class);
+ //Execute initializeAddedPartition() and it should not trigger updatePartitionStatsFast() as DO_NOT_UPDATE_STATS is true
+ HiveMetaStore.HMSHandler hms = new HiveMetaStore.HMSHandler("", conf, false);
+ Method m = hms.getClass().getDeclaredMethod("initializeAddedPartition", Table.class, Partition.class, boolean.class);
+ m.setAccessible(true);
+ //Invoke initializeAddedPartition();
+ m.invoke(hms, tbl, part, false);
+ verify(wh, never()).getFileStatusesForLocation(part.getSd().getLocation());
+
+ //Remove tbl's DO_NOT_UPDATE_STATS & set STATS_AUTO_GATHER = false
+ tbl.unsetParameters();
+ MetastoreConf.setBoolVar(conf, ConfVars.STATS_AUTO_GATHER, false);
+ m.invoke(hms, tbl, part, false);
+ verify(wh, never()).getFileStatusesForLocation(part.getSd().getLocation());
+
+ //Set STATS_AUTO_GATHER = true and set tbl as a VIRTUAL_VIEW
+ MetastoreConf.setBoolVar(conf, ConfVars.STATS_AUTO_GATHER, true);
+ tbl.setTableType("VIRTUAL_VIEW");
+ m.invoke(hms, tbl, part, false);
+ verify(wh, never()).getFileStatusesForLocation(part.getSd().getLocation());
+ }
}