You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by de...@apache.org on 2026/04/10 06:42:12 UTC
(hive) branch master updated: HIVE-29429: Support multiple parent classes in HMSHander (#6377)
This is an automated email from the ASF dual-hosted git repository.
dengzh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 6f063df3dd2 HIVE-29429: Support multiple parent classes in HMSHander (#6377)
6f063df3dd2 is described below
commit 6f063df3dd24b9e5160eebcf0fc91c466a6b7005
Author: dengzh <de...@gmail.com>
AuthorDate: Fri Apr 10 14:42:05 2026 +0800
HIVE-29429: Support multiple parent classes in HMSHander (#6377)
---
.../hive/metastore/TestMetastoreVersion.java | 4 +-
.../apache/hadoop/hive/metastore/HMSHandler.java | 3464 +-------------------
.../hadoop/hive/metastore/HMSHandlerContext.java | 7 +-
.../hadoop/hive/metastore/MetaStoreInit.java | 2 +-
.../hive/metastore/TSetIpAddressProcessor.java | 3 +-
.../metastore/TransactionalValidationListener.java | 2 +-
.../hadoop/hive/metastore/cache/CachedStore.java | 2 +-
.../metastore/directsql/DirectSqlUpdatePart.java | 2 +-
.../metastore/handler/AppendPartitionHandler.java | 2 +-
.../hadoop/hive/metastore/handler/BaseHandler.java | 1016 ++++++
.../hive/metastore/handler/DeprecatedHandler.java | 1244 +++++++
.../hive/metastore/handler/PrivilegeHandler.java | 687 ++++
.../metastore/handler/SetAggrStatsHandler.java | 2 +-
.../hive/metastore/handler/TransactionHandler.java | 567 ++++
.../hive/metastore/utils/MetaStoreServerUtils.java | 21 +
.../hadoop/hive/metastore/IpAddressListener.java | 3 +-
16 files changed, 3711 insertions(+), 3317 deletions(-)
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java
index 89c71bbffd0..c80897132ad 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java
@@ -25,6 +25,7 @@
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConfForTest;
import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.handler.BaseHandler;
import org.apache.hadoop.hive.ql.DriverFactory;
import org.apache.hadoop.hive.ql.IDriver;
import org.apache.hadoop.hive.ql.metadata.Hive;
@@ -55,8 +56,7 @@ public class TestMetastoreVersion {
@Before
public void setUp() throws Exception {
-
- Field defDb = HMSHandler.class.getDeclaredField("currentUrl");
+ Field defDb = BaseHandler.class.getDeclaredField("currentUrl");
defDb.setAccessible(true);
defDb.set(null, null);
// reset defaults
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java
index 522a62f90ef..842b9390af3 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java
@@ -17,14 +17,9 @@
*/
package org.apache.hadoop.hive.metastore;
-import com.codahale.metrics.Counter;
-import com.facebook.fb303.FacebookBase;
-import com.facebook.fb303.fb_status;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
-import com.google.common.base.Splitter;
import com.google.common.collect.Lists;
-import com.google.common.util.concurrent.Striped;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
@@ -41,11 +36,11 @@
import org.apache.hadoop.hive.metastore.handler.AbstractRequestHandler;
import org.apache.hadoop.hive.metastore.handler.AddPartitionsHandler;
import org.apache.hadoop.hive.metastore.handler.AppendPartitionHandler;
+import org.apache.hadoop.hive.metastore.handler.BaseHandler;
import org.apache.hadoop.hive.metastore.handler.DropPartitionsHandler;
import org.apache.hadoop.hive.metastore.handler.GetPartitionsHandler;
+import org.apache.hadoop.hive.metastore.handler.PrivilegeHandler;
import org.apache.hadoop.hive.metastore.messaging.EventMessage.EventType;
-import org.apache.hadoop.hive.metastore.metrics.Metrics;
-import org.apache.hadoop.hive.metastore.metrics.MetricsConstants;
import org.apache.hadoop.hive.metastore.metrics.PerfLogger;
import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy;
import org.apache.hadoop.hive.metastore.properties.PropertyException;
@@ -53,23 +48,16 @@
import org.apache.hadoop.hive.metastore.properties.PropertyMap;
import org.apache.hadoop.hive.metastore.properties.PropertyStore;
import org.apache.hadoop.hive.metastore.txn.*;
-import org.apache.hadoop.hive.metastore.txn.entities.CompactionInfo;
-import org.apache.hadoop.hive.metastore.utils.FileUtils;
import org.apache.hadoop.hive.metastore.utils.FilterUtils;
import org.apache.hadoop.hive.metastore.utils.JavaUtils;
import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils;
import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
-import org.apache.hadoop.hive.metastore.utils.MetastoreVersionInfo;
import org.apache.hadoop.hive.metastore.utils.SecurityUtils;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.util.ReflectionUtils;
import org.apache.thrift.TException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import javax.jdo.JDOException;
import java.io.IOException;
-import java.nio.ByteBuffer;
import java.util.*;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.Lock;
@@ -77,7 +65,6 @@
import java.util.regex.Pattern;
import java.util.stream.Collectors;
-import static org.apache.commons.lang3.StringUtils.isBlank;
import static org.apache.commons.lang3.StringUtils.join;
import static org.apache.hadoop.hive.common.repl.ReplConst.REPL_RESUME_STARTED_AFTER_FAILOVER;
@@ -88,50 +75,30 @@
import static org.apache.hadoop.hive.metastore.ExceptionHandler.rethrowException;
import static org.apache.hadoop.hive.metastore.ExceptionHandler.throwMetaException;
import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_CATALOG_NAME;
-import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_DATABASE_COMMENT;
import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_DATABASE_NAME;
import static org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars.HIVE_IN_TEST;
+import static org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils.getPartValsFromName;
import static org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils.isDbReplicationTarget;
import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.CAT_NAME;
import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.DB_NAME;
import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.getDefaultCatalog;
import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.parseDbName;
-import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.prependCatalogToDbName;
import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.prependNotNullCatToDbName;
import static org.apache.hadoop.hive.metastore.utils.StringUtils.normalizeIdentifier;
/**
* Default handler for all Hive Metastore methods. Implements methods defined in hive_metastore.thrift.
*/
-public class HMSHandler extends FacebookBase implements IHMSHandler {
+public class HMSHandler extends PrivilegeHandler {
public static final Logger LOG = LoggerFactory.getLogger(HMSHandler.class);
- private final Configuration conf; // stores datastore (jpox) properties,
- // right now they come from jpox.properties
-
- private static String currentUrl;
- private FileMetadataManager fileMetadataManager;
- private PartitionExpressionProxy expressionProxy;
+ private final boolean isInTest;
private StorageSchemaReader storageSchemaReader;
- private IMetaStoreMetadataTransformer transformer;
- private static DataConnectorProviderFactory dataconnectorFactory = null;
public static final String PARTITION_NUMBER_EXCEED_LIMIT_MSG =
"Number of partitions scanned (=%d) on table '%s' exceeds limit (=%d). This is controlled on the metastore server by %s.";
- public static final String ADMIN = "admin";
- public static final String PUBLIC = "public";
-
- static final int LOG_SAMPLE_PARTITIONS_MAX_SIZE = 4;
-
- static final int LOG_SAMPLE_PARTITIONS_HALF_SIZE = 2;
-
- static final String LOG_SAMPLE_PARTITIONS_SEPARATOR = ",";
-
private static final String AVRO_SERDE_CLASS = "org.apache.hadoop.hive.serde2.avro.AvroSerDe";
- private Warehouse wh; // hdfs warehouse
- private static Striped<Lock> tablelocks;
-
public static RawStore getRawStore() {
return HMSHandlerContext.getRawStore().orElse(null);
}
@@ -145,7 +112,7 @@ static void cleanupHandlerContext() {
cleanedRawStore.set(true);
});
} finally {
- HMSHandlerContext.getHMSHandler().ifPresent(HMSHandler::notifyMetaListenersOnShutDown);
+ HMSHandlerContext.getHMSHandler().ifPresent(BaseHandler::notifyMetaListenersOnShutDown);
if (cleanedRawStore.get()) {
logAndAudit("Done cleaning up thread local RawStore");
}
@@ -153,29 +120,6 @@ static void cleanupHandlerContext() {
}
}
- static final Logger auditLog = LoggerFactory.getLogger(
- HiveMetaStore.class.getName() + ".audit");
-
- private static void logAuditEvent(String cmd) {
- if (cmd == null) {
- return;
- }
-
- UserGroupInformation ugi;
- try {
- ugi = SecurityUtils.getUGI();
- } catch (Exception ex) {
- throw new RuntimeException(ex);
- }
-
- String address = getIPAddress();
- if (address == null) {
- address = "unknown-ip-addr";
- }
-
- auditLog.info("ugi={} ip={} cmd={} ", ugi.getUserName(), address, cmd);
- }
-
public static String getIPAddress() {
if (HiveMetaStore.useSasl) {
if (HiveMetaStore.saslServer != null && HiveMetaStore.saslServer.getRemoteAddress() != null) {
@@ -188,297 +132,15 @@ public static String getIPAddress() {
return null;
}
- /**
- * Internal function to notify listeners to revert back to old values of keys
- * that were modified during setMetaConf. This would get called from HiveMetaStore#cleanupHandlerContext
- */
- private void notifyMetaListenersOnShutDown() {
- try {
- Map<String, String> modifiedConf = HMSHandlerContext.getModifiedConfig();
- Configuration conf = HMSHandlerContext.getConfiguration()
- .orElseThrow(() -> new MetaException("Unexpected: modifiedConf is non-null but conf is null"));
- // Notify listeners of the changed value
- for (Map.Entry<String, String> entry : modifiedConf.entrySet()) {
- String key = entry.getKey();
- // curr value becomes old and vice-versa
- String currVal = entry.getValue();
- String oldVal = conf.get(key);
- if (!Objects.equals(oldVal, currVal)) {
- for (List<MetaStoreEventListener> eventListeners :
- new List[] { listeners, transactionalListeners }) {
- MetaStoreListenerNotifier.notifyEvent(eventListeners, EventType.CONFIG_CHANGE,
- new ConfigChangeEvent(this, key, oldVal, currVal));
- }
- }
- }
- logAndAudit("Meta listeners shutdown notification completed.");
- } catch (MetaException e) {
- LOG.error("Failed to notify meta listeners on shutdown: ", e);
- }
- }
-
- static void setThreadLocalIpAddress(String ipAddress) {
- HMSHandlerContext.setIpAddress(ipAddress);
- }
-
- // This will return null if the metastore is not being accessed from a metastore Thrift server,
- // or if the TTransport being used to connect is not an instance of TSocket, or if kereberos
- // is used
- static String getThreadLocalIpAddress() {
- return HMSHandlerContext.getIpAddress().orElse(null);
- }
-
- // Make it possible for tests to check that the right type of PartitionExpressionProxy was
- // instantiated.
- @VisibleForTesting
- PartitionExpressionProxy getExpressionProxy() {
- return expressionProxy;
- }
-
public HMSHandler(String name, Configuration conf) {
- super(name);
- this.conf = conf;
+ super(name, conf);
isInTest = MetastoreConf.getBoolVar(this.conf, HIVE_IN_TEST);
- if (tablelocks == null) {
- synchronized (HMSHandler.class) {
- if (tablelocks == null) {
- int numTableLocks = MetastoreConf.getIntVar(conf, ConfVars.METASTORE_NUM_STRIPED_TABLE_LOCKS);
- tablelocks = Striped.lock(numTableLocks);
- }
- }
- }
- }
-
- /**
- * Use {@link #getConf()} instead.
- * @return Configuration object
- */
- @Deprecated
- public Configuration getHiveConf() {
- return conf;
- }
-
- private AlterHandler alterHandler;
- private List<MetaStorePreEventListener> preListeners;
- private List<MetaStoreEventListener> listeners;
- private List<TransactionalMetaStoreEventListener> transactionalListeners;
- private List<MetaStoreEndFunctionListener> endFunctionListeners;
- private List<MetaStoreInitListener> initListeners;
- private MetaStoreFilterHook filterHook;
- private boolean isServerFilterEnabled = false;
-
- private final boolean isInTest;
-
- @Override
- public List<TransactionalMetaStoreEventListener> getTransactionalListeners() {
- return transactionalListeners;
- }
-
- @Override
- public List<MetaStoreEventListener> getListeners() {
- return listeners;
- }
-
- @Override
- public IMetaStoreMetadataTransformer getMetadataTransformer() {
- return transformer;
- }
-
- @Override
- public MetaStoreFilterHook getMetaFilterHook() {
- return filterHook;
}
@Override
public void init() throws MetaException {
- init(new Warehouse(conf));
- }
-
- @VisibleForTesting
- public void init(Warehouse wh) throws MetaException {
- Metrics.initialize(conf);
- initListeners = MetaStoreServerUtils.getMetaStoreListeners(
- MetaStoreInitListener.class, conf, MetastoreConf.getVar(conf, ConfVars.INIT_HOOKS));
- for (MetaStoreInitListener singleInitListener: initListeners) {
- MetaStoreInitContext context = new MetaStoreInitContext();
- singleInitListener.onInit(context);
- }
-
- String alterHandlerName = MetastoreConf.getVar(conf, ConfVars.ALTER_HANDLER);
- alterHandler = ReflectionUtils.newInstance(JavaUtils.getClass(
- alterHandlerName, AlterHandler.class), conf);
- this.wh = wh;
-
- synchronized (HMSHandler.class) {
- if (currentUrl == null || !currentUrl.equals(MetaStoreInit.getConnectionURL(conf))) {
- createDefaultDB();
- createDefaultRoles();
- addAdminUsers();
- currentUrl = MetaStoreInit.getConnectionURL(conf);
- updateMetrics();
- }
- }
-
- preListeners = MetaStoreServerUtils.getMetaStoreListeners(MetaStorePreEventListener.class,
- conf, MetastoreConf.getVar(conf, ConfVars.PRE_EVENT_LISTENERS));
- preListeners.add(0, new TransactionalValidationListener(conf));
- listeners = MetaStoreServerUtils.getMetaStoreListeners(MetaStoreEventListener.class, conf,
- MetastoreConf.getVar(conf, ConfVars.EVENT_LISTENERS));
- listeners.add(new SessionPropertiesListener(conf));
- transactionalListeners = new ArrayList() {{
- add(new AcidEventListener(conf));
- }};
- transactionalListeners.addAll(MetaStoreServerUtils.getMetaStoreListeners(
- TransactionalMetaStoreEventListener.class, conf,
- MetastoreConf.getVar(conf, ConfVars.TRANSACTIONAL_EVENT_LISTENERS)));
- if (Metrics.getRegistry() != null) {
- listeners.add(new HMSMetricsListener(conf));
- }
-
- boolean canCachedStoreCanUseEvent = false;
- for (MetaStoreEventListener listener : transactionalListeners) {
- if (listener.doesAddEventsToNotificationLogTable()) {
- canCachedStoreCanUseEvent = true;
- break;
- }
- }
- if (conf.getBoolean(ConfVars.METASTORE_CACHE_CAN_USE_EVENT.getVarname(), false) &&
- !canCachedStoreCanUseEvent) {
- throw new MetaException("CahcedStore can not use events for invalidation as there is no " +
- " TransactionalMetaStoreEventListener to add events to notification table");
- }
-
- endFunctionListeners = MetaStoreServerUtils.getMetaStoreListeners(
- MetaStoreEndFunctionListener.class, conf, MetastoreConf.getVar(conf, ConfVars.END_FUNCTION_LISTENERS));
-
- expressionProxy = PartFilterExprUtil.createExpressionProxy(conf);
- fileMetadataManager = new FileMetadataManager(this.getMS(), conf);
-
- isServerFilterEnabled = getIfServerFilterenabled();
- filterHook = isServerFilterEnabled ? loadFilterHooks() : null;
-
- String className = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.METASTORE_METADATA_TRANSFORMER_CLASS);
- if (className != null && !className.trim().isEmpty()) {
- try {
- transformer = JavaUtils.newInstance(JavaUtils.getClass(className.trim(), IMetaStoreMetadataTransformer.class),
- new Class[] {IHMSHandler.class}, new Object[] {this});
- } catch (Exception e) {
- LOG.error("Unable to create instance of class " + className, e);
- throw new IllegalArgumentException(e);
- }
- }
- dataconnectorFactory = DataConnectorProviderFactory.getInstance(this);
- }
-
- /**
- *
- * Filter is actually enabled only when the configured filter hook is configured, not default, and
- * enabled in configuration
- * @return
- */
- private boolean getIfServerFilterenabled() throws MetaException{
- boolean isEnabled = MetastoreConf.getBoolVar(conf, ConfVars.METASTORE_SERVER_FILTER_ENABLED);
-
- if (!isEnabled) {
- LOG.info("HMS server filtering is disabled by configuration");
- return false;
- }
-
- String filterHookClassName = MetastoreConf.getVar(conf, ConfVars.FILTER_HOOK);
-
- if (isBlank(filterHookClassName)) {
- throw new MetaException("HMS server filtering is enabled but no filter hook is configured");
- }
-
- if (filterHookClassName.trim().equalsIgnoreCase(DefaultMetaStoreFilterHookImpl.class.getName())) {
- throw new MetaException("HMS server filtering is enabled but the filter hook is DefaultMetaStoreFilterHookImpl, which does no filtering");
- }
-
- LOG.info("HMS server filtering is enabled. The filter class is " + filterHookClassName);
- return true;
- }
-
- private MetaStoreFilterHook loadFilterHooks() throws IllegalStateException {
- String errorMsg = "Unable to load filter hook at HMS server. ";
-
- String filterHookClassName = MetastoreConf.getVar(conf, ConfVars.FILTER_HOOK);
- Preconditions.checkState(!isBlank(filterHookClassName));
-
- try {
- return (MetaStoreFilterHook)Class.forName(
- filterHookClassName.trim(), true, JavaUtils.getClassLoader()).getConstructor(
- Configuration.class).newInstance(conf);
- } catch (Exception e) {
- LOG.error(errorMsg, e);
- throw new IllegalStateException(errorMsg + e.getMessage(), e);
- }
- }
-
- @Override
- public void setConf(Configuration conf) {
- HMSHandlerContext.setConfiguration(conf);
- // reload if DS related configuration is changed
- HMSHandlerContext.getRawStore().ifPresent(ms -> ms.setConf(conf));
- }
-
- @Override
- public Configuration getConf() {
- return HMSHandlerContext.getConfiguration()
- .orElseGet(
- () -> {
- Configuration configuration = new Configuration(this.conf);
- HMSHandlerContext.setConfiguration(configuration);
- return configuration;
- });
- }
-
- @Override
- public Warehouse getWh() {
- return wh;
- }
-
- @Override
- public void setMetaConf(String key, String value) throws MetaException {
- ConfVars confVar = MetastoreConf.getMetaConf(key);
- if (confVar == null) {
- throw new MetaException("Invalid configuration key " + key);
- }
- try {
- confVar.validate(value);
- } catch (IllegalArgumentException e) {
- throw new MetaException("Invalid configuration value " + value + " for key " + key +
- " by " + e.getMessage());
- }
- Configuration configuration = getConf();
- String oldValue = MetastoreConf.get(configuration, key);
- // Save prev val of the key on threadLocal
- Map<String, String> modifiedConf = HMSHandlerContext.getModifiedConfig();
- if (!modifiedConf.containsKey(key)) {
- modifiedConf.put(key, oldValue);
- }
- // Set invoking HMSHandler on threadLocal, this will be used later to notify
- // metaListeners in HiveMetaStore#cleanupHandlerContext
- if (!HMSHandlerContext.getHMSHandler().isPresent()) {
- HMSHandlerContext.setHMSHandler(this);
- }
- configuration.set(key, value);
- for (List<MetaStoreEventListener> eventListeners :
- new List[] { listeners, transactionalListeners }) {
- MetaStoreListenerNotifier.notifyEvent(eventListeners, EventType.CONFIG_CHANGE,
- new ConfigChangeEvent(this, key, oldValue, value));
- }
- if (ConfVars.TRY_DIRECT_SQL == confVar) {
- HMSHandler.LOG.info("Direct SQL optimization = {}", value);
- }
- }
-
- @Override
- public String getMetaConf(String key) throws MetaException {
- ConfVars confVar = MetastoreConf.getMetaConf(key);
- if (confVar == null) {
- throw new MetaException("Invalid configuration key " + key);
- }
- return getConf().get(key, confVar.getDefaultVal().toString());
+ super.init();
+ DataConnectorProviderFactory.getInstance(this);
}
/**
@@ -509,11 +171,6 @@ public static RawStore getMSForConf(Configuration conf) throws MetaException {
return ms;
}
- @Override
- public TxnStore getTxnHandler() {
- return HMSHandlerContext.getTxnStore(conf);
- }
-
static RawStore newRawStoreForConf(Configuration conf) throws MetaException {
Configuration newConf = new Configuration(conf);
String rawStoreClassName = MetastoreConf.getVar(newConf, ConfVars.RAW_STORE_IMPL);
@@ -544,309 +201,12 @@ public static void createDefaultCatalog(RawStore ms, Warehouse wh) throws MetaEx
}
}
- private void createDefaultDB_core(RawStore ms) throws MetaException, InvalidObjectException {
- try {
- ms.getDatabase(DEFAULT_CATALOG_NAME, DEFAULT_DATABASE_NAME);
- } catch (NoSuchObjectException e) {
- LOG.info("Started creating a default database with name: "+DEFAULT_DATABASE_NAME);
- Database db = new Database(DEFAULT_DATABASE_NAME, DEFAULT_DATABASE_COMMENT,
- wh.getDefaultDatabasePath(DEFAULT_DATABASE_NAME, true).toString(), null);
- db.setOwnerName(PUBLIC);
- db.setOwnerType(PrincipalType.ROLE);
- db.setCatalogName(DEFAULT_CATALOG_NAME);
- long time = System.currentTimeMillis() / 1000;
- db.setCreateTime((int) time);
- db.setType(DatabaseType.NATIVE);
- ms.createDatabase(db);
- LOG.info("Successfully created a default database with name: "+DEFAULT_DATABASE_NAME);
- }
- }
-
- /**
- * create default database if it doesn't exist.
- *
- * This is a potential contention when HiveServer2 using embedded metastore and Metastore
- * Server try to concurrently invoke createDefaultDB. If one failed, JDOException was caught
- * for one more time try, if failed again, simply ignored by warning, which meant another
- * succeeds.
- *
- * @throws MetaException
- */
- private void createDefaultDB() throws MetaException {
- try {
- RawStore ms = getMS();
- createDefaultCatalog(ms, wh);
- createDefaultDB_core(ms);
- } catch (JDOException e) {
- LOG.warn("Retrying creating default database after error: " + e.getMessage(), e);
- try {
- RawStore ms = getMS();
- createDefaultCatalog(ms, wh);
- createDefaultDB_core(ms);
- } catch (InvalidObjectException | InvalidOperationException e1) {
- throw new MetaException(e1.getMessage());
- }
- } catch (InvalidObjectException|InvalidOperationException e) {
- throw new MetaException(e.getMessage());
- }
- }
-
- /**
- * create default roles if they don't exist.
- *
- * This is a potential contention when HiveServer2 using embedded metastore and Metastore
- * Server try to concurrently invoke createDefaultRoles. If one failed, JDOException was caught
- * for one more time try, if failed again, simply ignored by warning, which meant another
- * succeeds.
- *
- * @throws MetaException
- */
- private void createDefaultRoles() throws MetaException {
- try {
- createDefaultRoles_core();
- } catch (JDOException e) {
- LOG.warn("Retrying creating default roles after error: " + e.getMessage(), e);
- createDefaultRoles_core();
- }
- }
-
- private void createDefaultRoles_core() throws MetaException {
-
- RawStore ms = getMS();
- try {
- ms.addRole(ADMIN, ADMIN);
- } catch (InvalidObjectException e) {
- LOG.debug(ADMIN +" role already exists",e);
- } catch (NoSuchObjectException e) {
- // This should never be thrown.
- LOG.warn("Unexpected exception while adding " +ADMIN+" roles" , e);
- }
- LOG.info("Added "+ ADMIN+ " role in metastore");
- try {
- ms.addRole(PUBLIC, PUBLIC);
- } catch (InvalidObjectException e) {
- LOG.debug(PUBLIC + " role already exists",e);
- } catch (NoSuchObjectException e) {
- // This should never be thrown.
- LOG.warn("Unexpected exception while adding "+PUBLIC +" roles" , e);
- }
- LOG.info("Added "+PUBLIC+ " role in metastore");
- // now grant all privs to admin
- PrivilegeBag privs = new PrivilegeBag();
- privs.addToPrivileges(new HiveObjectPrivilege( new HiveObjectRef(HiveObjectType.GLOBAL, null,
- null, null, null), ADMIN, PrincipalType.ROLE, new PrivilegeGrantInfo("All", 0, ADMIN,
- PrincipalType.ROLE, true), "SQL"));
- try {
- ms.grantPrivileges(privs);
- } catch (InvalidObjectException e) {
- // Surprisingly these privs are already granted.
- LOG.debug("Failed while granting global privs to admin", e);
- } catch (NoSuchObjectException e) {
- // Unlikely to be thrown.
- LOG.warn("Failed while granting global privs to admin", e);
- }
- }
-
- /**
- * add admin users if they don't exist.
- *
- * This is a potential contention when HiveServer2 using embedded metastore and Metastore
- * Server try to concurrently invoke addAdminUsers. If one failed, JDOException was caught for
- * one more time try, if failed again, simply ignored by warning, which meant another succeeds.
- *
- * @throws MetaException
- */
- private void addAdminUsers() throws MetaException {
- try {
- addAdminUsers_core();
- } catch (JDOException e) {
- LOG.warn("Retrying adding admin users after error: " + e.getMessage(), e);
- addAdminUsers_core();
- }
- }
-
- private void addAdminUsers_core() throws MetaException {
-
- // now add pre-configured users to admin role
- String userStr = MetastoreConf.getVar(conf,ConfVars.USERS_IN_ADMIN_ROLE,"").trim();
- if (userStr.isEmpty()) {
- LOG.info("No user is added in admin role, since config is empty");
- return;
- }
- // Since user names need to be valid unix user names, per IEEE Std 1003.1-2001 they cannot
- // contain comma, so we can safely split above string on comma.
-
- Iterator<String> users = Splitter.on(",").trimResults().omitEmptyStrings().split(userStr).iterator();
- if (!users.hasNext()) {
- LOG.info("No user is added in admin role, since config value "+ userStr +
- " is in incorrect format. We accept comma separated list of users.");
- return;
- }
- Role adminRole;
- RawStore ms = getMS();
- try {
- adminRole = ms.getRole(ADMIN);
- } catch (NoSuchObjectException e) {
- LOG.error("Failed to retrieve just added admin role",e);
- return;
- }
- while (users.hasNext()) {
- String userName = users.next();
- try {
- ms.grantRole(adminRole, userName, PrincipalType.USER, ADMIN, PrincipalType.ROLE, true);
- LOG.info("Added " + userName + " to admin role");
- } catch (NoSuchObjectException e) {
- LOG.error("Failed to add "+ userName + " in admin role",e);
- } catch (InvalidObjectException e) {
- LOG.debug(userName + " already in admin role", e);
- }
- }
- }
-
- private static void logAndAudit(final String m) {
- LOG.debug(m);
- logAuditEvent(m);
- }
-
- private String startFunction(String function, String extraLogInfo) {
- incrementCounter(function);
- logAndAudit((getThreadLocalIpAddress() == null ? "" : "source:" + getThreadLocalIpAddress() + " ") +
- function + extraLogInfo);
- com.codahale.metrics.Timer timer =
- Metrics.getOrCreateTimer(MetricsConstants.API_PREFIX + function);
- if (timer != null) {
- // Timer will be null we aren't using the metrics
- HMSHandlerContext.getTimerContexts().put(function, timer.time());
- }
- Counter counter = Metrics.getOrCreateCounter(MetricsConstants.ACTIVE_CALLS + function);
- if (counter != null) {
- counter.inc();
- }
- return function;
- }
-
- private String startFunction(String function) {
- return startFunction(function, "");
- }
-
- private void startTableFunction(String function, String catName, String db, String tbl) {
- startFunction(function, " : tbl=" +
- TableName.getQualified(catName, db, tbl));
- }
-
- private void startMultiTableFunction(String function, String db, List<String> tbls) {
- String tableNames = join(tbls, ",");
- startFunction(function, " : db=" + db + " tbls=" + tableNames);
- }
-
- private void startPartitionFunction(String function, String cat, String db, String tbl,
- List<String> partVals) {
- startFunction(function, " : tbl=" +
- TableName.getQualified(cat, db, tbl) + samplePartitionValues(partVals));
- }
-
- private void startPartitionFunction(String function, String catName, String db, String tbl,
- Map<String, String> partName) {
- startFunction(function, " : tbl=" +
- TableName.getQualified(catName, db, tbl) + " partition=" + partName);
- }
-
- private void startPartitionFunction(String function, String catName, String db, String tbl, int maxParts) {
- startFunction(function, " : tbl=" + TableName.getQualified(catName, db, tbl) + ": Max partitions =" + maxParts);
- }
-
- private void startPartitionFunction(String function, String catName, String db, String tbl, int maxParts,
- List<String> partVals) {
- startFunction(function, " : tbl=" + TableName.getQualified(catName, db, tbl) + ": Max partitions =" + maxParts
- + samplePartitionValues(partVals));
- }
-
- private void startPartitionFunction(String function, String catName, String db, String tbl, int maxParts,
- String filter) {
- startFunction(function,
- " : tbl=" + TableName.getQualified(catName, db, tbl) + ": Filter=" + filter + ": Max partitions ="
- + maxParts);
- }
-
- private void startPartitionFunction(String function, String catName, String db, String tbl, int maxParts,
- String expression, String defaultPartitionName) {
- startFunction(function, " : tbl=" + TableName.getQualified(catName, db, tbl) + ": Expression=" + expression
- + ": Default partition name=" + defaultPartitionName + ": Max partitions=" + maxParts);
- }
-
- private String getGroupsCountAndUsername(final String user_name, final List<String> group_names) {
- return ". Number of groups= " + (group_names == null ? 0 : group_names.size()) + ", user name= " + user_name;
- }
-
- private String samplePartitionValues(List<String> partVals) {
- if (CollectionUtils.isEmpty(partVals)) {
- return ": Partitions = []";
- }
- StringBuilder sb = new StringBuilder(": Number of Partitions = " + partVals.size());
- sb.append(": Partitions = [");
- if (partVals.size() > LOG_SAMPLE_PARTITIONS_MAX_SIZE) {
- // extracting starting 2 values, and ending 2 values
- sb.append(join(partVals.subList(0, LOG_SAMPLE_PARTITIONS_HALF_SIZE), LOG_SAMPLE_PARTITIONS_SEPARATOR));
- sb.append(" .... ");
- sb.append(join(partVals.subList(partVals.size() - LOG_SAMPLE_PARTITIONS_HALF_SIZE, partVals.size()),
- LOG_SAMPLE_PARTITIONS_SEPARATOR));
- } else {
- sb.append(join(partVals, LOG_SAMPLE_PARTITIONS_SEPARATOR));
- }
- sb.append("]");
- return sb.toString();
- }
-
- private void endFunction(String function, boolean successful, Exception e) {
- endFunction(function, successful, e, null);
- }
- private void endFunction(String function, boolean successful, Exception e,
- String inputTableName) {
- endFunction(function, new MetaStoreEndFunctionContext(successful, e, inputTableName));
- }
-
- private void endFunction(String function, MetaStoreEndFunctionContext context) {
- com.codahale.metrics.Timer.Context timerContext = HMSHandlerContext.getTimerContexts().remove(function);
- if (timerContext != null) {
- long timeTaken = timerContext.stop();
- LOG.debug((getThreadLocalIpAddress() == null ? "" : "source:" + getThreadLocalIpAddress() + " ") +
- function + " time taken(ns): " + timeTaken);
- }
- Counter counter = Metrics.getOrCreateCounter(MetricsConstants.ACTIVE_CALLS + function);
- if (counter != null) {
- counter.dec();
- }
-
- for (MetaStoreEndFunctionListener listener : endFunctionListeners) {
- listener.onEndFunction(function, context);
- }
- }
-
- @Override
- public fb_status getStatus() {
- return fb_status.ALIVE;
- }
-
@Override
public void shutdown() {
cleanupHandlerContext();
PerfLogger.getPerfLogger(false).cleanupPerfLogMetrics();
}
- @Override
- public AbstractMap<String, Long> getCounters() {
- AbstractMap<String, Long> counters = super.getCounters();
-
- // Allow endFunctionListeners to add any counters they have collected
- if (endFunctionListeners != null) {
- for (MetaStoreEndFunctionListener listener : endFunctionListeners) {
- listener.exportCounters(counters);
- }
- }
-
- return counters;
- }
-
@Override
public void create_catalog(CreateCatalogRequest rqst)
throws AlreadyExistsException, InvalidObjectException, MetaException {
@@ -1107,30 +467,6 @@ private void dropCatalogCore(String catName, boolean ifExists)
}
}
- @Override
- @Deprecated
- public void create_database(final Database db)
- throws AlreadyExistsException, InvalidObjectException, MetaException {
- CreateDatabaseRequest req = new CreateDatabaseRequest(db.getName());
- req.setDescription(db.getDescription());
- req.setLocationUri(db.getLocationUri());
- req.setParameters(db.getParameters());
- req.setPrivileges(db.getPrivileges());
- req.setOwnerName(db.getOwnerName());
- req.setOwnerType(db.getOwnerType());
- req.setCatalogName(db.getCatalogName());
- req.setCreateTime(db.getCreateTime());
- req.setManagedLocationUri(db.getManagedLocationUri());
- req.setType(db.getType());
- req.setDataConnectorName(db.getConnector_name());
- req.setRemote_dbname(db.getRemote_dbname());
-
- create_database_req(req);
- //location and manged location might be set/changed.
- db.setLocationUri(req.getLocationUri());
- db.setManagedLocationUri(req.getManagedLocationUri());
- }
-
@Override
public void create_database_req(final CreateDatabaseRequest createDatabaseRequest)
throws AlreadyExistsException, InvalidObjectException, MetaException {
@@ -1160,19 +496,6 @@ public void create_database_req(final CreateDatabaseRequest createDatabaseReques
}
}
- @Override
- @Deprecated
- public Database get_database(final String name)
- throws NoSuchObjectException, MetaException {
- GetDatabaseRequest request = new GetDatabaseRequest();
- String[] parsedDbName = parseDbName(name, conf);
- request.setName(parsedDbName[DB_NAME]);
- if (parsedDbName[CAT_NAME] != null) {
- request.setCatalogName(parsedDbName[CAT_NAME]);
- }
- return get_database_req(request);
- }
-
@Override
public Database get_database_core(String catName, final String name) throws NoSuchObjectException, MetaException {
Database db = null;
@@ -1214,13 +537,6 @@ public Database get_database_req(GetDatabaseRequest request) throws NoSuchObject
return db;
}
- @Override
- @Deprecated
- public void alter_database(final String dbName, final Database newDB) throws TException {
- AlterDatabaseRequest alterDbReq = new AlterDatabaseRequest(dbName, newDB);
- alter_database_req(alterDbReq);
- }
-
@Override
public void alter_database_req(final AlterDatabaseRequest alterDbReq) throws TException {
startFunction("alter_database_req " + alterDbReq.getOldDbName());
@@ -1311,20 +627,6 @@ private boolean isReplicationEventIdUpdate(Database oldDb, Database newDb) {
return newReplId != null && !newReplId.equalsIgnoreCase(oldReplId);
}
- @Override
- @Deprecated
- public void drop_database(final String dbName, final boolean deleteData, final boolean cascade)
- throws NoSuchObjectException, InvalidOperationException, MetaException {
- String[] parsedDbName = parseDbName(dbName, conf);
-
- DropDatabaseRequest req = new DropDatabaseRequest();
- req.setName(parsedDbName[DB_NAME]);
- req.setCatalogName(parsedDbName[CAT_NAME]);
- req.setDeleteData(deleteData);
- req.setCascade(cascade);
- drop_database_req(req);
- }
-
@Override
public AsyncOperationResp drop_database_req(final DropDatabaseRequest req)
throws NoSuchObjectException, InvalidOperationException, MetaException {
@@ -1733,14 +1035,6 @@ public boolean drop_type(final String name) throws MetaException, NoSuchObjectEx
return success;
}
- @Override
- public Map<String, Type> get_type_all(String name) throws MetaException {
- // TODO Auto-generated method stub
- startFunction("get_type_all", ": " + name);
- endFunction("get_type_all", false, null);
- throw new MetaException("Not yet implemented");
- }
-
@Override
public Table translate_table_dryrun(final CreateTableRequest req) throws AlreadyExistsException,
MetaException, InvalidObjectException, InvalidInputException {
@@ -1758,48 +1052,15 @@ public Table translate_table_dryrun(final CreateTableRequest req) throws Already
}
@Override
- @Deprecated
- public void create_table(final Table tbl) throws AlreadyExistsException,
- MetaException, InvalidObjectException, InvalidInputException {
- CreateTableRequest createTableReq = new CreateTableRequest(tbl);
- create_table_req(createTableReq);
- }
-
- @Override
- @Deprecated
- public void create_table_with_environment_context(final Table tbl,
- final EnvironmentContext envContext)
+ public void create_table_req(final CreateTableRequest req)
throws AlreadyExistsException, MetaException, InvalidObjectException,
InvalidInputException {
- startFunction("create_table_with_environment_context", ": " + tbl.getTableName());
+ Table tbl = req.getTable();
+ startFunction("create_table_req", ": " + tbl.toString());
boolean success = false;
Exception ex = null;
try {
- CreateTableRequest createTableReq = new CreateTableRequest(tbl);
- if (envContext != null) {
- createTableReq.setEnvContext(envContext);
- }
- create_table_req(createTableReq);
- } catch (Exception e) {
- ex = e;
- throw handleException(e).throwIfInstance(MetaException.class, InvalidObjectException.class)
- .throwIfInstance(AlreadyExistsException.class, InvalidInputException.class)
- .defaultMetaException();
- } finally {
- endFunction("create_table_with_environment_context", success, ex, tbl.getTableName());
- }
- }
-
- @Override
- public void create_table_req(final CreateTableRequest req)
- throws AlreadyExistsException, MetaException, InvalidObjectException,
- InvalidInputException {
- Table tbl = req.getTable();
- startFunction("create_table_req", ": " + tbl.toString());
- boolean success = false;
- Exception ex = null;
- try {
- success = AbstractRequestHandler.offer(this, req).success();
+ success = AbstractRequestHandler.offer(this, req).success();
} catch (Exception e) {
LOG.warn("create_table_req got ", e);
ex = e;
@@ -1812,39 +1073,6 @@ public void create_table_req(final CreateTableRequest req)
}
}
- @Override
- @Deprecated
- public void create_table_with_constraints(final Table tbl,
- final List<SQLPrimaryKey> primaryKeys, final List<SQLForeignKey> foreignKeys,
- List<SQLUniqueConstraint> uniqueConstraints,
- List<SQLNotNullConstraint> notNullConstraints,
- List<SQLDefaultConstraint> defaultConstraints,
- List<SQLCheckConstraint> checkConstraints)
- throws AlreadyExistsException, MetaException, InvalidObjectException,
- InvalidInputException {
- startFunction("create_table_with_constraints", ": " + tbl.toString());
- boolean success = false;
- Exception ex = null;
- try {
- CreateTableRequest req = new CreateTableRequest(tbl);
- req.setPrimaryKeys(primaryKeys);
- req.setForeignKeys(foreignKeys);
- req.setUniqueConstraints(uniqueConstraints);
- req.setNotNullConstraints(notNullConstraints);
- req.setDefaultConstraints(defaultConstraints);
- req.setCheckConstraints(checkConstraints);
- create_table_req(req);
- success = true;
- } catch (Exception e) {
- ex = e;
- throw handleException(e).throwIfInstance(MetaException.class, InvalidObjectException.class)
- .throwIfInstance(AlreadyExistsException.class, InvalidInputException.class)
- .defaultMetaException();
- } finally {
- endFunction("create_table_with_constraints", success, ex, tbl.getTableName());
- }
- }
-
@Override
public void drop_constraint(DropConstraintRequest req)
throws MetaException, InvalidObjectException {
@@ -2143,31 +1371,6 @@ public void add_check_constraint(AddCheckConstraintRequest req)
}
}
- @Override
- @Deprecated
- public void drop_table(final String dbname, final String name, final boolean deleteData)
- throws NoSuchObjectException, MetaException {
- String[] parsedDbName = parseDbName(dbname, conf);
- DropTableRequest dropTableReq = new DropTableRequest(parsedDbName[DB_NAME], name);
- dropTableReq.setDeleteData(deleteData);
- dropTableReq.setCatalogName(parsedDbName[CAT_NAME]);
- dropTableReq.setDropPartitions(true);
- drop_table_req(dropTableReq);
- }
-
- @Override
- @Deprecated
- public void drop_table_with_environment_context(final String dbname, final String name, final boolean deleteData,
- final EnvironmentContext envContext) throws NoSuchObjectException, MetaException {
- String[] parsedDbName = parseDbName(dbname, conf);
- DropTableRequest dropTableReq = new DropTableRequest(parsedDbName[DB_NAME], name);
- dropTableReq.setDeleteData(deleteData);
- dropTableReq.setCatalogName(parsedDbName[CAT_NAME]);
- dropTableReq.setDropPartitions(true);
- dropTableReq.setEnvContext(envContext);
- drop_table_req(dropTableReq);
- }
-
@Override
public AsyncOperationResp drop_table_req(DropTableRequest dropReq)
throws MetaException, NoSuchObjectException {
@@ -2194,21 +1397,6 @@ public AsyncOperationResp drop_table_req(DropTableRequest dropReq)
}
}
- @Override
- public CmRecycleResponse cm_recycle(final CmRecycleRequest request) throws MetaException {
- wh.recycleDirToCmPath(new Path(request.getDataPath()), request.isPurge());
- return new CmRecycleResponse();
- }
-
- @Override
- public void truncate_table(final String dbName, final String tableName, List<String> partNames)
- throws NoSuchObjectException, MetaException {
- // Deprecated path, won't work for txn tables.
- TruncateTableRequest truncateTableReq = new TruncateTableRequest(dbName, tableName);
- truncateTableReq.setPartNames(partNames);
- truncate_table_req(truncateTableReq);
- }
-
@Override
public TruncateTableResponse truncate_table_req(TruncateTableRequest req)
throws NoSuchObjectException, MetaException {
@@ -2564,11 +1752,6 @@ private List<Table> getTableObjectsInternal(String catName, String dbName,
return tables;
}
- @Override
- public Materialization get_materialization_invalidation_info(final CreationMetadata cm, final String validTxnList) throws MetaException {
- return getTxnHandler().getMaterializationInvalidationInfo(cm, validTxnList);
- }
-
@Override
public void update_creation_metadata(String catName, final String dbName, final String tableName, CreationMetadata cm) throws MetaException {
getMS().updateCreationMetadata(catName, dbName, tableName, cm);
@@ -2624,47 +1807,6 @@ public List<String> get_table_names_by_filter(
return tables;
}
- public void firePreEvent(PreEventContext event) throws MetaException {
- for (MetaStorePreEventListener listener : preListeners) {
- try {
- listener.onEvent(event);
- } catch (NoSuchObjectException e) {
- throw new MetaException(e.getMessage());
- } catch (InvalidOperationException e) {
- throw new MetaException(e.getMessage());
- }
- }
- }
-
- @Override
- @Deprecated
- public Partition append_partition(final String dbName, final String tableName,
- final List<String> part_vals) throws InvalidObjectException,
- AlreadyExistsException, MetaException {
- String[] parsedDbName = parseDbName(dbName, conf);
- AppendPartitionsRequest appendPartitionsReq = new AppendPartitionsRequest();
- appendPartitionsReq.setDbName(parsedDbName[DB_NAME]);
- appendPartitionsReq.setTableName(tableName);
- appendPartitionsReq.setPartVals(part_vals);
- appendPartitionsReq.setCatalogName(parsedDbName[CAT_NAME]);
- return append_partition_req(appendPartitionsReq);
- }
-
- @Override
- @Deprecated
- public Partition append_partition_with_environment_context(final String dbName,
- final String tableName, final List<String> part_vals, final EnvironmentContext envContext)
- throws InvalidObjectException, AlreadyExistsException, MetaException {
- String[] parsedDbName = parseDbName(dbName, conf);
- AppendPartitionsRequest appendPartitionsReq = new AppendPartitionsRequest();
- appendPartitionsReq.setDbName(parsedDbName[DB_NAME]);
- appendPartitionsReq.setTableName(tableName);
- appendPartitionsReq.setPartVals(part_vals);
- appendPartitionsReq.setCatalogName(parsedDbName[CAT_NAME]);
- appendPartitionsReq.setEnvironmentContext(envContext);
- return append_partition_req(appendPartitionsReq);
- }
-
@Override
public Partition append_partition_req(final AppendPartitionsRequest appendPartitionsReq)
throws InvalidObjectException, AlreadyExistsException, MetaException {
@@ -2687,10 +1829,6 @@ public Partition append_partition_req(final AppendPartitionsRequest appendPartit
}
}
- public Lock getTableLockFor(String dbName, String tblName) {
- return tablelocks.get(dbName + "." + tblName);
- }
-
@Override
public AddPartitionsResult add_partitions_req(AddPartitionsRequest request)
throws TException {
@@ -2735,49 +1873,6 @@ public AddPartitionsResult add_partitions_req(AddPartitionsRequest request)
return result;
}
- @Deprecated
- @Override
- public int add_partitions(final List<Partition> parts) throws MetaException,
- InvalidObjectException, AlreadyExistsException {
- if (parts == null) {
- throw new MetaException("Partition list cannot be null.");
- }
- if (parts.isEmpty()) {
- return 0;
- }
- String catName = parts.get(0).isSetCatName() ? parts.get(0).getCatName() : getDefaultCatalog(conf);
- String dbName = parts.get(0).getDbName();
- String tableName = parts.get(0).getTableName();
- startTableFunction("add_partitions", catName, dbName, tableName);
-
- Integer ret = null;
- Exception ex = null;
- try {
- // Old API assumed all partitions belong to the same table; keep the same assumption
- if (!parts.get(0).isSetCatName()) {
- parts.forEach(p -> p.setCatName(catName));
- }
- AddPartitionsRequest addPartitionsReq = new AddPartitionsRequest();
- addPartitionsReq.setDbName(parts.get(0).getDbName());
- addPartitionsReq.setTblName(parts.get(0).getTableName());
- addPartitionsReq.setParts(parts);
- addPartitionsReq.setIfNotExists(false);
- if (parts.get(0).isSetCatName()) {
- addPartitionsReq.setCatName(parts.get(0).getCatName());
- }
- ret = add_partitions_req(addPartitionsReq).getPartitions().size();
- assert ret == parts.size();
- } catch (Exception e) {
- ex = e;
- throw handleException(e)
- .throwIfInstance(MetaException.class, InvalidObjectException.class, AlreadyExistsException.class)
- .defaultMetaException();
- } finally {
- endFunction("add_partitions", ret != null, ex, tableName);
- }
- return ret;
- }
-
@Override
public int add_partitions_pspec(final List<PartitionSpec> partSpecs)
throws TException {
@@ -2827,31 +1922,6 @@ public int add_partitions_pspec(final List<PartitionSpec> partSpecs)
}
}
- @Deprecated
- @Override
- public Partition add_partition(final Partition part)
- throws InvalidObjectException, AlreadyExistsException, MetaException {
- return add_partition_with_environment_context(part, null);
- }
-
- @Deprecated
- @Override
- public Partition add_partition_with_environment_context( final Partition part, EnvironmentContext envContext)
- throws InvalidObjectException, AlreadyExistsException, MetaException {
- if (part == null) {
- throw new MetaException("Partition cannot be null.");
- }
- AddPartitionsRequest addPartitionsReq = new AddPartitionsRequest(part.getDbName(), part.getTableName(),
- new ArrayList<>(Arrays.asList(part)), false);
- addPartitionsReq.setEnvironmentContext(envContext);
- try {
- return add_partitions_req(addPartitionsReq).getPartitions().get(0);
- } catch (Exception e) {
- throw handleException(e).throwIfInstance(MetaException.class, InvalidObjectException.class,
- AlreadyExistsException.class).defaultMetaException();
- }
- }
-
@Override
public Partition exchange_partition(Map<String, String> partitionSpecs,
String sourceDbName, String sourceTableName, String destDbName,
@@ -3041,31 +2111,6 @@ public List<Partition> exchange_partitions(Map<String, String> partitionSpecs,
}
}
- private void throwUnsupportedExceptionIfRemoteDB(String dbName, String operationName) throws MetaException {
- if (isDatabaseRemote(dbName)) {
- throw new MetaException("Operation " + operationName + " not supported for REMOTE database " + dbName);
- }
- }
-
- private boolean isDatabaseRemote(String name) {
- try {
- String[] parsedDbName = parseDbName(name, conf);
- Database db = get_database_core(parsedDbName[CAT_NAME], parsedDbName[DB_NAME]);
- return MetaStoreUtils.isDatabaseRemote(db);
- } catch (Exception e) {
- return false;
- }
- }
-
- @Deprecated
- @Override
- public boolean drop_partition(final String db_name, final String tbl_name,
- final List<String> part_vals, final boolean deleteData)
- throws TException {
- return drop_partition_with_environment_context(db_name, tbl_name, part_vals, deleteData,
- null);
- }
-
@Override
public DropPartitionsResult drop_partitions_req(
DropPartitionsRequest request) throws TException {
@@ -3088,19 +2133,6 @@ public DropPartitionsResult drop_partitions_req(
}
}
- @Deprecated
- @Override
- public boolean drop_partition_with_environment_context(final String db_name, final String tbl_name, final List<String> part_vals,
- final boolean deleteData, final EnvironmentContext envContext) throws TException {
- String[] parsedDbName = parseDbName(db_name, conf);
- DropPartitionRequest dropPartitionReq = new DropPartitionRequest(parsedDbName[DB_NAME], tbl_name);
- dropPartitionReq.setCatName(parsedDbName[CAT_NAME]);
- dropPartitionReq.setPartVals(part_vals);
- dropPartitionReq.setDeleteData(deleteData);
- dropPartitionReq.setEnvironmentContext(envContext);
- return drop_partition_req(dropPartitionReq);
- }
-
@Override
public boolean drop_partition_req(final DropPartitionRequest dropPartitionReq) throws TException {
String dbName = dropPartitionReq.getDbName();
@@ -3133,25 +2165,6 @@ public boolean drop_partition_req(final DropPartitionRequest dropPartitionReq) t
return true;
}
- /**
- * Use {@link #get_partition_req(GetPartitionRequest)} ()} instead.
- *
- */
- @Override
- @Deprecated
- public Partition get_partition(final String db_name, final String tbl_name,
- final List<String> part_vals) throws MetaException, NoSuchObjectException {
- try {
- String[] parsedDbName = parseDbName(db_name, conf);
- GetPartitionRequest getPartitionRequest = new GetPartitionRequest(parsedDbName[DB_NAME], tbl_name, part_vals);
- getPartitionRequest.setCatName(parsedDbName[CAT_NAME]);
- return get_partition_req(getPartitionRequest).getPartition();
- } catch (TException e) {
- throw handleException(e).throwIfInstance(MetaException.class, NoSuchObjectException.class)
- .defaultMetaException();
- }
- }
-
@Override
public GetPartitionResponse get_partition_req(GetPartitionRequest req)
throws MetaException, NoSuchObjectException, TException {
@@ -3168,97 +2181,6 @@ public GetPartitionResponse get_partition_req(GetPartitionRequest req)
return new GetPartitionResponse(partitions.getFirst());
}
- /**
- * filters out the table meta for which read database access is not granted
- * @param catName catalog name
- * @param tableMetas list of table metas
- * @return filtered list of table metas
- * @throws RuntimeException
- * @throws NoSuchObjectException
- */
- private List<TableMeta> filterReadableTables(String catName, List<TableMeta> tableMetas)
- throws RuntimeException, NoSuchObjectException {
- List<TableMeta> finalT = new ArrayList<>();
- Map<String, Boolean> databaseNames = new HashMap();
- for (TableMeta tableMeta : tableMetas) {
- String fullDbName = prependCatalogToDbName(catName, tableMeta.getDbName(), conf);
- if (databaseNames.get(fullDbName) == null) {
- boolean isExecptionThrown = false;
- try {
- fireReadDatabasePreEvent(fullDbName);
- } catch (MetaException e) {
- isExecptionThrown = true;
- }
- databaseNames.put(fullDbName, isExecptionThrown);
- }
- if (!databaseNames.get(fullDbName)) {
- finalT.add(tableMeta);
- }
- }
- return finalT;
- }
-
- /**
- * Fire a pre-event for read database operation, if there are any
- * pre-event listeners registered
- */
- private void fireReadDatabasePreEvent(final String name)
- throws MetaException, RuntimeException, NoSuchObjectException {
- if(preListeners.size() > 0) {
- String[] parsedDbName = parseDbName(name, conf);
- Database db = null;
- try {
- db = get_database_core(parsedDbName[CAT_NAME], parsedDbName[DB_NAME]);
- if (db == null) {
- throw new NoSuchObjectException("Database: " + name + " not found");
- }
- } catch(MetaException | NoSuchObjectException e) {
- throw new RuntimeException(e);
- }
- firePreEvent(new PreReadDatabaseEvent(db, this));
- }
- }
-
- @Override
- @Deprecated
- public Partition get_partition_with_auth(final String db_name,
- final String tbl_name, final List<String> part_vals,
- final String user_name, final List<String> group_names)
- throws TException {
- String[] parsedDbName = parseDbName(db_name, conf);
- TableName tableName = new TableName(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name);
- String partName = GetPartitionsHandler.validatePartVals(this, tableName, part_vals);
- GetPartitionsPsWithAuthRequest gpar = new GetPartitionsPsWithAuthRequest(tableName.getDb(), tableName.getTable());
- gpar.setCatName(tableName.getCat());
- gpar.setUserName(user_name);
- gpar.setGroupNames(group_names);
- gpar.setPartNames(List.of(partName));
- List<Partition> partitions = GetPartitionsHandler.getPartitions(
- t -> startFunction("get_partition_with_auth",
- " : tbl=" + t + samplePartitionValues(part_vals) + getGroupsCountAndUsername(user_name,group_names)),
- rex -> endFunction("get_partition_with_auth",
- rex.getLeft() != null && rex.getLeft().success(), rex.getRight(), tbl_name),
- this, tableName, gpar, true);
- return partitions.getFirst();
- }
-
- /**
- * Use {@link #get_partitions_req(PartitionsRequest)} ()} instead.
- *
- */
- @Override
- @Deprecated
- public List<Partition> get_partitions(final String db_name, final String tbl_name,
- final short max_parts) throws NoSuchObjectException, MetaException {
- String[] parsedDbName = parseDbName(db_name, conf);
- TableName tableName = new TableName(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name);
- return GetPartitionsHandler.getPartitions(
- t -> startTableFunction("get_partitions", tableName.getCat(), tableName.getDb(), tableName.getTable()),
- rex -> endFunction("get_partitions",
- rex.getLeft() != null && rex.getLeft().success(), rex.getRight(), tableName.toString()),
- this, tableName, GetPartitionsHandler.createPartitionsRequest(tableName, max_parts), false);
- }
-
@Override
public PartitionsResponse get_partitions_req(PartitionsRequest req)
throws NoSuchObjectException, MetaException, TException {
@@ -3273,66 +2195,6 @@ public PartitionsResponse get_partitions_req(PartitionsRequest req)
return new PartitionsResponse(partitions);
}
- @Override
- @Deprecated
- public List<Partition> get_partitions_with_auth(final String dbName,
- final String tblName, final short maxParts, final String userName,
- final List<String> groupNames) throws TException {
- String[] parsedDbName = parseDbName(dbName, conf);
- TableName tableName = new TableName(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tblName);
- GetPartitionsPsWithAuthRequest getAuthReq = new GetPartitionsPsWithAuthRequest(tableName.getDb(), tableName.getTable());
- getAuthReq.setCatName(tableName.getCat());
- getAuthReq.setUserName(userName);
- getAuthReq.setGroupNames(groupNames);
- getAuthReq.setMaxParts(maxParts);
- return get_partitions_ps_with_auth_req(getAuthReq).getPartitions();
- }
-
- @Override
- @Deprecated
- public List<PartitionSpec> get_partitions_pspec(final String db_name, final String tbl_name, final int max_parts)
- throws NoSuchObjectException, MetaException {
-
- String[] parsedDbName = parseDbName(db_name, conf);
- String catName = parsedDbName[CAT_NAME];
- String dbName = parsedDbName[DB_NAME];
- String tableName = tbl_name.toLowerCase();
-
- startPartitionFunction("get_partitions_pspec", catName, dbName, tableName, max_parts);
-
- List<PartitionSpec> partitionSpecs = null;
- try {
- GetTableRequest getTableRequest = new GetTableRequest(dbName, tableName);
- getTableRequest.setCatName(catName);
- Table table = get_table_core(getTableRequest);
- // get_partitions will parse out the catalog and db names itself
- TableName t = new TableName(catName, dbName, tableName);
- GetPartitionsHandler<PartitionsRequest, Partition> getPartitionsHandler = AbstractRequestHandler.offer(this,
- new GetPartitionsHandler.GetPartitionsRequest<>(GetPartitionsHandler.createPartitionsRequest(t, max_parts), t));
- List<Partition> partitions = getPartitionsHandler.getResult().result();
- if (is_partition_spec_grouping_enabled(table)) {
- partitionSpecs = MetaStoreServerUtils
- .getPartitionspecsGroupedByStorageDescriptor(table, partitions);
- }
- else {
- PartitionSpec pSpec = new PartitionSpec();
- pSpec.setPartitionList(new PartitionListComposingSpec(partitions));
- pSpec.setCatName(normalizeIdentifier(catName));
- pSpec.setDbName(normalizeIdentifier(dbName));
- pSpec.setTableName(tableName);
- pSpec.setRootPath(table.getSd().getLocation());
- partitionSpecs = Arrays.asList(pSpec);
- }
-
- return partitionSpecs;
- } catch (Exception e) {
- throw handleException(e).throwIfInstance(NoSuchObjectException.class, MetaException.class).defaultMetaException();
- }
- finally {
- endFunction("get_partitions_pspec", partitionSpecs != null && !partitionSpecs.isEmpty(), null, tbl_name);
- }
- }
-
@Override
public GetPartitionsResponse get_partitions_with_specs(GetPartitionsRequest request)
throws MetaException, TException {
@@ -3380,24 +2242,6 @@ public GetPartitionsResponse get_partitions_with_specs(GetPartitionsRequest requ
return response;
}
- private static boolean is_partition_spec_grouping_enabled(Table table) {
-
- Map<String, String> parameters = table.getParameters();
- return parameters.containsKey("hive.hcatalog.partition.spec.grouping.enabled")
- && parameters.get("hive.hcatalog.partition.spec.grouping.enabled").equalsIgnoreCase("true");
- }
-
- @Override
- @Deprecated
- public List<String> get_partition_names(final String db_name, final String tbl_name,
- final short max_parts) throws NoSuchObjectException, MetaException {
- String[] parsedDbName = parseDbName(db_name, conf);
- PartitionsRequest partitionReq = new PartitionsRequest(parsedDbName[DB_NAME], tbl_name);
- partitionReq.setCatName(parsedDbName[CAT_NAME]);
- partitionReq.setMaxParts(max_parts);
- return fetch_partition_names_req(partitionReq);
- }
-
@Override
public List<String> fetch_partition_names_req(final PartitionsRequest req)
throws NoSuchObjectException, MetaException {
@@ -3451,36 +2295,6 @@ public PartitionValuesResponse get_partition_values(PartitionValuesRequest reque
}
}
- @Deprecated
- @Override
- public void alter_partition(final String db_name, final String tbl_name,
- final Partition new_part)
- throws TException {
- rename_partition(db_name, tbl_name, null, new_part);
- }
-
- @Deprecated
- @Override
- public void alter_partition_with_environment_context(final String dbName,
- final String tableName, final Partition newPartition,
- final EnvironmentContext envContext)
- throws TException {
- String[] parsedDbName = parseDbName(dbName, conf);
- alter_partition_core(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableName, null,
- newPartition, envContext, null);
- }
-
- @Deprecated
- @Override
- public void rename_partition(final String db_name, final String tbl_name,
- final List<String> part_vals, final Partition new_part)
- throws TException {
- // Call alter_partition_core without an environment context.
- String[] parsedDbName = parseDbName(db_name, conf);
- alter_partition_core(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name, part_vals, new_part,
- null, null);
- }
-
@Override
public RenamePartitionResponse rename_partition_req(RenamePartitionRequest req) throws TException {
EnvironmentContext context = new EnvironmentContext();
@@ -3492,74 +2306,6 @@ public RenamePartitionResponse rename_partition_req(RenamePartitionRequest req)
return new RenamePartitionResponse();
};
- private void alter_partition_core(String catName, String db_name, String tbl_name,
- List<String> part_vals, Partition new_part, EnvironmentContext envContext,
- String validWriteIds) throws TException {
- startTableFunction("alter_partition", catName, db_name, tbl_name);
-
- if (LOG.isInfoEnabled()) {
- LOG.info("New partition values:" + new_part.getValues());
- if (part_vals != null && part_vals.size() > 0) {
- LOG.info("Old Partition values:" + part_vals);
- }
- }
-
- // Adds the missing scheme/authority for the new partition location
- if (new_part.getSd() != null) {
- String newLocation = new_part.getSd().getLocation();
- if (org.apache.commons.lang3.StringUtils.isNotEmpty(newLocation)) {
- Path tblPath = wh.getDnsPath(new Path(newLocation));
- new_part.getSd().setLocation(tblPath.toString());
- }
- }
-
- // Make sure the new partition has the catalog value set
- if (!new_part.isSetCatName()) {
- new_part.setCatName(catName);
- }
-
- Partition oldPart = null;
- Exception ex = null;
- try {
- Table table = getMS().getTable(catName, db_name, tbl_name, null);
-
- firePreEvent(new PreAlterPartitionEvent(db_name, tbl_name, table, part_vals, new_part, this));
- if (part_vals != null && !part_vals.isEmpty()) {
- MetaStoreServerUtils.validatePartitionNameCharacters(new_part.getValues(), getConf());
- }
-
- oldPart = alterHandler.alterPartition(getMS(), wh, catName, db_name, tbl_name,
- part_vals, new_part, envContext, this, validWriteIds);
-
- if (!listeners.isEmpty()) {
- MetaStoreListenerNotifier.notifyEvent(listeners,
- EventType.ALTER_PARTITION,
- new AlterPartitionEvent(oldPart, new_part, table, false,
- true, new_part.getWriteId(), this),
- envContext);
- }
- } catch (Exception e) {
- ex = e;
- throw handleException(e).throwIfInstance(MetaException.class, InvalidOperationException.class)
- .convertIfInstance(InvalidObjectException.class, InvalidOperationException.class)
- .convertIfInstance(AlreadyExistsException.class, InvalidOperationException.class)
- .defaultMetaException();
- } finally {
- endFunction("alter_partition", oldPart != null, ex, tbl_name);
- }
- }
-
- @Deprecated
- @Override
- public void alter_partitions(final String db_name, final String tbl_name,
- final List<Partition> new_parts)
- throws TException {
- String[] o = parseDbName(db_name, conf);
- AlterPartitionsRequest req = new AlterPartitionsRequest(o[1], tbl_name, new_parts);
- req.setCatName(o[0]);
- alter_partitions_req(req);
- }
-
@Override
public AlterPartitionsResponse alter_partitions_req(AlterPartitionsRequest req) throws TException {
alter_partitions_with_environment_context(req.getCatName(),
@@ -3569,19 +2315,6 @@ public AlterPartitionsResponse alter_partitions_req(AlterPartitionsRequest req)
return new AlterPartitionsResponse();
}
- // The old API we are keeping for backward compat. Not used within Hive.
- @Deprecated
- @Override
- public void alter_partitions_with_environment_context(final String db_name, final String tbl_name,
- final List<Partition> new_parts, EnvironmentContext environmentContext)
- throws TException {
- String[] o = parseDbName(db_name, conf);
- AlterPartitionsRequest req = new AlterPartitionsRequest(o[1], tbl_name, new_parts);
- req.setCatName(o[0]);
- req.setEnvironmentContext(environmentContext);
- alter_partitions_req(req);
- }
-
private void alter_partitions_with_environment_context(String catName, String db_name, final String tbl_name,
final List<Partition> new_parts, EnvironmentContext environmentContext,
String writeIdList, long writeId)
@@ -3655,42 +2388,6 @@ private void alter_partitions_with_environment_context(String catName, String db
}
}
- @Override
- public String getVersion() throws TException {
- String version = MetastoreVersionInfo.getVersion();
- endFunction(startFunction("getVersion"), version != null, null);
- return version;
- }
-
- @Deprecated
- @Override
- public void alter_table(final String dbname, final String name,
- final Table newTable)
- throws InvalidOperationException, MetaException {
- // Do not set an environment context.
- String[] parsedDbName = parseDbName(dbname, conf);
- AlterTableRequest req = new AlterTableRequest(parsedDbName[DB_NAME], name, newTable);
- req.setCatName(parsedDbName[CAT_NAME]);
- alter_table_req(req);
- }
-
- @Deprecated
- @Override
- public void alter_table_with_cascade(final String dbname, final String name,
- final Table newTable, final boolean cascade)
- throws InvalidOperationException, MetaException {
- EnvironmentContext envContext = null;
- if (cascade) {
- envContext = new EnvironmentContext();
- envContext.putToProperties(StatsSetupConst.CASCADE, StatsSetupConst.TRUE);
- }
- String[] parsedDbName = parseDbName(dbname, conf);
- AlterTableRequest req = new AlterTableRequest(parsedDbName[DB_NAME], name, newTable);
- req.setCatName(parsedDbName[CAT_NAME]);
- req.setEnvironmentContext(envContext);
- alter_table_req(req);
- }
-
@Override
public AlterTableResponse alter_table_req(AlterTableRequest req)
throws InvalidOperationException, MetaException {
@@ -3701,19 +2398,6 @@ public AlterTableResponse alter_table_req(AlterTableRequest req)
return new AlterTableResponse();
}
- @Deprecated
- @Override
- public void alter_table_with_environment_context(final String dbname,
- final String name, final Table newTable,
- final EnvironmentContext envContext)
- throws InvalidOperationException, MetaException {
- String[] parsedDbName = parseDbName(dbname, conf);
- AlterTableRequest req = new AlterTableRequest(parsedDbName[DB_NAME], name, newTable);
- req.setCatName(parsedDbName[CAT_NAME]);
- req.setEnvironmentContext(envContext);
- alter_table_req(req);
- }
-
private void alter_table_core(String catName, String dbname, String name, Table newTable,
EnvironmentContext envContext, String validWriteIdList, List<String> processorCapabilities,
String processorId, String expectedPropertyKey, String expectedPropertyValue)
@@ -3924,7 +2608,7 @@ public List<String> get_all_tables(final String dbname) throws MetaException {
} catch (Exception e) { /* ignore */ }
try {
- if (getIfServerFilterenabled()) {
+ if (isServerFilterEnabled) {
List<TableMeta> filteredTableMetas = get_table_meta(dbname, "*", null);
ret = filteredTableMetas.stream()
.map(TableMeta::getTableName)
@@ -3941,29 +2625,6 @@ public List<String> get_all_tables(final String dbname) throws MetaException {
return ret;
}
- /**
- * Use {@link #get_fields_req(GetFieldsRequest)} ()} instead.
- *
- */
- @Override
- @Deprecated
- public List<FieldSchema> get_fields(String db, String tableName)
- throws MetaException, UnknownTableException, UnknownDBException {
- return get_fields_with_environment_context(db, tableName, null);
- }
-
- @Override
- @Deprecated
- public List<FieldSchema> get_fields_with_environment_context(String db, String tableName,
- final EnvironmentContext envContext)
- throws MetaException, UnknownTableException, UnknownDBException {
- String[] parsedDbName = parseDbName(db, conf);
- GetFieldsRequest req = new GetFieldsRequest(parsedDbName[DB_NAME], tableName);
- req.setCatName(parsedDbName[CAT_NAME]);
- req.setEnvContext(envContext);
- return get_fields_req(req).getFields();
- }
-
private List<FieldSchema> get_fields_with_environment_context_core(String db, String tableName, final EnvironmentContext envContext)
throws MetaException, UnknownTableException, UnknownDBException {
startFunction("get_fields_with_environment_context_core", ": db=" + db + "tbl=" + tableName);
@@ -3974,7 +2635,10 @@ private List<FieldSchema> get_fields_with_environment_context_core(String db, St
List<FieldSchema> ret = null;
Exception ex = null;
try {
- Table tbl = getTableForGetFields(parsedDbName, base_table_name);
+ GetTableRequest getTableRequest = new GetTableRequest(parsedDbName[DB_NAME], base_table_name);
+ getTableRequest.setCatName(parsedDbName[CAT_NAME]);
+ Table tbl = get_table_core(getTableRequest);
+ firePreEvent(new PreReadTableEvent(tbl, this));
String serdeLib = tbl.getSd().getSerdeInfo().getSerializationLib();
if (serdeLib == null ||
MetastoreConf.getStringCollection(conf,
@@ -3985,26 +2649,14 @@ private List<FieldSchema> get_fields_with_environment_context_core(String db, St
}
} catch (Exception e) {
ex = e;
- throw handleException(e).throwIfInstance(UnknownTableException.class, MetaException.class).defaultMetaException();
+ throw handleException(e).convertIfInstance(NoSuchObjectException.class, UnknownTableException.class)
+ .defaultMetaException();
} finally {
endFunction("get_fields_with_environment_context_core", ret != null, ex, tableName);
}
return ret;
}
- private Table getTableForGetFields(String[] parsedDbName, String baseTableName)
- throws UnknownTableException, MetaException {
- try {
- GetTableRequest getTableRequest = new GetTableRequest(parsedDbName[DB_NAME], baseTableName);
- getTableRequest.setCatName(parsedDbName[CAT_NAME]);
- Table tbl = get_table_core(getTableRequest);
- firePreEvent(new PreReadTableEvent(tbl, this));
- return tbl;
- } catch (NoSuchObjectException e) {
- throw new UnknownTableException(e.getMessage());
- }
- }
-
private List<FieldSchema> getFieldsUsingSchemaReader(Table tbl, String db, String tableName, String serdeLib,
EnvironmentContext envContext) throws MetaException {
try {
@@ -4050,46 +2702,19 @@ private StorageSchemaReader getStorageSchemaReader() throws MetaException {
return storageSchemaReader;
}
- /**
- * Use {@link #get_schema_req(GetSchemaRequest)} ()} instead.
- *
- */
- @Override
- @Deprecated
- public List<FieldSchema> get_schema(String db, String tableName)
- throws MetaException, UnknownTableException, UnknownDBException {
- return get_schema_with_environment_context(db,tableName, null);
- }
-
- /**
- * Return the schema of the table. This function includes partition columns
- * in addition to the regular columns.
- *
- * @param db
- * Name of the database
- * @param tableName
- * Name of the table
- * @param envContext
- * Store session based properties
- * @return List of columns, each column is a FieldSchema structure
- * @throws MetaException
- * @throws UnknownTableException
- * @throws UnknownDBException
- */
@Override
- @Deprecated
- public List<FieldSchema> get_schema_with_environment_context(String db, String tableName,
- final EnvironmentContext envContext)
+ public GetSchemaResponse get_schema_req(GetSchemaRequest req)
throws MetaException, UnknownTableException, UnknownDBException {
- String[] parsedDbName = parseDbName(db, conf);
- GetSchemaRequest req = new GetSchemaRequest(parsedDbName[DB_NAME], tableName);
- req.setCatName(parsedDbName[CAT_NAME]);
- req.setEnvContext(envContext);
- return get_schema_req(req).getFields();
+ String dbName = MetaStoreUtils.prependCatalogToDbName(req.getCatName(), req.getDbName(), conf);
+ List<FieldSchema> fields = get_schema_with_environment_context_core(
+ dbName, req.getTblName(), req.getEnvContext());
+ GetSchemaResponse res = new GetSchemaResponse();
+ res.setFields(fields);
+ return res;
}
private List<FieldSchema> get_schema_with_environment_context_core(String db, String tableName, final EnvironmentContext envContext)
- throws MetaException, UnknownTableException, UnknownDBException {
+ throws MetaException, UnknownTableException, UnknownDBException {
startFunction("get_schema_with_environment_context_core", ": db=" + db + "tbl=" + tableName);
boolean success = false;
Exception ex = null;
@@ -4108,7 +2733,7 @@ private List<FieldSchema> get_schema_with_environment_context_core(String db, St
}
// Pass unparsed db name here
List<FieldSchema> fieldSchemas = get_fields_with_environment_context_core(db, base_table_name,
- envContext);
+ envContext);
if (tbl == null || fieldSchemas == null) {
throw new UnknownTableException(tableName + " doesn't exist");
@@ -4124,184 +2749,13 @@ private List<FieldSchema> get_schema_with_environment_context_core(String db, St
} catch (Exception e) {
ex = e;
throw handleException(e)
- .throwIfInstance(UnknownDBException.class, UnknownTableException.class, MetaException.class)
- .defaultMetaException();
+ .throwIfInstance(UnknownDBException.class, UnknownTableException.class, MetaException.class)
+ .defaultMetaException();
} finally {
endFunction("get_schema_with_environment_context_core", success, ex, tableName);
}
}
- @Override
- public GetSchemaResponse get_schema_req(GetSchemaRequest req)
- throws MetaException, UnknownTableException, UnknownDBException {
- String dbName = MetaStoreUtils.prependCatalogToDbName(req.getCatName(), req.getDbName(), conf);
- List<FieldSchema> fields = get_schema_with_environment_context_core(
- dbName, req.getTblName(), req.getEnvContext());
- GetSchemaResponse res = new GetSchemaResponse();
- res.setFields(fields);
- return res;
- }
-
- @Override
- public String getCpuProfile(int profileDurationInSec) throws TException {
- return "";
- }
-
- /**
- * Returns the value of the given configuration variable name. If the
- * configuration variable with the given name doesn't exist, or if there
- * were an exception thrown while retrieving the variable, or if name is
- * null, defaultValue is returned.
- */
- @Override
- public String get_config_value(String name, String defaultValue)
- throws TException {
- startFunction("get_config_value", ": name=" + name + " defaultValue="
- + defaultValue);
- boolean success = false;
- Exception ex = null;
- try {
- if (name == null) {
- success = true;
- return defaultValue;
- }
- // Allow only keys that start with hive.*, hdfs.*, mapred.* for security
- // i.e. don't allow access to db password
- if (!Pattern.matches("(hive|hdfs|mapred|metastore).*", name)) {
- throw new ConfigValSecurityException("For security reasons, the "
- + "config key " + name + " cannot be accessed");
- }
-
- String toReturn = defaultValue;
- try {
- toReturn = MetastoreConf.get(conf, name);
- if (toReturn == null) {
- toReturn = defaultValue;
- }
- } catch (RuntimeException e) {
- LOG.error("RuntimeException thrown in get_config_value - msg: "
- + e.getMessage() + " cause: " + e.getCause());
- }
- success = true;
- return toReturn;
- } catch (Exception e) {
- ex = e;
- throw handleException(e).defaultTException();
- } finally {
- endFunction("get_config_value", success, ex);
- }
- }
-
- public static List<String> getPartValsFromName(Table t, String partName)
- throws MetaException, InvalidObjectException {
- Preconditions.checkArgument(t != null, "Table can not be null");
- // Unescape the partition name
- LinkedHashMap<String, String> hm = Warehouse.makeSpecFromName(partName);
-
- List<String> partVals = new ArrayList<>();
- for (FieldSchema field : t.getPartitionKeys()) {
- String key = field.getName();
- String val = hm.get(key);
- if (val == null) {
- throw new InvalidObjectException("incomplete partition name - missing " + key);
- }
- partVals.add(val);
- }
- return partVals;
- }
-
- @Override
- @Deprecated
- public Partition get_partition_by_name(final String db_name, final String tbl_name,
- final String part_name) throws TException {
- if (StringUtils.isBlank(part_name)) {
- throw new MetaException("The part_name in get_partition_by_name cannot be null or empty");
- }
- String[] parsedDbName = parseDbName(db_name, conf);
- TableName tableName = new TableName(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name);
- GetPartitionsByNamesRequest gpnr = new GetPartitionsByNamesRequest(tableName.getDb(), tableName.getTable());
- gpnr.setNames(List.of(part_name));
- List<Partition> partitions = GetPartitionsHandler.getPartitions(
- t -> startFunction("get_partition_by_name", ": tbl=" + t + " part=" + part_name),
- rex -> endFunction("get_partition_by_name",
- rex.getLeft() != null && rex.getLeft().success(), rex.getRight(), tableName.toString()),
- this, tableName, gpnr, true);
- return partitions.getFirst();
- }
-
- @Deprecated
- @Override
- public Partition append_partition_by_name(final String db_name, final String tbl_name,
- final String part_name) throws TException {
- return append_partition_by_name_with_environment_context(db_name, tbl_name, part_name, null);
- }
-
- @Deprecated
- @Override
- public Partition append_partition_by_name_with_environment_context(final String db_name,
- final String tbl_name, final String part_name, final EnvironmentContext env_context)
- throws TException {
- String[] parsedDbName = parseDbName(db_name, conf);
- AppendPartitionsRequest appendPartitionRequest = new AppendPartitionsRequest();
- appendPartitionRequest.setDbName(parsedDbName[DB_NAME]);
- appendPartitionRequest.setTableName(tbl_name);
- appendPartitionRequest.setName(part_name);
- appendPartitionRequest.setCatalogName(parsedDbName[CAT_NAME]);
- appendPartitionRequest.setEnvironmentContext(env_context);
- return append_partition_req(appendPartitionRequest);
- }
-
- @Deprecated
- @Override
- public boolean drop_partition_by_name(final String db_name, final String tbl_name,
- final String part_name, final boolean deleteData) throws TException {
- return drop_partition_by_name_with_environment_context(db_name, tbl_name, part_name,
- deleteData, null);
- }
-
- @Deprecated
- @Override
- public boolean drop_partition_by_name_with_environment_context(final String db_name,
- final String tbl_name, final String part_name, final boolean deleteData,
- final EnvironmentContext envContext) throws TException {
- String[] parsedDbName = parseDbName(db_name, conf);
- DropPartitionRequest dropPartitionReq = new DropPartitionRequest(parsedDbName[DB_NAME], tbl_name);
- dropPartitionReq.setCatName(parsedDbName[CAT_NAME]);
- dropPartitionReq.setPartName(part_name);
- dropPartitionReq.setDeleteData(deleteData);
- dropPartitionReq.setEnvironmentContext(envContext);
- return drop_partition_req(dropPartitionReq);
- }
-
- @Override
- @Deprecated
- public List<Partition> get_partitions_ps(final String db_name,
- final String tbl_name, final List<String> part_vals,
- final short max_parts) throws TException {
- return get_partitions_ps_with_auth(db_name, tbl_name, part_vals, max_parts, null, null);
- }
-
- /**
- * Use {@link #get_partitions_ps_with_auth_req(GetPartitionsPsWithAuthRequest)} ()} instead.
- *
- */
- @Override
- @Deprecated
- public List<Partition> get_partitions_ps_with_auth(final String db_name,
- final String tbl_name, final List<String> part_vals,
- final short max_parts, final String userName,
- final List<String> groupNames) throws TException {
- String[] parsedDbName = parseDbName(db_name, conf);
- TableName tableName = new TableName(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name);
- GetPartitionsPsWithAuthRequest getAuthReq = new GetPartitionsPsWithAuthRequest(tableName.getDb(), tableName.getTable());
- getAuthReq.setCatName(tableName.getCat());
- getAuthReq.setMaxParts(max_parts);
- getAuthReq.setUserName(userName);
- getAuthReq.setGroupNames(groupNames);
- getAuthReq.setPartVals(part_vals);
- return get_partitions_ps_with_auth_req(getAuthReq).getPartitions();
- }
-
@Override
public GetPartitionsPsWithAuthResponse get_partitions_ps_with_auth_req(GetPartitionsPsWithAuthRequest req)
throws MetaException, NoSuchObjectException, TException {
@@ -4315,36 +2769,6 @@ public GetPartitionsPsWithAuthResponse get_partitions_ps_with_auth_req(GetPartit
return new GetPartitionsPsWithAuthResponse(partitions);
}
- /**
- * Use {@link #get_partition_names_ps_req(GetPartitionNamesPsRequest)} ()} instead.
- *
- */
- @Override
- @Deprecated
- public List<String> get_partition_names_ps(final String db_name,
- final String tbl_name, final List<String> part_vals, final short max_parts)
- throws TException {
- String[] parsedDbName = parseDbName(db_name, conf);
- startPartitionFunction("get_partitions_names_ps", parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name,
- max_parts, part_vals);
-
- List<String> ret = null;
- Exception ex = null;
- try {
- GetPartitionNamesPsRequest getPartitionNamesPsRequest = new GetPartitionNamesPsRequest(parsedDbName[DB_NAME], tbl_name);
- getPartitionNamesPsRequest.setCatName(parsedDbName[CAT_NAME]);
- getPartitionNamesPsRequest.setPartValues(part_vals);
- getPartitionNamesPsRequest.setMaxParts(max_parts);
- ret = get_partition_names_ps_req(getPartitionNamesPsRequest).getNames();
- } catch (Exception e) {
- ex = e;
- rethrowException(e);
- } finally {
- endFunction("get_partitions_names_ps", ret != null, ex, tbl_name);
- }
- return ret;
- }
-
@Override
public GetPartitionNamesPsResponse get_partition_names_ps_req(GetPartitionNamesPsRequest req)
throws MetaException, NoSuchObjectException, TException {
@@ -4380,49 +2804,6 @@ public List<String> get_partition_names_req(PartitionsByExprRequest req)
this, tableName, req).result();
}
- @Override
- public List<String> partition_name_to_vals(String part_name) throws TException {
- if (part_name.length() == 0) {
- return Collections.emptyList();
- }
- LinkedHashMap<String, String> map = Warehouse.makeSpecFromName(part_name);
- return new ArrayList<>(map.values());
- }
-
- @Override
- public Map<String, String> partition_name_to_spec(String part_name) throws TException {
- if (part_name.length() == 0) {
- return new HashMap<>();
- }
- return Warehouse.makeSpecFromName(part_name);
- }
-
- @Override
- @Deprecated
- public ColumnStatistics get_table_column_statistics(String dbName, String tableName,
- String colName) throws TException {
- String[] parsedDbName = parseDbName(dbName, conf);
- parsedDbName[CAT_NAME] = parsedDbName[CAT_NAME].toLowerCase();
- parsedDbName[DB_NAME] = parsedDbName[DB_NAME].toLowerCase();
- tableName = tableName.toLowerCase();
- colName = colName.toLowerCase();
- startFunction("get_column_statistics_by_table", ": table=" +
- TableName.getQualified(parsedDbName[CAT_NAME], parsedDbName[DB_NAME],
- tableName) + " column=" + colName);
- ColumnStatistics statsObj = null;
- try {
- statsObj = getMS().getTableColumnStatistics(
- parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableName, Lists.newArrayList(colName),
- "hive", null);
- if (statsObj != null) {
- assert statsObj.getStatsObjSize() <= 1;
- }
- return statsObj;
- } finally {
- endFunction("get_column_statistics_by_table", statsObj != null, null, tableName);
- }
- }
-
@Override
public TableStatsResult get_table_statistics_req(TableStatsRequest request) throws TException {
String catName = request.isSetCatName() ? request.getCatName().toLowerCase() :
@@ -4454,39 +2835,6 @@ public TableStatsResult get_table_statistics_req(TableStatsRequest request) thro
return result;
}
- @Override
- @Deprecated
- public ColumnStatistics get_partition_column_statistics(String dbName, String tableName,
- String partName, String colName) throws TException {
- // Note: this method appears to be unused within Hive.
- // It doesn't take txn stats into account.
- dbName = dbName.toLowerCase();
- String[] parsedDbName = parseDbName(dbName, conf);
- tableName = tableName.toLowerCase();
- colName = colName.toLowerCase();
- startFunction("get_column_statistics_by_partition", ": table=" +
- TableName.getQualified(parsedDbName[CAT_NAME], parsedDbName[DB_NAME],
- tableName) + " partition=" + partName + " column=" + colName);
- ColumnStatistics statsObj = null;
-
- try {
- List<ColumnStatistics> list = getMS().getPartitionColumnStatistics(
- parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableName,
- Lists.newArrayList(partName), Lists.newArrayList(colName),
- "hive");
- if (list.isEmpty()) {
- return null;
- }
- if (list.size() != 1) {
- throw new MetaException(list.size() + " statistics for single column and partition");
- }
- statsObj = list.get(0);
- } finally {
- endFunction("get_column_statistics_by_partition", statsObj != null, null, tableName);
- }
- return statsObj;
- }
-
@Override
public PartitionsStatsResult get_partitions_statistics_req(PartitionsStatsRequest request)
throws TException {
@@ -4589,27 +2937,6 @@ public SetPartitionsStatsResponse update_partition_column_statistics_req(
return new SetPartitionsStatsResponse(ret);
}
- @Deprecated
- @Override
- public boolean delete_partition_column_statistics(String dbName, String tableName,
- String partName, String colName, String engine) throws TException {
- dbName = dbName.toLowerCase();
- String[] parsedDbName = parseDbName(dbName, conf);
- tableName = tableName.toLowerCase();
- DeleteColumnStatisticsRequest request = new DeleteColumnStatisticsRequest(parsedDbName[DB_NAME], tableName);
- if (colName != null) {
- colName = colName.toLowerCase();
- request.addToCol_names(colName);
- }
-
- request.setEngine(engine);
- request.setCat_name(parsedDbName[CAT_NAME]);
- if (partName != null) {
- request.addToPart_names(partName);
- }
- return delete_column_statistics_req(request);
- }
-
@Override
public boolean delete_column_statistics_req(DeleteColumnStatisticsRequest req) throws TException {
String dbName = normalizeIdentifier(req.getDb_name());
@@ -4692,42 +3019,6 @@ public boolean delete_column_statistics_req(DeleteColumnStatisticsRequest req) t
return ret;
}
- @Deprecated
- @Override
- public boolean delete_table_column_statistics(String dbName, String tableName, String colName, String engine)
- throws TException {
- dbName = dbName.toLowerCase();
- tableName = tableName.toLowerCase();
- String[] parsedDbName = parseDbName(dbName, conf);
- DeleteColumnStatisticsRequest request = new DeleteColumnStatisticsRequest(parsedDbName[DB_NAME], tableName);
- if (colName != null) {
- colName = colName.toLowerCase();
- request.addToCol_names(colName);
- }
- request.setEngine(engine);
- request.setCat_name(parsedDbName[CAT_NAME]);
- request.setTableLevel(true);
- return delete_column_statistics_req(request);
- }
-
- @Override
- public void update_transaction_statistics(UpdateTransactionalStatsRequest req) throws TException {
- getTxnHandler().updateTransactionStatistics(req);
- }
-
- @Override
- @Deprecated
- public List<Partition> get_partitions_by_filter(final String dbName, final String tblName,
- final String filter, final short maxParts)
- throws TException {
- String[] parsedDbName = parseDbName(dbName, conf);
- GetPartitionsByFilterRequest gfr =
- new GetPartitionsByFilterRequest(parsedDbName[DB_NAME], tblName, filter);
- gfr.setMaxParts(maxParts);
- gfr.setCatName(parsedDbName[CAT_NAME]);
- return get_partitions_by_filter_req(gfr);
- }
-
@Override
public List<Partition> get_partitions_by_filter_req(GetPartitionsByFilterRequest req) throws TException {
String catName = req.isSetCatName() ? req.getCatName() : getDefaultCatalog(conf);
@@ -4739,44 +3030,6 @@ public List<Partition> get_partitions_by_filter_req(GetPartitionsByFilterRequest
this, tableName, req).result();
}
- @Override
- @Deprecated
- public List<PartitionSpec> get_part_specs_by_filter(final String dbName, final String tblName,
- final String filter, final int maxParts)
- throws TException {
-
- String[] parsedDbName = parseDbName(dbName, conf);
- startPartitionFunction("get_partitions_by_filter_pspec", parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tblName,
- maxParts, filter);
- List<PartitionSpec> partitionSpecs = null;
- try {
- GetTableRequest getTableRequest = new GetTableRequest(parsedDbName[DB_NAME], tblName);
- getTableRequest.setCatName(parsedDbName[CAT_NAME]);
- Table table = get_table_core(getTableRequest);
- // Don't pass the parsed db name, as get_partitions_by_filter will parse it itself
- List<Partition> partitions = get_partitions_by_filter(dbName, tblName, filter, (short) maxParts);
-
- if (is_partition_spec_grouping_enabled(table)) {
- partitionSpecs = MetaStoreServerUtils
- .getPartitionspecsGroupedByStorageDescriptor(table, partitions);
- }
- else {
- PartitionSpec pSpec = new PartitionSpec();
- pSpec.setPartitionList(new PartitionListComposingSpec(partitions));
- pSpec.setRootPath(table.getSd().getLocation());
- pSpec.setCatName(parsedDbName[CAT_NAME]);
- pSpec.setDbName(parsedDbName[DB_NAME]);
- pSpec.setTableName(tblName);
- partitionSpecs = Arrays.asList(pSpec);
- }
-
- return partitionSpecs;
- }
- finally {
- endFunction("get_partitions_by_filter_pspec", partitionSpecs != null && !partitionSpecs.isEmpty(), null, tblName);
- }
- }
-
@Override
public PartitionsSpecByExprResult get_partitions_spec_by_expr(
PartitionsByExprRequest req) throws TException {
@@ -4814,46 +3067,6 @@ public PartitionsByExprResult get_partitions_by_expr(
return new PartitionsByExprResult(result.result(), result.hasUnknownPartitions());
}
- @Override
- @Deprecated
- public int get_num_partitions_by_filter(final String dbName,
- final String tblName, final String filter)
- throws TException {
- String[] parsedDbName = parseDbName(dbName, conf);
- if (parsedDbName[DB_NAME] == null || tblName == null) {
- throw new MetaException("The DB and table name cannot be null.");
- }
- startFunction("get_num_partitions_by_filter",
- " : tbl=" + TableName.getQualified(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tblName) + " Filter="
- + filter);
-
- int ret = -1;
- Exception ex = null;
- try {
- ret = getMS().getNumPartitionsByFilter(parsedDbName[CAT_NAME], parsedDbName[DB_NAME],
- tblName, filter);
- } catch (Exception e) {
- ex = e;
- rethrowException(e);
- } finally {
- endFunction("get_num_partitions_by_filter", ret != -1, ex, tblName);
- }
- return ret;
- }
-
- @Override
- @Deprecated
- public List<Partition> get_partitions_by_names(final String dbName, final String tblName,
- final List<String> partNames)
- throws TException {
- if (partNames == null) {
- throw new MetaException("The partNames is null");
- }
- GetPartitionsByNamesRequest request = new GetPartitionsByNamesRequest(dbName, tblName);
- request.setNames(partNames);
- return get_partitions_by_names_req(request).getPartitions();
- }
-
@Override
public GetPartitionsByNamesResult get_partitions_by_names_req(GetPartitionsByNamesRequest gpbnr)
throws TException {
@@ -4911,587 +3124,77 @@ public boolean set_properties(PropertySetRequest req) throws TException {
}
@Override
- public PrincipalPrivilegeSet get_privilege_set(HiveObjectRef hiveObject, String userName,
- List<String> groupNames) throws TException {
- firePreEvent(new PreAuthorizationCallEvent(this));
- String catName = hiveObject.isSetCatName() ? hiveObject.getCatName() : getDefaultCatalog(conf);
- HiveObjectType debug = hiveObject.getObjectType();
- if (hiveObject.getObjectType() == HiveObjectType.COLUMN) {
- String partName = getPartName(hiveObject);
- return this.get_column_privilege_set(catName, hiveObject.getDbName(), hiveObject
- .getObjectName(), partName, hiveObject.getColumnName(), userName,
- groupNames);
- } else if (hiveObject.getObjectType() == HiveObjectType.PARTITION) {
- String partName = getPartName(hiveObject);
- return this.get_partition_privilege_set(catName, hiveObject.getDbName(),
- hiveObject.getObjectName(), partName, userName, groupNames);
- } else if (hiveObject.getObjectType() == HiveObjectType.DATABASE) {
- return this.get_db_privilege_set(catName, hiveObject.getDbName(), userName,
- groupNames);
- } else if (hiveObject.getObjectType() == HiveObjectType.DATACONNECTOR) {
- return this.get_connector_privilege_set(catName, hiveObject.getObjectName(), userName, groupNames);
- } else if (hiveObject.getObjectType() == HiveObjectType.TABLE) {
- return this.get_table_privilege_set(catName, hiveObject.getDbName(), hiveObject
- .getObjectName(), userName, groupNames);
- } else if (hiveObject.getObjectType() == HiveObjectType.GLOBAL) {
- return this.get_user_privilege_set(userName, groupNames);
- }
- return null;
- }
+ public void markPartitionForEvent(final String db_name, final String tbl_name,
+ final Map<String, String> partName, final PartitionEventType evtType) throws TException {
- private String getPartName(HiveObjectRef hiveObject) throws MetaException {
- String partName = null;
- List<String> partValue = hiveObject.getPartValues();
- if (partValue != null && partValue.size() > 0) {
- try {
- String catName = hiveObject.isSetCatName() ? hiveObject.getCatName() :
- getDefaultCatalog(conf);
- GetTableRequest getTableRequest = new GetTableRequest(hiveObject.getDbName(), hiveObject.getObjectName());
- getTableRequest.setCatName(catName);
- Table table = get_table_core(getTableRequest);
- partName = Warehouse
- .makePartName(table.getPartitionKeys(), partValue);
- } catch (NoSuchObjectException e) {
- throw new MetaException(e.getMessage());
+ Table tbl = null;
+ Exception ex = null;
+ RawStore ms = getMS();
+ boolean success = false;
+ try {
+ String[] parsedDbName = parseDbName(db_name, conf);
+ ms.openTransaction();
+ startPartitionFunction("markPartitionForEvent", parsedDbName[CAT_NAME], parsedDbName[DB_NAME],
+ tbl_name, partName);
+ firePreEvent(new PreLoadPartitionDoneEvent(parsedDbName[CAT_NAME], parsedDbName[DB_NAME],
+ tbl_name, partName, this));
+ tbl = ms.markPartitionForEvent(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name,
+ partName, evtType);
+ if (null == tbl) {
+ throw new UnknownTableException("Table: " + tbl_name + " not found.");
}
- }
- return partName;
- }
- private PrincipalPrivilegeSet get_column_privilege_set(String catName, final String dbName,
- final String tableName, final String partName, final String columnName,
- final String userName, final List<String> groupNames) throws TException {
- incrementCounter("get_column_privilege_set");
+ if (transactionalListeners.size() > 0) {
+ LoadPartitionDoneEvent lpde = new LoadPartitionDoneEvent(true, tbl, partName, this);
+ for (MetaStoreEventListener transactionalListener : transactionalListeners) {
+ transactionalListener.onLoadPartitionDone(lpde);
+ }
+ }
- PrincipalPrivilegeSet ret;
- try {
- ret = getMS().getColumnPrivilegeSet(
- catName, dbName, tableName, partName, columnName, userName, groupNames);
- } catch (Exception e) {
- throw handleException(e).throwIfInstance(MetaException.class).defaultRuntimeException();
- }
- return ret;
- }
-
- private PrincipalPrivilegeSet get_db_privilege_set(String catName, final String dbName,
- final String userName, final List<String> groupNames) throws TException {
- incrementCounter("get_db_privilege_set");
-
- PrincipalPrivilegeSet ret;
- try {
- ret = getMS().getDBPrivilegeSet(catName, dbName, userName, groupNames);
- } catch (Exception e) {
- throw handleException(e).throwIfInstance(MetaException.class).defaultRuntimeException();
- }
- return ret;
- }
-
- private PrincipalPrivilegeSet get_connector_privilege_set(String catName, final String connectorName,
- final String userName, final List<String> groupNames) throws TException {
- incrementCounter("get_connector_privilege_set");
-
- PrincipalPrivilegeSet ret;
- try {
- ret = getMS().getConnectorPrivilegeSet(catName, connectorName, userName, groupNames);
- } catch (MetaException e) {
- throw e;
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- return ret;
-
- }
-
- private PrincipalPrivilegeSet get_partition_privilege_set(
- String catName, final String dbName, final String tableName, final String partName,
- final String userName, final List<String> groupNames)
- throws TException {
- incrementCounter("get_partition_privilege_set");
-
- PrincipalPrivilegeSet ret;
- try {
- ret = getMS().getPartitionPrivilegeSet(catName, dbName, tableName, partName,
- userName, groupNames);
- } catch (Exception e) {
- throw handleException(e).throwIfInstance(MetaException.class).defaultRuntimeException();
- }
- return ret;
- }
-
- private PrincipalPrivilegeSet get_table_privilege_set(String catName, final String dbName,
- final String tableName, final String userName,
- final List<String> groupNames) throws TException {
- incrementCounter("get_table_privilege_set");
-
- PrincipalPrivilegeSet ret;
- try {
- ret = getMS().getTablePrivilegeSet(catName, dbName, tableName, userName,
- groupNames);
- } catch (Exception e) {
- throw handleException(e).throwIfInstance(MetaException.class).defaultRuntimeException();
- }
- return ret;
- }
-
- @Override
- public boolean grant_role(final String roleName,
- final String principalName, final PrincipalType principalType,
- final String grantor, final PrincipalType grantorType, final boolean grantOption)
- throws TException {
- incrementCounter("add_role_member");
- firePreEvent(new PreAuthorizationCallEvent(this));
- if (PUBLIC.equals(roleName)) {
- throw new MetaException("No user can be added to " + PUBLIC +". Since all users implicitly"
- + " belong to " + PUBLIC + " role.");
- }
- Boolean ret;
- try {
- RawStore ms = getMS();
- Role role = ms.getRole(roleName);
- if(principalType == PrincipalType.ROLE){
- //check if this grant statement will end up creating a cycle
- if(isNewRoleAParent(principalName, roleName)){
- throw new MetaException("Cannot grant role " + principalName + " to " + roleName +
- " as " + roleName + " already belongs to the role " + principalName +
- ". (no cycles allowed)");
- }
- }
- ret = ms.grantRole(role, principalName, principalType, grantor, grantorType, grantOption);
- } catch (Exception e) {
- String exInfo = "Got exception: " + e.getClass().getName() + " " + e.getMessage();
- LOG.error(exInfo, e);
- throw handleException(e).throwIfInstance(MetaException.class)
- .toMetaExceptionIfInstance(exInfo, InvalidObjectException.class, NoSuchObjectException.class)
- .defaultTException();
- }
- return ret;
- }
-
-
-
- /**
- * Check if newRole is in parent hierarchy of curRole
- * @param newRole
- * @param curRole
- * @return true if newRole is curRole or present in its hierarchy
- * @throws MetaException
- */
- private boolean isNewRoleAParent(String newRole, String curRole) throws MetaException {
- if(newRole.equals(curRole)){
- return true;
- }
- //do this check recursively on all the parent roles of curRole
- List<Role> parentRoleMaps = getMS().listRoles(curRole, PrincipalType.ROLE);
- for(Role parentRole : parentRoleMaps){
- if(isNewRoleAParent(newRole, parentRole.getRoleName())){
- return true;
- }
- }
- return false;
- }
-
- @Override
- public List<Role> list_roles(final String principalName,
- final PrincipalType principalType) throws TException {
- incrementCounter("list_roles");
- firePreEvent(new PreAuthorizationCallEvent(this));
- return getMS().listRoles(principalName, principalType);
- }
-
- @Override
- public boolean create_role(final Role role) throws TException {
- incrementCounter("create_role");
- firePreEvent(new PreAuthorizationCallEvent(this));
- if (PUBLIC.equals(role.getRoleName())) {
- throw new MetaException(PUBLIC + " role implicitly exists. It can't be created.");
- }
- Boolean ret;
- try {
- ret = getMS().addRole(role.getRoleName(), role.getOwnerName());
- } catch (Exception e) {
- String exInfo = "Got exception: " + e.getClass().getName() + " " + e.getMessage();
- LOG.error(exInfo, e);
- throw handleException(e).throwIfInstance(MetaException.class)
- .toMetaExceptionIfInstance(exInfo, InvalidObjectException.class, NoSuchObjectException.class)
- .defaultTException();
- }
- return ret;
- }
-
- @Override
- public boolean drop_role(final String roleName) throws TException {
- incrementCounter("drop_role");
- firePreEvent(new PreAuthorizationCallEvent(this));
- if (ADMIN.equals(roleName) || PUBLIC.equals(roleName)) {
- throw new MetaException(PUBLIC + "," + ADMIN + " roles can't be dropped.");
- }
- Boolean ret;
- try {
- ret = getMS().removeRole(roleName);
- } catch (Exception e) {
- String exInfo = "Got exception: " + e.getClass().getName() + " " + e.getMessage();
- LOG.error(exInfo, e);
- throw handleException(e).throwIfInstance(MetaException.class)
- .toMetaExceptionIfInstance(exInfo, NoSuchObjectException.class)
- .defaultTException();
- }
- return ret;
- }
-
- @Override
- public List<String> get_role_names() throws TException {
- incrementCounter("get_role_names");
- firePreEvent(new PreAuthorizationCallEvent(this));
- List<String> ret;
- try {
- ret = getMS().listRoleNames();
- return ret;
- } catch (Exception e) {
- throw handleException(e).throwIfInstance(MetaException.class).defaultRuntimeException();
- }
- }
-
- @Override
- public boolean grant_privileges(final PrivilegeBag privileges) throws TException {
- incrementCounter("grant_privileges");
- firePreEvent(new PreAuthorizationCallEvent(this));
- Boolean ret;
- try {
- ret = getMS().grantPrivileges(privileges);
- } catch (Exception e) {
- String exInfo = "Got exception: " + e.getClass().getName() + " " + e.getMessage();
- LOG.error(exInfo, e);
- throw handleException(e).throwIfInstance(MetaException.class)
- .toMetaExceptionIfInstance(exInfo, InvalidObjectException.class, NoSuchObjectException.class)
- .defaultTException();
- }
- return ret;
- }
-
- @Override
- public boolean revoke_role(final String roleName, final String userName,
- final PrincipalType principalType) throws TException {
- return revoke_role(roleName, userName, principalType, false);
- }
-
- private boolean revoke_role(final String roleName, final String userName,
- final PrincipalType principalType, boolean grantOption) throws TException {
- incrementCounter("remove_role_member");
- firePreEvent(new PreAuthorizationCallEvent(this));
- if (PUBLIC.equals(roleName)) {
- throw new MetaException(PUBLIC + " role can't be revoked.");
- }
- Boolean ret;
- try {
- RawStore ms = getMS();
- Role mRole = ms.getRole(roleName);
- ret = ms.revokeRole(mRole, userName, principalType, grantOption);
- } catch (Exception e) {
- String exInfo = "Got exception: " + e.getClass().getName() + " " + e.getMessage();
- LOG.error(exInfo, e);
- throw handleException(e).throwIfInstance(MetaException.class)
- .toMetaExceptionIfInstance(exInfo, NoSuchObjectException.class)
- .defaultTException();
- }
- return ret;
- }
-
- @Override
- public GrantRevokeRoleResponse grant_revoke_role(GrantRevokeRoleRequest request)
- throws TException {
- GrantRevokeRoleResponse response = new GrantRevokeRoleResponse();
- boolean grantOption = false;
- if (request.isSetGrantOption()) {
- grantOption = request.isGrantOption();
- }
- switch (request.getRequestType()) {
- case GRANT: {
- boolean result = grant_role(request.getRoleName(),
- request.getPrincipalName(), request.getPrincipalType(),
- request.getGrantor(), request.getGrantorType(), grantOption);
- response.setSuccess(result);
- break;
- }
- case REVOKE: {
- boolean result = revoke_role(request.getRoleName(), request.getPrincipalName(),
- request.getPrincipalType(), grantOption);
- response.setSuccess(result);
- break;
+ success = ms.commitTransaction();
+ for (MetaStoreEventListener listener : listeners) {
+ listener.onLoadPartitionDone(new LoadPartitionDoneEvent(true, tbl, partName, this));
}
- default:
- throw new MetaException("Unknown request type " + request.getRequestType());
- }
-
- return response;
- }
-
- @Override
- public GrantRevokePrivilegeResponse grant_revoke_privileges(GrantRevokePrivilegeRequest request)
- throws TException {
- GrantRevokePrivilegeResponse response = new GrantRevokePrivilegeResponse();
- switch (request.getRequestType()) {
- case GRANT: {
- boolean result = grant_privileges(request.getPrivileges());
- response.setSuccess(result);
- break;
- }
- case REVOKE: {
- boolean revokeGrantOption = false;
- if (request.isSetRevokeGrantOption()) {
- revokeGrantOption = request.isRevokeGrantOption();
- }
- boolean result = revoke_privileges(request.getPrivileges(), revokeGrantOption);
- response.setSuccess(result);
- break;
+ } catch (Exception original) {
+ ex = original;
+ LOG.error("Exception caught in mark partition event ", original);
+ throw handleException(original)
+ .throwIfInstance(UnknownTableException.class, InvalidPartitionException.class, MetaException.class)
+ .defaultMetaException();
+ } finally {
+ if (!success) {
+ ms.rollbackTransaction();
}
- default:
- throw new MetaException("Unknown request type " + request.getRequestType());
- }
-
- return response;
- }
- @Override
- public GrantRevokePrivilegeResponse refresh_privileges(HiveObjectRef objToRefresh, String authorizer,
- GrantRevokePrivilegeRequest grantRequest)
- throws TException {
- incrementCounter("refresh_privileges");
- firePreEvent(new PreAuthorizationCallEvent(this));
- GrantRevokePrivilegeResponse response = new GrantRevokePrivilegeResponse();
- try {
- boolean result = getMS().refreshPrivileges(objToRefresh, authorizer, grantRequest.getPrivileges());
- response.setSuccess(result);
- } catch (Exception e) {
- throw handleException(e).throwIfInstance(MetaException.class).defaultRuntimeException();
+ endFunction("markPartitionForEvent", tbl != null, ex, tbl_name);
}
- return response;
}
@Override
- public boolean revoke_privileges(final PrivilegeBag privileges) throws TException {
- return revoke_privileges(privileges, false);
- }
+ public boolean isPartitionMarkedForEvent(final String db_name, final String tbl_name,
+ final Map<String, String> partName, final PartitionEventType evtType) throws TException {
- public boolean revoke_privileges(final PrivilegeBag privileges, boolean grantOption)
- throws TException {
- incrementCounter("revoke_privileges");
- firePreEvent(new PreAuthorizationCallEvent(this));
- Boolean ret;
+ String[] parsedDbName = parseDbName(db_name, conf);
+ startPartitionFunction("isPartitionMarkedForEvent", parsedDbName[CAT_NAME], parsedDbName[DB_NAME],
+ tbl_name, partName);
+ Boolean ret = null;
+ Exception ex = null;
try {
- ret = getMS().revokePrivileges(privileges, grantOption);
- } catch (Exception e) {
- String exInfo = "Got exception: " + e.getClass().getName() + " " + e.getMessage();
- LOG.error(exInfo, e);
- throw handleException(e).throwIfInstance(MetaException.class)
- .toMetaExceptionIfInstance(exInfo, InvalidObjectException.class, NoSuchObjectException.class)
- .defaultTException();
+ ret = getMS().isPartitionMarkedForEvent(parsedDbName[CAT_NAME], parsedDbName[DB_NAME],
+ tbl_name, partName, evtType);
+ } catch (Exception original) {
+ LOG.error("Exception caught for isPartitionMarkedForEvent ", original);
+ ex = original;
+ throw handleException(original).throwIfInstance(UnknownTableException.class, InvalidPartitionException.class)
+ .throwIfInstance(UnknownPartitionException.class, MetaException.class)
+ .defaultMetaException();
+ } finally {
+ endFunction("isPartitionMarkedForEvent", ret != null, ex, tbl_name);
}
- return ret;
- }
- private PrincipalPrivilegeSet get_user_privilege_set(final String userName,
- final List<String> groupNames) throws TException {
- incrementCounter("get_user_privilege_set");
- PrincipalPrivilegeSet ret;
- try {
- ret = getMS().getUserPrivilegeSet(userName, groupNames);
- } catch (Exception e) {
- throw handleException(e).throwIfInstance(MetaException.class).defaultRuntimeException();
- }
return ret;
}
- @Override
- public List<HiveObjectPrivilege> list_privileges(String principalName,
- PrincipalType principalType, HiveObjectRef hiveObject)
- throws TException {
- firePreEvent(new PreAuthorizationCallEvent(this));
- String catName = hiveObject.isSetCatName() ? hiveObject.getCatName() : getDefaultCatalog(conf);
- if (hiveObject.getObjectType() == null) {
- return getAllPrivileges(principalName, principalType, catName);
- }
- if (hiveObject.getObjectType() == HiveObjectType.GLOBAL) {
- return list_global_privileges(principalName, principalType);
- }
- if (hiveObject.getObjectType() == HiveObjectType.DATABASE) {
- return list_db_privileges(principalName, principalType, catName, hiveObject
- .getDbName());
- }
- if (hiveObject.getObjectType() == HiveObjectType.DATACONNECTOR) {
- return list_dc_privileges(principalName, principalType, hiveObject
- .getObjectName());
- }
- if (hiveObject.getObjectType() == HiveObjectType.TABLE) {
- return list_table_privileges(principalName, principalType,
- catName, hiveObject.getDbName(), hiveObject.getObjectName());
- }
- if (hiveObject.getObjectType() == HiveObjectType.PARTITION) {
- return list_partition_privileges(principalName, principalType,
- catName, hiveObject.getDbName(), hiveObject.getObjectName(), hiveObject
- .getPartValues());
- }
- if (hiveObject.getObjectType() == HiveObjectType.COLUMN) {
- if (hiveObject.getPartValues() == null || hiveObject.getPartValues().isEmpty()) {
- return list_table_column_privileges(principalName, principalType,
- catName, hiveObject.getDbName(), hiveObject.getObjectName(), hiveObject.getColumnName());
- }
- return list_partition_column_privileges(principalName, principalType,
- catName, hiveObject.getDbName(), hiveObject.getObjectName(), hiveObject
- .getPartValues(), hiveObject.getColumnName());
- }
- return null;
- }
-
- private List<HiveObjectPrivilege> getAllPrivileges(String principalName,
- PrincipalType principalType, String catName) throws TException {
- List<HiveObjectPrivilege> privs = new ArrayList<>();
- privs.addAll(list_global_privileges(principalName, principalType));
- privs.addAll(list_db_privileges(principalName, principalType, catName, null));
- privs.addAll(list_dc_privileges(principalName, principalType, null));
- privs.addAll(list_table_privileges(principalName, principalType, catName, null, null));
- privs.addAll(list_partition_privileges(principalName, principalType, catName, null, null, null));
- privs.addAll(list_table_column_privileges(principalName, principalType, catName, null, null, null));
- privs.addAll(list_partition_column_privileges(principalName, principalType,
- catName, null, null, null, null));
- return privs;
- }
-
- private List<HiveObjectPrivilege> list_table_column_privileges(
- final String principalName, final PrincipalType principalType, String catName,
- final String dbName, final String tableName, final String columnName) throws TException {
- incrementCounter("list_table_column_privileges");
-
- try {
- if (dbName == null) {
- return getMS().listPrincipalTableColumnGrantsAll(principalName, principalType);
- }
- if (principalName == null) {
- return getMS().listTableColumnGrantsAll(catName, dbName, tableName, columnName);
- }
- return getMS().listPrincipalTableColumnGrants(principalName, principalType,
- catName, dbName, tableName, columnName);
- } catch (Exception e) {
- throw handleException(e).throwIfInstance(MetaException.class).defaultRuntimeException();
- }
- }
-
- private List<HiveObjectPrivilege> list_partition_column_privileges(
- final String principalName, final PrincipalType principalType,
- String catName, final String dbName, final String tableName, final List<String> partValues,
- final String columnName) throws TException {
- incrementCounter("list_partition_column_privileges");
-
- try {
- if (dbName == null) {
- return getMS().listPrincipalPartitionColumnGrantsAll(principalName, principalType);
- }
- GetTableRequest getTableRequest = new GetTableRequest(dbName, tableName);
- getTableRequest.setCatName(catName);
- Table tbl = get_table_core(getTableRequest);
- String partName = Warehouse.makePartName(tbl.getPartitionKeys(), partValues);
- if (principalName == null) {
- return getMS().listPartitionColumnGrantsAll(catName, dbName, tableName, partName, columnName);
- }
-
- return getMS().listPrincipalPartitionColumnGrants(principalName, principalType, catName, dbName,
- tableName, partValues, partName, columnName);
- } catch (Exception e) {
- throw handleException(e).throwIfInstance(MetaException.class).defaultRuntimeException();
- }
- }
-
- private List<HiveObjectPrivilege> list_db_privileges(final String principalName,
- final PrincipalType principalType, String catName, final String dbName) throws TException {
- incrementCounter("list_security_db_grant");
-
- try {
- if (dbName == null) {
- return getMS().listPrincipalDBGrantsAll(principalName, principalType);
- }
- if (principalName == null) {
- return getMS().listDBGrantsAll(catName, dbName);
- } else {
- return getMS().listPrincipalDBGrants(principalName, principalType, catName, dbName);
- }
- } catch (Exception e) {
- throw handleException(e).throwIfInstance(MetaException.class).defaultRuntimeException();
- }
- }
-
- private List<HiveObjectPrivilege> list_dc_privileges(final String principalName,
- final PrincipalType principalType, final String dcName) throws TException {
- incrementCounter("list_security_dc_grant");
-
- try {
- if (dcName == null) {
- return getMS().listPrincipalDCGrantsAll(principalName, principalType);
- }
- if (principalName == null) {
- return getMS().listDCGrantsAll(dcName);
- } else {
- return getMS().listPrincipalDCGrants(principalName, principalType, dcName);
- }
- } catch (Exception e) {
- throw handleException(e).throwIfInstance(MetaException.class).defaultRuntimeException();
- }
- }
-
- private List<HiveObjectPrivilege> list_partition_privileges(
- final String principalName, final PrincipalType principalType,
- String catName, final String dbName, final String tableName, final List<String> partValues)
- throws TException {
- incrementCounter("list_security_partition_grant");
-
- try {
- if (dbName == null) {
- return getMS().listPrincipalPartitionGrantsAll(principalName, principalType);
- }
- GetTableRequest getTableRequest = new GetTableRequest(dbName, tableName);
- getTableRequest.setCatName(catName);
- Table tbl = get_table_core(getTableRequest);
- String partName = Warehouse.makePartName(tbl.getPartitionKeys(), partValues);
- if (principalName == null) {
- return getMS().listPartitionGrantsAll(catName, dbName, tableName, partName);
- }
- return getMS().listPrincipalPartitionGrants(
- principalName, principalType, catName, dbName, tableName, partValues, partName);
- } catch (Exception e) {
- throw handleException(e).throwIfInstance(MetaException.class).defaultRuntimeException();
- }
- }
-
- private List<HiveObjectPrivilege> list_table_privileges(
- final String principalName, final PrincipalType principalType,
- String catName, final String dbName, final String tableName) throws TException {
- incrementCounter("list_security_table_grant");
-
- try {
- if (dbName == null) {
- return getMS().listPrincipalTableGrantsAll(principalName, principalType);
- }
- if (principalName == null) {
- return getMS().listTableGrantsAll(catName, dbName, tableName);
- }
- return getMS().listAllTableGrants(principalName, principalType, catName, dbName, tableName);
- } catch (Exception e) {
- throw handleException(e).throwIfInstance(MetaException.class).defaultRuntimeException();
- }
- }
-
- private List<HiveObjectPrivilege> list_global_privileges(
- final String principalName, final PrincipalType principalType) throws TException {
- incrementCounter("list_security_user_grant");
-
- try {
- if (principalName == null) {
- return getMS().listGlobalGrantsAll();
- }
- return getMS().listPrincipalGlobalGrants(principalName, principalType);
- } catch (Exception e) {
- throw handleException(e).throwIfInstance(MetaException.class).defaultRuntimeException();
- }
- }
-
@Override
public void cancel_delegation_token(String token_str_form) throws TException {
startFunction("cancel_delegation_token");
@@ -5672,106 +3375,6 @@ public List<String> get_master_keys() throws TException {
return Arrays.asList(ret);
}
- @Override
- public void markPartitionForEvent(final String db_name, final String tbl_name,
- final Map<String, String> partName, final PartitionEventType evtType) throws TException {
-
- Table tbl = null;
- Exception ex = null;
- RawStore ms = getMS();
- boolean success = false;
- try {
- String[] parsedDbName = parseDbName(db_name, conf);
- ms.openTransaction();
- startPartitionFunction("markPartitionForEvent", parsedDbName[CAT_NAME], parsedDbName[DB_NAME],
- tbl_name, partName);
- firePreEvent(new PreLoadPartitionDoneEvent(parsedDbName[CAT_NAME], parsedDbName[DB_NAME],
- tbl_name, partName, this));
- tbl = ms.markPartitionForEvent(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name,
- partName, evtType);
- if (null == tbl) {
- throw new UnknownTableException("Table: " + tbl_name + " not found.");
- }
-
- if (transactionalListeners.size() > 0) {
- LoadPartitionDoneEvent lpde = new LoadPartitionDoneEvent(true, tbl, partName, this);
- for (MetaStoreEventListener transactionalListener : transactionalListeners) {
- transactionalListener.onLoadPartitionDone(lpde);
- }
- }
-
- success = ms.commitTransaction();
- for (MetaStoreEventListener listener : listeners) {
- listener.onLoadPartitionDone(new LoadPartitionDoneEvent(true, tbl, partName, this));
- }
- } catch (Exception original) {
- ex = original;
- LOG.error("Exception caught in mark partition event ", original);
- throw handleException(original)
- .throwIfInstance(UnknownTableException.class, InvalidPartitionException.class, MetaException.class)
- .defaultMetaException();
- } finally {
- if (!success) {
- ms.rollbackTransaction();
- }
-
- endFunction("markPartitionForEvent", tbl != null, ex, tbl_name);
- }
- }
-
- @Override
- public boolean isPartitionMarkedForEvent(final String db_name, final String tbl_name,
- final Map<String, String> partName, final PartitionEventType evtType) throws TException {
-
- String[] parsedDbName = parseDbName(db_name, conf);
- startPartitionFunction("isPartitionMarkedForEvent", parsedDbName[CAT_NAME], parsedDbName[DB_NAME],
- tbl_name, partName);
- Boolean ret = null;
- Exception ex = null;
- try {
- ret = getMS().isPartitionMarkedForEvent(parsedDbName[CAT_NAME], parsedDbName[DB_NAME],
- tbl_name, partName, evtType);
- } catch (Exception original) {
- LOG.error("Exception caught for isPartitionMarkedForEvent ", original);
- ex = original;
- throw handleException(original).throwIfInstance(UnknownTableException.class, InvalidPartitionException.class)
- .throwIfInstance(UnknownPartitionException.class, MetaException.class)
- .defaultMetaException();
- } finally {
- endFunction("isPartitionMarkedForEvent", ret != null, ex, tbl_name);
- }
-
- return ret;
- }
-
- @Override
- public List<String> set_ugi(String username, List<String> groupNames) throws TException {
- Collections.addAll(groupNames, username);
- return groupNames;
- }
-
- @Override
- public boolean partition_name_has_valid_characters(
- List<String> part_vals, boolean throw_exception) throws TException {
- startFunction("partition_name_has_valid_characters");
- boolean ret;
- Exception ex = null;
- try {
- if (throw_exception) {
- MetaStoreServerUtils.validatePartitionNameCharacters(part_vals, getConf());
- ret = true;
- } else {
- ret = MetaStoreServerUtils.partitionNameHasValidCharacters(part_vals, getConf());
- }
- } catch (Exception e) {
- ex = e;
- throw newMetaException(e);
- } finally {
- endFunction("partition_name_has_valid_characters", true, ex);
- }
- return ret;
- }
-
private void validateFunctionInfo(Function func) throws InvalidObjectException, MetaException {
if (func == null) {
throw new MetaException("Function cannot be null.");
@@ -5935,563 +3538,122 @@ public void alter_function(String dbName, String funcName, Function newFunc) thr
} finally {
if (!success) {
ms.rollbackTransaction();
- }
- }
- }
-
- private void validateForAlterFunction(String dbName, String funcName, Function newFunc)
- throws MetaException {
- if (dbName == null || funcName == null) {
- throw new MetaException("Database and function name cannot be null.");
- }
- try {
- validateFunctionInfo(newFunc);
- } catch (InvalidObjectException e) {
- // The validateFunctionInfo method is used by the create and alter function methods as well
- // and it can throw InvalidObjectException. But the InvalidObjectException is not defined
- // for the alter_function method in the Thrift interface, therefore a TApplicationException
- // will occur at the caller side. Re-throwing the InvalidObjectException as MetaException
- // would eliminate the TApplicationException at caller side.
- throw newMetaException(e);
- }
- }
-
- @Override
- public List<String> get_functions(String dbName, String pattern)
- throws MetaException {
- String[] parsedDbName = parseDbName(dbName, conf);
- GetFunctionsRequest request = new GetFunctionsRequest(parsedDbName[DB_NAME]);
- request.setCatalogName(parsedDbName[CAT_NAME]);
- request.setPattern(pattern);
- request.setReturnNames(true);
- GetFunctionsResponse resp = get_functions_req(request);
- return resp.getFunction_names();
- }
-
- @Override
- public GetFunctionsResponse get_functions_req(GetFunctionsRequest req)
- throws MetaException {
- startFunction("get_functions_req", ": db=" + req.getDbName()
- + " pat=" + req.getPattern());
-
- RawStore ms = getMS();
- Exception ex = null;
- GetFunctionsResponse response = new GetFunctionsResponse();
- String catName = req.isSetCatalogName() ? req.getCatalogName() : getDefaultCatalog(conf);
- try {
- List result = ms.getFunctionsRequest(catName, req.getDbName(),
- req.getPattern(), req.isReturnNames());
- if (req.isReturnNames()) {
- response.setFunction_names(result);
- } else {
- response.setFunctions(result);
- }
- } catch (Exception e) {
- ex = e;
- throw newMetaException(e);
- } finally {
- endFunction("get_functions", ex == null, ex);
- }
- return response;
- }
-
- @Override
- public GetAllFunctionsResponse get_all_functions()
- throws MetaException {
- GetAllFunctionsResponse response = new GetAllFunctionsResponse();
- startFunction("get_all_functions");
- RawStore ms = getMS();
- List<Function> allFunctions = null;
- Exception ex = null;
- try {
- // Leaving this as the 'hive' catalog (rather than choosing the default from the
- // configuration) because all the default UDFs are in that catalog, and I think that's
- // would people really want here.
- allFunctions = ms.getAllFunctions(DEFAULT_CATALOG_NAME);
- } catch (Exception e) {
- ex = e;
- throw newMetaException(e);
- } finally {
- endFunction("get_all_functions", allFunctions != null, ex);
- }
- response.setFunctions(allFunctions);
- return response;
- }
-
- @Override
- public Function get_function(String dbName, String funcName) throws TException {
- if (dbName == null || funcName == null) {
- throw new MetaException("Database and function name cannot be null.");
- }
- startFunction("get_function", ": " + dbName + "." + funcName);
-
- RawStore ms = getMS();
- Function func = null;
- Exception ex = null;
- String[] parsedDbName = parseDbName(dbName, conf);
-
- try {
- func = ms.getFunction(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], funcName);
- if (func == null) {
- throw new NoSuchObjectException(
- "Function " + dbName + "." + funcName + " does not exist");
- }
- } catch (Exception e) {
- ex = e;
- throw handleException(e).throwIfInstance(NoSuchObjectException.class).defaultMetaException();
- } finally {
- endFunction("get_function", func != null, ex);
- }
-
- return func;
- }
-
- // Transaction and locking methods
- @Override
- public GetOpenTxnsResponse get_open_txns() throws TException {
- return getTxnHandler().getOpenTxns();
- }
-
- @Override
- public GetOpenTxnsResponse get_open_txns_req(GetOpenTxnsRequest getOpenTxnsRequest) throws TException {
- return getTxnHandler().getOpenTxns(getOpenTxnsRequest.getExcludeTxnTypes());
- }
-
- // Transaction and locking methods
- @Override
- public GetOpenTxnsInfoResponse get_open_txns_info() throws TException {
- return getTxnHandler().getOpenTxnsInfo();
- }
-
- @Override
- public OpenTxnsResponse open_txns(OpenTxnRequest rqst) throws TException {
- OpenTxnsResponse response = getTxnHandler().openTxns(rqst);
- List<Long> txnIds = response.getTxn_ids();
- boolean isHiveReplTxn = rqst.isSetReplPolicy() && TxnType.DEFAULT.equals(rqst.getTxn_type());
- if (txnIds != null && listeners != null && !listeners.isEmpty() && !isHiveReplTxn) {
- MetaStoreListenerNotifier.notifyEvent(listeners, EventType.OPEN_TXN,
- new OpenTxnEvent(txnIds, this));
- }
- return response;
- }
-
- @Override
- public void abort_txn(AbortTxnRequest rqst) throws TException {
- getTxnHandler().abortTxn(rqst);
- boolean isHiveReplTxn = rqst.isSetReplPolicy() && TxnType.DEFAULT.equals(rqst.getTxn_type());
- if (listeners != null && !listeners.isEmpty() && !isHiveReplTxn) {
- // Not adding dbsUpdated to AbortTxnEvent because
- // only DbNotificationListener cares about it, and this is already
- // handled with transactional listeners in TxnHandler.
- MetaStoreListenerNotifier.notifyEvent(listeners, EventType.ABORT_TXN,
- new AbortTxnEvent(rqst.getTxnid(), this));
- }
- }
-
- @Override
- public void abort_txns(AbortTxnsRequest rqst) throws TException {
- getTxnHandler().abortTxns(rqst);
- if (listeners != null && !listeners.isEmpty()) {
- for (Long txnId : rqst.getTxn_ids()) {
- // See above abort_txn() note about not adding dbsUpdated.
- MetaStoreListenerNotifier.notifyEvent(listeners, EventType.ABORT_TXN,
- new AbortTxnEvent(txnId, this));
- }
- }
- }
-
- @Override
- public AbortCompactResponse abort_Compactions(AbortCompactionRequest rqst) throws TException {
- return getTxnHandler().abortCompactions(rqst);
- }
-
- @Override
- public long get_latest_txnid_in_conflict(long txnId) throws MetaException {
- return getTxnHandler().getLatestTxnIdInConflict(txnId);
- }
-
- @Override
- public void commit_txn(CommitTxnRequest rqst) throws TException {
- boolean isReplayedReplTxn = TxnType.REPL_CREATED.equals(rqst.getTxn_type());
- boolean isHiveReplTxn = rqst.isSetReplPolicy() && TxnType.DEFAULT.equals(rqst.getTxn_type());
- // in replication flow, the write notification log table will be updated here.
- if (rqst.isSetWriteEventInfos() && isReplayedReplTxn) {
- assert (rqst.isSetReplPolicy());
- long targetTxnId = getTxnHandler().getTargetTxnId(rqst.getReplPolicy(), rqst.getTxnid());
- if (targetTxnId < 0) {
- //looks like a retry
- return;
- }
- for (WriteEventInfo writeEventInfo : rqst.getWriteEventInfos()) {
- String[] filesAdded = ReplChangeManager.getListFromSeparatedString(writeEventInfo.getFiles());
- List<String> partitionValue = null;
- Partition ptnObj = null;
- String root;
- Table tbl = getTblObject(writeEventInfo.getDatabase(), writeEventInfo.getTable(), null);
-
- if (writeEventInfo.getPartition() != null && !writeEventInfo.getPartition().isEmpty()) {
- partitionValue = Warehouse.getPartValuesFromPartName(writeEventInfo.getPartition());
- ptnObj = getPartitionObj(writeEventInfo.getDatabase(), writeEventInfo.getTable(), partitionValue, tbl);
- root = ptnObj.getSd().getLocation();
- } else {
- root = tbl.getSd().getLocation();
- }
-
- InsertEventRequestData insertData = new InsertEventRequestData();
- insertData.setReplace(true);
-
- // The files in the commit txn message during load will have files with path corresponding to source
- // warehouse. Need to transform them to target warehouse using table or partition object location.
- for (String file : filesAdded) {
- String[] decodedPath = ReplChangeManager.decodeFileUri(file);
- String name = (new Path(decodedPath[0])).getName();
- Path newPath = FileUtils.getTransformedPath(name, decodedPath[3], root);
- insertData.addToFilesAdded(newPath.toUri().toString());
- insertData.addToSubDirectoryList(decodedPath[3]);
- try {
- insertData.addToFilesAddedChecksum(ReplChangeManager.checksumFor(newPath, newPath.getFileSystem(conf)));
- } catch (IOException e) {
- LOG.error("failed to get checksum for the file " + newPath + " with error: " + e.getMessage());
- throw new TException(e.getMessage());
- }
- }
-
- WriteNotificationLogRequest wnRqst = new WriteNotificationLogRequest(targetTxnId,
- writeEventInfo.getWriteId(), writeEventInfo.getDatabase(), writeEventInfo.getTable(), insertData);
- if (partitionValue != null) {
- wnRqst.setPartitionVals(partitionValue);
- }
- addTxnWriteNotificationLog(tbl, ptnObj, wnRqst);
- }
- }
- getTxnHandler().commitTxn(rqst);
- if (listeners != null && !listeners.isEmpty() && !isHiveReplTxn) {
- MetaStoreListenerNotifier.notifyEvent(listeners, EventType.COMMIT_TXN,
- new CommitTxnEvent(rqst.getTxnid(), this));
- Optional<CompactionInfo> compactionInfo = getTxnHandler().getCompactionByTxnId(rqst.getTxnid());
- if (compactionInfo.isPresent()) {
- MetaStoreListenerNotifier.notifyEvent(listeners, EventType.COMMIT_COMPACTION,
- new CommitCompactionEvent(rqst.getTxnid(), compactionInfo.get(), this));
- }
- }
- }
-
- @Override
- public void update_table_params(List<TableParamsUpdate> updates) throws TException {
- getMS().updateTableParams(updates);
- }
-
- @Override
- public void repl_tbl_writeid_state(ReplTblWriteIdStateRequest rqst) throws TException {
- getTxnHandler().replTableWriteIdState(rqst);
- }
-
- @Override
- public GetValidWriteIdsResponse get_valid_write_ids(GetValidWriteIdsRequest rqst) throws TException {
- return getTxnHandler().getValidWriteIds(rqst);
- }
-
- @Override
- public void add_write_ids_to_min_history(long txnId, Map<String, Long> validWriteIds) throws TException {
- getTxnHandler().addWriteIdsToMinHistory(txnId, validWriteIds);
- }
-
- @Override
- public void set_hadoop_jobid(String jobId, long cqId) throws MetaException {
- getTxnHandler().setHadoopJobId(jobId, cqId);
- }
-
- @Deprecated
- @Override
- public OptionalCompactionInfoStruct find_next_compact(String workerId) throws MetaException{
- return CompactionInfo.compactionInfoToOptionalStruct(
- getTxnHandler().findNextToCompact(workerId));
- }
-
- @Override
- public OptionalCompactionInfoStruct find_next_compact2(FindNextCompactRequest rqst) throws MetaException{
- return CompactionInfo.compactionInfoToOptionalStruct(
- getTxnHandler().findNextToCompact(rqst));
- }
-
- @Override
- public void mark_cleaned(CompactionInfoStruct cr) throws MetaException {
- getTxnHandler().markCleaned(CompactionInfo.compactionStructToInfo(cr));
- }
-
- @Override
- public void mark_compacted(CompactionInfoStruct cr) throws MetaException {
- getTxnHandler().markCompacted(CompactionInfo.compactionStructToInfo(cr));
- }
-
- @Override
- public void mark_failed(CompactionInfoStruct cr) throws MetaException {
- getTxnHandler().markFailed(CompactionInfo.compactionStructToInfo(cr));
- }
-
- @Override
- public void mark_refused(CompactionInfoStruct cr) throws MetaException {
- getTxnHandler().markRefused(CompactionInfo.compactionStructToInfo(cr));
- }
-
- @Override
- public boolean update_compaction_metrics_data(CompactionMetricsDataStruct struct) throws MetaException, TException {
- return getTxnHandler().updateCompactionMetricsData(CompactionMetricsDataConverter.structToData(struct));
- }
-
- @Override
- public void remove_compaction_metrics_data(CompactionMetricsDataRequest request)
- throws MetaException, TException {
- getTxnHandler().removeCompactionMetricsData(request.getDbName(), request.getTblName(), request.getPartitionName(),
- CompactionMetricsDataConverter.thriftCompactionMetricType2DbType(request.getType()));
- }
-
- @Override
- public List<String> find_columns_with_stats(CompactionInfoStruct cr) throws MetaException {
- return getTxnHandler().findColumnsWithStats(CompactionInfo.compactionStructToInfo(cr));
- }
-
- @Override
- public void update_compactor_state(CompactionInfoStruct cr, long highWaterMark) throws MetaException {
- getTxnHandler().updateCompactorState(
- CompactionInfo.compactionStructToInfo(cr), highWaterMark);
- }
-
- @Override
- public GetLatestCommittedCompactionInfoResponse get_latest_committed_compaction_info(
- GetLatestCommittedCompactionInfoRequest rqst) throws MetaException {
- if (rqst.getDbname() == null || rqst.getTablename() == null) {
- throw new MetaException("Database name and table name cannot be null.");
- }
- GetLatestCommittedCompactionInfoResponse response = getTxnHandler().getLatestCommittedCompactionInfo(rqst);
- return FilterUtils.filterCommittedCompactionInfoStructIfEnabled(isServerFilterEnabled, filterHook,
- getDefaultCatalog(conf), rqst.getDbname(), rqst.getTablename(), response);
- }
-
- @Override
- public AllocateTableWriteIdsResponse allocate_table_write_ids(
- AllocateTableWriteIdsRequest rqst) throws TException {
- AllocateTableWriteIdsResponse response = getTxnHandler().allocateTableWriteIds(rqst);
- if (listeners != null && !listeners.isEmpty()) {
- MetaStoreListenerNotifier.notifyEvent(listeners, EventType.ALLOC_WRITE_ID,
- new AllocWriteIdEvent(response.getTxnToWriteIds(), rqst.getDbName(),
- rqst.getTableName(), this));
- }
- return response;
- }
-
- @Override
- public MaxAllocatedTableWriteIdResponse get_max_allocated_table_write_id(MaxAllocatedTableWriteIdRequest rqst)
- throws MetaException {
- return getTxnHandler().getMaxAllocatedTableWriteId(rqst);
- }
-
- @Override
- public void seed_write_id(SeedTableWriteIdsRequest rqst) throws MetaException {
- getTxnHandler().seedWriteId(rqst);
- }
-
- @Override
- public void seed_txn_id(SeedTxnIdRequest rqst) throws MetaException {
- getTxnHandler().seedTxnId(rqst);
- }
-
- private void addTxnWriteNotificationLog(Table tableObj, Partition ptnObj, WriteNotificationLogRequest rqst)
- throws MetaException {
- String partition = ""; //Empty string is an invalid partition name. Can be used for non partitioned table.
- if (ptnObj != null) {
- partition = Warehouse.makePartName(tableObj.getPartitionKeys(), rqst.getPartitionVals());
- }
- AcidWriteEvent event = new AcidWriteEvent(partition, tableObj, ptnObj, rqst);
- getTxnHandler().addWriteNotificationLog(event);
- if (listeners != null && !listeners.isEmpty()) {
- MetaStoreListenerNotifier.notifyEvent(listeners, EventType.ACID_WRITE, event);
+ }
}
}
- private Table getTblObject(String db, String table, String catalog) throws MetaException, NoSuchObjectException {
- GetTableRequest req = new GetTableRequest(db, table);
- if (catalog != null) {
- req.setCatName(catalog);
+ private void validateForAlterFunction(String dbName, String funcName, Function newFunc)
+ throws MetaException {
+ if (dbName == null || funcName == null) {
+ throw new MetaException("Database and function name cannot be null.");
}
- req.setCapabilities(new ClientCapabilities(Lists.newArrayList(ClientCapability.TEST_CAPABILITY, ClientCapability.INSERT_ONLY_TABLES)));
- return get_table_req(req).getTable();
- }
-
- private Partition getPartitionObj(String db, String table, List<String> partitionVals, Table tableObj)
- throws MetaException, NoSuchObjectException {
- if (tableObj.isSetPartitionKeys() && !tableObj.getPartitionKeys().isEmpty()) {
- return get_partition(db, table, partitionVals);
+ try {
+ validateFunctionInfo(newFunc);
+ } catch (InvalidObjectException e) {
+ // The validateFunctionInfo method is used by the create and alter function methods as well
+ // and it can throw InvalidObjectException. But the InvalidObjectException is not defined
+ // for the alter_function method in the Thrift interface, therefore a TApplicationException
+ // will occur at the caller side. Re-throwing the InvalidObjectException as MetaException
+ // would eliminate the TApplicationException at caller side.
+ throw newMetaException(e);
}
- return null;
}
@Override
- public WriteNotificationLogResponse add_write_notification_log(WriteNotificationLogRequest rqst)
- throws TException {
- Table tableObj = getTblObject(rqst.getDb(), rqst.getTable(), null);
- Partition ptnObj = getPartitionObj(rqst.getDb(), rqst.getTable(), rqst.getPartitionVals(), tableObj);
- addTxnWriteNotificationLog(tableObj, ptnObj, rqst);
- return new WriteNotificationLogResponse();
+ public List<String> get_functions(String dbName, String pattern)
+ throws MetaException {
+ String[] parsedDbName = parseDbName(dbName, conf);
+ GetFunctionsRequest request = new GetFunctionsRequest(parsedDbName[DB_NAME]);
+ request.setCatalogName(parsedDbName[CAT_NAME]);
+ request.setPattern(pattern);
+ request.setReturnNames(true);
+ GetFunctionsResponse resp = get_functions_req(request);
+ return resp.getFunction_names();
}
@Override
- public WriteNotificationLogBatchResponse add_write_notification_log_in_batch(
- WriteNotificationLogBatchRequest batchRequest) throws TException {
- if (batchRequest.getRequestList().size() == 0) {
- return new WriteNotificationLogBatchResponse();
- }
-
- Table tableObj = getTblObject(batchRequest.getDb(), batchRequest.getTable(), batchRequest.getCatalog());
- BatchAcidWriteEvent event = new BatchAcidWriteEvent();
- List<String> partNameList = new ArrayList<>();
- List<Partition> ptnObjList;
-
- Map<String, WriteNotificationLogRequest> rqstMap = new HashMap<>();
- if (tableObj.getPartitionKeys().size() != 0) {
- // partitioned table
- for (WriteNotificationLogRequest rqst : batchRequest.getRequestList()) {
- String partition = Warehouse.makePartName(tableObj.getPartitionKeys(), rqst.getPartitionVals());
- partNameList.add(partition);
- // This is used to ignore those request for which the partition does not exists.
- rqstMap.put(partition, rqst);
- }
- ptnObjList = getMS().getPartitionsByNames(tableObj.getCatName(), tableObj.getDbName(),
- tableObj.getTableName(), partNameList);
- } else {
- ptnObjList = new ArrayList<>();
- for (WriteNotificationLogRequest ignored : batchRequest.getRequestList()) {
- ptnObjList.add(null);
- }
- }
+ public GetFunctionsResponse get_functions_req(GetFunctionsRequest req)
+ throws MetaException {
+ startFunction("get_functions_req", ": db=" + req.getDbName()
+ + " pat=" + req.getPattern());
- int idx = 0;
- for (Partition partObject : ptnObjList) {
- String partition = ""; //Empty string is an invalid partition name. Can be used for non partitioned table.
- WriteNotificationLogRequest request;
- if (partObject != null) {
- partition = Warehouse.makePartName(tableObj.getPartitionKeys(), partObject.getValues());
- request = rqstMap.get(partition);
+ RawStore ms = getMS();
+ Exception ex = null;
+ GetFunctionsResponse response = new GetFunctionsResponse();
+ String catName = req.isSetCatalogName() ? req.getCatalogName() : getDefaultCatalog(conf);
+ try {
+ List result = ms.getFunctionsRequest(catName, req.getDbName(),
+ req.getPattern(), req.isReturnNames());
+ if (req.isReturnNames()) {
+ response.setFunction_names(result);
} else {
- // for non partitioned table, we can get serially from the list.
- request = batchRequest.getRequestList().get(idx++);
- }
- event.addNotification(partition, tableObj, partObject, request);
- if (listeners != null && !listeners.isEmpty()) {
- MetaStoreListenerNotifier.notifyEvent(listeners, EventType.BATCH_ACID_WRITE,
- new BatchAcidWriteEvent(partition, tableObj, partObject, request));
+ response.setFunctions(result);
}
+ } catch (Exception e) {
+ ex = e;
+ throw newMetaException(e);
+ } finally {
+ endFunction("get_functions", ex == null, ex);
}
-
- getTxnHandler().addWriteNotificationLog(event);
- return new WriteNotificationLogBatchResponse();
- }
-
- @Override
- public LockResponse lock(LockRequest rqst) throws TException {
- return getTxnHandler().lock(rqst);
- }
-
- @Override
- public LockResponse check_lock(CheckLockRequest rqst) throws TException {
- return getTxnHandler().checkLock(rqst);
- }
-
- @Override
- public void unlock(UnlockRequest rqst) throws TException {
- getTxnHandler().unlock(rqst);
- }
-
- @Override
- public ShowLocksResponse show_locks(ShowLocksRequest rqst) throws TException {
- return getTxnHandler().showLocks(rqst);
- }
-
- @Override
- public void heartbeat(HeartbeatRequest ids) throws TException {
- getTxnHandler().heartbeat(ids);
- }
-
- @Override
- public HeartbeatTxnRangeResponse heartbeat_txn_range(HeartbeatTxnRangeRequest rqst)
- throws TException {
- return getTxnHandler().heartbeatTxnRange(rqst);
- }
- @Deprecated
- @Override
- public void compact(CompactionRequest rqst) throws TException {
- compact2(rqst);
- }
- @Override
- public CompactionResponse compact2(CompactionRequest rqst) throws TException {
- return getTxnHandler().compact(rqst);
- }
-
- @Override
- public ShowCompactResponse show_compact(ShowCompactRequest rqst) throws TException {
- ShowCompactResponse response = getTxnHandler().showCompact(rqst);
- response.setCompacts(FilterUtils.filterCompactionsIfEnabled(isServerFilterEnabled,
- filterHook, getDefaultCatalog(conf), response.getCompacts()));
return response;
}
@Override
- public boolean submit_for_cleanup(CompactionRequest rqst, long highestWriteId, long txnId)
- throws TException {
- return getTxnHandler().submitForCleanup(rqst, highestWriteId, txnId);
- }
-
- @Override
- public void flushCache() throws TException {
- getMS().flushCache();
- }
-
- @Override
- public void add_dynamic_partitions(AddDynamicPartitions rqst) throws TException {
- getTxnHandler().addDynamicPartitions(rqst);
- }
-
- @Override
- public GetPrincipalsInRoleResponse get_principals_in_role(GetPrincipalsInRoleRequest request)
- throws TException {
-
- incrementCounter("get_principals_in_role");
- firePreEvent(new PreAuthorizationCallEvent(this));
+ public GetAllFunctionsResponse get_all_functions()
+ throws MetaException {
+ GetAllFunctionsResponse response = new GetAllFunctionsResponse();
+ startFunction("get_all_functions");
+ RawStore ms = getMS();
+ List<Function> allFunctions = null;
Exception ex = null;
- GetPrincipalsInRoleResponse response = null;
try {
- response = new GetPrincipalsInRoleResponse(getMS().listRoleMembers(request.getRoleName()));
+ // Leaving this as the 'hive' catalog (rather than choosing the default from the
+ // configuration) because all the default UDFs are in that catalog, and I think that's
+ // would people really want here.
+ allFunctions = ms.getAllFunctions(DEFAULT_CATALOG_NAME);
} catch (Exception e) {
ex = e;
- rethrowException(e);
+ throw newMetaException(e);
} finally {
- endFunction("get_principals_in_role", ex == null, ex);
+ endFunction("get_all_functions", allFunctions != null, ex);
}
+ response.setFunctions(allFunctions);
return response;
}
@Override
- public GetRoleGrantsForPrincipalResponse get_role_grants_for_principal(
- GetRoleGrantsForPrincipalRequest request) throws TException {
+ public Function get_function(String dbName, String funcName) throws TException {
+ if (dbName == null || funcName == null) {
+ throw new MetaException("Database and function name cannot be null.");
+ }
+ startFunction("get_function", ": " + dbName + "." + funcName);
- incrementCounter("get_role_grants_for_principal");
- firePreEvent(new PreAuthorizationCallEvent(this));
+ RawStore ms = getMS();
+ Function func = null;
Exception ex = null;
- List<RolePrincipalGrant> roleMaps = null;
+ String[] parsedDbName = parseDbName(dbName, conf);
+
try {
- roleMaps = getMS().listRolesWithGrants(request.getPrincipal_name(), request.getPrincipal_type());
+ func = ms.getFunction(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], funcName);
+ if (func == null) {
+ throw new NoSuchObjectException(
+ "Function " + dbName + "." + funcName + " does not exist");
+ }
} catch (Exception e) {
ex = e;
- rethrowException(e);
+ throw handleException(e).throwIfInstance(NoSuchObjectException.class).defaultMetaException();
} finally {
- endFunction("get_role_grants_for_principal", ex == null, ex);
+ endFunction("get_function", func != null, ex);
}
- //List<RolePrincipalGrant> roleGrantsList = getRolePrincipalGrants(roleMaps);
- return new GetRoleGrantsForPrincipalResponse(roleMaps);
+ return func;
}
@Override
+ public void update_table_params(List<TableParamsUpdate> updates) throws TException {
+ getMS().updateTableParams(updates);
+ }
+
public AggrStats get_aggr_stats_for(PartitionsStatsRequest request) throws TException {
String catName = request.isSetCatName() ? request.getCatName().toLowerCase() :
getDefaultCatalog(conf);
@@ -6545,53 +3707,6 @@ public boolean set_aggr_stats_for(SetPartitionsStatsRequest req) throws TExcepti
}
}
- @Override
- public NotificationEventResponse get_next_notification(NotificationEventRequest rqst)
- throws TException {
- authorizeProxyPrivilege();
-
- RawStore ms = getMS();
- return ms.getNextNotification(rqst);
- }
-
- @Override
- public CurrentNotificationEventId get_current_notificationEventId() throws TException {
- authorizeProxyPrivilege();
-
- RawStore ms = getMS();
- return ms.getCurrentNotificationEventId();
- }
-
- @Override
- public NotificationEventsCountResponse get_notification_events_count(NotificationEventsCountRequest rqst)
- throws TException {
- authorizeProxyPrivilege();
-
- RawStore ms = getMS();
- return ms.getNotificationEventsCount(rqst);
- }
-
- private void authorizeProxyPrivilege() throws TException {
- // Skip the auth in embedded mode or if the auth is disabled
- if (!HiveMetaStore.isMetaStoreRemote() ||
- !MetastoreConf.getBoolVar(conf, ConfVars.EVENT_DB_NOTIFICATION_API_AUTH) || conf.getBoolean(HIVE_IN_TEST.getVarname(),
- false)) {
- return;
- }
- String user = null;
- try {
- user = SecurityUtils.getUGI().getShortUserName();
- } catch (Exception ex) {
- LOG.error("Cannot obtain username", ex);
- throw new TException(ex);
- }
- if (!MetaStoreServerUtils.checkUserHasHostProxyPrivileges(user, conf, getIPAddress())) {
- LOG.error("Not authorized to make the get_notification_events_count call. You can try to disable " +
- ConfVars.EVENT_DB_NOTIFICATION_API_AUTH.toString());
- throw new TException("User " + user + " is not allowed to perform this API call");
- }
- }
-
@Override
public FireEventResponse fire_listener_event(FireEventRequest rqst) throws TException {
switch (rqst.getData().getSetField()) {
@@ -6689,213 +3804,6 @@ public FireEventResponse fire_listener_event(FireEventRequest rqst) throws TExce
}
- @Override
- public GetFileMetadataByExprResult get_file_metadata_by_expr(GetFileMetadataByExprRequest req)
- throws TException {
- GetFileMetadataByExprResult result = new GetFileMetadataByExprResult();
- RawStore ms = getMS();
- if (!ms.isFileMetadataSupported()) {
- result.setIsSupported(false);
- result.setMetadata(Collections.emptyMap()); // Set the required field.
- return result;
- }
- result.setIsSupported(true);
-
- List<Long> fileIds = req.getFileIds();
- boolean needMetadata = !req.isSetDoGetFooters() || req.isDoGetFooters();
- FileMetadataExprType type = req.isSetType() ? req.getType() : FileMetadataExprType.ORC_SARG;
-
- ByteBuffer[] metadatas = needMetadata ? new ByteBuffer[fileIds.size()] : null;
- ByteBuffer[] ppdResults = new ByteBuffer[fileIds.size()];
- boolean[] eliminated = new boolean[fileIds.size()];
-
- getMS().getFileMetadataByExpr(fileIds, type, req.getExpr(), metadatas, ppdResults, eliminated);
- for (int i = 0; i < fileIds.size(); ++i) {
- if (!eliminated[i] && ppdResults[i] == null)
- {
- continue; // No metadata => no ppd.
- }
- MetadataPpdResult mpr = new MetadataPpdResult();
- ByteBuffer ppdResult = eliminated[i] ? null : handleReadOnlyBufferForThrift(ppdResults[i]);
- mpr.setIncludeBitset(ppdResult);
- if (needMetadata) {
- ByteBuffer metadata = eliminated[i] ? null : handleReadOnlyBufferForThrift(metadatas[i]);
- mpr.setMetadata(metadata);
- }
- result.putToMetadata(fileIds.get(i), mpr);
- }
- if (!result.isSetMetadata()) {
- result.setMetadata(Collections.emptyMap()); // Set the required field.
- }
- return result;
- }
-
- @Override
- public GetFileMetadataResult get_file_metadata(GetFileMetadataRequest req) throws TException {
- GetFileMetadataResult result = new GetFileMetadataResult();
- RawStore ms = getMS();
- if (!ms.isFileMetadataSupported()) {
- result.setIsSupported(false);
- result.setMetadata(Collections.emptyMap()); // Set the required field.
- return result;
- }
- result.setIsSupported(true);
- List<Long> fileIds = req.getFileIds();
- ByteBuffer[] metadatas = ms.getFileMetadata(fileIds);
- assert metadatas.length == fileIds.size();
- for (int i = 0; i < metadatas.length; ++i) {
- ByteBuffer bb = metadatas[i];
- if (bb == null) {
- continue;
- }
- bb = handleReadOnlyBufferForThrift(bb);
- result.putToMetadata(fileIds.get(i), bb);
- }
- if (!result.isSetMetadata()) {
- result.setMetadata(Collections.emptyMap()); // Set the required field.
- }
- return result;
- }
-
- private ByteBuffer handleReadOnlyBufferForThrift(ByteBuffer bb) {
- if (!bb.isReadOnly()) {
- return bb;
- }
- // Thrift cannot write read-only buffers... oh well.
- // TODO: actually thrift never writes to the buffer, so we could use reflection to
- // unset the unnecessary read-only flag if allocation/copy perf becomes a problem.
- ByteBuffer copy = ByteBuffer.allocate(bb.capacity());
- copy.put(bb);
- copy.flip();
- return copy;
- }
-
- @Override
- public PutFileMetadataResult put_file_metadata(PutFileMetadataRequest req) throws TException {
- RawStore ms = getMS();
- if (ms.isFileMetadataSupported()) {
- ms.putFileMetadata(req.getFileIds(), req.getMetadata(), req.getType());
- }
- return new PutFileMetadataResult();
- }
-
- @Override
- public ClearFileMetadataResult clear_file_metadata(ClearFileMetadataRequest req)
- throws TException {
- getMS().putFileMetadata(req.getFileIds(), null, null);
- return new ClearFileMetadataResult();
- }
-
- @Override
- public CacheFileMetadataResult cache_file_metadata(
- CacheFileMetadataRequest req) throws TException {
- RawStore ms = getMS();
- if (!ms.isFileMetadataSupported()) {
- return new CacheFileMetadataResult(false);
- }
- String dbName = req.getDbName(), tblName = req.getTblName(),
- partName = req.isSetPartName() ? req.getPartName() : null;
- boolean isAllPart = req.isSetIsAllParts() && req.isIsAllParts();
- ms.openTransaction();
- boolean success = false;
- try {
- Table tbl = ms.getTable(getDefaultCatalog(conf), dbName, tblName);
- if (tbl == null) {
- throw new NoSuchObjectException(dbName + "." + tblName + " not found");
- }
- boolean isPartitioned = tbl.isSetPartitionKeys() && tbl.getPartitionKeysSize() > 0;
- String tableInputFormat = tbl.isSetSd() ? tbl.getSd().getInputFormat() : null;
- if (!isPartitioned) {
- if (partName != null || isAllPart) {
- throw new MetaException("Table is not partitioned");
- }
- if (!tbl.isSetSd() || !tbl.getSd().isSetLocation()) {
- throw new MetaException(
- "Table does not have storage location; this operation is not supported on views");
- }
- FileMetadataExprType type = expressionProxy.getMetadataType(tableInputFormat);
- if (type == null) {
- throw new MetaException("The operation is not supported for " + tableInputFormat);
- }
- fileMetadataManager.queueCacheMetadata(tbl.getSd().getLocation(), type);
- success = true;
- } else {
- List<String> partNames;
- if (partName != null) {
- partNames = Lists.newArrayList(partName);
- } else if (isAllPart) {
- partNames = ms.listPartitionNames(getDefaultCatalog(conf), dbName, tblName, (short)-1);
- } else {
- throw new MetaException("Table is partitioned");
- }
- int batchSize = MetastoreConf.getIntVar(
- conf, ConfVars.BATCH_RETRIEVE_OBJECTS_MAX);
- int index = 0;
- int successCount = 0, failCount = 0;
- HashSet<String> failFormats = null;
- while (index < partNames.size()) {
- int currentBatchSize = Math.min(batchSize, partNames.size() - index);
- List<String> nameBatch = partNames.subList(index, index + currentBatchSize);
- index += currentBatchSize;
- List<Partition> parts = ms.getPartitionsByNames(getDefaultCatalog(conf), dbName, tblName, nameBatch);
- for (Partition part : parts) {
- if (!part.isSetSd() || !part.getSd().isSetLocation()) {
- throw new MetaException("Partition does not have storage location;" +
- " this operation is not supported on views");
- }
- String inputFormat = part.getSd().isSetInputFormat()
- ? part.getSd().getInputFormat() : tableInputFormat;
- FileMetadataExprType type = expressionProxy.getMetadataType(inputFormat);
- if (type == null) {
- ++failCount;
- if (failFormats == null) {
- failFormats = new HashSet<>();
- }
- failFormats.add(inputFormat);
- } else {
- ++successCount;
- fileMetadataManager.queueCacheMetadata(part.getSd().getLocation(), type);
- }
- }
- }
- success = true; // Regardless of the following exception
- if (failCount > 0) {
- String errorMsg = "The operation failed for " + failCount + " partitions and "
- + "succeeded for " + successCount + " partitions; unsupported formats: ";
- boolean isFirst = true;
- for (String s : failFormats) {
- if (!isFirst) {
- errorMsg += ", ";
- }
- isFirst = false;
- errorMsg += s;
- }
- throw new MetaException(errorMsg);
- }
- }
- } finally {
- if (success) {
- if (!ms.commitTransaction()) {
- throw new MetaException("Failed to commit");
- }
- } else {
- ms.rollbackTransaction();
- }
- }
- return new CacheFileMetadataResult(true);
- }
-
- @VisibleForTesting
- void updateMetrics() throws MetaException {
- if (Metrics.getRegistry() != null &&
- MetastoreConf.getBoolVar(conf, ConfVars.INIT_METADATA_COUNT_ENABLED)) {
- LOG.info("Begin calculating metadata count metrics.");
- Metrics.getOrCreateGauge(MetricsConstants.TOTAL_TABLES).set(getMS().getTableCount());
- Metrics.getOrCreateGauge(MetricsConstants.TOTAL_PARTITIONS).set(getMS().getPartitionCount());
- Metrics.getOrCreateGauge(MetricsConstants.TOTAL_DATABASES).set(getMS().getDatabaseCount());
- }
- }
-
@Override
public PrimaryKeysResponse get_primary_keys(PrimaryKeysRequest request) throws TException {
request.setCatName(request.isSetCatName() ? request.getCatName() : getDefaultCatalog(conf));
@@ -7024,16 +3932,6 @@ public AllTableConstraintsResponse get_all_table_constraints(AllTableConstraints
return new AllTableConstraintsResponse(ret);
}
- @Override
- public String get_metastore_db_uuid() throws TException {
- try {
- return getMS().getMetastoreDbUuid();
- } catch (MetaException e) {
- LOG.error("Exception thrown while querying metastore db uuid", e);
- throw e;
- }
- }
-
@Override
public WMCreateResourcePlanResponse create_resource_plan(WMCreateResourcePlanRequest request)
throws AlreadyExistsException, InvalidObjectException, MetaException, TException {
@@ -7723,32 +4621,6 @@ public SerDeInfo get_serde(GetSerdeRequest rqst) throws TException {
}
}
- @Deprecated
- @Override
- public LockResponse get_lock_materialization_rebuild(String dbName, String tableName, long txnId) throws TException {
- return get_lock_materialization_rebuild_req(new LockMaterializationRebuildRequest(DEFAULT_CATALOG_NAME, dbName,
- tableName, txnId));
- }
-
- @Override
- public LockResponse get_lock_materialization_rebuild_req(LockMaterializationRebuildRequest rqst)
- throws TException {
- return getTxnHandler().lockMaterializationRebuild(rqst);
- }
-
- @Deprecated
- @Override
- public boolean heartbeat_lock_materialization_rebuild(String dbName, String tableName, long txnId) throws TException {
- return heartbeat_lock_materialization_rebuild_req(new LockMaterializationRebuildRequest(DEFAULT_CATALOG_NAME,
- dbName, tableName, txnId));
- }
-
- @Override
- public boolean heartbeat_lock_materialization_rebuild_req(LockMaterializationRebuildRequest rqst) throws TException {
- return getTxnHandler().heartbeatLockMaterializationRebuild(rqst);
- }
-
- @Override
public void add_runtime_stats(RuntimeStat stat) throws TException {
startFunction("store_runtime_stats");
Exception ex = null;
@@ -7862,22 +4734,6 @@ private void abortReplCreatedOpenTxnsForDatabase(String query) throws TException
}
}
- @Override
- public ReplayedTxnsForPolicyResult get_replayed_txns_for_policy(String policyName) throws MetaException {
- Exception ex = null;
- ReplayedTxnsForPolicyResult ret = null;
- try {
- startFunction("get_replayed_txns_for_policy");
- ret = getTxnHandler().getReplayedTxnsForPolicy(policyName);
- } catch (Exception e) {
- ex = e;
- throw new MetaException("Failed to get replayed txns details for policy " + e.getMessage());
- } finally {
- endFunction("get_replayed_txns_for_policy", ret != null, ex);
- }
- return ret;
- }
-
@Override
public void scheduled_query_progress(ScheduledQueryProgressInfo info) throws MetaException, TException {
startFunction("scheduled_query_poll");
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandlerContext.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandlerContext.java
index c30b5b3828f..d52755786d4 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandlerContext.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandlerContext.java
@@ -22,6 +22,7 @@
import java.util.Optional;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.handler.BaseHandler;
import org.apache.hadoop.hive.metastore.txn.TxnStore;
import org.apache.hadoop.hive.metastore.txn.TxnUtils;
@@ -38,7 +39,7 @@ public final class HMSHandlerContext {
private TxnStore txnStore;
// Thread local HMSHandler used during shutdown to notify meta listeners
- private HMSHandler hmsHandler;
+ private BaseHandler hmsHandler;
// Thread local configuration is needed as many threads could make changes
// to the conf using the connection hook
@@ -61,7 +62,7 @@ public static Optional<RawStore> getRawStore() {
return Optional.ofNullable(context.get().rawStore);
}
- public static Optional<HMSHandler> getHMSHandler() {
+ public static Optional<BaseHandler> getHMSHandler() {
return Optional.ofNullable(context.get().hmsHandler);
}
@@ -96,7 +97,7 @@ public static void setTxnStore(TxnStore txnStore) {
context.get().txnStore = txnStore;
}
- public static void setHMSHandler(HMSHandler hmsHandler) {
+ public static void setHMSHandler(BaseHandler hmsHandler) {
context.get().hmsHandler = hmsHandler;
}
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreInit.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreInit.java
index 9417cf5a80e..5a4ceff0978 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreInit.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreInit.java
@@ -83,7 +83,7 @@ static boolean updateConnectionURL(Configuration originalConf, Configuration act
return false;
}
- static String getConnectionURL(Configuration conf) {
+ public static String getConnectionURL(Configuration conf) {
return MetastoreConf.getVar(conf, ConfVars.CONNECT_URL_KEY, "");
}
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/TSetIpAddressProcessor.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/TSetIpAddressProcessor.java
index 0b39353773f..e365d2a55e8 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/TSetIpAddressProcessor.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/TSetIpAddressProcessor.java
@@ -23,6 +23,7 @@
import org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore;
import org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore.Iface;
+import org.apache.hadoop.hive.metastore.handler.BaseHandler;
import org.apache.thrift.TException;
import org.apache.thrift.protocol.TProtocol;
import org.apache.thrift.transport.TSocket;
@@ -56,6 +57,6 @@ protected void setIpAddress(final TProtocol in) {
}
protected void setIpAddress(final Socket inSocket) {
- HMSHandler.setThreadLocalIpAddress(inSocket.getInetAddress().getHostAddress());
+ BaseHandler.setThreadLocalIpAddress(inSocket.getInetAddress().getHostAddress());
}
}
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
index 786cdd615fc..16404a9ca9c 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
@@ -60,7 +60,7 @@ public final class TransactionalValidationListener extends MetaStorePreEventList
private final Set<String> supportedCatalogs = new HashSet<String>();
- TransactionalValidationListener(Configuration conf) {
+ public TransactionalValidationListener(Configuration conf) {
super(conf);
supportedCatalogs.add("hive");
}
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
index 7df41ec68a3..04468bd2139 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
@@ -78,8 +78,8 @@
import com.google.common.annotations.VisibleForTesting;
-import static org.apache.hadoop.hive.metastore.HMSHandler.getPartValsFromName;
import static org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils.findStaleColumns;
+import static org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils.getPartValsFromName;
import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.getDefaultCatalog;
import static org.apache.hadoop.hive.metastore.utils.StringUtils.normalizeIdentifier;
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/directsql/DirectSqlUpdatePart.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/directsql/DirectSqlUpdatePart.java
index c22df8e54d9..4a7f831d8d0 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/directsql/DirectSqlUpdatePart.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/directsql/DirectSqlUpdatePart.java
@@ -73,11 +73,11 @@
import java.util.stream.Collectors;
import static org.apache.hadoop.hive.common.StatsSetupConst.COLUMN_STATS_ACCURATE;
-import static org.apache.hadoop.hive.metastore.HMSHandler.getPartValsFromName;
import static org.apache.hadoop.hive.metastore.directsql.MetastoreDirectSqlUtils.extractSqlClob;
import static org.apache.hadoop.hive.metastore.directsql.MetastoreDirectSqlUtils.extractSqlInt;
import static org.apache.hadoop.hive.metastore.directsql.MetastoreDirectSqlUtils.extractSqlLong;
import static org.apache.hadoop.hive.metastore.directsql.MetastoreDirectSqlUtils.getModelIdentity;
+import static org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils.getPartValsFromName;
/**
* This class contains the optimizations for MetaStore that rely on direct SQL access to
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/handler/AppendPartitionHandler.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/handler/AppendPartitionHandler.java
index dd26a3ded60..f7463410575 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/handler/AppendPartitionHandler.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/handler/AppendPartitionHandler.java
@@ -47,8 +47,8 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static org.apache.hadoop.hive.metastore.HMSHandler.getPartValsFromName;
import static org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils.canUpdateStats;
+import static org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils.getPartValsFromName;
import static org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils.updatePartitionStatsFast;
import static org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils.validatePartitionNameCharacters;
import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.getDefaultCatalog;
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/handler/BaseHandler.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/handler/BaseHandler.java
new file mode 100644
index 00000000000..b91af16a35c
--- /dev/null
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/handler/BaseHandler.java
@@ -0,0 +1,1016 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.handler;
+
+import com.codahale.metrics.Counter;
+import com.facebook.fb303.FacebookBase;
+import com.facebook.fb303.fb_status;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Splitter;
+import com.google.common.util.concurrent.Striped;
+
+import javax.jdo.JDOException;
+import java.util.AbstractMap;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.concurrent.locks.Lock;
+import java.util.regex.Pattern;
+
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.common.TableName;
+import org.apache.hadoop.hive.metastore.AcidEventListener;
+import org.apache.hadoop.hive.metastore.AlterHandler;
+import org.apache.hadoop.hive.metastore.DefaultMetaStoreFilterHookImpl;
+import org.apache.hadoop.hive.metastore.FileMetadataManager;
+import org.apache.hadoop.hive.metastore.HMSHandlerContext;
+import org.apache.hadoop.hive.metastore.HMSMetricsListener;
+import org.apache.hadoop.hive.metastore.HiveMetaStore;
+import org.apache.hadoop.hive.metastore.IHMSHandler;
+import org.apache.hadoop.hive.metastore.IMetaStoreMetadataTransformer;
+import org.apache.hadoop.hive.metastore.MetaStoreEndFunctionContext;
+import org.apache.hadoop.hive.metastore.MetaStoreEndFunctionListener;
+import org.apache.hadoop.hive.metastore.MetaStoreEventListener;
+import org.apache.hadoop.hive.metastore.MetaStoreFilterHook;
+import org.apache.hadoop.hive.metastore.MetaStoreInit;
+import org.apache.hadoop.hive.metastore.MetaStoreInitContext;
+import org.apache.hadoop.hive.metastore.MetaStoreInitListener;
+import org.apache.hadoop.hive.metastore.MetaStoreListenerNotifier;
+import org.apache.hadoop.hive.metastore.MetaStorePreEventListener;
+import org.apache.hadoop.hive.metastore.PartFilterExprUtil;
+import org.apache.hadoop.hive.metastore.PartitionExpressionProxy;
+import org.apache.hadoop.hive.metastore.RawStore;
+import org.apache.hadoop.hive.metastore.SessionPropertiesListener;
+import org.apache.hadoop.hive.metastore.TransactionalMetaStoreEventListener;
+import org.apache.hadoop.hive.metastore.TransactionalValidationListener;
+import org.apache.hadoop.hive.metastore.Warehouse;
+import org.apache.hadoop.hive.metastore.api.CmRecycleRequest;
+import org.apache.hadoop.hive.metastore.api.CmRecycleResponse;
+import org.apache.hadoop.hive.metastore.api.ConfigValSecurityException;
+import org.apache.hadoop.hive.metastore.api.CurrentNotificationEventId;
+import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.metastore.api.DatabaseType;
+import org.apache.hadoop.hive.metastore.api.HiveObjectPrivilege;
+import org.apache.hadoop.hive.metastore.api.HiveObjectRef;
+import org.apache.hadoop.hive.metastore.api.HiveObjectType;
+import org.apache.hadoop.hive.metastore.api.InvalidObjectException;
+import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
+import org.apache.hadoop.hive.metastore.api.NotificationEventRequest;
+import org.apache.hadoop.hive.metastore.api.NotificationEventResponse;
+import org.apache.hadoop.hive.metastore.api.NotificationEventsCountRequest;
+import org.apache.hadoop.hive.metastore.api.NotificationEventsCountResponse;
+import org.apache.hadoop.hive.metastore.api.PrincipalType;
+import org.apache.hadoop.hive.metastore.api.PrivilegeBag;
+import org.apache.hadoop.hive.metastore.api.PrivilegeGrantInfo;
+import org.apache.hadoop.hive.metastore.api.Role;
+import org.apache.hadoop.hive.metastore.api.TableMeta;
+import org.apache.hadoop.hive.metastore.api.Type;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.hadoop.hive.metastore.events.ConfigChangeEvent;
+import org.apache.hadoop.hive.metastore.events.PreEventContext;
+import org.apache.hadoop.hive.metastore.events.PreReadDatabaseEvent;
+import org.apache.hadoop.hive.metastore.messaging.EventMessage;
+import org.apache.hadoop.hive.metastore.metrics.Metrics;
+import org.apache.hadoop.hive.metastore.metrics.MetricsConstants;
+import org.apache.hadoop.hive.metastore.txn.TxnStore;
+import org.apache.hadoop.hive.metastore.utils.JavaUtils;
+import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils;
+import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
+import org.apache.hadoop.hive.metastore.utils.MetastoreVersionInfo;
+import org.apache.hadoop.hive.metastore.utils.SecurityUtils;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.thrift.TException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.commons.lang3.StringUtils.isBlank;
+import static org.apache.commons.lang3.StringUtils.join;
+import static org.apache.hadoop.hive.metastore.ExceptionHandler.handleException;
+import static org.apache.hadoop.hive.metastore.ExceptionHandler.newMetaException;
+import static org.apache.hadoop.hive.metastore.HMSHandler.createDefaultCatalog;
+import static org.apache.hadoop.hive.metastore.HMSHandler.getIPAddress;
+import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_CATALOG_NAME;
+import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_DATABASE_COMMENT;
+import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_DATABASE_NAME;
+import static org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars.HIVE_IN_TEST;
+import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.CAT_NAME;
+import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.DB_NAME;
+import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.parseDbName;
+import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.prependCatalogToDbName;
+
+/**
+ * This class serves as the super class for all handlers that implement the IHMSHandler
+ */
+public abstract class BaseHandler extends FacebookBase implements IHMSHandler {
+ private static final Logger LOG = LoggerFactory.getLogger(BaseHandler.class);
+ private static String currentUrl;
+ private static volatile Striped<Lock> tablelocks;
+
+ public static final String ADMIN = "admin";
+ public static final String PUBLIC = "public";
+
+ static final int LOG_SAMPLE_PARTITIONS_MAX_SIZE = 4;
+ static final int LOG_SAMPLE_PARTITIONS_HALF_SIZE = 2;
+ static final String LOG_SAMPLE_PARTITIONS_SEPARATOR = ",";
+
+ protected final Configuration conf;
+ protected FileMetadataManager fileMetadataManager;
+ protected PartitionExpressionProxy expressionProxy;
+ protected IMetaStoreMetadataTransformer transformer;
+ protected Warehouse wh; // hdfs warehouse
+ protected AlterHandler alterHandler;
+ protected List<MetaStorePreEventListener> preListeners;
+ protected List<MetaStoreEventListener> listeners;
+ protected List<TransactionalMetaStoreEventListener> transactionalListeners;
+ protected List<MetaStoreEndFunctionListener> endFunctionListeners;
+ protected List<MetaStoreInitListener> initListeners;
+ protected MetaStoreFilterHook filterHook;
+ protected boolean isServerFilterEnabled = false;
+
+ protected BaseHandler(String name, Configuration conf) {
+ super(name);
+ this.conf = conf;
+ }
+
+ @Override
+ public List<TransactionalMetaStoreEventListener> getTransactionalListeners() {
+ return transactionalListeners;
+ }
+
+ @Override
+ public List<MetaStoreEventListener> getListeners() {
+ return listeners;
+ }
+
+ @Override
+ public IMetaStoreMetadataTransformer getMetadataTransformer() {
+ return transformer;
+ }
+
+ @Override
+ public MetaStoreFilterHook getMetaFilterHook() {
+ return filterHook;
+ }
+
+ // Make it possible for tests to check that the right type of PartitionExpressionProxy was
+ // instantiated.
+ @VisibleForTesting
+ public PartitionExpressionProxy getExpressionProxy() {
+ return expressionProxy;
+ }
+
+ @Override
+ public void init() throws MetaException {
+ init(new Warehouse(conf));
+ }
+
+ @VisibleForTesting
+ public void init(Warehouse wh) throws MetaException {
+ Metrics.initialize(conf);
+
+ initListeners = MetaStoreServerUtils.getMetaStoreListeners(
+ MetaStoreInitListener.class, conf, MetastoreConf.getVar(conf, MetastoreConf.ConfVars.INIT_HOOKS));
+ for (MetaStoreInitListener singleInitListener: initListeners) {
+ MetaStoreInitContext context = new MetaStoreInitContext();
+ singleInitListener.onInit(context);
+ }
+
+ String alterHandlerName = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.ALTER_HANDLER);
+ alterHandler = ReflectionUtils.newInstance(JavaUtils.getClass(
+ alterHandlerName, AlterHandler.class), conf);
+ this.wh = wh;
+
+ initDefaultSchema();
+
+ preListeners = MetaStoreServerUtils.getMetaStoreListeners(MetaStorePreEventListener.class,
+ conf, MetastoreConf.getVar(conf, MetastoreConf.ConfVars.PRE_EVENT_LISTENERS));
+ preListeners.add(0, new TransactionalValidationListener(conf));
+ listeners = MetaStoreServerUtils.getMetaStoreListeners(MetaStoreEventListener.class, conf,
+ MetastoreConf.getVar(conf, MetastoreConf.ConfVars.EVENT_LISTENERS));
+ listeners.add(new SessionPropertiesListener(conf));
+ transactionalListeners = new ArrayList<>() {{
+ add(new AcidEventListener(conf));
+ }};
+ transactionalListeners.addAll(MetaStoreServerUtils.getMetaStoreListeners(
+ TransactionalMetaStoreEventListener.class, conf,
+ MetastoreConf.getVar(conf, MetastoreConf.ConfVars.TRANSACTIONAL_EVENT_LISTENERS)));
+ if (Metrics.getRegistry() != null) {
+ listeners.add(new HMSMetricsListener(conf));
+ }
+
+ boolean canCachedStoreCanUseEvent = false;
+ for (MetaStoreEventListener listener : transactionalListeners) {
+ if (listener.doesAddEventsToNotificationLogTable()) {
+ canCachedStoreCanUseEvent = true;
+ break;
+ }
+ }
+ if (conf.getBoolean(MetastoreConf.ConfVars.METASTORE_CACHE_CAN_USE_EVENT.getVarname(), false) &&
+ !canCachedStoreCanUseEvent) {
+ throw new MetaException("CachedStore can not use events for invalidation as there is no " +
+ " TransactionalMetaStoreEventListener to add events to notification table");
+ }
+
+ endFunctionListeners = MetaStoreServerUtils.getMetaStoreListeners(
+ MetaStoreEndFunctionListener.class, conf,
+ MetastoreConf.getVar(conf, MetastoreConf.ConfVars.END_FUNCTION_LISTENERS));
+
+ expressionProxy = PartFilterExprUtil.createExpressionProxy(conf);
+ fileMetadataManager = new FileMetadataManager(this.getMS(), conf);
+
+ isServerFilterEnabled = getIfServerFilterenabled();
+ filterHook = isServerFilterEnabled ? loadFilterHooks() : null;
+
+ String className = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.METASTORE_METADATA_TRANSFORMER_CLASS);
+ if (className != null && !className.trim().isEmpty()) {
+ try {
+ transformer = JavaUtils.newInstance(JavaUtils.getClass(className.trim(), IMetaStoreMetadataTransformer.class),
+ new Class[] {IHMSHandler.class}, new Object[] {this});
+ } catch (Exception e) {
+ LOG.error("Unable to create instance of class {}", className, e);
+ throw new IllegalArgumentException(e);
+ }
+ }
+ }
+
+ private void initDefaultSchema() throws MetaException {
+ synchronized (BaseHandler.class) {
+ if (currentUrl == null || !currentUrl.equals(MetaStoreInit.getConnectionURL(conf))) {
+ createDefaultDB();
+ createDefaultRoles();
+ addAdminUsers();
+ currentUrl = MetaStoreInit.getConnectionURL(conf);
+ updateMetrics();
+ }
+
+ if (tablelocks == null) {
+ int numTableLocks = MetastoreConf.getIntVar(conf,
+ MetastoreConf.ConfVars.METASTORE_NUM_STRIPED_TABLE_LOCKS);
+ tablelocks = Striped.lock(numTableLocks);
+ }
+ }
+ }
+
+ /**
+ *
+ * Filter is actually enabled only when the configured filter hook is configured, not default, and
+ * enabled in configuration
+ * @return
+ */
+ private boolean getIfServerFilterenabled() throws MetaException{
+ boolean isEnabled = MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.METASTORE_SERVER_FILTER_ENABLED);
+ if (!isEnabled) {
+ LOG.info("HMS server filtering is disabled by configuration");
+ return false;
+ }
+ String filterHookClassName = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.FILTER_HOOK);
+
+ if (isBlank(filterHookClassName)) {
+ throw new MetaException("HMS server filtering is enabled but no filter hook is configured");
+ }
+ if (filterHookClassName.trim().equalsIgnoreCase(DefaultMetaStoreFilterHookImpl.class.getName())) {
+ throw new MetaException("HMS server filtering is enabled but the filter hook is DefaultMetaStoreFilterHookImpl,"
+ + " which does no filtering");
+ }
+ LOG.info("HMS server filtering is enabled. The filter class is {}", filterHookClassName);
+ return true;
+ }
+
+ private MetaStoreFilterHook loadFilterHooks() throws IllegalStateException {
+ String errorMsg = "Unable to load filter hook at HMS server. ";
+
+ String filterHookClassName = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.FILTER_HOOK);
+ Preconditions.checkState(!isBlank(filterHookClassName));
+
+ try {
+ return (MetaStoreFilterHook)Class.forName(
+ filterHookClassName.trim(), true, JavaUtils.getClassLoader()).getConstructor(
+ Configuration.class).newInstance(conf);
+ } catch (Exception e) {
+ LOG.error(errorMsg, e);
+ throw new IllegalStateException(errorMsg + e.getMessage(), e);
+ }
+ }
+
+ @Override
+ public void setConf(Configuration conf) {
+ HMSHandlerContext.setConfiguration(conf);
+ // reload if DS related configuration is changed
+ HMSHandlerContext.getRawStore().ifPresent(ms -> ms.setConf(conf));
+ }
+
+ @Override
+ public Configuration getConf() {
+ return HMSHandlerContext.getConfiguration()
+ .orElseGet(
+ () -> {
+ Configuration configuration = new Configuration(this.conf);
+ HMSHandlerContext.setConfiguration(configuration);
+ return configuration;
+ });
+ }
+
+ @Override
+ public Warehouse getWh() {
+ return wh;
+ }
+
+ @Override
+ public fb_status getStatus() {
+ return fb_status.ALIVE;
+ }
+
+ @Override
+ public AbstractMap<String, Long> getCounters() {
+ AbstractMap<String, Long> counters = super.getCounters();
+
+ // Allow endFunctionListeners to add any counters they have collected
+ if (endFunctionListeners != null) {
+ for (MetaStoreEndFunctionListener listener : endFunctionListeners) {
+ listener.exportCounters(counters);
+ }
+ }
+
+ return counters;
+ }
+
+ @Override
+ public TxnStore getTxnHandler() {
+ return HMSHandlerContext.getTxnStore(conf);
+ }
+
+ @Override
+ public String getVersion() throws TException {
+ return MetastoreVersionInfo.getVersion();
+ }
+
+ protected static void logAndAudit(final String m) {
+ LOG.debug(m);
+ logAuditEvent(m);
+ }
+
+ // This will return null if the metastore is not being accessed from a metastore Thrift server,
+ // or if the TTransport being used to connect is not an instance of TSocket, or if kerberos
+ // is used
+ public static String getThreadLocalIpAddress() {
+ return HMSHandlerContext.getIpAddress().orElse(null);
+ }
+
+ public static void setThreadLocalIpAddress(String ipAddress) {
+ HMSHandlerContext.setIpAddress(ipAddress);
+ }
+
+ protected String startFunction(String function, String extraLogInfo) {
+ incrementCounter(function);
+ logAndAudit((getThreadLocalIpAddress() == null ? "" : "source:" + getThreadLocalIpAddress() + " ") +
+ function + extraLogInfo);
+ com.codahale.metrics.Timer timer =
+ Metrics.getOrCreateTimer(MetricsConstants.API_PREFIX + function);
+ if (timer != null) {
+ // Timer will be null we aren't using the metrics
+ HMSHandlerContext.getTimerContexts().put(function, timer.time());
+ }
+ Counter counter = Metrics.getOrCreateCounter(MetricsConstants.ACTIVE_CALLS + function);
+ if (counter != null) {
+ counter.inc();
+ }
+ return function;
+ }
+
+ protected String startFunction(String function) {
+ return startFunction(function, "");
+ }
+
+ protected void startTableFunction(String function, String catName, String db, String tbl) {
+ startFunction(function, " : tbl=" +
+ TableName.getQualified(catName, db, tbl));
+ }
+
+ protected void startMultiTableFunction(String function, String db, List<String> tbls) {
+ String tableNames = join(tbls, ",");
+ startFunction(function, " : db=" + db + " tbls=" + tableNames);
+ }
+
+ protected void startPartitionFunction(String function, String cat, String db, String tbl,
+ List<String> partVals) {
+ startFunction(function, " : tbl=" +
+ TableName.getQualified(cat, db, tbl) + samplePartitionValues(partVals));
+ }
+
+ protected void startPartitionFunction(String function, String catName, String db, String tbl,
+ Map<String, String> partName) {
+ startFunction(function, " : tbl=" +
+ TableName.getQualified(catName, db, tbl) + " partition=" + partName);
+ }
+
+ protected void startPartitionFunction(String function, String catName, String db, String tbl, int maxParts) {
+ startFunction(function, " : tbl=" + TableName.getQualified(catName, db, tbl) + ": Max partitions =" + maxParts);
+ }
+
+ protected void startPartitionFunction(String function, String catName, String db, String tbl, int maxParts,
+ List<String> partVals) {
+ startFunction(function, " : tbl=" + TableName.getQualified(catName, db, tbl) + ": Max partitions =" + maxParts
+ + samplePartitionValues(partVals));
+ }
+
+ protected void startPartitionFunction(String function, String catName, String db, String tbl, int maxParts,
+ String filter) {
+ startFunction(function,
+ " : tbl=" + TableName.getQualified(catName, db, tbl) + ": Filter=" + filter + ": Max partitions ="
+ + maxParts);
+ }
+
+ protected void startPartitionFunction(String function, String catName, String db, String tbl, int maxParts,
+ String expression, String defaultPartitionName) {
+ startFunction(function, " : tbl=" + TableName.getQualified(catName, db, tbl) + ": Expression=" + expression
+ + ": Default partition name=" + defaultPartitionName + ": Max partitions=" + maxParts);
+ }
+
+ protected String getGroupsCountAndUsername(final String user_name, final List<String> group_names) {
+ return ". Number of groups= " + (group_names == null ? 0 : group_names.size()) + ", user name= " + user_name;
+ }
+
+ protected String samplePartitionValues(List<String> partVals) {
+ if (CollectionUtils.isEmpty(partVals)) {
+ return ": Partitions = []";
+ }
+ StringBuilder sb = new StringBuilder(": Number of Partitions = " + partVals.size());
+ sb.append(": Partitions = [");
+ if (partVals.size() > LOG_SAMPLE_PARTITIONS_MAX_SIZE) {
+ // extracting starting 2 values, and ending 2 values
+ sb.append(join(partVals.subList(0, LOG_SAMPLE_PARTITIONS_HALF_SIZE), LOG_SAMPLE_PARTITIONS_SEPARATOR));
+ sb.append(" .... ");
+ sb.append(join(partVals.subList(partVals.size() - LOG_SAMPLE_PARTITIONS_HALF_SIZE, partVals.size()),
+ LOG_SAMPLE_PARTITIONS_SEPARATOR));
+ } else {
+ sb.append(join(partVals, LOG_SAMPLE_PARTITIONS_SEPARATOR));
+ }
+ sb.append("]");
+ return sb.toString();
+ }
+
+ protected void endFunction(String function, boolean successful, Exception e) {
+ endFunction(function, successful, e, null);
+ }
+
+ protected void endFunction(String function, boolean successful, Exception e,
+ String inputTableName) {
+ endFunction(function, new MetaStoreEndFunctionContext(successful, e, inputTableName));
+ }
+
+ protected void endFunction(String function, MetaStoreEndFunctionContext context) {
+ com.codahale.metrics.Timer.Context timerContext = HMSHandlerContext.getTimerContexts().remove(function);
+ if (timerContext != null) {
+ long timeTaken = timerContext.stop();
+ LOG.debug((getThreadLocalIpAddress() == null ? "" : "source:" + getThreadLocalIpAddress() + " ") +
+ function + " time taken(ns): " + timeTaken);
+ }
+ Counter counter = Metrics.getOrCreateCounter(MetricsConstants.ACTIVE_CALLS + function);
+ if (counter != null) {
+ counter.dec();
+ }
+
+ for (MetaStoreEndFunctionListener listener : endFunctionListeners) {
+ listener.onEndFunction(function, context);
+ }
+ }
+
+ static final Logger auditLog = LoggerFactory.getLogger(HiveMetaStore.class.getName() + ".audit");
+
+ private static void logAuditEvent(String cmd) {
+ if (cmd == null) {
+ return;
+ }
+
+ UserGroupInformation ugi;
+ try {
+ ugi = SecurityUtils.getUGI();
+ } catch (Exception ex) {
+ throw new RuntimeException(ex);
+ }
+
+ String address = getIPAddress();
+ if (address == null) {
+ address = "unknown-ip-addr";
+ }
+
+ auditLog.info("ugi={} ip={} cmd={} ", ugi.getUserName(), address, cmd);
+ }
+
+ @Override
+ public void flushCache() throws TException {
+ getMS().flushCache();
+ }
+
+ @Override
+ public List<String> partition_name_to_vals(String part_name) throws TException {
+ if (part_name.isEmpty()) {
+ return Collections.emptyList();
+ }
+ LinkedHashMap<String, String> map = Warehouse.makeSpecFromName(part_name);
+ return new ArrayList<>(map.values());
+ }
+
+ @Override
+ public Map<String, String> partition_name_to_spec(String part_name) throws TException {
+ if (part_name.isEmpty()) {
+ return new HashMap<>();
+ }
+ return Warehouse.makeSpecFromName(part_name);
+ }
+
+ @VisibleForTesting
+ void updateMetrics() throws MetaException {
+ if (Metrics.getRegistry() != null &&
+ MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.INIT_METADATA_COUNT_ENABLED)) {
+ LOG.info("Begin calculating metadata count metrics.");
+ Metrics.getOrCreateGauge(MetricsConstants.TOTAL_TABLES).set(getMS().getTableCount());
+ Metrics.getOrCreateGauge(MetricsConstants.TOTAL_PARTITIONS).set(getMS().getPartitionCount());
+ Metrics.getOrCreateGauge(MetricsConstants.TOTAL_DATABASES).set(getMS().getDatabaseCount());
+ }
+ }
+
+ /**
+ * create default database if it doesn't exist.
+ *
+ * This is a potential contention when HiveServer2 using embedded metastore and Metastore
+ * Server try to concurrently invoke createDefaultDB. If one failed, JDOException was caught
+ * for one more time try, if failed again, simply ignored by warning, which meant another
+ * succeeds.
+ *
+ * @throws MetaException
+ */
+ private void createDefaultDB() throws MetaException {
+ try {
+ RawStore ms = getMS();
+ createDefaultCatalog(ms, wh);
+ createDefaultDB_core(ms);
+ } catch (JDOException e) {
+ LOG.warn("Retrying creating default database after error: " + e.getMessage(), e);
+ try {
+ RawStore ms = getMS();
+ createDefaultCatalog(ms, wh);
+ createDefaultDB_core(ms);
+ } catch (InvalidObjectException | InvalidOperationException e1) {
+ throw new MetaException(e1.getMessage());
+ }
+ } catch (InvalidObjectException|InvalidOperationException e) {
+ throw new MetaException(e.getMessage());
+ }
+ }
+
+ /**
+ * create default roles if they don't exist.
+ *
+ * This is a potential contention when HiveServer2 using embedded metastore and Metastore
+ * Server try to concurrently invoke createDefaultRoles. If one failed, JDOException was caught
+ * for one more time try, if failed again, simply ignored by warning, which meant another
+ * succeeds.
+ *
+ * @throws MetaException
+ */
+ private void createDefaultRoles() throws MetaException {
+ try {
+ createDefaultRoles_core();
+ } catch (JDOException e) {
+ LOG.warn("Retrying creating default roles after error: " + e.getMessage(), e);
+ createDefaultRoles_core();
+ }
+ }
+
+ private void createDefaultRoles_core() throws MetaException {
+
+ RawStore ms = getMS();
+ try {
+ ms.addRole(ADMIN, ADMIN);
+ } catch (InvalidObjectException e) {
+ LOG.debug(ADMIN +" role already exists",e);
+ } catch (NoSuchObjectException e) {
+ // This should never be thrown.
+ LOG.warn("Unexpected exception while adding " +ADMIN+" roles" , e);
+ }
+ LOG.info("Added "+ ADMIN+ " role in metastore");
+ try {
+ ms.addRole(PUBLIC, PUBLIC);
+ } catch (InvalidObjectException e) {
+ LOG.debug(PUBLIC + " role already exists",e);
+ } catch (NoSuchObjectException e) {
+ // This should never be thrown.
+ LOG.warn("Unexpected exception while adding "+PUBLIC +" roles" , e);
+ }
+ LOG.info("Added "+PUBLIC+ " role in metastore");
+ // now grant all privs to admin
+ PrivilegeBag privs = new PrivilegeBag();
+ privs.addToPrivileges(new HiveObjectPrivilege( new HiveObjectRef(HiveObjectType.GLOBAL, null,
+ null, null, null), ADMIN, PrincipalType.ROLE, new PrivilegeGrantInfo("All", 0, ADMIN,
+ PrincipalType.ROLE, true), "SQL"));
+ try {
+ ms.grantPrivileges(privs);
+ } catch (InvalidObjectException e) {
+ // Surprisingly these privs are already granted.
+ LOG.debug("Failed while granting global privs to admin", e);
+ } catch (NoSuchObjectException e) {
+ // Unlikely to be thrown.
+ LOG.warn("Failed while granting global privs to admin", e);
+ }
+ }
+
+ /**
+ * add admin users if they don't exist.
+ *
+ * This is a potential contention when HiveServer2 using embedded metastore and Metastore
+ * Server try to concurrently invoke addAdminUsers. If one failed, JDOException was caught for
+ * one more time try, if failed again, simply ignored by warning, which meant another succeeds.
+ *
+ * @throws MetaException
+ */
+ private void addAdminUsers() throws MetaException {
+ try {
+ addAdminUsers_core();
+ } catch (JDOException e) {
+ LOG.warn("Retrying adding admin users after error: {}", e.getMessage(), e);
+ addAdminUsers_core();
+ }
+ }
+
+ private void addAdminUsers_core() throws MetaException {
+
+ // now add pre-configured users to admin role
+ String userStr = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.USERS_IN_ADMIN_ROLE,"").trim();
+ if (userStr.isEmpty()) {
+ LOG.info("No user is added in admin role, since config is empty");
+ return;
+ }
+ // Since user names need to be valid unix user names, per IEEE Std 1003.1-2001 they cannot
+ // contain comma, so we can safely split above string on comma.
+
+ Iterator<String> users = Splitter.on(",").trimResults().omitEmptyStrings().split(userStr).iterator();
+ if (!users.hasNext()) {
+ LOG.info("No user is added in admin role, since config value {} is in incorrect format." +
+ " We accept comma separated list of users.", userStr);
+ return;
+ }
+ Role adminRole;
+ RawStore ms = getMS();
+ try {
+ adminRole = ms.getRole(ADMIN);
+ } catch (NoSuchObjectException e) {
+ LOG.error("Failed to retrieve just added admin role",e);
+ return;
+ }
+ while (users.hasNext()) {
+ String userName = users.next();
+ try {
+ ms.grantRole(adminRole, userName, PrincipalType.USER, ADMIN, PrincipalType.ROLE, true);
+ LOG.info("Added {} to admin role", userName);
+ } catch (NoSuchObjectException e) {
+ LOG.error("Failed to add {} in admin role", userName, e);
+ } catch (InvalidObjectException e) {
+ LOG.debug("{} already in admin role", userName, e);
+ }
+ }
+ }
+
+ private void createDefaultDB_core(RawStore ms) throws MetaException, InvalidObjectException {
+ try {
+ ms.getDatabase(DEFAULT_CATALOG_NAME, DEFAULT_DATABASE_NAME);
+ } catch (NoSuchObjectException e) {
+ LOG.info("Started creating a default database with name: {}", DEFAULT_DATABASE_NAME);
+ Database db = new Database(DEFAULT_DATABASE_NAME, DEFAULT_DATABASE_COMMENT,
+ wh.getDefaultDatabasePath(DEFAULT_DATABASE_NAME, true).toString(), null);
+ db.setOwnerName(PUBLIC);
+ db.setOwnerType(PrincipalType.ROLE);
+ db.setCatalogName(DEFAULT_CATALOG_NAME);
+ long time = System.currentTimeMillis() / 1000;
+ db.setCreateTime((int) time);
+ db.setType(DatabaseType.NATIVE);
+ ms.createDatabase(db);
+ LOG.info("Successfully created a default database with name: {}", DEFAULT_DATABASE_NAME);
+ }
+ }
+
+ @Override
+ public String getCpuProfile(int i) throws TException {
+ return "";
+ }
+
+ protected void throwUnsupportedExceptionIfRemoteDB(String dbName, String operationName) throws MetaException {
+ if (isDatabaseRemote(dbName)) {
+ throw new MetaException("Operation " + operationName + " not supported for REMOTE database " + dbName);
+ }
+ }
+
+ protected boolean isDatabaseRemote(String name) {
+ try {
+ String[] parsedDbName = parseDbName(name, conf);
+ Database db = get_database_core(parsedDbName[CAT_NAME], parsedDbName[DB_NAME]);
+ return MetaStoreUtils.isDatabaseRemote(db);
+ } catch (Exception e) {
+ return false;
+ }
+ }
+
+ @Override
+ public List<String> set_ugi(String username, List<String> groupNames) throws TException {
+ Collections.addAll(groupNames, username);
+ return groupNames;
+ }
+
+ @Override
+ public boolean partition_name_has_valid_characters(
+ List<String> part_vals, boolean throw_exception) throws TException {
+ startFunction("partition_name_has_valid_characters");
+ boolean ret;
+ Exception ex = null;
+ try {
+ if (throw_exception) {
+ MetaStoreServerUtils.validatePartitionNameCharacters(part_vals, getConf());
+ ret = true;
+ } else {
+ ret = MetaStoreServerUtils.partitionNameHasValidCharacters(part_vals, getConf());
+ }
+ } catch (Exception e) {
+ ex = e;
+ throw newMetaException(e);
+ } finally {
+ endFunction("partition_name_has_valid_characters", true, ex);
+ }
+ return ret;
+ }
+
+ @Override
+ public Map<String, Type> get_type_all(String name) throws MetaException {
+ // TODO Auto-generated method stub
+ throw new MetaException("Not yet implemented");
+ }
+
+ /**
+ * Returns the value of the given configuration variable name. If the
+ * configuration variable with the given name doesn't exist, or if there
+ * were an exception thrown while retrieving the variable, or if name is
+ * null, defaultValue is returned.
+ */
+ @Override
+ public String get_config_value(String name, String defaultValue)
+ throws TException {
+ startFunction("get_config_value", ": name=" + name + " defaultValue="
+ + defaultValue);
+ boolean success = false;
+ Exception ex = null;
+ try {
+ if (name == null) {
+ success = true;
+ return defaultValue;
+ }
+ // Allow only keys that start with hive.*, hdfs.*, mapred.* for security
+ // i.e. don't allow access to db password
+ if (!Pattern.matches("(hive|hdfs|mapred|metastore).*", name)) {
+ throw new ConfigValSecurityException("For security reasons, the "
+ + "config key " + name + " cannot be accessed");
+ }
+
+ String toReturn = defaultValue;
+ try {
+ toReturn = MetastoreConf.get(conf, name);
+ if (toReturn == null) {
+ toReturn = defaultValue;
+ }
+ } catch (RuntimeException e) {
+ LOG.error("RuntimeException thrown in get_config_value - msg: {} cause: {}",
+ e.getMessage(), e.getCause());
+ }
+ success = true;
+ return toReturn;
+ } catch (Exception e) {
+ ex = e;
+ throw handleException(e).defaultTException();
+ } finally {
+ endFunction("get_config_value", success, ex);
+ }
+ }
+
+ @Override
+ public String get_metastore_db_uuid() throws TException {
+ return getMS().getMetastoreDbUuid();
+ }
+
+ public Lock getTableLockFor(String dbName, String tblName) {
+ return tablelocks.get(dbName + "." + tblName);
+ }
+
+ @Override
+ public NotificationEventResponse get_next_notification(NotificationEventRequest rqst)
+ throws TException {
+ authorizeProxyPrivilege();
+
+ RawStore ms = getMS();
+ return ms.getNextNotification(rqst);
+ }
+
+ @Override
+ public CurrentNotificationEventId get_current_notificationEventId() throws TException {
+ authorizeProxyPrivilege();
+
+ RawStore ms = getMS();
+ return ms.getCurrentNotificationEventId();
+ }
+
+ @Override
+ public NotificationEventsCountResponse get_notification_events_count(NotificationEventsCountRequest rqst)
+ throws TException {
+ authorizeProxyPrivilege();
+
+ RawStore ms = getMS();
+ return ms.getNotificationEventsCount(rqst);
+ }
+
+ private void authorizeProxyPrivilege() throws TException {
+ // Skip the auth in embedded mode or if the auth is disabled
+ if (!HiveMetaStore.isMetaStoreRemote() ||
+ !MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.EVENT_DB_NOTIFICATION_API_AUTH) ||
+ conf.getBoolean(HIVE_IN_TEST.getVarname(), false)) {
+ return;
+ }
+ String user = null;
+ try {
+ user = SecurityUtils.getUGI().getShortUserName();
+ } catch (Exception ex) {
+ LOG.error("Cannot obtain username", ex);
+ throw new TException(ex);
+ }
+ if (!MetaStoreServerUtils.checkUserHasHostProxyPrivileges(user, conf, getIPAddress())) {
+ LOG.error("Not authorized to make the get_notification_events_count call. You can try to disable " +
+ MetastoreConf.ConfVars.EVENT_DB_NOTIFICATION_API_AUTH.toString());
+ throw new TException("User " + user + " is not allowed to perform this API call");
+ }
+ }
+
+ @Override
+ public CmRecycleResponse cm_recycle(final CmRecycleRequest request) throws MetaException {
+ wh.recycleDirToCmPath(new Path(request.getDataPath()), request.isPurge());
+ return new CmRecycleResponse();
+ }
+
+ /**
+ * filters out the table meta for which read database access is not granted
+ * @param catName catalog name
+ * @param tableMetas list of table metas
+ * @return filtered list of table metas
+ * @throws RuntimeException
+ * @throws NoSuchObjectException
+ */
+ protected List<TableMeta> filterReadableTables(String catName, List<TableMeta> tableMetas)
+ throws RuntimeException, NoSuchObjectException {
+ List<TableMeta> finalT = new ArrayList<>();
+ Map<String, Boolean> databaseNames = new HashMap();
+ for (TableMeta tableMeta : tableMetas) {
+ String fullDbName = prependCatalogToDbName(catName, tableMeta.getDbName(), conf);
+ if (databaseNames.get(fullDbName) == null) {
+ boolean isExecptionThrown = false;
+ try {
+ fireReadDatabasePreEvent(fullDbName);
+ } catch (MetaException e) {
+ isExecptionThrown = true;
+ }
+ databaseNames.put(fullDbName, isExecptionThrown);
+ }
+ if (!databaseNames.get(fullDbName)) {
+ finalT.add(tableMeta);
+ }
+ }
+ return finalT;
+ }
+
+ @Override
+ public void setMetaConf(String key, String value) throws MetaException {
+ MetastoreConf.ConfVars confVar = MetastoreConf.getMetaConf(key);
+ if (confVar == null) {
+ throw new MetaException("Invalid configuration key " + key);
+ }
+ try {
+ confVar.validate(value);
+ } catch (IllegalArgumentException e) {
+ throw new MetaException("Invalid configuration value " + value + " for key " + key +
+ " by " + e.getMessage());
+ }
+ Configuration configuration = getConf();
+ String oldValue = MetastoreConf.get(configuration, key);
+ // Save prev val of the key on threadLocal
+ Map<String, String> modifiedConf = HMSHandlerContext.getModifiedConfig();
+ if (!modifiedConf.containsKey(key)) {
+ modifiedConf.put(key, oldValue);
+ }
+ // Set invoking HMSHandler on threadLocal, this will be used later to notify
+ // metaListeners in HiveMetaStore#cleanupHandlerContext
+ if (!HMSHandlerContext.getHMSHandler().isPresent()) {
+ HMSHandlerContext.setHMSHandler(this);
+ }
+ configuration.set(key, value);
+ for (List<MetaStoreEventListener> eventListeners :
+ new List[] { listeners, transactionalListeners }) {
+ MetaStoreListenerNotifier.notifyEvent(eventListeners, EventMessage.EventType.CONFIG_CHANGE,
+ new ConfigChangeEvent(this, key, oldValue, value));
+ }
+ if (MetastoreConf.ConfVars.TRY_DIRECT_SQL == confVar) {
+ LOG.info("Direct SQL optimization = {}", value);
+ }
+ }
+
+ /**
+ * Internal function to notify listeners to revert back to old values of keys
+ * that were modified during setMetaConf. This would get called from HiveMetaStore#cleanupHandlerContext
+ */
+ public void notifyMetaListenersOnShutDown() {
+ try {
+ Map<String, String> modifiedConf = HMSHandlerContext.getModifiedConfig();
+ Configuration conf = HMSHandlerContext.getConfiguration()
+ .orElseThrow(() -> new MetaException("Unexpected: modifiedConf is non-null but conf is null"));
+ // Notify listeners of the changed value
+ for (Map.Entry<String, String> entry : modifiedConf.entrySet()) {
+ String key = entry.getKey();
+ // curr value becomes old and vice-versa
+ String currVal = entry.getValue();
+ String oldVal = conf.get(key);
+ if (!Objects.equals(oldVal, currVal)) {
+ for (List<MetaStoreEventListener> eventListeners :
+ new List[] { listeners, transactionalListeners }) {
+ MetaStoreListenerNotifier.notifyEvent(eventListeners, EventMessage.EventType.CONFIG_CHANGE,
+ new ConfigChangeEvent(this, key, oldVal, currVal));
+ }
+ }
+ }
+ logAndAudit("Meta listeners shutdown notification completed.");
+ } catch (MetaException e) {
+ LOG.error("Failed to notify meta listeners on shutdown: ", e);
+ }
+ }
+
+ @Override
+ public String getMetaConf(String key) throws MetaException {
+ MetastoreConf.ConfVars confVar = MetastoreConf.getMetaConf(key);
+ if (confVar == null) {
+ throw new MetaException("Invalid configuration key " + key);
+ }
+ return getConf().get(key, confVar.getDefaultVal().toString());
+ }
+
+ /**
+ * Fire a pre-event for read database operation, if there are any
+ * pre-event listeners registered
+ */
+ protected void fireReadDatabasePreEvent(final String name)
+ throws MetaException, RuntimeException, NoSuchObjectException {
+ if(preListeners.size() > 0) {
+ String[] parsedDbName = parseDbName(name, conf);
+ Database db = null;
+ try {
+ db = get_database_core(parsedDbName[CAT_NAME], parsedDbName[DB_NAME]);
+ if (db == null) {
+ throw new NoSuchObjectException("Database: " + name + " not found");
+ }
+ } catch(MetaException | NoSuchObjectException e) {
+ throw new RuntimeException(e);
+ }
+ firePreEvent(new PreReadDatabaseEvent(db, this));
+ }
+ }
+
+ public void firePreEvent(PreEventContext event) throws MetaException {
+ for (MetaStorePreEventListener listener : preListeners) {
+ try {
+ listener.onEvent(event);
+ } catch (NoSuchObjectException e) {
+ throw new MetaException(e.getMessage());
+ } catch (InvalidOperationException e) {
+ throw new MetaException(e.getMessage());
+ }
+ }
+ }
+}
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/handler/DeprecatedHandler.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/handler/DeprecatedHandler.java
new file mode 100644
index 00000000000..e25c4c31f1e
--- /dev/null
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/handler/DeprecatedHandler.java
@@ -0,0 +1,1244 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.handler;
+
+import com.google.common.collect.Lists;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.common.StatsSetupConst;
+import org.apache.hadoop.hive.common.TableName;
+import org.apache.hadoop.hive.metastore.MetaStoreListenerNotifier;
+import org.apache.hadoop.hive.metastore.RawStore;
+import org.apache.hadoop.hive.metastore.api.AddPartitionsRequest;
+import org.apache.hadoop.hive.metastore.api.AlreadyExistsException;
+import org.apache.hadoop.hive.metastore.api.AlterDatabaseRequest;
+import org.apache.hadoop.hive.metastore.api.AlterPartitionsRequest;
+import org.apache.hadoop.hive.metastore.api.AlterTableRequest;
+import org.apache.hadoop.hive.metastore.api.AppendPartitionsRequest;
+import org.apache.hadoop.hive.metastore.api.CacheFileMetadataRequest;
+import org.apache.hadoop.hive.metastore.api.CacheFileMetadataResult;
+import org.apache.hadoop.hive.metastore.api.ClearFileMetadataRequest;
+import org.apache.hadoop.hive.metastore.api.ClearFileMetadataResult;
+import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
+import org.apache.hadoop.hive.metastore.api.CreateDatabaseRequest;
+import org.apache.hadoop.hive.metastore.api.CreateTableRequest;
+import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.metastore.api.DeleteColumnStatisticsRequest;
+import org.apache.hadoop.hive.metastore.api.DropDatabaseRequest;
+import org.apache.hadoop.hive.metastore.api.DropPartitionRequest;
+import org.apache.hadoop.hive.metastore.api.DropTableRequest;
+import org.apache.hadoop.hive.metastore.api.EnvironmentContext;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.FileMetadataExprType;
+import org.apache.hadoop.hive.metastore.api.GetDatabaseRequest;
+import org.apache.hadoop.hive.metastore.api.GetFieldsRequest;
+import org.apache.hadoop.hive.metastore.api.GetFileMetadataByExprRequest;
+import org.apache.hadoop.hive.metastore.api.GetFileMetadataByExprResult;
+import org.apache.hadoop.hive.metastore.api.GetFileMetadataRequest;
+import org.apache.hadoop.hive.metastore.api.GetFileMetadataResult;
+import org.apache.hadoop.hive.metastore.api.GetPartitionNamesPsRequest;
+import org.apache.hadoop.hive.metastore.api.GetPartitionRequest;
+import org.apache.hadoop.hive.metastore.api.GetPartitionsByFilterRequest;
+import org.apache.hadoop.hive.metastore.api.GetPartitionsByNamesRequest;
+import org.apache.hadoop.hive.metastore.api.GetPartitionsPsWithAuthRequest;
+import org.apache.hadoop.hive.metastore.api.GetSchemaRequest;
+import org.apache.hadoop.hive.metastore.api.GetTableRequest;
+import org.apache.hadoop.hive.metastore.api.InvalidInputException;
+import org.apache.hadoop.hive.metastore.api.InvalidObjectException;
+import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.MetadataPpdResult;
+import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
+import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.PartitionListComposingSpec;
+import org.apache.hadoop.hive.metastore.api.PartitionSpec;
+import org.apache.hadoop.hive.metastore.api.PartitionsRequest;
+import org.apache.hadoop.hive.metastore.api.PutFileMetadataRequest;
+import org.apache.hadoop.hive.metastore.api.PutFileMetadataResult;
+import org.apache.hadoop.hive.metastore.api.SQLCheckConstraint;
+import org.apache.hadoop.hive.metastore.api.SQLDefaultConstraint;
+import org.apache.hadoop.hive.metastore.api.SQLForeignKey;
+import org.apache.hadoop.hive.metastore.api.SQLNotNullConstraint;
+import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey;
+import org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.metastore.api.TruncateTableRequest;
+import org.apache.hadoop.hive.metastore.api.UnknownDBException;
+import org.apache.hadoop.hive.metastore.api.UnknownTableException;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.hadoop.hive.metastore.events.AlterPartitionEvent;
+import org.apache.hadoop.hive.metastore.events.PreAlterPartitionEvent;
+import org.apache.hadoop.hive.metastore.messaging.EventMessage;
+import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils;
+import org.apache.thrift.TException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.hadoop.hive.metastore.ExceptionHandler.handleException;
+import static org.apache.hadoop.hive.metastore.ExceptionHandler.rethrowException;
+import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.CAT_NAME;
+import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.DB_NAME;
+import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.getDefaultCatalog;
+import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.parseDbName;
+import static org.apache.hadoop.hive.metastore.utils.StringUtils.normalizeIdentifier;
+
+/**
+ * This handler collects all deprecated APIs, those APIs are
+ * supposed to be dropped in the future release including the thrift definition.
+ */
+public abstract class DeprecatedHandler extends BaseHandler {
+ private static final Logger LOG = LoggerFactory.getLogger(DeprecatedHandler.class);
+
+ protected DeprecatedHandler(String name, Configuration conf) {
+ super(name, conf);
+ }
+
+ @Override
+ @Deprecated
+ public void create_database(final Database db) throws TException {
+ CreateDatabaseRequest req = new CreateDatabaseRequest(db.getName());
+ req.setDescription(db.getDescription());
+ req.setLocationUri(db.getLocationUri());
+ req.setParameters(db.getParameters());
+ req.setPrivileges(db.getPrivileges());
+ req.setOwnerName(db.getOwnerName());
+ req.setOwnerType(db.getOwnerType());
+ req.setCatalogName(db.getCatalogName());
+ req.setCreateTime(db.getCreateTime());
+ req.setManagedLocationUri(db.getManagedLocationUri());
+ req.setType(db.getType());
+ req.setDataConnectorName(db.getConnector_name());
+ req.setRemote_dbname(db.getRemote_dbname());
+
+ create_database_req(req);
+ //location and manged location might be set/changed.
+ db.setLocationUri(req.getLocationUri());
+ db.setManagedLocationUri(req.getManagedLocationUri());
+ }
+
+ @Override
+ @Deprecated
+ public Database get_database(final String name) throws TException {
+ GetDatabaseRequest request = new GetDatabaseRequest();
+ String[] parsedDbName = parseDbName(name, conf);
+ request.setName(parsedDbName[DB_NAME]);
+ if (parsedDbName[CAT_NAME] != null) {
+ request.setCatalogName(parsedDbName[CAT_NAME]);
+ }
+ return get_database_req(request);
+ }
+
+ @Override
+ @Deprecated
+ public void alter_database(final String dbName, final Database newDB) throws TException {
+ AlterDatabaseRequest alterDbReq = new AlterDatabaseRequest(dbName, newDB);
+ alter_database_req(alterDbReq);
+ }
+
+ @Override
+ @Deprecated
+ public void drop_database(final String dbName, final boolean deleteData, final boolean cascade)
+ throws TException {
+ String[] parsedDbName = parseDbName(dbName, conf);
+
+ DropDatabaseRequest req = new DropDatabaseRequest();
+ req.setName(parsedDbName[DB_NAME]);
+ req.setCatalogName(parsedDbName[CAT_NAME]);
+ req.setDeleteData(deleteData);
+ req.setCascade(cascade);
+ drop_database_req(req);
+ }
+
+ @Override
+ @Deprecated
+ public void create_table(final Table tbl) throws TException {
+ CreateTableRequest createTableReq = new CreateTableRequest(tbl);
+ create_table_req(createTableReq);
+ }
+
+ @Override
+ @Deprecated
+ public void create_table_with_environment_context(final Table tbl,
+ final EnvironmentContext envContext)
+ throws AlreadyExistsException, MetaException, InvalidObjectException,
+ InvalidInputException {
+ startFunction("create_table_with_environment_context", ": " + tbl.getTableName());
+ boolean success = false;
+ Exception ex = null;
+ try {
+ CreateTableRequest createTableReq = new CreateTableRequest(tbl);
+ if (envContext != null) {
+ createTableReq.setEnvContext(envContext);
+ }
+ create_table_req(createTableReq);
+ } catch (Exception e) {
+ ex = e;
+ throw handleException(e).throwIfInstance(MetaException.class, InvalidObjectException.class)
+ .throwIfInstance(AlreadyExistsException.class, InvalidInputException.class)
+ .defaultMetaException();
+ } finally {
+ endFunction("create_table_with_environment_context", success, ex, tbl.getTableName());
+ }
+ }
+
+ @Override
+ @Deprecated
+ public void create_table_with_constraints(final Table tbl,
+ final List<SQLPrimaryKey> primaryKeys, final List<SQLForeignKey> foreignKeys,
+ List<SQLUniqueConstraint> uniqueConstraints,
+ List<SQLNotNullConstraint> notNullConstraints,
+ List<SQLDefaultConstraint> defaultConstraints,
+ List<SQLCheckConstraint> checkConstraints)
+ throws AlreadyExistsException, MetaException, InvalidObjectException,
+ InvalidInputException {
+ startFunction("create_table_with_constraints", ": " + tbl.toString());
+ boolean success = false;
+ Exception ex = null;
+ try {
+ CreateTableRequest req = new CreateTableRequest(tbl);
+ req.setPrimaryKeys(primaryKeys);
+ req.setForeignKeys(foreignKeys);
+ req.setUniqueConstraints(uniqueConstraints);
+ req.setNotNullConstraints(notNullConstraints);
+ req.setDefaultConstraints(defaultConstraints);
+ req.setCheckConstraints(checkConstraints);
+ create_table_req(req);
+ success = true;
+ } catch (Exception e) {
+ ex = e;
+ throw handleException(e).throwIfInstance(MetaException.class, InvalidObjectException.class)
+ .throwIfInstance(AlreadyExistsException.class, InvalidInputException.class)
+ .defaultMetaException();
+ } finally {
+ endFunction("create_table_with_constraints", success, ex, tbl.getTableName());
+ }
+ }
+
+ @Override
+ @Deprecated
+ public void drop_table(final String dbname, final String name, final boolean deleteData) throws TException {
+ String[] parsedDbName = parseDbName(dbname, conf);
+ DropTableRequest dropTableReq = new DropTableRequest(parsedDbName[DB_NAME], name);
+ dropTableReq.setDeleteData(deleteData);
+ dropTableReq.setCatalogName(parsedDbName[CAT_NAME]);
+ dropTableReq.setDropPartitions(true);
+ drop_table_req(dropTableReq);
+ }
+
+ @Override
+ @Deprecated
+ public void drop_table_with_environment_context(final String dbname, final String name, final boolean deleteData,
+ final EnvironmentContext envContext) throws TException {
+ String[] parsedDbName = parseDbName(dbname, conf);
+ DropTableRequest dropTableReq = new DropTableRequest(parsedDbName[DB_NAME], name);
+ dropTableReq.setDeleteData(deleteData);
+ dropTableReq.setCatalogName(parsedDbName[CAT_NAME]);
+ dropTableReq.setDropPartitions(true);
+ dropTableReq.setEnvContext(envContext);
+ drop_table_req(dropTableReq);
+ }
+
+ @Deprecated
+ @Override
+ public void truncate_table(final String dbName, final String tableName, List<String> partNames) throws TException {
+ // Deprecated path, won't work for txn tables.
+ TruncateTableRequest truncateTableReq = new TruncateTableRequest(dbName, tableName);
+ truncateTableReq.setPartNames(partNames);
+ truncate_table_req(truncateTableReq);
+ }
+
+ @Override
+ @Deprecated
+ public Partition append_partition(final String dbName, final String tableName,
+ final List<String> part_vals) throws TException {
+ String[] parsedDbName = parseDbName(dbName, conf);
+ AppendPartitionsRequest appendPartitionsReq = new AppendPartitionsRequest();
+ appendPartitionsReq.setDbName(parsedDbName[DB_NAME]);
+ appendPartitionsReq.setTableName(tableName);
+ appendPartitionsReq.setPartVals(part_vals);
+ appendPartitionsReq.setCatalogName(parsedDbName[CAT_NAME]);
+ return append_partition_req(appendPartitionsReq);
+ }
+
+ @Override
+ @Deprecated
+ public Partition append_partition_with_environment_context(final String dbName,
+ final String tableName, final List<String> part_vals, final EnvironmentContext envContext) throws TException {
+ String[] parsedDbName = parseDbName(dbName, conf);
+ AppendPartitionsRequest appendPartitionsReq = new AppendPartitionsRequest();
+ appendPartitionsReq.setDbName(parsedDbName[DB_NAME]);
+ appendPartitionsReq.setTableName(tableName);
+ appendPartitionsReq.setPartVals(part_vals);
+ appendPartitionsReq.setCatalogName(parsedDbName[CAT_NAME]);
+ appendPartitionsReq.setEnvironmentContext(envContext);
+ return append_partition_req(appendPartitionsReq);
+ }
+
+ @Deprecated
+ @Override
+ public int add_partitions(final List<Partition> parts) throws MetaException,
+ InvalidObjectException, AlreadyExistsException {
+ if (parts == null) {
+ throw new MetaException("Partition list cannot be null.");
+ }
+ if (parts.isEmpty()) {
+ return 0;
+ }
+ String catName = parts.get(0).isSetCatName() ? parts.get(0).getCatName() : getDefaultCatalog(conf);
+ String dbName = parts.get(0).getDbName();
+ String tableName = parts.get(0).getTableName();
+ startTableFunction("add_partitions", catName, dbName, tableName);
+
+ Integer ret = null;
+ Exception ex = null;
+ try {
+ // Old API assumed all partitions belong to the same table; keep the same assumption
+ if (!parts.get(0).isSetCatName()) {
+ parts.forEach(p -> p.setCatName(catName));
+ }
+ AddPartitionsRequest addPartitionsReq = new AddPartitionsRequest();
+ addPartitionsReq.setDbName(parts.get(0).getDbName());
+ addPartitionsReq.setTblName(parts.get(0).getTableName());
+ addPartitionsReq.setParts(parts);
+ addPartitionsReq.setIfNotExists(false);
+ if (parts.get(0).isSetCatName()) {
+ addPartitionsReq.setCatName(parts.get(0).getCatName());
+ }
+ ret = add_partitions_req(addPartitionsReq).getPartitions().size();
+ assert ret == parts.size();
+ } catch (Exception e) {
+ ex = e;
+ throw handleException(e)
+ .throwIfInstance(MetaException.class, InvalidObjectException.class, AlreadyExistsException.class)
+ .defaultMetaException();
+ } finally {
+ endFunction("add_partitions", ret != null, ex, tableName);
+ }
+ return ret;
+ }
+
+ @Deprecated
+ @Override
+ public Partition add_partition(final Partition part)
+ throws InvalidObjectException, AlreadyExistsException, MetaException {
+ return add_partition_with_environment_context(part, null);
+ }
+
+ @Deprecated
+ @Override
+ public Partition add_partition_with_environment_context( final Partition part, EnvironmentContext envContext)
+ throws InvalidObjectException, AlreadyExistsException, MetaException {
+ if (part == null) {
+ throw new MetaException("Partition cannot be null.");
+ }
+ AddPartitionsRequest addPartitionsReq = new AddPartitionsRequest(part.getDbName(), part.getTableName(),
+ new ArrayList<>(Arrays.asList(part)), false);
+ addPartitionsReq.setEnvironmentContext(envContext);
+ try {
+ return add_partitions_req(addPartitionsReq).getPartitions().get(0);
+ } catch (Exception e) {
+ throw handleException(e).throwIfInstance(MetaException.class, InvalidObjectException.class,
+ AlreadyExistsException.class).defaultMetaException();
+ }
+ }
+
+ @Deprecated
+ @Override
+ public boolean drop_partition(final String db_name, final String tbl_name,
+ final List<String> part_vals, final boolean deleteData)
+ throws TException {
+ return drop_partition_with_environment_context(db_name, tbl_name, part_vals, deleteData,
+ null);
+ }
+
+
+ @Deprecated
+ @Override
+ public boolean drop_partition_with_environment_context(final String db_name, final String tbl_name, final List<String> part_vals,
+ final boolean deleteData, final EnvironmentContext envContext) throws TException {
+ String[] parsedDbName = parseDbName(db_name, conf);
+ DropPartitionRequest dropPartitionReq = new DropPartitionRequest(parsedDbName[DB_NAME], tbl_name);
+ dropPartitionReq.setCatName(parsedDbName[CAT_NAME]);
+ dropPartitionReq.setPartVals(part_vals);
+ dropPartitionReq.setDeleteData(deleteData);
+ dropPartitionReq.setEnvironmentContext(envContext);
+ return drop_partition_req(dropPartitionReq);
+ }
+
+ /**
+ * Use {@link #get_partition_req(GetPartitionRequest)} ()} instead.
+ *
+ */
+ @Override
+ @Deprecated
+ public Partition get_partition(final String db_name, final String tbl_name,
+ final List<String> part_vals) throws MetaException, NoSuchObjectException {
+ try {
+ String[] parsedDbName = parseDbName(db_name, conf);
+ GetPartitionRequest getPartitionRequest = new GetPartitionRequest(parsedDbName[DB_NAME], tbl_name, part_vals);
+ getPartitionRequest.setCatName(parsedDbName[CAT_NAME]);
+ return get_partition_req(getPartitionRequest).getPartition();
+ } catch (TException e) {
+ throw handleException(e).throwIfInstance(MetaException.class, NoSuchObjectException.class)
+ .defaultMetaException();
+ }
+ }
+
+ @Override
+ @Deprecated
+ public Partition get_partition_with_auth(final String db_name,
+ final String tbl_name, final List<String> part_vals,
+ final String user_name, final List<String> group_names)
+ throws TException {
+ String[] parsedDbName = parseDbName(db_name, conf);
+ TableName tableName = new TableName(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name);
+ String partName = GetPartitionsHandler.validatePartVals(this, tableName, part_vals);
+ GetPartitionsPsWithAuthRequest gpar = new GetPartitionsPsWithAuthRequest(tableName.getDb(), tableName.getTable());
+ gpar.setCatName(tableName.getCat());
+ gpar.setUserName(user_name);
+ gpar.setGroupNames(group_names);
+ gpar.setPartNames(List.of(partName));
+ List<Partition> partitions = GetPartitionsHandler.getPartitions(
+ t -> startFunction("get_partition_with_auth",
+ " : tbl=" + t + samplePartitionValues(part_vals) + getGroupsCountAndUsername(user_name,group_names)),
+ rex -> endFunction("get_partition_with_auth",
+ rex.getLeft() != null && rex.getLeft().success(), rex.getRight(), tbl_name),
+ this, tableName, gpar, true);
+ return partitions.getFirst();
+ }
+
+ /**
+ * Use {@link #get_partitions_req(PartitionsRequest)} ()} instead.
+ *
+ */
+ @Override
+ @Deprecated
+ public List<Partition> get_partitions(final String db_name, final String tbl_name,
+ final short max_parts) throws NoSuchObjectException, MetaException {
+ String[] parsedDbName = parseDbName(db_name, conf);
+ TableName tableName = new TableName(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name);
+ return GetPartitionsHandler.getPartitions(
+ t -> startTableFunction("get_partitions", tableName.getCat(), tableName.getDb(), tableName.getTable()),
+ rex -> endFunction("get_partitions",
+ rex.getLeft() != null && rex.getLeft().success(), rex.getRight(), tableName.toString()),
+ this, tableName, GetPartitionsHandler.createPartitionsRequest(tableName, max_parts), false);
+ }
+
+ @Override
+ @Deprecated
+ public List<Partition> get_partitions_with_auth(final String dbName,
+ final String tblName, final short maxParts, final String userName,
+ final List<String> groupNames) throws TException {
+ String[] parsedDbName = parseDbName(dbName, conf);
+ TableName tableName = new TableName(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tblName);
+ GetPartitionsPsWithAuthRequest getAuthReq = new GetPartitionsPsWithAuthRequest(tableName.getDb(), tableName.getTable());
+ getAuthReq.setCatName(tableName.getCat());
+ getAuthReq.setUserName(userName);
+ getAuthReq.setGroupNames(groupNames);
+ getAuthReq.setMaxParts(maxParts);
+ return get_partitions_ps_with_auth_req(getAuthReq).getPartitions();
+ }
+
+ @Override
+ @Deprecated
+ public List<PartitionSpec> get_partitions_pspec(final String db_name, final String tbl_name, final int max_parts)
+ throws NoSuchObjectException, MetaException {
+
+ String[] parsedDbName = parseDbName(db_name, conf);
+ String catName = parsedDbName[CAT_NAME];
+ String dbName = parsedDbName[DB_NAME];
+ String tableName = tbl_name.toLowerCase();
+
+ startPartitionFunction("get_partitions_pspec", catName, dbName, tableName, max_parts);
+
+ List<PartitionSpec> partitionSpecs = null;
+ try {
+ GetTableRequest getTableRequest = new GetTableRequest(dbName, tableName);
+ getTableRequest.setCatName(catName);
+ Table table = get_table_core(getTableRequest);
+ // get_partitions will parse out the catalog and db names itself
+ TableName t = new TableName(catName, dbName, tableName);
+ GetPartitionsHandler<PartitionsRequest, Partition> getPartitionsHandler = AbstractRequestHandler.offer(this,
+ new GetPartitionsHandler.GetPartitionsRequest<>(GetPartitionsHandler.createPartitionsRequest(t, max_parts), t));
+ List<Partition> partitions = getPartitionsHandler.getResult().result();
+ if (is_partition_spec_grouping_enabled(table)) {
+ partitionSpecs = MetaStoreServerUtils
+ .getPartitionspecsGroupedByStorageDescriptor(table, partitions);
+ }
+ else {
+ PartitionSpec pSpec = new PartitionSpec();
+ pSpec.setPartitionList(new PartitionListComposingSpec(partitions));
+ pSpec.setCatName(normalizeIdentifier(catName));
+ pSpec.setDbName(normalizeIdentifier(dbName));
+ pSpec.setTableName(tableName);
+ pSpec.setRootPath(table.getSd().getLocation());
+ partitionSpecs = Arrays.asList(pSpec);
+ }
+
+ return partitionSpecs;
+ } catch (Exception e) {
+ throw handleException(e).throwIfInstance(NoSuchObjectException.class, MetaException.class).defaultMetaException();
+ }
+ finally {
+ endFunction("get_partitions_pspec", partitionSpecs != null && !partitionSpecs.isEmpty(), null, tbl_name);
+ }
+ }
+
+ @Override
+ @Deprecated
+ public List<String> get_partition_names(final String db_name, final String tbl_name,
+ final short max_parts) throws TException {
+ String[] parsedDbName = parseDbName(db_name, conf);
+ PartitionsRequest partitionReq = new PartitionsRequest(parsedDbName[DB_NAME], tbl_name);
+ partitionReq.setCatName(parsedDbName[CAT_NAME]);
+ partitionReq.setMaxParts(max_parts);
+ return fetch_partition_names_req(partitionReq);
+ }
+
+ @Deprecated
+ @Override
+ public void alter_partition(final String db_name, final String tbl_name,
+ final Partition new_part)
+ throws TException {
+ rename_partition(db_name, tbl_name, null, new_part);
+ }
+
+ @Deprecated
+ @Override
+ public void alter_partition_with_environment_context(final String dbName,
+ final String tableName, final Partition newPartition,
+ final EnvironmentContext envContext)
+ throws TException {
+ String[] parsedDbName = parseDbName(dbName, conf);
+ alter_partition_core(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableName, null,
+ newPartition, envContext, null);
+ }
+
+ @Deprecated
+ @Override
+ public void rename_partition(final String db_name, final String tbl_name,
+ final List<String> part_vals, final Partition new_part)
+ throws TException {
+ // Call alter_partition_core without an environment context.
+ String[] parsedDbName = parseDbName(db_name, conf);
+ alter_partition_core(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name, part_vals, new_part,
+ null, null);
+ }
+
+ protected void alter_partition_core(String catName, String db_name, String tbl_name,
+ List<String> part_vals, Partition new_part, EnvironmentContext envContext,
+ String validWriteIds) throws TException {
+ startTableFunction("alter_partition", catName, db_name, tbl_name);
+
+ if (LOG.isInfoEnabled()) {
+ LOG.info("New partition values:" + new_part.getValues());
+ if (part_vals != null && part_vals.size() > 0) {
+ LOG.info("Old Partition values:" + part_vals);
+ }
+ }
+
+ // Adds the missing scheme/authority for the new partition location
+ if (new_part.getSd() != null) {
+ String newLocation = new_part.getSd().getLocation();
+ if (org.apache.commons.lang3.StringUtils.isNotEmpty(newLocation)) {
+ Path tblPath = wh.getDnsPath(new Path(newLocation));
+ new_part.getSd().setLocation(tblPath.toString());
+ }
+ }
+
+ // Make sure the new partition has the catalog value set
+ if (!new_part.isSetCatName()) {
+ new_part.setCatName(catName);
+ }
+
+ Partition oldPart = null;
+ Exception ex = null;
+ try {
+ Table table = getMS().getTable(catName, db_name, tbl_name, null);
+
+ firePreEvent(new PreAlterPartitionEvent(db_name, tbl_name, table, part_vals, new_part, this));
+ if (part_vals != null && !part_vals.isEmpty()) {
+ MetaStoreServerUtils.validatePartitionNameCharacters(new_part.getValues(), getConf());
+ }
+
+ oldPart = alterHandler.alterPartition(getMS(), wh, catName, db_name, tbl_name,
+ part_vals, new_part, envContext, this, validWriteIds);
+
+ if (!listeners.isEmpty()) {
+ MetaStoreListenerNotifier.notifyEvent(listeners,
+ EventMessage.EventType.ALTER_PARTITION,
+ new AlterPartitionEvent(oldPart, new_part, table, false,
+ true, new_part.getWriteId(), this),
+ envContext);
+ }
+ } catch (Exception e) {
+ ex = e;
+ throw handleException(e).throwIfInstance(MetaException.class, InvalidOperationException.class)
+ .convertIfInstance(InvalidObjectException.class, InvalidOperationException.class)
+ .convertIfInstance(AlreadyExistsException.class, InvalidOperationException.class)
+ .defaultMetaException();
+ } finally {
+ endFunction("alter_partition", oldPart != null, ex, tbl_name);
+ }
+ }
+
+ @Deprecated
+ @Override
+ public void alter_partitions(final String db_name, final String tbl_name,
+ final List<Partition> new_parts)
+ throws TException {
+ String[] o = parseDbName(db_name, conf);
+ AlterPartitionsRequest req = new AlterPartitionsRequest(o[1], tbl_name, new_parts);
+ req.setCatName(o[0]);
+ alter_partitions_req(req);
+ }
+
+ // The old API we are keeping for backward compat. Not used within Hive.
+ @Deprecated
+ @Override
+ public void alter_partitions_with_environment_context(final String db_name, final String tbl_name,
+ final List<Partition> new_parts, EnvironmentContext environmentContext)
+ throws TException {
+ String[] o = parseDbName(db_name, conf);
+ AlterPartitionsRequest req = new AlterPartitionsRequest(o[1], tbl_name, new_parts);
+ req.setCatName(o[0]);
+ req.setEnvironmentContext(environmentContext);
+ alter_partitions_req(req);
+ }
+
+ @Deprecated
+ @Override
+ public void alter_table(final String dbname, final String name,
+ final Table newTable) throws TException {
+ // Do not set an environment context.
+ String[] parsedDbName = parseDbName(dbname, conf);
+ AlterTableRequest req = new AlterTableRequest(parsedDbName[DB_NAME], name, newTable);
+ req.setCatName(parsedDbName[CAT_NAME]);
+ alter_table_req(req);
+ }
+
+ @Deprecated
+ @Override
+ public void alter_table_with_cascade(final String dbname, final String name,
+ final Table newTable, final boolean cascade) throws TException {
+ EnvironmentContext envContext = null;
+ if (cascade) {
+ envContext = new EnvironmentContext();
+ envContext.putToProperties(StatsSetupConst.CASCADE, StatsSetupConst.TRUE);
+ }
+ String[] parsedDbName = parseDbName(dbname, conf);
+ AlterTableRequest req = new AlterTableRequest(parsedDbName[DB_NAME], name, newTable);
+ req.setCatName(parsedDbName[CAT_NAME]);
+ req.setEnvironmentContext(envContext);
+ alter_table_req(req);
+ }
+
+ @Deprecated
+ @Override
+ public void alter_table_with_environment_context(final String dbname,
+ final String name, final Table newTable,
+ final EnvironmentContext envContext) throws TException {
+ String[] parsedDbName = parseDbName(dbname, conf);
+ AlterTableRequest req = new AlterTableRequest(parsedDbName[DB_NAME], name, newTable);
+ req.setCatName(parsedDbName[CAT_NAME]);
+ req.setEnvironmentContext(envContext);
+ alter_table_req(req);
+ }
+
+ /**
+ * Use {@link #get_fields_req(GetFieldsRequest)} ()} instead.
+ *
+ */
+ @Override
+ @Deprecated
+ public List<FieldSchema> get_fields(String db, String tableName) throws TException {
+ return get_fields_with_environment_context(db, tableName, null);
+ }
+
+ @Override
+ @Deprecated
+ public List<FieldSchema> get_fields_with_environment_context(String db, String tableName,
+ final EnvironmentContext envContext) throws TException {
+ String[] parsedDbName = parseDbName(db, conf);
+ GetFieldsRequest req = new GetFieldsRequest(parsedDbName[DB_NAME], tableName);
+ req.setCatName(parsedDbName[CAT_NAME]);
+ req.setEnvContext(envContext);
+ return get_fields_req(req).getFields();
+ }
+
+
+ /**
+ * Use {@link #get_schema_req(GetSchemaRequest)} ()} instead.
+ *
+ */
+ @Override
+ @Deprecated
+ public List<FieldSchema> get_schema(String db, String tableName) throws TException {
+ return get_schema_with_environment_context(db,tableName, null);
+ }
+
+ /**
+ * Return the schema of the table. This function includes partition columns
+ * in addition to the regular columns.
+ *
+ * @param db
+ * Name of the database
+ * @param tableName
+ * Name of the table
+ * @param envContext
+ * Store session based properties
+ * @return List of columns, each column is a FieldSchema structure
+ * @throws MetaException
+ * @throws UnknownTableException
+ * @throws UnknownDBException
+ */
+ @Override
+ @Deprecated
+ public List<FieldSchema> get_schema_with_environment_context(String db, String tableName,
+ final EnvironmentContext envContext) throws TException {
+ String[] parsedDbName = parseDbName(db, conf);
+ GetSchemaRequest req = new GetSchemaRequest(parsedDbName[DB_NAME], tableName);
+ req.setCatName(parsedDbName[CAT_NAME]);
+ req.setEnvContext(envContext);
+ return get_schema_req(req).getFields();
+ }
+
+ @Override
+ @Deprecated
+ public Partition get_partition_by_name(final String db_name, final String tbl_name,
+ final String part_name) throws TException {
+ if (StringUtils.isBlank(part_name)) {
+ throw new MetaException("The part_name in get_partition_by_name cannot be null or empty");
+ }
+ String[] parsedDbName = parseDbName(db_name, conf);
+ TableName tableName = new TableName(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name);
+ GetPartitionsByNamesRequest gpnr = new GetPartitionsByNamesRequest(tableName.getDb(), tableName.getTable());
+ gpnr.setNames(List.of(part_name));
+ List<Partition> partitions = GetPartitionsHandler.getPartitions(
+ t -> startFunction("get_partition_by_name", ": tbl=" + t + " part=" + part_name),
+ rex -> endFunction("get_partition_by_name",
+ rex.getLeft() != null && rex.getLeft().success(), rex.getRight(), tableName.toString()),
+ this, tableName, gpnr, true);
+ return partitions.getFirst();
+ }
+
+ @Deprecated
+ @Override
+ public Partition append_partition_by_name(final String db_name, final String tbl_name,
+ final String part_name) throws TException {
+ return append_partition_by_name_with_environment_context(db_name, tbl_name, part_name, null);
+ }
+
+ @Deprecated
+ @Override
+ public Partition append_partition_by_name_with_environment_context(final String db_name,
+ final String tbl_name, final String part_name, final EnvironmentContext env_context)
+ throws TException {
+ String[] parsedDbName = parseDbName(db_name, conf);
+ AppendPartitionsRequest appendPartitionRequest = new AppendPartitionsRequest();
+ appendPartitionRequest.setDbName(parsedDbName[DB_NAME]);
+ appendPartitionRequest.setTableName(tbl_name);
+ appendPartitionRequest.setName(part_name);
+ appendPartitionRequest.setCatalogName(parsedDbName[CAT_NAME]);
+ appendPartitionRequest.setEnvironmentContext(env_context);
+ return append_partition_req(appendPartitionRequest);
+ }
+
+ @Deprecated
+ @Override
+ public boolean drop_partition_by_name(final String db_name, final String tbl_name,
+ final String part_name, final boolean deleteData) throws TException {
+ return drop_partition_by_name_with_environment_context(db_name, tbl_name, part_name,
+ deleteData, null);
+ }
+
+ @Deprecated
+ @Override
+ public boolean drop_partition_by_name_with_environment_context(final String db_name,
+ final String tbl_name, final String part_name, final boolean deleteData,
+ final EnvironmentContext envContext) throws TException {
+ String[] parsedDbName = parseDbName(db_name, conf);
+ DropPartitionRequest dropPartitionReq = new DropPartitionRequest(parsedDbName[DB_NAME], tbl_name);
+ dropPartitionReq.setCatName(parsedDbName[CAT_NAME]);
+ dropPartitionReq.setPartName(part_name);
+ dropPartitionReq.setDeleteData(deleteData);
+ dropPartitionReq.setEnvironmentContext(envContext);
+ return drop_partition_req(dropPartitionReq);
+ }
+
+ @Override
+ @Deprecated
+ public List<Partition> get_partitions_ps(final String db_name,
+ final String tbl_name, final List<String> part_vals,
+ final short max_parts) throws TException {
+ return get_partitions_ps_with_auth(db_name, tbl_name, part_vals, max_parts, null, null);
+ }
+
+ /**
+ * Use {@link #get_partitions_ps_with_auth_req(GetPartitionsPsWithAuthRequest)} ()} instead.
+ *
+ */
+ @Override
+ @Deprecated
+ public List<Partition> get_partitions_ps_with_auth(final String db_name,
+ final String tbl_name, final List<String> part_vals,
+ final short max_parts, final String userName,
+ final List<String> groupNames) throws TException {
+ String[] parsedDbName = parseDbName(db_name, conf);
+ TableName tableName = new TableName(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name);
+ GetPartitionsPsWithAuthRequest getAuthReq = new GetPartitionsPsWithAuthRequest(tableName.getDb(), tableName.getTable());
+ getAuthReq.setCatName(tableName.getCat());
+ getAuthReq.setMaxParts(max_parts);
+ getAuthReq.setUserName(userName);
+ getAuthReq.setGroupNames(groupNames);
+ getAuthReq.setPartVals(part_vals);
+ return get_partitions_ps_with_auth_req(getAuthReq).getPartitions();
+ }
+
+ /**
+ * Use {@link #get_partition_names_ps_req(GetPartitionNamesPsRequest)} ()} instead.
+ *
+ */
+ @Override
+ @Deprecated
+ public List<String> get_partition_names_ps(final String db_name,
+ final String tbl_name, final List<String> part_vals, final short max_parts)
+ throws TException {
+ String[] parsedDbName = parseDbName(db_name, conf);
+ startPartitionFunction("get_partitions_names_ps", parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name,
+ max_parts, part_vals);
+
+ List<String> ret = null;
+ Exception ex = null;
+ try {
+ GetPartitionNamesPsRequest getPartitionNamesPsRequest = new GetPartitionNamesPsRequest(parsedDbName[DB_NAME], tbl_name);
+ getPartitionNamesPsRequest.setCatName(parsedDbName[CAT_NAME]);
+ getPartitionNamesPsRequest.setPartValues(part_vals);
+ getPartitionNamesPsRequest.setMaxParts(max_parts);
+ ret = get_partition_names_ps_req(getPartitionNamesPsRequest).getNames();
+ } catch (Exception e) {
+ ex = e;
+ rethrowException(e);
+ } finally {
+ endFunction("get_partitions_names_ps", ret != null, ex, tbl_name);
+ }
+ return ret;
+ }
+
+ @Override
+ @Deprecated
+ public ColumnStatistics get_table_column_statistics(String dbName, String tableName,
+ String colName) throws TException {
+ String[] parsedDbName = parseDbName(dbName, conf);
+ parsedDbName[CAT_NAME] = parsedDbName[CAT_NAME].toLowerCase();
+ parsedDbName[DB_NAME] = parsedDbName[DB_NAME].toLowerCase();
+ tableName = tableName.toLowerCase();
+ colName = colName.toLowerCase();
+ startFunction("get_column_statistics_by_table", ": table=" +
+ TableName.getQualified(parsedDbName[CAT_NAME], parsedDbName[DB_NAME],
+ tableName) + " column=" + colName);
+ ColumnStatistics statsObj = null;
+ try {
+ statsObj = getMS().getTableColumnStatistics(
+ parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableName, Lists.newArrayList(colName),
+ "hive", null);
+ if (statsObj != null) {
+ assert statsObj.getStatsObjSize() <= 1;
+ }
+ return statsObj;
+ } finally {
+ endFunction("get_column_statistics_by_table", statsObj != null, null, tableName);
+ }
+ }
+
+ @Override
+ @Deprecated
+ public ColumnStatistics get_partition_column_statistics(String dbName, String tableName,
+ String partName, String colName) throws TException {
+ // Note: this method appears to be unused within Hive.
+ // It doesn't take txn stats into account.
+ dbName = dbName.toLowerCase();
+ String[] parsedDbName = parseDbName(dbName, conf);
+ tableName = tableName.toLowerCase();
+ colName = colName.toLowerCase();
+ startFunction("get_column_statistics_by_partition", ": table=" +
+ TableName.getQualified(parsedDbName[CAT_NAME], parsedDbName[DB_NAME],
+ tableName) + " partition=" + partName + " column=" + colName);
+ ColumnStatistics statsObj = null;
+
+ try {
+ List<ColumnStatistics> list = getMS().getPartitionColumnStatistics(
+ parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableName,
+ Lists.newArrayList(partName), Lists.newArrayList(colName),
+ "hive");
+ if (list.isEmpty()) {
+ return null;
+ }
+ if (list.size() != 1) {
+ throw new MetaException(list.size() + " statistics for single column and partition");
+ }
+ statsObj = list.get(0);
+ } finally {
+ endFunction("get_column_statistics_by_partition", statsObj != null, null, tableName);
+ }
+ return statsObj;
+ }
+
+ @Deprecated
+ @Override
+ public boolean delete_partition_column_statistics(String dbName, String tableName,
+ String partName, String colName, String engine) throws TException {
+ dbName = dbName.toLowerCase();
+ String[] parsedDbName = parseDbName(dbName, conf);
+ tableName = tableName.toLowerCase();
+ DeleteColumnStatisticsRequest request = new DeleteColumnStatisticsRequest(parsedDbName[DB_NAME], tableName);
+ if (colName != null) {
+ colName = colName.toLowerCase();
+ request.addToCol_names(colName);
+ }
+ request.setEngine(engine);
+ request.setCat_name(parsedDbName[CAT_NAME]);
+ if (partName != null) {
+ request.addToPart_names(partName);
+ }
+ return delete_column_statistics_req(request);
+ }
+
+ @Deprecated
+ @Override
+ public boolean delete_table_column_statistics(String dbName, String tableName, String colName, String engine)
+ throws TException {
+ dbName = dbName.toLowerCase();
+ tableName = tableName.toLowerCase();
+
+ String[] parsedDbName = parseDbName(dbName, conf);
+ DeleteColumnStatisticsRequest request = new DeleteColumnStatisticsRequest(parsedDbName[DB_NAME], tableName);
+ if (colName != null) {
+ colName = colName.toLowerCase();
+ request.addToCol_names(colName);
+ }
+ request.setEngine(engine);
+ request.setCat_name(parsedDbName[CAT_NAME]);
+ request.setTableLevel(true);
+ return delete_column_statistics_req(request);
+ }
+
+ @Override
+ @Deprecated
+ public List<Partition> get_partitions_by_filter(final String dbName, final String tblName,
+ final String filter, final short maxParts)
+ throws TException {
+ String[] parsedDbName = parseDbName(dbName, conf);
+ GetPartitionsByFilterRequest gfr =
+ new GetPartitionsByFilterRequest(parsedDbName[DB_NAME], tblName, filter);
+ gfr.setMaxParts(maxParts);
+ gfr.setCatName(parsedDbName[CAT_NAME]);
+ return get_partitions_by_filter_req(gfr);
+ }
+
+ @Override
+ @Deprecated
+ public List<PartitionSpec> get_part_specs_by_filter(final String dbName, final String tblName,
+ final String filter, final int maxParts)
+ throws TException {
+
+ String[] parsedDbName = parseDbName(dbName, conf);
+ startPartitionFunction("get_partitions_by_filter_pspec", parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tblName,
+ maxParts, filter);
+ List<PartitionSpec> partitionSpecs = null;
+ try {
+ GetTableRequest getTableRequest = new GetTableRequest(parsedDbName[DB_NAME], tblName);
+ getTableRequest.setCatName(parsedDbName[CAT_NAME]);
+ Table table = get_table_core(getTableRequest);
+ // Don't pass the parsed db name, as get_partitions_by_filter will parse it itself
+ List<Partition> partitions = get_partitions_by_filter(dbName, tblName, filter, (short) maxParts);
+
+ if (is_partition_spec_grouping_enabled(table)) {
+ partitionSpecs = MetaStoreServerUtils
+ .getPartitionspecsGroupedByStorageDescriptor(table, partitions);
+ }
+ else {
+ PartitionSpec pSpec = new PartitionSpec();
+ pSpec.setPartitionList(new PartitionListComposingSpec(partitions));
+ pSpec.setRootPath(table.getSd().getLocation());
+ pSpec.setCatName(parsedDbName[CAT_NAME]);
+ pSpec.setDbName(parsedDbName[DB_NAME]);
+ pSpec.setTableName(tblName);
+ partitionSpecs = Arrays.asList(pSpec);
+ }
+
+ return partitionSpecs;
+ }
+ finally {
+ endFunction("get_partitions_by_filter_pspec", partitionSpecs != null && !partitionSpecs.isEmpty(), null, tblName);
+ }
+ }
+
+ @Override
+ @Deprecated
+ public int get_num_partitions_by_filter(final String dbName,
+ final String tblName, final String filter)
+ throws TException {
+ String[] parsedDbName = parseDbName(dbName, conf);
+ if (parsedDbName[DB_NAME] == null || tblName == null) {
+ throw new MetaException("The DB and table name cannot be null.");
+ }
+ startFunction("get_num_partitions_by_filter",
+ " : tbl=" + TableName.getQualified(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tblName) + " Filter="
+ + filter);
+
+ int ret = -1;
+ Exception ex = null;
+ try {
+ ret = getMS().getNumPartitionsByFilter(parsedDbName[CAT_NAME], parsedDbName[DB_NAME],
+ tblName, filter);
+ } catch (Exception e) {
+ ex = e;
+ rethrowException(e);
+ } finally {
+ endFunction("get_num_partitions_by_filter", ret != -1, ex, tblName);
+ }
+ return ret;
+ }
+
+ @Override
+ @Deprecated
+ public List<Partition> get_partitions_by_names(final String dbName, final String tblName,
+ final List<String> partNames)
+ throws TException {
+ if (partNames == null) {
+ throw new MetaException("The partNames is null");
+ }
+ GetPartitionsByNamesRequest request = new GetPartitionsByNamesRequest(dbName, tblName);
+ request.setNames(partNames);
+ return get_partitions_by_names_req(request).getPartitions();
+ }
+
+ @Override
+ public GetFileMetadataByExprResult get_file_metadata_by_expr(GetFileMetadataByExprRequest req)
+ throws TException {
+ GetFileMetadataByExprResult result = new GetFileMetadataByExprResult();
+ RawStore ms = getMS();
+ if (!ms.isFileMetadataSupported()) {
+ result.setIsSupported(false);
+ result.setMetadata(Collections.emptyMap()); // Set the required field.
+ return result;
+ }
+ result.setIsSupported(true);
+
+ List<Long> fileIds = req.getFileIds();
+ boolean needMetadata = !req.isSetDoGetFooters() || req.isDoGetFooters();
+ FileMetadataExprType type = req.isSetType() ? req.getType() : FileMetadataExprType.ORC_SARG;
+
+ ByteBuffer[] metadatas = needMetadata ? new ByteBuffer[fileIds.size()] : null;
+ ByteBuffer[] ppdResults = new ByteBuffer[fileIds.size()];
+ boolean[] eliminated = new boolean[fileIds.size()];
+
+ getMS().getFileMetadataByExpr(fileIds, type, req.getExpr(), metadatas, ppdResults, eliminated);
+ for (int i = 0; i < fileIds.size(); ++i) {
+ if (!eliminated[i] && ppdResults[i] == null)
+ {
+ continue; // No metadata => no ppd.
+ }
+ MetadataPpdResult mpr = new MetadataPpdResult();
+ ByteBuffer ppdResult = eliminated[i] ? null : handleReadOnlyBufferForThrift(ppdResults[i]);
+ mpr.setIncludeBitset(ppdResult);
+ if (needMetadata) {
+ ByteBuffer metadata = eliminated[i] ? null : handleReadOnlyBufferForThrift(metadatas[i]);
+ mpr.setMetadata(metadata);
+ }
+ result.putToMetadata(fileIds.get(i), mpr);
+ }
+ if (!result.isSetMetadata()) {
+ result.setMetadata(Collections.emptyMap()); // Set the required field.
+ }
+ return result;
+ }
+
+ @Override
+ public GetFileMetadataResult get_file_metadata(GetFileMetadataRequest req) throws TException {
+ GetFileMetadataResult result = new GetFileMetadataResult();
+ RawStore ms = getMS();
+ if (!ms.isFileMetadataSupported()) {
+ result.setIsSupported(false);
+ result.setMetadata(Collections.emptyMap()); // Set the required field.
+ return result;
+ }
+ result.setIsSupported(true);
+ List<Long> fileIds = req.getFileIds();
+ ByteBuffer[] metadatas = ms.getFileMetadata(fileIds);
+ assert metadatas.length == fileIds.size();
+ for (int i = 0; i < metadatas.length; ++i) {
+ ByteBuffer bb = metadatas[i];
+ if (bb == null) {
+ continue;
+ }
+ bb = handleReadOnlyBufferForThrift(bb);
+ result.putToMetadata(fileIds.get(i), bb);
+ }
+ if (!result.isSetMetadata()) {
+ result.setMetadata(Collections.emptyMap()); // Set the required field.
+ }
+ return result;
+ }
+
+ private ByteBuffer handleReadOnlyBufferForThrift(ByteBuffer bb) {
+ if (!bb.isReadOnly()) {
+ return bb;
+ }
+ // Thrift cannot write read-only buffers... oh well.
+ // TODO: actually thrift never writes to the buffer, so we could use reflection to
+ // unset the unnecessary read-only flag if allocation/copy perf becomes a problem.
+ ByteBuffer copy = ByteBuffer.allocate(bb.capacity());
+ copy.put(bb);
+ copy.flip();
+ return copy;
+ }
+
+ @Override
+ public PutFileMetadataResult put_file_metadata(PutFileMetadataRequest req) throws TException {
+ RawStore ms = getMS();
+ if (ms.isFileMetadataSupported()) {
+ ms.putFileMetadata(req.getFileIds(), req.getMetadata(), req.getType());
+ }
+ return new PutFileMetadataResult();
+ }
+
+ @Override
+ public ClearFileMetadataResult clear_file_metadata(ClearFileMetadataRequest req)
+ throws TException {
+ getMS().putFileMetadata(req.getFileIds(), null, null);
+ return new ClearFileMetadataResult();
+ }
+
+ @Override
+ public CacheFileMetadataResult cache_file_metadata(
+ CacheFileMetadataRequest req) throws TException {
+ RawStore ms = getMS();
+ if (!ms.isFileMetadataSupported()) {
+ return new CacheFileMetadataResult(false);
+ }
+ String dbName = req.getDbName(), tblName = req.getTblName(),
+ partName = req.isSetPartName() ? req.getPartName() : null;
+ boolean isAllPart = req.isSetIsAllParts() && req.isIsAllParts();
+ ms.openTransaction();
+ boolean success = false;
+ try {
+ Table tbl = ms.getTable(getDefaultCatalog(conf), dbName, tblName);
+ if (tbl == null) {
+ throw new NoSuchObjectException(dbName + "." + tblName + " not found");
+ }
+ boolean isPartitioned = tbl.isSetPartitionKeys() && tbl.getPartitionKeysSize() > 0;
+ String tableInputFormat = tbl.isSetSd() ? tbl.getSd().getInputFormat() : null;
+ if (!isPartitioned) {
+ if (partName != null || isAllPart) {
+ throw new MetaException("Table is not partitioned");
+ }
+ if (!tbl.isSetSd() || !tbl.getSd().isSetLocation()) {
+ throw new MetaException(
+ "Table does not have storage location; this operation is not supported on views");
+ }
+ FileMetadataExprType type = expressionProxy.getMetadataType(tableInputFormat);
+ if (type == null) {
+ throw new MetaException("The operation is not supported for " + tableInputFormat);
+ }
+ fileMetadataManager.queueCacheMetadata(tbl.getSd().getLocation(), type);
+ success = true;
+ } else {
+ List<String> partNames;
+ if (partName != null) {
+ partNames = Lists.newArrayList(partName);
+ } else if (isAllPart) {
+ partNames = ms.listPartitionNames(getDefaultCatalog(conf), dbName, tblName, (short)-1);
+ } else {
+ throw new MetaException("Table is partitioned");
+ }
+ int batchSize = MetastoreConf.getIntVar(
+ conf, MetastoreConf.ConfVars.BATCH_RETRIEVE_OBJECTS_MAX);
+ int index = 0;
+ int successCount = 0, failCount = 0;
+ HashSet<String> failFormats = null;
+ while (index < partNames.size()) {
+ int currentBatchSize = Math.min(batchSize, partNames.size() - index);
+ List<String> nameBatch = partNames.subList(index, index + currentBatchSize);
+ index += currentBatchSize;
+ List<Partition> parts = ms.getPartitionsByNames(getDefaultCatalog(conf), dbName, tblName, nameBatch);
+ for (Partition part : parts) {
+ if (!part.isSetSd() || !part.getSd().isSetLocation()) {
+ throw new MetaException("Partition does not have storage location;" +
+ " this operation is not supported on views");
+ }
+ String inputFormat = part.getSd().isSetInputFormat()
+ ? part.getSd().getInputFormat() : tableInputFormat;
+ FileMetadataExprType type = expressionProxy.getMetadataType(inputFormat);
+ if (type == null) {
+ ++failCount;
+ if (failFormats == null) {
+ failFormats = new HashSet<>();
+ }
+ failFormats.add(inputFormat);
+ } else {
+ ++successCount;
+ fileMetadataManager.queueCacheMetadata(part.getSd().getLocation(), type);
+ }
+ }
+ }
+ success = true; // Regardless of the following exception
+ if (failCount > 0) {
+ String errorMsg = "The operation failed for " + failCount + " partitions and "
+ + "succeeded for " + successCount + " partitions; unsupported formats: ";
+ boolean isFirst = true;
+ for (String s : failFormats) {
+ if (!isFirst) {
+ errorMsg += ", ";
+ }
+ isFirst = false;
+ errorMsg += s;
+ }
+ throw new MetaException(errorMsg);
+ }
+ }
+ } finally {
+ if (success) {
+ if (!ms.commitTransaction()) {
+ throw new MetaException("Failed to commit");
+ }
+ } else {
+ ms.rollbackTransaction();
+ }
+ }
+ return new CacheFileMetadataResult(true);
+ }
+
+ private static boolean is_partition_spec_grouping_enabled(Table table) {
+ Map<String, String> parameters = table.getParameters();
+ return parameters.containsKey("hive.hcatalog.partition.spec.grouping.enabled")
+ && parameters.get("hive.hcatalog.partition.spec.grouping.enabled").equalsIgnoreCase("true");
+ }
+}
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/handler/PrivilegeHandler.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/handler/PrivilegeHandler.java
new file mode 100644
index 00000000000..74a0ba4b9f5
--- /dev/null
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/handler/PrivilegeHandler.java
@@ -0,0 +1,687 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.handler;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.RawStore;
+import org.apache.hadoop.hive.metastore.Warehouse;
+import org.apache.hadoop.hive.metastore.api.GetPrincipalsInRoleRequest;
+import org.apache.hadoop.hive.metastore.api.GetPrincipalsInRoleResponse;
+import org.apache.hadoop.hive.metastore.api.GetRoleGrantsForPrincipalRequest;
+import org.apache.hadoop.hive.metastore.api.GetRoleGrantsForPrincipalResponse;
+import org.apache.hadoop.hive.metastore.api.GetTableRequest;
+import org.apache.hadoop.hive.metastore.api.GrantRevokePrivilegeRequest;
+import org.apache.hadoop.hive.metastore.api.GrantRevokePrivilegeResponse;
+import org.apache.hadoop.hive.metastore.api.GrantRevokeRoleRequest;
+import org.apache.hadoop.hive.metastore.api.GrantRevokeRoleResponse;
+import org.apache.hadoop.hive.metastore.api.HiveObjectPrivilege;
+import org.apache.hadoop.hive.metastore.api.HiveObjectRef;
+import org.apache.hadoop.hive.metastore.api.HiveObjectType;
+import org.apache.hadoop.hive.metastore.api.InvalidObjectException;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
+import org.apache.hadoop.hive.metastore.api.PrincipalPrivilegeSet;
+import org.apache.hadoop.hive.metastore.api.PrincipalType;
+import org.apache.hadoop.hive.metastore.api.PrivilegeBag;
+import org.apache.hadoop.hive.metastore.api.Role;
+import org.apache.hadoop.hive.metastore.api.RolePrincipalGrant;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.metastore.events.PreAuthorizationCallEvent;
+import org.apache.thrift.TException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.hadoop.hive.metastore.ExceptionHandler.handleException;
+import static org.apache.hadoop.hive.metastore.ExceptionHandler.rethrowException;
+import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.getDefaultCatalog;
+
+/**
+ * Privilege and Role methods
+ */
+public abstract class PrivilegeHandler extends TransactionHandler {
+ private static final Logger LOG = LoggerFactory.getLogger(PrivilegeHandler.class);
+
+ protected PrivilegeHandler(String name, Configuration conf) {
+ super(name, conf);
+ }
+
+ @Override
+ public PrincipalPrivilegeSet get_privilege_set(HiveObjectRef hiveObject, String userName,
+ List<String> groupNames) throws TException {
+ firePreEvent(new PreAuthorizationCallEvent(this));
+ String catName = hiveObject.isSetCatName() ? hiveObject.getCatName() : getDefaultCatalog(conf);
+ HiveObjectType debug = hiveObject.getObjectType();
+ if (hiveObject.getObjectType() == HiveObjectType.COLUMN) {
+ String partName = getPartName(hiveObject);
+ return this.get_column_privilege_set(catName, hiveObject.getDbName(), hiveObject
+ .getObjectName(), partName, hiveObject.getColumnName(), userName,
+ groupNames);
+ } else if (hiveObject.getObjectType() == HiveObjectType.PARTITION) {
+ String partName = getPartName(hiveObject);
+ return this.get_partition_privilege_set(catName, hiveObject.getDbName(),
+ hiveObject.getObjectName(), partName, userName, groupNames);
+ } else if (hiveObject.getObjectType() == HiveObjectType.DATABASE) {
+ return this.get_db_privilege_set(catName, hiveObject.getDbName(), userName,
+ groupNames);
+ } else if (hiveObject.getObjectType() == HiveObjectType.DATACONNECTOR) {
+ return this.get_connector_privilege_set(catName, hiveObject.getObjectName(), userName, groupNames);
+ } else if (hiveObject.getObjectType() == HiveObjectType.TABLE) {
+ return this.get_table_privilege_set(catName, hiveObject.getDbName(), hiveObject
+ .getObjectName(), userName, groupNames);
+ } else if (hiveObject.getObjectType() == HiveObjectType.GLOBAL) {
+ return this.get_user_privilege_set(userName, groupNames);
+ }
+ return null;
+ }
+
+ private String getPartName(HiveObjectRef hiveObject) throws MetaException {
+ String partName = null;
+ List<String> partValue = hiveObject.getPartValues();
+ if (partValue != null && partValue.size() > 0) {
+ try {
+ String catName = hiveObject.isSetCatName() ? hiveObject.getCatName() :
+ getDefaultCatalog(conf);
+ GetTableRequest getTableRequest = new GetTableRequest(hiveObject.getDbName(), hiveObject.getObjectName());
+ getTableRequest.setCatName(catName);
+ Table table = get_table_core(getTableRequest);
+ partName = Warehouse
+ .makePartName(table.getPartitionKeys(), partValue);
+ } catch (NoSuchObjectException e) {
+ throw new MetaException(e.getMessage());
+ }
+ }
+ return partName;
+ }
+
+ private PrincipalPrivilegeSet get_column_privilege_set(String catName, final String dbName,
+ final String tableName, final String partName, final String columnName,
+ final String userName, final List<String> groupNames) throws TException {
+ incrementCounter("get_column_privilege_set");
+
+ PrincipalPrivilegeSet ret;
+ try {
+ ret = getMS().getColumnPrivilegeSet(
+ catName, dbName, tableName, partName, columnName, userName, groupNames);
+ } catch (Exception e) {
+ throw handleException(e).throwIfInstance(MetaException.class).defaultRuntimeException();
+ }
+ return ret;
+ }
+
+ @Override
+ public GetPrincipalsInRoleResponse get_principals_in_role(GetPrincipalsInRoleRequest request)
+ throws TException {
+
+ incrementCounter("get_principals_in_role");
+ firePreEvent(new PreAuthorizationCallEvent(this));
+ Exception ex = null;
+ GetPrincipalsInRoleResponse response = null;
+ try {
+ response = new GetPrincipalsInRoleResponse(getMS().listRoleMembers(request.getRoleName()));
+ } catch (Exception e) {
+ ex = e;
+ rethrowException(e);
+ } finally {
+ endFunction("get_principals_in_role", ex == null, ex);
+ }
+ return response;
+ }
+
+ @Override
+ public GetRoleGrantsForPrincipalResponse get_role_grants_for_principal(
+ GetRoleGrantsForPrincipalRequest request) throws TException {
+
+ incrementCounter("get_role_grants_for_principal");
+ firePreEvent(new PreAuthorizationCallEvent(this));
+ Exception ex = null;
+ List<RolePrincipalGrant> roleMaps = null;
+ try {
+ roleMaps = getMS().listRolesWithGrants(request.getPrincipal_name(), request.getPrincipal_type());
+ } catch (Exception e) {
+ ex = e;
+ rethrowException(e);
+ } finally {
+ endFunction("get_role_grants_for_principal", ex == null, ex);
+ }
+
+ //List<RolePrincipalGrant> roleGrantsList = getRolePrincipalGrants(roleMaps);
+ return new GetRoleGrantsForPrincipalResponse(roleMaps);
+ }
+
+ private PrincipalPrivilegeSet get_db_privilege_set(String catName, final String dbName,
+ final String userName, final List<String> groupNames) throws TException {
+ incrementCounter("get_db_privilege_set");
+
+ PrincipalPrivilegeSet ret;
+ try {
+ ret = getMS().getDBPrivilegeSet(catName, dbName, userName, groupNames);
+ } catch (Exception e) {
+ throw handleException(e).throwIfInstance(MetaException.class).defaultRuntimeException();
+ }
+ return ret;
+ }
+
+ private PrincipalPrivilegeSet get_connector_privilege_set(String catName, final String connectorName,
+ final String userName, final List<String> groupNames) throws TException {
+ incrementCounter("get_connector_privilege_set");
+
+ PrincipalPrivilegeSet ret;
+ try {
+ ret = getMS().getConnectorPrivilegeSet(catName, connectorName, userName, groupNames);
+ } catch (MetaException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ return ret;
+
+ }
+
+ private PrincipalPrivilegeSet get_partition_privilege_set(
+ String catName, final String dbName, final String tableName, final String partName,
+ final String userName, final List<String> groupNames)
+ throws TException {
+ incrementCounter("get_partition_privilege_set");
+
+ PrincipalPrivilegeSet ret;
+ try {
+ ret = getMS().getPartitionPrivilegeSet(catName, dbName, tableName, partName,
+ userName, groupNames);
+ } catch (Exception e) {
+ throw handleException(e).throwIfInstance(MetaException.class).defaultRuntimeException();
+ }
+ return ret;
+ }
+
+ private PrincipalPrivilegeSet get_table_privilege_set(String catName, final String dbName,
+ final String tableName, final String userName,
+ final List<String> groupNames) throws TException {
+ incrementCounter("get_table_privilege_set");
+
+ PrincipalPrivilegeSet ret;
+ try {
+ ret = getMS().getTablePrivilegeSet(catName, dbName, tableName, userName,
+ groupNames);
+ } catch (Exception e) {
+ throw handleException(e).throwIfInstance(MetaException.class).defaultRuntimeException();
+ }
+ return ret;
+ }
+
+ @Override
+ public boolean grant_role(final String roleName,
+ final String principalName, final PrincipalType principalType,
+ final String grantor, final PrincipalType grantorType, final boolean grantOption)
+ throws TException {
+ incrementCounter("add_role_member");
+ firePreEvent(new PreAuthorizationCallEvent(this));
+ if (PUBLIC.equals(roleName)) {
+ throw new MetaException("No user can be added to " + PUBLIC +". Since all users implicitly"
+ + " belong to " + PUBLIC + " role.");
+ }
+ Boolean ret;
+ try {
+ RawStore ms = getMS();
+ Role role = ms.getRole(roleName);
+ if(principalType == PrincipalType.ROLE){
+ //check if this grant statement will end up creating a cycle
+ if(isNewRoleAParent(principalName, roleName)){
+ throw new MetaException("Cannot grant role " + principalName + " to " + roleName +
+ " as " + roleName + " already belongs to the role " + principalName +
+ ". (no cycles allowed)");
+ }
+ }
+ ret = ms.grantRole(role, principalName, principalType, grantor, grantorType, grantOption);
+ } catch (Exception e) {
+ String exInfo = "Got exception: " + e.getClass().getName() + " " + e.getMessage();
+ LOG.error(exInfo, e);
+ throw handleException(e).throwIfInstance(MetaException.class)
+ .toMetaExceptionIfInstance(exInfo, InvalidObjectException.class, NoSuchObjectException.class)
+ .defaultTException();
+ }
+ return ret;
+ }
+
+ /**
+ * Check if newRole is in parent hierarchy of curRole
+ * @param newRole
+ * @param curRole
+ * @return true if newRole is curRole or present in its hierarchy
+ * @throws MetaException
+ */
+ private boolean isNewRoleAParent(String newRole, String curRole) throws MetaException {
+ if(newRole.equals(curRole)){
+ return true;
+ }
+ //do this check recursively on all the parent roles of curRole
+ List<Role> parentRoleMaps = getMS().listRoles(curRole, PrincipalType.ROLE);
+ for(Role parentRole : parentRoleMaps){
+ if(isNewRoleAParent(newRole, parentRole.getRoleName())){
+ return true;
+ }
+ }
+ return false;
+ }
+
+ @Override
+ public List<Role> list_roles(final String principalName,
+ final PrincipalType principalType) throws TException {
+ incrementCounter("list_roles");
+ firePreEvent(new PreAuthorizationCallEvent(this));
+ return getMS().listRoles(principalName, principalType);
+ }
+
+ @Override
+ public boolean create_role(final Role role) throws TException {
+ incrementCounter("create_role");
+ firePreEvent(new PreAuthorizationCallEvent(this));
+ if (PUBLIC.equals(role.getRoleName())) {
+ throw new MetaException(PUBLIC + " role implicitly exists. It can't be created.");
+ }
+ Boolean ret;
+ try {
+ ret = getMS().addRole(role.getRoleName(), role.getOwnerName());
+ } catch (Exception e) {
+ String exInfo = "Got exception: " + e.getClass().getName() + " " + e.getMessage();
+ LOG.error(exInfo, e);
+ throw handleException(e).throwIfInstance(MetaException.class)
+ .toMetaExceptionIfInstance(exInfo, InvalidObjectException.class, NoSuchObjectException.class)
+ .defaultTException();
+ }
+ return ret;
+ }
+
+ @Override
+ public boolean drop_role(final String roleName) throws TException {
+ incrementCounter("drop_role");
+ firePreEvent(new PreAuthorizationCallEvent(this));
+ if (ADMIN.equals(roleName) || PUBLIC.equals(roleName)) {
+ throw new MetaException(PUBLIC + "," + ADMIN + " roles can't be dropped.");
+ }
+ Boolean ret;
+ try {
+ ret = getMS().removeRole(roleName);
+ } catch (Exception e) {
+ String exInfo = "Got exception: " + e.getClass().getName() + " " + e.getMessage();
+ LOG.error(exInfo, e);
+ throw handleException(e).throwIfInstance(MetaException.class)
+ .toMetaExceptionIfInstance(exInfo, NoSuchObjectException.class)
+ .defaultTException();
+ }
+ return ret;
+ }
+
+ @Override
+ public List<String> get_role_names() throws TException {
+ incrementCounter("get_role_names");
+ firePreEvent(new PreAuthorizationCallEvent(this));
+ List<String> ret;
+ try {
+ ret = getMS().listRoleNames();
+ return ret;
+ } catch (Exception e) {
+ throw handleException(e).throwIfInstance(MetaException.class).defaultRuntimeException();
+ }
+ }
+
+ @Override
+ public boolean grant_privileges(final PrivilegeBag privileges) throws TException {
+ incrementCounter("grant_privileges");
+ firePreEvent(new PreAuthorizationCallEvent(this));
+ Boolean ret;
+ try {
+ ret = getMS().grantPrivileges(privileges);
+ } catch (Exception e) {
+ String exInfo = "Got exception: " + e.getClass().getName() + " " + e.getMessage();
+ LOG.error(exInfo, e);
+ throw handleException(e).throwIfInstance(MetaException.class)
+ .toMetaExceptionIfInstance(exInfo, InvalidObjectException.class, NoSuchObjectException.class)
+ .defaultTException();
+ }
+ return ret;
+ }
+
+ @Override
+ public boolean revoke_role(final String roleName, final String userName,
+ final PrincipalType principalType) throws TException {
+ return revoke_role(roleName, userName, principalType, false);
+ }
+
+ private boolean revoke_role(final String roleName, final String userName,
+ final PrincipalType principalType, boolean grantOption) throws TException {
+ incrementCounter("remove_role_member");
+ firePreEvent(new PreAuthorizationCallEvent(this));
+ if (PUBLIC.equals(roleName)) {
+ throw new MetaException(PUBLIC + " role can't be revoked.");
+ }
+ Boolean ret;
+ try {
+ RawStore ms = getMS();
+ Role mRole = ms.getRole(roleName);
+ ret = ms.revokeRole(mRole, userName, principalType, grantOption);
+ } catch (Exception e) {
+ String exInfo = "Got exception: " + e.getClass().getName() + " " + e.getMessage();
+ LOG.error(exInfo, e);
+ throw handleException(e).throwIfInstance(MetaException.class)
+ .toMetaExceptionIfInstance(exInfo, NoSuchObjectException.class)
+ .defaultTException();
+ }
+ return ret;
+ }
+
+ @Override
+ public GrantRevokeRoleResponse grant_revoke_role(GrantRevokeRoleRequest request)
+ throws TException {
+ GrantRevokeRoleResponse response = new GrantRevokeRoleResponse();
+ boolean grantOption = false;
+ if (request.isSetGrantOption()) {
+ grantOption = request.isGrantOption();
+ }
+ switch (request.getRequestType()) {
+ case GRANT: {
+ boolean result = grant_role(request.getRoleName(),
+ request.getPrincipalName(), request.getPrincipalType(),
+ request.getGrantor(), request.getGrantorType(), grantOption);
+ response.setSuccess(result);
+ break;
+ }
+ case REVOKE: {
+ boolean result = revoke_role(request.getRoleName(), request.getPrincipalName(),
+ request.getPrincipalType(), grantOption);
+ response.setSuccess(result);
+ break;
+ }
+ default:
+ throw new MetaException("Unknown request type " + request.getRequestType());
+ }
+
+ return response;
+ }
+
+ @Override
+ public GrantRevokePrivilegeResponse grant_revoke_privileges(GrantRevokePrivilegeRequest request)
+ throws TException {
+ GrantRevokePrivilegeResponse response = new GrantRevokePrivilegeResponse();
+ switch (request.getRequestType()) {
+ case GRANT: {
+ boolean result = grant_privileges(request.getPrivileges());
+ response.setSuccess(result);
+ break;
+ }
+ case REVOKE: {
+ boolean revokeGrantOption = false;
+ if (request.isSetRevokeGrantOption()) {
+ revokeGrantOption = request.isRevokeGrantOption();
+ }
+ boolean result = revoke_privileges(request.getPrivileges(), revokeGrantOption);
+ response.setSuccess(result);
+ break;
+ }
+ default:
+ throw new MetaException("Unknown request type " + request.getRequestType());
+ }
+
+ return response;
+ }
+
+ @Override
+ public GrantRevokePrivilegeResponse refresh_privileges(HiveObjectRef objToRefresh, String authorizer,
+ GrantRevokePrivilegeRequest grantRequest)
+ throws TException {
+ incrementCounter("refresh_privileges");
+ firePreEvent(new PreAuthorizationCallEvent(this));
+ GrantRevokePrivilegeResponse response = new GrantRevokePrivilegeResponse();
+ try {
+ boolean result = getMS().refreshPrivileges(objToRefresh, authorizer, grantRequest.getPrivileges());
+ response.setSuccess(result);
+ } catch (Exception e) {
+ throw handleException(e).throwIfInstance(MetaException.class).defaultRuntimeException();
+ }
+ return response;
+ }
+
+ @Override
+ public boolean revoke_privileges(final PrivilegeBag privileges) throws TException {
+ return revoke_privileges(privileges, false);
+ }
+
+ public boolean revoke_privileges(final PrivilegeBag privileges, boolean grantOption)
+ throws TException {
+ incrementCounter("revoke_privileges");
+ firePreEvent(new PreAuthorizationCallEvent(this));
+ Boolean ret;
+ try {
+ ret = getMS().revokePrivileges(privileges, grantOption);
+ } catch (Exception e) {
+ String exInfo = "Got exception: " + e.getClass().getName() + " " + e.getMessage();
+ LOG.error(exInfo, e);
+ throw handleException(e).throwIfInstance(MetaException.class)
+ .toMetaExceptionIfInstance(exInfo, InvalidObjectException.class, NoSuchObjectException.class)
+ .defaultTException();
+ }
+ return ret;
+ }
+
+ private PrincipalPrivilegeSet get_user_privilege_set(final String userName,
+ final List<String> groupNames) throws TException {
+ incrementCounter("get_user_privilege_set");
+ PrincipalPrivilegeSet ret;
+ try {
+ ret = getMS().getUserPrivilegeSet(userName, groupNames);
+ } catch (Exception e) {
+ throw handleException(e).throwIfInstance(MetaException.class).defaultRuntimeException();
+ }
+ return ret;
+ }
+
+ @Override
+ public List<HiveObjectPrivilege> list_privileges(String principalName,
+ PrincipalType principalType, HiveObjectRef hiveObject)
+ throws TException {
+ firePreEvent(new PreAuthorizationCallEvent(this));
+ String catName = hiveObject.isSetCatName() ? hiveObject.getCatName() : getDefaultCatalog(conf);
+ if (hiveObject.getObjectType() == null) {
+ return getAllPrivileges(principalName, principalType, catName);
+ }
+ if (hiveObject.getObjectType() == HiveObjectType.GLOBAL) {
+ return list_global_privileges(principalName, principalType);
+ }
+ if (hiveObject.getObjectType() == HiveObjectType.DATABASE) {
+ return list_db_privileges(principalName, principalType, catName, hiveObject
+ .getDbName());
+ }
+ if (hiveObject.getObjectType() == HiveObjectType.DATACONNECTOR) {
+ return list_dc_privileges(principalName, principalType, hiveObject
+ .getObjectName());
+ }
+ if (hiveObject.getObjectType() == HiveObjectType.TABLE) {
+ return list_table_privileges(principalName, principalType,
+ catName, hiveObject.getDbName(), hiveObject.getObjectName());
+ }
+ if (hiveObject.getObjectType() == HiveObjectType.PARTITION) {
+ return list_partition_privileges(principalName, principalType,
+ catName, hiveObject.getDbName(), hiveObject.getObjectName(), hiveObject
+ .getPartValues());
+ }
+ if (hiveObject.getObjectType() == HiveObjectType.COLUMN) {
+ if (hiveObject.getPartValues() == null || hiveObject.getPartValues().isEmpty()) {
+ return list_table_column_privileges(principalName, principalType,
+ catName, hiveObject.getDbName(), hiveObject.getObjectName(), hiveObject.getColumnName());
+ }
+ return list_partition_column_privileges(principalName, principalType,
+ catName, hiveObject.getDbName(), hiveObject.getObjectName(), hiveObject
+ .getPartValues(), hiveObject.getColumnName());
+ }
+ return null;
+ }
+
+ private List<HiveObjectPrivilege> getAllPrivileges(String principalName,
+ PrincipalType principalType, String catName) throws TException {
+ List<HiveObjectPrivilege> privs = new ArrayList<>();
+ privs.addAll(list_global_privileges(principalName, principalType));
+ privs.addAll(list_db_privileges(principalName, principalType, catName, null));
+ privs.addAll(list_dc_privileges(principalName, principalType, null));
+ privs.addAll(list_table_privileges(principalName, principalType, catName, null, null));
+ privs.addAll(list_partition_privileges(principalName, principalType, catName, null, null, null));
+ privs.addAll(list_table_column_privileges(principalName, principalType, catName, null, null, null));
+ privs.addAll(list_partition_column_privileges(principalName, principalType,
+ catName, null, null, null, null));
+ return privs;
+ }
+
+ private List<HiveObjectPrivilege> list_table_column_privileges(
+ final String principalName, final PrincipalType principalType, String catName,
+ final String dbName, final String tableName, final String columnName) throws TException {
+ incrementCounter("list_table_column_privileges");
+
+ try {
+ if (dbName == null) {
+ return getMS().listPrincipalTableColumnGrantsAll(principalName, principalType);
+ }
+ if (principalName == null) {
+ return getMS().listTableColumnGrantsAll(catName, dbName, tableName, columnName);
+ }
+ return getMS().listPrincipalTableColumnGrants(principalName, principalType,
+ catName, dbName, tableName, columnName);
+ } catch (Exception e) {
+ throw handleException(e).throwIfInstance(MetaException.class).defaultRuntimeException();
+ }
+ }
+
+ private List<HiveObjectPrivilege> list_partition_column_privileges(
+ final String principalName, final PrincipalType principalType,
+ String catName, final String dbName, final String tableName, final List<String> partValues,
+ final String columnName) throws TException {
+ incrementCounter("list_partition_column_privileges");
+
+ try {
+ if (dbName == null) {
+ return getMS().listPrincipalPartitionColumnGrantsAll(principalName, principalType);
+ }
+ GetTableRequest getTableRequest = new GetTableRequest(dbName, tableName);
+ getTableRequest.setCatName(catName);
+ Table tbl = get_table_core(getTableRequest);
+ String partName = Warehouse.makePartName(tbl.getPartitionKeys(), partValues);
+ if (principalName == null) {
+ return getMS().listPartitionColumnGrantsAll(catName, dbName, tableName, partName, columnName);
+ }
+
+ return getMS().listPrincipalPartitionColumnGrants(principalName, principalType, catName, dbName,
+ tableName, partValues, partName, columnName);
+ } catch (Exception e) {
+ throw handleException(e).throwIfInstance(MetaException.class).defaultRuntimeException();
+ }
+ }
+
+ private List<HiveObjectPrivilege> list_db_privileges(final String principalName,
+ final PrincipalType principalType, String catName, final String dbName) throws TException {
+ incrementCounter("list_security_db_grant");
+
+ try {
+ if (dbName == null) {
+ return getMS().listPrincipalDBGrantsAll(principalName, principalType);
+ }
+ if (principalName == null) {
+ return getMS().listDBGrantsAll(catName, dbName);
+ } else {
+ return getMS().listPrincipalDBGrants(principalName, principalType, catName, dbName);
+ }
+ } catch (Exception e) {
+ throw handleException(e).throwIfInstance(MetaException.class).defaultRuntimeException();
+ }
+ }
+
+ private List<HiveObjectPrivilege> list_dc_privileges(final String principalName,
+ final PrincipalType principalType, final String dcName) throws TException {
+ incrementCounter("list_security_dc_grant");
+
+ try {
+ if (dcName == null) {
+ return getMS().listPrincipalDCGrantsAll(principalName, principalType);
+ }
+ if (principalName == null) {
+ return getMS().listDCGrantsAll(dcName);
+ } else {
+ return getMS().listPrincipalDCGrants(principalName, principalType, dcName);
+ }
+ } catch (Exception e) {
+ throw handleException(e).throwIfInstance(MetaException.class).defaultRuntimeException();
+ }
+ }
+
+ private List<HiveObjectPrivilege> list_partition_privileges(
+ final String principalName, final PrincipalType principalType,
+ String catName, final String dbName, final String tableName, final List<String> partValues)
+ throws TException {
+ incrementCounter("list_security_partition_grant");
+
+ try {
+ if (dbName == null) {
+ return getMS().listPrincipalPartitionGrantsAll(principalName, principalType);
+ }
+ GetTableRequest getTableRequest = new GetTableRequest(dbName, tableName);
+ getTableRequest.setCatName(catName);
+ Table tbl = get_table_core(getTableRequest);
+ String partName = Warehouse.makePartName(tbl.getPartitionKeys(), partValues);
+ if (principalName == null) {
+ return getMS().listPartitionGrantsAll(catName, dbName, tableName, partName);
+ }
+ return getMS().listPrincipalPartitionGrants(
+ principalName, principalType, catName, dbName, tableName, partValues, partName);
+ } catch (Exception e) {
+ throw handleException(e).throwIfInstance(MetaException.class).defaultRuntimeException();
+ }
+ }
+
+ private List<HiveObjectPrivilege> list_table_privileges(
+ final String principalName, final PrincipalType principalType,
+ String catName, final String dbName, final String tableName) throws TException {
+ incrementCounter("list_security_table_grant");
+
+ try {
+ if (dbName == null) {
+ return getMS().listPrincipalTableGrantsAll(principalName, principalType);
+ }
+ if (principalName == null) {
+ return getMS().listTableGrantsAll(catName, dbName, tableName);
+ }
+ return getMS().listAllTableGrants(principalName, principalType, catName, dbName, tableName);
+ } catch (Exception e) {
+ throw handleException(e).throwIfInstance(MetaException.class).defaultRuntimeException();
+ }
+ }
+
+ private List<HiveObjectPrivilege> list_global_privileges(
+ final String principalName, final PrincipalType principalType) throws TException {
+ incrementCounter("list_security_user_grant");
+
+ try {
+ if (principalName == null) {
+ return getMS().listGlobalGrantsAll();
+ }
+ return getMS().listPrincipalGlobalGrants(principalName, principalType);
+ } catch (Exception e) {
+ throw handleException(e).throwIfInstance(MetaException.class).defaultRuntimeException();
+ }
+ }
+
+}
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/handler/SetAggrStatsHandler.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/handler/SetAggrStatsHandler.java
index 3b43d689553..d1734ef72ea 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/handler/SetAggrStatsHandler.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/handler/SetAggrStatsHandler.java
@@ -53,7 +53,7 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static org.apache.hadoop.hive.metastore.HMSHandler.getPartValsFromName;
+import static org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils.getPartValsFromName;
import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.getDefaultCatalog;
import static org.apache.hadoop.hive.metastore.utils.StringUtils.normalizeIdentifier;
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/handler/TransactionHandler.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/handler/TransactionHandler.java
new file mode 100644
index 00000000000..087a258920d
--- /dev/null
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/handler/TransactionHandler.java
@@ -0,0 +1,567 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.handler;
+
+import com.google.common.collect.Lists;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.metastore.MetaStoreListenerNotifier;
+import org.apache.hadoop.hive.metastore.ReplChangeManager;
+import org.apache.hadoop.hive.metastore.Warehouse;
+import org.apache.hadoop.hive.metastore.api.AbortCompactResponse;
+import org.apache.hadoop.hive.metastore.api.AbortCompactionRequest;
+import org.apache.hadoop.hive.metastore.api.AbortTxnRequest;
+import org.apache.hadoop.hive.metastore.api.AbortTxnsRequest;
+import org.apache.hadoop.hive.metastore.api.AddDynamicPartitions;
+import org.apache.hadoop.hive.metastore.api.AllocateTableWriteIdsRequest;
+import org.apache.hadoop.hive.metastore.api.AllocateTableWriteIdsResponse;
+import org.apache.hadoop.hive.metastore.api.CheckLockRequest;
+import org.apache.hadoop.hive.metastore.api.ClientCapabilities;
+import org.apache.hadoop.hive.metastore.api.ClientCapability;
+import org.apache.hadoop.hive.metastore.api.CommitTxnRequest;
+import org.apache.hadoop.hive.metastore.api.CompactionInfoStruct;
+import org.apache.hadoop.hive.metastore.api.CompactionMetricsDataRequest;
+import org.apache.hadoop.hive.metastore.api.CompactionMetricsDataStruct;
+import org.apache.hadoop.hive.metastore.api.CompactionRequest;
+import org.apache.hadoop.hive.metastore.api.CompactionResponse;
+import org.apache.hadoop.hive.metastore.api.CreationMetadata;
+import org.apache.hadoop.hive.metastore.api.FindNextCompactRequest;
+import org.apache.hadoop.hive.metastore.api.GetLatestCommittedCompactionInfoRequest;
+import org.apache.hadoop.hive.metastore.api.GetLatestCommittedCompactionInfoResponse;
+import org.apache.hadoop.hive.metastore.api.GetOpenTxnsInfoResponse;
+import org.apache.hadoop.hive.metastore.api.GetOpenTxnsRequest;
+import org.apache.hadoop.hive.metastore.api.GetOpenTxnsResponse;
+import org.apache.hadoop.hive.metastore.api.GetTableRequest;
+import org.apache.hadoop.hive.metastore.api.GetValidWriteIdsRequest;
+import org.apache.hadoop.hive.metastore.api.GetValidWriteIdsResponse;
+import org.apache.hadoop.hive.metastore.api.HeartbeatRequest;
+import org.apache.hadoop.hive.metastore.api.HeartbeatTxnRangeRequest;
+import org.apache.hadoop.hive.metastore.api.HeartbeatTxnRangeResponse;
+import org.apache.hadoop.hive.metastore.api.InsertEventRequestData;
+import org.apache.hadoop.hive.metastore.api.LockMaterializationRebuildRequest;
+import org.apache.hadoop.hive.metastore.api.LockRequest;
+import org.apache.hadoop.hive.metastore.api.LockResponse;
+import org.apache.hadoop.hive.metastore.api.Materialization;
+import org.apache.hadoop.hive.metastore.api.MaxAllocatedTableWriteIdRequest;
+import org.apache.hadoop.hive.metastore.api.MaxAllocatedTableWriteIdResponse;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.OpenTxnRequest;
+import org.apache.hadoop.hive.metastore.api.OpenTxnsResponse;
+import org.apache.hadoop.hive.metastore.api.OptionalCompactionInfoStruct;
+import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.ReplTblWriteIdStateRequest;
+import org.apache.hadoop.hive.metastore.api.ReplayedTxnsForPolicyResult;
+import org.apache.hadoop.hive.metastore.api.SeedTableWriteIdsRequest;
+import org.apache.hadoop.hive.metastore.api.SeedTxnIdRequest;
+import org.apache.hadoop.hive.metastore.api.ShowCompactRequest;
+import org.apache.hadoop.hive.metastore.api.ShowCompactResponse;
+import org.apache.hadoop.hive.metastore.api.ShowLocksRequest;
+import org.apache.hadoop.hive.metastore.api.ShowLocksResponse;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.metastore.api.TxnType;
+import org.apache.hadoop.hive.metastore.api.UnlockRequest;
+import org.apache.hadoop.hive.metastore.api.UpdateTransactionalStatsRequest;
+import org.apache.hadoop.hive.metastore.api.WriteEventInfo;
+import org.apache.hadoop.hive.metastore.api.WriteNotificationLogBatchRequest;
+import org.apache.hadoop.hive.metastore.api.WriteNotificationLogBatchResponse;
+import org.apache.hadoop.hive.metastore.api.WriteNotificationLogRequest;
+import org.apache.hadoop.hive.metastore.api.WriteNotificationLogResponse;
+import org.apache.hadoop.hive.metastore.events.AbortTxnEvent;
+import org.apache.hadoop.hive.metastore.events.AcidWriteEvent;
+import org.apache.hadoop.hive.metastore.events.AllocWriteIdEvent;
+import org.apache.hadoop.hive.metastore.events.BatchAcidWriteEvent;
+import org.apache.hadoop.hive.metastore.events.CommitCompactionEvent;
+import org.apache.hadoop.hive.metastore.events.CommitTxnEvent;
+import org.apache.hadoop.hive.metastore.events.OpenTxnEvent;
+import org.apache.hadoop.hive.metastore.messaging.EventMessage;
+import org.apache.hadoop.hive.metastore.txn.CompactionMetricsDataConverter;
+import org.apache.hadoop.hive.metastore.txn.entities.CompactionInfo;
+import org.apache.hadoop.hive.metastore.utils.FileUtils;
+import org.apache.hadoop.hive.metastore.utils.FilterUtils;
+import org.apache.thrift.TException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.getDefaultCatalog;
+
+/**
+ * Transaction and locking methods
+ */
+public abstract class TransactionHandler extends DeprecatedHandler {
+ private static final Logger LOG = LoggerFactory.getLogger(TransactionHandler.class);
+
+ protected TransactionHandler(String name, Configuration conf) {
+ super(name, conf);
+ }
+
+ @Override
+ public GetOpenTxnsResponse get_open_txns() throws TException {
+ return getTxnHandler().getOpenTxns();
+ }
+
+ @Override
+ public GetOpenTxnsResponse get_open_txns_req(GetOpenTxnsRequest getOpenTxnsRequest) throws TException {
+ return getTxnHandler().getOpenTxns(getOpenTxnsRequest.getExcludeTxnTypes());
+ }
+
+ // Transaction and locking methods
+ @Override
+ public GetOpenTxnsInfoResponse get_open_txns_info() throws TException {
+ return getTxnHandler().getOpenTxnsInfo();
+ }
+
+ @Override
+ public OpenTxnsResponse open_txns(OpenTxnRequest rqst) throws TException {
+ OpenTxnsResponse response = getTxnHandler().openTxns(rqst);
+ List<Long> txnIds = response.getTxn_ids();
+ boolean isHiveReplTxn = rqst.isSetReplPolicy() && TxnType.DEFAULT.equals(rqst.getTxn_type());
+ if (txnIds != null && listeners != null && !listeners.isEmpty() && !isHiveReplTxn) {
+ MetaStoreListenerNotifier.notifyEvent(listeners, EventMessage.EventType.OPEN_TXN,
+ new OpenTxnEvent(txnIds, this));
+ }
+ return response;
+ }
+
+ @Override
+ public void abort_txn(AbortTxnRequest rqst) throws TException {
+ getTxnHandler().abortTxn(rqst);
+ boolean isHiveReplTxn = rqst.isSetReplPolicy() && TxnType.DEFAULT.equals(rqst.getTxn_type());
+ if (listeners != null && !listeners.isEmpty() && !isHiveReplTxn) {
+ // Not adding dbsUpdated to AbortTxnEvent because
+ // only DbNotificationListener cares about it, and this is already
+ // handled with transactional listeners in TxnHandler.
+ MetaStoreListenerNotifier.notifyEvent(listeners, EventMessage.EventType.ABORT_TXN,
+ new AbortTxnEvent(rqst.getTxnid(), this));
+ }
+ }
+
+ @Override
+ public void abort_txns(AbortTxnsRequest rqst) throws TException {
+ getTxnHandler().abortTxns(rqst);
+ if (listeners != null && !listeners.isEmpty()) {
+ for (Long txnId : rqst.getTxn_ids()) {
+ // See above abort_txn() note about not adding dbsUpdated.
+ MetaStoreListenerNotifier.notifyEvent(listeners, EventMessage.EventType.ABORT_TXN,
+ new AbortTxnEvent(txnId, this));
+ }
+ }
+ }
+
+ @Override
+ public AbortCompactResponse abort_Compactions(AbortCompactionRequest rqst) throws TException {
+ return getTxnHandler().abortCompactions(rqst);
+ }
+
+ @Override
+ public long get_latest_txnid_in_conflict(long txnId) throws MetaException {
+ return getTxnHandler().getLatestTxnIdInConflict(txnId);
+ }
+
+ @Override
+ public void commit_txn(CommitTxnRequest rqst) throws TException {
+ boolean isReplayedReplTxn = TxnType.REPL_CREATED.equals(rqst.getTxn_type());
+ boolean isHiveReplTxn = rqst.isSetReplPolicy() && TxnType.DEFAULT.equals(rqst.getTxn_type());
+ // in replication flow, the write notification log table will be updated here.
+ if (rqst.isSetWriteEventInfos() && isReplayedReplTxn) {
+ assert (rqst.isSetReplPolicy());
+ long targetTxnId = getTxnHandler().getTargetTxnId(rqst.getReplPolicy(), rqst.getTxnid());
+ if (targetTxnId < 0) {
+ //looks like a retry
+ return;
+ }
+ for (WriteEventInfo writeEventInfo : rqst.getWriteEventInfos()) {
+ String[] filesAdded = ReplChangeManager.getListFromSeparatedString(writeEventInfo.getFiles());
+ List<String> partitionValue = null;
+ Partition ptnObj = null;
+ String root;
+ Table tbl = getTblObject(writeEventInfo.getDatabase(), writeEventInfo.getTable(), null);
+
+ if (writeEventInfo.getPartition() != null && !writeEventInfo.getPartition().isEmpty()) {
+ partitionValue = Warehouse.getPartValuesFromPartName(writeEventInfo.getPartition());
+ ptnObj = getPartitionObj(writeEventInfo.getDatabase(), writeEventInfo.getTable(), partitionValue, tbl);
+ root = ptnObj.getSd().getLocation();
+ } else {
+ root = tbl.getSd().getLocation();
+ }
+
+ InsertEventRequestData insertData = new InsertEventRequestData();
+ insertData.setReplace(true);
+
+ // The files in the commit txn message during load will have files with path corresponding to source
+ // warehouse. Need to transform them to target warehouse using table or partition object location.
+ for (String file : filesAdded) {
+ String[] decodedPath = ReplChangeManager.decodeFileUri(file);
+ String name = (new Path(decodedPath[0])).getName();
+ Path newPath = FileUtils.getTransformedPath(name, decodedPath[3], root);
+ insertData.addToFilesAdded(newPath.toUri().toString());
+ insertData.addToSubDirectoryList(decodedPath[3]);
+ try {
+ insertData.addToFilesAddedChecksum(ReplChangeManager.checksumFor(newPath, newPath.getFileSystem(conf)));
+ } catch (IOException e) {
+ LOG.error("failed to get checksum for the file " + newPath + " with error: " + e.getMessage());
+ throw new TException(e.getMessage());
+ }
+ }
+
+ WriteNotificationLogRequest wnRqst = new WriteNotificationLogRequest(targetTxnId,
+ writeEventInfo.getWriteId(), writeEventInfo.getDatabase(), writeEventInfo.getTable(), insertData);
+ if (partitionValue != null) {
+ wnRqst.setPartitionVals(partitionValue);
+ }
+ addTxnWriteNotificationLog(tbl, ptnObj, wnRqst);
+ }
+ }
+ getTxnHandler().commitTxn(rqst);
+ if (listeners != null && !listeners.isEmpty() && !isHiveReplTxn) {
+ MetaStoreListenerNotifier.notifyEvent(listeners, EventMessage.EventType.COMMIT_TXN,
+ new CommitTxnEvent(rqst.getTxnid(), this));
+ Optional<CompactionInfo> compactionInfo = getTxnHandler().getCompactionByTxnId(rqst.getTxnid());
+ if (compactionInfo.isPresent()) {
+ MetaStoreListenerNotifier.notifyEvent(listeners, EventMessage.EventType.COMMIT_COMPACTION,
+ new CommitCompactionEvent(rqst.getTxnid(), compactionInfo.get(), this));
+ }
+ }
+ }
+
+ @Override
+ public void repl_tbl_writeid_state(ReplTblWriteIdStateRequest rqst) throws TException {
+ getTxnHandler().replTableWriteIdState(rqst);
+ }
+
+ @Override
+ public GetValidWriteIdsResponse get_valid_write_ids(GetValidWriteIdsRequest rqst) throws TException {
+ return getTxnHandler().getValidWriteIds(rqst);
+ }
+
+ @Override
+ public void add_write_ids_to_min_history(long txnId, Map<String, Long> validWriteIds) throws TException {
+ getTxnHandler().addWriteIdsToMinHistory(txnId, validWriteIds);
+ }
+
+ @Override
+ public void set_hadoop_jobid(String jobId, long cqId) throws MetaException {
+ getTxnHandler().setHadoopJobId(jobId, cqId);
+ }
+
+ @Deprecated
+ @Override
+ public OptionalCompactionInfoStruct find_next_compact(String workerId) throws MetaException{
+ return CompactionInfo.compactionInfoToOptionalStruct(
+ getTxnHandler().findNextToCompact(workerId));
+ }
+
+ @Override
+ public OptionalCompactionInfoStruct find_next_compact2(FindNextCompactRequest rqst) throws MetaException{
+ return CompactionInfo.compactionInfoToOptionalStruct(
+ getTxnHandler().findNextToCompact(rqst));
+ }
+
+ @Override
+ public void mark_cleaned(CompactionInfoStruct cr) throws MetaException {
+ getTxnHandler().markCleaned(CompactionInfo.compactionStructToInfo(cr));
+ }
+
+ @Override
+ public void mark_compacted(CompactionInfoStruct cr) throws MetaException {
+ getTxnHandler().markCompacted(CompactionInfo.compactionStructToInfo(cr));
+ }
+
+ @Override
+ public void mark_failed(CompactionInfoStruct cr) throws MetaException {
+ getTxnHandler().markFailed(CompactionInfo.compactionStructToInfo(cr));
+ }
+
+ @Override
+ public void mark_refused(CompactionInfoStruct cr) throws MetaException {
+ getTxnHandler().markRefused(CompactionInfo.compactionStructToInfo(cr));
+ }
+
+ @Override
+ public boolean update_compaction_metrics_data(CompactionMetricsDataStruct struct) throws MetaException, TException {
+ return getTxnHandler().updateCompactionMetricsData(CompactionMetricsDataConverter.structToData(struct));
+ }
+
+ @Override
+ public void remove_compaction_metrics_data(CompactionMetricsDataRequest request)
+ throws MetaException, TException {
+ getTxnHandler().removeCompactionMetricsData(request.getDbName(), request.getTblName(), request.getPartitionName(),
+ CompactionMetricsDataConverter.thriftCompactionMetricType2DbType(request.getType()));
+ }
+
+ @Override
+ public List<String> find_columns_with_stats(CompactionInfoStruct cr) throws MetaException {
+ return getTxnHandler().findColumnsWithStats(CompactionInfo.compactionStructToInfo(cr));
+ }
+
+ @Override
+ public void update_compactor_state(CompactionInfoStruct cr, long highWaterMark) throws MetaException {
+ getTxnHandler().updateCompactorState(
+ CompactionInfo.compactionStructToInfo(cr), highWaterMark);
+ }
+
+ @Override
+ public GetLatestCommittedCompactionInfoResponse get_latest_committed_compaction_info(
+ GetLatestCommittedCompactionInfoRequest rqst) throws MetaException {
+ if (rqst.getDbname() == null || rqst.getTablename() == null) {
+ throw new MetaException("Database name and table name cannot be null.");
+ }
+ GetLatestCommittedCompactionInfoResponse response = getTxnHandler().getLatestCommittedCompactionInfo(rqst);
+ return FilterUtils.filterCommittedCompactionInfoStructIfEnabled(isServerFilterEnabled, filterHook,
+ getDefaultCatalog(conf), rqst.getDbname(), rqst.getTablename(), response);
+ }
+
+ @Override
+ public AllocateTableWriteIdsResponse allocate_table_write_ids(
+ AllocateTableWriteIdsRequest rqst) throws TException {
+ AllocateTableWriteIdsResponse response = getTxnHandler().allocateTableWriteIds(rqst);
+ if (listeners != null && !listeners.isEmpty()) {
+ MetaStoreListenerNotifier.notifyEvent(listeners, EventMessage.EventType.ALLOC_WRITE_ID,
+ new AllocWriteIdEvent(response.getTxnToWriteIds(), rqst.getDbName(),
+ rqst.getTableName(), this));
+ }
+ return response;
+ }
+
+ @Override
+ public MaxAllocatedTableWriteIdResponse get_max_allocated_table_write_id(MaxAllocatedTableWriteIdRequest rqst)
+ throws MetaException {
+ return getTxnHandler().getMaxAllocatedTableWriteId(rqst);
+ }
+
+ @Override
+ public void seed_write_id(SeedTableWriteIdsRequest rqst) throws MetaException {
+ getTxnHandler().seedWriteId(rqst);
+ }
+
+ @Override
+ public void seed_txn_id(SeedTxnIdRequest rqst) throws MetaException {
+ getTxnHandler().seedTxnId(rqst);
+ }
+
+ private void addTxnWriteNotificationLog(Table tableObj, Partition ptnObj, WriteNotificationLogRequest rqst)
+ throws MetaException {
+ String partition = ""; //Empty string is an invalid partition name. Can be used for non partitioned table.
+ if (ptnObj != null) {
+ partition = Warehouse.makePartName(tableObj.getPartitionKeys(), rqst.getPartitionVals());
+ }
+ AcidWriteEvent event = new AcidWriteEvent(partition, tableObj, ptnObj, rqst);
+ getTxnHandler().addWriteNotificationLog(event);
+ if (listeners != null && !listeners.isEmpty()) {
+ MetaStoreListenerNotifier.notifyEvent(listeners, EventMessage.EventType.ACID_WRITE, event);
+ }
+ }
+
+ private Table getTblObject(String db, String table, String catalog) throws TException {
+ GetTableRequest req = new GetTableRequest(db, table);
+ if (catalog != null) {
+ req.setCatName(catalog);
+ }
+ req.setCapabilities(new ClientCapabilities(Lists.newArrayList(ClientCapability.TEST_CAPABILITY, ClientCapability.INSERT_ONLY_TABLES)));
+ return get_table_req(req).getTable();
+ }
+
+ private Partition getPartitionObj(String db, String table, List<String> partitionVals, Table tableObj)
+ throws TException {
+ if (tableObj.isSetPartitionKeys() && !tableObj.getPartitionKeys().isEmpty()) {
+ return get_partition(db, table, partitionVals);
+ }
+ return null;
+ }
+
+ @Override
+ public WriteNotificationLogResponse add_write_notification_log(WriteNotificationLogRequest rqst)
+ throws TException {
+ Table tableObj = getTblObject(rqst.getDb(), rqst.getTable(), null);
+ Partition ptnObj = getPartitionObj(rqst.getDb(), rqst.getTable(), rqst.getPartitionVals(), tableObj);
+ addTxnWriteNotificationLog(tableObj, ptnObj, rqst);
+ return new WriteNotificationLogResponse();
+ }
+
+ @Override
+ public WriteNotificationLogBatchResponse add_write_notification_log_in_batch(
+ WriteNotificationLogBatchRequest batchRequest) throws TException {
+ if (batchRequest.getRequestList().size() == 0) {
+ return new WriteNotificationLogBatchResponse();
+ }
+
+ Table tableObj = getTblObject(batchRequest.getDb(), batchRequest.getTable(), batchRequest.getCatalog());
+ BatchAcidWriteEvent event = new BatchAcidWriteEvent();
+ List<String> partNameList = new ArrayList<>();
+ List<Partition> ptnObjList;
+
+ Map<String, WriteNotificationLogRequest> rqstMap = new HashMap<>();
+ if (tableObj.getPartitionKeys().size() != 0) {
+ // partitioned table
+ for (WriteNotificationLogRequest rqst : batchRequest.getRequestList()) {
+ String partition = Warehouse.makePartName(tableObj.getPartitionKeys(), rqst.getPartitionVals());
+ partNameList.add(partition);
+ // This is used to ignore those request for which the partition does not exists.
+ rqstMap.put(partition, rqst);
+ }
+ ptnObjList = getMS().getPartitionsByNames(tableObj.getCatName(), tableObj.getDbName(),
+ tableObj.getTableName(), partNameList);
+ } else {
+ ptnObjList = new ArrayList<>();
+ for (WriteNotificationLogRequest ignored : batchRequest.getRequestList()) {
+ ptnObjList.add(null);
+ }
+ }
+
+ int idx = 0;
+ for (Partition partObject : ptnObjList) {
+ String partition = ""; //Empty string is an invalid partition name. Can be used for non partitioned table.
+ WriteNotificationLogRequest request;
+ if (partObject != null) {
+ partition = Warehouse.makePartName(tableObj.getPartitionKeys(), partObject.getValues());
+ request = rqstMap.get(partition);
+ } else {
+ // for non partitioned table, we can get serially from the list.
+ request = batchRequest.getRequestList().get(idx++);
+ }
+ event.addNotification(partition, tableObj, partObject, request);
+ if (listeners != null && !listeners.isEmpty()) {
+ MetaStoreListenerNotifier.notifyEvent(listeners, EventMessage.EventType.BATCH_ACID_WRITE,
+ new BatchAcidWriteEvent(partition, tableObj, partObject, request));
+ }
+ }
+
+ getTxnHandler().addWriteNotificationLog(event);
+ return new WriteNotificationLogBatchResponse();
+ }
+
+ @Override
+ public LockResponse lock(LockRequest rqst) throws TException {
+ return getTxnHandler().lock(rqst);
+ }
+
+ @Override
+ public LockResponse check_lock(CheckLockRequest rqst) throws TException {
+ return getTxnHandler().checkLock(rqst);
+ }
+
+ @Override
+ public void unlock(UnlockRequest rqst) throws TException {
+ getTxnHandler().unlock(rqst);
+ }
+
+ @Override
+ public ShowLocksResponse show_locks(ShowLocksRequest rqst) throws TException {
+ return getTxnHandler().showLocks(rqst);
+ }
+
+ @Override
+ public void heartbeat(HeartbeatRequest ids) throws TException {
+ getTxnHandler().heartbeat(ids);
+ }
+
+ @Override
+ public HeartbeatTxnRangeResponse heartbeat_txn_range(HeartbeatTxnRangeRequest rqst)
+ throws TException {
+ return getTxnHandler().heartbeatTxnRange(rqst);
+ }
+ @Deprecated
+ @Override
+ public void compact(CompactionRequest rqst) throws TException {
+ compact2(rqst);
+ }
+ @Override
+ public CompactionResponse compact2(CompactionRequest rqst) throws TException {
+ return getTxnHandler().compact(rqst);
+ }
+
+ @Override
+ public ShowCompactResponse show_compact(ShowCompactRequest rqst) throws TException {
+ ShowCompactResponse response = getTxnHandler().showCompact(rqst);
+ response.setCompacts(FilterUtils.filterCompactionsIfEnabled(isServerFilterEnabled,
+ filterHook, getDefaultCatalog(conf), response.getCompacts()));
+ return response;
+ }
+
+ @Override
+ public boolean submit_for_cleanup(CompactionRequest rqst, long highestWriteId, long txnId)
+ throws TException {
+ return getTxnHandler().submitForCleanup(rqst, highestWriteId, txnId);
+ }
+
+ @Override
+ public void add_dynamic_partitions(AddDynamicPartitions rqst) throws TException {
+ getTxnHandler().addDynamicPartitions(rqst);
+ }
+
+ @Override
+ public ReplayedTxnsForPolicyResult get_replayed_txns_for_policy(String policyName) throws MetaException {
+ Exception ex = null;
+ ReplayedTxnsForPolicyResult ret = null;
+ try {
+ startFunction("get_replayed_txns_for_policy");
+ ret = getTxnHandler().getReplayedTxnsForPolicy(policyName);
+ } catch (Exception e) {
+ ex = e;
+ throw new MetaException(
+ "Failed to get replayed txns details for policy '" + policyName + "': " + e.getMessage());
+ } finally {
+ endFunction("get_replayed_txns_for_policy", ret != null, ex);
+ }
+ return ret;
+ }
+
+ @Override
+ public LockResponse get_lock_materialization_rebuild(String dbName, String tableName, long txnId)
+ throws TException {
+ return getTxnHandler().lockMaterializationRebuild(dbName, tableName, txnId);
+ }
+
+ @Deprecated
+ @Override
+ public boolean heartbeat_lock_materialization_rebuild(String dbName, String tableName, long txnId)
+ throws TException {
+ return getTxnHandler().heartbeatLockMaterializationRebuild(dbName, tableName, txnId);
+ }
+
+ @Override
+ public void update_transaction_statistics(UpdateTransactionalStatsRequest req) throws TException {
+ getTxnHandler().updateTransactionStatistics(req);
+ }
+
+ @Override
+ public Materialization get_materialization_invalidation_info(final CreationMetadata cm,
+ final String validTxnList) throws MetaException {
+ return getTxnHandler().getMaterializationInvalidationInfo(cm, validTxnList);
+ }
+
+ @Override
+ public LockResponse get_lock_materialization_rebuild_req(LockMaterializationRebuildRequest rqst)
+ throws TException {
+ return getTxnHandler().lockMaterializationRebuild(rqst);
+ }
+
+ @Override
+ public boolean heartbeat_lock_materialization_rebuild_req(LockMaterializationRebuildRequest rqst)
+ throws TException {
+ return getTxnHandler().heartbeatLockMaterializationRebuild(rqst);
+ }
+}
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreServerUtils.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreServerUtils.java
index 04e1f2a9c80..56b2d9ba59b 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreServerUtils.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreServerUtils.java
@@ -42,6 +42,7 @@
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@@ -1823,4 +1824,24 @@ public static boolean canUpdateStats(Configuration conf, Table tbl) {
return true;
}
+ public static List<String> getPartValsFromName(Table t, String partName)
+ throws MetaException, InvalidObjectException {
+ if (t == null) {
+ throw new MetaException("Table cannot be null");
+ }
+ // Unescape the partition name
+ LinkedHashMap<String, String> hm = Warehouse.makeSpecFromName(partName);
+
+ List<String> partVals = new ArrayList<>();
+ for (FieldSchema field : t.getPartitionKeys()) {
+ String key = field.getName();
+ String val = hm.get(key);
+ if (val == null) {
+ throw new InvalidObjectException("incomplete partition name - missing " + key);
+ }
+ partVals.add(val);
+ }
+ return partVals;
+ }
+
}
diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/IpAddressListener.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/IpAddressListener.java
index 428d8606c60..75d7ae2b4d0 100644
--- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/IpAddressListener.java
+++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/IpAddressListener.java
@@ -32,6 +32,7 @@
import org.apache.hadoop.hive.metastore.events.DropPartitionEvent;
import org.apache.hadoop.hive.metastore.events.DropTableEvent;
import org.apache.hadoop.hive.metastore.events.LoadPartitionDoneEvent;
+import org.apache.hadoop.hive.metastore.handler.BaseHandler;
import org.junit.Assert;
/** An implementation for MetaStoreEventListener which checks that the IP Address stored in
@@ -48,7 +49,7 @@ public IpAddressListener(Configuration config) {
private void checkIpAddress() {
try {
String localhostIp = InetAddress.getByName(LOCAL_HOST).getHostAddress();
- Assert.assertEquals(localhostIp, HMSHandler.getThreadLocalIpAddress());
+ Assert.assertEquals(localhostIp, BaseHandler.getThreadLocalIpAddress());
} catch (UnknownHostException e) {
Assert.assertTrue("InetAddress.getLocalHost threw an exception: " + e.getMessage(), false);
}