You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by vi...@apache.org on 2018/08/27 21:49:53 UTC
hive git commit: HIVE-20388 : Move common classes out of
metastore-server (Alexander Kolbasov reviewed by Vihang Karajgaonkar)
Repository: hive
Updated Branches:
refs/heads/master 06fb30235 -> a3f0f99d5
HIVE-20388 : Move common classes out of metastore-server (Alexander Kolbasov reviewed by Vihang Karajgaonkar)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a3f0f99d
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a3f0f99d
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a3f0f99d
Branch: refs/heads/master
Commit: a3f0f99d53ab916143c49cb8611f701bfa70e59c
Parents: 06fb302
Author: Alexander Kolbasov <ak...@cloudera.com>
Authored: Mon Aug 27 14:41:12 2018 -0700
Committer: Vihang Karajgaonkar <vi...@apache.org>
Committed: Mon Aug 27 14:41:12 2018 -0700
----------------------------------------------------------------------
.../hadoop/hive/ql/lockmgr/DbTxnManager.java | 7 +-
.../hive/ql/stats/StatsUpdaterThread.java | 3 +-
.../ql/util/HiveStrictManagedMigration.java | 3 +-
.../hive/service/auth/HiveAuthFactory.java | 10 +-
.../hive/metastore/txn/TxnCommonUtils.java | 127 +++++++++++++++++++
.../hadoop/hive/metastore/HiveAlterHandler.java | 8 +-
.../hadoop/hive/metastore/HiveMetaStore.java | 4 +-
.../hive/metastore/HiveMetaStoreClient.java | 8 +-
.../MaterializationsRebuildLockCleanerTask.java | 3 +-
.../MetastoreDelegationTokenManager.java | 6 +-
.../hadoop/hive/metastore/txn/TxnHandler.java | 2 +-
.../hadoop/hive/metastore/txn/TxnUtils.java | 104 +--------------
.../metastore/utils/MetaStoreServerUtils.java | 85 ++++++++++++-
.../hive/metastore/utils/MetaStoreUtils.java | 46 +------
.../hive/metastore/utils/SecurityUtils.java | 53 +-------
.../HiveMetaStoreClientPreCatalog.java | 8 +-
16 files changed, 246 insertions(+), 231 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/a3f0f99d/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java
index f746d2d..4e3d74c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java
@@ -18,13 +18,11 @@
package org.apache.hadoop.hive.ql.lockmgr;
import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.common.JavaUtils;
import org.apache.hadoop.hive.common.ValidTxnList;
import org.apache.hadoop.hive.common.ValidTxnWriteIdList;
-import org.apache.hadoop.hive.common.ValidWriteIdList;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.IMetaStoreClient;
import org.apache.hadoop.hive.metastore.LockRequestBuilder;
@@ -37,7 +35,7 @@ import org.apache.hadoop.hive.metastore.api.NoSuchTxnException;
import org.apache.hadoop.hive.metastore.api.TxnAbortedException;
import org.apache.hadoop.hive.metastore.api.TxnToWriteId;
import org.apache.hadoop.hive.metastore.api.CommitTxnRequest;
-import org.apache.hadoop.hive.metastore.txn.TxnUtils;
+import org.apache.hadoop.hive.metastore.txn.TxnCommonUtils;
import org.apache.hadoop.hive.ql.Context;
import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.QueryPlan;
@@ -50,7 +48,6 @@ import org.apache.hadoop.hive.ql.plan.LockDatabaseDesc;
import org.apache.hadoop.hive.ql.plan.LockTableDesc;
import org.apache.hadoop.hive.ql.plan.UnlockDatabaseDesc;
import org.apache.hadoop.hive.ql.plan.UnlockTableDesc;
-import org.apache.hadoop.hive.ql.session.*;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hive.common.util.ShutdownHookManager;
import org.apache.thrift.TException;
@@ -676,7 +673,7 @@ public final class DbTxnManager extends HiveTxnManagerImpl {
assert isTxnOpen();
assert validTxnList != null && !validTxnList.isEmpty();
try {
- return TxnUtils.createValidTxnWriteIdList(
+ return TxnCommonUtils.createValidTxnWriteIdList(
txnId, getMS().getValidWriteIds(tableList, validTxnList));
} catch (TException e) {
throw new LockException(ErrorMsg.METASTORE_COMMUNICATION_FAILED.getMsg(), e);
http://git-wip-us.apache.org/repos/asf/hive/blob/a3f0f99d/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUpdaterThread.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUpdaterThread.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUpdaterThread.java
index a50ec18..981f6ae 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUpdaterThread.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUpdaterThread.java
@@ -52,6 +52,7 @@ import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf.StatsUpdateMode;
+import org.apache.hadoop.hive.metastore.txn.TxnCommonUtils;
import org.apache.hadoop.hive.metastore.txn.TxnStore;
import org.apache.hadoop.hive.metastore.txn.TxnUtils;
import org.apache.hadoop.hive.ql.DriverUtils;
@@ -500,7 +501,7 @@ public class StatsUpdaterThread extends Thread implements MetaStoreThread {
// TODO: acid utils don't support catalogs
GetValidWriteIdsRequest req = new GetValidWriteIdsRequest(
Lists.newArrayList(fullTableName.getDbTable()), null);
- return TxnUtils.createValidReaderWriteIdList(
+ return TxnCommonUtils.createValidReaderWriteIdList(
txnHandler.getValidWriteIds(req).getTblValidWriteIds().get(0));
}
http://git-wip-us.apache.org/repos/asf/hive/blob/a3f0f99d/ql/src/java/org/apache/hadoop/hive/ql/util/HiveStrictManagedMigration.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/util/HiveStrictManagedMigration.java b/ql/src/java/org/apache/hadoop/hive/ql/util/HiveStrictManagedMigration.java
index 54df541..13a8af7 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/util/HiveStrictManagedMigration.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/util/HiveStrictManagedMigration.java
@@ -54,6 +54,7 @@ import org.apache.hadoop.hive.metastore.IMetaStoreClient;
import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.TransactionalValidationListener;
import org.apache.hadoop.hive.metastore.Warehouse;
+import org.apache.hadoop.hive.metastore.txn.TxnCommonUtils;
import org.apache.hadoop.hive.metastore.txn.TxnUtils;
import org.apache.hadoop.hive.metastore.utils.HiveStrictManagedUtils;
import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
@@ -994,7 +995,7 @@ public class HiveStrictManagedMigration {
String fqn = table.getDbName() + "." + table.getTableName();
List<TableValidWriteIds> writeIdsObj = msc.getValidWriteIds(
Lists.newArrayList(fqn), txns.toString());
- String validWriteIds = TxnUtils.createValidTxnWriteIdList(txnId, writeIdsObj)
+ String validWriteIds = TxnCommonUtils.createValidTxnWriteIdList(txnId, writeIdsObj)
.getTableValidWriteIdList(fqn).writeToString();
long writeId = msc.allocateTableWriteId(txnId, table.getDbName(), table.getTableName());
result = new TxnCtx(writeId, validWriteIds, txnId);
http://git-wip-us.apache.org/repos/asf/hive/blob/a3f0f99d/service/src/java/org/apache/hive/service/auth/HiveAuthFactory.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/service/auth/HiveAuthFactory.java b/service/src/java/org/apache/hive/service/auth/HiveAuthFactory.java
index d23e438..f5f6d49 100644
--- a/service/src/java/org/apache/hive/service/auth/HiveAuthFactory.java
+++ b/service/src/java/org/apache/hive/service/auth/HiveAuthFactory.java
@@ -27,21 +27,15 @@ import javax.security.auth.login.LoginException;
import javax.security.sasl.AuthenticationException;
import javax.security.sasl.Sasl;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.conf.HiveConfUtil;
+import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils;
import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.shims.HadoopShims.KerberosNameShim;
import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.hive.metastore.security.DBTokenStore;
-import org.apache.hadoop.hive.metastore.security.DelegationTokenStore;
import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge;
-import org.apache.hadoop.hive.metastore.security.MemoryTokenStore;
import org.apache.hadoop.hive.metastore.security.MetastoreDelegationTokenManager;
-import org.apache.hadoop.hive.metastore.security.ZooKeeperTokenStore;
-import org.apache.hadoop.hive.metastore.utils.SecurityUtils;
import org.apache.hadoop.security.SaslRpcServer.AuthMethod;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
@@ -97,7 +91,7 @@ public class HiveAuthFactory {
delegationTokenManager = new MetastoreDelegationTokenManager();
try {
Object baseHandler = null;
- String tokenStoreClass = SecurityUtils.getTokenStoreClassName(conf);
+ String tokenStoreClass = MetaStoreServerUtils.getTokenStoreClassName(conf);
if (tokenStoreClass.equals(DBTokenStore.class.getName())) {
// IMetaStoreClient is needed to access token store if DBTokenStore is to be used. It
http://git-wip-us.apache.org/repos/asf/hive/blob/a3f0f99d/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnCommonUtils.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnCommonUtils.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnCommonUtils.java
new file mode 100644
index 0000000..94cb18d
--- /dev/null
+++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnCommonUtils.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.txn;
+
+import org.apache.hadoop.hive.common.ValidReadTxnList;
+import org.apache.hadoop.hive.common.ValidReaderWriteIdList;
+import org.apache.hadoop.hive.common.ValidTxnList;
+import org.apache.hadoop.hive.common.ValidTxnWriteIdList;
+import org.apache.hadoop.hive.metastore.api.GetOpenTxnsResponse;
+import org.apache.hadoop.hive.metastore.api.TableValidWriteIds;
+
+import java.util.BitSet;
+import java.util.Collections;
+import java.util.List;
+
+public class TxnCommonUtils {
+ /**
+ * Transform a {@link org.apache.hadoop.hive.metastore.api.GetOpenTxnsResponse} to a
+ * {@link org.apache.hadoop.hive.common.ValidTxnList}. This assumes that the caller intends to
+ * read the files, and thus treats both open and aborted transactions as invalid.
+ * @param txns txn list from the metastore
+ * @param currentTxn Current transaction that the user has open. If this is greater than 0 it
+ * will be removed from the exceptions list so that the user sees his own
+ * transaction as valid.
+ * @return a valid txn list.
+ */
+ public static ValidTxnList createValidReadTxnList(GetOpenTxnsResponse txns, long currentTxn) {
+ /*
+ * The highWaterMark should be min(currentTxn,txns.getTxn_high_water_mark()) assuming currentTxn>0
+ * otherwise if currentTxn=7 and 8 commits before 7, then 7 will see result of 8 which
+ * doesn't make sense for Snapshot Isolation. Of course for Read Committed, the list should
+ * include the latest committed set.
+ */
+ long highWaterMark = (currentTxn > 0) ? Math.min(currentTxn, txns.getTxn_high_water_mark())
+ : txns.getTxn_high_water_mark();
+
+ // Open txns are already sorted in ascending order. This list may or may not include HWM
+ // but it is guaranteed that list won't have txn > HWM. But, if we overwrite the HWM with currentTxn
+ // then need to truncate the exceptions list accordingly.
+ List<Long> openTxns = txns.getOpen_txns();
+
+ // We care only about open/aborted txns below currentTxn and hence the size should be determined
+ // for the exceptions list. The currentTxn will be missing in openTxns list only in rare case like
+ // txn is aborted by AcidHouseKeeperService and compactor actually cleans up the aborted txns.
+ // So, for such cases, we get negative value for sizeToHwm with found position for currentTxn, and so,
+ // we just negate it to get the size.
+ int sizeToHwm = (currentTxn > 0) ? Collections.binarySearch(openTxns, currentTxn) : openTxns.size();
+ sizeToHwm = (sizeToHwm < 0) ? (-sizeToHwm) : sizeToHwm;
+ long[] exceptions = new long[sizeToHwm];
+ BitSet inAbortedBits = BitSet.valueOf(txns.getAbortedBits());
+ BitSet outAbortedBits = new BitSet();
+ long minOpenTxnId = Long.MAX_VALUE;
+ int i = 0;
+ for (long txn : openTxns) {
+ // For snapshot isolation, we don't care about txns greater than current txn and so stop here.
+ // Also, we need not include current txn to exceptions list.
+ if ((currentTxn > 0) && (txn >= currentTxn)) {
+ break;
+ }
+ if (inAbortedBits.get(i)) {
+ outAbortedBits.set(i);
+ } else if (minOpenTxnId == Long.MAX_VALUE) {
+ minOpenTxnId = txn;
+ }
+ exceptions[i++] = txn;
+ }
+ return new ValidReadTxnList(exceptions, outAbortedBits, highWaterMark, minOpenTxnId);
+ }
+
+ /**
+ * Transform a {@link org.apache.hadoop.hive.metastore.api.GetValidWriteIdsResponse} to a
+ * {@link org.apache.hadoop.hive.common.ValidTxnWriteIdList}. This assumes that the caller intends to
+ * read the files, and thus treats both open and aborted transactions as invalid.
+ * @param currentTxnId current txn ID for which we get the valid write ids list
+ * @param list valid write ids list from the metastore
+ * @return a valid write IDs list for the whole transaction.
+ */
+ public static ValidTxnWriteIdList createValidTxnWriteIdList(Long currentTxnId,
+ List<TableValidWriteIds> validIds) {
+ ValidTxnWriteIdList validTxnWriteIdList = new ValidTxnWriteIdList(currentTxnId);
+ for (TableValidWriteIds tableWriteIds : validIds) {
+ validTxnWriteIdList.addTableValidWriteIdList(createValidReaderWriteIdList(tableWriteIds));
+ }
+ return validTxnWriteIdList;
+ }
+
+ /**
+ * Transform a {@link TableValidWriteIds} to a
+ * {@link org.apache.hadoop.hive.common.ValidReaderWriteIdList}. This assumes that the caller intends to
+ * read the files, and thus treats both open and aborted write ids as invalid.
+ * @param tableWriteIds valid write ids for the given table from the metastore
+ * @return a valid write IDs list for the input table
+ */
+ public static ValidReaderWriteIdList createValidReaderWriteIdList(TableValidWriteIds tableWriteIds) {
+ String fullTableName = tableWriteIds.getFullTableName();
+ long highWater = tableWriteIds.getWriteIdHighWaterMark();
+ List<Long> invalids = tableWriteIds.getInvalidWriteIds();
+ BitSet abortedBits = BitSet.valueOf(tableWriteIds.getAbortedBits());
+ long[] exceptions = new long[invalids.size()];
+ int i = 0;
+ for (long writeId : invalids) {
+ exceptions[i++] = writeId;
+ }
+ if (tableWriteIds.isSetMinOpenWriteId()) {
+ return new ValidReaderWriteIdList(fullTableName, exceptions, abortedBits, highWater,
+ tableWriteIds.getMinOpenWriteId());
+ } else {
+ return new ValidReaderWriteIdList(fullTableName, exceptions, abortedBits, highWater);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/a3f0f99d/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
index c551b80..6a1cd3b 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
@@ -316,7 +316,7 @@ public class HiveAlterHandler implements AlterHandler {
}
} else {
// operations other than table rename
- if (MetaStoreUtils.requireCalStats(null, null, newt, environmentContext) &&
+ if (MetaStoreServerUtils.requireCalStats(null, null, newt, environmentContext) &&
!isPartitionedTable) {
Database db = msdb.getDatabase(catName, newDbName);
// Update table stats. For partitioned table, we update stats in alterPartition()
@@ -467,7 +467,7 @@ public class HiveAlterHandler implements AlterHandler {
"Unable to alter partition because table or database does not exist.");
}
oldPart = msdb.getPartition(catName, dbname, name, new_part.getValues());
- if (MetaStoreUtils.requireCalStats(oldPart, new_part, tbl, environmentContext)) {
+ if (MetaStoreServerUtils.requireCalStats(oldPart, new_part, tbl, environmentContext)) {
// if stats are same, no need to update
if (MetaStoreServerUtils.isFastStatsSame(oldPart, new_part)) {
MetaStoreServerUtils.updateBasicState(environmentContext, new_part.getParameters());
@@ -611,7 +611,7 @@ public class HiveAlterHandler implements AlterHandler {
new_part.getSd().setLocation(oldPart.getSd().getLocation());
}
- if (MetaStoreUtils.requireCalStats(oldPart, new_part, tbl, environmentContext)) {
+ if (MetaStoreServerUtils.requireCalStats(oldPart, new_part, tbl, environmentContext)) {
MetaStoreServerUtils.updatePartitionStatsFast(
new_part, tbl, wh, false, true, environmentContext, false);
}
@@ -711,7 +711,7 @@ public class HiveAlterHandler implements AlterHandler {
oldParts.add(oldTmpPart);
partValsList.add(tmpPart.getValues());
- if (MetaStoreUtils.requireCalStats(oldTmpPart, tmpPart, tbl, environmentContext)) {
+ if (MetaStoreServerUtils.requireCalStats(oldTmpPart, tmpPart, tbl, environmentContext)) {
// Check if stats are same, no need to update
if (MetaStoreServerUtils.isFastStatsSame(oldTmpPart, tmpPart)) {
MetaStoreServerUtils.updateBasicState(environmentContext, tmpPart.getParameters());
http://git-wip-us.apache.org/repos/asf/hive/blob/a3f0f99d/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
index 6b660f5..33b22a9 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
@@ -1862,7 +1862,7 @@ public class HiveMetaStore extends ThriftHiveMetastore {
}
}
if (MetastoreConf.getBoolVar(conf, ConfVars.STATS_AUTO_GATHER) &&
- !MetaStoreServerUtils.isView(tbl)) {
+ !MetaStoreUtils.isView(tbl)) {
MetaStoreServerUtils.updateTableStatsSlow(db, tbl, wh, madeDir, false, envContext);
}
@@ -3804,7 +3804,7 @@ public class HiveMetaStore extends ThriftHiveMetastore {
updateStatsTbl = !Boolean.valueOf(tblParams.get(StatsSetupConst.DO_NOT_UPDATE_STATS));
}
if (!MetastoreConf.getBoolVar(conf, ConfVars.STATS_AUTO_GATHER) ||
- MetaStoreServerUtils.isView(tbl) ||
+ MetaStoreUtils.isView(tbl) ||
!updateStatsTbl) {
return false;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/a3f0f99d/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
index faf6810..a0ff79c 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
@@ -62,7 +62,7 @@ import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars;
import org.apache.hadoop.hive.metastore.hooks.URIResolverHook;
import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy;
import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge;
-import org.apache.hadoop.hive.metastore.txn.TxnUtils;
+import org.apache.hadoop.hive.metastore.txn.TxnCommonUtils;
import org.apache.hadoop.hive.metastore.utils.JavaUtils;
import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
import org.apache.hadoop.hive.metastore.utils.ObjectPair;
@@ -2726,19 +2726,19 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable {
@Override
public ValidTxnList getValidTxns() throws TException {
- return TxnUtils.createValidReadTxnList(client.get_open_txns(), 0);
+ return TxnCommonUtils.createValidReadTxnList(client.get_open_txns(), 0);
}
@Override
public ValidTxnList getValidTxns(long currentTxn) throws TException {
- return TxnUtils.createValidReadTxnList(client.get_open_txns(), currentTxn);
+ return TxnCommonUtils.createValidReadTxnList(client.get_open_txns(), currentTxn);
}
@Override
public ValidWriteIdList getValidWriteIds(String fullTableName) throws TException {
GetValidWriteIdsRequest rqst = new GetValidWriteIdsRequest(Collections.singletonList(fullTableName), null);
GetValidWriteIdsResponse validWriteIds = client.get_valid_write_ids(rqst);
- return TxnUtils.createValidReaderWriteIdList(validWriteIds.getTblValidWriteIds().get(0));
+ return TxnCommonUtils.createValidReaderWriteIdList(validWriteIds.getTblValidWriteIds().get(0));
}
@Override
http://git-wip-us.apache.org/repos/asf/hive/blob/a3f0f99d/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MaterializationsRebuildLockCleanerTask.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MaterializationsRebuildLockCleanerTask.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MaterializationsRebuildLockCleanerTask.java
index 9ce7d6d..d35c960 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MaterializationsRebuildLockCleanerTask.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MaterializationsRebuildLockCleanerTask.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hive.metastore;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.common.ValidTxnList;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.hadoop.hive.metastore.txn.TxnCommonUtils;
import org.apache.hadoop.hive.metastore.txn.TxnStore;
import org.apache.hadoop.hive.metastore.txn.TxnUtils;
import org.slf4j.Logger;
@@ -62,7 +63,7 @@ public class MaterializationsRebuildLockCleanerTask implements MetastoreTaskThre
TxnStore.MutexAPI.LockHandle handle = null;
try {
handle = txnHandler.getMutexAPI().acquireLock(TxnStore.MUTEX_KEY.MaterializationRebuild.name());
- ValidTxnList validTxnList = TxnUtils.createValidReadTxnList(txnHandler.getOpenTxns(), 0);
+ ValidTxnList validTxnList = TxnCommonUtils.createValidReadTxnList(txnHandler.getOpenTxns(), 0);
long removedCnt = txnHandler.cleanupMaterializationRebuildLocks(validTxnList,
MetastoreConf.getTimeVar(conf, MetastoreConf.ConfVars.TXN_TIMEOUT, TimeUnit.MILLISECONDS));
if (removedCnt > 0) {
http://git-wip-us.apache.org/repos/asf/hive/blob/a3f0f99d/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/security/MetastoreDelegationTokenManager.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/security/MetastoreDelegationTokenManager.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/security/MetastoreDelegationTokenManager.java
index 8d84fd7..b35dc7c 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/security/MetastoreDelegationTokenManager.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/security/MetastoreDelegationTokenManager.java
@@ -20,13 +20,11 @@
package org.apache.hadoop.hive.metastore.security;
import java.io.IOException;
-import java.security.PrivilegedExceptionAction;
import java.util.concurrent.TimeUnit;
-import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
-import org.apache.hadoop.hive.metastore.utils.SecurityUtils;
+import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.UserGroupInformation;
@@ -139,7 +137,7 @@ public class MetastoreDelegationTokenManager {
}
private DelegationTokenStore getTokenStore(Configuration conf) throws IOException {
- String tokenStoreClassName = SecurityUtils.getTokenStoreClassName(conf);
+ String tokenStoreClassName = MetaStoreServerUtils.getTokenStoreClassName(conf);
try {
Class<? extends DelegationTokenStore> storeClass =
Class.forName(tokenStoreClassName).asSubclass(DelegationTokenStore.class);
http://git-wip-us.apache.org/repos/asf/hive/blob/a3f0f99d/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
index 015dfd9..2df61b4 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
@@ -1252,7 +1252,7 @@ abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI {
validTxnList = new ValidReadTxnList(rqst.getValidTxnList());
} else {
// Passing 0 for currentTxn means, this validTxnList is not wrt to any txn
- validTxnList = TxnUtils.createValidReadTxnList(getOpenTxns(), 0);
+ validTxnList = TxnCommonUtils.createValidReadTxnList(getOpenTxns(), 0);
}
try {
/**
http://git-wip-us.apache.org/repos/asf/hive/blob/a3f0f99d/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java
index bd202ed..3bb1f0c 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java
@@ -20,19 +20,16 @@ package org.apache.hadoop.hive.metastore.txn;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.common.ValidCompactorWriteIdList;
import org.apache.hadoop.hive.common.ValidReaderWriteIdList;
-import org.apache.hadoop.hive.common.ValidReadTxnList;
-import org.apache.hadoop.hive.common.ValidTxnList;
-import org.apache.hadoop.hive.common.ValidTxnWriteIdList;
-import org.apache.hadoop.hive.common.ValidWriteIdList;
import org.apache.hadoop.hive.metastore.TransactionalValidationListener;
-import org.apache.hadoop.hive.metastore.api.*;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.metastore.api.TableValidWriteIds;
+import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars;
import org.apache.hadoop.hive.metastore.utils.JavaUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.Collections;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.BitSet;
@@ -49,101 +46,6 @@ public class TxnUtils {
static final public char STAT_OBSOLETE = 's';
/**
- * Transform a {@link org.apache.hadoop.hive.metastore.api.GetOpenTxnsResponse} to a
- * {@link org.apache.hadoop.hive.common.ValidTxnList}. This assumes that the caller intends to
- * read the files, and thus treats both open and aborted transactions as invalid.
- * @param txns txn list from the metastore
- * @param currentTxn Current transaction that the user has open. If this is greater than 0 it
- * will be removed from the exceptions list so that the user sees his own
- * transaction as valid.
- * @return a valid txn list.
- */
- public static ValidTxnList createValidReadTxnList(GetOpenTxnsResponse txns, long currentTxn) {
- /*
- * The highWaterMark should be min(currentTxn,txns.getTxn_high_water_mark()) assuming currentTxn>0
- * otherwise if currentTxn=7 and 8 commits before 7, then 7 will see result of 8 which
- * doesn't make sense for Snapshot Isolation. Of course for Read Committed, the list should
- * include the latest committed set.
- */
- long highWaterMark = (currentTxn > 0) ? Math.min(currentTxn, txns.getTxn_high_water_mark())
- : txns.getTxn_high_water_mark();
-
- // Open txns are already sorted in ascending order. This list may or may not include HWM
- // but it is guaranteed that list won't have txn > HWM. But, if we overwrite the HWM with currentTxn
- // then need to truncate the exceptions list accordingly.
- List<Long> openTxns = txns.getOpen_txns();
-
- // We care only about open/aborted txns below currentTxn and hence the size should be determined
- // for the exceptions list. The currentTxn will be missing in openTxns list only in rare case like
- // txn is aborted by AcidHouseKeeperService and compactor actually cleans up the aborted txns.
- // So, for such cases, we get negative value for sizeToHwm with found position for currentTxn, and so,
- // we just negate it to get the size.
- int sizeToHwm = (currentTxn > 0) ? Collections.binarySearch(openTxns, currentTxn) : openTxns.size();
- sizeToHwm = (sizeToHwm < 0) ? (-sizeToHwm) : sizeToHwm;
- long[] exceptions = new long[sizeToHwm];
- BitSet inAbortedBits = BitSet.valueOf(txns.getAbortedBits());
- BitSet outAbortedBits = new BitSet();
- long minOpenTxnId = Long.MAX_VALUE;
- int i = 0;
- for (long txn : openTxns) {
- // For snapshot isolation, we don't care about txns greater than current txn and so stop here.
- // Also, we need not include current txn to exceptions list.
- if ((currentTxn > 0) && (txn >= currentTxn)) {
- break;
- }
- if (inAbortedBits.get(i)) {
- outAbortedBits.set(i);
- } else if (minOpenTxnId == Long.MAX_VALUE) {
- minOpenTxnId = txn;
- }
- exceptions[i++] = txn;
- }
- return new ValidReadTxnList(exceptions, outAbortedBits, highWaterMark, minOpenTxnId);
- }
-
- /**
- * Transform a {@link org.apache.hadoop.hive.metastore.api.GetValidWriteIdsResponse} to a
- * {@link org.apache.hadoop.hive.common.ValidTxnWriteIdList}. This assumes that the caller intends to
- * read the files, and thus treats both open and aborted transactions as invalid.
- * @param currentTxnId current txn ID for which we get the valid write ids list
- * @param list valid write ids list from the metastore
- * @return a valid write IDs list for the whole transaction.
- */
- public static ValidTxnWriteIdList createValidTxnWriteIdList(Long currentTxnId,
- List<TableValidWriteIds> validIds) {
- ValidTxnWriteIdList validTxnWriteIdList = new ValidTxnWriteIdList(currentTxnId);
- for (TableValidWriteIds tableWriteIds : validIds) {
- validTxnWriteIdList.addTableValidWriteIdList(createValidReaderWriteIdList(tableWriteIds));
- }
- return validTxnWriteIdList;
- }
-
- /**
- * Transform a {@link org.apache.hadoop.hive.metastore.api.TableValidWriteIds} to a
- * {@link org.apache.hadoop.hive.common.ValidReaderWriteIdList}. This assumes that the caller intends to
- * read the files, and thus treats both open and aborted write ids as invalid.
- * @param tableWriteIds valid write ids for the given table from the metastore
- * @return a valid write IDs list for the input table
- */
- public static ValidReaderWriteIdList createValidReaderWriteIdList(TableValidWriteIds tableWriteIds) {
- String fullTableName = tableWriteIds.getFullTableName();
- long highWater = tableWriteIds.getWriteIdHighWaterMark();
- List<Long> invalids = tableWriteIds.getInvalidWriteIds();
- BitSet abortedBits = BitSet.valueOf(tableWriteIds.getAbortedBits());
- long[] exceptions = new long[invalids.size()];
- int i = 0;
- for (long writeId : invalids) {
- exceptions[i++] = writeId;
- }
- if (tableWriteIds.isSetMinOpenWriteId()) {
- return new ValidReaderWriteIdList(fullTableName, exceptions, abortedBits, highWater,
- tableWriteIds.getMinOpenWriteId());
- } else {
- return new ValidReaderWriteIdList(fullTableName, exceptions, abortedBits, highWater);
- }
- }
-
- /**
* Transform a {@link org.apache.hadoop.hive.metastore.api.TableValidWriteIds} to a
* {@link org.apache.hadoop.hive.common.ValidCompactorWriteIdList}. This assumes that the caller intends to
* compact the files, and thus treats only open transactions/write ids as invalid. Additionally any
http://git-wip-us.apache.org/repos/asf/hive/blob/a3f0f99d/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreServerUtils.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreServerUtils.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreServerUtils.java
index 506d9ab..10ff9df 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreServerUtils.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreServerUtils.java
@@ -79,7 +79,10 @@ import org.apache.hadoop.hive.metastore.columnstats.merge.ColumnStatsMerger;
import org.apache.hadoop.hive.metastore.columnstats.merge.ColumnStatsMergerFactory;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy;
+import org.apache.hadoop.hive.metastore.security.DBTokenStore;
import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge;
+import org.apache.hadoop.hive.metastore.security.MemoryTokenStore;
+import org.apache.hadoop.hive.metastore.security.ZooKeeperTokenStore;
import org.apache.hadoop.security.authorize.DefaultImpersonationProvider;
import org.apache.hadoop.security.authorize.ProxyUsers;
import org.apache.hadoop.util.MachineList;
@@ -105,6 +108,7 @@ public class MetaStoreServerUtils {
return org.apache.commons.lang.StringUtils.defaultString(string);
}
};
+ private static final String DELEGATION_TOKEN_STORE_CLS = "hive.cluster.delegation.token.store.class";
/**
* We have a need to sanity-check the map before conversion from persisted objects to
@@ -347,13 +351,6 @@ public class MetaStoreServerUtils {
return true;
}
- public static boolean isView(Table table) {
- if (table == null) {
- return false;
- }
- return TableType.VIRTUAL_VIEW.toString().equals(table.getTableType());
- }
-
/**
* @param partParams
* @return True if the passed Parameters Map contains values for all "Fast Stats".
@@ -906,6 +903,80 @@ public class MetaStoreServerUtils {
return Character.isLetterOrDigit(c) || c == '_';
}
+ // check if stats need to be (re)calculated
+ public static boolean requireCalStats(Partition oldPart,
+ Partition newPart, Table tbl,
+ EnvironmentContext environmentContext) {
+
+ if (environmentContext != null
+ && environmentContext.isSetProperties()
+ && StatsSetupConst.TRUE.equals(environmentContext.getProperties().get(
+ StatsSetupConst.DO_NOT_UPDATE_STATS))) {
+ return false;
+ }
+
+ if (MetaStoreUtils.isView(tbl)) {
+ return false;
+ }
+
+ if (oldPart == null && newPart == null) {
+ return true;
+ }
+
+ // requires to calculate stats if new partition doesn't have it
+ if ((newPart == null) || (newPart.getParameters() == null)
+ || !containsAllFastStats(newPart.getParameters())) {
+ return true;
+ }
+
+ if (environmentContext != null && environmentContext.isSetProperties()) {
+ String statsType = environmentContext.getProperties().get(StatsSetupConst.STATS_GENERATED);
+ // no matter STATS_GENERATED is USER or TASK, all need to re-calculate the stats:
+ // USER: alter table .. update statistics
+ // TASK: from some sql operation which could collect and compute stats
+ if (StatsSetupConst.TASK.equals(statsType) || StatsSetupConst.USER.equals(statsType)) {
+ return true;
+ }
+ }
+
+ // requires to calculate stats if new and old have different fast stats
+ return !isFastStatsSame(oldPart, newPart);
+ }
+
+ /**
+ * This method should be used to return the metastore specific tokenstore class name to main
+ * backwards compatibility
+ *
+ * @param conf - HiveConf object
+ * @return the tokenStoreClass name from the HiveConf. It maps the hive specific tokenstoreclass
+ * name to metastore module specific class name. For eg:
+ * hive.cluster.delegation.token.store.class is set to
+ * org.apache.hadoop.hive.thrift.MemoryTokenStore it returns the equivalent tokenstore
+ * class defined in the metastore module which is
+ * org.apache.hadoop.hive.metastore.security.MemoryTokenStore Similarly,
+ * org.apache.hadoop.hive.thrift.DBTokenStore maps to
+ * org.apache.hadoop.hive.metastore.security.DBTokenStore and
+ * org.apache.hadoop.hive.thrift.ZooKeeperTokenStore maps to
+ * org.apache.hadoop.hive.metastore.security.ZooKeeperTokenStore
+ */
+ public static String getTokenStoreClassName(Configuration conf) {
+ String tokenStoreClass = conf.get(DELEGATION_TOKEN_STORE_CLS, "");
+ if (StringUtils.isBlank(tokenStoreClass)) {
+ // default tokenstore is MemoryTokenStore
+ return MemoryTokenStore.class.getName();
+ }
+ switch (tokenStoreClass) {
+ case "org.apache.hadoop.hive.thrift.DBTokenStore":
+ return DBTokenStore.class.getName();
+ case "org.apache.hadoop.hive.thrift.MemoryTokenStore":
+ return MemoryTokenStore.class.getName();
+ case "org.apache.hadoop.hive.thrift.ZooKeeperTokenStore":
+ return ZooKeeperTokenStore.class.getName();
+ default:
+ return tokenStoreClass;
+ }
+ }
+
// ColumnStatisticsObj with info about its db, table, partition (if table is partitioned)
public static class ColStatsObjWithSourceInfo {
private final ColumnStatisticsObj colStatsObj;
http://git-wip-us.apache.org/repos/asf/hive/blob/a3f0f99d/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java
index 5233bee..c681a87 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java
@@ -245,46 +245,6 @@ public class MetaStoreUtils {
return "TRUE".equalsIgnoreCase(tableParams.get(prop));
}
- // check if stats need to be (re)calculated
- public static boolean requireCalStats(Partition oldPart,
- Partition newPart, Table tbl,
- EnvironmentContext environmentContext) {
-
- if (environmentContext != null
- && environmentContext.isSetProperties()
- && StatsSetupConst.TRUE.equals(environmentContext.getProperties().get(
- StatsSetupConst.DO_NOT_UPDATE_STATS))) {
- return false;
- }
-
- if (MetaStoreServerUtils.isView(tbl)) {
- return false;
- }
-
- if (oldPart == null && newPart == null) {
- return true;
- }
-
- // requires to calculate stats if new partition doesn't have it
- if ((newPart == null) || (newPart.getParameters() == null)
- || !MetaStoreServerUtils.containsAllFastStats(newPart.getParameters())) {
- return true;
- }
-
- if (environmentContext != null && environmentContext.isSetProperties()) {
- String statsType = environmentContext.getProperties().get(StatsSetupConst.STATS_GENERATED);
- // no matter STATS_GENERATED is USER or TASK, all need to re-calculate the stats:
- // USER: alter table .. update statistics
- // TASK: from some sql operation which could collect and compute stats
- if (StatsSetupConst.TASK.equals(statsType) || StatsSetupConst.USER.equals(statsType)) {
- return true;
- }
- }
-
- // requires to calculate stats if new and old have different fast stats
- return !MetaStoreServerUtils.isFastStatsSame(oldPart, newPart);
- }
-
/** Duplicates AcidUtils; used in a couple places in metastore. */
public static boolean isInsertOnlyTableParam(Map<String, String> params) {
@@ -939,4 +899,10 @@ public class MetaStoreUtils {
return catName;
}
+ public static boolean isView(Table table) {
+ if (table == null) {
+ return false;
+ }
+ return TableType.VIRTUAL_VIEW.toString().equals(table.getTableType());
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/a3f0f99d/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/SecurityUtils.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/SecurityUtils.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/SecurityUtils.java
index 0b0cfbd..bae1ec3 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/SecurityUtils.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/SecurityUtils.java
@@ -17,13 +17,8 @@
*/
package org.apache.hadoop.hive.metastore.utils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.metastore.security.DBTokenStore;
import org.apache.hadoop.hive.metastore.security.DelegationTokenIdentifier;
import org.apache.hadoop.hive.metastore.security.DelegationTokenSelector;
-import org.apache.hadoop.hive.metastore.security.MemoryTokenStore;
-import org.apache.hadoop.hive.metastore.security.ZooKeeperTokenStore;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
@@ -31,31 +26,29 @@ import org.apache.hadoop.security.authentication.util.KerberosUtil;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.security.token.TokenSelector;
-import org.apache.zookeeper.client.ZooKeeperSaslClient;
-
-import javax.security.auth.login.AppConfigurationEntry;
import org.apache.thrift.transport.TSSLTransportFactory;
import org.apache.thrift.transport.TServerSocket;
import org.apache.thrift.transport.TSocket;
import org.apache.thrift.transport.TTransport;
import org.apache.thrift.transport.TTransportException;
+import org.apache.zookeeper.client.ZooKeeperSaslClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.net.ssl.SSLParameters;
import javax.net.ssl.SSLServerSocket;
import javax.net.ssl.SSLSocket;
-import javax.security.auth.login.LoginException;
+import javax.security.auth.login.AppConfigurationEntry;
import javax.security.auth.login.AppConfigurationEntry.LoginModuleControlFlag;
-
+import javax.security.auth.login.LoginException;
import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
import java.net.InetSocketAddress;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
public class SecurityUtils {
private static final Logger LOG = LoggerFactory.getLogger(SecurityUtils.class);
@@ -192,42 +185,6 @@ public class SecurityUtils {
return delegationToken;
}
- private static final String DELEGATION_TOKEN_STORE_CLS = "hive.cluster.delegation.token.store.class";
-
- /**
- * This method should be used to return the metastore specific tokenstore class name to main
- * backwards compatibility
- *
- * @param conf - HiveConf object
- * @return the tokenStoreClass name from the HiveConf. It maps the hive specific tokenstoreclass
- * name to metastore module specific class name. For eg:
- * hive.cluster.delegation.token.store.class is set to
- * org.apache.hadoop.hive.thrift.MemoryTokenStore it returns the equivalent tokenstore
- * class defined in the metastore module which is
- * org.apache.hadoop.hive.metastore.security.MemoryTokenStore Similarly,
- * org.apache.hadoop.hive.thrift.DBTokenStore maps to
- * org.apache.hadoop.hive.metastore.security.DBTokenStore and
- * org.apache.hadoop.hive.thrift.ZooKeeperTokenStore maps to
- * org.apache.hadoop.hive.metastore.security.ZooKeeperTokenStore
- */
- public static String getTokenStoreClassName(Configuration conf) {
- String tokenStoreClass = conf.get(DELEGATION_TOKEN_STORE_CLS, "");
- if (StringUtils.isBlank(tokenStoreClass)) {
- // default tokenstore is MemoryTokenStore
- return MemoryTokenStore.class.getName();
- }
- switch (tokenStoreClass) {
- case "org.apache.hadoop.hive.thrift.DBTokenStore":
- return DBTokenStore.class.getName();
- case "org.apache.hadoop.hive.thrift.MemoryTokenStore":
- return MemoryTokenStore.class.getName();
- case "org.apache.hadoop.hive.thrift.ZooKeeperTokenStore":
- return ZooKeeperTokenStore.class.getName();
- default:
- return tokenStoreClass;
- }
- }
-
/**
* @return the user name set in hadoop.job.ugi param or the current user from System
http://git-wip-us.apache.org/repos/asf/hive/blob/a3f0f99d/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java
index 6e0d9c1..4093aa7 100644
--- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java
+++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java
@@ -62,7 +62,7 @@ import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars;
import org.apache.hadoop.hive.metastore.hooks.URIResolverHook;
import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy;
import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge;
-import org.apache.hadoop.hive.metastore.txn.TxnUtils;
+import org.apache.hadoop.hive.metastore.txn.TxnCommonUtils;
import org.apache.hadoop.hive.metastore.utils.JavaUtils;
import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
import org.apache.hadoop.hive.metastore.utils.ObjectPair;
@@ -2187,19 +2187,19 @@ public class HiveMetaStoreClientPreCatalog implements IMetaStoreClient, AutoClos
@Override
public ValidTxnList getValidTxns() throws TException {
- return TxnUtils.createValidReadTxnList(client.get_open_txns(), 0);
+ return TxnCommonUtils.createValidReadTxnList(client.get_open_txns(), 0);
}
@Override
public ValidTxnList getValidTxns(long currentTxn) throws TException {
- return TxnUtils.createValidReadTxnList(client.get_open_txns(), currentTxn);
+ return TxnCommonUtils.createValidReadTxnList(client.get_open_txns(), currentTxn);
}
@Override
public ValidWriteIdList getValidWriteIds(String fullTableName) throws TException {
GetValidWriteIdsRequest rqst = new GetValidWriteIdsRequest(Collections.singletonList(fullTableName), null);
GetValidWriteIdsResponse validWriteIds = client.get_valid_write_ids(rqst);
- return TxnUtils.createValidReaderWriteIdList(validWriteIds.getTblValidWriteIds().get(0));
+ return TxnCommonUtils.createValidReaderWriteIdList(validWriteIds.getTblValidWriteIds().get(0));
}
@Override