You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2017/10/10 00:52:05 UTC
[31/61] [abbrv] hive git commit: HIVE-17561 Move TxnStore and
implementations to standalone metastore (Alan Gates,
reviewed by Eugene Koifman)
http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnStore.java
deleted file mode 100644
index 3eb3827..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnStore.java
+++ /dev/null
@@ -1,433 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.metastore.txn;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.hive.common.classification.InterfaceAudience;
-import org.apache.hadoop.hive.common.classification.InterfaceStability;
-import org.apache.hadoop.hive.common.classification.RetrySemantics;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.api.*;
-
-import java.sql.SQLException;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Set;
-
-/**
- * A handler to answer transaction related calls that come into the metastore
- * server.
- *
- * Note on log messages: Please include txnid:X and lockid info using
- * {@link org.apache.hadoop.hive.common.JavaUtils#txnIdToString(long)}
- * and {@link org.apache.hadoop.hive.common.JavaUtils#lockIdToString(long)} in all messages.
- * The txnid:X and lockid:Y matches how Thrift object toString() methods are generated,
- * so keeping the format consistent makes grep'ing the logs much easier.
- *
- * Note on HIVE_LOCKS.hl_last_heartbeat.
- * For locks that are part of transaction, we set this 0 (would rather set it to NULL but
- * Currently the DB schema has this NOT NULL) and only update/read heartbeat from corresponding
- * transaction in TXNS.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public interface TxnStore {
-
- enum MUTEX_KEY {Initiator, Cleaner, HouseKeeper, CompactionHistory, CheckLock,
- WriteSetCleaner, CompactionScheduler}
- // Compactor states (Should really be enum)
- String INITIATED_RESPONSE = "initiated";
- String WORKING_RESPONSE = "working";
- String CLEANING_RESPONSE = "ready for cleaning";
- String FAILED_RESPONSE = "failed";
- String SUCCEEDED_RESPONSE = "succeeded";
- String ATTEMPTED_RESPONSE = "attempted";
-
- public static final int TIMED_OUT_TXN_ABORT_BATCH_SIZE = 50000;
-
- public void setConf(HiveConf conf);
-
- /**
- * Get information about open transactions. This gives extensive information about the
- * transactions rather than just the list of transactions. This should be used when the need
- * is to see information about the transactions (e.g. show transactions).
- * @return information about open transactions
- * @throws MetaException
- */
- @RetrySemantics.ReadOnly
- public GetOpenTxnsInfoResponse getOpenTxnsInfo() throws MetaException;
-
- /**
- * Get list of valid transactions. This gives just the list of transactions that are open.
- * @return list of open transactions, as well as a high water mark.
- * @throws MetaException
- */
- @RetrySemantics.ReadOnly
- public GetOpenTxnsResponse getOpenTxns() throws MetaException;
-
- /**
- * Get the count for open transactions.
- * @throws MetaException
- */
- @RetrySemantics.ReadOnly
- public void countOpenTxns() throws MetaException;
-
- /**
- * Open a set of transactions
- * @param rqst request to open transactions
- * @return information on opened transactions
- * @throws MetaException
- */
- @RetrySemantics.Idempotent
- public OpenTxnsResponse openTxns(OpenTxnRequest rqst) throws MetaException;
-
- /**
- * Abort (rollback) a transaction.
- * @param rqst info on transaction to abort
- * @throws NoSuchTxnException
- * @throws MetaException
- */
- @RetrySemantics.Idempotent
- public void abortTxn(AbortTxnRequest rqst) throws NoSuchTxnException, MetaException, TxnAbortedException;
-
- /**
- * Abort (rollback) a list of transactions in one request.
- * @param rqst info on transactions to abort
- * @throws NoSuchTxnException
- * @throws MetaException
- */
- @RetrySemantics.Idempotent
- public void abortTxns(AbortTxnsRequest rqst) throws NoSuchTxnException, MetaException;
-
- /**
- * Commit a transaction
- * @param rqst info on transaction to commit
- * @throws NoSuchTxnException
- * @throws TxnAbortedException
- * @throws MetaException
- */
- @RetrySemantics.Idempotent
- public void commitTxn(CommitTxnRequest rqst)
- throws NoSuchTxnException, TxnAbortedException, MetaException;
-
- /**
- * Obtain a lock.
- * @param rqst information on the lock to obtain. If the requester is part of a transaction
- * the txn information must be included in the lock request.
- * @return info on the lock, including whether it was obtained.
- * @throws NoSuchTxnException
- * @throws TxnAbortedException
- * @throws MetaException
- */
- @RetrySemantics.CannotRetry
- public LockResponse lock(LockRequest rqst)
- throws NoSuchTxnException, TxnAbortedException, MetaException;
-
- /**
- * Check whether a lock has been obtained. This is used after {@link #lock} returned a wait
- * state.
- * @param rqst info on the lock to check
- * @return info on the state of the lock
- * @throws NoSuchTxnException
- * @throws NoSuchLockException
- * @throws TxnAbortedException
- * @throws MetaException
- */
- @RetrySemantics.SafeToRetry
- public LockResponse checkLock(CheckLockRequest rqst)
- throws NoSuchTxnException, NoSuchLockException, TxnAbortedException, MetaException;
-
- /**
- * Unlock a lock. It is not legal to call this if the caller is part of a txn. In that case
- * the txn should be committed or aborted instead. (Note someday this will change since
- * multi-statement transactions will allow unlocking in the transaction.)
- * @param rqst lock to unlock
- * @throws NoSuchLockException
- * @throws TxnOpenException
- * @throws MetaException
- */
- @RetrySemantics.Idempotent
- public void unlock(UnlockRequest rqst)
- throws NoSuchLockException, TxnOpenException, MetaException;
-
- /**
- * Get information on current locks.
- * @param rqst lock information to retrieve
- * @return lock information.
- * @throws MetaException
- */
- @RetrySemantics.ReadOnly
- public ShowLocksResponse showLocks(ShowLocksRequest rqst) throws MetaException;
-
- /**
- * Send a heartbeat for a lock or a transaction
- * @param ids lock and/or txn id to heartbeat
- * @throws NoSuchTxnException
- * @throws NoSuchLockException
- * @throws TxnAbortedException
- * @throws MetaException
- */
- @RetrySemantics.SafeToRetry
- public void heartbeat(HeartbeatRequest ids)
- throws NoSuchTxnException, NoSuchLockException, TxnAbortedException, MetaException;
-
- /**
- * Heartbeat a group of transactions together
- * @param rqst set of transactions to heartbat
- * @return info on txns that were heartbeated
- * @throws MetaException
- */
- @RetrySemantics.SafeToRetry
- public HeartbeatTxnRangeResponse heartbeatTxnRange(HeartbeatTxnRangeRequest rqst)
- throws MetaException;
-
- /**
- * Submit a compaction request into the queue. This is called when a user manually requests a
- * compaction.
- * @param rqst information on what to compact
- * @return id of the compaction that has been started or existing id if this resource is already scheduled
- * @throws MetaException
- */
- @RetrySemantics.Idempotent
- public CompactionResponse compact(CompactionRequest rqst) throws MetaException;
-
- /**
- * Show list of current compactions
- * @param rqst info on which compactions to show
- * @return compaction information
- * @throws MetaException
- */
- @RetrySemantics.ReadOnly
- public ShowCompactResponse showCompact(ShowCompactRequest rqst) throws MetaException;
-
- /**
- * Add information on a set of dynamic partitions that participated in a transaction.
- * @param rqst dynamic partition info.
- * @throws NoSuchTxnException
- * @throws TxnAbortedException
- * @throws MetaException
- */
- @RetrySemantics.SafeToRetry
- public void addDynamicPartitions(AddDynamicPartitions rqst)
- throws NoSuchTxnException, TxnAbortedException, MetaException;
-
- /**
- * Clean up corresponding records in metastore tables
- * @param type Hive object type
- * @param db database object
- * @param table table object
- * @param partitionIterator partition iterator
- * @throws MetaException
- */
- @RetrySemantics.Idempotent
- public void cleanupRecords(HiveObjectType type, Database db, Table table,
- Iterator<Partition> partitionIterator) throws MetaException;
-
- /**
- * Timeout transactions and/or locks. This should only be called by the compactor.
- */
- @RetrySemantics.Idempotent
- public void performTimeOuts();
-
- /**
- * This will look through the completed_txn_components table and look for partitions or tables
- * that may be ready for compaction. Also, look through txns and txn_components tables for
- * aborted transactions that we should add to the list.
- * @param maxAborted Maximum number of aborted queries to allow before marking this as a
- * potential compaction.
- * @return list of CompactionInfo structs. These will not have id, type,
- * or runAs set since these are only potential compactions not actual ones.
- */
- @RetrySemantics.ReadOnly
- public Set<CompactionInfo> findPotentialCompactions(int maxAborted) throws MetaException;
-
- /**
- * Sets the user to run as. This is for the case
- * where the request was generated by the user and so the worker must set this value later.
- * @param cq_id id of this entry in the queue
- * @param user user to run the jobs as
- */
- @RetrySemantics.Idempotent
- public void setRunAs(long cq_id, String user) throws MetaException;
-
- /**
- * This will grab the next compaction request off of
- * the queue, and assign it to the worker.
- * @param workerId id of the worker calling this, will be recorded in the db
- * @return an info element for this compaction request, or null if there is no work to do now.
- */
- @RetrySemantics.ReadOnly
- public CompactionInfo findNextToCompact(String workerId) throws MetaException;
-
- /**
- * This will mark an entry in the queue as compacted
- * and put it in the ready to clean state.
- * @param info info on the compaction entry to mark as compacted.
- */
- @RetrySemantics.SafeToRetry
- public void markCompacted(CompactionInfo info) throws MetaException;
-
- /**
- * Find entries in the queue that are ready to
- * be cleaned.
- * @return information on the entry in the queue.
- */
- @RetrySemantics.ReadOnly
- public List<CompactionInfo> findReadyToClean() throws MetaException;
-
- /**
- * This will remove an entry from the queue after
- * it has been compacted.
- *
- * @param info info on the compaction entry to remove
- */
- @RetrySemantics.CannotRetry
- public void markCleaned(CompactionInfo info) throws MetaException;
-
- /**
- * Mark a compaction entry as failed. This will move it to the compaction history queue with a
- * failed status. It will NOT clean up aborted transactions in the table/partition associated
- * with this compaction.
- * @param info information on the compaction that failed.
- * @throws MetaException
- */
- @RetrySemantics.CannotRetry
- public void markFailed(CompactionInfo info) throws MetaException;
-
- /**
- * Clean up aborted transactions from txns that have no components in txn_components. The reson such
- * txns exist can be that now work was done in this txn (e.g. Streaming opened TransactionBatch and
- * abandoned it w/o doing any work) or due to {@link #markCleaned(CompactionInfo)} being called.
- */
- @RetrySemantics.SafeToRetry
- public void cleanEmptyAbortedTxns() throws MetaException;
-
- /**
- * This will take all entries assigned to workers
- * on a host return them to INITIATED state. The initiator should use this at start up to
- * clean entries from any workers that were in the middle of compacting when the metastore
- * shutdown. It does not reset entries from worker threads on other hosts as those may still
- * be working.
- * @param hostname Name of this host. It is assumed this prefixes the thread's worker id,
- * so that like hostname% will match the worker id.
- */
- @RetrySemantics.Idempotent
- public void revokeFromLocalWorkers(String hostname) throws MetaException;
-
- /**
- * This call will return all compaction queue
- * entries assigned to a worker but over the timeout back to the initiated state.
- * This should be called by the initiator on start up and occasionally when running to clean up
- * after dead threads. At start up {@link #revokeFromLocalWorkers(String)} should be called
- * first.
- * @param timeout number of milliseconds since start time that should elapse before a worker is
- * declared dead.
- */
- @RetrySemantics.Idempotent
- public void revokeTimedoutWorkers(long timeout) throws MetaException;
-
- /**
- * Queries metastore DB directly to find columns in the table which have statistics information.
- * If {@code ci} includes partition info then per partition stats info is examined, otherwise
- * table level stats are examined.
- * @throws MetaException
- */
- @RetrySemantics.ReadOnly
- public List<String> findColumnsWithStats(CompactionInfo ci) throws MetaException;
-
- /**
- * Record the highest txn id that the {@code ci} compaction job will pay attention to.
- */
- @RetrySemantics.Idempotent
- public void setCompactionHighestTxnId(CompactionInfo ci, long highestTxnId) throws MetaException;
-
- /**
- * For any given compactable entity (partition, table if not partitioned) the history of compactions
- * may look like "sssfffaaasffss", for example. The idea is to retain the tail (most recent) of the
- * history such that a configurable number of each type of state is present. Any other entries
- * can be purged. This scheme has advantage of always retaining the last failure/success even if
- * it's not recent.
- * @throws MetaException
- */
- @RetrySemantics.SafeToRetry
- public void purgeCompactionHistory() throws MetaException;
-
- /**
- * WriteSet tracking is used to ensure proper transaction isolation. This method deletes the
- * transaction metadata once it becomes unnecessary.
- */
- @RetrySemantics.SafeToRetry
- public void performWriteSetGC();
-
- /**
- * Determine if there are enough consecutive failures compacting a table or partition that no
- * new automatic compactions should be scheduled. User initiated compactions do not do this
- * check.
- * @param ci Table or partition to check.
- * @return true if it is ok to compact, false if there have been too many failures.
- * @throws MetaException
- */
- @RetrySemantics.ReadOnly
- public boolean checkFailedCompactions(CompactionInfo ci) throws MetaException;
-
- @VisibleForTesting
- public int numLocksInLockTable() throws SQLException, MetaException;
-
- @VisibleForTesting
- long setTimeout(long milliseconds);
-
- @RetrySemantics.Idempotent
- public MutexAPI getMutexAPI();
-
- /**
- * This is primarily designed to provide coarse grained mutex support to operations running
- * inside the Metastore (of which there could be several instances). The initial goal is to
- * ensure that various sub-processes of the Compactor don't step on each other.
- *
- * In RDMBS world each {@code LockHandle} uses a java.sql.Connection so use it sparingly.
- */
- public static interface MutexAPI {
- /**
- * The {@code key} is name of the lock. Will acquire and exclusive lock or block. It retuns
- * a handle which must be used to release the lock. Each invocation returns a new handle.
- */
- public LockHandle acquireLock(String key) throws MetaException;
-
- /**
- * Same as {@link #acquireLock(String)} but takes an already existing handle as input. This
- * will associate the lock on {@code key} with the same handle. All locks associated with
- * the same handle will be released together.
- * @param handle not NULL
- */
- public void acquireLock(String key, LockHandle handle) throws MetaException;
- public static interface LockHandle {
- /**
- * Releases all locks associated with this handle.
- */
- public void releaseLocks();
- }
- }
-
- /**
- * Once a {@link java.util.concurrent.ThreadPoolExecutor} Worker submits a job to the cluster,
- * it calls this to update the metadata.
- * @param id {@link CompactionInfo#id}
- */
- @RetrySemantics.Idempotent
- public void setHadoopJobId(String hadoopJobId, long id);
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java b/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java
deleted file mode 100644
index 30b155f..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java
+++ /dev/null
@@ -1,242 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.metastore.txn;
-
-import org.apache.hadoop.hive.common.ValidCompactorTxnList;
-import org.apache.hadoop.hive.common.ValidReadTxnList;
-import org.apache.hadoop.hive.common.ValidTxnList;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.MetaStoreUtils;
-import org.apache.hadoop.hive.metastore.api.GetOpenTxnsInfoResponse;
-import org.apache.hadoop.hive.metastore.api.GetOpenTxnsResponse;
-import org.apache.hadoop.hive.metastore.api.Table;
-import org.apache.hadoop.hive.metastore.api.TxnInfo;
-import org.apache.hadoop.hive.metastore.api.TxnState;
-import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Arrays;
-import java.util.BitSet;
-import java.util.List;
-import java.util.Map;
-
-public class TxnUtils {
- private static final Logger LOG = LoggerFactory.getLogger(TxnUtils.class);
-
- /**
- * Transform a {@link org.apache.hadoop.hive.metastore.api.GetOpenTxnsResponse} to a
- * {@link org.apache.hadoop.hive.common.ValidTxnList}. This assumes that the caller intends to
- * read the files, and thus treats both open and aborted transactions as invalid.
- * @param txns txn list from the metastore
- * @param currentTxn Current transaction that the user has open. If this is greater than 0 it
- * will be removed from the exceptions list so that the user sees his own
- * transaction as valid.
- * @return a valid txn list.
- */
- public static ValidTxnList createValidReadTxnList(GetOpenTxnsResponse txns, long currentTxn) {
- /*todo: should highWater be min(currentTxn,txns.getTxn_high_water_mark()) assuming currentTxn>0
- * otherwise if currentTxn=7 and 8 commits before 7, then 7 will see result of 8 which
- * doesn't make sense for Snapshot Isolation. Of course for Read Committed, the list should
- * inlude the latest committed set.*/
- long highWater = txns.getTxn_high_water_mark();
- List<Long> open = txns.getOpen_txns();
- BitSet abortedBits = BitSet.valueOf(txns.getAbortedBits());
- long[] exceptions = new long[open.size() - (currentTxn > 0 ? 1 : 0)];
- int i = 0;
- for(long txn: open) {
- if (currentTxn > 0 && currentTxn == txn) continue;
- exceptions[i++] = txn;
- }
- if(txns.isSetMin_open_txn()) {
- return new ValidReadTxnList(exceptions, abortedBits, highWater, txns.getMin_open_txn());
- }
- else {
- return new ValidReadTxnList(exceptions, abortedBits, highWater);
- }
- }
-
- /**
- * Transform a {@link org.apache.hadoop.hive.metastore.api.GetOpenTxnsInfoResponse} to a
- * {@link org.apache.hadoop.hive.common.ValidTxnList}. This assumes that the caller intends to
- * compact the files, and thus treats only open transactions as invalid. Additionally any
- * txnId > highestOpenTxnId is also invalid. This is to avoid creating something like
- * delta_17_120 where txnId 80, for example, is still open.
- * @param txns txn list from the metastore
- * @return a valid txn list.
- */
- public static ValidTxnList createValidCompactTxnList(GetOpenTxnsInfoResponse txns) {
- //highWater is the last txn id that has been allocated
- long highWater = txns.getTxn_high_water_mark();
- long minOpenTxn = Long.MAX_VALUE;
- long[] exceptions = new long[txns.getOpen_txnsSize()];
- int i = 0;
- for (TxnInfo txn : txns.getOpen_txns()) {
- if (txn.getState() == TxnState.OPEN) {
- minOpenTxn = Math.min(minOpenTxn, txn.getId());
- }
- else {
- //only need aborted since we don't consider anything above minOpenTxn
- exceptions[i++] = txn.getId();
- }
- }
- if(i < exceptions.length) {
- exceptions = Arrays.copyOf(exceptions, i);
- }
- highWater = minOpenTxn == Long.MAX_VALUE ? highWater : minOpenTxn - 1;
- BitSet bitSet = new BitSet(exceptions.length);
- bitSet.set(0, exceptions.length); // for ValidCompactorTxnList, everything in exceptions are aborted
- if(minOpenTxn == Long.MAX_VALUE) {
- return new ValidCompactorTxnList(exceptions, bitSet, highWater);
- }
- else {
- return new ValidCompactorTxnList(exceptions, bitSet, highWater, minOpenTxn);
- }
- }
-
- /**
- * Get an instance of the TxnStore that is appropriate for this store
- * @param conf configuration
- * @return txn store
- */
- public static TxnStore getTxnStore(HiveConf conf) {
- String className = conf.getVar(HiveConf.ConfVars.METASTORE_TXN_STORE_IMPL);
- try {
- TxnStore handler = ((Class<? extends TxnHandler>) MetaStoreUtils.getClass(
- className)).newInstance();
- handler.setConf(conf);
- return handler;
- } catch (Exception e) {
- LOG.error("Unable to instantiate raw store directly in fastpath mode", e);
- throw new RuntimeException(e);
- }
- }
-
- /** Checks if a table is a valid ACID table.
- * Note, users are responsible for using the correct TxnManager. We do not look at
- * SessionState.get().getTxnMgr().supportsAcid() here
- * @param table table
- * @return true if table is a legit ACID table, false otherwise
- */
- public static boolean isAcidTable(Table table) {
- if (table == null) {
- return false;
- }
- Map<String, String> parameters = table.getParameters();
- String tableIsTransactional = parameters.get(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL);
- return tableIsTransactional != null && tableIsTransactional.equalsIgnoreCase("true");
- }
-
- /**
- * Build a query (or queries if one query is too big) with specified "prefix" and "suffix",
- * while populating the IN list into multiple OR clauses, e.g. id in (1,2,3) OR id in (4,5,6)
- * For NOT IN case, NOT IN list is broken into multiple AND clauses.
- * @param queries array of complete query strings
- * @param prefix part of the query that comes before IN list
- * @param suffix part of the query that comes after IN list
- * @param inList the list containing IN list values
- * @param inColumn column name of IN list operator
- * @param addParens add a pair of parenthesis outside the IN lists
- * e.g. ( id in (1,2,3) OR id in (4,5,6) )
- * @param notIn clause to be broken up is NOT IN
- */
- public static void buildQueryWithINClause(HiveConf conf, List<String> queries, StringBuilder prefix,
- StringBuilder suffix, List<Long> inList,
- String inColumn, boolean addParens, boolean notIn) {
- if (inList == null || inList.size() == 0) {
- throw new IllegalArgumentException("The IN list is empty!");
- }
- int batchSize = conf.getIntVar(HiveConf.ConfVars.METASTORE_DIRECT_SQL_MAX_ELEMENTS_IN_CLAUSE);
- int numWholeBatches = inList.size() / batchSize;
- StringBuilder buf = new StringBuilder();
- buf.append(prefix);
- if (addParens) {
- buf.append("(");
- }
- buf.append(inColumn);
- if (notIn) {
- buf.append(" not in (");
- } else {
- buf.append(" in (");
- }
-
- for (int i = 0; i <= numWholeBatches; i++) {
- if (i * batchSize == inList.size()) {
- // At this point we just realized we don't need another query
- break;
- }
-
- if (needNewQuery(conf, buf)) {
- // Wrap up current query string
- if (addParens) {
- buf.append(")");
- }
- buf.append(suffix);
- queries.add(buf.toString());
-
- // Prepare a new query string
- buf.setLength(0);
- }
-
- if (i > 0) {
- if (notIn) {
- if (buf.length() == 0) {
- buf.append(prefix);
- if (addParens) {
- buf.append("(");
- }
- } else {
- buf.append(" and ");
- }
- buf.append(inColumn);
- buf.append(" not in (");
- } else {
- if (buf.length() == 0) {
- buf.append(prefix);
- if (addParens) {
- buf.append("(");
- }
- } else {
- buf.append(" or ");
- }
- buf.append(inColumn);
- buf.append(" in (");
- }
- }
-
- for (int j = i * batchSize; j < (i + 1) * batchSize && j < inList.size(); j++) {
- buf.append(inList.get(j)).append(",");
- }
- buf.setCharAt(buf.length() - 1, ')');
- }
-
- if (addParens) {
- buf.append(")");
- }
- buf.append(suffix);
- queries.add(buf.toString());
- }
-
- /** Estimate if the size of a string will exceed certain limit */
- private static boolean needNewQuery(HiveConf conf, StringBuilder sb) {
- int queryMemoryLimit = conf.getIntVar(HiveConf.ConfVars.METASTORE_DIRECT_SQL_MAX_QUERY_LENGTH);
- // http://www.javamex.com/tutorials/memory/string_memory_usage.shtml
- long sizeInBytes = 8 * (((sb.length() * 2) + 45) / 8);
- return sizeInBytes / 1024 > queryMemoryLimit;
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/metastore/src/test/org/apache/hadoop/hive/metastore/datasource/TestDataSourceProviderFactory.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/datasource/TestDataSourceProviderFactory.java b/metastore/src/test/org/apache/hadoop/hive/metastore/datasource/TestDataSourceProviderFactory.java
deleted file mode 100644
index daea544..0000000
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/datasource/TestDataSourceProviderFactory.java
+++ /dev/null
@@ -1,173 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.metastore.datasource;
-
-import com.jolbox.bonecp.BoneCPDataSource;
-import com.zaxxer.hikari.HikariDataSource;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-import javax.sql.DataSource;
-import java.sql.SQLException;
-
-public class TestDataSourceProviderFactory {
-
- private HiveConf conf;
-
- @Before
- public void init() {
- conf = new HiveConf();
- conf.setVar(HiveConf.ConfVars.METASTORE_CONNECTION_USER_NAME, "dummyUser");
- conf.setVar(HiveConf.ConfVars.METASTOREPWD, "dummyPass");
- }
-
- @Test
- public void testNoDataSourceCreatedWithoutProps() throws SQLException {
-
- DataSourceProvider dsp = DataSourceProviderFactory.getDataSourceProvider(conf);
- Assert.assertNull(dsp);
-
- conf.setVar(HiveConf.ConfVars.METASTORE_CONNECTION_POOLING_TYPE, BoneCPDataSourceProvider.BONECP);
-
- dsp = DataSourceProviderFactory.getDataSourceProvider(conf);
- Assert.assertNull(dsp);
- }
-
- @Test
- public void testCreateBoneCpDataSource() throws SQLException {
-
- conf.setVar(HiveConf.ConfVars.METASTORE_CONNECTION_POOLING_TYPE, BoneCPDataSourceProvider.BONECP);
- conf.set(BoneCPDataSourceProvider.BONECP + ".firstProp", "value");
- conf.set(BoneCPDataSourceProvider.BONECP + ".secondProp", "value");
-
- DataSourceProvider dsp = DataSourceProviderFactory.getDataSourceProvider(conf);
- Assert.assertNotNull(dsp);
-
- DataSource ds = dsp.create(conf);
- Assert.assertTrue(ds instanceof BoneCPDataSource);
- }
-
- @Test
- public void testSetBoneCpStringProperty() throws SQLException {
-
- conf.setVar(HiveConf.ConfVars.METASTORE_CONNECTION_POOLING_TYPE, BoneCPDataSourceProvider.BONECP);
- conf.set(BoneCPDataSourceProvider.BONECP + ".initSQL", "select 1 from dual");
-
- DataSourceProvider dsp = DataSourceProviderFactory.getDataSourceProvider(conf);
- Assert.assertNotNull(dsp);
-
- DataSource ds = dsp.create(conf);
- Assert.assertTrue(ds instanceof BoneCPDataSource);
- Assert.assertEquals("select 1 from dual", ((BoneCPDataSource)ds).getInitSQL());
- }
-
- @Test
- public void testSetBoneCpNumberProperty() throws SQLException {
-
- conf.setVar(HiveConf.ConfVars.METASTORE_CONNECTION_POOLING_TYPE, BoneCPDataSourceProvider.BONECP);
- conf.set(BoneCPDataSourceProvider.BONECP + ".acquireRetryDelayInMs", "599");
-
- DataSourceProvider dsp = DataSourceProviderFactory.getDataSourceProvider(conf);
- Assert.assertNotNull(dsp);
-
- DataSource ds = dsp.create(conf);
- Assert.assertTrue(ds instanceof BoneCPDataSource);
- Assert.assertEquals(599L, ((BoneCPDataSource)ds).getAcquireRetryDelayInMs());
- }
-
- @Test
- public void testSetBoneCpBooleanProperty() throws SQLException {
-
- conf.setVar(HiveConf.ConfVars.METASTORE_CONNECTION_POOLING_TYPE, BoneCPDataSourceProvider.BONECP);
- conf.set(BoneCPDataSourceProvider.BONECP + ".disableJMX", "true");
-
- DataSourceProvider dsp = DataSourceProviderFactory.getDataSourceProvider(conf);
- Assert.assertNotNull(dsp);
-
- DataSource ds = dsp.create(conf);
- Assert.assertTrue(ds instanceof BoneCPDataSource);
- Assert.assertEquals(true, ((BoneCPDataSource)ds).isDisableJMX());
- }
-
- @Test
- public void testCreateHikariCpDataSource() throws SQLException {
-
- conf.setVar(HiveConf.ConfVars.METASTORE_CONNECTION_POOLING_TYPE, HikariCPDataSourceProvider.HIKARI);
- // This is needed to prevent the HikariDataSource from trying to connect to the DB
- conf.set(HikariCPDataSourceProvider.HIKARI + ".initializationFailTimeout", "-1");
-
- DataSourceProvider dsp = DataSourceProviderFactory.getDataSourceProvider(conf);
- Assert.assertNotNull(dsp);
-
- DataSource ds = dsp.create(conf);
- Assert.assertTrue(ds instanceof HikariDataSource);
- }
-
- @Test
- public void testSetHikariCpStringProperty() throws SQLException {
-
- conf.setVar(HiveConf.ConfVars.METASTORE_CONNECTION_POOLING_TYPE, HikariCPDataSourceProvider.HIKARI);
- conf.set(HikariCPDataSourceProvider.HIKARI + ".connectionInitSql", "select 1 from dual");
- conf.set(HikariCPDataSourceProvider.HIKARI + ".initializationFailTimeout", "-1");
-
- DataSourceProvider dsp = DataSourceProviderFactory.getDataSourceProvider(conf);
- Assert.assertNotNull(dsp);
-
- DataSource ds = dsp.create(conf);
- Assert.assertTrue(ds instanceof HikariDataSource);
- Assert.assertEquals("select 1 from dual", ((HikariDataSource)ds).getConnectionInitSql());
- }
-
- @Test
- public void testSetHikariCpNumberProperty() throws SQLException {
-
- conf.setVar(HiveConf.ConfVars.METASTORE_CONNECTION_POOLING_TYPE, HikariCPDataSourceProvider.HIKARI);
- conf.set(HikariCPDataSourceProvider.HIKARI + ".idleTimeout", "59999");
- conf.set(HikariCPDataSourceProvider.HIKARI + ".initializationFailTimeout", "-1");
-
- DataSourceProvider dsp = DataSourceProviderFactory.getDataSourceProvider(conf);
- Assert.assertNotNull(dsp);
-
- DataSource ds = dsp.create(conf);
- Assert.assertTrue(ds instanceof HikariDataSource);
- Assert.assertEquals(59999L, ((HikariDataSource)ds).getIdleTimeout());
- }
-
- @Test
- public void testSetHikariCpBooleanProperty() throws SQLException {
-
- conf.setVar(HiveConf.ConfVars.METASTORE_CONNECTION_POOLING_TYPE, HikariCPDataSourceProvider.HIKARI);
- conf.set(HikariCPDataSourceProvider.HIKARI + ".allowPoolSuspension", "false");
- conf.set(HikariCPDataSourceProvider.HIKARI + ".initializationFailTimeout", "-1");
-
- DataSourceProvider dsp = DataSourceProviderFactory.getDataSourceProvider(conf);
- Assert.assertNotNull(dsp);
-
- DataSource ds = dsp.create(conf);
- Assert.assertTrue(ds instanceof HikariDataSource);
- Assert.assertEquals(false, ((HikariDataSource)ds).isAllowPoolSuspension());
- }
- @Test(expected = IllegalArgumentException.class)
- public void testBoneCPConfigCannotBeSet() {
- conf.addToRestrictList(BoneCPDataSourceProvider.BONECP);
- conf.verifyAndSet(BoneCPDataSourceProvider.BONECP + ".disableJMX", "true");
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandlerNegative.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandlerNegative.java b/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandlerNegative.java
deleted file mode 100644
index 639669e..0000000
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandlerNegative.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.metastore.txn;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.junit.Test;
-
-import static junit.framework.Assert.assertNotNull;
-import static junit.framework.Assert.assertTrue;
-
-public class TestTxnHandlerNegative {
- static final private Logger LOG = LoggerFactory.getLogger(TestTxnHandlerNegative.class);
-
- /**
- * this intentionally sets a bad URL for connection to test error handling logic
- * in TxnHandler
- * @throws Exception
- */
- @Test
- public void testBadConnection() throws Exception {
- HiveConf conf = new HiveConf();
- conf.setVar(HiveConf.ConfVars.METASTORECONNECTURLKEY, "blah");
- RuntimeException e = null;
- try {
- TxnUtils.getTxnStore(conf);
- }
- catch(RuntimeException ex) {
- LOG.info("Expected error: " + ex.getMessage(), ex);
- e = ex;
- }
- assertNotNull(e);
- assertTrue(
- e.getMessage().contains("No suitable driver found for blah")
- || e.getMessage().contains("Failed to get driver instance for jdbcUrl=blah")
- );
- }
-}