You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2018/07/18 18:52:13 UTC
[17/48] hive git commit: HIVE-20006: Make materializations
invalidation cache work with multiple active remote metastores (Jesus Camacho
Rodriguez, reviewed by Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/1b5903b0/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
index 3785f89..9dd3787 100644
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
+++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
@@ -26,10 +26,10 @@ import java.sql.SQLException;
import java.sql.SQLFeatureNotSupportedException;
import java.sql.Savepoint;
import java.sql.Statement;
+import java.time.Instant;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.BitSet;
-import java.util.Calendar;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
@@ -40,7 +40,6 @@ import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.SortedSet;
-import java.util.TimeZone;
import java.util.TreeSet;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Semaphore;
@@ -59,11 +58,10 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.common.ValidReadTxnList;
import org.apache.hadoop.hive.common.ValidReaderWriteIdList;
import org.apache.hadoop.hive.common.ValidTxnList;
+import org.apache.hadoop.hive.common.ValidTxnWriteIdList;
import org.apache.hadoop.hive.common.ValidWriteIdList;
import org.apache.hadoop.hive.common.classification.RetrySemantics;
import org.apache.hadoop.hive.metastore.DatabaseProduct;
-import org.apache.hadoop.hive.metastore.MaterializationsInvalidationCache;
-import org.apache.hadoop.hive.metastore.MaterializationsRebuildLockHandler;
import org.apache.hadoop.hive.metastore.Warehouse;
import org.apache.hadoop.hive.metastore.MetaStoreListenerNotifier;
import org.apache.hadoop.hive.metastore.TransactionalMetaStoreEventListener;
@@ -869,10 +867,7 @@ abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI {
@RetrySemantics.Idempotent("No-op if already committed")
public void commitTxn(CommitTxnRequest rqst)
throws NoSuchTxnException, TxnAbortedException, MetaException {
- MaterializationsRebuildLockHandler materializationsRebuildLockHandler =
- MaterializationsRebuildLockHandler.get();
- List<TransactionRegistryInfo> txnComponents = new ArrayList<>();
- boolean isUpdateDelete = false;
+ char isUpdateDelete = 'N';
long txnid = rqst.getTxnid();
long sourceTxnId = -1;
@@ -936,7 +931,7 @@ abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI {
"tc_operation_type " + conflictSQLSuffix));
}
if (rs != null && rs.next()) {
- isUpdateDelete = true;
+ isUpdateDelete = 'Y';
close(rs);
//if here it means currently committing txn performed update/delete and we should check WW conflict
/**
@@ -1033,8 +1028,8 @@ abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI {
// Move the record from txn_components into completed_txn_components so that the compactor
// knows where to look to compact.
s = "insert into COMPLETED_TXN_COMPONENTS (ctc_txnid, ctc_database, " +
- "ctc_table, ctc_partition, ctc_writeid) select tc_txnid, tc_database, tc_table, " +
- "tc_partition, tc_writeid from TXN_COMPONENTS where tc_txnid = " + txnid;
+ "ctc_table, ctc_partition, ctc_writeid, ctc_update_delete) select tc_txnid, tc_database, tc_table, " +
+ "tc_partition, tc_writeid, '" + isUpdateDelete + "' from TXN_COMPONENTS where tc_txnid = " + txnid;
LOG.debug("Going to execute insert <" + s + ">");
if ((stmt.executeUpdate(s)) < 1) {
@@ -1050,10 +1045,11 @@ abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI {
rows.add(txnid + "," + quoteString(writeEventInfo.getDatabase()) + "," +
quoteString(writeEventInfo.getTable()) + "," +
quoteString(writeEventInfo.getPartition()) + "," +
- writeEventInfo.getWriteId());
+ writeEventInfo.getWriteId() + "," +
+ "'" + isUpdateDelete + "'");
}
List<String> queries = sqlGenerator.createInsertValuesStmt("COMPLETED_TXN_COMPONENTS " +
- "(ctc_txnid," + " ctc_database, ctc_table, ctc_partition, ctc_writeid)", rows);
+ "(ctc_txnid," + " ctc_database, ctc_table, ctc_partition, ctc_writeid, ctc_update_delete)", rows);
for (String q : queries) {
LOG.debug("Going to execute insert <" + q + "> ");
stmt.execute(q);
@@ -1066,18 +1062,6 @@ abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI {
stmt.executeUpdate(s);
}
- // Obtain information that we need to update registry
- s = "select ctc_database, ctc_table, ctc_writeid, ctc_timestamp from COMPLETED_TXN_COMPONENTS" +
- " where ctc_txnid = " + txnid;
-
- LOG.debug("Going to extract table modification information for invalidation cache <" + s + ">");
- rs = stmt.executeQuery(s);
- while (rs.next()) {
- // We only enter in this loop if the transaction actually affected any table
- txnComponents.add(new TransactionRegistryInfo(rs.getString(1), rs.getString(2),
- rs.getLong(3), rs.getTimestamp(4, Calendar.getInstance(TimeZone.getTimeZone("UTC"))).getTime()));
- }
-
// cleanup all txn related metadata
s = "delete from TXN_COMPONENTS where tc_txnid = " + txnid;
LOG.debug("Going to execute update <" + s + ">");
@@ -1092,29 +1076,19 @@ abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI {
LOG.debug("Going to execute update <" + s + ">");
stmt.executeUpdate(s);
LOG.info("Removed committed transaction: (" + txnid + ") from MIN_HISTORY_LEVEL");
+
+ s = "delete from MATERIALIZATION_REBUILD_LOCKS where mrl_txn_id = " + txnid;
+ LOG.debug("Going to execute update <" + s + ">");
+ stmt.executeUpdate(s);
+
if (transactionalListeners != null) {
MetaStoreListenerNotifier.notifyEventWithDirectSql(transactionalListeners,
EventMessage.EventType.COMMIT_TXN, new CommitTxnEvent(txnid, null), dbConn, sqlGenerator);
}
- MaterializationsInvalidationCache materializationsInvalidationCache =
- MaterializationsInvalidationCache.get();
- for (TransactionRegistryInfo info : txnComponents) {
- if (materializationsInvalidationCache.containsMaterialization(info.dbName, info.tblName) &&
- !materializationsRebuildLockHandler.readyToCommitResource(info.dbName, info.tblName, txnid)) {
- throw new MetaException(
- "Another process is rebuilding the materialized view " + info.fullyQualifiedName);
- }
- }
LOG.debug("Going to commit");
close(rs);
dbConn.commit();
-
- // Update registry with modifications
- for (TransactionRegistryInfo info : txnComponents) {
- materializationsInvalidationCache.notifyTableModification(
- info.dbName, info.tblName, info.writeId, info.timestamp, isUpdateDelete);
- }
} catch (SQLException e) {
LOG.debug("Going to rollback");
rollbackDBConn(dbConn);
@@ -1125,9 +1099,6 @@ abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI {
close(commitIdRs);
close(lockHandle, stmt, dbConn);
unlockInternal();
- for (TransactionRegistryInfo info : txnComponents) {
- materializationsRebuildLockHandler.unlockResource(info.dbName, info.tblName, txnid);
- }
}
} catch (RetryException e) {
commitTxn(rqst);
@@ -1694,16 +1665,30 @@ abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI {
}
/**
- * Gets the information of the first transaction for the given table
- * after the transaction with the input id was committed (if any).
+ * Get invalidation info for the materialization. Currently, the materialization information
+ * only contains information about whether there was update/delete operations on the source
+ * tables used by the materialization since it was created.
*/
@Override
@RetrySemantics.ReadOnly
- public BasicTxnInfo getFirstCompletedTransactionForTableAfterCommit(
- String inputDbName, String inputTableName, ValidWriteIdList txnList)
- throws MetaException {
- final List<Long> openTxns = Arrays.asList(ArrayUtils.toObject(txnList.getInvalidWriteIds()));
+ public Materialization getMaterializationInvalidationInfo(
+ CreationMetadata creationMetadata, String validTxnListStr) throws MetaException {
+ if (creationMetadata.getTablesUsed().isEmpty()) {
+ // Bail out
+ LOG.warn("Materialization creation metadata does not contain any table");
+ return null;
+ }
+
+ // Parse validTxnList
+ final ValidReadTxnList validTxnList =
+ new ValidReadTxnList(validTxnListStr);
+
+ // Parse validReaderWriteIdList from creation metadata
+ final ValidTxnWriteIdList validReaderWriteIdList =
+ new ValidTxnWriteIdList(creationMetadata.getValidTxnList());
+ // We are composing a query that returns a single row if an update happened after
+ // the materialization was created. Otherwise, query returns 0 rows.
Connection dbConn = null;
Statement stmt = null;
ResultSet rs = null;
@@ -1711,32 +1696,207 @@ abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI {
dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
stmt = dbConn.createStatement();
stmt.setMaxRows(1);
- String s = "select ctc_timestamp, ctc_writeid, ctc_database, ctc_table "
- + "from COMPLETED_TXN_COMPONENTS "
- + "where ctc_database=" + quoteString(inputDbName) + " and ctc_table=" + quoteString(inputTableName)
- + " and ctc_writeid > " + txnList.getHighWatermark()
- + (txnList.getInvalidWriteIds().length == 0 ?
- " " : " or ctc_writeid IN(" + StringUtils.join(",", openTxns) + ") ")
- + "order by ctc_timestamp asc";
+ StringBuilder query = new StringBuilder();
+ // compose a query that select transactions containing an update...
+ query.append("select ctc_update_delete from COMPLETED_TXN_COMPONENTS where ctc_update_delete='Y' AND (");
+ int i = 0;
+ for (String fullyQualifiedName : creationMetadata.getTablesUsed()) {
+ // ...for each of the tables that are part of the materialized view,
+ // where the transaction had to be committed after the materialization was created...
+ if (i != 0) {
+ query.append("OR");
+ }
+ String[] names = TxnUtils.getDbTableName(fullyQualifiedName);
+ query.append(" (ctc_database=" + quoteString(names[0]) + " AND ctc_table=" + quoteString(names[1]));
+ ValidWriteIdList tblValidWriteIdList =
+ validReaderWriteIdList.getTableValidWriteIdList(fullyQualifiedName);
+ if (tblValidWriteIdList == null) {
+ LOG.warn("ValidWriteIdList for table {} not present in creation metadata, this should not happen");
+ return null;
+ }
+ query.append(" AND (ctc_writeid > " + tblValidWriteIdList.getHighWatermark());
+ query.append(tblValidWriteIdList.getInvalidWriteIds().length == 0 ? ") " :
+ " OR ctc_writeid IN(" + StringUtils.join(",",
+ Arrays.asList(ArrayUtils.toObject(tblValidWriteIdList.getInvalidWriteIds()))) + ") ");
+ query.append(") ");
+ i++;
+ }
+ // ... and where the transaction has already been committed as per snapshot taken
+ // when we are running current query
+ query.append(") AND ctc_txnid <= " + validTxnList.getHighWatermark());
+ query.append(validTxnList.getInvalidTransactions().length == 0 ? " " :
+ " AND ctc_txnid NOT IN(" + StringUtils.join(",",
+ Arrays.asList(ArrayUtils.toObject(validTxnList.getInvalidTransactions()))) + ") ");
+
+ // Execute query
+ String s = query.toString();
if (LOG.isDebugEnabled()) {
LOG.debug("Going to execute query <" + s + ">");
}
rs = stmt.executeQuery(s);
- if(!rs.next()) {
- return new BasicTxnInfo(true);
+ return new Materialization(rs.next());
+ } catch (SQLException ex) {
+ LOG.warn("getMaterializationInvalidationInfo failed due to " + getMessage(ex), ex);
+ throw new MetaException("Unable to retrieve materialization invalidation information due to " +
+ StringUtils.stringifyException(ex));
+ } finally {
+ close(rs, stmt, dbConn);
+ }
+ }
+
+ @Override
+ public LockResponse lockMaterializationRebuild(String dbName, String tableName, long txnId)
+ throws MetaException {
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Acquiring lock for materialization rebuild with txnId={} for {}", txnId, Warehouse.getQualifiedName(dbName,tableName));
+ }
+
+ TxnStore.MutexAPI.LockHandle handle = null;
+ Connection dbConn = null;
+ Statement stmt = null;
+ ResultSet rs = null;
+ try {
+ lockInternal();
+ /**
+ * MUTEX_KEY.MaterializationRebuild lock ensures that there is only 1 entry in
+ * Initiated/Working state for any resource. This ensures we do not run concurrent
+ * rebuild operations on any materialization.
+ */
+ handle = getMutexAPI().acquireLock(MUTEX_KEY.MaterializationRebuild.name());
+ dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
+ stmt = dbConn.createStatement();
+
+ String selectQ = "select mrl_txn_id from MATERIALIZATION_REBUILD_LOCKS where" +
+ " mrl_db_name =" + quoteString(dbName) +
+ " AND mrl_tbl_name=" + quoteString(tableName);
+ LOG.debug("Going to execute query <" + selectQ + ">");
+ rs = stmt.executeQuery(selectQ);
+ if(rs.next()) {
+ LOG.info("Ignoring request to rebuild " + dbName + "/" + tableName +
+ " since it is already being rebuilt");
+ return new LockResponse(txnId, LockState.NOT_ACQUIRED);
}
- final BasicTxnInfo txnInfo = new BasicTxnInfo(false);
- txnInfo.setTime(rs.getTimestamp(1, Calendar.getInstance(TimeZone.getTimeZone("UTC"))).getTime());
- txnInfo.setTxnid(rs.getLong(2));
- txnInfo.setDbname(rs.getString(3));
- txnInfo.setTablename(rs.getString(4));
- return txnInfo;
+ String insertQ = "insert into MATERIALIZATION_REBUILD_LOCKS " +
+ "(mrl_txn_id, mrl_db_name, mrl_tbl_name, mrl_last_heartbeat) values (" + txnId +
+ ", '" + dbName + "', '" + tableName + "', " + Instant.now().toEpochMilli() + ")";
+ LOG.debug("Going to execute update <" + insertQ + ">");
+ stmt.executeUpdate(insertQ);
+ LOG.debug("Going to commit");
+ dbConn.commit();
+ return new LockResponse(txnId, LockState.ACQUIRED);
} catch (SQLException ex) {
- LOG.warn("getLastCompletedTransactionForTable failed due to " + getMessage(ex), ex);
- throw new MetaException("Unable to retrieve commits information due to " + StringUtils.stringifyException(ex));
+ LOG.warn("lockMaterializationRebuild failed due to " + getMessage(ex), ex);
+ throw new MetaException("Unable to retrieve materialization invalidation information due to " +
+ StringUtils.stringifyException(ex));
} finally {
close(rs, stmt, dbConn);
+ if(handle != null) {
+ handle.releaseLocks();
+ }
+ unlockInternal();
+ }
+ }
+
+ @Override
+ public boolean heartbeatLockMaterializationRebuild(String dbName, String tableName, long txnId)
+ throws MetaException {
+ try {
+ Connection dbConn = null;
+ Statement stmt = null;
+ ResultSet rs = null;
+ try {
+ lockInternal();
+ dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
+ stmt = dbConn.createStatement();
+ String s = "update MATERIALIZATION_REBUILD_LOCKS" +
+ " set mrl_last_heartbeat = " + Instant.now().toEpochMilli() +
+ " where mrl_txn_id = " + txnId +
+ " AND mrl_db_name =" + quoteString(dbName) +
+ " AND mrl_tbl_name=" + quoteString(tableName);
+ LOG.debug("Going to execute update <" + s + ">");
+ int rc = stmt.executeUpdate(s);
+ if (rc < 1) {
+ LOG.debug("Going to rollback");
+ dbConn.rollback();
+ LOG.info("No lock found for rebuild of " + Warehouse.getQualifiedName(dbName, tableName) +
+ " when trying to heartbeat");
+ // It could not be renewed, return that information
+ return false;
+ }
+ LOG.debug("Going to commit");
+ dbConn.commit();
+ // It could be renewed, return that information
+ return true;
+ } catch (SQLException e) {
+ LOG.debug("Going to rollback");
+ rollbackDBConn(dbConn);
+ checkRetryable(dbConn, e,
+ "heartbeatLockMaterializationRebuild(" + Warehouse.getQualifiedName(dbName, tableName) + ", " + txnId + ")");
+ throw new MetaException("Unable to heartbeat rebuild lock due to " +
+ StringUtils.stringifyException(e));
+ } finally {
+ close(rs, stmt, dbConn);
+ unlockInternal();
+ }
+ } catch (RetryException e) {
+ return heartbeatLockMaterializationRebuild(dbName, tableName ,txnId);
+ }
+ }
+
+ @Override
+ public long cleanupMaterializationRebuildLocks(ValidTxnList validTxnList, long timeout) throws MetaException {
+ try {
+ // Aux values
+ long cnt = 0L;
+ List<Long> txnIds = new ArrayList<>();
+ long timeoutTime = Instant.now().toEpochMilli() - timeout;
+
+ Connection dbConn = null;
+ Statement stmt = null;
+ ResultSet rs = null;
+ try {
+ lockInternal();
+ dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
+ stmt = dbConn.createStatement();
+
+ String selectQ = "select mrl_txn_id, mrl_last_heartbeat from MATERIALIZATION_REBUILD_LOCKS";
+ LOG.debug("Going to execute query <" + selectQ + ">");
+ rs = stmt.executeQuery(selectQ);
+ while(rs.next()) {
+ long lastHeartbeat = rs.getLong(2);
+ if (lastHeartbeat < timeoutTime) {
+ // The heartbeat has timeout, double check whether we can remove it
+ long txnId = rs.getLong(1);
+ if (validTxnList.isTxnValid(txnId) || validTxnList.isTxnAborted(txnId)) {
+ // Txn was committed (but notification was not received) or it was aborted.
+ // Either case, we can clean it up
+ txnIds.add(txnId);
+ }
+ }
+ }
+ if (!txnIds.isEmpty()) {
+ String deleteQ = "delete from MATERIALIZATION_REBUILD_LOCKS where" +
+ " mrl_txn_id IN(" + StringUtils.join(",", txnIds) + ") ";
+ LOG.debug("Going to execute update <" + deleteQ + ">");
+ cnt = stmt.executeUpdate(deleteQ);
+ }
+ LOG.debug("Going to commit");
+ dbConn.commit();
+ return cnt;
+ } catch (SQLException e) {
+ LOG.debug("Going to rollback");
+ rollbackDBConn(dbConn);
+ checkRetryable(dbConn, e, "cleanupMaterializationRebuildLocks");
+ throw new MetaException("Unable to clean rebuild locks due to " +
+ StringUtils.stringifyException(e));
+ } finally {
+ close(rs, stmt, dbConn);
+ unlockInternal();
+ }
+ } catch (RetryException e) {
+ return cleanupMaterializationRebuildLocks(validTxnList, timeout);
}
}
@@ -2009,6 +2169,7 @@ abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI {
private static String normalizeCase(String s) {
return s == null ? null : s.toLowerCase();
}
+
private LockResponse checkLockWithRetry(Connection dbConn, long extLockId, long txnId)
throws NoSuchLockException, NoSuchTxnException, TxnAbortedException, MetaException {
try {
@@ -4887,20 +5048,4 @@ abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI {
}
};
- private class TransactionRegistryInfo {
- final String dbName;
- final String tblName;
- final String fullyQualifiedName;
- final long writeId;
- final long timestamp;
-
- public TransactionRegistryInfo (String dbName, String tblName, long writeId, long timestamp) {
- this.dbName = dbName;
- this.tblName = tblName;
- this.fullyQualifiedName = Warehouse.getQualifiedName(dbName, tblName);
- this.writeId = writeId;
- this.timestamp = timestamp;
- }
- }
-
}
http://git-wip-us.apache.org/repos/asf/hive/blob/1b5903b0/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnStore.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnStore.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnStore.java
index d972d10..33f24fb 100644
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnStore.java
+++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnStore.java
@@ -21,6 +21,7 @@ import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.hive.common.ValidTxnList;
import org.apache.hadoop.hive.common.ValidWriteIdList;
import org.apache.hadoop.hive.common.classification.RetrySemantics;
import org.apache.hadoop.hive.metastore.api.*;
@@ -29,6 +30,7 @@ import org.apache.hadoop.hive.metastore.events.AcidWriteEvent;
import java.sql.SQLException;
import java.util.Iterator;
import java.util.List;
+import java.util.Map;
import java.util.Set;
/**
@@ -41,7 +43,7 @@ public interface TxnStore extends Configurable {
enum MUTEX_KEY {
Initiator, Cleaner, HouseKeeper, CompactionHistory, CheckLock,
- WriteSetCleaner, CompactionScheduler, WriteIdAllocator
+ WriteSetCleaner, CompactionScheduler, WriteIdAllocator, MaterializationRebuild
}
// Compactor states (Should really be enum)
String INITIATED_RESPONSE = "initiated";
@@ -128,21 +130,33 @@ public interface TxnStore extends Configurable {
void replTableWriteIdState(ReplTblWriteIdStateRequest rqst) throws MetaException;
/**
- * Get the first transaction corresponding to given database and table after transactions
- * referenced in the transaction snapshot.
- * @return
+ * Get invalidation info for the materialization. Currently, the materialization information
+ * only contains information about whether there was update/delete operations on the source
+ * tables used by the materialization since it was created.
+ * @param cm creation metadata for the materialization
+ * @param validTxnList valid transaction list for snapshot taken for current query
* @throws MetaException
*/
@RetrySemantics.Idempotent
- BasicTxnInfo getFirstCompletedTransactionForTableAfterCommit(
- String inputDbName, String inputTableName, ValidWriteIdList txnList)
+ Materialization getMaterializationInvalidationInfo(
+ final CreationMetadata cm, final String validTxnList)
throws MetaException;
- /**
- * Gets the list of valid write ids for the given table wrt to current txn
- * @param rqst info on transaction and list of table names associated with given transaction
- * @throws NoSuchTxnException
- * @throws MetaException
- */
+
+ LockResponse lockMaterializationRebuild(String dbName, String tableName, long txnId)
+ throws MetaException;
+
+ boolean heartbeatLockMaterializationRebuild(String dbName, String tableName, long txnId)
+ throws MetaException;
+
+ long cleanupMaterializationRebuildLocks(ValidTxnList validTxnList, long timeout)
+ throws MetaException;
+
+ /**
+ * Gets the list of valid write ids for the given table wrt to current txn
+ * @param rqst info on transaction and list of table names associated with given transaction
+ * @throws NoSuchTxnException
+ * @throws MetaException
+ */
@RetrySemantics.ReadOnly
GetValidWriteIdsResponse getValidWriteIds(GetValidWriteIdsRequest rqst)
throws NoSuchTxnException, MetaException;
http://git-wip-us.apache.org/repos/asf/hive/blob/1b5903b0/standalone-metastore/metastore-common/src/main/sql/derby/hive-schema-3.1.0.derby.sql
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/main/sql/derby/hive-schema-3.1.0.derby.sql b/standalone-metastore/metastore-common/src/main/sql/derby/hive-schema-3.1.0.derby.sql
index a696d06..5e8693e 100644
--- a/standalone-metastore/metastore-common/src/main/sql/derby/hive-schema-3.1.0.derby.sql
+++ b/standalone-metastore/metastore-common/src/main/sql/derby/hive-schema-3.1.0.derby.sql
@@ -197,7 +197,8 @@ CREATE TABLE "APP"."MV_CREATION_METADATA" (
"CAT_NAME" VARCHAR(256) NOT NULL,
"DB_NAME" VARCHAR(128) NOT NULL,
"TBL_NAME" VARCHAR(256) NOT NULL,
- "TXN_LIST" CLOB
+ "TXN_LIST" CLOB,
+ "MATERIALIZATION_TIME" BIGINT NOT NULL
);
CREATE TABLE "APP"."MV_TABLES_USED" (
@@ -526,7 +527,8 @@ CREATE TABLE COMPLETED_TXN_COMPONENTS (
CTC_TABLE varchar(256),
CTC_PARTITION varchar(767),
CTC_TIMESTAMP timestamp DEFAULT CURRENT_TIMESTAMP NOT NULL,
- CTC_WRITEID bigint
+ CTC_WRITEID bigint,
+ CTC_UPDATE_DELETE char(1) NOT NULL
);
CREATE INDEX COMPLETED_TXN_COMPONENTS_IDX ON COMPLETED_TXN_COMPONENTS (CTC_DATABASE, CTC_TABLE, CTC_PARTITION);
@@ -645,6 +647,14 @@ CREATE TABLE MIN_HISTORY_LEVEL (
CREATE INDEX MIN_HISTORY_LEVEL_IDX ON MIN_HISTORY_LEVEL (MHL_MIN_OPEN_TXNID);
+CREATE TABLE MATERIALIZATION_REBUILD_LOCKS (
+ MRL_TXN_ID BIGINT NOT NULL,
+ MRL_DB_NAME VARCHAR(128) NOT NULL,
+ MRL_TBL_NAME VARCHAR(256) NOT NULL,
+ MRL_LAST_HEARTBEAT BIGINT NOT NULL,
+ PRIMARY KEY(MRL_TXN_ID)
+);
+
CREATE TABLE "APP"."I_SCHEMA" (
"SCHEMA_ID" bigint primary key,
"SCHEMA_TYPE" integer not null,
http://git-wip-us.apache.org/repos/asf/hive/blob/1b5903b0/standalone-metastore/metastore-common/src/main/sql/derby/hive-schema-4.0.0.derby.sql
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/main/sql/derby/hive-schema-4.0.0.derby.sql b/standalone-metastore/metastore-common/src/main/sql/derby/hive-schema-4.0.0.derby.sql
index 7cab4fb..5ba71c4 100644
--- a/standalone-metastore/metastore-common/src/main/sql/derby/hive-schema-4.0.0.derby.sql
+++ b/standalone-metastore/metastore-common/src/main/sql/derby/hive-schema-4.0.0.derby.sql
@@ -197,7 +197,8 @@ CREATE TABLE "APP"."MV_CREATION_METADATA" (
"CAT_NAME" VARCHAR(256) NOT NULL,
"DB_NAME" VARCHAR(128) NOT NULL,
"TBL_NAME" VARCHAR(256) NOT NULL,
- "TXN_LIST" CLOB
+ "TXN_LIST" CLOB,
+ "MATERIALIZATION_TIME" BIGINT NOT NULL
);
CREATE TABLE "APP"."MV_TABLES_USED" (
@@ -526,7 +527,8 @@ CREATE TABLE COMPLETED_TXN_COMPONENTS (
CTC_TABLE varchar(256),
CTC_PARTITION varchar(767),
CTC_TIMESTAMP timestamp DEFAULT CURRENT_TIMESTAMP NOT NULL,
- CTC_WRITEID bigint
+ CTC_WRITEID bigint,
+ CTC_UPDATE_DELETE char(1) NOT NULL
);
CREATE INDEX COMPLETED_TXN_COMPONENTS_IDX ON COMPLETED_TXN_COMPONENTS (CTC_DATABASE, CTC_TABLE, CTC_PARTITION);
@@ -645,6 +647,14 @@ CREATE TABLE MIN_HISTORY_LEVEL (
CREATE INDEX MIN_HISTORY_LEVEL_IDX ON MIN_HISTORY_LEVEL (MHL_MIN_OPEN_TXNID);
+CREATE TABLE MATERIALIZATION_REBUILD_LOCKS (
+ MRL_TXN_ID BIGINT NOT NULL,
+ MRL_DB_NAME VARCHAR(128) NOT NULL,
+ MRL_TBL_NAME VARCHAR(256) NOT NULL,
+ MRL_LAST_HEARTBEAT BIGINT NOT NULL,
+ PRIMARY KEY(MRL_TXN_ID)
+);
+
CREATE TABLE "APP"."I_SCHEMA" (
"SCHEMA_ID" bigint primary key,
"SCHEMA_TYPE" integer not null,
http://git-wip-us.apache.org/repos/asf/hive/blob/1b5903b0/standalone-metastore/metastore-common/src/main/sql/derby/upgrade-3.0.0-to-3.1.0.derby.sql
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/main/sql/derby/upgrade-3.0.0-to-3.1.0.derby.sql b/standalone-metastore/metastore-common/src/main/sql/derby/upgrade-3.0.0-to-3.1.0.derby.sql
index 7058ab0..2b200f2 100644
--- a/standalone-metastore/metastore-common/src/main/sql/derby/upgrade-3.0.0-to-3.1.0.derby.sql
+++ b/standalone-metastore/metastore-common/src/main/sql/derby/upgrade-3.0.0-to-3.1.0.derby.sql
@@ -45,5 +45,24 @@ CREATE TABLE TXN_WRITE_NOTIFICATION_LOG (
);
INSERT INTO SEQUENCE_TABLE (SEQUENCE_NAME, NEXT_VAL) VALUES ('org.apache.hadoop.hive.metastore.model.MTxnWriteNotificationLog', 1);
+-- HIVE-19027
+-- add column MATERIALIZATION_TIME (bigint) to MV_CREATION_METADATA table
+ALTER TABLE "APP"."MV_CREATION_METADATA" ADD COLUMN "MATERIALIZATION_TIME" BIGINT;
+UPDATE "APP"."MV_CREATION_METADATA" SET "MATERIALIZATION_TIME" = 0;
+ALTER TABLE "APP"."MV_CREATION_METADATA" ALTER COLUMN "MATERIALIZATION_TIME" NOT NULL;
+
+-- add column CTC_UPDATE_DELETE (char) to COMPLETED_TXN_COMPONENTS table
+ALTER TABLE COMPLETED_TXN_COMPONENTS ADD COLUMN CTC_UPDATE_DELETE char(1);
+UPDATE COMPLETED_TXN_COMPONENTS SET CTC_UPDATE_DELETE = 'N';
+ALTER TABLE COMPLETED_TXN_COMPONENTS ALTER COLUMN CTC_UPDATE_DELETE NOT NULL;
+
+CREATE TABLE MATERIALIZATION_REBUILD_LOCKS (
+ MRL_TXN_ID BIGINT NOT NULL,
+ MRL_DB_NAME VARCHAR(128) NOT NULL,
+ MRL_TBL_NAME VARCHAR(256) NOT NULL,
+ MRL_LAST_HEARTBEAT BIGINT NOT NULL,
+ PRIMARY KEY(MRL_TXN_ID)
+);
+
-- This needs to be the last thing done. Insert any changes above this line.
UPDATE "APP".VERSION SET SCHEMA_VERSION='3.1.0', VERSION_COMMENT='Hive release version 3.1.0' where VER_ID=1;
http://git-wip-us.apache.org/repos/asf/hive/blob/1b5903b0/standalone-metastore/metastore-common/src/main/sql/mssql/hive-schema-3.1.0.mssql.sql
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/main/sql/mssql/hive-schema-3.1.0.mssql.sql b/standalone-metastore/metastore-common/src/main/sql/mssql/hive-schema-3.1.0.mssql.sql
index d7722dc..446ee6e 100644
--- a/standalone-metastore/metastore-common/src/main/sql/mssql/hive-schema-3.1.0.mssql.sql
+++ b/standalone-metastore/metastore-common/src/main/sql/mssql/hive-schema-3.1.0.mssql.sql
@@ -388,7 +388,8 @@ CREATE TABLE MV_CREATION_METADATA
CAT_NAME nvarchar(256) NOT NULL,
DB_NAME nvarchar(128) NOT NULL,
TBL_NAME nvarchar(256) NOT NULL,
- TXN_LIST text NULL
+ TXN_LIST text NULL,
+ MATERIALIZATION_TIME bigint NOT NULL
);
ALTER TABLE MV_CREATION_METADATA ADD CONSTRAINT MV_CREATION_METADATA_PK PRIMARY KEY (MV_CREATION_METADATA_ID);
@@ -1034,7 +1035,8 @@ CREATE TABLE COMPLETED_TXN_COMPONENTS(
CTC_TABLE nvarchar(128) NULL,
CTC_PARTITION nvarchar(767) NULL,
CTC_TIMESTAMP datetime2 DEFAULT CURRENT_TIMESTAMP NOT NULL,
- CTC_WRITEID bigint
+ CTC_WRITEID bigint,
+ CTC_UPDATE_DELETE char(1) NOT NULL
);
CREATE INDEX COMPLETED_TXN_COMPONENTS_IDX ON COMPLETED_TXN_COMPONENTS (CTC_DATABASE, CTC_TABLE, CTC_PARTITION);
@@ -1191,6 +1193,17 @@ PRIMARY KEY CLUSTERED
CREATE INDEX MIN_HISTORY_LEVEL_IDX ON MIN_HISTORY_LEVEL (MHL_MIN_OPEN_TXNID);
+CREATE TABLE MATERIALIZATION_REBUILD_LOCKS (
+ MRL_TXN_ID bigint NOT NULL,
+ MRL_DB_NAME nvarchar(128) NOT NULL,
+ MRL_TBL_NAME nvarchar(256) NOT NULL,
+ MRL_LAST_HEARTBEAT bigint NOT NULL,
+PRIMARY KEY CLUSTERED
+(
+ MRL_TXN_ID ASC
+)
+);
+
CREATE TABLE "I_SCHEMA" (
"SCHEMA_ID" bigint primary key,
"SCHEMA_TYPE" int not null,
http://git-wip-us.apache.org/repos/asf/hive/blob/1b5903b0/standalone-metastore/metastore-common/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql b/standalone-metastore/metastore-common/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql
index a81fc40..bbc8ea2 100644
--- a/standalone-metastore/metastore-common/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql
+++ b/standalone-metastore/metastore-common/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql
@@ -248,7 +248,6 @@ CREATE TABLE TAB_COL_STATS
ALTER TABLE TAB_COL_STATS ADD CONSTRAINT TAB_COL_STATS_PK PRIMARY KEY (CS_ID);
CREATE INDEX TAB_COL_STATS_IDX ON TAB_COL_STATS (CAT_NAME, DB_NAME, TABLE_NAME, COLUMN_NAME);
-
-- Table TYPES for classes [org.apache.hadoop.hive.metastore.model.MType]
CREATE TABLE TYPES
(
@@ -389,7 +388,8 @@ CREATE TABLE MV_CREATION_METADATA
CAT_NAME nvarchar(256) NOT NULL,
DB_NAME nvarchar(128) NOT NULL,
TBL_NAME nvarchar(256) NOT NULL,
- TXN_LIST text NULL
+ TXN_LIST text NULL,
+ MATERIALIZATION_TIME bigint NOT NULL
);
ALTER TABLE MV_CREATION_METADATA ADD CONSTRAINT MV_CREATION_METADATA_PK PRIMARY KEY (MV_CREATION_METADATA_ID);
@@ -1035,7 +1035,8 @@ CREATE TABLE COMPLETED_TXN_COMPONENTS(
CTC_TABLE nvarchar(128) NULL,
CTC_PARTITION nvarchar(767) NULL,
CTC_TIMESTAMP datetime2 DEFAULT CURRENT_TIMESTAMP NOT NULL,
- CTC_WRITEID bigint
+ CTC_WRITEID bigint,
+ CTC_UPDATE_DELETE char(1) NOT NULL
);
CREATE INDEX COMPLETED_TXN_COMPONENTS_IDX ON COMPLETED_TXN_COMPONENTS (CTC_DATABASE, CTC_TABLE, CTC_PARTITION);
@@ -1192,6 +1193,17 @@ PRIMARY KEY CLUSTERED
CREATE INDEX MIN_HISTORY_LEVEL_IDX ON MIN_HISTORY_LEVEL (MHL_MIN_OPEN_TXNID);
+CREATE TABLE MATERIALIZATION_REBUILD_LOCKS (
+ MRL_TXN_ID bigint NOT NULL,
+ MRL_DB_NAME nvarchar(128) NOT NULL,
+ MRL_TBL_NAME nvarchar(256) NOT NULL,
+ MRL_LAST_HEARTBEAT bigint NOT NULL,
+PRIMARY KEY CLUSTERED
+(
+ MRL_TXN_ID ASC
+)
+);
+
CREATE TABLE "I_SCHEMA" (
"SCHEMA_ID" bigint primary key,
"SCHEMA_TYPE" int not null,
http://git-wip-us.apache.org/repos/asf/hive/blob/1b5903b0/standalone-metastore/metastore-common/src/main/sql/mssql/upgrade-3.0.0-to-3.1.0.mssql.sql
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/main/sql/mssql/upgrade-3.0.0-to-3.1.0.mssql.sql b/standalone-metastore/metastore-common/src/main/sql/mssql/upgrade-3.0.0-to-3.1.0.mssql.sql
index 41f23f7..d44cfdb 100644
--- a/standalone-metastore/metastore-common/src/main/sql/mssql/upgrade-3.0.0-to-3.1.0.mssql.sql
+++ b/standalone-metastore/metastore-common/src/main/sql/mssql/upgrade-3.0.0-to-3.1.0.mssql.sql
@@ -46,6 +46,25 @@ CREATE TABLE TXN_WRITE_NOTIFICATION_LOG (
ALTER TABLE TXN_WRITE_NOTIFICATION_LOG ADD CONSTRAINT TXN_WRITE_NOTIFICATION_LOG_PK PRIMARY KEY (WNL_TXNID, WNL_DATABASE, WNL_TABLE, WNL_PARTITION);
INSERT INTO SEQUENCE_TABLE (SEQUENCE_NAME, NEXT_VAL) VALUES ('org.apache.hadoop.hive.metastore.model.MTxnWriteNotificationLog', 1);
+-- HIVE-19027
+-- add column MATERIALIZATION_TIME (bigint) to MV_CREATION_METADATA table
+ALTER TABLE MV_CREATION_METADATA ADD MATERIALIZATION_TIME bigint NOT NULL DEFAULT(0);
+
+-- add column CTC_UPDATE_DELETE (char) to COMPLETED_TXN_COMPONENTS table
+ALTER TABLE COMPLETED_TXN_COMPONENTS ADD CTC_UPDATE_DELETE char(1) NOT NULL DEFAULT('N');
+
+CREATE TABLE MATERIALIZATION_REBUILD_LOCKS (
+ MRL_TXN_ID bigint NOT NULL,
+ MRL_DB_NAME nvarchar(128) NOT NULL,
+ MRL_TBL_NAME nvarchar(256) NOT NULL,
+ MRL_LAST_HEARTBEAT bigint NOT NULL,
+PRIMARY KEY CLUSTERED
+(
+ MRL_TXN_ID ASC
+)
+);
+
+
-- These lines need to be last. Insert any changes above.
UPDATE VERSION SET SCHEMA_VERSION='3.1.0', VERSION_COMMENT='Hive release version 3.1.0' where VER_ID=1;
SELECT 'Finished upgrading MetaStore schema from 3.0.0 to 3.1.0' AS MESSAGE;
http://git-wip-us.apache.org/repos/asf/hive/blob/1b5903b0/standalone-metastore/metastore-common/src/main/sql/mysql/hive-schema-3.1.0.mysql.sql
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/main/sql/mysql/hive-schema-3.1.0.mysql.sql b/standalone-metastore/metastore-common/src/main/sql/mysql/hive-schema-3.1.0.mysql.sql
index 29d4a43..75612a7 100644
--- a/standalone-metastore/metastore-common/src/main/sql/mysql/hive-schema-3.1.0.mysql.sql
+++ b/standalone-metastore/metastore-common/src/main/sql/mysql/hive-schema-3.1.0.mysql.sql
@@ -603,6 +603,7 @@ CREATE TABLE IF NOT EXISTS `MV_CREATION_METADATA` (
`DB_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL,
`TBL_NAME` varchar(256) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL,
`TXN_LIST` TEXT DEFAULT NULL,
+ `MATERIALIZATION_TIME` bigint(20) NOT NULL,
PRIMARY KEY (`MV_CREATION_METADATA_ID`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
/*!40101 SET character_set_client = @saved_cs_client */;
@@ -1006,7 +1007,8 @@ CREATE TABLE COMPLETED_TXN_COMPONENTS (
CTC_TABLE varchar(256),
CTC_PARTITION varchar(767),
CTC_TIMESTAMP timestamp DEFAULT CURRENT_TIMESTAMP NOT NULL,
- CTC_WRITEID bigint
+ CTC_WRITEID bigint,
+ CTC_UPDATE_DELETE char(1) NOT NULL
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
CREATE INDEX COMPLETED_TXN_COMPONENTS_IDX ON COMPLETED_TXN_COMPONENTS (CTC_DATABASE, CTC_TABLE, CTC_PARTITION) USING BTREE;
@@ -1124,6 +1126,14 @@ CREATE TABLE MIN_HISTORY_LEVEL (
CREATE INDEX MIN_HISTORY_LEVEL_IDX ON MIN_HISTORY_LEVEL (MHL_MIN_OPEN_TXNID);
+CREATE TABLE MATERIALIZATION_REBUILD_LOCKS (
+ MRL_TXN_ID bigint NOT NULL,
+ MRL_DB_NAME VARCHAR(128) NOT NULL,
+ MRL_TBL_NAME VARCHAR(256) NOT NULL,
+ MRL_LAST_HEARTBEAT bigint NOT NULL,
+ PRIMARY KEY(MRL_TXN_ID)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;
+
CREATE TABLE `I_SCHEMA` (
`SCHEMA_ID` BIGINT PRIMARY KEY,
`SCHEMA_TYPE` INTEGER NOT NULL,
http://git-wip-us.apache.org/repos/asf/hive/blob/1b5903b0/standalone-metastore/metastore-common/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql b/standalone-metastore/metastore-common/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql
index 968f4a4..d53e7fc 100644
--- a/standalone-metastore/metastore-common/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql
+++ b/standalone-metastore/metastore-common/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql
@@ -603,6 +603,7 @@ CREATE TABLE IF NOT EXISTS `MV_CREATION_METADATA` (
`DB_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL,
`TBL_NAME` varchar(256) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL,
`TXN_LIST` TEXT DEFAULT NULL,
+ `MATERIALIZATION_TIME` bigint(20) NOT NULL,
PRIMARY KEY (`MV_CREATION_METADATA_ID`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
/*!40101 SET character_set_client = @saved_cs_client */;
@@ -1006,7 +1007,8 @@ CREATE TABLE COMPLETED_TXN_COMPONENTS (
CTC_TABLE varchar(256),
CTC_PARTITION varchar(767),
CTC_TIMESTAMP timestamp DEFAULT CURRENT_TIMESTAMP NOT NULL,
- CTC_WRITEID bigint
+ CTC_WRITEID bigint,
+ CTC_UPDATE_DELETE char(1) NOT NULL
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
CREATE INDEX COMPLETED_TXN_COMPONENTS_IDX ON COMPLETED_TXN_COMPONENTS (CTC_DATABASE, CTC_TABLE, CTC_PARTITION) USING BTREE;
@@ -1124,6 +1126,14 @@ CREATE TABLE MIN_HISTORY_LEVEL (
CREATE INDEX MIN_HISTORY_LEVEL_IDX ON MIN_HISTORY_LEVEL (MHL_MIN_OPEN_TXNID);
+CREATE TABLE MATERIALIZATION_REBUILD_LOCKS (
+ MRL_TXN_ID bigint NOT NULL,
+ MRL_DB_NAME VARCHAR(128) NOT NULL,
+ MRL_TBL_NAME VARCHAR(256) NOT NULL,
+ MRL_LAST_HEARTBEAT bigint NOT NULL,
+ PRIMARY KEY(MRL_TXN_ID)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;
+
CREATE TABLE `I_SCHEMA` (
`SCHEMA_ID` BIGINT PRIMARY KEY,
`SCHEMA_TYPE` INTEGER NOT NULL,
http://git-wip-us.apache.org/repos/asf/hive/blob/1b5903b0/standalone-metastore/metastore-common/src/main/sql/mysql/upgrade-3.0.0-to-3.1.0.mysql.sql
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/main/sql/mysql/upgrade-3.0.0-to-3.1.0.mysql.sql b/standalone-metastore/metastore-common/src/main/sql/mysql/upgrade-3.0.0-to-3.1.0.mysql.sql
index e103bef..7752e89 100644
--- a/standalone-metastore/metastore-common/src/main/sql/mysql/upgrade-3.0.0-to-3.1.0.mysql.sql
+++ b/standalone-metastore/metastore-common/src/main/sql/mysql/upgrade-3.0.0-to-3.1.0.mysql.sql
@@ -46,6 +46,26 @@ CREATE TABLE TXN_WRITE_NOTIFICATION_LOG (
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
INSERT INTO `SEQUENCE_TABLE` (`SEQUENCE_NAME`, `NEXT_VAL`) VALUES ('org.apache.hadoop.hive.metastore.model.MTxnWriteNotificationLog', 1);
+-- HIVE-19027
+-- add column MATERIALIZATION_TIME (bigint) to MV_CREATION_METADATA table
+ALTER TABLE `MV_CREATION_METADATA` ADD `MATERIALIZATION_TIME` BIGINT;
+UPDATE `MV_CREATION_METADATA` SET `MATERIALIZATION_TIME` = 0;
+ALTER TABLE `MV_CREATION_METADATA` MODIFY COLUMN `MATERIALIZATION_TIME` BIGINT NOT NULL;
+
+-- add column CTC_UPDATE_DELETE (char) to COMPLETED_TXN_COMPONENTS table
+ALTER TABLE COMPLETED_TXN_COMPONENTS ADD CTC_UPDATE_DELETE char(1);
+UPDATE COMPLETED_TXN_COMPONENTS SET CTC_UPDATE_DELETE = 'N';
+ALTER TABLE COMPLETED_TXN_COMPONENTS MODIFY COLUMN CTC_UPDATE_DELETE char(1) NOT NULL;
+
+CREATE TABLE MATERIALIZATION_REBUILD_LOCKS (
+ MRL_TXN_ID BIGINT NOT NULL,
+ MRL_DB_NAME VARCHAR(128) NOT NULL,
+ MRL_TBL_NAME VARCHAR(256) NOT NULL,
+ MRL_LAST_HEARTBEAT BIGINT NOT NULL,
+ PRIMARY KEY(MRL_TXN_ID)
+);
+
+
-- These lines need to be last. Insert any changes above.
UPDATE VERSION SET SCHEMA_VERSION='3.1.0', VERSION_COMMENT='Hive release version 3.1.0' where VER_ID=1;
SELECT 'Finished upgrading MetaStore schema from 3.0.0 to 3.1.0' AS ' ';
http://git-wip-us.apache.org/repos/asf/hive/blob/1b5903b0/standalone-metastore/metastore-common/src/main/sql/oracle/hive-schema-3.1.0.oracle.sql
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/main/sql/oracle/hive-schema-3.1.0.oracle.sql b/standalone-metastore/metastore-common/src/main/sql/oracle/hive-schema-3.1.0.oracle.sql
index 9adea31..a4720c8 100644
--- a/standalone-metastore/metastore-common/src/main/sql/oracle/hive-schema-3.1.0.oracle.sql
+++ b/standalone-metastore/metastore-common/src/main/sql/oracle/hive-schema-3.1.0.oracle.sql
@@ -410,7 +410,8 @@ CREATE TABLE MV_CREATION_METADATA
CAT_NAME VARCHAR2(256) NOT NULL,
DB_NAME VARCHAR2(128) NOT NULL,
TBL_NAME VARCHAR2(256) NOT NULL,
- TXN_LIST CLOB NULL
+ TXN_LIST CLOB NULL,
+ MATERIALIZATION_TIME NUMBER NOT NULL
);
ALTER TABLE MV_CREATION_METADATA ADD CONSTRAINT MV_CREATION_METADATA_PK PRIMARY KEY (MV_CREATION_METADATA_ID);
@@ -983,7 +984,8 @@ CREATE TABLE COMPLETED_TXN_COMPONENTS (
CTC_TABLE VARCHAR2(256),
CTC_PARTITION VARCHAR2(767),
CTC_TIMESTAMP timestamp DEFAULT CURRENT_TIMESTAMP NOT NULL,
- CTC_WRITEID NUMBER(19)
+ CTC_WRITEID NUMBER(19),
+ CTC_UPDATE_DELETE CHAR(1) NOT NULL
) ROWDEPENDENCIES;
CREATE INDEX COMPLETED_TXN_COMPONENTS_INDEX ON COMPLETED_TXN_COMPONENTS (CTC_DATABASE, CTC_TABLE, CTC_PARTITION);
@@ -1100,6 +1102,14 @@ CREATE TABLE MIN_HISTORY_LEVEL (
CREATE INDEX MIN_HISTORY_LEVEL_IDX ON MIN_HISTORY_LEVEL (MHL_MIN_OPEN_TXNID);
+CREATE TABLE MATERIALIZATION_REBUILD_LOCKS (
+ MRL_TXN_ID NUMBER NOT NULL,
+ MRL_DB_NAME VARCHAR(128) NOT NULL,
+ MRL_TBL_NAME VARCHAR(256) NOT NULL,
+ MRL_LAST_HEARTBEAT NUMBER NOT NULL,
+ PRIMARY KEY(MRL_TXN_ID)
+);
+
CREATE TABLE "I_SCHEMA" (
"SCHEMA_ID" number primary key,
"SCHEMA_TYPE" number not null,
http://git-wip-us.apache.org/repos/asf/hive/blob/1b5903b0/standalone-metastore/metastore-common/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql b/standalone-metastore/metastore-common/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql
index faca669..e58ee33 100644
--- a/standalone-metastore/metastore-common/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql
+++ b/standalone-metastore/metastore-common/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql
@@ -410,7 +410,8 @@ CREATE TABLE MV_CREATION_METADATA
CAT_NAME VARCHAR2(256) NOT NULL,
DB_NAME VARCHAR2(128) NOT NULL,
TBL_NAME VARCHAR2(256) NOT NULL,
- TXN_LIST CLOB NULL
+ TXN_LIST CLOB NULL,
+ MATERIALIZATION_TIME NUMBER NOT NULL
);
ALTER TABLE MV_CREATION_METADATA ADD CONSTRAINT MV_CREATION_METADATA_PK PRIMARY KEY (MV_CREATION_METADATA_ID);
@@ -983,7 +984,8 @@ CREATE TABLE COMPLETED_TXN_COMPONENTS (
CTC_TABLE VARCHAR2(256),
CTC_PARTITION VARCHAR2(767),
CTC_TIMESTAMP timestamp DEFAULT CURRENT_TIMESTAMP NOT NULL,
- CTC_WRITEID NUMBER(19)
+ CTC_WRITEID NUMBER(19),
+ CTC_UPDATE_DELETE CHAR(1) NOT NULL
) ROWDEPENDENCIES;
CREATE INDEX COMPLETED_TXN_COMPONENTS_INDEX ON COMPLETED_TXN_COMPONENTS (CTC_DATABASE, CTC_TABLE, CTC_PARTITION);
@@ -1100,6 +1102,14 @@ CREATE TABLE MIN_HISTORY_LEVEL (
CREATE INDEX MIN_HISTORY_LEVEL_IDX ON MIN_HISTORY_LEVEL (MHL_MIN_OPEN_TXNID);
+CREATE TABLE MATERIALIZATION_REBUILD_LOCKS (
+ MRL_TXN_ID NUMBER NOT NULL,
+ MRL_DB_NAME VARCHAR(128) NOT NULL,
+ MRL_TBL_NAME VARCHAR(256) NOT NULL,
+ MRL_LAST_HEARTBEAT NUMBER NOT NULL,
+ PRIMARY KEY(MRL_TXN_ID)
+);
+
CREATE TABLE "I_SCHEMA" (
"SCHEMA_ID" number primary key,
"SCHEMA_TYPE" number not null,
http://git-wip-us.apache.org/repos/asf/hive/blob/1b5903b0/standalone-metastore/metastore-common/src/main/sql/oracle/upgrade-3.0.0-to-3.1.0.oracle.sql
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/main/sql/oracle/upgrade-3.0.0-to-3.1.0.oracle.sql b/standalone-metastore/metastore-common/src/main/sql/oracle/upgrade-3.0.0-to-3.1.0.oracle.sql
index cf8699b..e4efe4d 100644
--- a/standalone-metastore/metastore-common/src/main/sql/oracle/upgrade-3.0.0-to-3.1.0.oracle.sql
+++ b/standalone-metastore/metastore-common/src/main/sql/oracle/upgrade-3.0.0-to-3.1.0.oracle.sql
@@ -46,6 +46,25 @@ CREATE TABLE TXN_WRITE_NOTIFICATION_LOG (
);
INSERT INTO SEQUENCE_TABLE (SEQUENCE_NAME, NEXT_VAL) VALUES ('org.apache.hadoop.hive.metastore.model.MTxnWriteNotificationLog', 1);
+-- HIVE-19027
+-- add column MATERIALIZATION_TIME (bigint) to MV_CREATION_METADATA table
+ALTER TABLE MV_CREATION_METADATA ADD MATERIALIZATION_TIME NUMBER NULL;
+UPDATE MV_CREATION_METADATA SET MATERIALIZATION_TIME = 0;
+ALTER TABLE MV_CREATION_METADATA MODIFY(MATERIALIZATION_TIME NOT NULL);
+
+-- add column CTC_UPDATE_DELETE (char) to COMPLETED_TXN_COMPONENTS table
+ALTER TABLE COMPLETED_TXN_COMPONENTS ADD CTC_UPDATE_DELETE char(1) NULL;
+UPDATE COMPLETED_TXN_COMPONENTS SET CTC_UPDATE_DELETE = 'N';
+ALTER TABLE COMPLETED_TXN_COMPONENTS MODIFY(CTC_UPDATE_DELETE NOT NULL);
+
+CREATE TABLE MATERIALIZATION_REBUILD_LOCKS (
+ MRL_TXN_ID NUMBER NOT NULL,
+ MRL_DB_NAME VARCHAR(128) NOT NULL,
+ MRL_TBL_NAME VARCHAR(256) NOT NULL,
+ MRL_LAST_HEARTBEAT NUMBER NOT NULL,
+ PRIMARY KEY(MRL_TXN_ID)
+);
+
-- These lines need to be last. Insert any changes above.
UPDATE VERSION SET SCHEMA_VERSION='3.1.0', VERSION_COMMENT='Hive release version 3.1.0' where VER_ID=1;
SELECT 'Finished upgrading MetaStore schema from 3.0.0 to 3.1.0' AS Status from dual;
http://git-wip-us.apache.org/repos/asf/hive/blob/1b5903b0/standalone-metastore/metastore-common/src/main/sql/postgres/hive-schema-3.1.0.postgres.sql
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/main/sql/postgres/hive-schema-3.1.0.postgres.sql b/standalone-metastore/metastore-common/src/main/sql/postgres/hive-schema-3.1.0.postgres.sql
index 7a8a419..a74c388 100644
--- a/standalone-metastore/metastore-common/src/main/sql/postgres/hive-schema-3.1.0.postgres.sql
+++ b/standalone-metastore/metastore-common/src/main/sql/postgres/hive-schema-3.1.0.postgres.sql
@@ -404,7 +404,8 @@ CREATE TABLE "MV_CREATION_METADATA" (
"CAT_NAME" character varying(256) NOT NULL,
"DB_NAME" character varying(128) NOT NULL,
"TBL_NAME" character varying(256) NOT NULL,
- "TXN_LIST" text
+ "TXN_LIST" text,
+ "MATERIALIZATION_TIME" bigint NOT NULL
);
--
@@ -1673,7 +1674,8 @@ CREATE TABLE COMPLETED_TXN_COMPONENTS (
CTC_TABLE varchar(256),
CTC_PARTITION varchar(767),
CTC_TIMESTAMP timestamp DEFAULT CURRENT_TIMESTAMP NOT NULL,
- CTC_WRITEID bigint
+ CTC_WRITEID bigint,
+ CTC_UPDATE_DELETE char(1) NOT NULL
);
CREATE INDEX COMPLETED_TXN_COMPONENTS_INDEX ON COMPLETED_TXN_COMPONENTS USING btree (CTC_DATABASE, CTC_TABLE, CTC_PARTITION);
@@ -1790,6 +1792,14 @@ CREATE TABLE MIN_HISTORY_LEVEL (
CREATE INDEX MIN_HISTORY_LEVEL_IDX ON MIN_HISTORY_LEVEL (MHL_MIN_OPEN_TXNID);
+CREATE TABLE MATERIALIZATION_REBUILD_LOCKS (
+ MRL_TXN_ID bigint NOT NULL,
+ MRL_DB_NAME varchar(128) NOT NULL,
+ MRL_TBL_NAME varchar(256) NOT NULL,
+ MRL_LAST_HEARTBEAT bigint NOT NULL,
+ PRIMARY KEY(MRL_TXN_ID)
+);
+
CREATE TABLE "I_SCHEMA" (
"SCHEMA_ID" bigint primary key,
"SCHEMA_TYPE" integer not null,
http://git-wip-us.apache.org/repos/asf/hive/blob/1b5903b0/standalone-metastore/metastore-common/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql b/standalone-metastore/metastore-common/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql
index 2e7ac5a..5d1a525 100644
--- a/standalone-metastore/metastore-common/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql
+++ b/standalone-metastore/metastore-common/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql
@@ -404,7 +404,8 @@ CREATE TABLE "MV_CREATION_METADATA" (
"CAT_NAME" character varying(256) NOT NULL,
"DB_NAME" character varying(128) NOT NULL,
"TBL_NAME" character varying(256) NOT NULL,
- "TXN_LIST" text
+ "TXN_LIST" text,
+ "MATERIALIZATION_TIME" bigint NOT NULL
);
--
@@ -1282,6 +1283,11 @@ CREATE INDEX "TBLS_N50" ON "TBLS" USING btree ("SD_ID");
CREATE INDEX "TBL_COL_PRIVS_N49" ON "TBL_COL_PRIVS" USING btree ("TBL_ID");
+--
+-- Name: TAB_COL_STATS_IDX; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace:
+--
+
+CREATE INDEX "TAB_COL_STATS_IDX" ON "TAB_COL_STATS" USING btree ("CAT_NAME", "DB_NAME","TABLE_NAME","COLUMN_NAME");
--
-- Name: TBL_PRIVS_N49; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace:
@@ -1303,13 +1309,6 @@ CREATE INDEX "TYPE_FIELDS_N49" ON "TYPE_FIELDS" USING btree ("TYPE_NAME");
CREATE INDEX "TAB_COL_STATS_N49" ON "TAB_COL_STATS" USING btree ("TBL_ID");
--
--- Name: TAB_COL_STATS_IDX; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace:
---
-
-CREATE INDEX "TAB_COL_STATS_IDX" ON "TAB_COL_STATS" USING btree ("CAT_NAME", "DB_NAME","TABLE_NAME","COLUMN_NAME");
-
-
---
-- Name: PART_COL_STATS_N49; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace:
--
@@ -1675,7 +1674,8 @@ CREATE TABLE COMPLETED_TXN_COMPONENTS (
CTC_TABLE varchar(256),
CTC_PARTITION varchar(767),
CTC_TIMESTAMP timestamp DEFAULT CURRENT_TIMESTAMP NOT NULL,
- CTC_WRITEID bigint
+ CTC_WRITEID bigint,
+ CTC_UPDATE_DELETE char(1) NOT NULL
);
CREATE INDEX COMPLETED_TXN_COMPONENTS_INDEX ON COMPLETED_TXN_COMPONENTS USING btree (CTC_DATABASE, CTC_TABLE, CTC_PARTITION);
@@ -1792,6 +1792,14 @@ CREATE TABLE MIN_HISTORY_LEVEL (
CREATE INDEX MIN_HISTORY_LEVEL_IDX ON MIN_HISTORY_LEVEL (MHL_MIN_OPEN_TXNID);
+CREATE TABLE MATERIALIZATION_REBUILD_LOCKS (
+ MRL_TXN_ID bigint NOT NULL,
+ MRL_DB_NAME varchar(128) NOT NULL,
+ MRL_TBL_NAME varchar(256) NOT NULL,
+ MRL_LAST_HEARTBEAT bigint NOT NULL,
+ PRIMARY KEY(MRL_TXN_ID)
+);
+
CREATE TABLE "I_SCHEMA" (
"SCHEMA_ID" bigint primary key,
"SCHEMA_TYPE" integer not null,
http://git-wip-us.apache.org/repos/asf/hive/blob/1b5903b0/standalone-metastore/metastore-common/src/main/sql/postgres/upgrade-3.0.0-to-3.1.0.postgres.sql
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/main/sql/postgres/upgrade-3.0.0-to-3.1.0.postgres.sql b/standalone-metastore/metastore-common/src/main/sql/postgres/upgrade-3.0.0-to-3.1.0.postgres.sql
index 445c3a2..dadf065 100644
--- a/standalone-metastore/metastore-common/src/main/sql/postgres/upgrade-3.0.0-to-3.1.0.postgres.sql
+++ b/standalone-metastore/metastore-common/src/main/sql/postgres/upgrade-3.0.0-to-3.1.0.postgres.sql
@@ -48,6 +48,25 @@ CREATE TABLE "TXN_WRITE_NOTIFICATION_LOG" (
);
INSERT INTO "SEQUENCE_TABLE" ("SEQUENCE_NAME", "NEXT_VAL") VALUES ('org.apache.hadoop.hive.metastore.model.MTxnWriteNotificationLog', 1);
+-- HIVE-19027
+-- add column MATERIALIZATION_TIME (bigint) to MV_CREATION_METADATA table
+ALTER TABLE "MV_CREATION_METADATA" ADD COLUMN "MATERIALIZATION_TIME" bigint NULL;
+UPDATE "MV_CREATION_METADATA" SET "MATERIALIZATION_TIME" = 0;
+ALTER TABLE "MV_CREATION_METADATA" ALTER COLUMN "MATERIALIZATION_TIME" SET NOT NULL;
+
+-- add column CTC_UPDATE_DELETE (char) to COMPLETED_TXN_COMPONENTS table
+ALTER TABLE COMPLETED_TXN_COMPONENTS ADD COLUMN CTC_UPDATE_DELETE char(1) NULL;
+UPDATE COMPLETED_TXN_COMPONENTS SET CTC_UPDATE_DELETE = 'N';
+ALTER TABLE COMPLETED_TXN_COMPONENTS ALTER COLUMN CTC_UPDATE_DELETE SET NOT NULL;
+
+CREATE TABLE MATERIALIZATION_REBUILD_LOCKS (
+ MRL_TXN_ID bigint NOT NULL,
+ MRL_DB_NAME varchar(128) NOT NULL,
+ MRL_TBL_NAME varchar(256) NOT NULL,
+ MRL_LAST_HEARTBEAT bigint NOT NULL,
+ PRIMARY KEY(MRL_TXN_ID)
+);
+
-- These lines need to be last. Insert any changes above.
UPDATE "VERSION" SET "SCHEMA_VERSION"='3.1.0', "VERSION_COMMENT"='Hive release version 3.1.0' where "VER_ID"=1;
SELECT 'Finished upgrading MetaStore schema from 3.0.0 to 3.1.0';
http://git-wip-us.apache.org/repos/asf/hive/blob/1b5903b0/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift b/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift
index 1ca6454..8965059 100644
--- a/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift
+++ b/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift
@@ -1073,6 +1073,7 @@ struct CreationMetadata {
3: required string tblName,
4: required set<string> tablesUsed,
5: optional string validTxnList,
+ 6: optional i64 materializationTime
}
struct NotificationEventRequest {
@@ -1273,10 +1274,7 @@ struct TableMeta {
}
struct Materialization {
- 1: required set<string> tablesUsed;
- 2: optional string validTxnList
- 3: optional i64 invalidationTime;
- 4: optional bool sourceTablesUpdateDeleteModified;
+ 1: required bool sourceTablesUpdateDeleteModified;
}
// Data types for workload management.
@@ -1728,7 +1726,7 @@ service ThriftHiveMetastore extends fb303.FacebookService
GetTableResult get_table_req(1:GetTableRequest req) throws (1:MetaException o1, 2:NoSuchObjectException o2)
GetTablesResult get_table_objects_by_name_req(1:GetTablesRequest req)
throws (1:MetaException o1, 2:InvalidOperationException o2, 3:UnknownDBException o3)
- map<string, Materialization> get_materialization_invalidation_info(1:string dbname, 2:list<string> tbl_names)
+ Materialization get_materialization_invalidation_info(1:CreationMetadata creation_metadata, 2:string validTxnList)
throws (1:MetaException o1, 2:InvalidOperationException o2, 3:UnknownDBException o3)
void update_creation_metadata(1: string catName, 2:string dbname, 3:string tbl_name, 4:CreationMetadata creation_metadata)
throws (1:MetaException o1, 2:InvalidOperationException o2, 3:UnknownDBException o3)
http://git-wip-us.apache.org/repos/asf/hive/blob/1b5903b0/standalone-metastore/metastore-common/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java b/standalone-metastore/metastore-common/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java
index 53c4d24..d91f737 100644
--- a/standalone-metastore/metastore-common/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java
+++ b/standalone-metastore/metastore-common/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java
@@ -164,8 +164,6 @@ public class HiveMetaStoreClientPreCatalog implements IMetaStoreClient, AutoClos
// instantiate the metastore server handler directly instead of connecting
// through the network
client = HiveMetaStore.newRetryingHMSHandler("hive client", this.conf, true);
- // Initialize materializations invalidation cache (only for local metastore)
- MaterializationsInvalidationCache.get().init(conf, (IHMSHandler) client);
isConnected = true;
snapshotActiveConf();
return;
@@ -1442,10 +1440,9 @@ public class HiveMetaStoreClientPreCatalog implements IMetaStoreClient, AutoClos
/** {@inheritDoc} */
@Override
- public Map<String, Materialization> getMaterializationsInvalidationInfo(String dbName, List<String> viewNames)
+ public Materialization getMaterializationInvalidationInfo(CreationMetadata cm, String validTxnList)
throws MetaException, InvalidOperationException, UnknownDBException, TException {
- return client.get_materialization_invalidation_info(
- dbName, filterHook.filterTableNames(null, dbName, viewNames));
+ return client.get_materialization_invalidation_info(cm, validTxnList);
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/hive/blob/1b5903b0/standalone-metastore/metastore-common/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMaterializationsCacheCleaner.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMaterializationsCacheCleaner.java b/standalone-metastore/metastore-common/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMaterializationsCacheCleaner.java
deleted file mode 100644
index 8debcce..0000000
--- a/standalone-metastore/metastore-common/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMaterializationsCacheCleaner.java
+++ /dev/null
@@ -1,328 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.metastore;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.metastore.api.BasicTxnInfo;
-import org.apache.hadoop.hive.metastore.api.CreationMetadata;
-import org.apache.hadoop.hive.metastore.api.Materialization;
-import org.apache.hadoop.hive.metastore.api.Table;
-import org.junit.Assert;
-import org.junit.FixMethodOrder;
-import org.junit.Test;
-import org.junit.runners.MethodSorters;
-
-import java.util.Map;
-
-import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_CATALOG_NAME;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-/**
- * Unit tests for {@link org.apache.hadoop.hive.metastore.MaterializationsInvalidationCache}.
- * The tests focus on arrival of notifications (possibly out of order) and the logic
- * to clean up the materializations cache. Tests need to be executed in a certain order
- * to avoid interactions among them, as the invalidation cache is a singleton.
- */
-@FixMethodOrder(MethodSorters.NAME_ASCENDING)
-public class TestMetaStoreMaterializationsCacheCleaner {
-
- private static final String DB_NAME = "hive3252";
- private static final String TBL_NAME_1 = "tmptbl1";
- private static final String TBL_NAME_2 = "tmptbl2";
- private static final String TBL_NAME_3 = "tmptbl3";
- private static final String MV_NAME_1 = "mv1";
- private static final String MV_NAME_2 = "mv2";
-
-
- @Test
- public void testCleanerScenario1() throws Exception {
- // create mock raw store
- Configuration conf = new Configuration();
- conf.set("metastore.materializations.invalidation.impl", "DISABLE");
- // create mock handler
- final IHMSHandler handler = mock(IHMSHandler.class);
- // initialize invalidation cache (set conf to disable)
- MaterializationsInvalidationCache.get().init(conf, handler);
-
- // This is a dummy test, invalidation cache is not supposed to
- // record any information.
- MaterializationsInvalidationCache.get().notifyTableModification(
- DB_NAME, TBL_NAME_1, 1, 1, false);
- int id = 2;
- BasicTxnInfo txn2 = createTxnInfo(DB_NAME, TBL_NAME_1, id);
- MaterializationsInvalidationCache.get().notifyTableModification(
- DB_NAME, TBL_NAME_1, id, id, false);
- // Create tbl2 (nothing to do)
- id = 3;
- BasicTxnInfo txn3 = createTxnInfo(DB_NAME, TBL_NAME_1, id);
- MaterializationsInvalidationCache.get().notifyTableModification(
- DB_NAME, TBL_NAME_2, id, id, false);
- // Cleanup (current = 4, duration = 4) -> Does nothing
- long removed = MaterializationsInvalidationCache.get().cleanup(0L);
- Assert.assertEquals(0L, removed);
- // Create mv1
- Table mv1 = mock(Table.class);
- when(mv1.getDbName()).thenReturn(DB_NAME);
- when(mv1.getTableName()).thenReturn(MV_NAME_1);
- CreationMetadata mockCM1 = new CreationMetadata(
- DEFAULT_CATALOG_NAME, DB_NAME, MV_NAME_1,
- ImmutableSet.of(
- DB_NAME + "." + TBL_NAME_1,
- DB_NAME + "." + TBL_NAME_2));
- // Create txn list (highWatermark=4;minOpenTxn=Long.MAX_VALUE)
- mockCM1.setValidTxnList("3:" + Long.MAX_VALUE + "::");
- when(mv1.getCreationMetadata()).thenReturn(mockCM1);
- MaterializationsInvalidationCache.get().createMaterializedView(mockCM1.getDbName(), mockCM1.getTblName(),
- mockCM1.getTablesUsed(), mockCM1.getValidTxnList());
- // Format <txnId>$<table_name>:<hwm>:<minOpenWriteId>:<open_writeids>:<abort_writeids>$<table_name>
- Map<String, Materialization> invalidationInfos =
- MaterializationsInvalidationCache.get().getMaterializationInvalidationInfo(
- DB_NAME, ImmutableList.of(MV_NAME_1));
- Assert.assertTrue(invalidationInfos.isEmpty());
- id = 10;
- BasicTxnInfo txn10 = createTxnInfo(DB_NAME, TBL_NAME_2, id);
- MaterializationsInvalidationCache.get().notifyTableModification(
- DB_NAME, TBL_NAME_2, id, id, false);
- id = 9;
- BasicTxnInfo txn9 = createTxnInfo(DB_NAME, TBL_NAME_1, id);
- MaterializationsInvalidationCache.get().notifyTableModification(
- DB_NAME, TBL_NAME_1, id, id, false);
- // Cleanup (current = 12, duration = 4) -> Removes txn1, txn2, txn3
- removed = MaterializationsInvalidationCache.get().cleanup(8L);
- Assert.assertEquals(0L, removed);
- invalidationInfos =
- MaterializationsInvalidationCache.get().getMaterializationInvalidationInfo(
- DB_NAME, ImmutableList.of(MV_NAME_1));
- Assert.assertTrue(invalidationInfos.isEmpty());
- // Create mv2
- Table mv2 = mock(Table.class);
- when(mv2.getDbName()).thenReturn(DB_NAME);
- when(mv2.getTableName()).thenReturn(MV_NAME_2);
- CreationMetadata mockCM2 = new CreationMetadata(
- DEFAULT_CATALOG_NAME, DB_NAME, MV_NAME_2,
- ImmutableSet.of(
- DB_NAME + "." + TBL_NAME_1,
- DB_NAME + "." + TBL_NAME_2));
- // Create txn list (highWatermark=10;minOpenTxn=Long.MAX_VALUE)
- mockCM2.setValidTxnList("10:" + Long.MAX_VALUE + "::");
- when(mv2.getCreationMetadata()).thenReturn(mockCM2);
- MaterializationsInvalidationCache.get().createMaterializedView(mockCM2.getDbName(), mockCM2.getTblName(),
- mockCM2.getTablesUsed(), mockCM2.getValidTxnList());
- when(mv2.getCreationMetadata()).thenReturn(mockCM2);
- invalidationInfos =
- MaterializationsInvalidationCache.get().getMaterializationInvalidationInfo(
- DB_NAME, ImmutableList.of(MV_NAME_1, MV_NAME_2));
- Assert.assertTrue(invalidationInfos.isEmpty());
- // Create tbl3 (nothing to do)
- MaterializationsInvalidationCache.get().notifyTableModification(
- DB_NAME, TBL_NAME_3, 11, 11, false);
- MaterializationsInvalidationCache.get().notifyTableModification(
- DB_NAME, TBL_NAME_3, 18, 18, false);
- MaterializationsInvalidationCache.get().notifyTableModification(
- DB_NAME, TBL_NAME_1, 14, 14, false);
- MaterializationsInvalidationCache.get().notifyTableModification(
- DB_NAME, TBL_NAME_1, 17, 17, false);
- MaterializationsInvalidationCache.get().notifyTableModification(
- DB_NAME, TBL_NAME_2, 16, 16, false);
- // Cleanup (current = 20, duration = 4) -> Removes txn10, txn11
- removed = MaterializationsInvalidationCache.get().cleanup(16L);
- Assert.assertEquals(0L, removed);
- invalidationInfos =
- MaterializationsInvalidationCache.get().getMaterializationInvalidationInfo(
- DB_NAME, ImmutableList.of(MV_NAME_1, MV_NAME_2));
- Assert.assertTrue(invalidationInfos.isEmpty());
- MaterializationsInvalidationCache.get().notifyTableModification(
- DB_NAME, TBL_NAME_1, 12, 12, false);
- MaterializationsInvalidationCache.get().notifyTableModification(
- DB_NAME, TBL_NAME_2, 15, 15, false);
- MaterializationsInvalidationCache.get().notifyTableModification(
- DB_NAME, TBL_NAME_2, 7, 7, false);
- invalidationInfos =
- MaterializationsInvalidationCache.get().getMaterializationInvalidationInfo(
- DB_NAME, ImmutableList.of(MV_NAME_1, MV_NAME_2));
- Assert.assertTrue(invalidationInfos.isEmpty());
- // Cleanup (current = 24, duration = 4) -> Removes txn9, txn14, txn15, txn16, txn17, txn18
- removed = MaterializationsInvalidationCache.get().cleanup(20L);
- Assert.assertEquals(0L, removed);
- invalidationInfos =
- MaterializationsInvalidationCache.get().getMaterializationInvalidationInfo(
- DB_NAME, ImmutableList.of(MV_NAME_1, MV_NAME_2));
- Assert.assertTrue(invalidationInfos.isEmpty());
- // Cleanup (current = 28, duration = 4) -> Removes txn9
- removed = MaterializationsInvalidationCache.get().cleanup(24L);
- Assert.assertEquals(0L, removed);
- }
-
- @Test
- public void testCleanerScenario2() throws Exception {
- // create mock raw store
- Configuration conf = new Configuration();
- conf.set("metastore.materializations.invalidation.impl", "DEFAULT");
- // create mock handler
- final IHMSHandler handler = mock(IHMSHandler.class);
- // initialize invalidation cache (set conf to default)
- MaterializationsInvalidationCache.get().init(conf, handler);
-
- // Scenario consists of the following steps:
- // Create tbl1
- // (t = 1) Insert row in tbl1
- // (t = 2) Insert row in tbl1
- // Create tbl2
- // (t = 3) Insert row in tbl2
- // Cleanup (current = 4, duration = 4) -> Does nothing
- // Create mv1
- // (t = 10) Insert row in tbl2
- // (t = 9) Insert row in tbl1 (out of order)
- // Cleanup (current = 12, duration = 4) -> Removes txn1, txn2, txn3
- // Create mv2
- // Create tbl3
- // (t = 11) Insert row in tbl3
- // (t = 18) Insert row in tbl3
- // (t = 14) Insert row in tbl1
- // (t = 17) Insert row in tbl1
- // (t = 16) Insert row in tbl2
- // Cleanup (current = 20, duration = 4) -> Removes txn10, txn11
- // (t = 12) Insert row in tbl1
- // (t = 15) Insert row in tbl2
- // (t = 7) Insert row in tbl2
- // Cleanup (current = 24, duration = 4) -> Removes txn9, txn14, txn15, txn16, txn17, txn18
- // Create tbl1 (nothing to do)
- MaterializationsInvalidationCache.get().notifyTableModification(
- DB_NAME, TBL_NAME_1, 1, 1, false);
- int id = 2;
- BasicTxnInfo txn2 = createTxnInfo(DB_NAME, TBL_NAME_1, id);
- MaterializationsInvalidationCache.get().notifyTableModification(
- DB_NAME, TBL_NAME_1, id, id, false);
- // Create tbl2 (nothing to do)
- id = 3;
- BasicTxnInfo txn3 = createTxnInfo(DB_NAME, TBL_NAME_1, id);
- MaterializationsInvalidationCache.get().notifyTableModification(
- DB_NAME, TBL_NAME_2, id, id, false);
- // Cleanup (current = 4, duration = 4) -> Does nothing
- long removed = MaterializationsInvalidationCache.get().cleanup(0L);
- Assert.assertEquals(0L, removed);
- // Create mv1
- Table mv1 = mock(Table.class);
- when(mv1.getDbName()).thenReturn(DB_NAME);
- when(mv1.getTableName()).thenReturn(MV_NAME_1);
- CreationMetadata mockCM1 = new CreationMetadata(
- DEFAULT_CATALOG_NAME, DB_NAME, MV_NAME_1,
- ImmutableSet.of(
- DB_NAME + "." + TBL_NAME_1,
- DB_NAME + "." + TBL_NAME_2));
- // Create txn list (highWatermark=4;minOpenTxn=Long.MAX_VALUE)
- mockCM1.setValidTxnList("3$" + DB_NAME + "." + TBL_NAME_1 + ":3:" + Long.MAX_VALUE + "::" +
- "$" + DB_NAME + "." + TBL_NAME_2 + ":3:" + Long.MAX_VALUE + "::");
- when(mv1.getCreationMetadata()).thenReturn(mockCM1);
- MaterializationsInvalidationCache.get().createMaterializedView(mockCM1.getDbName(), mockCM1.getTblName(),
- mockCM1.getTablesUsed(), mockCM1.getValidTxnList());
- Map<String, Materialization> invalidationInfos =
- MaterializationsInvalidationCache.get().getMaterializationInvalidationInfo(
- DB_NAME, ImmutableList.of(MV_NAME_1));
- Assert.assertEquals(0L, invalidationInfos.get(MV_NAME_1).getInvalidationTime());
- id = 10;
- BasicTxnInfo txn10 = createTxnInfo(DB_NAME, TBL_NAME_2, id);
- MaterializationsInvalidationCache.get().notifyTableModification(
- DB_NAME, TBL_NAME_2, id, id, false);
- id = 9;
- BasicTxnInfo txn9 = createTxnInfo(DB_NAME, TBL_NAME_1, id);
- MaterializationsInvalidationCache.get().notifyTableModification(
- DB_NAME, TBL_NAME_1, id, id, false);
- // Cleanup (current = 12, duration = 4) -> Removes txn1, txn2, txn3
- removed = MaterializationsInvalidationCache.get().cleanup(8L);
- Assert.assertEquals(3L, removed);
- invalidationInfos =
- MaterializationsInvalidationCache.get().getMaterializationInvalidationInfo(
- DB_NAME, ImmutableList.of(MV_NAME_1));
- Assert.assertEquals(9L, invalidationInfos.get(MV_NAME_1).getInvalidationTime());
- // Create mv2
- Table mv2 = mock(Table.class);
- when(mv2.getDbName()).thenReturn(DB_NAME);
- when(mv2.getTableName()).thenReturn(MV_NAME_2);
- CreationMetadata mockCM2 = new CreationMetadata(
- DEFAULT_CATALOG_NAME, DB_NAME, MV_NAME_2,
- ImmutableSet.of(
- DB_NAME + "." + TBL_NAME_1,
- DB_NAME + "." + TBL_NAME_2));
- // Create txn list (highWatermark=10;minOpenTxn=Long.MAX_VALUE)
- mockCM2.setValidTxnList("10$" + DB_NAME + "." + TBL_NAME_1 + ":10:" + Long.MAX_VALUE + "::" +
- "$" + DB_NAME + "." + TBL_NAME_2 + ":10:" + Long.MAX_VALUE + "::");
- when(mv2.getCreationMetadata()).thenReturn(mockCM2);
- MaterializationsInvalidationCache.get().createMaterializedView(mockCM2.getDbName(), mockCM2.getTblName(),
- mockCM2.getTablesUsed(), mockCM2.getValidTxnList());
- when(mv2.getCreationMetadata()).thenReturn(mockCM2);
- invalidationInfos =
- MaterializationsInvalidationCache.get().getMaterializationInvalidationInfo(
- DB_NAME, ImmutableList.of(MV_NAME_1, MV_NAME_2));
- Assert.assertEquals(9L, invalidationInfos.get(MV_NAME_1).getInvalidationTime());
- Assert.assertEquals(0L, invalidationInfos.get(MV_NAME_2).getInvalidationTime());
- // Create tbl3 (nothing to do)
- MaterializationsInvalidationCache.get().notifyTableModification(
- DB_NAME, TBL_NAME_3, 11, 11, false);
- MaterializationsInvalidationCache.get().notifyTableModification(
- DB_NAME, TBL_NAME_3, 18, 18, false);
- MaterializationsInvalidationCache.get().notifyTableModification(
- DB_NAME, TBL_NAME_1, 14, 14, false);
- MaterializationsInvalidationCache.get().notifyTableModification(
- DB_NAME, TBL_NAME_1, 17, 17, false);
- MaterializationsInvalidationCache.get().notifyTableModification(
- DB_NAME, TBL_NAME_2, 16, 16, false);
- // Cleanup (current = 20, duration = 4) -> Removes txn10, txn11
- removed = MaterializationsInvalidationCache.get().cleanup(16L);
- Assert.assertEquals(2L, removed);
- invalidationInfos =
- MaterializationsInvalidationCache.get().getMaterializationInvalidationInfo(
- DB_NAME, ImmutableList.of(MV_NAME_1, MV_NAME_2));
- Assert.assertEquals(9L, invalidationInfos.get(MV_NAME_1).getInvalidationTime());
- Assert.assertEquals(14L, invalidationInfos.get(MV_NAME_2).getInvalidationTime());
- MaterializationsInvalidationCache.get().notifyTableModification(
- DB_NAME, TBL_NAME_1, 12, 12, false);
- MaterializationsInvalidationCache.get().notifyTableModification(
- DB_NAME, TBL_NAME_2, 15, 15, false);
- MaterializationsInvalidationCache.get().notifyTableModification(
- DB_NAME, TBL_NAME_2, 7, 7, false);
- invalidationInfos =
- MaterializationsInvalidationCache.get().getMaterializationInvalidationInfo(
- DB_NAME, ImmutableList.of(MV_NAME_1, MV_NAME_2));
- Assert.assertEquals(7L, invalidationInfos.get(MV_NAME_1).getInvalidationTime());
- Assert.assertEquals(12L, invalidationInfos.get(MV_NAME_2).getInvalidationTime());
- // Cleanup (current = 24, duration = 4) -> Removes txn9, txn14, txn15, txn16, txn17, txn18
- removed = MaterializationsInvalidationCache.get().cleanup(20L);
- Assert.assertEquals(6L, removed);
- invalidationInfos =
- MaterializationsInvalidationCache.get().getMaterializationInvalidationInfo(
- DB_NAME, ImmutableList.of(MV_NAME_1, MV_NAME_2));
- Assert.assertEquals(7L, invalidationInfos.get(MV_NAME_1).getInvalidationTime());
- Assert.assertEquals(12L, invalidationInfos.get(MV_NAME_2).getInvalidationTime());
- // Cleanup (current = 28, duration = 4) -> Removes txn9
- removed = MaterializationsInvalidationCache.get().cleanup(24L);
- Assert.assertEquals(0L, removed);
- }
-
- private static BasicTxnInfo createTxnInfo(String dbName, String tableName, int i) {
- BasicTxnInfo r = new BasicTxnInfo();
- r.setDbname(dbName);
- r.setTablename(tableName);
- r.setTxnid(i);
- r.setTime(i);
- return r;
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/1b5903b0/standalone-metastore/metastore-common/src/test/java/org/apache/hadoop/hive/metastore/client/TestTablesCreateDropAlterTruncate.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/test/java/org/apache/hadoop/hive/metastore/client/TestTablesCreateDropAlterTruncate.java b/standalone-metastore/metastore-common/src/test/java/org/apache/hadoop/hive/metastore/client/TestTablesCreateDropAlterTruncate.java
index efa3e7c..816a735 100644
--- a/standalone-metastore/metastore-common/src/test/java/org/apache/hadoop/hive/metastore/client/TestTablesCreateDropAlterTruncate.java
+++ b/standalone-metastore/metastore-common/src/test/java/org/apache/hadoop/hive/metastore/client/TestTablesCreateDropAlterTruncate.java
@@ -1211,6 +1211,7 @@ public class TestTablesCreateDropAlterTruncate extends MetaStoreClientTest {
// Update the metadata for the materialized view
CreationMetadata cm = client.getTable(catName, dbName, tableNames[3]).getCreationMetadata();
cm.addToTablesUsed(dbName + "." + tableNames[1]);
+ cm.unsetMaterializationTime();
client.updateCreationMetadata(catName, dbName, tableNames[3], cm);
List<String> partNames = new ArrayList<>();