You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2018/06/18 22:03:42 UTC
[51/67] [abbrv] hive git commit: HIVE-19532 : 04 patch (Steve Yeom)
http://git-wip-us.apache.org/repos/asf/hive/blob/1d46608e/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java
index 4e3068d..f2642cf 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hive.metastore.txn;
+import org.apache.hadoop.hive.common.StatsSetupConst;
import org.apache.hadoop.hive.common.classification.RetrySemantics;
import org.apache.hadoop.hive.metastore.api.CompactionType;
import org.apache.hadoop.hive.metastore.api.MetaException;
@@ -576,8 +577,8 @@ class CompactionTxnHandler extends TxnHandler {
dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
stmt = dbConn.createStatement();
String s = "select txn_id from TXNS where " +
- "txn_id not in (select tc_txnid from TXN_COMPONENTS) and " +
- "txn_state = '" + TXN_ABORTED + "'";
+ "txn_id not in (select tc_txnid from TXN_COMPONENTS) and " +
+ "txn_state = '" + TXN_ABORTED + "'";
LOG.debug("Going to execute query <" + s + ">");
rs = stmt.executeQuery(s);
List<Long> txnids = new ArrayList<>();
@@ -587,10 +588,71 @@ class CompactionTxnHandler extends TxnHandler {
return;
}
Collections.sort(txnids);//easier to read logs
+
List<String> queries = new ArrayList<>();
StringBuilder prefix = new StringBuilder();
StringBuilder suffix = new StringBuilder();
+ // Turn off COLUMN_STATS_ACCURATE for txnids' components in TBLS and PARTITIONS
+ for (Long txnId : txnids) {
+ // Get table ids for the current txnId.
+ s = "select tbl_id from TBLS where txn_id = " + txnId;
+ LOG.debug("Going to execute query <" + s + ">");
+ rs = stmt.executeQuery(s);
+ List<Long> tblIds = new ArrayList<>();
+ while (rs.next()) {
+ tblIds.add(rs.getLong(1));
+ }
+ close(rs);
+ if(tblIds.size() <= 0) {
+ continue;
+ }
+
+ // Update COLUMN_STATS_AcCURATE.BASIC_STATS to false for each tableId.
+ prefix.append("delete from TABLE_PARAMS " +
+ " where param_key = '" + "COLUMN_STATS_ACCURATE" + "' and ");
+ suffix.append("");
+ TxnUtils.buildQueryWithINClause(conf, queries, prefix, suffix, tblIds, "tbl_id", true, false);
+
+ for (String query : queries) {
+ LOG.debug("Going to execute update <" + query + ">");
+ int rc = stmt.executeUpdate(query);
+ LOG.info("Turned off " + rc + " COLUMN_STATE_ACCURATE.BASIC_STATS states from TBLS");
+ }
+
+ queries.clear();
+ prefix.setLength(0);
+ suffix.setLength(0);
+
+ // Get partition ids for the current txnId.
+ s = "select part_id from PARTITIONS where txn_id = " + txnId;
+ LOG.debug("Going to execute query <" + s + ">");
+ rs = stmt.executeQuery(s);
+ List<Long> ptnIds = new ArrayList<>();
+ while (rs.next()) ptnIds.add(rs.getLong(1));
+ close(rs);
+ if(ptnIds.size() <= 0) {
+ continue;
+ }
+
+ // Update COLUMN_STATS_AcCURATE.BASIC_STATS to false for each ptnId.
+ prefix.append("delete from PARTITION_PARAMS " +
+ " where param_key = '" + "COLUMN_STATS_ACCURATE" + "' and ");
+ suffix.append("");
+ TxnUtils.buildQueryWithINClause(conf, queries, prefix, suffix, ptnIds, "part_id", true, false);
+
+ for (String query : queries) {
+ LOG.debug("Going to execute update <" + query + ">");
+ int rc = stmt.executeUpdate(query);
+ LOG.info("Turned off " + rc + " COLUMN_STATE_ACCURATE.BASIC_STATS states from PARTITIONS");
+ }
+
+ queries.clear();
+ prefix.setLength(0);
+ suffix.setLength(0);
+ }
+
+ // Delete from TXNS.
prefix.append("delete from TXNS where ");
suffix.append("");
http://git-wip-us.apache.org/repos/asf/hive/blob/1d46608e/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java
index 50bfca3..bfbd928 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java
@@ -28,9 +28,12 @@ import java.sql.Statement;
import java.util.Properties;
import com.google.common.annotations.VisibleForTesting;
+import jline.internal.Log;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars;
+import org.apache.zookeeper.txn.TxnHeader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -195,6 +198,68 @@ public final class TxnDbUtil {
);
try {
+ stmt.execute("CREATE TABLE \"APP\".\"TBLS\" (\"TBL_ID\" BIGINT NOT NULL, " +
+ " \"CREATE_TIME\" INTEGER NOT NULL, \"DB_ID\" BIGINT, \"LAST_ACCESS_TIME\" INTEGER NOT NULL, " +
+ " \"OWNER\" VARCHAR(767), \"OWNER_TYPE\" VARCHAR(10), \"RETENTION\" INTEGER NOT NULL, " +
+ " \"SD_ID\" BIGINT, \"TBL_NAME\" VARCHAR(256), \"TBL_TYPE\" VARCHAR(128), " +
+ " \"VIEW_EXPANDED_TEXT\" LONG VARCHAR, \"VIEW_ORIGINAL_TEXT\" LONG VARCHAR, " +
+ " \"IS_REWRITE_ENABLED\" CHAR(1) NOT NULL DEFAULT \'N\', \"TXN_ID\" BIGINT DEFAULT 0, " +
+ " \"WRITEID_LIST\" CLOB, " +
+ " PRIMARY KEY (TBL_ID))"
+ );
+ } catch (SQLException e) {
+ if (e.getMessage() != null && e.getMessage().contains("already exists")) {
+ LOG.info("TBLS table already exist, ignoring");
+ } else {
+ throw e;
+ }
+ }
+
+ try {
+ stmt.execute("CREATE TABLE \"APP\".\"PARTITIONS\" (" +
+ " \"PART_ID\" BIGINT NOT NULL, \"CREATE_TIME\" INTEGER NOT NULL, " +
+ " \"LAST_ACCESS_TIME\" INTEGER NOT NULL, \"PART_NAME\" VARCHAR(767), " +
+ " \"SD_ID\" BIGINT, \"TBL_ID\" BIGINT, \"TXN_ID\" BIGINT DEFAULT 0, " +
+ " \"WRITEID_LIST\" CLOB, " +
+ " PRIMARY KEY (PART_ID))"
+ );
+ } catch (SQLException e) {
+ if (e.getMessage() != null && e.getMessage().contains("already exists")) {
+ LOG.info("PARTITIONS table already exist, ignoring");
+ } else {
+ throw e;
+ }
+ }
+
+ try {
+ stmt.execute("CREATE TABLE \"APP\".\"TABLE_PARAMS\" (" +
+ " \"TBL_ID\" BIGINT NOT NULL, \"PARAM_KEY\" VARCHAR(256) NOT NULL, " +
+ " \"PARAM_VALUE\" CLOB, " +
+ " PRIMARY KEY (TBL_ID, PARAM_KEY))"
+ );
+ } catch (SQLException e) {
+ if (e.getMessage() != null && e.getMessage().contains("already exists")) {
+ LOG.info("TABLE_PARAMS table already exist, ignoring");
+ } else {
+ throw e;
+ }
+ }
+
+ try {
+ stmt.execute("CREATE TABLE \"APP\".\"PARTITION_PARAMS\" (" +
+ " \"PART_ID\" BIGINT NOT NULL, \"PARAM_KEY\" VARCHAR(256) NOT NULL, " +
+ " \"PARAM_VALUE\" VARCHAR(4000), " +
+ " PRIMARY KEY (PART_ID, PARAM_KEY))"
+ );
+ } catch (SQLException e) {
+ if (e.getMessage() != null && e.getMessage().contains("already exists")) {
+ LOG.info("PARTITION_PARAMS table already exist, ignoring");
+ } else {
+ throw e;
+ }
+ }
+
+ try {
stmt.execute("CREATE TABLE \"APP\".\"SEQUENCE_TABLE\" (\"SEQUENCE_NAME\" VARCHAR(256) NOT " +
"NULL, \"NEXT_VAL\" BIGINT NOT NULL)"
@@ -376,6 +441,35 @@ public final class TxnDbUtil {
}
/**
+ * Return true if the transaction of the given txnId is open.
+ * @param conf HiveConf
+ * @param txnId transaction id to search for
+ * @return
+ * @throws Exception
+ */
+ public static boolean isOpenOrAbortedTransaction(Configuration conf, long txnId) throws Exception {
+ Connection conn = null;
+ PreparedStatement stmt = null;
+ ResultSet rs = null;
+ try {
+ conn = getConnection(conf);
+ conn.setAutoCommit(false);
+ conn.setTransactionIsolation(Connection.TRANSACTION_READ_COMMITTED);
+
+ stmt = conn.prepareStatement("SELECT txn_id FROM TXNS WHERE txn_id = ?");
+ stmt.setLong(1, txnId);
+ rs = stmt.executeQuery();
+ if (!rs.next()) {
+ return false;
+ } else {
+ return true;
+ }
+ } finally {
+ closeResources(conn, stmt, rs);
+ }
+ }
+
+ /**
* Utility method used to run COUNT queries like "select count(*) from ..." against metastore tables
* @param countQuery countQuery text
* @return count countQuery result
http://git-wip-us.apache.org/repos/asf/hive/blob/1d46608e/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java
index fa291d5..aac5811 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java
@@ -25,11 +25,7 @@ import org.apache.hadoop.hive.common.ValidTxnList;
import org.apache.hadoop.hive.common.ValidTxnWriteIdList;
import org.apache.hadoop.hive.common.ValidWriteIdList;
import org.apache.hadoop.hive.metastore.TransactionalValidationListener;
-import org.apache.hadoop.hive.metastore.api.GetOpenTxnsResponse;
-import org.apache.hadoop.hive.metastore.api.GetValidWriteIdsResponse;
-import org.apache.hadoop.hive.metastore.api.Table;
-import org.apache.hadoop.hive.metastore.api.TableValidWriteIds;
-import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
+import org.apache.hadoop.hive.metastore.api.*;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars;
import org.apache.hadoop.hive.metastore.utils.JavaUtils;
@@ -46,6 +42,12 @@ import java.util.Map;
public class TxnUtils {
private static final Logger LOG = LoggerFactory.getLogger(TxnUtils.class);
+ // Transactional stats states
+ static final public char STAT_OPEN = 'o';
+ static final public char STAT_INVALID = 'i';
+ static final public char STAT_COMMITTED = 'c';
+ static final public char STAT_OBSOLETE = 's';
+
/**
* Transform a {@link org.apache.hadoop.hive.metastore.api.GetOpenTxnsResponse} to a
* {@link org.apache.hadoop.hive.common.ValidTxnList}. This assumes that the caller intends to
@@ -223,6 +225,14 @@ public class TxnUtils {
return tableIsTransactional != null && tableIsTransactional.equalsIgnoreCase("true");
}
+ public static boolean isTransactionalTable(Map<String, String> parameters) {
+ if (parameters == null) {
+ return false;
+ }
+ String tableIsTransactional = parameters.get(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL);
+ return tableIsTransactional != null && tableIsTransactional.equalsIgnoreCase("true");
+ }
+
/**
* Should produce the same result as
* {@link org.apache.hadoop.hive.ql.io.AcidUtils#isAcidTable(org.apache.hadoop.hive.ql.metadata.Table)}.
http://git-wip-us.apache.org/repos/asf/hive/blob/1d46608e/standalone-metastore/src/main/resources/package.jdo
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/resources/package.jdo b/standalone-metastore/src/main/resources/package.jdo
index 1be3e98..3997f53 100644
--- a/standalone-metastore/src/main/resources/package.jdo
+++ b/standalone-metastore/src/main/resources/package.jdo
@@ -210,6 +210,12 @@
<field name="tableType">
<column name="TBL_TYPE" length="128" jdbc-type="VARCHAR"/>
</field>
+ <field name="txnId">
+ <column name="TXN_ID"/>
+ </field>
+ <field name="writeIdList">
+ <column name="WRITEID_LIST" jdbc-type="CLOB" allows-null="true"/>
+ </field>
</class>
<class name="MCreationMetadata" identity-type="datastore" table="MV_CREATION_METADATA" detachable="true">
@@ -489,6 +495,12 @@
<column name="PARAM_VALUE" length="4000" jdbc-type="VARCHAR"/>
</value>
</field>
+ <field name="txnId">
+ <column name="TXN_ID"/>
+ </field>
+ <field name="writeIdList">
+ <column name="WRITEID_LIST" jdbc-type="CLOB" allows-null="true"/>
+ </field>
</class>
<class name="MIndex" table="IDXS" identity-type="datastore" detachable="true">
@@ -989,6 +1001,9 @@
<field name="lastAnalyzed">
<column name="LAST_ANALYZED" jdbc-type="BIGINT" allows-null="false"/>
</field>
+ <field name="txnId">
+ <column name="TXN_ID"/>
+ </field>
</class>
<class name="MPartitionColumnStatistics" table="PART_COL_STATS" identity-type="datastore" detachable="true">
@@ -1059,6 +1074,9 @@
<field name="lastAnalyzed">
<column name="LAST_ANALYZED" jdbc-type="BIGINT" allows-null="false"/>
</field>
+ <field name="txnId">
+ <column name="TXN_ID"/>
+ </field>
</class>
<class name="MVersionTable" table="VERSION" identity-type="datastore" detachable="true">
<datastore-identity>
http://git-wip-us.apache.org/repos/asf/hive/blob/1d46608e/standalone-metastore/src/main/sql/derby/hive-schema-3.0.0.derby.sql
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/sql/derby/hive-schema-3.0.0.derby.sql b/standalone-metastore/src/main/sql/derby/hive-schema-3.0.0.derby.sql
index e818e1b..280fd4a 100644
--- a/standalone-metastore/src/main/sql/derby/hive-schema-3.0.0.derby.sql
+++ b/standalone-metastore/src/main/sql/derby/hive-schema-3.0.0.derby.sql
@@ -47,7 +47,7 @@ CREATE TABLE "APP"."IDXS" ("INDEX_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT
CREATE TABLE "APP"."INDEX_PARAMS" ("INDEX_ID" BIGINT NOT NULL, "PARAM_KEY" VARCHAR(256) NOT NULL, "PARAM_VALUE" VARCHAR(4000));
-CREATE TABLE "APP"."PARTITIONS" ("PART_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT NULL, "LAST_ACCESS_TIME" INTEGER NOT NULL, "PART_NAME" VARCHAR(767), "SD_ID" BIGINT, "TBL_ID" BIGINT);
+CREATE TABLE "APP"."PARTITIONS" ("PART_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT NULL, "LAST_ACCESS_TIME" INTEGER NOT NULL, "PART_NAME" VARCHAR(767), "SD_ID" BIGINT, "TBL_ID" BIGINT, "TXN_ID" BIGINT DEFAULT 0, "WRITEID_LIST" CLOB);
CREATE TABLE "APP"."SERDES" ("SERDE_ID" BIGINT NOT NULL, "NAME" VARCHAR(128), "SLIB" VARCHAR(4000), "DESCRIPTION" VARCHAR(4000), "SERIALIZER_CLASS" VARCHAR(4000), "DESERIALIZER_CLASS" VARCHAR(4000), SERDE_TYPE INTEGER);
@@ -75,7 +75,7 @@ CREATE TABLE "APP"."COLUMNS" ("SD_ID" BIGINT NOT NULL, "COMMENT" VARCHAR(256), "
CREATE TABLE "APP"."ROLES" ("ROLE_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT NULL, "OWNER_NAME" VARCHAR(128), "ROLE_NAME" VARCHAR(128));
-CREATE TABLE "APP"."TBLS" ("TBL_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT NULL, "DB_ID" BIGINT, "LAST_ACCESS_TIME" INTEGER NOT NULL, "OWNER" VARCHAR(767), "OWNER_TYPE" VARCHAR(10), "RETENTION" INTEGER NOT NULL, "SD_ID" BIGINT, "TBL_NAME" VARCHAR(256), "TBL_TYPE" VARCHAR(128), "VIEW_EXPANDED_TEXT" LONG VARCHAR, "VIEW_ORIGINAL_TEXT" LONG VARCHAR, "IS_REWRITE_ENABLED" CHAR(1) NOT NULL DEFAULT 'N');
+CREATE TABLE "APP"."TBLS" ("TBL_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT NULL, "DB_ID" BIGINT, "LAST_ACCESS_TIME" INTEGER NOT NULL, "OWNER" VARCHAR(767), "OWNER_TYPE" VARCHAR(10), "RETENTION" INTEGER NOT NULL, "SD_ID" BIGINT, "TBL_NAME" VARCHAR(256), "TBL_TYPE" VARCHAR(128), "VIEW_EXPANDED_TEXT" LONG VARCHAR, "VIEW_ORIGINAL_TEXT" LONG VARCHAR, "IS_REWRITE_ENABLED" CHAR(1) NOT NULL DEFAULT 'N', "TXN_ID" BIGINT DEFAULT 0, "WRITEID_LIST" CLOB);
CREATE TABLE "APP"."PARTITION_KEYS" ("TBL_ID" BIGINT NOT NULL, "PKEY_COMMENT" VARCHAR(4000), "PKEY_NAME" VARCHAR(128) NOT NULL, "PKEY_TYPE" VARCHAR(767) NOT NULL, "INTEGER_IDX" INTEGER NOT NULL);
@@ -106,7 +106,8 @@ CREATE TABLE "APP"."TAB_COL_STATS"(
"LAST_ANALYZED" BIGINT,
"CS_ID" BIGINT NOT NULL,
"TBL_ID" BIGINT NOT NULL,
- "BIT_VECTOR" BLOB
+ "BIT_VECTOR" BLOB,
+ "TXN_ID" BIGINT DEFAULT 0
);
CREATE TABLE "APP"."TABLE_PARAMS" ("TBL_ID" BIGINT NOT NULL, "PARAM_KEY" VARCHAR(256) NOT NULL, "PARAM_VALUE" CLOB);
@@ -155,7 +156,8 @@ CREATE TABLE "APP"."PART_COL_STATS"(
"NUM_FALSES" BIGINT,
"LAST_ANALYZED" BIGINT,
"CS_ID" BIGINT NOT NULL,
- "PART_ID" BIGINT NOT NULL
+ "PART_ID" BIGINT NOT NULL,
+ "TXN_ID" BIGINT DEFAULT 0
);
CREATE TABLE "APP"."VERSION" ("VER_ID" BIGINT NOT NULL, "SCHEMA_VERSION" VARCHAR(127) NOT NULL, "VERSION_COMMENT" VARCHAR(255));
@@ -373,7 +375,6 @@ ALTER TABLE "APP"."MV_CREATION_METADATA" ADD CONSTRAINT "MV_CREATION_METADATA_PK
ALTER TABLE "APP"."CTLGS" ADD CONSTRAINT "CTLG_PK" PRIMARY KEY ("CTLG_ID");
-
-- foreign
ALTER TABLE "APP"."IDXS" ADD CONSTRAINT "IDXS_FK1" FOREIGN KEY ("ORIG_TBL_ID") REFERENCES "APP"."TBLS" ("TBL_ID") ON DELETE NO ACTION ON UPDATE NO ACTION;
http://git-wip-us.apache.org/repos/asf/hive/blob/1d46608e/standalone-metastore/src/main/sql/derby/hive-schema-4.0.0.derby.sql
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/sql/derby/hive-schema-4.0.0.derby.sql b/standalone-metastore/src/main/sql/derby/hive-schema-4.0.0.derby.sql
index bb69105..f92f13c 100644
--- a/standalone-metastore/src/main/sql/derby/hive-schema-4.0.0.derby.sql
+++ b/standalone-metastore/src/main/sql/derby/hive-schema-4.0.0.derby.sql
@@ -47,7 +47,7 @@ CREATE TABLE "APP"."IDXS" ("INDEX_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT
CREATE TABLE "APP"."INDEX_PARAMS" ("INDEX_ID" BIGINT NOT NULL, "PARAM_KEY" VARCHAR(256) NOT NULL, "PARAM_VALUE" VARCHAR(4000));
-CREATE TABLE "APP"."PARTITIONS" ("PART_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT NULL, "LAST_ACCESS_TIME" INTEGER NOT NULL, "PART_NAME" VARCHAR(767), "SD_ID" BIGINT, "TBL_ID" BIGINT);
+CREATE TABLE "APP"."PARTITIONS" ("PART_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT NULL, "LAST_ACCESS_TIME" INTEGER NOT NULL, "PART_NAME" VARCHAR(767), "SD_ID" BIGINT, "TBL_ID" BIGINT, "TXN_ID" BIGINT DEFAULT 0, "WRITEID_LIST" CLOB);
CREATE TABLE "APP"."SERDES" ("SERDE_ID" BIGINT NOT NULL, "NAME" VARCHAR(128), "SLIB" VARCHAR(4000), "DESCRIPTION" VARCHAR(4000), "SERIALIZER_CLASS" VARCHAR(4000), "DESERIALIZER_CLASS" VARCHAR(4000), SERDE_TYPE INTEGER);
@@ -75,7 +75,7 @@ CREATE TABLE "APP"."COLUMNS" ("SD_ID" BIGINT NOT NULL, "COMMENT" VARCHAR(256), "
CREATE TABLE "APP"."ROLES" ("ROLE_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT NULL, "OWNER_NAME" VARCHAR(128), "ROLE_NAME" VARCHAR(128));
-CREATE TABLE "APP"."TBLS" ("TBL_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT NULL, "DB_ID" BIGINT, "LAST_ACCESS_TIME" INTEGER NOT NULL, "OWNER" VARCHAR(767), "OWNER_TYPE" VARCHAR(10), "RETENTION" INTEGER NOT NULL, "SD_ID" BIGINT, "TBL_NAME" VARCHAR(256), "TBL_TYPE" VARCHAR(128), "VIEW_EXPANDED_TEXT" LONG VARCHAR, "VIEW_ORIGINAL_TEXT" LONG VARCHAR, "IS_REWRITE_ENABLED" CHAR(1) NOT NULL DEFAULT 'N');
+CREATE TABLE "APP"."TBLS" ("TBL_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT NULL, "DB_ID" BIGINT, "LAST_ACCESS_TIME" INTEGER NOT NULL, "OWNER" VARCHAR(767), "OWNER_TYPE" VARCHAR(10), "RETENTION" INTEGER NOT NULL, "SD_ID" BIGINT, "TBL_NAME" VARCHAR(256), "TBL_TYPE" VARCHAR(128), "VIEW_EXPANDED_TEXT" LONG VARCHAR, "VIEW_ORIGINAL_TEXT" LONG VARCHAR, "IS_REWRITE_ENABLED" CHAR(1) NOT NULL DEFAULT 'N', "TXN_ID" BIGINT DEFAULT 0, "WRITEID_LIST" CLOB);
CREATE TABLE "APP"."PARTITION_KEYS" ("TBL_ID" BIGINT NOT NULL, "PKEY_COMMENT" VARCHAR(4000), "PKEY_NAME" VARCHAR(128) NOT NULL, "PKEY_TYPE" VARCHAR(767) NOT NULL, "INTEGER_IDX" INTEGER NOT NULL);
@@ -106,7 +106,8 @@ CREATE TABLE "APP"."TAB_COL_STATS"(
"LAST_ANALYZED" BIGINT,
"CS_ID" BIGINT NOT NULL,
"TBL_ID" BIGINT NOT NULL,
- "BIT_VECTOR" BLOB
+ "BIT_VECTOR" BLOB,
+ "TXN_ID" BIGINT DEFAULT 0
);
CREATE TABLE "APP"."TABLE_PARAMS" ("TBL_ID" BIGINT NOT NULL, "PARAM_KEY" VARCHAR(256) NOT NULL, "PARAM_VALUE" CLOB);
@@ -155,7 +156,8 @@ CREATE TABLE "APP"."PART_COL_STATS"(
"NUM_FALSES" BIGINT,
"LAST_ANALYZED" BIGINT,
"CS_ID" BIGINT NOT NULL,
- "PART_ID" BIGINT NOT NULL
+ "PART_ID" BIGINT NOT NULL,
+ "TXN_ID" BIGINT DEFAULT 0
);
CREATE TABLE "APP"."VERSION" ("VER_ID" BIGINT NOT NULL, "SCHEMA_VERSION" VARCHAR(127) NOT NULL, "VERSION_COMMENT" VARCHAR(255));
http://git-wip-us.apache.org/repos/asf/hive/blob/1d46608e/standalone-metastore/src/main/sql/derby/upgrade-3.1.0-to-4.0.0.derby.sql
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/sql/derby/upgrade-3.1.0-to-4.0.0.derby.sql b/standalone-metastore/src/main/sql/derby/upgrade-3.1.0-to-4.0.0.derby.sql
index a511376..94f8192 100644
--- a/standalone-metastore/src/main/sql/derby/upgrade-3.1.0-to-4.0.0.derby.sql
+++ b/standalone-metastore/src/main/sql/derby/upgrade-3.1.0-to-4.0.0.derby.sql
@@ -1,5 +1,11 @@
-- Upgrade MetaStore schema from 3.1.0 to 4.0.0
-
+-- HIVE-19416
+ALTER TABLE "APP"."TBLS" ADD WRITEID_LIST CLOB;
+ALTER TABLE "APP"."TBLS" ADD TXN_ID bigint DEFAULT 0;
+ALTER TABLE "APP"."PARTITIONS" ADD WRITEID_LIST CLOB;
+ALTER TABLE "APP"."PARTITIONS" ADD TXN_ID bigint DEFAULT 0;
+ALTER TABLE "APP"."TAB_COL_STATS" ADD TXN_ID bigint DEFAULT 0;
+ALTER TABLE "APP"."PART_COL_STATS" ADD TXN_ID bigint DEFAULT 0;
-- This needs to be the last thing done. Insert any changes above this line.
UPDATE "APP".VERSION SET SCHEMA_VERSION='4.0.0', VERSION_COMMENT='Hive release version 4.0.0' where VER_ID=1;
http://git-wip-us.apache.org/repos/asf/hive/blob/1d46608e/standalone-metastore/src/main/sql/mssql/hive-schema-3.0.0.mssql.sql
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/sql/mssql/hive-schema-3.0.0.mssql.sql b/standalone-metastore/src/main/sql/mssql/hive-schema-3.0.0.mssql.sql
index c88fb18..f20f910 100644
--- a/standalone-metastore/src/main/sql/mssql/hive-schema-3.0.0.mssql.sql
+++ b/standalone-metastore/src/main/sql/mssql/hive-schema-3.0.0.mssql.sql
@@ -94,7 +94,8 @@ CREATE TABLE PART_COL_STATS
PART_ID bigint NULL,
PARTITION_NAME nvarchar(767) NOT NULL,
"TABLE_NAME" nvarchar(256) NOT NULL,
- "CAT_NAME" nvarchar(256) NOT NULL
+ "CAT_NAME" nvarchar(256) NOT NULL,
+ TXN_ID bigint NULL
);
ALTER TABLE PART_COL_STATS ADD CONSTRAINT PART_COL_STATS_PK PRIMARY KEY (CS_ID);
@@ -144,7 +145,9 @@ CREATE TABLE PARTITIONS
LAST_ACCESS_TIME int NOT NULL,
PART_NAME nvarchar(767) NULL,
SD_ID bigint NULL,
- TBL_ID bigint NULL
+ TBL_ID bigint NULL,
+ TXN_ID bigint NULL,
+ WRITEID_LIST text NULL
);
ALTER TABLE PARTITIONS ADD CONSTRAINT PARTITIONS_PK PRIMARY KEY (PART_ID);
@@ -238,7 +241,8 @@ CREATE TABLE TAB_COL_STATS
NUM_TRUES bigint NULL,
TBL_ID bigint NULL,
"TABLE_NAME" nvarchar(256) NOT NULL,
- "CAT_NAME" nvarchar(256) NOT NULL
+ "CAT_NAME" nvarchar(256) NOT NULL,
+ TXN_ID bigint NULL
);
ALTER TABLE TAB_COL_STATS ADD CONSTRAINT TAB_COL_STATS_PK PRIMARY KEY (CS_ID);
@@ -369,7 +373,9 @@ CREATE TABLE TBLS
TBL_TYPE nvarchar(128) NULL,
VIEW_EXPANDED_TEXT text NULL,
VIEW_ORIGINAL_TEXT text NULL,
- IS_REWRITE_ENABLED bit NOT NULL DEFAULT 0
+ IS_REWRITE_ENABLED bit NOT NULL DEFAULT 0,
+ TXN_ID bigint NULL,
+ WRITEID_LIST text NULL
);
ALTER TABLE TBLS ADD CONSTRAINT TBLS_PK PRIMARY KEY (TBL_ID);
http://git-wip-us.apache.org/repos/asf/hive/blob/1d46608e/standalone-metastore/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql b/standalone-metastore/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql
index 922e8fe..22637c5 100644
--- a/standalone-metastore/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql
+++ b/standalone-metastore/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql
@@ -94,7 +94,8 @@ CREATE TABLE PART_COL_STATS
PART_ID bigint NULL,
PARTITION_NAME nvarchar(767) NOT NULL,
"TABLE_NAME" nvarchar(256) NOT NULL,
- "CAT_NAME" nvarchar(256) NOT NULL
+ "CAT_NAME" nvarchar(256) NOT NULL,
+ TXN_ID bigint NULL
);
ALTER TABLE PART_COL_STATS ADD CONSTRAINT PART_COL_STATS_PK PRIMARY KEY (CS_ID);
@@ -145,7 +146,9 @@ CREATE TABLE PARTITIONS
LAST_ACCESS_TIME int NOT NULL,
PART_NAME nvarchar(767) NULL,
SD_ID bigint NULL,
- TBL_ID bigint NULL
+ TBL_ID bigint NULL,
+ TXN_ID bigint NULL,
+ WRITEID_LIST text NULL
);
ALTER TABLE PARTITIONS ADD CONSTRAINT PARTITIONS_PK PRIMARY KEY (PART_ID);
@@ -242,7 +245,8 @@ CREATE TABLE TAB_COL_STATS
NUM_TRUES bigint NULL,
TBL_ID bigint NULL,
"TABLE_NAME" nvarchar(256) NOT NULL,
- "CAT_NAME" nvarchar(256) NOT NULL
+ "CAT_NAME" nvarchar(256) NOT NULL,
+ TXN_ID bigint NULL
);
ALTER TABLE TAB_COL_STATS ADD CONSTRAINT TAB_COL_STATS_PK PRIMARY KEY (CS_ID);
@@ -377,7 +381,9 @@ CREATE TABLE TBLS
TBL_TYPE nvarchar(128) NULL,
VIEW_EXPANDED_TEXT text NULL,
VIEW_ORIGINAL_TEXT text NULL,
- IS_REWRITE_ENABLED bit NOT NULL DEFAULT 0
+ IS_REWRITE_ENABLED bit NOT NULL DEFAULT 0,
+ TXN_ID bigint NULL,
+ WRITEID_LIST text NULL
);
ALTER TABLE TBLS ADD CONSTRAINT TBLS_PK PRIMARY KEY (TBL_ID);
http://git-wip-us.apache.org/repos/asf/hive/blob/1d46608e/standalone-metastore/src/main/sql/mssql/upgrade-3.1.0-to-4.0.0.mssql.sql
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/sql/mssql/upgrade-3.1.0-to-4.0.0.mssql.sql b/standalone-metastore/src/main/sql/mssql/upgrade-3.1.0-to-4.0.0.mssql.sql
index 27b7026..f0cbf6c 100644
--- a/standalone-metastore/src/main/sql/mssql/upgrade-3.1.0-to-4.0.0.mssql.sql
+++ b/standalone-metastore/src/main/sql/mssql/upgrade-3.1.0-to-4.0.0.mssql.sql
@@ -1,5 +1,13 @@
SELECT 'Upgrading MetaStore schema from 3.1.0 to 4.0.0' AS MESSAGE;
+-- HIVE-19416
+ALTER TABLE TBLS ADD WRITEID_LIST text NULL;
+ALTER TABLE TBLS ADD TXN_ID bigint NULL;
+ALTER TABLE PARTITIONS ADD WRITEID_LIST text NULL;
+ALTER TABLE PARTITIONS ADD TXN_ID bigint NULL;
+ALTER TABLE TAB_COL_STATS ADD TXN_ID bigint NULL;
+ALTER TABLE PART_COL_STATS ADD TXN_ID bigint NULL;
+
-- These lines need to be last. Insert any changes above.
UPDATE VERSION SET SCHEMA_VERSION='4.0.0', VERSION_COMMENT='Hive release version 4.0.0' where VER_ID=1;
SELECT 'Finished upgrading MetaStore schema from 3.1.0 to 4.0.0' AS MESSAGE;
http://git-wip-us.apache.org/repos/asf/hive/blob/1d46608e/standalone-metastore/src/main/sql/mysql/hive-schema-3.0.0.mysql.sql
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/sql/mysql/hive-schema-3.0.0.mysql.sql b/standalone-metastore/src/main/sql/mysql/hive-schema-3.0.0.mysql.sql
index c54df55..6e34ab5 100644
--- a/standalone-metastore/src/main/sql/mysql/hive-schema-3.0.0.mysql.sql
+++ b/standalone-metastore/src/main/sql/mysql/hive-schema-3.0.0.mysql.sql
@@ -222,6 +222,8 @@ CREATE TABLE IF NOT EXISTS `PARTITIONS` (
`PART_NAME` varchar(767) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL,
`SD_ID` bigint(20) DEFAULT NULL,
`TBL_ID` bigint(20) DEFAULT NULL,
+ `TXN_ID` bigint(20) DEFAULT 0,
+ `WRITEID_LIST` text DEFAULT NULL,
PRIMARY KEY (`PART_ID`),
UNIQUE KEY `UNIQUEPARTITION` (`PART_NAME`,`TBL_ID`),
KEY `PARTITIONS_N49` (`TBL_ID`),
@@ -625,6 +627,8 @@ CREATE TABLE IF NOT EXISTS `TBLS` (
`VIEW_EXPANDED_TEXT` mediumtext,
`VIEW_ORIGINAL_TEXT` mediumtext,
`IS_REWRITE_ENABLED` bit(1) NOT NULL DEFAULT 0,
+ `TXN_ID` bigint(20) DEFAULT 0,
+ `WRITEID_LIST` text DEFAULT NULL,
PRIMARY KEY (`TBL_ID`),
UNIQUE KEY `UNIQUETABLE` (`TBL_NAME`,`DB_ID`),
KEY `TBLS_N50` (`SD_ID`),
@@ -720,6 +724,7 @@ CREATE TABLE IF NOT EXISTS `TAB_COL_STATS` (
`NUM_TRUES` bigint(20),
`NUM_FALSES` bigint(20),
`LAST_ANALYZED` bigint(20) NOT NULL,
+ `TXN_ID` bigint(20) DEFAULT 0,
PRIMARY KEY (`CS_ID`),
CONSTRAINT `TAB_COL_STATS_FK` FOREIGN KEY (`TBL_ID`) REFERENCES `TBLS` (`TBL_ID`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
@@ -750,6 +755,7 @@ CREATE TABLE IF NOT EXISTS `PART_COL_STATS` (
`NUM_TRUES` bigint(20),
`NUM_FALSES` bigint(20),
`LAST_ANALYZED` bigint(20) NOT NULL,
+ `TXN_ID` bigint(20) DEFAULT 0,
PRIMARY KEY (`CS_ID`),
CONSTRAINT `PART_COL_STATS_FK` FOREIGN KEY (`PART_ID`) REFERENCES `PARTITIONS` (`PART_ID`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
http://git-wip-us.apache.org/repos/asf/hive/blob/1d46608e/standalone-metastore/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql b/standalone-metastore/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql
index 6c40e6e..f8f229d 100644
--- a/standalone-metastore/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql
+++ b/standalone-metastore/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql
@@ -224,6 +224,8 @@ CREATE TABLE IF NOT EXISTS `PARTITIONS` (
`PART_NAME` varchar(767) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL,
`SD_ID` bigint(20) DEFAULT NULL,
`TBL_ID` bigint(20) DEFAULT NULL,
+ `TXN_ID` bigint(20) DEFAULT 0,
+ `WRITEID_LIST` text DEFAULT NULL,
PRIMARY KEY (`PART_ID`),
UNIQUE KEY `UNIQUEPARTITION` (`PART_NAME`,`TBL_ID`),
KEY `PARTITIONS_N49` (`TBL_ID`),
@@ -629,6 +631,8 @@ CREATE TABLE IF NOT EXISTS `TBLS` (
`VIEW_EXPANDED_TEXT` mediumtext,
`VIEW_ORIGINAL_TEXT` mediumtext,
`IS_REWRITE_ENABLED` bit(1) NOT NULL DEFAULT 0,
+ `TXN_ID` bigint(20) DEFAULT 0,
+ `WRITEID_LIST` text DEFAULT NULL,
PRIMARY KEY (`TBL_ID`),
UNIQUE KEY `UNIQUETABLE` (`TBL_NAME`,`DB_ID`),
KEY `TBLS_N50` (`SD_ID`),
@@ -726,6 +730,7 @@ CREATE TABLE IF NOT EXISTS `TAB_COL_STATS` (
`NUM_TRUES` bigint(20),
`NUM_FALSES` bigint(20),
`LAST_ANALYZED` bigint(20) NOT NULL,
+ `TXN_ID` bigint(20) DEFAULT 0,
PRIMARY KEY (`CS_ID`),
CONSTRAINT `TAB_COL_STATS_FK` FOREIGN KEY (`TBL_ID`) REFERENCES `TBLS` (`TBL_ID`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
@@ -757,6 +762,7 @@ CREATE TABLE IF NOT EXISTS `PART_COL_STATS` (
`NUM_TRUES` bigint(20),
`NUM_FALSES` bigint(20),
`LAST_ANALYZED` bigint(20) NOT NULL,
+ `TXN_ID` bigint(20) DEFAULT 0,
PRIMARY KEY (`CS_ID`),
CONSTRAINT `PART_COL_STATS_FK` FOREIGN KEY (`PART_ID`) REFERENCES `PARTITIONS` (`PART_ID`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
http://git-wip-us.apache.org/repos/asf/hive/blob/1d46608e/standalone-metastore/src/main/sql/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/sql/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql b/standalone-metastore/src/main/sql/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql
index 9b87563..5877c93 100644
--- a/standalone-metastore/src/main/sql/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql
+++ b/standalone-metastore/src/main/sql/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql
@@ -323,4 +323,4 @@ ALTER TABLE TXN_COMPONENTS MODIFY COLUMN TC_TABLE varchar(128) NULL;
UPDATE VERSION SET SCHEMA_VERSION='3.0.0', VERSION_COMMENT='Hive release version 3.0.0' where VER_ID=1;
SELECT 'Finished upgrading MetaStore schema from 2.3.0 to 3.0.0' AS ' ';
-ALTER TABLE `TBLS` ADD COLUMN `OWNER_TYPE` VARCHAR(10) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL;
\ No newline at end of file
+ALTER TABLE `TBLS` ADD COLUMN `OWNER_TYPE` VARCHAR(10) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL;
http://git-wip-us.apache.org/repos/asf/hive/blob/1d46608e/standalone-metastore/src/main/sql/mysql/upgrade-3.1.0-to-4.0.0.mysql.sql
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/sql/mysql/upgrade-3.1.0-to-4.0.0.mysql.sql b/standalone-metastore/src/main/sql/mysql/upgrade-3.1.0-to-4.0.0.mysql.sql
index b3789f9..4ca584c 100644
--- a/standalone-metastore/src/main/sql/mysql/upgrade-3.1.0-to-4.0.0.mysql.sql
+++ b/standalone-metastore/src/main/sql/mysql/upgrade-3.1.0-to-4.0.0.mysql.sql
@@ -1,5 +1,13 @@
SELECT 'Upgrading MetaStore schema from 3.1.0 to 4.0.0' AS ' ';
+-- HIVE-19416
+ALTER TABLE TBLS ADD TXN_ID bigint;
+ALTER TABLE TBLS ADD WRITEID_LIST CLOB;
+ALTER TABLE PARTITIONS ADD TXN_ID bigint;
+ALTER TABLE PARTITIONS ADD WRITEID_LIST CLOB;
+ALTER TABLE TAB_COL_STATS ADD TXN_ID bigint;
+ALTER TABLE PART_COL_STATS ADD TXN_ID bigint;
+
-- These lines need to be last. Insert any changes above.
UPDATE VERSION SET SCHEMA_VERSION='4.0.0', VERSION_COMMENT='Hive release version 4.0.0' where VER_ID=1;
SELECT 'Finished upgrading MetaStore schema from 3.1.0 to 4.0.0' AS ' ';
http://git-wip-us.apache.org/repos/asf/hive/blob/1d46608e/standalone-metastore/src/main/sql/oracle/hive-schema-3.0.0.oracle.sql
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/sql/oracle/hive-schema-3.0.0.oracle.sql b/standalone-metastore/src/main/sql/oracle/hive-schema-3.0.0.oracle.sql
index 63cc1f7..abdb984 100644
--- a/standalone-metastore/src/main/sql/oracle/hive-schema-3.0.0.oracle.sql
+++ b/standalone-metastore/src/main/sql/oracle/hive-schema-3.0.0.oracle.sql
@@ -162,7 +162,9 @@ CREATE TABLE PARTITIONS
LAST_ACCESS_TIME NUMBER (10) NOT NULL,
PART_NAME VARCHAR2(767) NULL,
SD_ID NUMBER NULL,
- TBL_ID NUMBER NULL
+ TBL_ID NUMBER NULL,
+ TXN_ID NUMBER NULL,
+ WRITEID_LIST CLOB NULL
);
ALTER TABLE PARTITIONS ADD CONSTRAINT PARTITIONS_PK PRIMARY KEY (PART_ID);
@@ -392,7 +394,9 @@ CREATE TABLE TBLS
TBL_TYPE VARCHAR2(128) NULL,
VIEW_EXPANDED_TEXT CLOB NULL,
VIEW_ORIGINAL_TEXT CLOB NULL,
- IS_REWRITE_ENABLED NUMBER(1) DEFAULT 0 NOT NULL CHECK (IS_REWRITE_ENABLED IN (1,0))
+ IS_REWRITE_ENABLED NUMBER(1) DEFAULT 0 NOT NULL CHECK (IS_REWRITE_ENABLED IN (1,0)),
+ TXN_ID NUMBER NULL,
+ WRITEID_LIST CLOB NULL
);
ALTER TABLE TBLS ADD CONSTRAINT TBLS_PK PRIMARY KEY (TBL_ID);
@@ -525,7 +529,8 @@ CREATE TABLE TAB_COL_STATS (
MAX_COL_LEN NUMBER,
NUM_TRUES NUMBER,
NUM_FALSES NUMBER,
- LAST_ANALYZED NUMBER NOT NULL
+ LAST_ANALYZED NUMBER NOT NULL,
+ TXN_ID NUMBER NULL
);
CREATE TABLE VERSION (
@@ -563,7 +568,8 @@ CREATE TABLE PART_COL_STATS (
MAX_COL_LEN NUMBER,
NUM_TRUES NUMBER,
NUM_FALSES NUMBER,
- LAST_ANALYZED NUMBER NOT NULL
+ LAST_ANALYZED NUMBER NOT NULL,
+ TXN_ID NUMBER NULL
);
ALTER TABLE PART_COL_STATS ADD CONSTRAINT PART_COL_STATS_PKEY PRIMARY KEY (CS_ID);
@@ -1134,7 +1140,6 @@ CREATE TABLE RUNTIME_STATS (
CREATE INDEX IDX_RUNTIME_STATS_CREATE_TIME ON RUNTIME_STATS(CREATE_TIME);
-
-- -----------------------------------------------------------------
-- Record schema version. Should be the last step in the init script
-- -----------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/1d46608e/standalone-metastore/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql b/standalone-metastore/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql
index e12150a..a143fd2 100644
--- a/standalone-metastore/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql
+++ b/standalone-metastore/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql
@@ -163,7 +163,9 @@ CREATE TABLE PARTITIONS
LAST_ACCESS_TIME NUMBER (10) NOT NULL,
PART_NAME VARCHAR2(767) NULL,
SD_ID NUMBER NULL,
- TBL_ID NUMBER NULL
+ TBL_ID NUMBER NULL,
+ TXN_ID NUMBER NULL,
+ WRITEID_LIST CLOB NULL
);
ALTER TABLE PARTITIONS ADD CONSTRAINT PARTITIONS_PK PRIMARY KEY (PART_ID);
@@ -398,7 +400,9 @@ CREATE TABLE TBLS
TBL_TYPE VARCHAR2(128) NULL,
VIEW_EXPANDED_TEXT CLOB NULL,
VIEW_ORIGINAL_TEXT CLOB NULL,
- IS_REWRITE_ENABLED NUMBER(1) DEFAULT 0 NOT NULL CHECK (IS_REWRITE_ENABLED IN (1,0))
+ IS_REWRITE_ENABLED NUMBER(1) DEFAULT 0 NOT NULL CHECK (IS_REWRITE_ENABLED IN (1,0)),
+ TXN_ID NUMBER NULL,
+ WRITEID_LIST CLOB NULL
);
ALTER TABLE TBLS ADD CONSTRAINT TBLS_PK PRIMARY KEY (TBL_ID);
@@ -531,7 +535,8 @@ CREATE TABLE TAB_COL_STATS (
MAX_COL_LEN NUMBER,
NUM_TRUES NUMBER,
NUM_FALSES NUMBER,
- LAST_ANALYZED NUMBER NOT NULL
+ LAST_ANALYZED NUMBER NOT NULL,
+ TXN_ID NUMBER NULL
);
ALTER TABLE TAB_COL_STATS ADD CONSTRAINT TAB_COL_STATS_PKEY PRIMARY KEY (CS_ID);
@@ -571,7 +576,8 @@ CREATE TABLE PART_COL_STATS (
MAX_COL_LEN NUMBER,
NUM_TRUES NUMBER,
NUM_FALSES NUMBER,
- LAST_ANALYZED NUMBER NOT NULL
+ LAST_ANALYZED NUMBER NOT NULL,
+ TXN_ID NUMBER NULL
);
ALTER TABLE PART_COL_STATS ADD CONSTRAINT PART_COL_STATS_PKEY PRIMARY KEY (CS_ID);
http://git-wip-us.apache.org/repos/asf/hive/blob/1d46608e/standalone-metastore/src/main/sql/oracle/upgrade-2.3.0-to-3.0.0.oracle.sql
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/sql/oracle/upgrade-2.3.0-to-3.0.0.oracle.sql b/standalone-metastore/src/main/sql/oracle/upgrade-2.3.0-to-3.0.0.oracle.sql
index ce3437f..5b767bc 100644
--- a/standalone-metastore/src/main/sql/oracle/upgrade-2.3.0-to-3.0.0.oracle.sql
+++ b/standalone-metastore/src/main/sql/oracle/upgrade-2.3.0-to-3.0.0.oracle.sql
@@ -339,4 +339,4 @@ UPDATE COMPLETED_TXN_COMPONENTS SET CTC_WRITEID = CTC_TXNID;
UPDATE VERSION SET SCHEMA_VERSION='3.0.0', VERSION_COMMENT='Hive release version 3.0.0' where VER_ID=1;
SELECT 'Finished upgrading MetaStore schema from 2.3.0 to 3.0.0' AS Status from dual;
-ALTER TABLE TBLS ADD OWNER_TYPE VARCHAR2(10) NULL;
\ No newline at end of file
+ALTER TABLE TBLS ADD OWNER_TYPE VARCHAR2(10) NULL;
http://git-wip-us.apache.org/repos/asf/hive/blob/1d46608e/standalone-metastore/src/main/sql/oracle/upgrade-3.1.0-to-4.0.0.oracle.sql
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/sql/oracle/upgrade-3.1.0-to-4.0.0.oracle.sql b/standalone-metastore/src/main/sql/oracle/upgrade-3.1.0-to-4.0.0.oracle.sql
index 6fa5e2d..7ac4d40 100644
--- a/standalone-metastore/src/main/sql/oracle/upgrade-3.1.0-to-4.0.0.oracle.sql
+++ b/standalone-metastore/src/main/sql/oracle/upgrade-3.1.0-to-4.0.0.oracle.sql
@@ -1,5 +1,12 @@
SELECT 'Upgrading MetaStore schema from 3.1.0 to 4.0.0' AS Status from dual;
+ALTER TABLE TBLS ADD TXN_ID number NULL;
+ALTER TABLE TBLS ADD WRITEID_LIST CLOB NULL;
+ALTER TABLE PARTITIONS ADD TXN_ID number NULL;
+ALTER TABLE PARTITIONS ADD WRITEID_LIST CLOB NULL;
+ALTER TABLE TAB_COL_STATS ADD TXN_ID number NULL;
+ALTER TABLE PART_COL_STATS ADD TXN_ID number NULL;
+
-- These lines need to be last. Insert any changes above.
UPDATE VERSION SET SCHEMA_VERSION='4.0.0', VERSION_COMMENT='Hive release version 4.0.0' where VER_ID=1;
SELECT 'Finished upgrading MetaStore schema from 3.1.0 to 4.0.0' AS Status from dual;
http://git-wip-us.apache.org/repos/asf/hive/blob/1d46608e/standalone-metastore/src/main/sql/postgres/hive-schema-3.0.0.postgres.sql
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/sql/postgres/hive-schema-3.0.0.postgres.sql b/standalone-metastore/src/main/sql/postgres/hive-schema-3.0.0.postgres.sql
index 97697f8..449f295 100644
--- a/standalone-metastore/src/main/sql/postgres/hive-schema-3.0.0.postgres.sql
+++ b/standalone-metastore/src/main/sql/postgres/hive-schema-3.0.0.postgres.sql
@@ -166,7 +166,9 @@ CREATE TABLE "PARTITIONS" (
"LAST_ACCESS_TIME" bigint NOT NULL,
"PART_NAME" character varying(767) DEFAULT NULL::character varying,
"SD_ID" bigint,
- "TBL_ID" bigint
+ "TBL_ID" bigint,
+ "TXN_ID" bigint,
+ "WRITEID_LIST" text
);
@@ -388,7 +390,9 @@ CREATE TABLE "TBLS" (
"TBL_TYPE" character varying(128) DEFAULT NULL::character varying,
"VIEW_EXPANDED_TEXT" text,
"VIEW_ORIGINAL_TEXT" text,
- "IS_REWRITE_ENABLED" boolean NOT NULL DEFAULT false
+ "IS_REWRITE_ENABLED" boolean NOT NULL DEFAULT false,
+ "TXN_ID" bigint,
+ "WRITEID_LIST" text
);
--
@@ -539,7 +543,8 @@ CREATE TABLE "TAB_COL_STATS" (
"MAX_COL_LEN" bigint,
"NUM_TRUES" bigint,
"NUM_FALSES" bigint,
- "LAST_ANALYZED" bigint NOT NULL
+ "LAST_ANALYZED" bigint NOT NULL,
+ "TXN_ID" bigint
);
--
@@ -577,7 +582,8 @@ CREATE TABLE "PART_COL_STATS" (
"MAX_COL_LEN" bigint,
"NUM_TRUES" bigint,
"NUM_FALSES" bigint,
- "LAST_ANALYZED" bigint NOT NULL
+ "LAST_ANALYZED" bigint NOT NULL,
+ "TXN_ID" bigint
);
--
@@ -1074,6 +1080,8 @@ ALTER TABLE ONLY "WM_MAPPING"
ALTER TABLE ONLY "WM_MAPPING"
ADD CONSTRAINT "UNIQUE_WM_MAPPING" UNIQUE ("RP_ID", "ENTITY_TYPE", "ENTITY_NAME");
+-- Transactional table stats PK constraints
+
--
-- Name: BUCKETING_COLS_N49; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace:
--
@@ -1618,6 +1626,8 @@ ALTER TABLE ONLY "MV_TABLES_USED"
ALTER TABLE ONLY "MV_TABLES_USED"
ADD CONSTRAINT "MV_TABLES_USED_FK2" FOREIGN KEY ("TBL_ID") REFERENCES "TBLS" ("TBL_ID") DEFERRABLE;
+-- Transactional table stats FK constraints
+
--
-- Name: public; Type: ACL; Schema: -; Owner: hiveuser
--
@@ -1822,7 +1832,6 @@ CREATE TABLE RUNTIME_STATS (
CREATE INDEX IDX_RUNTIME_STATS_CREATE_TIME ON RUNTIME_STATS(CREATE_TIME);
-
-- -----------------------------------------------------------------
-- Record schema version. Should be the last step in the init script
-- -----------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/1d46608e/standalone-metastore/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql b/standalone-metastore/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql
index b73e1d1..0ead590 100644
--- a/standalone-metastore/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql
+++ b/standalone-metastore/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql
@@ -168,7 +168,9 @@ CREATE TABLE "PARTITIONS" (
"LAST_ACCESS_TIME" bigint NOT NULL,
"PART_NAME" character varying(767) DEFAULT NULL::character varying,
"SD_ID" bigint,
- "TBL_ID" bigint
+ "TBL_ID" bigint,
+ "TXN_ID" bigint,
+ "WRITEID_LIST" text
);
@@ -392,7 +394,9 @@ CREATE TABLE "TBLS" (
"TBL_TYPE" character varying(128) DEFAULT NULL::character varying,
"VIEW_EXPANDED_TEXT" text,
"VIEW_ORIGINAL_TEXT" text,
- "IS_REWRITE_ENABLED" boolean NOT NULL DEFAULT false
+ "IS_REWRITE_ENABLED" boolean NOT NULL DEFAULT false,
+ "TXN_ID" bigint,
+ "WRITEID_LIST" text
);
--
@@ -545,7 +549,8 @@ CREATE TABLE "TAB_COL_STATS" (
"MAX_COL_LEN" bigint,
"NUM_TRUES" bigint,
"NUM_FALSES" bigint,
- "LAST_ANALYZED" bigint NOT NULL
+ "LAST_ANALYZED" bigint NOT NULL,
+ "TXN_ID" bigint
);
--
@@ -583,7 +588,8 @@ CREATE TABLE "PART_COL_STATS" (
"MAX_COL_LEN" bigint,
"NUM_TRUES" bigint,
"NUM_FALSES" bigint,
- "LAST_ANALYZED" bigint NOT NULL
+ "LAST_ANALYZED" bigint NOT NULL,
+ "TXN_ID" bigint
);
--
http://git-wip-us.apache.org/repos/asf/hive/blob/1d46608e/standalone-metastore/src/main/sql/postgres/upgrade-3.1.0-to-4.0.0.postgres.sql
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/sql/postgres/upgrade-3.1.0-to-4.0.0.postgres.sql b/standalone-metastore/src/main/sql/postgres/upgrade-3.1.0-to-4.0.0.postgres.sql
index 40d2795..f2bae02 100644
--- a/standalone-metastore/src/main/sql/postgres/upgrade-3.1.0-to-4.0.0.postgres.sql
+++ b/standalone-metastore/src/main/sql/postgres/upgrade-3.1.0-to-4.0.0.postgres.sql
@@ -1,5 +1,13 @@
SELECT 'Upgrading MetaStore schema from 3.1.0 to 4.0.0';
+-- HIVE-19416
+ALTER TABLE "TBLS" ADD "TXN_ID" bigint;
+ALTER TABLE "TBLS" ADD "WRITEID_LIST" text;
+ALTER TABLE "PARTITIONS" ADD "TXN_ID" bigint;
+ALTER TABLE "PARTITIONS" ADD "WRITEID_LIST" text;
+ALTER TABLE "TAB_COL_STATS" ADD "TXN_ID" bigint;
+ALTER TABLE "PART_COL_STATS" ADD "TXN_ID" bigint;
+
-- These lines need to be last. Insert any changes above.
UPDATE "VERSION" SET "SCHEMA_VERSION"='4.0.0', "VERSION_COMMENT"='Hive release version 4.0.0' where "VER_ID"=1;
SELECT 'Finished upgrading MetaStore schema from 3.1.0 to 4.0.0';
http://git-wip-us.apache.org/repos/asf/hive/blob/1d46608e/standalone-metastore/src/main/thrift/hive_metastore.thrift
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/thrift/hive_metastore.thrift b/standalone-metastore/src/main/thrift/hive_metastore.thrift
index 6e503eb..ecd2001 100644
--- a/standalone-metastore/src/main/thrift/hive_metastore.thrift
+++ b/standalone-metastore/src/main/thrift/hive_metastore.thrift
@@ -233,6 +233,12 @@ enum SchemaVersionState {
DELETED = 8
}
+enum IsolationLevelCompliance {
+ YES = 1,
+ NO = 2,
+ UNKNOWN = 3
+}
+
struct HiveObjectRef{
1: HiveObjectType objectType,
2: string dbName,
@@ -430,7 +436,10 @@ struct Table {
15: optional bool rewriteEnabled, // rewrite enabled or not
16: optional CreationMetadata creationMetadata, // only for MVs, it stores table names used and txn list at MV creation
17: optional string catName, // Name of the catalog the table is in
- 18: optional PrincipalType ownerType = PrincipalType.USER // owner type of this table (default to USER for backward compatibility)
+ 18: optional PrincipalType ownerType = PrincipalType.USER, // owner type of this table (default to USER for backward compatibility)
+ 19: optional i64 txnId=-1,
+ 20: optional string validWriteIdList,
+ 21: optional IsolationLevelCompliance isStatsCompliant
}
struct Partition {
@@ -442,7 +451,10 @@ struct Partition {
6: StorageDescriptor sd,
7: map<string, string> parameters,
8: optional PrincipalPrivilegeSet privileges,
- 9: optional string catName
+ 9: optional string catName,
+ 10: optional i64 txnId=-1,
+ 11: optional string validWriteIdList,
+ 12: optional IsolationLevelCompliance isStatsCompliant
}
struct PartitionWithoutSD {
@@ -469,7 +481,10 @@ struct PartitionSpec {
3: string rootPath,
4: optional PartitionSpecWithSharedSD sharedSDPartitionSpec,
5: optional PartitionListComposingSpec partitionList,
- 6: optional string catName
+ 6: optional string catName,
+ 7: optional i64 txnId=-1,
+ 8: optional string validWriteIdList,
+ 9: optional IsolationLevelCompliance isStatsCompliant
}
// column statistics
@@ -564,17 +579,24 @@ struct ColumnStatisticsDesc {
struct ColumnStatistics {
1: required ColumnStatisticsDesc statsDesc,
-2: required list<ColumnStatisticsObj> statsObj;
+2: required list<ColumnStatisticsObj> statsObj,
+3: optional i64 txnId=-1, // transaction id of the query that sends this structure
+4: optional string validWriteIdList, // valid write id list for the table for which this struct is being sent
+5: optional IsolationLevelCompliance isStatsCompliant // Are the stats isolation-level-compliant with the
+ // the calling query?
}
struct AggrStats {
1: required list<ColumnStatisticsObj> colStats,
-2: required i64 partsFound // number of partitions for which stats were found
+2: required i64 partsFound, // number of partitions for which stats were found
+3: optional IsolationLevelCompliance isStatsCompliant
}
struct SetPartitionsStatsRequest {
1: required list<ColumnStatistics> colStats,
-2: optional bool needMerge //stats need to be merged with the existing stats
+2: optional bool needMerge, //stats need to be merged with the existing stats
+3: optional i64 txnId=-1, // transaction id of the query that sends this structure
+4: optional string validWriteIdList // valid write id list for the table for which this struct is being sent
}
// schema of the table/query results etc.
@@ -703,18 +725,22 @@ struct PartitionsByExprRequest {
}
struct TableStatsResult {
- 1: required list<ColumnStatisticsObj> tableStats
+ 1: required list<ColumnStatisticsObj> tableStats,
+ 2: optional IsolationLevelCompliance isStatsCompliant
}
struct PartitionsStatsResult {
- 1: required map<string, list<ColumnStatisticsObj>> partStats
+ 1: required map<string, list<ColumnStatisticsObj>> partStats,
+ 2: optional IsolationLevelCompliance isStatsCompliant
}
struct TableStatsRequest {
1: required string dbName,
2: required string tblName,
3: required list<string> colNames
- 4: optional string catName
+ 4: optional string catName,
+ 5: optional i64 txnId=-1, // transaction id of the query that sends this structure
+ 6: optional string validWriteIdList // valid write id list for the table for which this struct is being sent
}
struct PartitionsStatsRequest {
@@ -722,12 +748,15 @@ struct PartitionsStatsRequest {
2: required string tblName,
3: required list<string> colNames,
4: required list<string> partNames,
- 5: optional string catName
+ 5: optional string catName,
+ 6: optional i64 txnId=-1, // transaction id of the query that sends this structure
+ 7: optional string validWriteIdList // valid write id list for the table for which this struct is being sent
}
// Return type for add_partitions_req
struct AddPartitionsResult {
1: optional list<Partition> partitions,
+ 2: optional IsolationLevelCompliance isStatsCompliant
}
// Request type for add_partitions_req
@@ -737,7 +766,9 @@ struct AddPartitionsRequest {
3: required list<Partition> parts,
4: required bool ifNotExists,
5: optional bool needResult=true,
- 6: optional string catName
+ 6: optional string catName,
+ 7: optional i64 txnId=-1,
+ 8: optional string validWriteIdList
}
// Return type for drop_partitions_req
@@ -1209,11 +1240,14 @@ struct GetTableRequest {
1: required string dbName,
2: required string tblName,
3: optional ClientCapabilities capabilities,
- 4: optional string catName
+ 4: optional string catName,
+ 5: optional i64 txnId=-1,
+ 6: optional string validWriteIdList
}
struct GetTableResult {
- 1: required Table table
+ 1: required Table table,
+ 2: optional IsolationLevelCompliance isStatsCompliant
}
struct GetTablesRequest {
@@ -1544,6 +1578,18 @@ struct GetRuntimeStatsRequest {
2: required i32 maxCreateTime
}
+struct AlterPartitionsRequest {
+ 1: required string dbName,
+ 2: required string tableName,
+ 3: required list<Partition> partitions,
+ 4: required EnvironmentContext environmentContext,
+ 5: optional i64 txnId=-1,
+ 6: optional string validWriteIdList
+}
+
+struct AlterPartitionsResponse {
+}
+
// Exceptions.
exception MetaException {
@@ -1874,7 +1920,9 @@ service ThriftHiveMetastore extends fb303.FacebookService
// prehooks are fired together followed by all post hooks
void alter_partitions(1:string db_name, 2:string tbl_name, 3:list<Partition> new_parts)
throws (1:InvalidOperationException o1, 2:MetaException o2)
- void alter_partitions_with_environment_context(1:string db_name, 2:string tbl_name, 3:list<Partition> new_parts, 4:EnvironmentContext environment_context) throws (1:InvalidOperationException o1, 2:MetaException o2)
+
+ AlterPartitionsResponse alter_partitions_with_environment_context(1:AlterPartitionsRequest req)
+ throws (1:InvalidOperationException o1, 2:MetaException o2)
void alter_partition_with_environment_context(1:string db_name,
2:string tbl_name, 3:Partition new_part,
http://git-wip-us.apache.org/repos/asf/hive/blob/1d46608e/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java
index 7c7429d..6985736 100644
--- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java
+++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java
@@ -19,11 +19,7 @@
package org.apache.hadoop.hive.metastore;
import org.apache.hadoop.hive.common.TableName;
-import org.apache.hadoop.hive.metastore.api.CreationMetadata;
-import org.apache.hadoop.hive.metastore.api.ISchemaName;
-import org.apache.hadoop.hive.metastore.api.SchemaVersionDescriptor;
-import org.apache.hadoop.hive.metastore.api.Catalog;
-import org.apache.hadoop.hive.metastore.api.WMFullResourcePlan;
+import org.apache.hadoop.hive.metastore.api.*;
import java.nio.ByteBuffer;
import java.util.ArrayList;
@@ -33,58 +29,6 @@ import java.util.Map;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.metastore.api.AggrStats;
-import org.apache.hadoop.hive.metastore.api.AlreadyExistsException;
-import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
-import org.apache.hadoop.hive.metastore.api.CurrentNotificationEventId;
-import org.apache.hadoop.hive.metastore.api.Database;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.FileMetadataExprType;
-import org.apache.hadoop.hive.metastore.api.Function;
-import org.apache.hadoop.hive.metastore.api.HiveObjectPrivilege;
-import org.apache.hadoop.hive.metastore.api.HiveObjectRef;
-import org.apache.hadoop.hive.metastore.api.ISchema;
-import org.apache.hadoop.hive.metastore.api.InvalidInputException;
-import org.apache.hadoop.hive.metastore.api.InvalidObjectException;
-import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
-import org.apache.hadoop.hive.metastore.api.InvalidPartitionException;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
-import org.apache.hadoop.hive.metastore.api.NotificationEvent;
-import org.apache.hadoop.hive.metastore.api.NotificationEventRequest;
-import org.apache.hadoop.hive.metastore.api.NotificationEventResponse;
-import org.apache.hadoop.hive.metastore.api.NotificationEventsCountRequest;
-import org.apache.hadoop.hive.metastore.api.NotificationEventsCountResponse;
-import org.apache.hadoop.hive.metastore.api.Partition;
-import org.apache.hadoop.hive.metastore.api.PartitionEventType;
-import org.apache.hadoop.hive.metastore.api.PartitionValuesResponse;
-import org.apache.hadoop.hive.metastore.api.PrincipalPrivilegeSet;
-import org.apache.hadoop.hive.metastore.api.PrincipalType;
-import org.apache.hadoop.hive.metastore.api.PrivilegeBag;
-import org.apache.hadoop.hive.metastore.api.WMNullablePool;
-import org.apache.hadoop.hive.metastore.api.WMNullableResourcePlan;
-import org.apache.hadoop.hive.metastore.api.WMResourcePlan;
-import org.apache.hadoop.hive.metastore.api.WMTrigger;
-import org.apache.hadoop.hive.metastore.api.WMValidateResourcePlanResponse;
-import org.apache.hadoop.hive.metastore.api.Role;
-import org.apache.hadoop.hive.metastore.api.RolePrincipalGrant;
-import org.apache.hadoop.hive.metastore.api.RuntimeStat;
-import org.apache.hadoop.hive.metastore.api.SQLCheckConstraint;
-import org.apache.hadoop.hive.metastore.api.SQLDefaultConstraint;
-import org.apache.hadoop.hive.metastore.api.SQLForeignKey;
-import org.apache.hadoop.hive.metastore.api.SQLNotNullConstraint;
-import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey;
-import org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint;
-import org.apache.hadoop.hive.metastore.api.SchemaVersion;
-import org.apache.hadoop.hive.metastore.api.SerDeInfo;
-import org.apache.hadoop.hive.metastore.api.Table;
-import org.apache.hadoop.hive.metastore.api.TableMeta;
-import org.apache.hadoop.hive.metastore.api.Type;
-import org.apache.hadoop.hive.metastore.api.UnknownDBException;
-import org.apache.hadoop.hive.metastore.api.UnknownPartitionException;
-import org.apache.hadoop.hive.metastore.api.UnknownTableException;
-import org.apache.hadoop.hive.metastore.api.WMMapping;
-import org.apache.hadoop.hive.metastore.api.WMPool;
import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy;
import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.ColStatsObjWithSourceInfo;
import org.apache.thrift.TException;
@@ -247,6 +191,12 @@ public class DummyRawStoreControlledCommit implements RawStore, Configurable {
}
@Override
+ public Table getTable(String catName, String dbName, String tableName, long txnId, String writeIdList)
+ throws MetaException {
+ return objectStore.getTable(catName, dbName, tableName, txnId, writeIdList);
+ }
+
+ @Override
public boolean addPartition(Partition part)
throws InvalidObjectException, MetaException {
return objectStore.addPartition(part);
@@ -259,6 +209,13 @@ public class DummyRawStoreControlledCommit implements RawStore, Configurable {
}
@Override
+ public Partition getPartition(String catName, String dbName, String tableName,
+ List<String> partVals, long txnId, String writeIdList)
+ throws MetaException, NoSuchObjectException {
+ return objectStore.getPartition(catName, dbName, tableName, partVals, txnId, writeIdList);
+ }
+
+ @Override
public boolean dropPartition(String catName, String dbName, String tableName, List<String> partVals)
throws MetaException, NoSuchObjectException,
InvalidObjectException, InvalidInputException {
@@ -343,9 +300,11 @@ public class DummyRawStoreControlledCommit implements RawStore, Configurable {
@Override
public void alterPartitions(String catName, String dbName, String tblName,
- List<List<String>> partValsList, List<Partition> newParts)
+ List<List<String>> partValsList, List<Partition> newParts,
+ long txnId, String writeIdList)
throws InvalidObjectException, MetaException {
- objectStore.alterPartitions(catName, dbName, tblName, partValsList, newParts);
+ objectStore.alterPartitions(
+ catName, dbName, tblName, partValsList, newParts, txnId, writeIdList);
}
@Override
@@ -647,6 +606,15 @@ public class DummyRawStoreControlledCommit implements RawStore, Configurable {
}
@Override
+ public ColumnStatistics getTableColumnStatistics(String catName, String dbName,
+ String tableName, List<String> colNames,
+ long txnId, String writeIdList)
+ throws MetaException, NoSuchObjectException {
+ return objectStore.getTableColumnStatistics(
+ catName, dbName, tableName, colNames, txnId, writeIdList);
+ }
+
+ @Override
public boolean deleteTableColumnStatistics(String catName, String dbName, String tableName,
String colName)
throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException {
@@ -739,6 +707,15 @@ public class DummyRawStoreControlledCommit implements RawStore, Configurable {
}
@Override
+ public List<ColumnStatistics> getPartitionColumnStatistics(
+ String catName, String dbName, String tblName, List<String> partNames,
+ List<String> colNames, long txnId, String writeIdList)
+ throws MetaException, NoSuchObjectException {
+ return objectStore.getPartitionColumnStatistics(
+ catName, dbName, tblName , colNames, partNames, txnId, writeIdList);
+ }
+
+ @Override
public boolean doesPartitionExist(String catName, String dbName, String tableName,
List<FieldSchema> partKeys, List<String> partVals)
throws MetaException, NoSuchObjectException {
@@ -807,6 +784,15 @@ public class DummyRawStoreControlledCommit implements RawStore, Configurable {
}
@Override
+ public AggrStats get_aggr_stats_for(String catName, String dbName,
+ String tblName, List<String> partNames,
+ List<String> colNames,
+ long txnId, String writeIdList)
+ throws MetaException, NoSuchObjectException {
+ return null;
+ }
+
+ @Override
public NotificationEventResponse getNextNotification(NotificationEventRequest rqst) {
return objectStore.getNextNotification(rqst);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/1d46608e/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
index e4f2a17..37e9920 100644
--- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
+++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
@@ -19,11 +19,7 @@
package org.apache.hadoop.hive.metastore;
import org.apache.hadoop.hive.common.TableName;
-import org.apache.hadoop.hive.metastore.api.CreationMetadata;
-import org.apache.hadoop.hive.metastore.api.ISchemaName;
-import org.apache.hadoop.hive.metastore.api.SchemaVersionDescriptor;
-import org.apache.hadoop.hive.metastore.api.Catalog;
-import org.apache.hadoop.hive.metastore.api.WMFullResourcePlan;
+import org.apache.hadoop.hive.metastore.api.*;
import java.nio.ByteBuffer;
import java.util.Collections;
@@ -31,58 +27,6 @@ import java.util.List;
import java.util.Map;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.metastore.api.AggrStats;
-import org.apache.hadoop.hive.metastore.api.AlreadyExistsException;
-import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
-import org.apache.hadoop.hive.metastore.api.CurrentNotificationEventId;
-import org.apache.hadoop.hive.metastore.api.Database;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.FileMetadataExprType;
-import org.apache.hadoop.hive.metastore.api.Function;
-import org.apache.hadoop.hive.metastore.api.HiveObjectPrivilege;
-import org.apache.hadoop.hive.metastore.api.HiveObjectRef;
-import org.apache.hadoop.hive.metastore.api.ISchema;
-import org.apache.hadoop.hive.metastore.api.InvalidInputException;
-import org.apache.hadoop.hive.metastore.api.InvalidObjectException;
-import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
-import org.apache.hadoop.hive.metastore.api.InvalidPartitionException;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
-import org.apache.hadoop.hive.metastore.api.NotificationEvent;
-import org.apache.hadoop.hive.metastore.api.NotificationEventRequest;
-import org.apache.hadoop.hive.metastore.api.NotificationEventResponse;
-import org.apache.hadoop.hive.metastore.api.NotificationEventsCountRequest;
-import org.apache.hadoop.hive.metastore.api.NotificationEventsCountResponse;
-import org.apache.hadoop.hive.metastore.api.Partition;
-import org.apache.hadoop.hive.metastore.api.PartitionEventType;
-import org.apache.hadoop.hive.metastore.api.PartitionValuesResponse;
-import org.apache.hadoop.hive.metastore.api.PrincipalPrivilegeSet;
-import org.apache.hadoop.hive.metastore.api.PrincipalType;
-import org.apache.hadoop.hive.metastore.api.PrivilegeBag;
-import org.apache.hadoop.hive.metastore.api.WMNullablePool;
-import org.apache.hadoop.hive.metastore.api.WMNullableResourcePlan;
-import org.apache.hadoop.hive.metastore.api.WMResourcePlan;
-import org.apache.hadoop.hive.metastore.api.WMTrigger;
-import org.apache.hadoop.hive.metastore.api.WMValidateResourcePlanResponse;
-import org.apache.hadoop.hive.metastore.api.Role;
-import org.apache.hadoop.hive.metastore.api.RolePrincipalGrant;
-import org.apache.hadoop.hive.metastore.api.RuntimeStat;
-import org.apache.hadoop.hive.metastore.api.SQLCheckConstraint;
-import org.apache.hadoop.hive.metastore.api.SQLDefaultConstraint;
-import org.apache.hadoop.hive.metastore.api.SQLForeignKey;
-import org.apache.hadoop.hive.metastore.api.SQLNotNullConstraint;
-import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey;
-import org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint;
-import org.apache.hadoop.hive.metastore.api.SchemaVersion;
-import org.apache.hadoop.hive.metastore.api.SerDeInfo;
-import org.apache.hadoop.hive.metastore.api.Table;
-import org.apache.hadoop.hive.metastore.api.TableMeta;
-import org.apache.hadoop.hive.metastore.api.Type;
-import org.apache.hadoop.hive.metastore.api.UnknownDBException;
-import org.apache.hadoop.hive.metastore.api.UnknownPartitionException;
-import org.apache.hadoop.hive.metastore.api.UnknownTableException;
-import org.apache.hadoop.hive.metastore.api.WMMapping;
-import org.apache.hadoop.hive.metastore.api.WMPool;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy;
import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
@@ -243,6 +187,12 @@ public class DummyRawStoreForJdoConnection implements RawStore {
}
@Override
+ public Table getTable(String catalogName, String dbName, String tableName,
+ long txnid, String writeIdList) throws MetaException {
+ return null;
+ }
+
+ @Override
public boolean addPartition(Partition part) throws InvalidObjectException, MetaException {
return false;
@@ -256,6 +206,13 @@ public class DummyRawStoreForJdoConnection implements RawStore {
}
@Override
+ public Partition getPartition(String catName, String dbName, String tableName, List<String> part_vals,
+ long txnid, String writeIdList)
+ throws MetaException, NoSuchObjectException {
+ return null;
+ }
+
+ @Override
public boolean dropPartition(String catName, String dbName, String tableName, List<String> part_vals)
throws MetaException {
@@ -344,10 +301,10 @@ public class DummyRawStoreForJdoConnection implements RawStore {
@Override
public void alterPartitions(String catName, String db_name, String tbl_name,
- List<List<String>> part_vals_list, List<Partition> new_parts)
+ List<List<String>> part_vals_list, List<Partition> new_parts,
+ long txnId, String writeIdList)
throws InvalidObjectException, MetaException {
-
}
@Override
@@ -700,6 +657,14 @@ public class DummyRawStoreForJdoConnection implements RawStore {
}
@Override
+ public ColumnStatistics getTableColumnStatistics(
+ String catName, String dbName, String tableName, List<String> colName,
+ long txnid, String writeIdList)
+ throws MetaException, NoSuchObjectException {
+ return null;
+ }
+
+ @Override
public boolean deleteTableColumnStatistics(String catName, String dbName, String tableName,
String colName)
throws NoSuchObjectException, MetaException, InvalidObjectException {
@@ -749,6 +714,14 @@ public class DummyRawStoreForJdoConnection implements RawStore {
}
@Override
+ public List<ColumnStatistics> getPartitionColumnStatistics(
+ String catName, String dbName, String tblName, List<String> partNames,
+ List<String> colNames, long txnid, String writeIdList)
+ throws MetaException, NoSuchObjectException {
+ return Collections.emptyList();
+ }
+
+ @Override
public boolean doesPartitionExist(String catName, String dbName, String tableName,
List<FieldSchema> partKeys, List<String> partVals)
throws MetaException, NoSuchObjectException {
@@ -812,6 +785,14 @@ public class DummyRawStoreForJdoConnection implements RawStore {
}
@Override
+ public AggrStats get_aggr_stats_for(
+ String catName, String dbName, String tblName, List<String> partNames,
+ List<String> colNames, long txnid, String writeIdList)
+ throws MetaException, NoSuchObjectException {
+ return null;
+ }
+
+ @Override
public NotificationEventResponse getNextNotification(NotificationEventRequest rqst) {
return null;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/1d46608e/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java
index 2d87a2f..3899f03 100644
--- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java
+++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hive.metastore;
import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_DATABASE_NAME;
import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.getDefaultCatalog;
+import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.prependCatalogToDbName;
import java.io.IOException;
import java.lang.reflect.Constructor;
@@ -1429,6 +1430,17 @@ public class HiveMetaStoreClientPreCatalog implements IMetaStoreClient, AutoClos
return fastpath ? t : deepCopy(filterHook.filterTable(t));
}
+ @Override
+ public Table getTable(String dbName, String tableName, long txnId, String validWriteIdList)
+ throws MetaException, TException, NoSuchObjectException {
+ GetTableRequest req = new GetTableRequest(dbName, tableName);
+ req.setCapabilities(version);
+ req.setTxnId(txnId);
+ req.setValidWriteIdList(validWriteIdList);
+ Table t = client.get_table_req(req).getTable();
+ return fastpath ? t : deepCopy(filterHook.filterTable(t));
+ }
+
/** {@inheritDoc} */
@Override
public List<Table> getTableObjectsByName(String dbName, List<String> tableNames)
@@ -1612,13 +1624,33 @@ public class HiveMetaStoreClientPreCatalog implements IMetaStoreClient, AutoClos
@Override
public void alter_partitions(String dbName, String tblName, List<Partition> newParts)
throws InvalidOperationException, MetaException, TException {
- client.alter_partitions_with_environment_context(dbName, tblName, newParts, null);
+ client.alter_partitions(dbName, tblName, newParts);
}
@Override
public void alter_partitions(String dbName, String tblName, List<Partition> newParts, EnvironmentContext environmentContext)
throws InvalidOperationException, MetaException, TException {
- client.alter_partitions_with_environment_context(dbName, tblName, newParts, environmentContext);
+ AlterPartitionsRequest req = new AlterPartitionsRequest();
+ req.setDbName(dbName);
+ req.setTableName(tblName);
+ req.setPartitions(newParts);
+ req.setEnvironmentContext(environmentContext);
+ client.alter_partitions_with_environment_context(req);
+ }
+
+ @Override
+ public void alter_partitions(String dbName, String tblName, List<Partition> newParts,
+ EnvironmentContext environmentContext,
+ long txnId, String writeIdList)
+ throws InvalidOperationException, MetaException, TException {
+ AlterPartitionsRequest req = new AlterPartitionsRequest();
+ req.setDbName(dbName);
+ req.setTableName(tblName);
+ req.setPartitions(newParts);
+ req.setEnvironmentContext(environmentContext);
+ req.setTxnId(txnId);
+ req.setValidWriteIdList(writeIdList);
+ client.alter_partitions_with_environment_context(req);
}
@Override
@@ -1727,6 +1759,17 @@ public class HiveMetaStoreClientPreCatalog implements IMetaStoreClient, AutoClos
new TableStatsRequest(dbName, tableName, colNames)).getTableStats();
}
+ @Override
+ public List<ColumnStatisticsObj> getTableColumnStatistics(
+ String dbName, String tableName, List<String> colNames, long txnId, String validWriteIdList)
+ throws NoSuchObjectException, MetaException, TException {
+ TableStatsRequest tsr = new TableStatsRequest(dbName, tableName, colNames);
+ tsr.setTxnId(txnId);
+ tsr.setValidWriteIdList(validWriteIdList);
+
+ return client.get_table_statistics_req(tsr).getTableStats();
+ }
+
/** {@inheritDoc} */
@Override
public Map<String, List<ColumnStatisticsObj>> getPartitionColumnStatistics(
@@ -1736,6 +1779,18 @@ public class HiveMetaStoreClientPreCatalog implements IMetaStoreClient, AutoClos
new PartitionsStatsRequest(dbName, tableName, colNames, partNames)).getPartStats();
}
+ @Override
+ public Map<String, List<ColumnStatisticsObj>> getPartitionColumnStatistics(
+ String dbName, String tableName, List<String> partNames,
+ List<String> colNames, long txnId, String validWriteIdList)
+ throws NoSuchObjectException, MetaException, TException {
+ PartitionsStatsRequest psr = new PartitionsStatsRequest(dbName, tableName, colNames, partNames);
+ psr.setTxnId(txnId);
+ psr.setValidWriteIdList(validWriteIdList);
+ return client.get_partitions_statistics_req(
+ psr).getPartStats();
+ }
+
/** {@inheritDoc} */
@Override
public boolean deletePartitionColumnStatistics(String dbName, String tableName, String partName,
@@ -2593,6 +2648,21 @@ public class HiveMetaStoreClientPreCatalog implements IMetaStoreClient, AutoClos
}
@Override
+ public AggrStats getAggrColStatsFor(
+ String dbName, String tblName, List<String> colNames,
+ List<String> partName, long txnId, String writeIdList)
+ throws NoSuchObjectException, MetaException, TException {
+ if (colNames.isEmpty() || partName.isEmpty()) {
+ LOG.debug("Columns is empty or partNames is empty : Short-circuiting stats eval on client side.");
+ return new AggrStats(new ArrayList<>(),0); // Nothing to aggregate
+ }
+ PartitionsStatsRequest req = new PartitionsStatsRequest(dbName, tblName, colNames, partName);
+ req.setTxnId(txnId);
+ req.setValidWriteIdList(writeIdList);
+ return client.get_aggr_stats_for(req);
+ }
+
+ @Override
public Iterable<Entry<Long, ByteBuffer>> getFileMetadata(
final List<Long> fileIds) throws TException {
return new MetastoreMapIterable<Long, ByteBuffer>() {
@@ -3000,6 +3070,12 @@ public class HiveMetaStoreClientPreCatalog implements IMetaStoreClient, AutoClos
}
@Override
+ public Table getTable(String catName, String dbName, String tableName,
+ long txnId, String validWriteIdList) throws TException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
public List<Table> getTableObjectsByName(String catName, String dbName,
List<String> tableNames) throws MetaException,
InvalidOperationException, UnknownDBException, TException {
@@ -3226,7 +3302,8 @@ public class HiveMetaStoreClientPreCatalog implements IMetaStoreClient, AutoClos
@Override
public void alter_partitions(String catName, String dbName, String tblName,
List<Partition> newParts,
- EnvironmentContext environmentContext) throws
+ EnvironmentContext environmentContext,
+ long txnId, String writeIdList) throws
InvalidOperationException, MetaException, TException {
throw new UnsupportedOperationException();
}
@@ -3259,6 +3336,14 @@ public class HiveMetaStoreClientPreCatalog implements IMetaStoreClient, AutoClos
}
@Override
+ public List<ColumnStatisticsObj> getTableColumnStatistics(
+ String catName, String dbName, String tableName, List<String> colNames,
+ long txnId, String validWriteIdList)
+ throws NoSuchObjectException, MetaException, TException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
public Map<String, List<ColumnStatisticsObj>> getPartitionColumnStatistics(String catName,
String dbName,
String tableName,
@@ -3269,6 +3354,14 @@ public class HiveMetaStoreClientPreCatalog implements IMetaStoreClient, AutoClos
}
@Override
+ public Map<String, List<ColumnStatisticsObj>> getPartitionColumnStatistics(
+ String catName, String dbName, String tableName, List<String> partNames,
+ List<String> colNames, long txnId, String validWriteIdList)
+ throws NoSuchObjectException, MetaException, TException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
public boolean deletePartitionColumnStatistics(String catName, String dbName, String tableName,
String partName, String colName) throws
NoSuchObjectException, MetaException, InvalidObjectException, TException,
@@ -3316,6 +3409,14 @@ public class HiveMetaStoreClientPreCatalog implements IMetaStoreClient, AutoClos
}
@Override
+ public AggrStats getAggrColStatsFor(String catName, String dbName, String tblName,
+ List<String> colNames, List<String> partNames,
+ long txnId, String writeIdList)
+ throws NoSuchObjectException, MetaException, TException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
public void dropConstraint(String catName, String dbName, String tableName,
String constraintName) throws MetaException, NoSuchObjectException,
TException {
@@ -3420,4 +3521,5 @@ public class HiveMetaStoreClientPreCatalog implements IMetaStoreClient, AutoClos
public List<RuntimeStat> getRuntimeStats(int maxWeight, int maxCreateTime) throws TException {
throw new UnsupportedOperationException();
}
+
}
http://git-wip-us.apache.org/repos/asf/hive/blob/1d46608e/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestAlterPartitions.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestAlterPartitions.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestAlterPartitions.java
index 54bf3d7..f19b505 100644
--- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestAlterPartitions.java
+++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestAlterPartitions.java
@@ -835,7 +835,8 @@ public class TestAlterPartitions extends MetaStoreClientTest {
public void testAlterPartitionsWithEnvironmentCtxBogusCatalogName() throws Exception {
createTable4PartColsParts(client);
Partition part = client.listPartitions(DB_NAME, TABLE_NAME, (short)-1).get(0);
- client.alter_partitions("nosuch", DB_NAME, TABLE_NAME, Lists.newArrayList(part), new EnvironmentContext());
+ client.alter_partitions("nosuch", DB_NAME, TABLE_NAME, Lists.newArrayList(part), new EnvironmentContext(),
+ -1, null);
}
@Test(expected = InvalidOperationException.class)
http://git-wip-us.apache.org/repos/asf/hive/blob/1d46608e/storage-api/src/java/org/apache/hadoop/hive/common/ValidTxnWriteIdList.java
----------------------------------------------------------------------
diff --git a/storage-api/src/java/org/apache/hadoop/hive/common/ValidTxnWriteIdList.java b/storage-api/src/java/org/apache/hadoop/hive/common/ValidTxnWriteIdList.java
index 9867a81..cfe01fe 100644
--- a/storage-api/src/java/org/apache/hadoop/hive/common/ValidTxnWriteIdList.java
+++ b/storage-api/src/java/org/apache/hadoop/hive/common/ValidTxnWriteIdList.java
@@ -63,6 +63,10 @@ public class ValidTxnWriteIdList {
return null;
}
+ public boolean isEmpty() {
+ return tablesValidWriteIdList.isEmpty();
+ }
+
// Each ValidWriteIdList is separated with "$" and each one maps to one table
// Format <txnId>$<table_name>:<hwm>:<minOpenWriteId>:<open_writeids>:<abort_writeids>$<table_name>...
private void readFromString(String src) {
http://git-wip-us.apache.org/repos/asf/hive/blob/1d46608e/storage-api/src/java/org/apache/hive/common/util/TxnIdUtils.java
----------------------------------------------------------------------
diff --git a/storage-api/src/java/org/apache/hive/common/util/TxnIdUtils.java b/storage-api/src/java/org/apache/hive/common/util/TxnIdUtils.java
index 17f3777..dc50f1b 100644
--- a/storage-api/src/java/org/apache/hive/common/util/TxnIdUtils.java
+++ b/storage-api/src/java/org/apache/hive/common/util/TxnIdUtils.java
@@ -36,8 +36,24 @@ public class TxnIdUtils {
}
return checkEquivalentCommittedIds(
- older.getHighWatermark(), older.getInvalidWriteIds(),
- newer.getHighWatermark(), newer.getInvalidWriteIds());
+ older.getHighWatermark(), older.getInvalidWriteIds(),
+ newer.getHighWatermark(), newer.getInvalidWriteIds());
+ }
+
+ /**
+ * Check if the give two write id lists are for concurrent writes
+ * on the table.
+ */
+ public static boolean areTheseConcurrentWrites(
+ ValidWriteIdList older, ValidWriteIdList newer, long statsWriteId) {
+ if (!older.getTableName().equalsIgnoreCase(newer.getTableName())) {
+ return false;
+ }
+
+ assert(older.getHighWatermark() <= newer.getHighWatermark());
+
+ // TODO: Just return false for now.
+ return false;
}
/**