You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by th...@apache.org on 2013/09/26 04:05:49 UTC
svn commit: r1526348 - in /hive/branches/branch-0.12: ./
common/src/java/org/apache/hadoop/hive/conf/ conf/
metastore/scripts/upgrade/derby/ metastore/scripts/upgrade/mysql/
metastore/scripts/upgrade/oracle/ metastore/scripts/upgrade/postgres/
metastor...
Author: thejas
Date: Thu Sep 26 02:05:48 2013
New Revision: 1526348
URL: http://svn.apache.org/r1526348
Log:
HIVE-3764 : Support metastore version consistency check (Prasad Mujumdar via Ashutosh Chauhan)
Added:
hive/branches/branch-0.12/metastore/scripts/upgrade/derby/014-HIVE-3764.derby.sql
hive/branches/branch-0.12/metastore/scripts/upgrade/derby/upgrade.order.derby
hive/branches/branch-0.12/metastore/scripts/upgrade/mysql/014-HIVE-3764.mysql.sql
hive/branches/branch-0.12/metastore/scripts/upgrade/mysql/upgrade.order.mysql
hive/branches/branch-0.12/metastore/scripts/upgrade/oracle/014-HIVE-3764.oracle.sql
hive/branches/branch-0.12/metastore/scripts/upgrade/oracle/upgrade-0.10.0-to-0.11.0.mysql.sql
hive/branches/branch-0.12/metastore/scripts/upgrade/oracle/upgrade.order.oracle
hive/branches/branch-0.12/metastore/scripts/upgrade/postgres/014-HIVE-3764.postgres.sql
hive/branches/branch-0.12/metastore/scripts/upgrade/postgres/upgrade.order.postgres
hive/branches/branch-0.12/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java
hive/branches/branch-0.12/metastore/src/model/org/apache/hadoop/hive/metastore/model/MVersionTable.java
hive/branches/branch-0.12/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java
Modified:
hive/branches/branch-0.12/build-common.xml
hive/branches/branch-0.12/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
hive/branches/branch-0.12/conf/hive-default.xml.template
hive/branches/branch-0.12/metastore/scripts/upgrade/derby/hive-schema-0.12.0.derby.sql
hive/branches/branch-0.12/metastore/scripts/upgrade/derby/upgrade-0.10.0-to-0.11.0.derby.sql
hive/branches/branch-0.12/metastore/scripts/upgrade/derby/upgrade-0.11.0-to-0.12.0.derby.sql
hive/branches/branch-0.12/metastore/scripts/upgrade/mysql/hive-schema-0.12.0.mysql.sql
hive/branches/branch-0.12/metastore/scripts/upgrade/mysql/upgrade-0.11.0-to-0.12.0.mysql.sql
hive/branches/branch-0.12/metastore/scripts/upgrade/oracle/hive-schema-0.12.0.oracle.sql
hive/branches/branch-0.12/metastore/scripts/upgrade/oracle/upgrade-0.11.0-to-0.12.0.oracle.sql
hive/branches/branch-0.12/metastore/scripts/upgrade/postgres/hive-schema-0.12.0.postgres.sql
hive/branches/branch-0.12/metastore/scripts/upgrade/postgres/upgrade-0.11.0-to-0.12.0.postgres.sql
hive/branches/branch-0.12/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
hive/branches/branch-0.12/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
hive/branches/branch-0.12/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java
hive/branches/branch-0.12/metastore/src/model/package.jdo
hive/branches/branch-0.12/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java
hive/branches/branch-0.12/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
Modified: hive/branches/branch-0.12/build-common.xml
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/build-common.xml?rev=1526348&r1=1526347&r2=1526348&view=diff
==============================================================================
--- hive/branches/branch-0.12/build-common.xml (original)
+++ hive/branches/branch-0.12/build-common.xml Thu Sep 26 02:05:48 2013
@@ -475,6 +475,7 @@
<sysproperty key="java.net.preferIPv4Stack" value="${java.net.preferIPv4Stack}"/>
<sysproperty key="hadoop.bin.path" value="${test.hadoop.bin.path}${junit.script.extension}"/>
<sysproperty key="test.concurrency.num.threads" value="${test.concurrency.num.threads}"/>
+ <sysproperty key="hive.home" value="${hive.root}/build/dist"/>
<jvmarg line="${junit.jvm.args}"/>
<classpath refid="test.local.classpath"/>
Modified: hive/branches/branch-0.12/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1526348&r1=1526347&r2=1526348&view=diff
==============================================================================
--- hive/branches/branch-0.12/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hive/branches/branch-0.12/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Thu Sep 26 02:05:48 2013
@@ -323,6 +323,8 @@ public class HiveConf extends Configurat
METASTORE_VALIDATE_CONSTRAINTS("datanucleus.validateConstraints", false),
METASTORE_STORE_MANAGER_TYPE("datanucleus.storeManagerType", "rdbms"),
METASTORE_AUTO_CREATE_SCHEMA("datanucleus.autoCreateSchema", true),
+ METASTORE_FIXED_DATASTORE("datanucleus.fixedDatastore", false),
+ METASTORE_SCHEMA_VERIFICATION("hive.metastore.schema.verification", false),
METASTORE_AUTO_START_MECHANISM_MODE("datanucleus.autoStartMechanismMode", "checked"),
METASTORE_TRANSACTION_ISOLATION("datanucleus.transactionIsolation", "read-committed"),
METASTORE_CACHE_LEVEL2("datanucleus.cache.level2", false),
@@ -1141,6 +1143,11 @@ public class HiveConf extends Configurat
auxJars = this.get(ConfVars.HIVEAUXJARS.varname);
}
+ if (getBoolVar(ConfVars.METASTORE_SCHEMA_VERIFICATION)) {
+ setBoolVar(ConfVars.METASTORE_AUTO_CREATE_SCHEMA, false);
+ setBoolVar(ConfVars.METASTORE_FIXED_DATASTORE, true);
+ }
+
// setup list of conf vars that are not allowed to change runtime
String restrictListStr = this.get(ConfVars.HIVE_CONF_RESTRICTED_LIST.toString());
if (restrictListStr != null) {
Modified: hive/branches/branch-0.12/conf/hive-default.xml.template
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/conf/hive-default.xml.template?rev=1526348&r1=1526347&r2=1526348&view=diff
==============================================================================
--- hive/branches/branch-0.12/conf/hive-default.xml.template (original)
+++ hive/branches/branch-0.12/conf/hive-default.xml.template Thu Sep 26 02:05:48 2013
@@ -1976,4 +1976,17 @@
</description>
</property>
+<property>
+ <name>hive.metastore.schema.verification</name>
+ <value>true</value>
+ <description>
+ Enforce metastore schema version consistency.
+ True: Verify that version information stored in metastore matches with one from Hive jars. Also disable automatic
+ schema migration attempt. Users are required to manully migrate schema after Hive upgrade which ensures
+ proper metastore schema migration. (Default)
+ False: Warn if the version information stored in metastore doesn't match with one from in Hive jars.
+ </description>
+</property>
+
+
</configuration>
Added: hive/branches/branch-0.12/metastore/scripts/upgrade/derby/014-HIVE-3764.derby.sql
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/metastore/scripts/upgrade/derby/014-HIVE-3764.derby.sql?rev=1526348&view=auto
==============================================================================
--- hive/branches/branch-0.12/metastore/scripts/upgrade/derby/014-HIVE-3764.derby.sql (added)
+++ hive/branches/branch-0.12/metastore/scripts/upgrade/derby/014-HIVE-3764.derby.sql Thu Sep 26 02:05:48 2013
@@ -0,0 +1,8 @@
+--
+-- Hive HIVE-3764
+-- Support metastore version consistency check
+
+CREATE TABLE "APP"."VERSION" ("VER_ID" BIGINT NOT NULL, "SCHEMA_VERSION" VARCHAR(127) NOT NULL, "COMMENT" VARCHAR(255));
+ALTER TABLE "APP"."VERSION" ADD CONSTRAINT "VERSION_PK" PRIMARY KEY ("VER_ID");
+
+INSERT INTO "APP"."VERSION" (VER_ID, SCHEMA_VERSION, COMMENT) VALUES (1, '', 'Initial value');
Modified: hive/branches/branch-0.12/metastore/scripts/upgrade/derby/hive-schema-0.12.0.derby.sql
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/metastore/scripts/upgrade/derby/hive-schema-0.12.0.derby.sql?rev=1526348&r1=1526347&r2=1526348&view=diff
==============================================================================
--- hive/branches/branch-0.12/metastore/scripts/upgrade/derby/hive-schema-0.12.0.derby.sql (original)
+++ hive/branches/branch-0.12/metastore/scripts/upgrade/derby/hive-schema-0.12.0.derby.sql Thu Sep 26 02:05:48 2013
@@ -98,6 +98,8 @@ CREATE TABLE "APP"."TAB_COL_STATS"("DB_N
CREATE TABLE "APP"."PART_COL_STATS"("DB_NAME" VARCHAR(128) NOT NULL,"TABLE_NAME" VARCHAR(128) NOT NULL, "PARTITION_NAME" VARCHAR(767) NOT NULL, "COLUMN_NAME" VARCHAR(128) NOT NULL, "COLUMN_TYPE" VARCHAR(128) NOT NULL, "LONG_LOW_VALUE" BIGINT, "LONG_HIGH_VALUE" BIGINT, "DOUBLE_LOW_VALUE" DOUBLE, "DOUBLE_HIGH_VALUE" DOUBLE, "BIG_DECIMAL_LOW_VALUE" VARCHAR(4000), "BIG_DECIMAL_HIGH_VALUE" VARCHAR(4000),"NUM_DISTINCTS" BIGINT, "NUM_NULLS" BIGINT NOT NULL, "AVG_COL_LEN" DOUBLE, "MAX_COL_LEN" BIGINT, "NUM_TRUES" BIGINT, "NUM_FALSES" BIGINT, "LAST_ANALYZED" BIGINT, "CS_ID" BIGINT NOT NULL, "PART_ID" BIGINT NOT NULL);
+CREATE TABLE "APP"."VERSION" ("VER_ID" BIGINT NOT NULL, "SCHEMA_VERSION" VARCHAR(127) NOT NULL, "VERSION_COMMENT" VARCHAR(255));
+
-- ----------------------------------------------
-- DDL Statements for indexes
-- ----------------------------------------------
@@ -284,6 +286,8 @@ ALTER TABLE "APP"."TAB_COL_STATS" ADD CO
ALTER TABLE "APP"."PART_COL_STATS" ADD CONSTRAINT "PART_COL_STATS_FK" FOREIGN KEY ("PART_ID") REFERENCES PARTITIONS("PART_ID") ON DELETE NO ACTION ON UPDATE NO ACTION;
+ALTER TABLE "APP"."VERSION" ADD CONSTRAINT "VERSION_PK" PRIMARY KEY ("VER_ID");
+
-- ----------------------------------------------
-- DDL Statements for checks
-- ----------------------------------------------
@@ -292,3 +296,4 @@ ALTER TABLE "APP"."IDXS" ADD CONSTRAINT
ALTER TABLE "APP"."SDS" ADD CONSTRAINT "SQL110318025505550" CHECK (IS_COMPRESSED IN ('Y','N'));
+INSERT INTO "APP"."VERSION" (VER_ID, SCHEMA_VERSION, VERSION_COMMENT) VALUES (1, '0.12.0', 'Hive release version 0.12.0');
Modified: hive/branches/branch-0.12/metastore/scripts/upgrade/derby/upgrade-0.10.0-to-0.11.0.derby.sql
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/metastore/scripts/upgrade/derby/upgrade-0.10.0-to-0.11.0.derby.sql?rev=1526348&r1=1526347&r2=1526348&view=diff
==============================================================================
--- hive/branches/branch-0.12/metastore/scripts/upgrade/derby/upgrade-0.10.0-to-0.11.0.derby.sql (original)
+++ hive/branches/branch-0.12/metastore/scripts/upgrade/derby/upgrade-0.10.0-to-0.11.0.derby.sql Thu Sep 26 02:05:48 2013
@@ -1,2 +1 @@
-- Upgrade MetaStore schema from 0.10.0 to 0.11.0
-
Modified: hive/branches/branch-0.12/metastore/scripts/upgrade/derby/upgrade-0.11.0-to-0.12.0.derby.sql
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/metastore/scripts/upgrade/derby/upgrade-0.11.0-to-0.12.0.derby.sql?rev=1526348&r1=1526347&r2=1526348&view=diff
==============================================================================
--- hive/branches/branch-0.12/metastore/scripts/upgrade/derby/upgrade-0.11.0-to-0.12.0.derby.sql (original)
+++ hive/branches/branch-0.12/metastore/scripts/upgrade/derby/upgrade-0.11.0-to-0.12.0.derby.sql Thu Sep 26 02:05:48 2013
@@ -1,2 +1,4 @@
-- Upgrade MetaStore schema from 0.11.0 to 0.12.0
RUN '013-HIVE-3255.derby.sql';
+RUN '014-HIVE-3764.derby.sql';
+UPDATE "APP".VERSION SET SCHEMA_VERSION='0.12.0', VERSION_COMMENT='Hive release version 0.12.0' where VER_ID=1;
Added: hive/branches/branch-0.12/metastore/scripts/upgrade/derby/upgrade.order.derby
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/metastore/scripts/upgrade/derby/upgrade.order.derby?rev=1526348&view=auto
==============================================================================
--- hive/branches/branch-0.12/metastore/scripts/upgrade/derby/upgrade.order.derby (added)
+++ hive/branches/branch-0.12/metastore/scripts/upgrade/derby/upgrade.order.derby Thu Sep 26 02:05:48 2013
@@ -0,0 +1,7 @@
+0.5.0-to-0.6.0
+0.6.0-to-0.7.0
+0.7.0-to-0.8.0
+0.8.0-to-0.9.0
+0.9.0-to-0.10.0
+0.10.0-to-0.11.0
+0.11.0-to-0.12.0
Added: hive/branches/branch-0.12/metastore/scripts/upgrade/mysql/014-HIVE-3764.mysql.sql
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/metastore/scripts/upgrade/mysql/014-HIVE-3764.mysql.sql?rev=1526348&view=auto
==============================================================================
--- hive/branches/branch-0.12/metastore/scripts/upgrade/mysql/014-HIVE-3764.mysql.sql (added)
+++ hive/branches/branch-0.12/metastore/scripts/upgrade/mysql/014-HIVE-3764.mysql.sql Thu Sep 26 02:05:48 2013
@@ -0,0 +1,9 @@
+-- Table structure for VERSION
+CREATE TABLE IF NOT EXISTS `VERSION` (
+ `VER_ID` BIGINT NOT NULL,
+ `SCHEMA_VERSION` VARCHAR(127) NOT NULL,
+ `COMMENT` VARCHAR(255),
+ PRIMARY KEY (`VER_ID`)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;
+
+INSERT INTO VERSION (VER_ID, SCHEMA_VERSION, COMMENT) VALUES (1, '', 'Initial value');
Modified: hive/branches/branch-0.12/metastore/scripts/upgrade/mysql/hive-schema-0.12.0.mysql.sql
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/metastore/scripts/upgrade/mysql/hive-schema-0.12.0.mysql.sql?rev=1526348&r1=1526347&r2=1526348&view=diff
==============================================================================
--- hive/branches/branch-0.12/metastore/scripts/upgrade/mysql/hive-schema-0.12.0.mysql.sql (original)
+++ hive/branches/branch-0.12/metastore/scripts/upgrade/mysql/hive-schema-0.12.0.mysql.sql Thu Sep 26 02:05:48 2013
@@ -751,6 +751,17 @@ CREATE TABLE IF NOT EXISTS `DELEGATION_T
PRIMARY KEY (`TOKEN_IDENT`)
) ENGINE=INNODB DEFAULT CHARSET=latin1;
+--
+-- Table structure for VERSION
+--
+CREATE TABLE IF NOT EXISTS `VERSION` (
+ `VER_ID` BIGINT NOT NULL,
+ `SCHEMA_VERSION` VARCHAR(127) NOT NULL,
+ `VERSION_COMMENT` VARCHAR(255),
+ PRIMARY KEY (`VER_ID`)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;
+
+INSERT INTO VERSION (VER_ID, SCHEMA_VERSION, VERSION_COMMENT) VALUES (1, '0.12.0', 'Hive release version 0.12.0');
/*!40101 SET character_set_client = @saved_cs_client */;
/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
Modified: hive/branches/branch-0.12/metastore/scripts/upgrade/mysql/upgrade-0.11.0-to-0.12.0.mysql.sql
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/metastore/scripts/upgrade/mysql/upgrade-0.11.0-to-0.12.0.mysql.sql?rev=1526348&r1=1526347&r2=1526348&view=diff
==============================================================================
--- hive/branches/branch-0.12/metastore/scripts/upgrade/mysql/upgrade-0.11.0-to-0.12.0.mysql.sql (original)
+++ hive/branches/branch-0.12/metastore/scripts/upgrade/mysql/upgrade-0.11.0-to-0.12.0.mysql.sql Thu Sep 26 02:05:48 2013
@@ -1,3 +1,5 @@
SELECT 'Upgrading MetaStore schema from 0.11.0 to 0.12.0' AS ' ';
SOURCE 013-HIVE-3255.mysql.sql;
+SOURCE 014-HIVE-3764.mysql.sql;
+UPDATE VERSION SET SCHEMA_VERSION='0.12.0', VERSION_COMMENT='Hive release version 0.12.0' where VER_ID=1;
SELECT 'Finished upgrading MetaStore schema from 0.11.0 to 0.12.0' AS ' ';
Added: hive/branches/branch-0.12/metastore/scripts/upgrade/mysql/upgrade.order.mysql
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/metastore/scripts/upgrade/mysql/upgrade.order.mysql?rev=1526348&view=auto
==============================================================================
--- hive/branches/branch-0.12/metastore/scripts/upgrade/mysql/upgrade.order.mysql (added)
+++ hive/branches/branch-0.12/metastore/scripts/upgrade/mysql/upgrade.order.mysql Thu Sep 26 02:05:48 2013
@@ -0,0 +1,7 @@
+0.5.0-to-0.6.0
+0.6.0-to-0.7.0
+0.7.0-to-0.8.0
+0.8.0-to-0.9.0
+0.9.0-to-0.10.0
+0.10.0-to-0.11.0
+0.11.0-to-0.12.0
Added: hive/branches/branch-0.12/metastore/scripts/upgrade/oracle/014-HIVE-3764.oracle.sql
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/metastore/scripts/upgrade/oracle/014-HIVE-3764.oracle.sql?rev=1526348&view=auto
==============================================================================
--- hive/branches/branch-0.12/metastore/scripts/upgrade/oracle/014-HIVE-3764.oracle.sql (added)
+++ hive/branches/branch-0.12/metastore/scripts/upgrade/oracle/014-HIVE-3764.oracle.sql Thu Sep 26 02:05:48 2013
@@ -0,0 +1,10 @@
+-- HIVE-3764 Support metastore version consistency check
+
+CREATE TABLE IF NOT EXISTS VERSION (
+ VER_ID NUMBER NOT NULL,
+ SCHEMA_VERSION VARCHAR(127) NOT NULL,
+ COMMENT VARCHAR(255)
+)
+ALTER TABLE VERSION ADD CONSTRAINT VERSION_PK PRIMARY KEY (VER_ID);
+
+INSERT INTO VERSION (VER_ID, SCHEMA_VERSION, COMMENT) VALUES (1, '', 'Initial value');
Modified: hive/branches/branch-0.12/metastore/scripts/upgrade/oracle/hive-schema-0.12.0.oracle.sql
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/metastore/scripts/upgrade/oracle/hive-schema-0.12.0.oracle.sql?rev=1526348&r1=1526347&r2=1526348&view=diff
==============================================================================
--- hive/branches/branch-0.12/metastore/scripts/upgrade/oracle/hive-schema-0.12.0.oracle.sql (original)
+++ hive/branches/branch-0.12/metastore/scripts/upgrade/oracle/hive-schema-0.12.0.oracle.sql Thu Sep 26 02:05:48 2013
@@ -483,6 +483,13 @@ CREATE TABLE TAB_COL_STATS (
LAST_ANALYZED NUMBER NOT NULL
);
+CREATE TABLE IF NOT EXISTS VERSION (
+ VER_ID NUMBER NOT NULL,
+ SCHEMA_VERSION VARCHAR(127) NOT NULL,
+ VERSION_COMMENT VARCHAR(255)
+)
+ALTER TABLE VERSION ADD CONSTRAINT VERSION_PK PRIMARY KEY (VER_ID);
+
ALTER TABLE TAB_COL_STATS ADD CONSTRAINT TAB_COL_STATS_PKEY PRIMARY KEY (CS_ID);
ALTER TABLE TAB_COL_STATS ADD CONSTRAINT TAB_COL_STATS_FK FOREIGN KEY (TBL_ID) REFERENCES TBLS (TBL_ID) INITIALLY DEFERRED ;
@@ -707,4 +714,5 @@ CREATE INDEX TBLS_N50 ON TBLS (SD_ID);
-- Constraints for table PARTITION_EVENTS for class(es) [org.apache.hadoop.hive.metastore.model.MPartitionEvent]
CREATE INDEX PARTITIONEVENTINDEX ON PARTITION_EVENTS (PARTITION_NAME);
+INSERT INTO VERSION (VER_ID, SCHEMA_VERSION, VERSION_COMMENT) VALUES (1, '0.12.0', 'Hive release version 0.12.0');
Added: hive/branches/branch-0.12/metastore/scripts/upgrade/oracle/upgrade-0.10.0-to-0.11.0.mysql.sql
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/metastore/scripts/upgrade/oracle/upgrade-0.10.0-to-0.11.0.mysql.sql?rev=1526348&view=auto
==============================================================================
--- hive/branches/branch-0.12/metastore/scripts/upgrade/oracle/upgrade-0.10.0-to-0.11.0.mysql.sql (added)
+++ hive/branches/branch-0.12/metastore/scripts/upgrade/oracle/upgrade-0.10.0-to-0.11.0.mysql.sql Thu Sep 26 02:05:48 2013
@@ -0,0 +1,2 @@
+SELECT 'Upgrading MetaStore schema from 0.10.0 to 0.11.0' AS ' ';
+SELECT 'Finished upgrading MetaStore schema from 0.10.0 to 0.11.0' AS Status from dual;
Modified: hive/branches/branch-0.12/metastore/scripts/upgrade/oracle/upgrade-0.11.0-to-0.12.0.oracle.sql
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/metastore/scripts/upgrade/oracle/upgrade-0.11.0-to-0.12.0.oracle.sql?rev=1526348&r1=1526347&r2=1526348&view=diff
==============================================================================
--- hive/branches/branch-0.12/metastore/scripts/upgrade/oracle/upgrade-0.11.0-to-0.12.0.oracle.sql (original)
+++ hive/branches/branch-0.12/metastore/scripts/upgrade/oracle/upgrade-0.11.0-to-0.12.0.oracle.sql Thu Sep 26 02:05:48 2013
@@ -1,3 +1,5 @@
SELECT 'Upgrading MetaStore schema from 0.11.0 to 0.12.0' AS ' ';
@013-HIVE-3255.oracle.sql;
+@014-HIVE-3764.oracle.sql;
+UPDATE VERSION SET SCHEMA_VERSION='0.12.0', VERSION_COMMENT='Hive release version 0.12.0' where VER_ID=1;
SELECT 'Finished upgrading MetaStore schema from 0.11.0 to 0.12.0' AS Status from dual;
Added: hive/branches/branch-0.12/metastore/scripts/upgrade/oracle/upgrade.order.oracle
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/metastore/scripts/upgrade/oracle/upgrade.order.oracle?rev=1526348&view=auto
==============================================================================
--- hive/branches/branch-0.12/metastore/scripts/upgrade/oracle/upgrade.order.oracle (added)
+++ hive/branches/branch-0.12/metastore/scripts/upgrade/oracle/upgrade.order.oracle Thu Sep 26 02:05:48 2013
@@ -0,0 +1,3 @@
+0.9.0-to-0.10.0
+0.10.0-to-0.11.0
+0.11.0-to-0.12.0
Added: hive/branches/branch-0.12/metastore/scripts/upgrade/postgres/014-HIVE-3764.postgres.sql
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/metastore/scripts/upgrade/postgres/014-HIVE-3764.postgres.sql?rev=1526348&view=auto
==============================================================================
--- hive/branches/branch-0.12/metastore/scripts/upgrade/postgres/014-HIVE-3764.postgres.sql (added)
+++ hive/branches/branch-0.12/metastore/scripts/upgrade/postgres/014-HIVE-3764.postgres.sql Thu Sep 26 02:05:48 2013
@@ -0,0 +1,12 @@
+--
+-- Table structure for VERSION
+--
+CREATE TABLE "VERSION" (
+ "VER_ID" bigint,
+ "SCHEMA_VERSION" character varying(127) NOT NULL,
+ "COMMENT" character varying(255) NOT NULL,
+ PRIMARY KEY ("VER_ID")
+);
+ALTER TABLE ONLY "VERSION" ADD CONSTRAINT "VERSION_pkey" PRIMARY KEY ("VER_ID");
+
+INSERT INTO VERSION (VER_ID, SCHEMA_VERSION, COMMENT) VALUES (1, '', 'Initial value');
Modified: hive/branches/branch-0.12/metastore/scripts/upgrade/postgres/hive-schema-0.12.0.postgres.sql
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/metastore/scripts/upgrade/postgres/hive-schema-0.12.0.postgres.sql?rev=1526348&r1=1526347&r2=1526348&view=diff
==============================================================================
--- hive/branches/branch-0.12/metastore/scripts/upgrade/postgres/hive-schema-0.12.0.postgres.sql (original)
+++ hive/branches/branch-0.12/metastore/scripts/upgrade/postgres/hive-schema-0.12.0.postgres.sql Thu Sep 26 02:05:48 2013
@@ -516,6 +516,16 @@ CREATE TABLE "TAB_COL_STATS" (
);
--
+-- Table structure for VERSION
+--
+CREATE TABLE "VERSION" (
+ "VER_ID" bigint,
+ "SCHEMA_VERSION" character varying(127) NOT NULL,
+ "VERSION_COMMENT" character varying(255) NOT NULL,
+ PRIMARY KEY ("VER_ID")
+);
+
+--
-- Name: PART_COL_STATS Type: TABLE; Schema: public; Owner: hiveuser; Tablespace:
--
@@ -1379,6 +1389,9 @@ ALTER TABLE ONLY "TAB_COL_STATS" ADD CON
--
ALTER TABLE ONLY "PART_COL_STATS" ADD CONSTRAINT "PART_COL_STATS_fkey" FOREIGN KEY("PART_ID") REFERENCES "PARTITIONS"("PART_ID") DEFERRABLE;
+
+ALTER TABLE ONLY "VERSION" ADD CONSTRAINT "VERSION_pkey" PRIMARY KEY ("VER_ID");
+
--
-- Name: public; Type: ACL; Schema: -; Owner: hiveuser
--
@@ -1387,6 +1400,7 @@ REVOKE ALL ON SCHEMA public FROM PUBLIC;
GRANT ALL ON SCHEMA public TO PUBLIC;
+INSERT INTO VERSION (VER_ID, SCHEMA_VERSION, VERSION_COMMENT) VALUES (1, '0.12.0', 'Hive release version 0.12.0');
--
-- PostgreSQL database dump complete
--
Modified: hive/branches/branch-0.12/metastore/scripts/upgrade/postgres/upgrade-0.11.0-to-0.12.0.postgres.sql
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/metastore/scripts/upgrade/postgres/upgrade-0.11.0-to-0.12.0.postgres.sql?rev=1526348&r1=1526347&r2=1526348&view=diff
==============================================================================
--- hive/branches/branch-0.12/metastore/scripts/upgrade/postgres/upgrade-0.11.0-to-0.12.0.postgres.sql (original)
+++ hive/branches/branch-0.12/metastore/scripts/upgrade/postgres/upgrade-0.11.0-to-0.12.0.postgres.sql Thu Sep 26 02:05:48 2013
@@ -1,3 +1,5 @@
SELECT 'Upgrading MetaStore schema from 0.11.0 to 0.12.0';
\i 013-HIVE-3255.postgres.sql;
+\i 014-HIVE-3764.postgres.sql;
+UPDATE VERSION SET SCHEMA_VERSION='0.12.0', VERSION_COMMENT='Hive release version 0.12.0' where VER_ID=1;
SELECT 'Finished upgrading MetaStore schema from 0.11.0 to 0.12.0';
Added: hive/branches/branch-0.12/metastore/scripts/upgrade/postgres/upgrade.order.postgres
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/metastore/scripts/upgrade/postgres/upgrade.order.postgres?rev=1526348&view=auto
==============================================================================
--- hive/branches/branch-0.12/metastore/scripts/upgrade/postgres/upgrade.order.postgres (added)
+++ hive/branches/branch-0.12/metastore/scripts/upgrade/postgres/upgrade.order.postgres Thu Sep 26 02:05:48 2013
@@ -0,0 +1,7 @@
+0.5.0-to-0.6.0
+0.6.0-to-0.7.0
+0.7.0-to-0.8.0
+0.8.0-to-0.9.0
+0.9.0-to-0.10.0
+0.10.0-to-0.11.0
+0.11.0-to-0.12.0
Modified: hive/branches/branch-0.12/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java?rev=1526348&r1=1526347&r2=1526348&view=diff
==============================================================================
--- hive/branches/branch-0.12/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java (original)
+++ hive/branches/branch-0.12/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java Thu Sep 26 02:05:48 2013
@@ -400,6 +400,7 @@ public class HiveMetaStore extends Thrif
RawStore ms = threadLocalMS.get();
if (ms == null) {
ms = newRawStore();
+ ms.verifySchema();
threadLocalMS.set(ms);
ms = threadLocalMS.get();
}
Added: hive/branches/branch-0.12/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java?rev=1526348&view=auto
==============================================================================
--- hive/branches/branch-0.12/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java (added)
+++ hive/branches/branch-0.12/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java Thu Sep 26 02:05:48 2013
@@ -0,0 +1,138 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hive.common.util.HiveVersionInfo;
+
+
+public class MetaStoreSchemaInfo {
+ private static String SQL_FILE_EXTENSION=".sql";
+ private static String UPGRADE_FILE_PREFIX="upgrade-";
+ private static String INIT_FILE_PREFIX="hive-schema-";
+ private static String VERSION_UPGRADE_LIST = "upgrade.order";
+ private final String dbType;
+ private final String hiveSchemaVersions[];
+ private final HiveConf hiveConf;
+ private final String hiveHome;
+
+ public MetaStoreSchemaInfo(String hiveHome, HiveConf hiveConf, String dbType) throws HiveMetaException {
+ this.hiveHome = hiveHome;
+ this.dbType = dbType;
+ this.hiveConf = hiveConf;
+ // load upgrade order for the given dbType
+ List<String> upgradeOrderList = new ArrayList<String>();
+ String upgradeListFile = getMetaStoreScriptDir() + File.separator +
+ VERSION_UPGRADE_LIST + "." + dbType;
+ try {
+ BufferedReader bfReader =
+ new BufferedReader(new FileReader(upgradeListFile));
+ String currSchemaVersion;
+ while ((currSchemaVersion = bfReader.readLine()) != null) {
+ upgradeOrderList.add(currSchemaVersion.trim());
+ }
+ } catch (FileNotFoundException e) {
+ throw new HiveMetaException("File " + upgradeListFile + "not found ", e);
+ } catch (IOException e) {
+ throw new HiveMetaException("Error reading " + upgradeListFile, e);
+ }
+ hiveSchemaVersions = upgradeOrderList.toArray(new String[0]);
+ }
+
+ /***
+ * Get the list of sql scripts required to upgrade from the give version to current
+ * @param fromVersion
+ * @return
+ * @throws HiveMetaException
+ */
+ public List<String> getUpgradeScripts(String fromVersion)
+ throws HiveMetaException {
+ List <String> upgradeScriptList = new ArrayList<String>();
+
+ // check if we are already at current schema level
+ if (getHiveSchemaVersion().equals(fromVersion)) {
+ return upgradeScriptList;
+ }
+ // Find the list of scripts to execute for this upgrade
+ int firstScript = hiveSchemaVersions.length;
+ for (int i=0; i < hiveSchemaVersions.length; i++) {
+ if (hiveSchemaVersions[i].startsWith(fromVersion)) {
+ firstScript = i;
+ }
+ }
+ if (firstScript == hiveSchemaVersions.length) {
+ throw new HiveMetaException("Unknown version specified for upgrade " +
+ fromVersion + " Metastore schema may be too old or newer");
+ }
+
+ for (int i=firstScript; i < hiveSchemaVersions.length; i++) {
+ String scriptFile = generateUpgradeFileName(hiveSchemaVersions[i]);
+ upgradeScriptList.add(scriptFile);
+ }
+ return upgradeScriptList;
+ }
+
+ /***
+ * Get the name of the script to initialize the schema for given version
+ * @param toVersion Target version. If it's null, then the current server version is used
+ * @return
+ * @throws HiveMetaException
+ */
+ public String generateInitFileName(String toVersion) throws HiveMetaException {
+ if (toVersion == null) {
+ toVersion = getHiveSchemaVersion();
+ }
+ String initScriptName = INIT_FILE_PREFIX + toVersion + "." +
+ dbType + SQL_FILE_EXTENSION;
+ // check if the file exists
+ if (!(new File(getMetaStoreScriptDir() + File.separatorChar +
+ initScriptName).exists())) {
+ throw new HiveMetaException("Unknown version specified for initialization: " + toVersion);
+ }
+ return initScriptName;
+ }
+
+ /**
+ * Find the directory of metastore scripts
+ * @return
+ */
+ public String getMetaStoreScriptDir() {
+ return hiveHome + File.separatorChar +
+ "scripts" + File.separatorChar + "metastore" +
+ File.separatorChar + "upgrade" + File.separatorChar + dbType;
+ }
+
+ // format the upgrade script name eg upgrade-x-y-dbType.sql
+ private String generateUpgradeFileName(String fileVersion) {
+ return UPGRADE_FILE_PREFIX + fileVersion + "." + dbType + SQL_FILE_EXTENSION;
+ }
+
+ // Current hive version, remove the 'SNAPSHOT' part if needed
+ public static String getHiveSchemaVersion() {
+ return HiveVersionInfo.getVersion().replace("-SNAPSHOT", "");
+ }
+
+}
Modified: hive/branches/branch-0.12/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java?rev=1526348&r1=1526347&r2=1526348&view=diff
==============================================================================
--- hive/branches/branch-0.12/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java (original)
+++ hive/branches/branch-0.12/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java Thu Sep 26 02:05:48 2013
@@ -19,7 +19,6 @@
package org.apache.hadoop.hive.metastore;
import static org.apache.commons.lang.StringUtils.join;
-import static org.apache.commons.lang.StringUtils.repeat;
import java.net.URI;
import java.net.URISyntaxException;
@@ -34,10 +33,11 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Properties;
import java.util.Set;
-import java.util.TreeMap;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
+import javax.jdo.JDODataStoreException;
import javax.jdo.JDOHelper;
import javax.jdo.JDOObjectNotFoundException;
import javax.jdo.PersistenceManager;
@@ -119,10 +119,12 @@ import org.apache.hadoop.hive.metastore.
import org.apache.hadoop.hive.metastore.model.MTableColumnStatistics;
import org.apache.hadoop.hive.metastore.model.MTablePrivilege;
import org.apache.hadoop.hive.metastore.model.MType;
+import org.apache.hadoop.hive.metastore.model.MVersionTable;
import org.apache.hadoop.hive.metastore.parser.ExpressionTree.ANTLRNoCaseStringStream;
import org.apache.hadoop.hive.metastore.parser.FilterLexer;
import org.apache.hadoop.hive.metastore.parser.FilterParser;
import org.apache.hadoop.util.StringUtils;
+import org.datanucleus.store.rdbms.exceptions.MissingTableException;
/**
* This class is the interface between the application logic and the database
@@ -163,6 +165,7 @@ public class ObjectStore implements RawS
int openTrasactionCalls = 0;
private Transaction currentTransaction = null;
private TXN_STATUS transactionStatus = TXN_STATUS.NO_STATE;
+ private final AtomicBoolean isSchemaVerified = new AtomicBoolean(false);
public ObjectStore() {
}
@@ -5609,4 +5612,131 @@ public class ObjectStore implements RawS
return masterKeys;
}
+ // compare hive version and metastore version
+ @Override
+ public void verifySchema() throws MetaException {
+ // If the schema version is already checked, then go ahead and use this metastore
+ if (isSchemaVerified.get()) {
+ return;
+ }
+ checkSchema();
+ }
+
+ private synchronized void checkSchema() throws MetaException {
+ // recheck if it got verified by another thread while we were waiting
+ if (isSchemaVerified.get()) {
+ return;
+ }
+
+ boolean strictValidation =
+ HiveConf.getBoolVar(getConf(), HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION);
+ // read the schema version stored in metastore db
+ String schemaVer = getMetaStoreSchemaVersion();
+ if (schemaVer == null) {
+ // metastore has no schema version information
+ if (strictValidation) {
+ throw new MetaException("Version information not found in metastore. ");
+ } else {
+ LOG.warn("Version information not found in metastore. "
+ + HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION.toString() +
+ " is not enabled so recording the schema version " +
+ MetaStoreSchemaInfo.getHiveSchemaVersion());
+ setMetaStoreSchemaVersion(MetaStoreSchemaInfo.getHiveSchemaVersion(),
+ "Set by MetaStore");
+ }
+ } else {
+ // metastore schema version is different than Hive distribution needs
+ if (strictValidation) {
+ if (!schemaVer.equalsIgnoreCase(MetaStoreSchemaInfo.getHiveSchemaVersion())) {
+ throw new MetaException("Hive Schema version "
+ + MetaStoreSchemaInfo.getHiveSchemaVersion() +
+ " does not match metastore's schema version " + schemaVer +
+ " Metastore is not upgraded or corrupt");
+ } else {
+ LOG.warn("Metastore version was " + schemaVer + " " +
+ HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION.toString() +
+ " is not enabled so recording the new schema version " +
+ MetaStoreSchemaInfo.getHiveSchemaVersion());
+ setMetaStoreSchemaVersion(MetaStoreSchemaInfo.getHiveSchemaVersion(),
+ "Set by MetaStore");
+ }
+ }
+ }
+ isSchemaVerified.set(true);
+ return;
+ }
+
+ // load the schema version stored in metastore db
+ @Override
+ public String getMetaStoreSchemaVersion() throws MetaException {
+
+ MVersionTable mSchemaVer;
+ try {
+ mSchemaVer = getMSchemaVersion();
+ } catch (NoSuchObjectException e) {
+ return null;
+ }
+ return mSchemaVer.getSchemaVersion();
+ }
+
+ @SuppressWarnings("unchecked")
+ private MVersionTable getMSchemaVersion()
+ throws NoSuchObjectException, MetaException {
+ boolean committed = false;
+ List<MVersionTable> mVerTables = new ArrayList<MVersionTable>();
+
+ try {
+ openTransaction();
+ Query query = pm.newQuery(MVersionTable.class);
+
+ try {
+ mVerTables = (List<MVersionTable>)query.execute();
+ pm.retrieveAll(mVerTables);
+ } catch (JDODataStoreException e) {
+ if (e.getCause() instanceof MissingTableException) {
+ throw new MetaException("Version table not found. " +
+ "The metastore is not upgraded to " + MetaStoreSchemaInfo.getHiveSchemaVersion());
+ } else {
+ throw e;
+ }
+ }
+ committed = commitTransaction();
+ } finally {
+ if (!committed) {
+ rollbackTransaction();
+ }
+ }
+ if (mVerTables.isEmpty()) {
+ throw new NoSuchObjectException("No matching version found");
+ }
+ if (mVerTables.size() > 1) {
+ throw new MetaException("Metastore contains multiple versions");
+ }
+ return mVerTables.get(0);
+ }
+
+ @Override
+ public void setMetaStoreSchemaVersion(String schemaVersion, String comment) throws MetaException {
+ MVersionTable mSchemaVer;
+ boolean commited = false;
+
+ try {
+ mSchemaVer = getMSchemaVersion();
+ } catch (NoSuchObjectException e) {
+ // if the version doesn't exist, then create it
+ mSchemaVer = new MVersionTable();
+ }
+
+ mSchemaVer.setSchemaVersion(schemaVersion);
+ mSchemaVer.setVersionComment(comment);
+ try {
+ openTransaction();
+ pm.makePersistent(mSchemaVer);
+ commited = commitTransaction();
+ } finally {
+ if (!commited) {
+ rollbackTransaction();
+ }
+ }
+ }
}
Modified: hive/branches/branch-0.12/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java?rev=1526348&r1=1526347&r2=1526348&view=diff
==============================================================================
--- hive/branches/branch-0.12/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java (original)
+++ hive/branches/branch-0.12/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java Thu Sep 26 02:05:48 2013
@@ -436,4 +436,10 @@ public interface RawStore extends Config
public abstract String[] getMasterKeys();
+ public abstract void verifySchema() throws MetaException;
+
+ public abstract String getMetaStoreSchemaVersion() throws MetaException;
+
+ public abstract void setMetaStoreSchemaVersion(String version, String comment) throws MetaException;
+
}
Added: hive/branches/branch-0.12/metastore/src/model/org/apache/hadoop/hive/metastore/model/MVersionTable.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/metastore/src/model/org/apache/hadoop/hive/metastore/model/MVersionTable.java?rev=1526348&view=auto
==============================================================================
--- hive/branches/branch-0.12/metastore/src/model/org/apache/hadoop/hive/metastore/model/MVersionTable.java (added)
+++ hive/branches/branch-0.12/metastore/src/model/org/apache/hadoop/hive/metastore/model/MVersionTable.java Thu Sep 26 02:05:48 2013
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.model;
+
+public class MVersionTable {
+ private String schemaVersion;
+ private String versionComment;
+
+ public MVersionTable() {}
+
+ public MVersionTable(String schemaVersion, String versionComment) {
+ this.schemaVersion = schemaVersion;
+ this.versionComment = versionComment;
+ }
+ /**
+ * @return the versionComment
+ */
+ public String getVersionComment() {
+ return versionComment;
+ }
+ /**
+ * @param versionComment the versionComment to set
+ */
+ public void setVersionComment(String versionComment) {
+ this.versionComment = versionComment;
+ }
+
+ /**
+ * @return the schemaVersion
+ */
+ public String getSchemaVersion() {
+ return schemaVersion;
+ }
+ /**
+ * @param schemaVersion the schemaVersion to set
+ */
+ public void setSchemaVersion(String schemaVersion) {
+ this.schemaVersion = schemaVersion;
+ }
+
+}
Modified: hive/branches/branch-0.12/metastore/src/model/package.jdo
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/metastore/src/model/package.jdo?rev=1526348&r1=1526347&r2=1526348&view=diff
==============================================================================
--- hive/branches/branch-0.12/metastore/src/model/package.jdo (original)
+++ hive/branches/branch-0.12/metastore/src/model/package.jdo Thu Sep 26 02:05:48 2013
@@ -899,6 +899,17 @@
<column name="LAST_ANALYZED" jdbc-type="BIGINT" allows-null="false"/>
</field>
</class>
+ <class name="MVersionTable" table="VERSION" identity-type="datastore" detachable="true">
+ <datastore-identity>
+ <column name="VER_ID"/>
+ </datastore-identity>
+ <field name ="schemaVersion">
+ <column name="SCHEMA_VERSION" length="127" jdbc-type="VARCHAR" allows-null="false"/>
+ </field>
+ <field name ="versionComment">
+ <column name="VERSION_COMMENT" length="255" jdbc-type="VARCHAR" allows-null="false"/>
+ </field>
+ </class>
</package>
</jdo>
Modified: hive/branches/branch-0.12/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java?rev=1526348&r1=1526347&r2=1526348&view=diff
==============================================================================
--- hive/branches/branch-0.12/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java (original)
+++ hive/branches/branch-0.12/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java Thu Sep 26 02:05:48 2013
@@ -18,8 +18,8 @@
package org.apache.hadoop.hive.metastore;
-import java.util.List;
import java.util.ArrayList;
+import java.util.List;
import java.util.Map;
import org.apache.hadoop.conf.Configurable;
@@ -532,11 +532,11 @@ public class DummyRawStoreControlledComm
public boolean addToken(String tokenIdentifier, String delegationToken) {
return false;
}
-
+
public boolean removeToken(String tokenIdentifier) {
return false;
}
-
+
public String getToken(String tokenIdentifier) {
return "";
}
@@ -560,4 +560,19 @@ public class DummyRawStoreControlledComm
return new String[0];
}
+ @Override
+ public void verifySchema() throws MetaException {
+ }
+
+ @Override
+ public String getMetaStoreSchemaVersion() throws MetaException {
+ return objectStore.getMetaStoreSchemaVersion();
+ }
+
+ @Override
+ public void setMetaStoreSchemaVersion(String schemaVersion, String comment) throws MetaException {
+ objectStore.setMetaStoreSchemaVersion(schemaVersion, comment);
+
+ }
+
}
Modified: hive/branches/branch-0.12/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java?rev=1526348&r1=1526347&r2=1526348&view=diff
==============================================================================
--- hive/branches/branch-0.12/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java (original)
+++ hive/branches/branch-0.12/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java Thu Sep 26 02:05:48 2013
@@ -593,6 +593,19 @@ public class DummyRawStoreForJdoConnecti
throws NoSuchObjectException, MetaException, InvalidObjectException {
return false;
}
+
+ @Override
+ public void verifySchema() throws MetaException {
+ }
+
+ @Override
+ public String getMetaStoreSchemaVersion() throws MetaException {
+ return null;
+ }
+
+ @Override
+ public void setMetaStoreSchemaVersion(String version, String comment) throws MetaException {
+ }
}
Added: hive/branches/branch-0.12/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java?rev=1526348&view=auto
==============================================================================
--- hive/branches/branch-0.12/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java (added)
+++ hive/branches/branch-0.12/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java Thu Sep 26 02:05:48 2013
@@ -0,0 +1,166 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore;
+
+import junit.framework.TestCase;
+
+import org.apache.hadoop.hive.cli.CliSessionState;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.ql.Driver;
+import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
+import org.apache.hadoop.hive.ql.session.SessionState;
+
+public class TestMetastoreVersion extends TestCase {
+
+ protected HiveConf hiveConf;
+ private Driver driver;
+ private String hiveHome;
+
+ @Override
+ protected void setUp() throws Exception {
+ super.setUp();
+ hiveConf = new HiveConf(this.getClass());
+ System.setProperty("hive.metastore.event.listeners",
+ DummyListener.class.getName());
+ System.setProperty("hive.metastore.pre.event.listeners",
+ DummyPreListener.class.getName());
+ hiveHome = System.getProperty("hive.home");
+ }
+
+ /***
+ * Test config defaults
+ */
+ public void testDefaults() {
+ hiveConf = new HiveConf(this.getClass());
+ assertFalse(hiveConf.getBoolVar(HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION));
+ assertTrue(hiveConf.getBoolVar(HiveConf.ConfVars.METASTORE_AUTO_CREATE_SCHEMA));
+ assertFalse(hiveConf.getBoolVar(HiveConf.ConfVars.METASTORE_FIXED_DATASTORE));
+ }
+
+ /***
+ * Test schema verification property
+ * @throws Exception
+ */
+ public void testVersionRestriction () throws Exception {
+ System.setProperty(HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION.toString(), "true");
+ hiveConf = new HiveConf(this.getClass());
+ assertTrue(hiveConf.getBoolVar(HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION));
+ assertFalse(hiveConf.getBoolVar(HiveConf.ConfVars.METASTORE_AUTO_CREATE_SCHEMA));
+ assertTrue(hiveConf.getBoolVar(HiveConf.ConfVars.METASTORE_FIXED_DATASTORE));
+
+ SessionState.start(new CliSessionState(hiveConf));
+ driver = new Driver(hiveConf);
+ // driver execution should fail since the schema didn't get created
+ CommandProcessorResponse proc = driver.run("show tables");
+ assertFalse(proc.getResponseCode() == 0);
+ }
+
+ /***
+ * Test that with no verification, hive populates the schema and version correctly
+ * @throws Exception
+ */
+ public void testMetastoreVersion () throws Exception {
+ // let the schema and version be auto created
+ System.setProperty(HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION.toString(), "false");
+ hiveConf = new HiveConf(this.getClass());
+ SessionState.start(new CliSessionState(hiveConf));
+ driver = new Driver(hiveConf);
+ driver.run("show tables");
+
+ // correct version stored by Metastore during startup
+ assertEquals(MetaStoreSchemaInfo.getHiveSchemaVersion(), getVersion(hiveConf));
+ setVersion(hiveConf, "foo");
+ assertEquals("foo", getVersion(hiveConf));
+ }
+
+ /***
+ * Test that with verification enabled, hive works when the correct schema is already populated
+ * @throws Exception
+ */
+ public void testVersionMatching () throws Exception {
+ System.setProperty(HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION.toString(), "false");
+ hiveConf = new HiveConf(this.getClass());
+ SessionState.start(new CliSessionState(hiveConf));
+ driver = new Driver(hiveConf);
+ driver.run("show tables");
+
+ hiveConf.setBoolVar(HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION, true);
+ setVersion(hiveConf, MetaStoreSchemaInfo.getHiveSchemaVersion());
+ driver = new Driver(hiveConf);
+ CommandProcessorResponse proc = driver.run("show tables");
+ assertTrue(proc.getResponseCode() == 0);
+ }
+
+ /**
+ * Store garbage version in metastore and verify that hive fails when verification is on
+ * @throws Exception
+ */
+ public void testVersionMisMatch () throws Exception {
+ System.setProperty(HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION.toString(), "false");
+ hiveConf = new HiveConf(this.getClass());
+ SessionState.start(new CliSessionState(hiveConf));
+ driver = new Driver(hiveConf);
+ driver.run("show tables");
+
+ System.setProperty(HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION.toString(), "true");
+ hiveConf = new HiveConf(this.getClass());
+ setVersion(hiveConf, "fooVersion");
+ SessionState.start(new CliSessionState(hiveConf));
+ driver = new Driver(hiveConf);
+ CommandProcessorResponse proc = driver.run("show tables");
+ assertFalse(proc.getResponseCode() == 0);
+ }
+
+ // write the given version to metastore
+ private String getVersion(HiveConf conf) throws HiveMetaException {
+ MetaStoreSchemaInfo schemInfo = new MetaStoreSchemaInfo(hiveHome, conf, "derby");
+ return getMetaStoreVersion();
+ }
+
+ // write the given version to metastore
+ private void setVersion(HiveConf conf, String version) throws HiveMetaException {
+ MetaStoreSchemaInfo schemInfo = new MetaStoreSchemaInfo(hiveHome, conf, "derby");
+ setMetaStoreVersion(version, "setVersion test");
+ }
+
+ // Load the version stored in the metastore db
+ public String getMetaStoreVersion() throws HiveMetaException {
+ ObjectStore objStore = new ObjectStore();
+ objStore.setConf(hiveConf);
+ try {
+ return objStore.getMetaStoreSchemaVersion();
+ } catch (MetaException e) {
+ throw new HiveMetaException("Failed to get version", e);
+ }
+ }
+
+ // Store the given version and comment in the metastore
+ public void setMetaStoreVersion(String newVersion, String comment) throws HiveMetaException {
+ ObjectStore objStore = new ObjectStore();
+ objStore.setConf(hiveConf);
+ try {
+ objStore.setMetaStoreSchemaVersion(newVersion, comment);
+ } catch (MetaException e) {
+ throw new HiveMetaException("Failed to set version", e);
+ }
+ }
+
+
+}
+