You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by th...@apache.org on 2016/07/12 07:12:49 UTC
hive git commit: HIVE-14152 : datanucleus.autoStartMechanismMode
should set to Ignored to allow rolling downgrade (Thejas M Nair,
reviewed by Sushanth Sowmyan)
Repository: hive
Updated Branches:
refs/heads/master a61c35184 -> 5fb8a8b88
HIVE-14152 : datanucleus.autoStartMechanismMode should set to Ignored to allow rolling downgrade (Thejas M Nair, reviewed by Sushanth Sowmyan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/5fb8a8b8
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/5fb8a8b8
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/5fb8a8b8
Branch: refs/heads/master
Commit: 5fb8a8b8884d9b7318d61b77cc9b98ea6cadd570
Parents: a61c351
Author: Thejas Nair <th...@hortonworks.com>
Authored: Tue Jul 12 00:12:40 2016 -0700
Committer: Thejas Nair <th...@hortonworks.com>
Committed: Tue Jul 12 00:12:40 2016 -0700
----------------------------------------------------------------------
beeline/src/test/resources/hive-site.xml | 13 +++++++----
.../org/apache/hadoop/hive/conf/HiveConf.java | 13 +++++------
data/conf/hive-site.xml | 5 ++++
data/conf/llap/hive-site.xml | 5 ++++
data/conf/spark/standalone/hive-site.xml | 5 ++++
data/conf/spark/yarn-client/hive-site.xml | 5 ++++
.../hive/metastore/TestMetastoreVersion.java | 4 +++-
.../hadoop/hive/metastore/ObjectStore.java | 24 ++++++++++++++++++--
ql/src/test/templates/TestPerfCliDriver.vm | 1 +
9 files changed, 61 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/5fb8a8b8/beeline/src/test/resources/hive-site.xml
----------------------------------------------------------------------
diff --git a/beeline/src/test/resources/hive-site.xml b/beeline/src/test/resources/hive-site.xml
index 453b0ea..5f310d6 100644
--- a/beeline/src/test/resources/hive-site.xml
+++ b/beeline/src/test/resources/hive-site.xml
@@ -24,10 +24,15 @@
<description>Internal marker for test. Used for masking env-dependent values</description>
</property>
-<property>
- <name>datanucleus.schema.autoCreateAll</name>
- <value>true</value>
-</property>
+ <property>
+ <name>datanucleus.schema.autoCreateAll</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>hive.metastore.schema.verification</name>
+ <value>false</value>
+ </property>
<property>
<name>javax.jdo.option.ConnectionURL</name>
http://git-wip-us.apache.org/repos/asf/hive/blob/5fb8a8b8/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 283ef2e..bae1825 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -232,7 +232,6 @@ public class HiveConf extends Configuration {
HiveConf.ConfVars.METASTORE_VALIDATE_CONSTRAINTS,
HiveConf.ConfVars.METASTORE_STORE_MANAGER_TYPE,
HiveConf.ConfVars.METASTORE_AUTO_CREATE_ALL,
- HiveConf.ConfVars.METASTORE_AUTO_START_MECHANISM_MODE,
HiveConf.ConfVars.METASTORE_TRANSACTION_ISOLATION,
HiveConf.ConfVars.METASTORE_CACHE_LEVEL2,
HiveConf.ConfVars.METASTORE_CACHE_LEVEL2_TYPE,
@@ -712,18 +711,18 @@ public class HiveConf extends Configuration {
"validates existing schema against code. turn this on if you want to verify existing schema"),
METASTORE_STORE_MANAGER_TYPE("datanucleus.storeManagerType", "rdbms", "metadata store type"),
METASTORE_AUTO_CREATE_ALL("datanucleus.schema.autoCreateAll", false,
- "creates necessary schema on a startup if one doesn't exist. set this to false, after creating it once"),
- METASTORE_SCHEMA_VERIFICATION("hive.metastore.schema.verification", false,
+ "Auto creates necessary schema on a startup if one doesn't exist. Set this to false, after creating it once."
+ + "To enable auto create also set hive.metastore.schema.verification=false. Auto creation is not "
+ + "recommended for production use cases, run schematool command instead." ),
+ METASTORE_SCHEMA_VERIFICATION("hive.metastore.schema.verification", true,
"Enforce metastore schema version consistency.\n" +
- "True: Verify that version information stored in metastore matches with one from Hive jars. Also disable automatic\n" +
+ "True: Verify that version information stored in is compatible with one from Hive jars. Also disable automatic\n" +
" schema migration attempt. Users are required to manually migrate schema after Hive upgrade which ensures\n" +
" proper metastore schema migration. (Default)\n" +
"False: Warn if the version information stored in metastore doesn't match with one from in Hive jars."),
- METASTORE_SCHEMA_VERIFICATION_RECORD_VERSION("hive.metastore.schema.verification.record.version", true,
+ METASTORE_SCHEMA_VERIFICATION_RECORD_VERSION("hive.metastore.schema.verification.record.version", false,
"When true the current MS version is recorded in the VERSION table. If this is disabled and verification is\n" +
" enabled the MS will be unusable."),
- METASTORE_AUTO_START_MECHANISM_MODE("datanucleus.autoStartMechanismMode", "checked",
- "throw exception if metadata tables are incorrect"),
METASTORE_TRANSACTION_ISOLATION("datanucleus.transactionIsolation", "read-committed",
"Default transaction isolation level for identity generation."),
METASTORE_CACHE_LEVEL2("datanucleus.cache.level2", false,
http://git-wip-us.apache.org/repos/asf/hive/blob/5fb8a8b8/data/conf/hive-site.xml
----------------------------------------------------------------------
diff --git a/data/conf/hive-site.xml b/data/conf/hive-site.xml
index 041b67d..775f5c2 100644
--- a/data/conf/hive-site.xml
+++ b/data/conf/hive-site.xml
@@ -64,6 +64,11 @@
</property>
<property>
+ <name>hive.metastore.schema.verification</name>
+ <value>false</value>
+</property>
+
+<property>
<name>javax.jdo.option.ConnectionURL</name>
<value>jdbc:derby:;databaseName=${test.tmp.dir}/junit_metastore_db;create=true</value>
</property>
http://git-wip-us.apache.org/repos/asf/hive/blob/5fb8a8b8/data/conf/llap/hive-site.xml
----------------------------------------------------------------------
diff --git a/data/conf/llap/hive-site.xml b/data/conf/llap/hive-site.xml
index 72bdcfb..8b6dc95 100644
--- a/data/conf/llap/hive-site.xml
+++ b/data/conf/llap/hive-site.xml
@@ -74,6 +74,11 @@
</property>
<property>
+ <name>hive.metastore.schema.verification</name>
+ <value>false</value>
+</property>
+
+<property>
<name>javax.jdo.option.ConnectionURL</name>
<value>jdbc:derby:;databaseName=${test.tmp.dir}/junit_metastore_db;create=true</value>
</property>
http://git-wip-us.apache.org/repos/asf/hive/blob/5fb8a8b8/data/conf/spark/standalone/hive-site.xml
----------------------------------------------------------------------
diff --git a/data/conf/spark/standalone/hive-site.xml b/data/conf/spark/standalone/hive-site.xml
index bc8c0e5..989e65e 100644
--- a/data/conf/spark/standalone/hive-site.xml
+++ b/data/conf/spark/standalone/hive-site.xml
@@ -50,6 +50,11 @@
</property>
<property>
+ <name>hive.metastore.schema.verification</name>
+ <value>false</value>
+</property>
+
+<property>
<name>javax.jdo.option.ConnectionURL</name>
<value>jdbc:derby:;databaseName=${test.tmp.dir}/junit_metastore_db;create=true</value>
</property>
http://git-wip-us.apache.org/repos/asf/hive/blob/5fb8a8b8/data/conf/spark/yarn-client/hive-site.xml
----------------------------------------------------------------------
diff --git a/data/conf/spark/yarn-client/hive-site.xml b/data/conf/spark/yarn-client/hive-site.xml
index f1d9ddc..8c66f87 100644
--- a/data/conf/spark/yarn-client/hive-site.xml
+++ b/data/conf/spark/yarn-client/hive-site.xml
@@ -50,6 +50,11 @@
</property>
<property>
+ <name>hive.metastore.schema.verification</name>
+ <value>false</value>
+</property>
+
+<property>
<name>javax.jdo.option.ConnectionURL</name>
<value>jdbc:derby:;databaseName=${test.tmp.dir}/junit_metastore_db;create=true</value>
</property>
http://git-wip-us.apache.org/repos/asf/hive/blob/5fb8a8b8/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java
index 1352198..6d1673d 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java
@@ -107,12 +107,14 @@ public class TestMetastoreVersion extends TestCase {
}
/***
- * Test that with no verification, hive populates the schema and version correctly
+ * Test that with no verification, and record verification enabled, hive populates the schema
+ * and version correctly
* @throws Exception
*/
public void testMetastoreVersion () throws Exception {
// let the schema and version be auto created
System.setProperty(HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION.toString(), "false");
+ System.setProperty(HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION_RECORD_VERSION.toString(), "true");
hiveConf = new HiveConf(this.getClass());
SessionState.start(new CliSessionState(hiveConf));
driver = new Driver(hiveConf);
http://git-wip-us.apache.org/repos/asf/hive/blob/5fb8a8b8/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
index 82de857..5adfa02 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
@@ -385,6 +385,7 @@ public class ObjectStore implements RawStore, Configurable {
@SuppressWarnings("nls")
private static Properties getDataSourceProps(Configuration conf) {
Properties prop = new Properties();
+ correctAutoStartMechanism(conf);
Iterator<Map.Entry<String, String>> iter = conf.iterator();
while (iter.hasNext()) {
@@ -420,6 +421,24 @@ public class ObjectStore implements RawStore, Configurable {
return prop;
}
+ /**
+ * Update conf to set datanucleus.autoStartMechanismMode=ignored.
+ * This is necessary to able to use older version of hive against
+ * an upgraded but compatible metastore schema in db from new version
+ * of hive
+ * @param conf
+ */
+ private static void correctAutoStartMechanism(Configuration conf) {
+ final String autoStartKey = "datanucleus.autoStartMechanismMode";
+ final String autoStartIgnore = "ignored";
+ String currentAutoStartVal = conf.get(autoStartKey);
+ if(currentAutoStartVal != null && !currentAutoStartVal.equalsIgnoreCase(autoStartIgnore)) {
+ LOG.warn(autoStartKey + " is set to unsupported value " + conf.get(autoStartKey) +
+ " . Setting it to value " + autoStartIgnore);
+ }
+ conf.set(autoStartKey, autoStartIgnore);
+ }
+
private static synchronized PersistenceManagerFactory getPMF() {
if (pmf == null) {
pmf = JDOHelper.getPersistenceManagerFactory(prop);
@@ -7769,10 +7788,10 @@ public class ObjectStore implements RawStore, Configurable {
"Set by MetaStore " + USER + "@" + HOSTNAME);
}
} else {
- // metastore schema version is different than Hive distribution needs
if (MetaStoreSchemaInfo.isVersionCompatible(hiveSchemaVer, dbSchemaVer)) {
LOG.debug("Found expected HMS version of " + dbSchemaVer);
} else {
+ // metastore schema version is different than Hive distribution needs
if (strictValidation) {
throw new MetaException("Hive Schema version " + hiveSchemaVer +
" does not match metastore's schema version " + dbSchemaVer +
@@ -7781,7 +7800,7 @@ public class ObjectStore implements RawStore, Configurable {
LOG.error("Version information found in metastore differs " + dbSchemaVer +
" from expected schema version " + hiveSchemaVer +
". Schema verififcation is disabled " +
- HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION + " so setting version.");
+ HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION);
setMetaStoreSchemaVersion(hiveSchemaVer,
"Set by MetaStore " + USER + "@" + HOSTNAME);
}
@@ -7858,6 +7877,7 @@ public class ObjectStore implements RawStore, Configurable {
"version = " + schemaVersion + ", comment = " + comment);
return;
}
+ LOG.warn("Setting metastore schema version in db to " + schemaVersion);
try {
mSchemaVer = getMSchemaVersion();
http://git-wip-us.apache.org/repos/asf/hive/blob/5fb8a8b8/ql/src/test/templates/TestPerfCliDriver.vm
----------------------------------------------------------------------
diff --git a/ql/src/test/templates/TestPerfCliDriver.vm b/ql/src/test/templates/TestPerfCliDriver.vm
index 97dde2c..d2946cb 100644
--- a/ql/src/test/templates/TestPerfCliDriver.vm
+++ b/ql/src/test/templates/TestPerfCliDriver.vm
@@ -57,6 +57,7 @@ public class $className extends TestCase {
static {
System.setProperty("datanucleus.schema.autoCreateAll", "true");
+ System.setProperty("hive.metastore.schema.verification", "false");
MiniClusterType miniMR = MiniClusterType.valueForString("$clusterMode");
String hiveConfDir = "$hiveConfDir";
String initScript = "$initScript";