You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by th...@apache.org on 2013/10/04 02:56:49 UTC
svn commit: r1529047 - in /hive/branches/branch-0.12:
beeline/src/java/org/apache/hive/beeline/
beeline/src/test/org/apache/hive/beeline/src/test/
metastore/scripts/upgrade/oracle/ metastore/scripts/upgrade/postgres/
metastore/src/test/org/apache/hadoo...
Author: thejas
Date: Fri Oct 4 00:56:49 2013
New Revision: 1529047
URL: http://svn.apache.org/r1529047
Log:
HIVE-5419 : Fix schema tool issues with Oracle metastore (Prasad Mujumdar via Ashutosh Chauhan)
Modified:
hive/branches/branch-0.12/beeline/src/java/org/apache/hive/beeline/HiveSchemaHelper.java
hive/branches/branch-0.12/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java
hive/branches/branch-0.12/beeline/src/test/org/apache/hive/beeline/src/test/TestSchemaTool.java
hive/branches/branch-0.12/metastore/scripts/upgrade/oracle/014-HIVE-3764.oracle.sql
hive/branches/branch-0.12/metastore/scripts/upgrade/oracle/hive-schema-0.12.0.oracle.sql
hive/branches/branch-0.12/metastore/scripts/upgrade/oracle/upgrade-0.11.0-to-0.12.0.oracle.sql
hive/branches/branch-0.12/metastore/scripts/upgrade/postgres/hive-schema-0.12.0.postgres.sql
hive/branches/branch-0.12/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java
Modified: hive/branches/branch-0.12/beeline/src/java/org/apache/hive/beeline/HiveSchemaHelper.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/beeline/src/java/org/apache/hive/beeline/HiveSchemaHelper.java?rev=1529047&r1=1529046&r2=1529047&view=diff
==============================================================================
--- hive/branches/branch-0.12/beeline/src/java/org/apache/hive/beeline/HiveSchemaHelper.java (original)
+++ hive/branches/branch-0.12/beeline/src/java/org/apache/hive/beeline/HiveSchemaHelper.java Fri Oct 4 00:56:49 2013
@@ -22,7 +22,7 @@ import java.util.IllegalFormatException;
public class HiveSchemaHelper {
public static final String DB_DERBY = "derby";
public static final String DB_MYSQL = "mysql";
- public static final String DB_POSTGRACE = "postgrace";
+ public static final String DB_POSTGRACE = "postgres";
public static final String DB_ORACLE = "oracle";
public interface NestedScriptParser {
@@ -225,8 +225,8 @@ public class HiveSchemaHelper {
if (!isNestedScript(dbCommand)) {
throw new IllegalArgumentException("Not a nested script format " + dbCommand);
}
- // remove ending ';'
- return dbCommand.replace(";", "");
+ // remove ending ';' and starting '@'
+ return dbCommand.replace(";", "").replace(ORACLE_NESTING_TOKEN, "");
}
@Override
Modified: hive/branches/branch-0.12/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java?rev=1529047&r1=1529046&r2=1529047&view=diff
==============================================================================
--- hive/branches/branch-0.12/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java (original)
+++ hive/branches/branch-0.12/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java Fri Oct 4 00:56:49 2013
@@ -366,6 +366,7 @@ public class HiveSchemaTool {
beeLine.getOpts().setSilent(true);
}
beeLine.getOpts().setAllowMultiLineCommand(false);
+ beeLine.getOpts().setIsolation("TRANSACTION_READ_COMMITTED");
int status = beeLine.begin(argList.toArray(new String[0]), null);
if (status != 0) {
throw new IOException("Schema script failed, errorcode " + status);
Modified: hive/branches/branch-0.12/beeline/src/test/org/apache/hive/beeline/src/test/TestSchemaTool.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/beeline/src/test/org/apache/hive/beeline/src/test/TestSchemaTool.java?rev=1529047&r1=1529046&r2=1529047&view=diff
==============================================================================
--- hive/branches/branch-0.12/beeline/src/test/org/apache/hive/beeline/src/test/TestSchemaTool.java (original)
+++ hive/branches/branch-0.12/beeline/src/test/org/apache/hive/beeline/src/test/TestSchemaTool.java Fri Oct 4 00:56:49 2013
@@ -30,6 +30,7 @@ import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.HiveMetaException;
+import org.apache.hadoop.hive.metastore.MetaStoreSchemaInfo;
import org.apache.hive.beeline.HiveSchemaHelper;
import org.apache.hive.beeline.HiveSchemaHelper.NestedScriptParser;
import org.apache.hive.beeline.HiveSchemaTool;
@@ -100,7 +101,8 @@ public class TestSchemaTool extends Test
* @throws Exception
*/
public void testSchemaInit() throws Exception {
- schemaTool.doInit("0.12.0");
+ schemaTool.doInit(MetaStoreSchemaInfo.getHiveSchemaVersion());
+ schemaTool.verifySchemaVersion();
}
/**
@@ -321,6 +323,51 @@ public class TestSchemaTool extends Test
assertEquals(expectedSQL, flattenedSql);
}
+ /**
+ * Test nested script formatting
+ * @throws Exception
+ */
+ public void testNestedScriptsForOracle() throws Exception {
+ String childTab1 = "childTab1";
+ String childTab2 = "childTab2";
+ String parentTab = "fooTab";
+
+ String childTestScript1[] = {
+ "-- this is a comment ",
+ "DROP TABLE IF EXISTS " + childTab1 + ";",
+ "CREATE TABLE " + childTab1 + "(id INTEGER);",
+ "DROP TABLE " + childTab1 + ";"
+ };
+ String childTestScript2[] = {
+ "-- this is a comment",
+ "DROP TABLE IF EXISTS " + childTab2 + ";",
+ "CREATE TABLE " + childTab2 + "(id INTEGER);",
+ "-- this is also a comment",
+ "DROP TABLE " + childTab2 + ";"
+ };
+
+ String parentTestScript[] = {
+ " -- this is a comment",
+ "DROP TABLE IF EXISTS " + parentTab + ";",
+ " -- this is another comment ",
+ "CREATE TABLE " + parentTab + "(id INTEGER);",
+ "@" + generateTestScript(childTestScript1).getName() + ";",
+ "DROP TABLE " + parentTab + ";",
+ "@" + generateTestScript(childTestScript2).getName() + ";",
+ "--ending comment ",
+ };
+
+ File testScriptFile = generateTestScript(parentTestScript);
+ String flattenedSql = HiveSchemaTool.buildCommand(
+ HiveSchemaHelper.getDbCommandParser("oracle"),
+ testScriptFile.getParentFile().getPath(), testScriptFile.getName());
+ assertFalse(flattenedSql.contains("@"));
+ assertFalse(flattenedSql.contains("comment"));
+ assertTrue(flattenedSql.contains(childTab1));
+ assertTrue(flattenedSql.contains(childTab2));
+ assertTrue(flattenedSql.contains(parentTab));
+ }
+
private File generateTestScript(String [] stmts) throws IOException {
File testScriptFile = File.createTempFile("schematest", ".sql");
testScriptFile.deleteOnExit();
Modified: hive/branches/branch-0.12/metastore/scripts/upgrade/oracle/014-HIVE-3764.oracle.sql
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/metastore/scripts/upgrade/oracle/014-HIVE-3764.oracle.sql?rev=1529047&r1=1529046&r2=1529047&view=diff
==============================================================================
--- hive/branches/branch-0.12/metastore/scripts/upgrade/oracle/014-HIVE-3764.oracle.sql (original)
+++ hive/branches/branch-0.12/metastore/scripts/upgrade/oracle/014-HIVE-3764.oracle.sql Fri Oct 4 00:56:49 2013
@@ -1,10 +1,10 @@
-- HIVE-3764 Support metastore version consistency check
-CREATE TABLE IF NOT EXISTS VERSION (
+CREATE TABLE VERSION (
VER_ID NUMBER NOT NULL,
SCHEMA_VERSION VARCHAR(127) NOT NULL,
VERSION_COMMENT VARCHAR(255)
-)
+);
ALTER TABLE VERSION ADD CONSTRAINT VERSION_PK PRIMARY KEY (VER_ID);
-INSERT INTO VERSION (VER_ID, SCHEMA_VERSION, VERSION_COMMENT) VALUES (1, '', 'Initial value');
+INSERT INTO VERSION (VER_ID, SCHEMA_VERSION, VERSION_COMMENT) VALUES (1, ' ', 'Initial value');
Modified: hive/branches/branch-0.12/metastore/scripts/upgrade/oracle/hive-schema-0.12.0.oracle.sql
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/metastore/scripts/upgrade/oracle/hive-schema-0.12.0.oracle.sql?rev=1529047&r1=1529046&r2=1529047&view=diff
==============================================================================
--- hive/branches/branch-0.12/metastore/scripts/upgrade/oracle/hive-schema-0.12.0.oracle.sql (original)
+++ hive/branches/branch-0.12/metastore/scripts/upgrade/oracle/hive-schema-0.12.0.oracle.sql Fri Oct 4 00:56:49 2013
@@ -483,11 +483,11 @@ CREATE TABLE TAB_COL_STATS (
LAST_ANALYZED NUMBER NOT NULL
);
-CREATE TABLE IF NOT EXISTS VERSION (
+CREATE TABLE VERSION (
VER_ID NUMBER NOT NULL,
SCHEMA_VERSION VARCHAR(127) NOT NULL,
VERSION_COMMENT VARCHAR(255)
-)
+);
ALTER TABLE VERSION ADD CONSTRAINT VERSION_PK PRIMARY KEY (VER_ID);
ALTER TABLE TAB_COL_STATS ADD CONSTRAINT TAB_COL_STATS_PKEY PRIMARY KEY (CS_ID);
Modified: hive/branches/branch-0.12/metastore/scripts/upgrade/oracle/upgrade-0.11.0-to-0.12.0.oracle.sql
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/metastore/scripts/upgrade/oracle/upgrade-0.11.0-to-0.12.0.oracle.sql?rev=1529047&r1=1529046&r2=1529047&view=diff
==============================================================================
--- hive/branches/branch-0.12/metastore/scripts/upgrade/oracle/upgrade-0.11.0-to-0.12.0.oracle.sql (original)
+++ hive/branches/branch-0.12/metastore/scripts/upgrade/oracle/upgrade-0.11.0-to-0.12.0.oracle.sql Fri Oct 4 00:56:49 2013
@@ -1,4 +1,4 @@
-SELECT 'Upgrading MetaStore schema from 0.11.0 to 0.12.0' AS ' ';
+SELECT 'Upgrading MetaStore schema from 0.11.0 to 0.12.0' AS Status from dual;
@013-HIVE-3255.oracle.sql;
@014-HIVE-3764.oracle.sql;
UPDATE VERSION SET SCHEMA_VERSION='0.12.0', VERSION_COMMENT='Hive release version 0.12.0' where VER_ID=1;
Modified: hive/branches/branch-0.12/metastore/scripts/upgrade/postgres/hive-schema-0.12.0.postgres.sql
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/metastore/scripts/upgrade/postgres/hive-schema-0.12.0.postgres.sql?rev=1529047&r1=1529046&r2=1529047&view=diff
==============================================================================
--- hive/branches/branch-0.12/metastore/scripts/upgrade/postgres/hive-schema-0.12.0.postgres.sql (original)
+++ hive/branches/branch-0.12/metastore/scripts/upgrade/postgres/hive-schema-0.12.0.postgres.sql Fri Oct 4 00:56:49 2013
@@ -521,7 +521,7 @@ CREATE TABLE "TAB_COL_STATS" (
CREATE TABLE "VERSION" (
"VER_ID" bigint,
"SCHEMA_VERSION" character varying(127) NOT NULL,
- "VERSION_COMMENT" character varying(255) NOT NULL,
+ "VERSION_COMMENT" character varying(255) NOT NULL
);
--
Modified: hive/branches/branch-0.12/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java?rev=1529047&r1=1529046&r2=1529047&view=diff
==============================================================================
--- hive/branches/branch-0.12/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java (original)
+++ hive/branches/branch-0.12/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java Fri Oct 4 00:56:49 2013
@@ -17,8 +17,13 @@
*/
package org.apache.hadoop.hive.metastore;
+import java.io.File;
+import java.lang.reflect.Field;
+import java.util.Random;
+
import junit.framework.TestCase;
+import org.apache.commons.io.FileUtils;
import org.apache.hadoop.hive.cli.CliSessionState;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.api.MetaException;
@@ -31,18 +36,35 @@ public class TestMetastoreVersion extend
protected HiveConf hiveConf;
private Driver driver;
private String hiveHome;
+ private String testMetastoreDB;
+ Random randomNum = new Random();
@Override
protected void setUp() throws Exception {
super.setUp();
+ Field defDb = HiveMetaStore.HMSHandler.class.getDeclaredField("createDefaultDB");
+ defDb.setAccessible(true);
+ defDb.setBoolean(null, false);
hiveConf = new HiveConf(this.getClass());
System.setProperty("hive.metastore.event.listeners",
DummyListener.class.getName());
System.setProperty("hive.metastore.pre.event.listeners",
DummyPreListener.class.getName());
+ testMetastoreDB = System.getProperty("java.io.tmpdir") +
+ File.separator + "test_metastore-" + randomNum.nextInt();
+ System.setProperty(HiveConf.ConfVars.METASTORECONNECTURLKEY.varname,
+ "jdbc:derby:" + testMetastoreDB + ";create=true");
hiveHome = System.getProperty("hive.home");
}
+ @Override
+ protected void tearDown() throws Exception {
+ File metaStoreDir = new File(testMetastoreDB);
+ if (metaStoreDir.exists()) {
+ FileUtils.deleteDirectory(metaStoreDir);
+ }
+ }
+
/***
* Test config defaults
*/