You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2013/10/03 00:24:32 UTC
svn commit: r1528655 - in /hive/trunk:
beeline/src/java/org/apache/hive/beeline/
beeline/src/test/org/apache/hive/beeline/src/test/
metastore/scripts/upgrade/derby/ metastore/scripts/upgrade/oracle/
metastore/scripts/upgrade/postgres/ metastore/src/tes...
Author: hashutosh
Date: Wed Oct 2 22:24:32 2013
New Revision: 1528655
URL: http://svn.apache.org/r1528655
Log:
HIVE-5419 : Fix schema tool issues with Oracle metastore (Prasad Mujumdar via Ashutosh Chauhan)
Modified:
hive/trunk/beeline/src/java/org/apache/hive/beeline/HiveSchemaHelper.java
hive/trunk/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java
hive/trunk/beeline/src/test/org/apache/hive/beeline/src/test/TestSchemaTool.java
hive/trunk/metastore/scripts/upgrade/derby/hive-schema-0.13.0.derby.sql
hive/trunk/metastore/scripts/upgrade/oracle/014-HIVE-3764.oracle.sql
hive/trunk/metastore/scripts/upgrade/oracle/hive-schema-0.12.0.oracle.sql
hive/trunk/metastore/scripts/upgrade/oracle/hive-schema-0.13.0.oracle.sql
hive/trunk/metastore/scripts/upgrade/oracle/upgrade-0.11.0-to-0.12.0.oracle.sql
hive/trunk/metastore/scripts/upgrade/oracle/upgrade-0.12.0-to-0.13.0.oracle.sql
hive/trunk/metastore/scripts/upgrade/postgres/hive-schema-0.12.0.postgres.sql
hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java
Modified: hive/trunk/beeline/src/java/org/apache/hive/beeline/HiveSchemaHelper.java
URL: http://svn.apache.org/viewvc/hive/trunk/beeline/src/java/org/apache/hive/beeline/HiveSchemaHelper.java?rev=1528655&r1=1528654&r2=1528655&view=diff
==============================================================================
--- hive/trunk/beeline/src/java/org/apache/hive/beeline/HiveSchemaHelper.java (original)
+++ hive/trunk/beeline/src/java/org/apache/hive/beeline/HiveSchemaHelper.java Wed Oct 2 22:24:32 2013
@@ -22,7 +22,7 @@ import java.util.IllegalFormatException;
public class HiveSchemaHelper {
public static final String DB_DERBY = "derby";
public static final String DB_MYSQL = "mysql";
- public static final String DB_POSTGRACE = "postgrace";
+ public static final String DB_POSTGRACE = "postgres";
public static final String DB_ORACLE = "oracle";
public interface NestedScriptParser {
@@ -225,8 +225,8 @@ public class HiveSchemaHelper {
if (!isNestedScript(dbCommand)) {
throw new IllegalArgumentException("Not a nested script format " + dbCommand);
}
- // remove ending ';'
- return dbCommand.replace(";", "");
+ // remove ending ';' and starting '@'
+ return dbCommand.replace(";", "").replace(ORACLE_NESTING_TOKEN, "");
}
@Override
Modified: hive/trunk/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java
URL: http://svn.apache.org/viewvc/hive/trunk/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java?rev=1528655&r1=1528654&r2=1528655&view=diff
==============================================================================
--- hive/trunk/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java (original)
+++ hive/trunk/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java Wed Oct 2 22:24:32 2013
@@ -366,6 +366,7 @@ public class HiveSchemaTool {
beeLine.getOpts().setSilent(true);
}
beeLine.getOpts().setAllowMultiLineCommand(false);
+ beeLine.getOpts().setIsolation("TRANSACTION_READ_COMMITTED");
int status = beeLine.begin(argList.toArray(new String[0]), null);
if (status != 0) {
throw new IOException("Schema script failed, errorcode " + status);
Modified: hive/trunk/beeline/src/test/org/apache/hive/beeline/src/test/TestSchemaTool.java
URL: http://svn.apache.org/viewvc/hive/trunk/beeline/src/test/org/apache/hive/beeline/src/test/TestSchemaTool.java?rev=1528655&r1=1528654&r2=1528655&view=diff
==============================================================================
--- hive/trunk/beeline/src/test/org/apache/hive/beeline/src/test/TestSchemaTool.java (original)
+++ hive/trunk/beeline/src/test/org/apache/hive/beeline/src/test/TestSchemaTool.java Wed Oct 2 22:24:32 2013
@@ -30,6 +30,7 @@ import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.HiveMetaException;
+import org.apache.hadoop.hive.metastore.MetaStoreSchemaInfo;
import org.apache.hive.beeline.HiveSchemaHelper;
import org.apache.hive.beeline.HiveSchemaHelper.NestedScriptParser;
import org.apache.hive.beeline.HiveSchemaTool;
@@ -100,7 +101,8 @@ public class TestSchemaTool extends Test
* @throws Exception
*/
public void testSchemaInit() throws Exception {
- schemaTool.doInit("0.12.0");
+ schemaTool.doInit(MetaStoreSchemaInfo.getHiveSchemaVersion());
+ schemaTool.verifySchemaVersion();
}
/**
@@ -321,6 +323,51 @@ public class TestSchemaTool extends Test
assertEquals(expectedSQL, flattenedSql);
}
+ /**
+ * Test nested script formatting
+ * @throws Exception
+ */
+ public void testNestedScriptsForOracle() throws Exception {
+ String childTab1 = "childTab1";
+ String childTab2 = "childTab2";
+ String parentTab = "fooTab";
+
+ String childTestScript1[] = {
+ "-- this is a comment ",
+ "DROP TABLE IF EXISTS " + childTab1 + ";",
+ "CREATE TABLE " + childTab1 + "(id INTEGER);",
+ "DROP TABLE " + childTab1 + ";"
+ };
+ String childTestScript2[] = {
+ "-- this is a comment",
+ "DROP TABLE IF EXISTS " + childTab2 + ";",
+ "CREATE TABLE " + childTab2 + "(id INTEGER);",
+ "-- this is also a comment",
+ "DROP TABLE " + childTab2 + ";"
+ };
+
+ String parentTestScript[] = {
+ " -- this is a comment",
+ "DROP TABLE IF EXISTS " + parentTab + ";",
+ " -- this is another comment ",
+ "CREATE TABLE " + parentTab + "(id INTEGER);",
+ "@" + generateTestScript(childTestScript1).getName() + ";",
+ "DROP TABLE " + parentTab + ";",
+ "@" + generateTestScript(childTestScript2).getName() + ";",
+ "--ending comment ",
+ };
+
+ File testScriptFile = generateTestScript(parentTestScript);
+ String flattenedSql = HiveSchemaTool.buildCommand(
+ HiveSchemaHelper.getDbCommandParser("oracle"),
+ testScriptFile.getParentFile().getPath(), testScriptFile.getName());
+ assertFalse(flattenedSql.contains("@"));
+ assertFalse(flattenedSql.contains("comment"));
+ assertTrue(flattenedSql.contains(childTab1));
+ assertTrue(flattenedSql.contains(childTab2));
+ assertTrue(flattenedSql.contains(parentTab));
+ }
+
private File generateTestScript(String [] stmts) throws IOException {
File testScriptFile = File.createTempFile("schematest", ".sql");
testScriptFile.deleteOnExit();
Modified: hive/trunk/metastore/scripts/upgrade/derby/hive-schema-0.13.0.derby.sql
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/scripts/upgrade/derby/hive-schema-0.13.0.derby.sql?rev=1528655&r1=1528654&r2=1528655&view=diff
==============================================================================
--- hive/trunk/metastore/scripts/upgrade/derby/hive-schema-0.13.0.derby.sql (original)
+++ hive/trunk/metastore/scripts/upgrade/derby/hive-schema-0.13.0.derby.sql Wed Oct 2 22:24:32 2013
@@ -296,4 +296,4 @@ ALTER TABLE "APP"."IDXS" ADD CONSTRAINT
ALTER TABLE "APP"."SDS" ADD CONSTRAINT "SQL110318025505550" CHECK (IS_COMPRESSED IN ('Y','N'));
-INSERT INTO "APP"."VERSION" (VER_ID, SCHEMA_VERSION, COMMENT) VALUES (1, '0.13.0', 'Hive release version 0.13.0');
+INSERT INTO "APP"."VERSION" (VER_ID, SCHEMA_VERSION, VERSION_COMMENT) VALUES (1, '0.13.0', 'Hive release version 0.13.0');
Modified: hive/trunk/metastore/scripts/upgrade/oracle/014-HIVE-3764.oracle.sql
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/scripts/upgrade/oracle/014-HIVE-3764.oracle.sql?rev=1528655&r1=1528654&r2=1528655&view=diff
==============================================================================
--- hive/trunk/metastore/scripts/upgrade/oracle/014-HIVE-3764.oracle.sql (original)
+++ hive/trunk/metastore/scripts/upgrade/oracle/014-HIVE-3764.oracle.sql Wed Oct 2 22:24:32 2013
@@ -1,10 +1,10 @@
-- HIVE-3764 Support metastore version consistency check
-CREATE TABLE IF NOT EXISTS VERSION (
+CREATE TABLE VERSION (
VER_ID NUMBER NOT NULL,
SCHEMA_VERSION VARCHAR(127) NOT NULL,
VERSION_COMMENT VARCHAR(255)
-)
+);
ALTER TABLE VERSION ADD CONSTRAINT VERSION_PK PRIMARY KEY (VER_ID);
-INSERT INTO VERSION (VER_ID, SCHEMA_VERSION, VERSION_COMMENT) VALUES (1, '', 'Initial value');
+INSERT INTO VERSION (VER_ID, SCHEMA_VERSION, VERSION_COMMENT) VALUES (1, ' ', 'Initial value');
Modified: hive/trunk/metastore/scripts/upgrade/oracle/hive-schema-0.12.0.oracle.sql
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/scripts/upgrade/oracle/hive-schema-0.12.0.oracle.sql?rev=1528655&r1=1528654&r2=1528655&view=diff
==============================================================================
--- hive/trunk/metastore/scripts/upgrade/oracle/hive-schema-0.12.0.oracle.sql (original)
+++ hive/trunk/metastore/scripts/upgrade/oracle/hive-schema-0.12.0.oracle.sql Wed Oct 2 22:24:32 2013
@@ -483,11 +483,11 @@ CREATE TABLE TAB_COL_STATS (
LAST_ANALYZED NUMBER NOT NULL
);
-CREATE TABLE IF NOT EXISTS VERSION (
+CREATE TABLE VERSION (
VER_ID NUMBER NOT NULL,
SCHEMA_VERSION VARCHAR(127) NOT NULL,
VERSION_COMMENT VARCHAR(255)
-)
+);
ALTER TABLE VERSION ADD CONSTRAINT VERSION_PK PRIMARY KEY (VER_ID);
ALTER TABLE TAB_COL_STATS ADD CONSTRAINT TAB_COL_STATS_PKEY PRIMARY KEY (CS_ID);
Modified: hive/trunk/metastore/scripts/upgrade/oracle/hive-schema-0.13.0.oracle.sql
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/scripts/upgrade/oracle/hive-schema-0.13.0.oracle.sql?rev=1528655&r1=1528654&r2=1528655&view=diff
==============================================================================
--- hive/trunk/metastore/scripts/upgrade/oracle/hive-schema-0.13.0.oracle.sql (original)
+++ hive/trunk/metastore/scripts/upgrade/oracle/hive-schema-0.13.0.oracle.sql Wed Oct 2 22:24:32 2013
@@ -483,11 +483,11 @@ CREATE TABLE TAB_COL_STATS (
LAST_ANALYZED NUMBER NOT NULL
);
-CREATE TABLE IF NOT EXISTS VERSION (
+CREATE TABLE VERSION (
VER_ID NUMBER NOT NULL,
SCHEMA_VERSION VARCHAR(127) NOT NULL,
- COMMENT VARCHAR(255)
-)
+ VERSION_COMMENT VARCHAR(255)
+);
ALTER TABLE VERSION ADD CONSTRAINT VERSION_PK PRIMARY KEY (VER_ID);
ALTER TABLE TAB_COL_STATS ADD CONSTRAINT TAB_COL_STATS_PKEY PRIMARY KEY (CS_ID);
Modified: hive/trunk/metastore/scripts/upgrade/oracle/upgrade-0.11.0-to-0.12.0.oracle.sql
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/scripts/upgrade/oracle/upgrade-0.11.0-to-0.12.0.oracle.sql?rev=1528655&r1=1528654&r2=1528655&view=diff
==============================================================================
--- hive/trunk/metastore/scripts/upgrade/oracle/upgrade-0.11.0-to-0.12.0.oracle.sql (original)
+++ hive/trunk/metastore/scripts/upgrade/oracle/upgrade-0.11.0-to-0.12.0.oracle.sql Wed Oct 2 22:24:32 2013
@@ -1,4 +1,4 @@
-SELECT 'Upgrading MetaStore schema from 0.11.0 to 0.12.0' AS ' ';
+SELECT 'Upgrading MetaStore schema from 0.11.0 to 0.12.0' AS Status from dual;
@013-HIVE-3255.oracle.sql;
@014-HIVE-3764.oracle.sql;
UPDATE VERSION SET SCHEMA_VERSION='0.12.0', VERSION_COMMENT='Hive release version 0.12.0' where VER_ID=1;
Modified: hive/trunk/metastore/scripts/upgrade/oracle/upgrade-0.12.0-to-0.13.0.oracle.sql
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/scripts/upgrade/oracle/upgrade-0.12.0-to-0.13.0.oracle.sql?rev=1528655&r1=1528654&r2=1528655&view=diff
==============================================================================
--- hive/trunk/metastore/scripts/upgrade/oracle/upgrade-0.12.0-to-0.13.0.oracle.sql (original)
+++ hive/trunk/metastore/scripts/upgrade/oracle/upgrade-0.12.0-to-0.13.0.oracle.sql Wed Oct 2 22:24:32 2013
@@ -1,3 +1,3 @@
-SELECT 'Upgrading MetaStore schema from 0.11.0 to 0.12.0' AS ' ';
+SELECT 'Upgrading MetaStore schema from 0.11.0 to 0.12.0' AS Status from dual;
UPDATE VERSION SET SCHEMA_VERSION='0.13.0', VERSION_COMMENT='Hive release version 0.13.0' where VER_ID=1;
SELECT 'Finished upgrading MetaStore schema from 0.11.0 to 0.12.0' AS Status from dual;
Modified: hive/trunk/metastore/scripts/upgrade/postgres/hive-schema-0.12.0.postgres.sql
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/scripts/upgrade/postgres/hive-schema-0.12.0.postgres.sql?rev=1528655&r1=1528654&r2=1528655&view=diff
==============================================================================
--- hive/trunk/metastore/scripts/upgrade/postgres/hive-schema-0.12.0.postgres.sql (original)
+++ hive/trunk/metastore/scripts/upgrade/postgres/hive-schema-0.12.0.postgres.sql Wed Oct 2 22:24:32 2013
@@ -521,7 +521,7 @@ CREATE TABLE "TAB_COL_STATS" (
CREATE TABLE "VERSION" (
"VER_ID" bigint,
"SCHEMA_VERSION" character varying(127) NOT NULL,
- "VERSION_COMMENT" character varying(255) NOT NULL,
+ "VERSION_COMMENT" character varying(255) NOT NULL
);
--
Modified: hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java?rev=1528655&r1=1528654&r2=1528655&view=diff
==============================================================================
--- hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java (original)
+++ hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java Wed Oct 2 22:24:32 2013
@@ -17,8 +17,13 @@
*/
package org.apache.hadoop.hive.metastore;
+import java.io.File;
+import java.lang.reflect.Field;
+import java.util.Random;
+
import junit.framework.TestCase;
+import org.apache.commons.io.FileUtils;
import org.apache.hadoop.hive.cli.CliSessionState;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.api.MetaException;
@@ -31,18 +36,35 @@ public class TestMetastoreVersion extend
protected HiveConf hiveConf;
private Driver driver;
private String hiveHome;
+ private String testMetastoreDB;
+ Random randomNum = new Random();
@Override
protected void setUp() throws Exception {
super.setUp();
+ Field defDb = HiveMetaStore.HMSHandler.class.getDeclaredField("createDefaultDB");
+ defDb.setAccessible(true);
+ defDb.setBoolean(null, false);
hiveConf = new HiveConf(this.getClass());
System.setProperty("hive.metastore.event.listeners",
DummyListener.class.getName());
System.setProperty("hive.metastore.pre.event.listeners",
DummyPreListener.class.getName());
+ testMetastoreDB = System.getProperty("java.io.tmpdir") +
+ File.separator + "test_metastore-" + randomNum.nextInt();
+ System.setProperty(HiveConf.ConfVars.METASTORECONNECTURLKEY.varname,
+ "jdbc:derby:" + testMetastoreDB + ";create=true");
hiveHome = System.getProperty("hive.home");
}
+ @Override
+ protected void tearDown() throws Exception {
+ File metaStoreDir = new File(testMetastoreDB);
+ if (metaStoreDir.exists()) {
+ FileUtils.deleteDirectory(metaStoreDir);
+ }
+ }
+
/***
* Test config defaults
*/