You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2018/07/03 17:47:39 UTC
[11/46] hive git commit: HIVE-19711 Refactor Hive Schema Tool (Miklos
Gergely via Alan Gates).
http://git-wip-us.apache.org/repos/asf/hive/blob/d83a0be9/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaTool.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaTool.java b/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaTool.java
deleted file mode 100644
index 3b22f15..0000000
--- a/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaTool.java
+++ /dev/null
@@ -1,801 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.beeline;
-
-import java.io.BufferedWriter;
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.io.PrintStream;
-import java.net.URI;
-import java.sql.Connection;
-import java.sql.DatabaseMetaData;
-import java.sql.SQLException;
-import java.util.Random;
-
-import junit.framework.TestCase;
-
-import org.apache.commons.dbcp.DelegatingConnection;
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.HiveMetaException;
-import org.apache.hadoop.hive.metastore.IMetaStoreSchemaInfo;
-import org.apache.hadoop.hive.metastore.MetaStoreSchemaInfoFactory;
-import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper;
-import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper.NestedScriptParser;
-import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper.PostgresCommandParser;
-import org.apache.hadoop.hive.shims.ShimLoader;
-
-public class TestSchemaTool extends TestCase {
- private HiveSchemaTool schemaTool;
- private Connection conn;
- private HiveConf hiveConf;
- private String testMetastoreDB;
- private PrintStream errStream;
- private PrintStream outStream;
-
- @Override
- protected void setUp() throws Exception {
- super.setUp();
- testMetastoreDB = System.getProperty("java.io.tmpdir") +
- File.separator + "test_metastore-" + new Random().nextInt();
- System.setProperty(HiveConf.ConfVars.METASTORECONNECTURLKEY.varname,
- "jdbc:derby:" + testMetastoreDB + ";create=true");
- hiveConf = new HiveConf(this.getClass());
- schemaTool = new HiveSchemaTool(
- System.getProperty("test.tmp.dir", "target/tmp"), hiveConf, "derby", null);
- schemaTool.setUserName(
- schemaTool.getHiveConf().get(HiveConf.ConfVars.METASTORE_CONNECTION_USER_NAME.varname));
- schemaTool.setPassWord(ShimLoader.getHadoopShims().getPassword(schemaTool.getHiveConf(),
- HiveConf.ConfVars.METASTOREPWD.varname));
- System.setProperty("beeLine.system.exit", "true");
- errStream = System.err;
- outStream = System.out;
- conn = schemaTool.getConnectionToMetastore(false);
- }
-
- @Override
- protected void tearDown() throws Exception {
- File metaStoreDir = new File(testMetastoreDB);
- if (metaStoreDir.exists()) {
- FileUtils.forceDeleteOnExit(metaStoreDir);
- }
- System.setOut(outStream);
- System.setErr(errStream);
- if (conn != null) {
- conn.close();
- }
- }
-
- /**
- * Test the sequence validation functionality
- * @throws Exception
- */
- public void testValidateSequences() throws Exception {
- schemaTool.doInit();
-
- // Test empty database
- boolean isValid = schemaTool.validateSequences(conn);
- assertTrue(isValid);
-
- // Test valid case
- String[] scripts = new String[] {
- "insert into CTLGS values(99, 'test_cat_1', 'description', 'hdfs://myhost.com:8020/user/hive/warehouse/mydb')",
- "insert into SEQUENCE_TABLE values('org.apache.hadoop.hive.metastore.model.MDatabase', 100)",
- "insert into DBS values(99, 'test db1', 'hdfs:///tmp', 'db1', 'test', 'test', 'test_cat_1')"
- };
- File scriptFile = generateTestScript(scripts);
- schemaTool.runBeeLine(scriptFile.getPath());
- isValid = schemaTool.validateSequences(conn);
- assertTrue(isValid);
-
- // Test invalid case
- scripts = new String[] {
- "delete from SEQUENCE_TABLE",
- "delete from DBS",
- "insert into SEQUENCE_TABLE values('org.apache.hadoop.hive.metastore.model.MDatabase', 100)",
- "insert into DBS values(102, 'test db1', 'hdfs:///tmp', 'db1', 'test', 'test', 'test_cat_1')"
- };
- scriptFile = generateTestScript(scripts);
- schemaTool.runBeeLine(scriptFile.getPath());
- isValid = schemaTool.validateSequences(conn);
- assertFalse(isValid);
- }
-
- /**
- * Test to validate that all tables exist in the HMS metastore.
- * @throws Exception
- */
- public void testValidateSchemaTables() throws Exception {
- schemaTool.doInit("2.0.0");
-
- boolean isValid = (boolean)schemaTool.validateSchemaTables(conn);
- assertTrue(isValid);
-
- // upgrade from 2.0.0 schema and re-validate
- schemaTool.doUpgrade("2.0.0");
- isValid = (boolean)schemaTool.validateSchemaTables(conn);
- assertTrue(isValid);
-
- // Simulate a missing table scenario by renaming a couple of tables
- String[] scripts = new String[] {
- "RENAME TABLE SEQUENCE_TABLE to SEQUENCE_TABLE_RENAMED",
- "RENAME TABLE NUCLEUS_TABLES to NUCLEUS_TABLES_RENAMED"
- };
-
- File scriptFile = generateTestScript(scripts);
- schemaTool.runBeeLine(scriptFile.getPath());
- isValid = schemaTool.validateSchemaTables(conn);
- assertFalse(isValid);
-
- // Restored the renamed tables
- scripts = new String[] {
- "RENAME TABLE SEQUENCE_TABLE_RENAMED to SEQUENCE_TABLE",
- "RENAME TABLE NUCLEUS_TABLES_RENAMED to NUCLEUS_TABLES"
- };
-
- scriptFile = generateTestScript(scripts);
- schemaTool.runBeeLine(scriptFile.getPath());
- isValid = schemaTool.validateSchemaTables(conn);
- assertTrue(isValid);
-
- // Check that an exception from getMetaData() is reported correctly
- try {
- // Make a Connection object that will throw an exception
- BadMetaDataConnection bad = new BadMetaDataConnection(conn);
- schemaTool.validateSchemaTables(bad);
- fail("did not get expected exception");
- } catch (HiveMetaException hme) {
- String message = hme.getMessage();
- assertTrue("Bad HiveMetaException message :" + message,
- message.contains("Failed to retrieve schema tables from Hive Metastore DB"));
- Throwable cause = hme.getCause();
- assertNotNull("HiveMetaException did not contain a cause", cause);
- String causeMessage = cause.getMessage();
- assertTrue("Bad SQLException message: " + causeMessage, causeMessage.contains(
- BadMetaDataConnection.FAILURE_TEXT));
- }
- }
-
- /*
- * Test the validation of incorrect NULL values in the tables
- * @throws Exception
- */
- public void testValidateNullValues() throws Exception {
- schemaTool.doInit();
-
- // Test empty database
- boolean isValid = schemaTool.validateColumnNullValues(conn);
- assertTrue(isValid);
-
- // Test valid case
- createTestHiveTableSchemas();
- isValid = schemaTool.validateColumnNullValues(conn);
-
- // Test invalid case
- String[] scripts = new String[] {
- "update TBLS set SD_ID=null"
- };
- File scriptFile = generateTestScript(scripts);
- schemaTool.runBeeLine(scriptFile.getPath());
- isValid = schemaTool.validateColumnNullValues(conn);
- assertFalse(isValid);
- }
-
- /**
- * Test dryrun of schema initialization
- * @throws Exception
- */
- public void testSchemaInitDryRun() throws Exception {
- schemaTool.setDryRun(true);
- schemaTool.doInit("0.7.0");
- schemaTool.setDryRun(false);
- try {
- schemaTool.verifySchemaVersion();
- } catch (HiveMetaException e) {
- // The connection should fail since it the dry run
- return;
- }
- fail("Dry run shouldn't create actual metastore");
- }
-
- /**
- * Test dryrun of schema upgrade
- * @throws Exception
- */
- public void testSchemaUpgradeDryRun() throws Exception {
- schemaTool.doInit("0.7.0");
-
- schemaTool.setDryRun(true);
- schemaTool.doUpgrade("0.7.0");
- schemaTool.setDryRun(false);
- try {
- schemaTool.verifySchemaVersion();
- } catch (HiveMetaException e) {
- // The connection should fail since it the dry run
- return;
- }
- fail("Dry run shouldn't upgrade metastore schema");
- }
-
- /**
- * Test schema initialization
- * @throws Exception
- */
- public void testSchemaInit() throws Exception {
- IMetaStoreSchemaInfo metastoreSchemaInfo = MetaStoreSchemaInfoFactory.get(hiveConf,
- System.getProperty("test.tmp.dir", "target/tmp"), "derby");
- schemaTool.doInit(metastoreSchemaInfo.getHiveSchemaVersion());
- schemaTool.verifySchemaVersion();
- }
-
- /**
- * Test validation for schema versions
- * @throws Exception
- */
- public void testValidateSchemaVersions() throws Exception {
- schemaTool.doInit();
- boolean isValid = schemaTool.validateSchemaVersions();
- // Test an invalid case with multiple versions
- String[] scripts = new String[] {
- "insert into VERSION values(100, '2.2.0', 'Hive release version 2.2.0')"
- };
- File scriptFile = generateTestScript(scripts);
- schemaTool.runBeeLine(scriptFile.getPath());
- isValid = schemaTool.validateSchemaVersions();
- assertFalse(isValid);
-
- scripts = new String[] {
- "delete from VERSION where VER_ID = 100"
- };
- scriptFile = generateTestScript(scripts);
- schemaTool.runBeeLine(scriptFile.getPath());
- isValid = schemaTool.validateSchemaVersions();
- assertTrue(isValid);
-
- // Test an invalid case without version
- scripts = new String[] {
- "delete from VERSION"
- };
- scriptFile = generateTestScript(scripts);
- schemaTool.runBeeLine(scriptFile.getPath());
- isValid = schemaTool.validateSchemaVersions();
- assertFalse(isValid);
- }
-
- /**
- * Test schema upgrade
- * @throws Exception
- */
- public void testSchemaUpgrade() throws Exception {
- boolean foundException = false;
- // Initialize 0.7.0 schema
- schemaTool.doInit("0.7.0");
- // verify that driver fails due to older version schema
- try {
- schemaTool.verifySchemaVersion();
- } catch (HiveMetaException e) {
- // Expected to fail due to old schema
- foundException = true;
- }
- if (!foundException) {
- throw new Exception(
- "Hive operations shouldn't pass with older version schema");
- }
-
- // Generate dummy pre-upgrade script with errors
- String invalidPreUpgradeScript = writeDummyPreUpgradeScript(
- 0, "upgrade-0.11.0-to-0.12.0.derby.sql", "foo bar;");
- // Generate dummy pre-upgrade scripts with valid SQL
- String validPreUpgradeScript0 = writeDummyPreUpgradeScript(
- 0, "upgrade-0.12.0-to-0.13.0.derby.sql",
- "CREATE TABLE schema_test0 (id integer);");
- String validPreUpgradeScript1 = writeDummyPreUpgradeScript(
- 1, "upgrade-0.12.0-to-0.13.0.derby.sql",
- "CREATE TABLE schema_test1 (id integer);");
-
- // Capture system out and err
- schemaTool.setVerbose(true);
- OutputStream stderr = new ByteArrayOutputStream();
- PrintStream errPrintStream = new PrintStream(stderr);
- System.setErr(errPrintStream);
- OutputStream stdout = new ByteArrayOutputStream();
- PrintStream outPrintStream = new PrintStream(stdout);
- System.setOut(outPrintStream);
-
- // Upgrade schema from 0.7.0 to latest
- schemaTool.doUpgrade("0.7.0");
-
- // Verify that the schemaTool ran pre-upgrade scripts and ignored errors
- assertTrue(stderr.toString().contains(invalidPreUpgradeScript));
- assertTrue(stderr.toString().contains("foo"));
- assertFalse(stderr.toString().contains(validPreUpgradeScript0));
- assertFalse(stderr.toString().contains(validPreUpgradeScript1));
- assertTrue(stdout.toString().contains(validPreUpgradeScript0));
- assertTrue(stdout.toString().contains(validPreUpgradeScript1));
-
- // Verify that driver works fine with latest schema
- schemaTool.verifySchemaVersion();
- }
-
- /**
- * Test script formatting
- * @throws Exception
- */
- public void testScripts() throws Exception {
- String testScript[] = {
- "-- this is a comment",
- "DROP TABLE IF EXISTS fooTab;",
- "/*!1234 this is comment code like mysql */;",
- "CREATE TABLE fooTab(id INTEGER);",
- "DROP TABLE footab;",
- "-- ending comment"
- };
- String resultScript[] = {
- "DROP TABLE IF EXISTS fooTab",
- "/*!1234 this is comment code like mysql */",
- "CREATE TABLE fooTab(id INTEGER)",
- "DROP TABLE footab",
- };
- String expectedSQL = StringUtils.join(resultScript, System.getProperty("line.separator")) +
- System.getProperty("line.separator");
- File testScriptFile = generateTestScript(testScript);
- String flattenedSql = HiveSchemaHelper.getDbCommandParser("derby", false)
- .buildCommand(testScriptFile.getParentFile().getPath(),
- testScriptFile.getName());
-
- assertEquals(expectedSQL, flattenedSql);
- }
-
- /**
- * Test nested script formatting
- * @throws Exception
- */
- public void testNestedScriptsForDerby() throws Exception {
- String childTab1 = "childTab1";
- String childTab2 = "childTab2";
- String parentTab = "fooTab";
-
- String childTestScript1[] = {
- "-- this is a comment ",
- "DROP TABLE IF EXISTS " + childTab1 + ";",
- "CREATE TABLE " + childTab1 + "(id INTEGER);",
- "DROP TABLE " + childTab1 + ";"
- };
- String childTestScript2[] = {
- "-- this is a comment",
- "DROP TABLE IF EXISTS " + childTab2 + ";",
- "CREATE TABLE " + childTab2 + "(id INTEGER);",
- "-- this is also a comment",
- "DROP TABLE " + childTab2 + ";"
- };
-
- String parentTestScript[] = {
- " -- this is a comment",
- "DROP TABLE IF EXISTS " + parentTab + ";",
- " -- this is another comment ",
- "CREATE TABLE " + parentTab + "(id INTEGER);",
- "RUN '" + generateTestScript(childTestScript1).getName() + "';",
- "DROP TABLE " + parentTab + ";",
- "RUN '" + generateTestScript(childTestScript2).getName() + "';",
- "--ending comment ",
- };
-
- File testScriptFile = generateTestScript(parentTestScript);
- String flattenedSql = HiveSchemaHelper.getDbCommandParser("derby", false)
- .buildCommand(testScriptFile.getParentFile().getPath(),
- testScriptFile.getName());
- assertFalse(flattenedSql.contains("RUN"));
- assertFalse(flattenedSql.contains("comment"));
- assertTrue(flattenedSql.contains(childTab1));
- assertTrue(flattenedSql.contains(childTab2));
- assertTrue(flattenedSql.contains(parentTab));
- }
-
- /**
- * Test nested script formatting
- * @throws Exception
- */
- public void testNestedScriptsForMySQL() throws Exception {
- String childTab1 = "childTab1";
- String childTab2 = "childTab2";
- String parentTab = "fooTab";
-
- String childTestScript1[] = {
- "/* this is a comment code */",
- "DROP TABLE IF EXISTS " + childTab1 + ";",
- "CREATE TABLE " + childTab1 + "(id INTEGER);",
- "DROP TABLE " + childTab1 + ";"
- };
- String childTestScript2[] = {
- "/* this is a special exec code */;",
- "DROP TABLE IF EXISTS " + childTab2 + ";",
- "CREATE TABLE " + childTab2 + "(id INTEGER);",
- "-- this is a comment",
- "DROP TABLE " + childTab2 + ";"
- };
-
- String parentTestScript[] = {
- " -- this is a comment",
- "DROP TABLE IF EXISTS " + parentTab + ";",
- " /* this is special exec code */;",
- "CREATE TABLE " + parentTab + "(id INTEGER);",
- "SOURCE " + generateTestScript(childTestScript1).getName() + ";",
- "DROP TABLE " + parentTab + ";",
- "SOURCE " + generateTestScript(childTestScript2).getName() + ";",
- "--ending comment ",
- };
-
- File testScriptFile = generateTestScript(parentTestScript);
- String flattenedSql = HiveSchemaHelper.getDbCommandParser("mysql", false)
- .buildCommand(testScriptFile.getParentFile().getPath(),
- testScriptFile.getName());
- assertFalse(flattenedSql.contains("RUN"));
- assertFalse(flattenedSql.contains("comment"));
- assertTrue(flattenedSql.contains(childTab1));
- assertTrue(flattenedSql.contains(childTab2));
- assertTrue(flattenedSql.contains(parentTab));
- }
-
- /**
- * Test script formatting
- * @throws Exception
- */
- public void testScriptWithDelimiter() throws Exception {
- String testScript[] = {
- "-- this is a comment",
- "DROP TABLE IF EXISTS fooTab;",
- "DELIMITER $$",
- "/*!1234 this is comment code like mysql */$$",
- "CREATE TABLE fooTab(id INTEGER)$$",
- "CREATE PROCEDURE fooProc()",
- "SELECT * FROM fooTab;",
- "CALL barProc();",
- "END PROCEDURE$$",
- "DELIMITER ;",
- "DROP TABLE footab;",
- "-- ending comment"
- };
- String resultScript[] = {
- "DROP TABLE IF EXISTS fooTab",
- "/*!1234 this is comment code like mysql */",
- "CREATE TABLE fooTab(id INTEGER)",
- "CREATE PROCEDURE fooProc()" + " " +
- "SELECT * FROM fooTab;" + " " +
- "CALL barProc();" + " " +
- "END PROCEDURE",
- "DROP TABLE footab",
- };
- String expectedSQL = StringUtils.join(resultScript, System.getProperty("line.separator")) +
- System.getProperty("line.separator");
- File testScriptFile = generateTestScript(testScript);
- NestedScriptParser testDbParser = HiveSchemaHelper.getDbCommandParser("mysql", false);
- String flattenedSql = testDbParser.buildCommand(testScriptFile.getParentFile().getPath(),
- testScriptFile.getName());
-
- assertEquals(expectedSQL, flattenedSql);
- }
-
- /**
- * Test script formatting
- * @throws Exception
- */
- public void testScriptMultiRowComment() throws Exception {
- String testScript[] = {
- "-- this is a comment",
- "DROP TABLE IF EXISTS fooTab;",
- "DELIMITER $$",
- "/*!1234 this is comment code like mysql */$$",
- "CREATE TABLE fooTab(id INTEGER)$$",
- "DELIMITER ;",
- "/* multiline comment started ",
- " * multiline comment continue",
- " * multiline comment ended */",
- "DROP TABLE footab;",
- "-- ending comment"
- };
- String parsedScript[] = {
- "DROP TABLE IF EXISTS fooTab",
- "/*!1234 this is comment code like mysql */",
- "CREATE TABLE fooTab(id INTEGER)",
- "DROP TABLE footab",
- };
-
- String expectedSQL = StringUtils.join(parsedScript, System.getProperty("line.separator")) +
- System.getProperty("line.separator");
- File testScriptFile = generateTestScript(testScript);
- NestedScriptParser testDbParser = HiveSchemaHelper.getDbCommandParser("mysql", false);
- String flattenedSql = testDbParser.buildCommand(testScriptFile.getParentFile().getPath(),
- testScriptFile.getName());
-
- assertEquals(expectedSQL, flattenedSql);
- }
-
- /**
- * Test nested script formatting
- * @throws Exception
- */
- public void testNestedScriptsForOracle() throws Exception {
- String childTab1 = "childTab1";
- String childTab2 = "childTab2";
- String parentTab = "fooTab";
-
- String childTestScript1[] = {
- "-- this is a comment ",
- "DROP TABLE IF EXISTS " + childTab1 + ";",
- "CREATE TABLE " + childTab1 + "(id INTEGER);",
- "DROP TABLE " + childTab1 + ";"
- };
- String childTestScript2[] = {
- "-- this is a comment",
- "DROP TABLE IF EXISTS " + childTab2 + ";",
- "CREATE TABLE " + childTab2 + "(id INTEGER);",
- "-- this is also a comment",
- "DROP TABLE " + childTab2 + ";"
- };
-
- String parentTestScript[] = {
- " -- this is a comment",
- "DROP TABLE IF EXISTS " + parentTab + ";",
- " -- this is another comment ",
- "CREATE TABLE " + parentTab + "(id INTEGER);",
- "@" + generateTestScript(childTestScript1).getName() + ";",
- "DROP TABLE " + parentTab + ";",
- "@" + generateTestScript(childTestScript2).getName() + ";",
- "--ending comment ",
- };
-
- File testScriptFile = generateTestScript(parentTestScript);
- String flattenedSql = HiveSchemaHelper.getDbCommandParser("oracle", false)
- .buildCommand(testScriptFile.getParentFile().getPath(),
- testScriptFile.getName());
- assertFalse(flattenedSql.contains("@"));
- assertFalse(flattenedSql.contains("comment"));
- assertTrue(flattenedSql.contains(childTab1));
- assertTrue(flattenedSql.contains(childTab2));
- assertTrue(flattenedSql.contains(parentTab));
- }
-
- /**
- * Test script formatting
- * @throws Exception
- */
- public void testPostgresFilter() throws Exception {
- String testScript[] = {
- "-- this is a comment",
- "DROP TABLE IF EXISTS fooTab;",
- HiveSchemaHelper.PostgresCommandParser.POSTGRES_STANDARD_STRINGS_OPT + ";",
- "CREATE TABLE fooTab(id INTEGER);",
- "DROP TABLE footab;",
- "-- ending comment"
- };
-
- String expectedScriptWithOptionPresent[] = {
- "DROP TABLE IF EXISTS fooTab",
- HiveSchemaHelper.PostgresCommandParser.POSTGRES_STANDARD_STRINGS_OPT,
- "CREATE TABLE fooTab(id INTEGER)",
- "DROP TABLE footab",
- };
-
- NestedScriptParser noDbOptParser = HiveSchemaHelper
- .getDbCommandParser("postgres", false);
- String expectedSQL = StringUtils.join(
- expectedScriptWithOptionPresent, System.getProperty("line.separator")) +
- System.getProperty("line.separator");
- File testScriptFile = generateTestScript(testScript);
- String flattenedSql = noDbOptParser.buildCommand(
- testScriptFile.getParentFile().getPath(), testScriptFile.getName());
- assertEquals(expectedSQL, flattenedSql);
-
- String expectedScriptWithOptionAbsent[] = {
- "DROP TABLE IF EXISTS fooTab",
- "CREATE TABLE fooTab(id INTEGER)",
- "DROP TABLE footab",
- };
-
- NestedScriptParser dbOptParser = HiveSchemaHelper.getDbCommandParser(
- "postgres",
- PostgresCommandParser.POSTGRES_SKIP_STANDARD_STRINGS_DBOPT,
- null, null, null, null, false);
- expectedSQL = StringUtils.join(
- expectedScriptWithOptionAbsent, System.getProperty("line.separator")) +
- System.getProperty("line.separator");
- testScriptFile = generateTestScript(testScript);
- flattenedSql = dbOptParser.buildCommand(
- testScriptFile.getParentFile().getPath(), testScriptFile.getName());
- assertEquals(expectedSQL, flattenedSql);
- }
-
- /**
- * Test validate uri of locations
- * @throws Exception
- */
- public void testValidateLocations() throws Exception {
- schemaTool.doInit();
- URI defaultRoot = new URI("hdfs://myhost.com:8020");
- URI defaultRoot2 = new URI("s3://myhost2.com:8888");
- //check empty DB
- boolean isValid = schemaTool.validateLocations(conn, null);
- assertTrue(isValid);
- isValid = schemaTool.validateLocations(conn, new URI[] {defaultRoot,defaultRoot2});
- assertTrue(isValid);
-
- // Test valid case
- String[] scripts = new String[] {
- "insert into CTLGS values(3, 'test_cat_2', 'description', 'hdfs://myhost.com:8020/user/hive/warehouse/mydb')",
- "insert into DBS values(2, 'my db', 'hdfs://myhost.com:8020/user/hive/warehouse/mydb', 'mydb', 'public', 'role', 'test_cat_2')",
- "insert into DBS values(7, 'db with bad port', 'hdfs://myhost.com:8020/', 'haDB', 'public', 'role', 'test_cat_2')",
- "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (1,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020/user/hive/warehouse/mydb',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)",
- "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (2,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020/user/admin/2015_11_18',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)",
- "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (3,null,'org.apache.hadoop.mapred.TextInputFormat','N','N',null,-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)",
- "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (4000,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020/',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)",
- "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (2 ,1435255431,2,0 ,'hive',0,1,'mytal','MANAGED_TABLE',NULL,NULL,'n')",
- "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (3 ,1435255431,2,0 ,'hive',0,3,'myView','VIRTUAL_VIEW','select a.col1,a.col2 from foo','select * from foo','n')",
- "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (4012 ,1435255431,7,0 ,'hive',0,4000,'mytal4012','MANAGED_TABLE',NULL,NULL,'n')",
- "insert into PARTITIONS(PART_ID,CREATE_TIME,LAST_ACCESS_TIME, PART_NAME,SD_ID,TBL_ID) values(1, 1441402388,0, 'd1=1/d2=1',2,2)",
- "insert into SKEWED_STRING_LIST values(1)",
- "insert into SKEWED_STRING_LIST values(2)",
- "insert into SKEWED_COL_VALUE_LOC_MAP values(1,1,'hdfs://myhost.com:8020/user/hive/warehouse/mytal/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME/')",
- "insert into SKEWED_COL_VALUE_LOC_MAP values(2,2,'s3://myhost.com:8020/user/hive/warehouse/mytal/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME/')"
- };
- File scriptFile = generateTestScript(scripts);
- schemaTool.runBeeLine(scriptFile.getPath());
- isValid = schemaTool.validateLocations(conn, null);
- assertTrue(isValid);
- isValid = schemaTool.validateLocations(conn, new URI[] {defaultRoot, defaultRoot2});
- assertTrue(isValid);
- scripts = new String[] {
- "delete from SKEWED_COL_VALUE_LOC_MAP",
- "delete from SKEWED_STRING_LIST",
- "delete from PARTITIONS",
- "delete from TBLS",
- "delete from SDS",
- "delete from DBS",
- "insert into DBS values(2, 'my db', '/user/hive/warehouse/mydb', 'mydb', 'public', 'role', 'test_cat_2')",
- "insert into DBS values(4, 'my db2', 'hdfs://myhost.com:8020', '', 'public', 'role', 'test_cat_2')",
- "insert into DBS values(6, 'db with bad port', 'hdfs://myhost.com:8020:', 'zDB', 'public', 'role', 'test_cat_2')",
- "insert into DBS values(7, 'db with bad port', 'hdfs://mynameservice.com/', 'haDB', 'public', 'role', 'test_cat_2')",
- "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (1,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://yourhost.com:8020/user/hive/warehouse/mydb',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)",
- "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (2,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','file:///user/admin/2015_11_18',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)",
- "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (2 ,1435255431,2,0 ,'hive',0,1,'mytal','MANAGED_TABLE',NULL,NULL,'n')",
- "insert into PARTITIONS(PART_ID,CREATE_TIME,LAST_ACCESS_TIME, PART_NAME,SD_ID,TBL_ID) values(1, 1441402388,0, 'd1=1/d2=1',2,2)",
- "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (3000,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','yourhost.com:8020/user/hive/warehouse/mydb',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)",
- "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (4000,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020/',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)",
- "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (4001,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)",
- "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (4003,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)",
- "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (4004,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)",
- "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (4002,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020/',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)",
- "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (5000,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','file:///user/admin/2016_11_18',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)",
- "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (3000 ,1435255431,2,0 ,'hive',0,3000,'mytal3000','MANAGED_TABLE',NULL,NULL,'n')",
- "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (4011 ,1435255431,4,0 ,'hive',0,4001,'mytal4011','MANAGED_TABLE',NULL,NULL,'n')",
- "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (4012 ,1435255431,4,0 ,'hive',0,4002,'','MANAGED_TABLE',NULL,NULL,'n')",
- "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (4013 ,1435255431,4,0 ,'hive',0,4003,'mytal4013','MANAGED_TABLE',NULL,NULL,'n')",
- "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (4014 ,1435255431,2,0 ,'hive',0,4003,'','MANAGED_TABLE',NULL,NULL,'n')",
- "insert into PARTITIONS(PART_ID,CREATE_TIME,LAST_ACCESS_TIME, PART_NAME,SD_ID,TBL_ID) values(4001, 1441402388,0, 'd1=1/d2=4001',4001,4011)",
- "insert into PARTITIONS(PART_ID,CREATE_TIME,LAST_ACCESS_TIME, PART_NAME,SD_ID,TBL_ID) values(4002, 1441402388,0, 'd1=1/d2=4002',4002,4012)",
- "insert into PARTITIONS(PART_ID,CREATE_TIME,LAST_ACCESS_TIME, PART_NAME,SD_ID,TBL_ID) values(4003, 1441402388,0, 'd1=1/d2=4003',4003,4013)",
- "insert into PARTITIONS(PART_ID,CREATE_TIME,LAST_ACCESS_TIME, PART_NAME,SD_ID,TBL_ID) values(4004, 1441402388,0, 'd1=1/d2=4004',4004,4014)",
- "insert into PARTITIONS(PART_ID,CREATE_TIME,LAST_ACCESS_TIME, PART_NAME,SD_ID,TBL_ID) values(5000, 1441402388,0, 'd1=1/d2=5000',5000,2)",
- "insert into SKEWED_STRING_LIST values(1)",
- "insert into SKEWED_STRING_LIST values(2)",
- "insert into SKEWED_COL_VALUE_LOC_MAP values(1,1,'hdfs://yourhost.com:8020/user/hive/warehouse/mytal/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME/')",
- "insert into SKEWED_COL_VALUE_LOC_MAP values(2,2,'file:///user/admin/warehouse/mytal/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME/')"
- };
- scriptFile = generateTestScript(scripts);
- schemaTool.runBeeLine(scriptFile.getPath());
- isValid = schemaTool.validateLocations(conn, null);
- assertFalse(isValid);
- isValid = schemaTool.validateLocations(conn, new URI[] {defaultRoot, defaultRoot2});
- assertFalse(isValid);
- }
-
- public void testHiveMetastoreDbPropertiesTable() throws HiveMetaException, IOException {
- schemaTool.doInit("3.0.0");
- validateMetastoreDbPropertiesTable();
- }
-
- public void testMetastoreDbPropertiesAfterUpgrade() throws HiveMetaException, IOException {
- schemaTool.doInit("2.0.0");
- schemaTool.doUpgrade();
- validateMetastoreDbPropertiesTable();
- }
-
- private File generateTestScript(String [] stmts) throws IOException {
- File testScriptFile = File.createTempFile("schematest", ".sql");
- testScriptFile.deleteOnExit();
- FileWriter fstream = new FileWriter(testScriptFile.getPath());
- BufferedWriter out = new BufferedWriter(fstream);
- for (String line: stmts) {
- out.write(line);
- out.newLine();
- }
- out.close();
- return testScriptFile;
- }
-
- private void validateMetastoreDbPropertiesTable() throws HiveMetaException, IOException {
- boolean isValid = (boolean) schemaTool.validateSchemaTables(conn);
- assertTrue(isValid);
- // adding same property key twice should throw unique key constraint violation exception
- String[] scripts = new String[] {
- "insert into METASTORE_DB_PROPERTIES values ('guid', 'test-uuid-1', 'dummy uuid 1')",
- "insert into METASTORE_DB_PROPERTIES values ('guid', 'test-uuid-2', 'dummy uuid 2')", };
- File scriptFile = generateTestScript(scripts);
- Exception ex = null;
- try {
- schemaTool.runBeeLine(scriptFile.getPath());
- } catch (Exception iox) {
- ex = iox;
- }
- assertTrue(ex != null && ex instanceof IOException);
- }
- /**
- * Write out a dummy pre-upgrade script with given SQL statement.
- */
- private String writeDummyPreUpgradeScript(int index, String upgradeScriptName,
- String sql) throws Exception {
- String preUpgradeScript = "pre-" + index + "-" + upgradeScriptName;
- String dummyPreScriptPath = System.getProperty("test.tmp.dir", "target/tmp") +
- File.separatorChar + "scripts" + File.separatorChar + "metastore" +
- File.separatorChar + "upgrade" + File.separatorChar + "derby" +
- File.separatorChar + preUpgradeScript;
- FileWriter fstream = new FileWriter(dummyPreScriptPath);
- BufferedWriter out = new BufferedWriter(fstream);
- out.write(sql + System.getProperty("line.separator") + ";");
- out.close();
- return preUpgradeScript;
- }
-
- /**
- * Insert the records in DB to simulate a hive table
- * @throws IOException
- */
- private void createTestHiveTableSchemas() throws IOException {
- String[] scripts = new String[] {
- "insert into CTLGS values(2, 'my_catalog', 'description', 'hdfs://myhost.com:8020/user/hive/warehouse/mydb')",
- "insert into DBS values(2, 'my db', 'hdfs://myhost.com:8021/user/hive/warehouse/mydb', 'mydb', 'public', 'role', 'my_catalog')",
- "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (1,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020/user/hive/warehouse/mydb',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)",
- "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (2,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020/user/admin/2015_11_18',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null)",
- "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (2 ,1435255431,2,0 ,'hive',0,1,'mytal','MANAGED_TABLE',NULL,NULL,'n')",
- "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (3 ,1435255431,2,0 ,'hive',0,2,'aTable','MANAGED_TABLE',NULL,NULL,'n')",
- "insert into PARTITIONS(PART_ID,CREATE_TIME,LAST_ACCESS_TIME, PART_NAME,SD_ID,TBL_ID) values(1, 1441402388,0, 'd1=1/d2=1',2,2)"
- };
- File scriptFile = generateTestScript(scripts);
- schemaTool.runBeeLine(scriptFile.getPath());
- }
-
- /**
- * A mock Connection class that throws an exception out of getMetaData().
- */
- class BadMetaDataConnection extends DelegatingConnection {
- static final String FAILURE_TEXT = "fault injected";
-
- BadMetaDataConnection(Connection connection) {
- super(connection);
- }
-
- @Override
- public DatabaseMetaData getMetaData() throws SQLException {
- throw new SQLException(FAILURE_TEXT);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/d83a0be9/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaToolCatalogOps.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaToolCatalogOps.java b/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaToolCatalogOps.java
deleted file mode 100644
index f5bc570..0000000
--- a/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaToolCatalogOps.java
+++ /dev/null
@@ -1,417 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hive.beeline;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.HiveMetaException;
-import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
-import org.apache.hadoop.hive.metastore.IMetaStoreClient;
-import org.apache.hadoop.hive.metastore.api.Catalog;
-import org.apache.hadoop.hive.metastore.api.Database;
-import org.apache.hadoop.hive.metastore.api.Function;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.Partition;
-import org.apache.hadoop.hive.metastore.api.Table;
-import org.apache.hadoop.hive.metastore.client.builder.CatalogBuilder;
-import org.apache.hadoop.hive.metastore.client.builder.DatabaseBuilder;
-import org.apache.hadoop.hive.metastore.client.builder.FunctionBuilder;
-import org.apache.hadoop.hive.metastore.client.builder.PartitionBuilder;
-import org.apache.hadoop.hive.metastore.client.builder.TableBuilder;
-import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
-import org.apache.thrift.TException;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.Set;
-
-import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_CATALOG_NAME;
-import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_DATABASE_NAME;
-
-public class TestSchemaToolCatalogOps {
- private static final Logger LOG = LoggerFactory.getLogger(TestSchemaToolCatalogOps.class);
- private static HiveSchemaTool schemaTool;
- private static HiveConf conf;
- private IMetaStoreClient client;
- private static String testMetastoreDB;
-
- @BeforeClass
- public static void initDb() throws HiveMetaException, IOException {
- conf = new HiveConf();
- MetastoreConf.setBoolVar(conf, MetastoreConf.ConfVars.AUTO_CREATE_ALL, false);
- MetastoreConf.setLongVar(conf, MetastoreConf.ConfVars.HMS_HANDLER_ATTEMPTS, 1);
- MetastoreConf.setLongVar(conf, MetastoreConf.ConfVars.THRIFT_CONNECTION_RETRIES, 1);
- testMetastoreDB = System.getProperty("java.io.tmpdir") +
- File.separator + "testschematoolcatopsdb";
- MetastoreConf.setVar(conf, MetastoreConf.ConfVars.CONNECT_URL_KEY,
- "jdbc:derby:" + testMetastoreDB + ";create=true");
- schemaTool = new HiveSchemaTool(
- System.getProperty("test.tmp.dir", "target/tmp"), conf, "derby", null);
- schemaTool.setUserName(MetastoreConf.getVar(conf, MetastoreConf.ConfVars.CONNECTION_USER_NAME));
- schemaTool.setPassWord(MetastoreConf.getPassword(conf, MetastoreConf.ConfVars.PWD));
- schemaTool.doInit(); // Pre-install the database so all the tables are there.
-
- }
-
- @AfterClass
- public static void removeDb() throws Exception {
- File metaStoreDir = new File(testMetastoreDB);
- if (metaStoreDir.exists()) {
- FileUtils.forceDeleteOnExit(metaStoreDir);
- }
- }
-
- @Before
- public void createClient() throws MetaException {
- client = new HiveMetaStoreClient(conf);
- }
-
- @Test
- public void createCatalog() throws HiveMetaException, TException {
- String catName = "my_test_catalog";
- String location = "file:///tmp/my_test_catalog";
- String description = "very descriptive";
- schemaTool.createCatalog(catName, location, description, false);
-
- Catalog cat = client.getCatalog(catName);
- Assert.assertEquals(location, cat.getLocationUri());
- Assert.assertEquals(description, cat.getDescription());
- }
-
- @Test(expected = HiveMetaException.class)
- public void createExistingCatalog() throws HiveMetaException {
- schemaTool.createCatalog("hive", "somewhere", "", false);
- }
-
- @Test
- public void createExistingCatalogWithIfNotExists() throws HiveMetaException {
- String catName = "my_existing_test_catalog";
- String location = "file:///tmp/my_test_catalog";
- String description = "very descriptive";
- schemaTool.createCatalog(catName, location, description, false);
-
- schemaTool.createCatalog(catName, location, description, true);
- }
-
- @Test
- public void alterCatalog() throws HiveMetaException, TException {
- String catName = "an_alterable_catalog";
- String location = "file:///tmp/an_alterable_catalog";
- String description = "description";
- schemaTool.createCatalog(catName, location, description, false);
-
- location = "file:///tmp/somewhere_else";
- schemaTool.alterCatalog(catName, location, null);
- Catalog cat = client.getCatalog(catName);
- Assert.assertEquals(location, cat.getLocationUri());
- Assert.assertEquals(description, cat.getDescription());
-
- description = "a better description";
- schemaTool.alterCatalog(catName, null, description);
- cat = client.getCatalog(catName);
- Assert.assertEquals(location, cat.getLocationUri());
- Assert.assertEquals(description, cat.getDescription());
-
- location = "file:///tmp/a_third_location";
- description = "best description yet";
- schemaTool.alterCatalog(catName, location, description);
- cat = client.getCatalog(catName);
- Assert.assertEquals(location, cat.getLocationUri());
- Assert.assertEquals(description, cat.getDescription());
- }
-
- @Test(expected = HiveMetaException.class)
- public void alterBogusCatalog() throws HiveMetaException {
- schemaTool.alterCatalog("nosuch", "file:///tmp/somewhere", "whatever");
- }
-
- @Test(expected = HiveMetaException.class)
- public void alterCatalogNoChange() throws HiveMetaException {
- String catName = "alter_cat_no_change";
- String location = "file:///tmp/alter_cat_no_change";
- String description = "description";
- schemaTool.createCatalog(catName, location, description, false);
-
- schemaTool.alterCatalog(catName, null, null);
- }
-
- @Test
- public void moveDatabase() throws HiveMetaException, TException {
- String toCatName = "moveDbCat";
- String dbName = "moveDbDb";
- String tableName = "moveDbTable";
- String funcName = "movedbfunc";
- String partVal = "moveDbKey";
-
- new CatalogBuilder()
- .setName(toCatName)
- .setLocation("file:///tmp")
- .create(client);
-
- Database db = new DatabaseBuilder()
- .setCatalogName(DEFAULT_CATALOG_NAME)
- .setName(dbName)
- .create(client, conf);
-
- new FunctionBuilder()
- .inDb(db)
- .setName(funcName)
- .setClass("org.apache.hive.myudf")
- .create(client, conf);
-
- Table table = new TableBuilder()
- .inDb(db)
- .setTableName(tableName)
- .addCol("a", "int")
- .addPartCol("p", "string")
- .create(client, conf);
-
- new PartitionBuilder()
- .inTable(table)
- .addValue(partVal)
- .addToTable(client, conf);
-
- schemaTool.moveDatabase(DEFAULT_CATALOG_NAME, toCatName, dbName);
-
- Database fetchedDb = client.getDatabase(toCatName, dbName);
- Assert.assertNotNull(fetchedDb);
- Assert.assertEquals(toCatName.toLowerCase(), fetchedDb.getCatalogName());
-
- Function fetchedFunction = client.getFunction(toCatName, dbName, funcName);
- Assert.assertNotNull(fetchedFunction);
- Assert.assertEquals(toCatName.toLowerCase(), fetchedFunction.getCatName());
- Assert.assertEquals(dbName.toLowerCase(), fetchedFunction.getDbName());
-
- Table fetchedTable = client.getTable(toCatName, dbName, tableName);
- Assert.assertNotNull(fetchedTable);
- Assert.assertEquals(toCatName.toLowerCase(), fetchedTable.getCatName());
- Assert.assertEquals(dbName.toLowerCase(), fetchedTable.getDbName());
-
- Partition fetchedPart =
- client.getPartition(toCatName, dbName, tableName, Collections.singletonList(partVal));
- Assert.assertNotNull(fetchedPart);
- Assert.assertEquals(toCatName.toLowerCase(), fetchedPart.getCatName());
- Assert.assertEquals(dbName.toLowerCase(), fetchedPart.getDbName());
- Assert.assertEquals(tableName.toLowerCase(), fetchedPart.getTableName());
- }
-
- @Test
- public void moveDatabaseWithExistingDbOfSameNameAlreadyInTargetCatalog()
- throws TException, HiveMetaException {
- String catName = "clobberCatalog";
- new CatalogBuilder()
- .setName(catName)
- .setLocation("file:///tmp")
- .create(client);
- try {
- schemaTool.moveDatabase(catName, DEFAULT_CATALOG_NAME, DEFAULT_DATABASE_NAME);
- Assert.fail("Attempt to move default database should have failed.");
- } catch (HiveMetaException e) {
- // good
- }
-
- // Make sure nothing really moved
- Set<String> dbNames = new HashSet<>(client.getAllDatabases(DEFAULT_CATALOG_NAME));
- Assert.assertTrue(dbNames.contains(DEFAULT_DATABASE_NAME));
- }
-
- @Test(expected = HiveMetaException.class)
- public void moveNonExistentDatabase() throws TException, HiveMetaException {
- String catName = "moveNonExistentDb";
- new CatalogBuilder()
- .setName(catName)
- .setLocation("file:///tmp")
- .create(client);
- schemaTool.moveDatabase(catName, DEFAULT_CATALOG_NAME, "nosuch");
- }
-
- @Test
- public void moveDbToNonExistentCatalog() throws TException, HiveMetaException {
- String dbName = "doomedToHomelessness";
- new DatabaseBuilder()
- .setName(dbName)
- .create(client, conf);
- try {
- schemaTool.moveDatabase(DEFAULT_CATALOG_NAME, "nosuch", dbName);
- Assert.fail("Attempt to move database to non-existent catalog should have failed.");
- } catch (HiveMetaException e) {
- // good
- }
-
- // Make sure nothing really moved
- Set<String> dbNames = new HashSet<>(client.getAllDatabases(DEFAULT_CATALOG_NAME));
- Assert.assertTrue(dbNames.contains(dbName.toLowerCase()));
- }
-
- @Test
- public void moveTable() throws TException, HiveMetaException {
- String toCatName = "moveTableCat";
- String toDbName = "moveTableDb";
- String tableName = "moveTableTable";
- String partVal = "moveTableKey";
-
- new CatalogBuilder()
- .setName(toCatName)
- .setLocation("file:///tmp")
- .create(client);
-
- new DatabaseBuilder()
- .setCatalogName(toCatName)
- .setName(toDbName)
- .create(client, conf);
-
- Table table = new TableBuilder()
- .setTableName(tableName)
- .addCol("a", "int")
- .addPartCol("p", "string")
- .create(client, conf);
-
- new PartitionBuilder()
- .inTable(table)
- .addValue(partVal)
- .addToTable(client, conf);
-
- schemaTool.moveTable(DEFAULT_CATALOG_NAME, toCatName, DEFAULT_DATABASE_NAME, toDbName, tableName);
-
- Table fetchedTable = client.getTable(toCatName, toDbName, tableName);
- Assert.assertNotNull(fetchedTable);
- Assert.assertEquals(toCatName.toLowerCase(), fetchedTable.getCatName());
- Assert.assertEquals(toDbName.toLowerCase(), fetchedTable.getDbName());
-
- Partition fetchedPart =
- client.getPartition(toCatName, toDbName, tableName, Collections.singletonList(partVal));
- Assert.assertNotNull(fetchedPart);
- Assert.assertEquals(toCatName.toLowerCase(), fetchedPart.getCatName());
- Assert.assertEquals(toDbName.toLowerCase(), fetchedPart.getDbName());
- Assert.assertEquals(tableName.toLowerCase(), fetchedPart.getTableName());
- }
-
- @Test
- public void moveTableWithinCatalog() throws TException, HiveMetaException {
- String toDbName = "moveTableWithinCatalogDb";
- String tableName = "moveTableWithinCatalogTable";
- String partVal = "moveTableWithinCatalogKey";
-
- new DatabaseBuilder()
- .setName(toDbName)
- .create(client, conf);
-
- Table table = new TableBuilder()
- .setTableName(tableName)
- .addCol("a", "int")
- .addPartCol("p", "string")
- .create(client, conf);
-
- new PartitionBuilder()
- .inTable(table)
- .addValue(partVal)
- .addToTable(client, conf);
-
- schemaTool.moveTable(DEFAULT_CATALOG_NAME, DEFAULT_CATALOG_NAME, DEFAULT_DATABASE_NAME, toDbName, tableName);
-
- Table fetchedTable = client.getTable(DEFAULT_CATALOG_NAME, toDbName, tableName);
- Assert.assertNotNull(fetchedTable);
- Assert.assertEquals(DEFAULT_CATALOG_NAME, fetchedTable.getCatName());
- Assert.assertEquals(toDbName.toLowerCase(), fetchedTable.getDbName());
-
- Partition fetchedPart =
- client.getPartition(DEFAULT_CATALOG_NAME, toDbName, tableName, Collections.singletonList(partVal));
- Assert.assertNotNull(fetchedPart);
- Assert.assertEquals(DEFAULT_CATALOG_NAME, fetchedPart.getCatName());
- Assert.assertEquals(toDbName.toLowerCase(), fetchedPart.getDbName());
- Assert.assertEquals(tableName.toLowerCase(), fetchedPart.getTableName());
- }
-
- @Test
- public void moveTableWithExistingTableOfSameNameAlreadyInTargetDatabase()
- throws TException, HiveMetaException {
- String toDbName = "clobberTableDb";
- String tableName = "clobberTableTable";
-
- Database toDb = new DatabaseBuilder()
- .setName(toDbName)
- .create(client, conf);
-
- new TableBuilder()
- .setTableName(tableName)
- .addCol("a", "int")
- .create(client, conf);
-
- new TableBuilder()
- .inDb(toDb)
- .setTableName(tableName)
- .addCol("b", "varchar(32)")
- .create(client, conf);
-
- try {
- schemaTool.moveTable(DEFAULT_CATALOG_NAME, DEFAULT_CATALOG_NAME, DEFAULT_DATABASE_NAME,
- toDbName, tableName);
- Assert.fail("Attempt to move table should have failed.");
- } catch (HiveMetaException e) {
- // good
- }
-
- // Make sure nothing really moved
- Set<String> tableNames = new HashSet<>(client.getAllTables(DEFAULT_CATALOG_NAME, DEFAULT_DATABASE_NAME));
- Assert.assertTrue(tableNames.contains(tableName.toLowerCase()));
-
- // Make sure the table in the target database didn't get clobbered
- Table fetchedTable = client.getTable(DEFAULT_CATALOG_NAME, toDbName, tableName);
- Assert.assertEquals("b", fetchedTable.getSd().getCols().get(0).getName());
- }
-
- @Test(expected = HiveMetaException.class)
- public void moveNonExistentTable() throws TException, HiveMetaException {
- String toDbName = "moveNonExistentTable";
- new DatabaseBuilder()
- .setName(toDbName)
- .create(client, conf);
- schemaTool.moveTable(DEFAULT_CATALOG_NAME, DEFAULT_CATALOG_NAME, DEFAULT_DATABASE_NAME, toDbName,
- "nosuch");
- }
-
- @Test
- public void moveTableToNonExistentDb() throws TException, HiveMetaException {
- String tableName = "doomedToWander";
- new TableBuilder()
- .setTableName(tableName)
- .addCol("a", "int")
- .create(client, conf);
-
- try {
- schemaTool.moveTable(DEFAULT_CATALOG_NAME, DEFAULT_CATALOG_NAME, DEFAULT_DATABASE_NAME,
- "nosuch", tableName);
- Assert.fail("Attempt to move table to non-existent table should have failed.");
- } catch (HiveMetaException e) {
- // good
- }
-
- // Make sure nothing really moved
- Set<String> tableNames = new HashSet<>(client.getAllTables(DEFAULT_CATALOG_NAME, DEFAULT_DATABASE_NAME));
- Assert.assertTrue(tableNames.contains(tableName.toLowerCase()));
- }
-}