You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ai...@apache.org on 2016/12/09 15:24:00 UTC
hive git commit: HIVE-15392: Refactoring the validate function of
HiveSchemaTool to make the output consistent (Aihua Xu,
reviewed by Chaoyu Tang)
Repository: hive
Updated Branches:
refs/heads/master 24f48f124 -> 999dc553e
HIVE-15392: Refactoring the validate function of HiveSchemaTool to make the output consistent (Aihua Xu, reviewed by Chaoyu Tang)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/999dc553
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/999dc553
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/999dc553
Branch: refs/heads/master
Commit: 999dc553e17e12a95540ac67dc08cbf7c2917fa2
Parents: 24f48f1
Author: Aihua Xu <ai...@apache.org>
Authored: Thu Dec 8 13:14:02 2016 -0500
Committer: Aihua Xu <ai...@apache.org>
Committed: Fri Dec 9 10:23:36 2016 -0500
----------------------------------------------------------------------
.../org/apache/hive/beeline/HiveSchemaTool.java | 161 +++++++------------
.../org/apache/hive/beeline/TestSchemaTool.java | 47 +++---
2 files changed, 81 insertions(+), 127 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/999dc553/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java
----------------------------------------------------------------------
diff --git a/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java b/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java
index 88b6b2b..32a7b32 100644
--- a/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java
+++ b/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java
@@ -127,7 +127,7 @@ public class HiveSchemaTool {
System.exit(1);
}
- private Connection getConnectionToMetastore(boolean printInfo)
+ Connection getConnectionToMetastore(boolean printInfo)
throws HiveMetaException {
return HiveSchemaHelper.getConnectionToMetastore(userName,
passWord, printInfo, hiveConf);
@@ -179,21 +179,15 @@ public class HiveSchemaTool {
} catch (SQLException e) {
throw new HiveMetaException("Failed to get schema version.", e);
}
- finally {
- try {
- metastoreConn.close();
- } catch (SQLException e) {
- System.err.println("Failed to close the metastore connection");
- e.printStackTrace(System.err);
- }
- }
}
- boolean validateLocations(String defaultLocPrefix) throws HiveMetaException {
+ boolean validateLocations(Connection conn, String defaultLocPrefix) throws HiveMetaException {
+ System.out.println("Validating database/table/partition locations");
boolean rtn;
- rtn = checkMetaStoreDBLocation(defaultLocPrefix);
- rtn = checkMetaStoreTableLocation(defaultLocPrefix) && rtn;
- rtn = checkMetaStorePartitionLocation(defaultLocPrefix) && rtn;
+ rtn = checkMetaStoreDBLocation(conn, defaultLocPrefix);
+ rtn = checkMetaStoreTableLocation(conn, defaultLocPrefix) && rtn;
+ rtn = checkMetaStorePartitionLocation(conn, defaultLocPrefix) && rtn;
+ System.out.println((rtn ? "Succeeded" : "Failed") + " in database/table/partition location validation");
return rtn;
}
@@ -203,24 +197,22 @@ public class HiveSchemaTool {
}
// read schema version from metastore
- private boolean checkMetaStoreDBLocation(String locHeader)
+ private boolean checkMetaStoreDBLocation(Connection conn, String locHeader)
throws HiveMetaException {
String defaultPrefix = locHeader;
String dbLoc;
boolean isValid = true;
int numOfInvalid = 0;
- Connection metastoreConn = getConnectionToMetastore(true);
if (getDbCommandParser(dbType).needsQuotedIdentifier()) {
dbLoc = "select dbt.\"DB_ID\", dbt.\"NAME\", dbt.\"DB_LOCATION_URI\" from \"DBS\" dbt";
} else {
dbLoc = "select dbt.DB_ID, dbt.NAME, dbt.DB_LOCATION_URI from DBS dbt";
}
- String locValue;
- String dbName;
- try(Statement stmt = metastoreConn.createStatement();
+
+ try(Statement stmt = conn.createStatement();
ResultSet res = stmt.executeQuery(dbLoc)) {
while (res.next()) {
- locValue = res.getString(3);
+ String locValue = res.getString(3);
if (locValue == null) {
System.err.println("NULL Location for DB with " + getNameOrID(res,2,1));
numOfInvalid++;
@@ -251,14 +243,6 @@ public class HiveSchemaTool {
} catch (SQLException e) {
throw new HiveMetaException("Failed to get DB Location Info.", e);
}
- finally {
- try {
- metastoreConn.close();
- } catch (SQLException e) {
- System.err.println("Failed to close the metastore connection");
- e.printStackTrace(System.err);
- }
- }
if (numOfInvalid > 0) {
isValid = false;
System.err.println("Total number of invalid DB locations is: "+ numOfInvalid);
@@ -266,12 +250,11 @@ public class HiveSchemaTool {
return isValid;
}
- private boolean checkMetaStoreTableLocation(String locHeader)
+ private boolean checkMetaStoreTableLocation(Connection conn, String locHeader)
throws HiveMetaException {
String defaultPrefix = locHeader;
String tabLoc, tabIDRange;
boolean isValid = true;
- Connection metastoreConn = getConnectionToMetastore(true);
int numOfInvalid = 0;
if (getDbCommandParser(dbType).needsQuotedIdentifier()) {
tabIDRange = "select max(\"TBL_ID\"), min(\"TBL_ID\") from \"TBLS\" ";
@@ -286,13 +269,12 @@ public class HiveSchemaTool {
} else {
tabLoc = "select tbl.TBL_ID, tbl.TBL_NAME, sd.LOCATION, dbt.DB_ID, dbt.NAME from TBLS tbl join SDS sd on tbl.SD_ID = sd.SD_ID and tbl.TBL_ID >= ? and tbl.TBL_ID <= ? inner join DBS dbt on tbl.DB_ID = dbt.DB_ID";
}
- String locValue;
- String tabName;
- long maxID = 0, minID = 0, curID;
+
+ long maxID = 0, minID = 0;
long rtnSize = 2000;
try {
- Statement stmt = metastoreConn.createStatement();
+ Statement stmt = conn.createStatement();
ResultSet res = stmt.executeQuery(tabIDRange);
if (res.next()) {
maxID = res.getLong(1);
@@ -300,14 +282,13 @@ public class HiveSchemaTool {
}
res.close();
stmt.close();
- curID = minID;
- PreparedStatement pStmt = metastoreConn.prepareStatement(tabLoc);
+ PreparedStatement pStmt = conn.prepareStatement(tabLoc);
while (minID <= maxID) {
pStmt.setLong(1, minID);
pStmt.setLong(2, minID + rtnSize);
res = pStmt.executeQuery();
while (res.next()) {
- locValue = res.getString(3);
+ String locValue = res.getString(3);
if (locValue == null) {
System.err.println("In DB with " + getNameOrID(res,5,4));
System.err.println("NULL Location for TABLE with " + getNameOrID(res,2,1));
@@ -346,14 +327,6 @@ public class HiveSchemaTool {
} catch (SQLException e) {
throw new HiveMetaException("Failed to get Table Location Info.", e);
}
- finally {
- try {
- metastoreConn.close();
- } catch (SQLException e) {
- System.err.println("Failed to close the metastore connection");
- e.printStackTrace(System.err);
- }
- }
if (numOfInvalid > 0) {
isValid = false;
System.err.println("Total number of invalid TABLE locations is: "+ numOfInvalid);
@@ -361,13 +334,12 @@ public class HiveSchemaTool {
return isValid;
}
- private boolean checkMetaStorePartitionLocation(String locHeader)
+ private boolean checkMetaStorePartitionLocation(Connection conn, String locHeader)
throws HiveMetaException {
String defaultPrefix = locHeader;
String partLoc, partIDRange;
boolean isValid = true;
int numOfInvalid = 0;
- Connection metastoreConn = getConnectionToMetastore(true);
if (getDbCommandParser(dbType).needsQuotedIdentifier()) {
partIDRange = "select max(\"PART_ID\"), min(\"PART_ID\") from \"PARTITIONS\" ";
} else {
@@ -384,13 +356,12 @@ public class HiveSchemaTool {
+ "inner join SDS sd on pt.SD_ID = sd.SD_ID and pt.PART_ID >= ? and pt.PART_ID <= ? "
+ "inner join TBLS tbl on tbl.TBL_ID = pt.TBL_ID inner join DBS dbt on tbl.DB_ID = dbt.DB_ID ";
}
- String locValue;
- String tabName;
- long maxID = 0, minID = 0, curID;
+
+ long maxID = 0, minID = 0;
long rtnSize = 2000;
try {
- Statement stmt = metastoreConn.createStatement();
+ Statement stmt = conn.createStatement();
ResultSet res = stmt.executeQuery(partIDRange);
if (res.next()) {
maxID = res.getLong(1);
@@ -398,14 +369,13 @@ public class HiveSchemaTool {
}
res.close();
stmt.close();
- curID = minID;
- PreparedStatement pStmt = metastoreConn.prepareStatement(partLoc);
+ PreparedStatement pStmt = conn.prepareStatement(partLoc);
while (minID <= maxID) {
pStmt.setLong(1, minID);
pStmt.setLong(2, minID + rtnSize);
res = pStmt.executeQuery();
while (res.next()) {
- locValue = res.getString(3);
+ String locValue = res.getString(3);
if (locValue == null) {
System.err.println("In DB with " + getNameOrID(res,7,6) + ", TABLE with " + getNameOrID(res,5,4));
System.err.println("NULL Location for PARTITION with " + getNameOrID(res,2,1));
@@ -442,14 +412,6 @@ public class HiveSchemaTool {
} catch (SQLException e) {
throw new HiveMetaException("Failed to get Partiton Location Info.", e);
}
- finally {
- try {
- metastoreConn.close();
- } catch (SQLException e) {
- System.err.println("Failed to close the metastore connection");
- e.printStackTrace(System.err);
- }
- }
if (numOfInvalid > 0) {
isValid = false;
System.err.println("Total number of invalid PARTITION locations is: "+ numOfInvalid);
@@ -582,16 +544,28 @@ public class HiveSchemaTool {
}
public void doValidate() throws HiveMetaException {
- System.out.print("Starting metastore validation");
- validateSchemaVersions();
- validateSequences();
- validateSchemaTables();
- validateLocations(null);
- validateColumnNullValues();
- System.out.print("Done with metastore validation");
+ System.out.println("Starting metastore validation");
+ Connection conn = getConnectionToMetastore(false);
+ try {
+ validateSchemaVersions(conn);
+ validateSequences(conn);
+ validateSchemaTables(conn);
+ validateLocations(conn, null);
+ validateColumnNullValues(conn);
+ } finally {
+ if (conn != null) {
+ try {
+ conn.close();
+ } catch (SQLException e) {
+ throw new HiveMetaException("Failed to close metastore connection", e);
+ }
+ }
+ }
+
+ System.out.println("Done with metastore validation");
}
- boolean validateSequences() throws HiveMetaException {
+ boolean validateSequences(Connection conn) throws HiveMetaException {
Map<String, Pair<String, String>> seqNameToTable =
new ImmutableMap.Builder<String, Pair<String, String>>()
.put("MDatabase", Pair.of("DBS", "DB_ID"))
@@ -611,7 +585,7 @@ public class HiveSchemaTool {
.build();
System.out.println("Validating sequence number for SEQUENCE_TABLE");
- Connection conn = getConnectionToMetastore(true);
+
boolean isValid = true;
try {
Statement stmt = conn.createStatement();
@@ -638,25 +612,17 @@ public class HiveSchemaTool {
}
}
+ System.out.println((isValid ? "Succeeded" :"Failed") + " in sequence number validation for SEQUENCE_TABLE");
return isValid;
} catch(SQLException e) {
throw new HiveMetaException("Failed to validate sequence number for SEQUENCE_TABLE", e);
- } finally {
- if (conn != null) {
- try {
- conn.close();
- } catch (SQLException e) {
- throw new HiveMetaException("Failed to close metastore connection", e);
- }
- }
}
}
- boolean validateSchemaVersions() throws HiveMetaException {
+ boolean validateSchemaVersions(Connection conn) throws HiveMetaException {
System.out.println("Validating schema version");
try {
- String newSchemaVersion = getMetaStoreSchemaVersion(
- getConnectionToMetastore(false), true);
+ String newSchemaVersion = getMetaStoreSchemaVersion(conn, true);
assertCompatibleVersion(MetaStoreSchemaInfo.getHiveSchemaVersion(), newSchemaVersion);
} catch (HiveMetaException hme) {
if (hme.getMessage().contains("Metastore schema version is not compatible")
@@ -672,19 +638,17 @@ public class HiveSchemaTool {
return true;
}
- boolean validateSchemaTables() throws HiveMetaException {
+ boolean validateSchemaTables(Connection conn) throws HiveMetaException {
ResultSet rs = null;
DatabaseMetaData metadata = null;
List<String> dbTables = new ArrayList<String>();
List<String> schemaTables = new ArrayList<String>();
List<String> subScripts = new ArrayList<String>();
- Connection hmsConn = getConnectionToMetastore(false);
- String version = getMetaStoreSchemaVersion(hmsConn);
- hmsConn = getConnectionToMetastore(false);
+ String version = getMetaStoreSchemaVersion(conn);
System.out.println("Validating tables in the schema for version " + version);
try {
- metadata = hmsConn.getMetaData();
+ metadata = conn.getMetaData();
String[] types = {"TABLE"};
rs = metadata.getTables(null, null, "%", types);
String table = null;
@@ -704,15 +668,6 @@ public class HiveSchemaTool {
throw new HiveMetaException("Failed to close resultset", e);
}
}
-
- if (hmsConn != null) {
- try {
- hmsConn.close();
-
- } catch (SQLException e) {
- throw new HiveMetaException("Failed to close metastore connection", e);
- }
- }
}
// parse the schema file to determine the tables that are expected to exist
@@ -721,11 +676,11 @@ public class HiveSchemaTool {
String schemaFile = baseDir + "/oracle/hive-schema-" + version + ".oracle.sql";
try {
- LOG.info("Parsing schema script " + schemaFile);
+ LOG.debug("Parsing schema script " + schemaFile);
subScripts.addAll(findCreateTable(schemaFile, schemaTables));
while (subScripts.size() > 0) {
schemaFile = baseDir + "/oracle/" + subScripts.remove(0);
- LOG.info("Parsing subscript " + schemaFile);
+ LOG.debug("Parsing subscript " + schemaFile);
subScripts.addAll(findCreateTable(schemaFile, schemaTables));
}
} catch (Exception e) {
@@ -742,7 +697,7 @@ public class HiveSchemaTool {
+ " ] are missing from the database schema.");
return false;
} else {
- System.out.println("Schema table validation successful");
+ System.out.println("Succeeded in schema table validation");
return true;
}
}
@@ -778,9 +733,8 @@ public class HiveSchemaTool {
return subs;
}
- boolean validateColumnNullValues() throws HiveMetaException {
+ boolean validateColumnNullValues(Connection conn) throws HiveMetaException {
System.out.println("Validating columns for incorrect NULL values");
- Connection conn = getConnectionToMetastore(true);
boolean isValid = true;
try {
Statement stmt = conn.createStatement();
@@ -797,15 +751,10 @@ public class HiveSchemaTool {
System.err.println("Value of SD_ID in TBLS should not be NULL: hive table - " + tableName + " tableId - " + tableId + " tableType - " + tableType);
}
+ System.out.println((isValid ? "Succeeded" : "Failed") + " in column validation for incorrect NULL values");
return isValid;
} catch(SQLException e) {
throw new HiveMetaException("Failed to validate columns for incorrect NULL values", e);
- } finally {
- try {
- conn.close();
- } catch (SQLException e) {
- throw new HiveMetaException("Failed to close metastore connection", e);
- }
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/999dc553/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaTool.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaTool.java b/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaTool.java
index ac2c927..3d585ac 100644
--- a/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaTool.java
+++ b/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaTool.java
@@ -25,6 +25,7 @@ import java.io.FileWriter;
import java.io.IOException;
import java.io.OutputStream;
import java.io.PrintStream;
+import java.sql.Connection;
import java.util.Random;
import junit.framework.TestCase;
@@ -39,6 +40,7 @@ import org.apache.hive.beeline.HiveSchemaHelper.PostgresCommandParser;
public class TestSchemaTool extends TestCase {
private HiveSchemaTool schemaTool;
+ private Connection conn;
private HiveConf hiveConf;
private String testMetastoreDB;
private PrintStream errStream;
@@ -57,6 +59,7 @@ public class TestSchemaTool extends TestCase {
System.setProperty("beeLine.system.exit", "true");
errStream = System.err;
outStream = System.out;
+ conn = schemaTool.getConnectionToMetastore(false);
}
@Override
@@ -67,6 +70,9 @@ public class TestSchemaTool extends TestCase {
}
System.setOut(outStream);
System.setErr(errStream);
+ if (conn != null) {
+ conn.close();
+ }
}
/**
@@ -77,7 +83,7 @@ public class TestSchemaTool extends TestCase {
schemaTool.doInit();
// Test empty database
- boolean isValid = schemaTool.validateSequences();
+ boolean isValid = schemaTool.validateSequences(conn);
assertTrue(isValid);
// Test valid case
@@ -87,7 +93,7 @@ public class TestSchemaTool extends TestCase {
};
File scriptFile = generateTestScript(scripts);
schemaTool.runBeeLine(scriptFile.getPath());
- isValid = schemaTool.validateSequences();
+ isValid = schemaTool.validateSequences(conn);
assertTrue(isValid);
// Test invalid case
@@ -99,7 +105,7 @@ public class TestSchemaTool extends TestCase {
};
scriptFile = generateTestScript(scripts);
schemaTool.runBeeLine(scriptFile.getPath());
- isValid = schemaTool.validateSequences();
+ isValid = schemaTool.validateSequences(conn);
assertFalse(isValid);
}
@@ -110,12 +116,12 @@ public class TestSchemaTool extends TestCase {
public void testValidateSchemaTables() throws Exception {
schemaTool.doInit("2.0.0");
- boolean isValid = (boolean)schemaTool.validateSchemaTables();
+ boolean isValid = (boolean)schemaTool.validateSchemaTables(conn);
assertTrue(isValid);
// upgrade to 2.2.0 schema and re-validate
schemaTool.doUpgrade("2.2.0");
- isValid = (boolean)schemaTool.validateSchemaTables();
+ isValid = (boolean)schemaTool.validateSchemaTables(conn);
assertTrue(isValid);
// Simulate a missing table scenario by renaming a couple of tables
@@ -126,7 +132,7 @@ public class TestSchemaTool extends TestCase {
File scriptFile = generateTestScript(scripts);
schemaTool.runBeeLine(scriptFile.getPath());
- isValid = schemaTool.validateSchemaTables();
+ isValid = schemaTool.validateSchemaTables(conn);
assertFalse(isValid);
// Restored the renamed tables
@@ -137,7 +143,7 @@ public class TestSchemaTool extends TestCase {
scriptFile = generateTestScript(scripts);
schemaTool.runBeeLine(scriptFile.getPath());
- isValid = schemaTool.validateSchemaTables();
+ isValid = schemaTool.validateSchemaTables(conn);
assertTrue(isValid);
}
@@ -149,12 +155,12 @@ public class TestSchemaTool extends TestCase {
schemaTool.doInit();
// Test empty database
- boolean isValid = schemaTool.validateColumnNullValues();
+ boolean isValid = schemaTool.validateColumnNullValues(conn);
assertTrue(isValid);
// Test valid case
createTestHiveTableSchemas();
- isValid = schemaTool.validateColumnNullValues();
+ isValid = schemaTool.validateColumnNullValues(conn);
// Test invalid case
String[] scripts = new String[] {
@@ -162,7 +168,7 @@ public class TestSchemaTool extends TestCase {
};
File scriptFile = generateTestScript(scripts);
schemaTool.runBeeLine(scriptFile.getPath());
- isValid = schemaTool.validateColumnNullValues();
+ isValid = schemaTool.validateColumnNullValues(conn);
assertFalse(isValid);
}
@@ -217,14 +223,14 @@ public class TestSchemaTool extends TestCase {
*/
public void testValidateSchemaVersions() throws Exception {
schemaTool.doInit();
- boolean isValid = schemaTool.validateSchemaVersions();
+ boolean isValid = schemaTool.validateSchemaVersions(conn);
// Test an invalid case with multiple versions
String[] scripts = new String[] {
"insert into VERSION values(100, '2.2.0', 'Hive release version 2.2.0')"
};
File scriptFile = generateTestScript(scripts);
schemaTool.runBeeLine(scriptFile.getPath());
- isValid = schemaTool.validateSchemaVersions();
+ isValid = schemaTool.validateSchemaVersions(conn);
assertFalse(isValid);
scripts = new String[] {
@@ -232,7 +238,7 @@ public class TestSchemaTool extends TestCase {
};
scriptFile = generateTestScript(scripts);
schemaTool.runBeeLine(scriptFile.getPath());
- isValid = schemaTool.validateSchemaVersions();
+ isValid = schemaTool.validateSchemaVersions(conn);
assertTrue(isValid);
// Test an invalid case without version
@@ -241,7 +247,7 @@ public class TestSchemaTool extends TestCase {
};
scriptFile = generateTestScript(scripts);
schemaTool.runBeeLine(scriptFile.getPath());
- isValid = schemaTool.validateSchemaVersions();
+ isValid = schemaTool.validateSchemaVersions(conn);
assertFalse(isValid);
}
@@ -596,12 +602,11 @@ public class TestSchemaTool extends TestCase {
schemaTool.doInit();
String defaultRoot = "hdfs://myhost.com:8020";
//check empty DB
- boolean isValid = schemaTool.validateLocations(null);
+ boolean isValid = schemaTool.validateLocations(conn, null);
assertTrue(isValid);
- isValid = schemaTool.validateLocations(defaultRoot);
+ isValid = schemaTool.validateLocations(conn, defaultRoot);
assertTrue(isValid);
- String dbmydbLocation = defaultRoot + "/user/hive/warehouse/mydb";
// Test valid case
String[] scripts = new String[] {
"insert into DBS values(2, 'my db', 'hdfs://myhost.com:8020/user/hive/warehouse/mydb', 'mydb', 'public', 'role')",
@@ -613,9 +618,9 @@ public class TestSchemaTool extends TestCase {
};
File scriptFile = generateTestScript(scripts);
schemaTool.runBeeLine(scriptFile.getPath());
- isValid = schemaTool.validateLocations(null);
+ isValid = schemaTool.validateLocations(conn, null);
assertTrue(isValid);
- isValid = schemaTool.validateLocations(defaultRoot);
+ isValid = schemaTool.validateLocations(conn, defaultRoot);
assertTrue(isValid);
scripts = new String[] {
"delete from PARTITIONS",
@@ -634,9 +639,9 @@ public class TestSchemaTool extends TestCase {
};
scriptFile = generateTestScript(scripts);
schemaTool.runBeeLine(scriptFile.getPath());
- isValid = schemaTool.validateLocations(null);
+ isValid = schemaTool.validateLocations(conn, null);
assertFalse(isValid);
- isValid = schemaTool.validateLocations(defaultRoot);
+ isValid = schemaTool.validateLocations(conn, defaultRoot);
assertFalse(isValid);
}