You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2018/07/11 02:04:43 UTC
[11/32] hive git commit: HIVE-20060 Refactor HiveSchemaTool and
MetastoreSchemaTool (Alan Gates, reviewed by Daniel Dai)
http://git-wip-us.apache.org/repos/asf/hive/blob/eae5225f/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskInfo.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskInfo.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskInfo.java
new file mode 100644
index 0000000..cd1d57b
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskInfo.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.tools;
+
+import org.apache.hadoop.hive.metastore.HiveMetaException;
+import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper.MetaStoreConnectionInfo;
+
+/**
+ * Print Hive version and schema version.
+ */
+class SchemaToolTaskInfo extends SchemaToolTask {
+ @Override
+ void setCommandLineArguments(SchemaToolCommandLine cl) {
+ // do nothing
+ }
+
+ @Override
+ void execute() throws HiveMetaException {
+ String hiveVersion = schemaTool.getMetaStoreSchemaInfo().getHiveSchemaVersion();
+ MetaStoreConnectionInfo connectionInfo = schemaTool.getConnectionInfo(true);
+ String dbVersion = schemaTool.getMetaStoreSchemaInfo().getMetaStoreSchemaVersion(connectionInfo);
+
+ System.out.println("Hive distribution version:\t " + hiveVersion);
+ System.out.println("Metastore schema version:\t " + dbVersion);
+
+ schemaTool.assertCompatibleVersion(hiveVersion, dbVersion);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/eae5225f/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskInit.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskInit.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskInit.java
new file mode 100644
index 0000000..e3fa495
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskInit.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.tools;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hive.metastore.HiveMetaException;
+
+/**
+ * Initialize the metastore schema.
+ */
+class SchemaToolTaskInit extends SchemaToolTask {
+ private boolean validate = true;
+ private String toVersion;
+
+ @Override
+ void setCommandLineArguments(SchemaToolCommandLine cl) {
+ if (cl.hasOption("initSchemaTo")) {
+ this.toVersion = cl.getOptionValue("initSchemaTo");
+ this.validate = false;
+ }
+ }
+
+ private void ensureToVersion() throws HiveMetaException {
+ if (toVersion != null) {
+ return;
+ }
+
+ // If null then current hive version is used
+ toVersion = schemaTool.getMetaStoreSchemaInfo().getHiveSchemaVersion();
+ System.out.println("Initializing the schema to: " + toVersion);
+ }
+
+ @Override
+ void execute() throws HiveMetaException {
+ ensureToVersion();
+
+ schemaTool.testConnectionToMetastore();
+ System.out.println("Starting metastore schema initialization to " + toVersion);
+
+ String initScriptDir = schemaTool.getMetaStoreSchemaInfo().getMetaStoreScriptDir();
+ String initScriptFile = schemaTool.getMetaStoreSchemaInfo().generateInitFileName(toVersion);
+
+ try {
+ System.out.println("Initialization script " + initScriptFile);
+ if (!schemaTool.isDryRun()) {
+ schemaTool.execSql(initScriptDir, initScriptFile);
+ System.out.println("Initialization script completed");
+ }
+ } catch (IOException e) {
+ throw new HiveMetaException("Schema initialization FAILED! Metastore state would be inconsistent!", e);
+ }
+
+ if (validate) {
+ schemaTool.verifySchemaVersion();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/eae5225f/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskMoveDatabase.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskMoveDatabase.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskMoveDatabase.java
new file mode 100644
index 0000000..8a9b9d1
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskMoveDatabase.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.tools;
+
+import static org.apache.hadoop.hive.metastore.utils.StringUtils.normalizeIdentifier;
+
+import java.sql.Connection;
+import java.sql.SQLException;
+import java.sql.Statement;
+
+import org.apache.hadoop.hive.metastore.HiveMetaException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Print Hive version and schema version.
+ */
+class SchemaToolTaskMoveDatabase extends SchemaToolTask {
+ private static final Logger LOG = LoggerFactory.getLogger(SchemaToolTaskMoveDatabase.class.getName());
+
+ private String fromCatName;
+ private String toCatName;
+ private String dbName;
+
+ @Override
+ void setCommandLineArguments(SchemaToolCommandLine cl) {
+ fromCatName = normalizeIdentifier(cl.getOptionValue("fromCatalog"));
+ toCatName = normalizeIdentifier(cl.getOptionValue("toCatalog"));
+ dbName = normalizeIdentifier(cl.getOptionValue("moveDatabase"));
+ }
+
+ @Override
+ void execute() throws HiveMetaException {
+ System.out.println(String.format("Moving database %s from catalog %s to catalog %s",
+ dbName, fromCatName, toCatName));
+ Connection conn = schemaTool.getConnectionToMetastore(true);
+ boolean success = false;
+ try {
+ conn.setAutoCommit(false);
+ try (Statement stmt = conn.createStatement()) {
+ updateCatalogNameInTable(stmt, "DBS", "CTLG_NAME", "NAME", fromCatName, toCatName, dbName, false);
+ updateCatalogNameInTable(stmt, "TAB_COL_STATS", "CAT_NAME", "DB_NAME", fromCatName, toCatName, dbName, true);
+ updateCatalogNameInTable(stmt, "PART_COL_STATS", "CAT_NAME", "DB_NAME", fromCatName, toCatName, dbName, true);
+ updateCatalogNameInTable(stmt, "PARTITION_EVENTS", "CAT_NAME", "DB_NAME", fromCatName, toCatName, dbName, true);
+ updateCatalogNameInTable(stmt, "NOTIFICATION_LOG", "CAT_NAME", "DB_NAME", fromCatName, toCatName, dbName, true);
+ conn.commit();
+ success = true;
+ }
+ } catch (SQLException e) {
+ throw new HiveMetaException("Failed to move database", e);
+ } finally {
+ try {
+ if (!success) {
+ conn.rollback();
+ }
+ } catch (SQLException e) {
+ // Not really much we can do here.
+ LOG.error("Failed to rollback, everything will probably go bad from here.");
+ }
+ }
+ }
+
+ private static final String UPDATE_CATALOG_NAME_STMT =
+ "update <q>%s<q> " +
+ " set <q>%s<q> = '%s' " +
+ " where <q>%s<q> = '%s' " +
+ " and <q>%s<q> = '%s'";
+
+ private void updateCatalogNameInTable(Statement stmt, String tableName, String catColName, String dbColName,
+ String fromCatName, String toCatName, String dbName, boolean zeroUpdatesOk)
+ throws HiveMetaException, SQLException {
+ String update = String.format(schemaTool.quote(UPDATE_CATALOG_NAME_STMT), tableName, catColName, toCatName,
+ catColName, fromCatName, dbColName, dbName);
+ LOG.debug("Going to run " + update);
+ int numUpdated = stmt.executeUpdate(update);
+ if (numUpdated != 1 && !(zeroUpdatesOk && numUpdated == 0)) {
+ throw new HiveMetaException("Failed to properly update the " + tableName +
+ " table. Expected to update 1 row but instead updated " + numUpdated);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/eae5225f/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskMoveTable.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskMoveTable.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskMoveTable.java
new file mode 100644
index 0000000..a8f9228
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskMoveTable.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.tools;
+
+import static org.apache.hadoop.hive.metastore.utils.StringUtils.normalizeIdentifier;
+
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+
+import org.apache.hadoop.hive.metastore.HiveMetaException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Print Hive version and schema version.
+ */
+class SchemaToolTaskMoveTable extends SchemaToolTask {
+ private static final Logger LOG = LoggerFactory.getLogger(SchemaToolTaskMoveTable.class.getName());
+
+ private String fromCat;
+ private String toCat;
+ private String fromDb;
+ private String toDb;
+ private String tableName;
+
+ @Override
+ void setCommandLineArguments(SchemaToolCommandLine cl) {
+ fromCat = normalizeIdentifier(cl.getOptionValue("fromCatalog"));
+ toCat = normalizeIdentifier(cl.getOptionValue("toCatalog"));
+ fromDb = normalizeIdentifier(cl.getOptionValue("fromDatabase"));
+ toDb = normalizeIdentifier(cl.getOptionValue("toDatabase"));
+ tableName = normalizeIdentifier(cl.getOptionValue("moveTable"));
+ }
+
+ @Override
+ void execute() throws HiveMetaException {
+ Connection conn = schemaTool.getConnectionToMetastore(true);
+ boolean success = false;
+ try {
+ conn.setAutoCommit(false);
+ try (Statement stmt = conn.createStatement()) {
+ updateTableId(stmt);
+ updateDbNameForTable(stmt, "TAB_COL_STATS", "TABLE_NAME", fromCat, toCat, fromDb, toDb, tableName);
+ updateDbNameForTable(stmt, "PART_COL_STATS", "TABLE_NAME", fromCat, toCat, fromDb, toDb, tableName);
+ updateDbNameForTable(stmt, "PARTITION_EVENTS", "TBL_NAME", fromCat, toCat, fromDb, toDb, tableName);
+ updateDbNameForTable(stmt, "NOTIFICATION_LOG", "TBL_NAME", fromCat, toCat, fromDb, toDb, tableName);
+ conn.commit();
+ success = true;
+ }
+ } catch (SQLException se) {
+ throw new HiveMetaException("Failed to move table", se);
+ } finally {
+ try {
+ if (!success) {
+ conn.rollback();
+ }
+ } catch (SQLException e) {
+ // Not really much we can do here.
+ LOG.error("Failed to rollback, everything will probably go bad from here.");
+ }
+
+ }
+ }
+
+ private static final String UPDATE_TABLE_ID_STMT =
+ "update <q>TBLS<q> " +
+ " set <q>DB_ID<q> = %d " +
+ " where <q>DB_ID<q> = %d " +
+ " and <q>TBL_NAME<q> = '%s'";
+
+ private void updateTableId(Statement stmt) throws SQLException, HiveMetaException {
+ // Find the old database id
+ long oldDbId = getDbId(stmt, fromDb, fromCat);
+
+ // Find the new database id
+ long newDbId = getDbId(stmt, toDb, toCat);
+
+ String update = String.format(schemaTool.quote(UPDATE_TABLE_ID_STMT), newDbId, oldDbId, tableName);
+ LOG.debug("Going to run " + update);
+ int numUpdated = stmt.executeUpdate(update);
+ if (numUpdated != 1) {
+ throw new HiveMetaException(
+ "Failed to properly update TBLS table. Expected to update " +
+ "1 row but instead updated " + numUpdated);
+ }
+ }
+
+ private static final String DB_ID_QUERY =
+ "select <q>DB_ID<q> " +
+ " from <q>DBS<q> " +
+ " where <q>NAME<q> = '%s' " +
+ " and <q>CTLG_NAME<q> = '%s'";
+
+ private long getDbId(Statement stmt, String db, String catalog) throws SQLException, HiveMetaException {
+ String query = String.format(schemaTool.quote(DB_ID_QUERY), db, catalog);
+ LOG.debug("Going to run " + query);
+ try (ResultSet rs = stmt.executeQuery(query)) {
+ if (!rs.next()) {
+ throw new HiveMetaException("Unable to find database " + fromDb);
+ }
+ return rs.getLong(1);
+ }
+ }
+
+ private static final String UPDATE_DB_NAME_STMT =
+ "update <q>%s<q> " +
+ " set <q>CAT_NAME<q> = '%s', " +
+ " <q>DB_NAME<q> = '%s' " +
+ " where <q>CAT_NAME<q> = '%s' " +
+ " and <q>DB_NAME<q> = '%s' " +
+ " and <q>%s<q> = '%s'";
+
+ private void updateDbNameForTable(Statement stmt, String tableName, String tableColumnName, String fromCat,
+ String toCat, String fromDb, String toDb, String hiveTblName) throws HiveMetaException, SQLException {
+ String update = String.format(schemaTool.quote(UPDATE_DB_NAME_STMT), tableName, toCat, toDb, fromCat, fromDb,
+ tableColumnName, hiveTblName);
+
+ LOG.debug("Going to run " + update);
+ int numUpdated = stmt.executeUpdate(update);
+ if (numUpdated > 1 || numUpdated < 0) {
+ throw new HiveMetaException("Failed to properly update the " + tableName +
+ " table. Expected to update 1 row but instead updated " + numUpdated);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/eae5225f/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskUpgrade.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskUpgrade.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskUpgrade.java
new file mode 100644
index 0000000..5e71609
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskUpgrade.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.tools;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.hadoop.hive.metastore.HiveMetaException;
+import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper.MetaStoreConnectionInfo;
+
+/**
+ * Perform metastore schema upgrade.
+ */
+class SchemaToolTaskUpgrade extends SchemaToolTask {
+ private String fromVersion;
+
+ @Override
+ void setCommandLineArguments(SchemaToolCommandLine cl) {
+ if (cl.hasOption("upgradeSchemaFrom")) {
+ this.fromVersion = cl.getOptionValue("upgradeSchemaFrom");
+ }
+ }
+
+ private void ensureFromVersion() throws HiveMetaException {
+ if (fromVersion != null) {
+ return;
+ }
+
+ // If null, then read from the metastore
+ MetaStoreConnectionInfo connectionInfo = schemaTool.getConnectionInfo(false);
+ fromVersion = schemaTool.getMetaStoreSchemaInfo().getMetaStoreSchemaVersion(connectionInfo);
+ if (fromVersion == null || fromVersion.isEmpty()) {
+ throw new HiveMetaException("Schema version not stored in the metastore. " +
+ "Metastore schema is too old or corrupt. Try specifying the version manually");
+ }
+ System.out.println("Upgrading from the version " + fromVersion);
+ }
+
+ @Override
+ void execute() throws HiveMetaException {
+ ensureFromVersion();
+
+ if (schemaTool.getMetaStoreSchemaInfo().getHiveSchemaVersion().equals(fromVersion)) {
+ System.out.println("No schema upgrade required from version " + fromVersion);
+ return;
+ }
+
+ // Find the list of scripts to execute for this upgrade
+ List<String> upgradeScripts = schemaTool.getMetaStoreSchemaInfo().getUpgradeScripts(fromVersion);
+ schemaTool.testConnectionToMetastore();
+ System.out.println("Starting upgrade metastore schema from version " + fromVersion + " to " +
+ schemaTool.getMetaStoreSchemaInfo().getHiveSchemaVersion());
+ String scriptDir = schemaTool.getMetaStoreSchemaInfo().getMetaStoreScriptDir();
+ try {
+ for (String scriptFile : upgradeScripts) {
+ System.out.println("Upgrade script " + scriptFile);
+ if (!schemaTool.isDryRun()) {
+ runPreUpgrade(scriptDir, scriptFile);
+ schemaTool.execSql(scriptDir, scriptFile);
+ System.out.println("Completed " + scriptFile);
+ }
+ }
+ } catch (IOException e) {
+ throw new HiveMetaException("Upgrade FAILED! Metastore state would be inconsistent !!", e);
+ }
+
+ // Revalidated the new version after upgrade
+ schemaTool.verifySchemaVersion();
+ }
+
+ /**
+ * Run pre-upgrade scripts corresponding to a given upgrade script,
+ * if any exist. The errors from pre-upgrade are ignored.
+ * Pre-upgrade scripts typically contain setup statements which
+ * may fail on some database versions and failure is ignorable.
+ *
+ * @param scriptDir upgrade script directory name
+ * @param scriptFile upgrade script file name
+ */
+ private void runPreUpgrade(String scriptDir, String scriptFile) {
+ for (int i = 0;; i++) {
+ String preUpgradeScript = schemaTool.getMetaStoreSchemaInfo().getPreUpgradeScriptName(i, scriptFile);
+ File preUpgradeScriptFile = new File(scriptDir, preUpgradeScript);
+ if (!preUpgradeScriptFile.isFile()) {
+ break;
+ }
+
+ try {
+ schemaTool.execSql(scriptDir, preUpgradeScript);
+ System.out.println("Completed " + preUpgradeScript);
+ } catch (Exception e) {
+ // Ignore the pre-upgrade script errors
+ System.err.println("Warning in pre-upgrade script " + preUpgradeScript + ": " + e.getMessage());
+ if (schemaTool.isVerbose()) {
+ e.printStackTrace();
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/eae5225f/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskValidate.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskValidate.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskValidate.java
new file mode 100644
index 0000000..d86b457
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskValidate.java
@@ -0,0 +1,630 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.tools;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.net.URI;
+import java.sql.Connection;
+import java.sql.DatabaseMetaData;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.SQLFeatureNotSupportedException;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.commons.lang.ArrayUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.metastore.HiveMetaException;
+import org.apache.hadoop.hive.metastore.TableType;
+import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper.MetaStoreConnectionInfo;
+import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper.NestedScriptParser;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableMap;
+
+/**
+ * Print Hive version and schema version.
+ */
+class SchemaToolTaskValidate extends SchemaToolTask {
+ private static final Logger LOG = LoggerFactory.getLogger(SchemaToolTaskValidate.class.getName());
+
+ @Override
+ void setCommandLineArguments(SchemaToolCommandLine cl) {
+ // do nothing
+ }
+
+ @Override
+ void execute() throws HiveMetaException {
+ System.out.println("Starting metastore validation\n");
+ Connection conn = schemaTool.getConnectionToMetastore(false);
+ boolean success = true;
+ try {
+ success &= validateSchemaVersions();
+ success &= validateSequences(conn);
+ success &= validateSchemaTables(conn);
+ success &= validateLocations(conn, schemaTool.getValidationServers());
+ success &= validateColumnNullValues(conn);
+ } finally {
+ if (conn != null) {
+ try {
+ conn.close();
+ } catch (SQLException e) {
+ throw new HiveMetaException("Failed to close metastore connection", e);
+ }
+ }
+ }
+
+ System.out.print("Done with metastore validation: ");
+ if (!success) {
+ System.out.println("[FAIL]");
+ throw new HiveMetaException("Validation failed");
+ } else {
+ System.out.println("[SUCCESS]");
+ }
+ }
+
+ boolean validateSchemaVersions() throws HiveMetaException {
+ System.out.println("Validating schema version");
+ try {
+ String hiveSchemaVersion = schemaTool.getMetaStoreSchemaInfo().getHiveSchemaVersion();
+ MetaStoreConnectionInfo connectionInfo = schemaTool.getConnectionInfo(false);
+ String newSchemaVersion = schemaTool.getMetaStoreSchemaInfo().getMetaStoreSchemaVersion(connectionInfo);
+ schemaTool.assertCompatibleVersion(hiveSchemaVersion, newSchemaVersion);
+ } catch (HiveMetaException hme) {
+ if (hme.getMessage().contains("Metastore schema version is not compatible") ||
+ hme.getMessage().contains("Multiple versions were found in metastore") ||
+ hme.getMessage().contains("Could not find version info in metastore VERSION table")) {
+ System.err.println(hme.getMessage());
+ System.out.println("[FAIL]\n");
+ return false;
+ } else {
+ throw hme;
+ }
+ }
+ System.out.println("[SUCCESS]\n");
+ return true;
+ }
+
+ private static final String QUERY_SEQ =
+ " select t.<q>NEXT_VAL<q>" +
+ " from <q>SEQUENCE_TABLE<q> t " +
+ " where t.<q>SEQUENCE_NAME<q> = ? " +
+ "order by t.<q>SEQUENCE_NAME<q>";
+
+ private static final String QUERY_MAX_ID =
+ "select max(<q>%s<q>)" +
+ " from <q>%s<q>";
+
+ @VisibleForTesting
+ boolean validateSequences(Connection conn) throws HiveMetaException {
+ Map<String, Pair<String, String>> seqNameToTable =
+ new ImmutableMap.Builder<String, Pair<String, String>>()
+ .put("MDatabase", Pair.of("DBS", "DB_ID"))
+ .put("MRole", Pair.of("ROLES", "ROLE_ID"))
+ .put("MGlobalPrivilege", Pair.of("GLOBAL_PRIVS", "USER_GRANT_ID"))
+ .put("MTable", Pair.of("TBLS","TBL_ID"))
+ .put("MStorageDescriptor", Pair.of("SDS", "SD_ID"))
+ .put("MSerDeInfo", Pair.of("SERDES", "SERDE_ID"))
+ .put("MColumnDescriptor", Pair.of("CDS", "CD_ID"))
+ .put("MTablePrivilege", Pair.of("TBL_PRIVS", "TBL_GRANT_ID"))
+ .put("MTableColumnStatistics", Pair.of("TAB_COL_STATS", "CS_ID"))
+ .put("MPartition", Pair.of("PARTITIONS", "PART_ID"))
+ .put("MPartitionColumnStatistics", Pair.of("PART_COL_STATS", "CS_ID"))
+ .put("MFunction", Pair.of("FUNCS", "FUNC_ID"))
+ .put("MIndex", Pair.of("IDXS", "INDEX_ID"))
+ .put("MStringList", Pair.of("SKEWED_STRING_LIST", "STRING_LIST_ID"))
+ .build();
+
+ System.out.println("Validating sequence number for SEQUENCE_TABLE");
+
+ boolean isValid = true;
+ try {
+ Statement stmt = conn.createStatement();
+ for (Map.Entry<String, Pair<String, String>> e : seqNameToTable.entrySet()) {
+ String tableName = e.getValue().getLeft();
+ String tableKey = e.getValue().getRight();
+ String fullSequenceName = "org.apache.hadoop.hive.metastore.model." + e.getKey();
+ String seqQuery = schemaTool.quote(QUERY_SEQ);
+ String maxIdQuery = String.format(schemaTool.quote(QUERY_MAX_ID), tableKey, tableName);
+
+ ResultSet res = stmt.executeQuery(maxIdQuery);
+ if (res.next()) {
+ long maxId = res.getLong(1);
+ if (maxId > 0) {
+ PreparedStatement stmtSeq = conn.prepareStatement(seqQuery);
+ stmtSeq.setString(1, fullSequenceName);
+ ResultSet resSeq = stmtSeq.executeQuery();
+ if (!resSeq.next()) {
+ isValid = false;
+ System.err.println("Missing SEQUENCE_NAME " + e.getKey() + " from SEQUENCE_TABLE");
+ } else if (resSeq.getLong(1) < maxId) {
+ isValid = false;
+ System.err.println("NEXT_VAL for " + e.getKey() + " in SEQUENCE_TABLE < max(" + tableKey +
+ ") in " + tableName);
+ }
+ }
+ }
+ }
+
+ System.out.println(isValid ? "[SUCCESS]\n" :"[FAIL]\n");
+ return isValid;
+ } catch (SQLException e) {
+ throw new HiveMetaException("Failed to validate sequence number for SEQUENCE_TABLE", e);
+ }
+ }
+
+ @VisibleForTesting
+ boolean validateSchemaTables(Connection conn) throws HiveMetaException {
+ System.out.println("Validating metastore schema tables");
+ String version = null;
+ try {
+ MetaStoreConnectionInfo connectionInfo = schemaTool.getConnectionInfo(false);
+ version = schemaTool.getMetaStoreSchemaInfo().getMetaStoreSchemaVersion(connectionInfo);
+ } catch (HiveMetaException he) {
+ System.err.println("Failed to determine schema version from Hive Metastore DB. " + he.getMessage());
+ System.out.println("Failed in schema table validation.");
+ LOG.debug("Failed to determine schema version from Hive Metastore DB," + he.getMessage(), he);
+ return false;
+ }
+
+ Connection hmsConn = schemaTool.getConnectionToMetastore(false);
+
+ LOG.debug("Validating tables in the schema for version " + version);
+ List<String> dbTables = new ArrayList<>();
+ ResultSet rs = null;
+ try {
+ String schema = null;
+ try {
+ schema = hmsConn.getSchema();
+ } catch (SQLFeatureNotSupportedException e) {
+ LOG.debug("schema is not supported");
+ }
+
+ DatabaseMetaData metadata = conn.getMetaData();
+ rs = metadata.getTables(null, schema, "%", new String[] {"TABLE"});
+
+ while (rs.next()) {
+ String table = rs.getString("TABLE_NAME");
+ dbTables.add(table.toLowerCase());
+ LOG.debug("Found table " + table + " in HMS dbstore");
+ }
+ } catch (SQLException e) {
+ throw new HiveMetaException("Failed to retrieve schema tables from Hive Metastore DB," +
+ e.getMessage(), e);
+ } finally {
+ if (rs != null) {
+ try {
+ rs.close();
+ } catch (SQLException e) {
+ throw new HiveMetaException("Failed to close resultset", e);
+ }
+ }
+ }
+
+ // parse the schema file to determine the tables that are expected to exist
+ // we are using oracle schema because it is simpler to parse, no quotes or backticks etc
+ List<String> schemaTables = new ArrayList<>();
+ List<String> subScripts = new ArrayList<>();
+
+ String baseDir = new File(schemaTool.getMetaStoreSchemaInfo().getMetaStoreScriptDir()).getParent();
+ String schemaFile = new File(schemaTool.getMetaStoreSchemaInfo().getMetaStoreScriptDir(),
+ schemaTool.getMetaStoreSchemaInfo().generateInitFileName(version)).getPath();
+ try {
+ LOG.debug("Parsing schema script " + schemaFile);
+ subScripts.addAll(findCreateTable(schemaFile, schemaTables));
+ while (subScripts.size() > 0) {
+ schemaFile = baseDir + "/" + schemaTool.getDbType() + "/" + subScripts.remove(0);
+ LOG.debug("Parsing subscript " + schemaFile);
+ subScripts.addAll(findCreateTable(schemaFile, schemaTables));
+ }
+ } catch (Exception e) {
+ System.err.println("Exception in parsing schema file. Cause:" + e.getMessage());
+ System.out.println("Failed in schema table validation.");
+ return false;
+ }
+
+ LOG.debug("Schema tables:[ " + Arrays.toString(schemaTables.toArray()) + " ]");
+ LOG.debug("DB tables:[ " + Arrays.toString(dbTables.toArray()) + " ]");
+
+ // now diff the lists
+ schemaTables.removeAll(dbTables);
+ if (schemaTables.size() > 0) {
+ Collections.sort(schemaTables);
+ System.err.println("Table(s) [ " + Arrays.toString(schemaTables.toArray()) + " ] " +
+ "are missing from the metastore database schema.");
+ System.out.println("[FAIL]\n");
+ return false;
+ } else {
+ System.out.println("[SUCCESS]\n");
+ return true;
+ }
+ }
+
+ @VisibleForTesting
+ List<String> findCreateTable(String path, List<String> tableList) throws Exception {
+ if (!(new File(path)).exists()) {
+ throw new Exception(path + " does not exist. Potentially incorrect version in the metastore VERSION table");
+ }
+
+ List<String> subs = new ArrayList<>();
+ NestedScriptParser sp = HiveSchemaHelper.getDbCommandParser(schemaTool.getDbType(), false);
+ Pattern regexp = Pattern.compile("CREATE TABLE(\\s+IF NOT EXISTS)?\\s+(\\S+).*");
+
+ try (BufferedReader reader = new BufferedReader(new FileReader(path))) {
+ String line = null;
+ while ((line = reader.readLine()) != null) {
+ if (sp.isNestedScript(line)) {
+ String subScript = sp.getScriptName(line);
+ LOG.debug("Schema subscript " + subScript + " found");
+ subs.add(subScript);
+ continue;
+ }
+ line = line.replaceAll("( )+", " "); //suppress multi-spaces
+ line = line.replaceAll("\\(", " ");
+ line = line.replaceAll("IF NOT EXISTS ", "");
+ line = line.replaceAll("`", "");
+ line = line.replaceAll("'", "");
+ line = line.replaceAll("\"", "");
+ Matcher matcher = regexp.matcher(line);
+
+ if (matcher.find()) {
+ String table = matcher.group(2);
+ if (schemaTool.getDbType().equals("derby")) {
+ table = table.replaceAll("APP\\.", "");
+ }
+ tableList.add(table.toLowerCase());
+ LOG.debug("Found table " + table + " in the schema");
+ }
+ }
+ } catch (IOException ex){
+ throw new Exception(ex.getMessage());
+ }
+
+ return subs;
+ }
+
+ @VisibleForTesting
+ boolean validateLocations(Connection conn, URI[] defaultServers) throws HiveMetaException {
+ System.out.println("Validating DFS locations");
+ boolean rtn = true;
+ rtn &= checkMetaStoreDBLocation(conn, defaultServers);
+ rtn &= checkMetaStoreTableLocation(conn, defaultServers);
+ rtn &= checkMetaStorePartitionLocation(conn, defaultServers);
+ rtn &= checkMetaStoreSkewedColumnsLocation(conn, defaultServers);
+ System.out.println(rtn ? "[SUCCESS]\n" : "[FAIL]\n");
+ return rtn;
+ }
+
+ private static final String QUERY_DB_LOCATION =
+ " select dbt.<q>DB_ID<q>, " +
+ " dbt.<q>NAME<q>, " +
+ " dbt.<q>DB_LOCATION_URI<q> " +
+ " from <q>DBS<q> dbt " +
+ "order by dbt.<q>DB_ID<q> ";
+
+ private boolean checkMetaStoreDBLocation(Connection conn, URI[] defaultServers) throws HiveMetaException {
+ String dbLocQuery = schemaTool.quote(QUERY_DB_LOCATION);
+
+ int numOfInvalid = 0;
+ try (Statement stmt = conn.createStatement();
+ ResultSet res = stmt.executeQuery(dbLocQuery)) {
+ while (res.next()) {
+ String locValue = res.getString(3);
+ String dbName = getNameOrID(res, 2, 1);
+ if (!checkLocation("Database " + dbName, locValue, defaultServers)) {
+ numOfInvalid++;
+ }
+ }
+ } catch (SQLException e) {
+ throw new HiveMetaException("Failed to get DB Location Info.", e);
+ }
+ return numOfInvalid == 0;
+ }
+
+ private static final String TAB_ID_RANGE_QUERY =
+ "select max(<q>TBL_ID<q>), " +
+ " min(<q>TBL_ID<q>) " +
+ " from <q>TBLS<q> ";
+
+ private static final String TAB_LOC_QUERY =
+ " select tbl.<q>TBL_ID<q>, " +
+ " tbl.<q>TBL_NAME<q>, " +
+ " sd.<q>LOCATION<q>, " +
+ " dbt.<q>DB_ID<q>, " +
+ " dbt.<q>NAME<q> " +
+ " from <q>TBLS<q> tbl " +
+ "inner join <q>SDS<q> sd on sd.<q>SD_ID<q> = tbl.<q>SD_ID<q> " +
+ "inner join <q>DBS<q> dbt on tbl.<q>DB_ID<q> = dbt.<q>DB_ID<q> " +
+ " where tbl.<q>TBL_TYPE<q> != '%s' " +
+ " and tbl.<q>TBL_ID<q> >= ? " +
+ " and tbl.<q>TBL_ID<q> <= ? " +
+ " order by tbl.<q>TBL_ID<q> ";
+
+ private static final int TAB_LOC_CHECK_SIZE = 2000;
+
+ private boolean checkMetaStoreTableLocation(Connection conn, URI[] defaultServers)
+ throws HiveMetaException {
+ String tabIDRangeQuery = schemaTool.quote(TAB_ID_RANGE_QUERY);
+ String tabLocQuery = String.format(schemaTool.quote(TAB_LOC_QUERY), TableType.VIRTUAL_VIEW);
+
+ try {
+ long maxID = 0, minID = 0;
+ try (Statement stmt = conn.createStatement();
+ ResultSet res = stmt.executeQuery(tabIDRangeQuery)) {
+ if (res.next()) {
+ maxID = res.getLong(1);
+ minID = res.getLong(2);
+ }
+ }
+
+ int numOfInvalid = 0;
+ try (PreparedStatement pStmt = conn.prepareStatement(tabLocQuery)) {
+ while (minID <= maxID) {
+ pStmt.setLong(1, minID);
+ pStmt.setLong(2, minID + TAB_LOC_CHECK_SIZE);
+ try (ResultSet res = pStmt.executeQuery()) {
+ while (res.next()) {
+ String locValue = res.getString(3);
+ String entity = "Database " + getNameOrID(res, 5, 4) + ", Table " + getNameOrID(res, 2, 1);
+ if (!checkLocation(entity, locValue, defaultServers)) {
+ numOfInvalid++;
+ }
+ }
+ }
+ minID += TAB_LOC_CHECK_SIZE + 1;
+ }
+ }
+
+ return numOfInvalid == 0;
+ } catch (SQLException e) {
+ throw new HiveMetaException("Failed to get Table Location Info.", e);
+ }
+ }
+
+ private static final String QUERY_PART_ID_RANGE =
+ "select max(<q>PART_ID<q>)," +
+ " min(<q>PART_ID<q>)" +
+ " from <q>PARTITIONS<q> ";
+
+ private static final String QUERY_PART_LOC =
+ " select pt.<q>PART_ID<q>, " +
+ " pt.<q>PART_NAME<q>, " +
+ " sd.<q>LOCATION<q>, " +
+ " tbl.<q>TBL_ID<q>, " +
+ " tbl.<q>TBL_NAME<q>, " +
+ " dbt.<q>DB_ID<q>, " +
+ " dbt.<q>NAME<q> " +
+ " from <q>PARTITIONS<q> pt " +
+ "inner join <q>SDS<q> sd on sd.<q>SD_ID<q> = pt.<q>SD_ID<q> " +
+ "inner join <q>TBLS<q> tbl on tbl.<q>TBL_ID<q> = pt.<q>TBL_ID<q> " +
+ "inner join <q>DBS<q> dbt on dbt.<q>DB_ID<q> = tbl.<q>DB_ID<q> " +
+ " where pt.<q>PART_ID<q> >= ? " +
+ " and pt.<q>PART_ID<q> <= ? " +
+ " order by tbl.<q>TBL_ID<q> ";
+
+ private static final int PART_LOC_CHECK_SIZE = 2000;
+
+ private boolean checkMetaStorePartitionLocation(Connection conn, URI[] defaultServers)
+ throws HiveMetaException {
+ String queryPartIDRange = schemaTool.quote(QUERY_PART_ID_RANGE);
+ String queryPartLoc = schemaTool.quote(QUERY_PART_LOC);
+
+ try {
+ long maxID = 0, minID = 0;
+ try (Statement stmt = conn.createStatement();
+ ResultSet res = stmt.executeQuery(queryPartIDRange)) {
+ if (res.next()) {
+ maxID = res.getLong(1);
+ minID = res.getLong(2);
+ }
+ }
+
+ int numOfInvalid = 0;
+ try (PreparedStatement pStmt = conn.prepareStatement(queryPartLoc)) {
+ while (minID <= maxID) {
+ pStmt.setLong(1, minID);
+ pStmt.setLong(2, minID + PART_LOC_CHECK_SIZE);
+ try (ResultSet res = pStmt.executeQuery()) {
+ while (res.next()) {
+ String locValue = res.getString(3);
+ String entity = "Database " + getNameOrID(res, 7, 6) + ", Table " + getNameOrID(res, 5, 4) +
+ ", Partition " + getNameOrID(res, 2, 1);
+ if (!checkLocation(entity, locValue, defaultServers)) {
+ numOfInvalid++;
+ }
+ }
+ }
+ minID += PART_LOC_CHECK_SIZE + 1;
+ }
+ }
+
+ return numOfInvalid == 0;
+ } catch (SQLException e) {
+ throw new HiveMetaException("Failed to get Partition Location Info.", e);
+ }
+ }
+
+ private static final String QUERY_SKEWED_COL_ID_RANGE =
+ "select max(<q>STRING_LIST_ID_KID<q>), " +
+ " min(<q>STRING_LIST_ID_KID<q>) " +
+ " from <q>SKEWED_COL_VALUE_LOC_MAP<q> ";
+
+ private static final String QUERY_SKEWED_COL_LOC =
+ " select t.<q>TBL_NAME<q>, " +
+ " t.<q>TBL_ID<q>, " +
+ " sk.<q>STRING_LIST_ID_KID<q>, " +
+ " sk.<q>LOCATION<q>, " +
+ " db.<q>NAME<q>, " +
+ " db.<q>DB_ID<q> " +
+ " from <q>TBLS<q> t " +
+ " join <q>SDS<q> s on s.<q>SD_ID<q> = t.<q>SD_ID<q> " +
+ " join <q>DBS<q> db on db.<q>DB_ID<q> = t.<q>DB_ID<q> " +
+ " join <q>SKEWED_COL_VALUE_LOC_MAP<q> sk on sk.<q>SD_ID<q> = s.<q>SD_ID<q> " +
+ " where sk.<q>STRING_LIST_ID_KID<q> >= ? " +
+ " and sk.<q>STRING_LIST_ID_KID<q> <= ? " +
+ "order by t.<q>TBL_ID<q> ";
+
+ private static final int SKEWED_COL_LOC_CHECK_SIZE = 2000;
+
+ private boolean checkMetaStoreSkewedColumnsLocation(Connection conn, URI[] defaultServers)
+ throws HiveMetaException {
+ String querySkewedColIDRange = schemaTool.quote(QUERY_SKEWED_COL_ID_RANGE);
+ String querySkewedColLoc = schemaTool.quote(QUERY_SKEWED_COL_LOC);
+
+ try {
+ long maxID = 0, minID = 0;
+ try (Statement stmt = conn.createStatement();
+ ResultSet res = stmt.executeQuery(querySkewedColIDRange)) {
+ if (res.next()) {
+ maxID = res.getLong(1);
+ minID = res.getLong(2);
+ }
+ }
+
+ int numOfInvalid = 0;
+ try (PreparedStatement pStmt = conn.prepareStatement(querySkewedColLoc)) {
+ while (minID <= maxID) {
+ pStmt.setLong(1, minID);
+ pStmt.setLong(2, minID + SKEWED_COL_LOC_CHECK_SIZE);
+ try (ResultSet res = pStmt.executeQuery()) {
+ while (res.next()) {
+ String locValue = res.getString(4);
+ String entity = "Database " + getNameOrID(res, 5, 6) + ", Table " + getNameOrID(res, 1, 2) +
+ ", String list " + res.getString(3);
+ if (!checkLocation(entity, locValue, defaultServers)) {
+ numOfInvalid++;
+ }
+ }
+ }
+ minID += SKEWED_COL_LOC_CHECK_SIZE + 1;
+ }
+ }
+
+ return numOfInvalid == 0;
+ } catch (SQLException e) {
+ throw new HiveMetaException("Failed to get skewed columns location info.", e);
+ }
+ }
+
+ /**
+ * Check if the location is valid for the given entity.
+ * @param entity the entity to represent a database, partition or table
+ * @param entityLocation the location
+ * @param defaultServers a list of the servers that the location needs to match.
+ * The location host needs to match one of the given servers.
+ * If empty, then no check against such list.
+ * @return true if the location is valid
+ */
+ private boolean checkLocation(String entity, String entityLocation, URI[] defaultServers) {
+ boolean isValid = true;
+
+ if (entityLocation == null) {
+ System.err.println(entity + ", Error: empty location");
+ isValid = false;
+ } else {
+ try {
+ URI currentUri = new Path(entityLocation).toUri();
+ String scheme = currentUri.getScheme();
+ String path = currentUri.getPath();
+ if (StringUtils.isEmpty(scheme)) {
+ System.err.println(entity + ", Location: "+ entityLocation + ", Error: missing location scheme.");
+ isValid = false;
+ } else if (StringUtils.isEmpty(path)) {
+ System.err.println(entity + ", Location: "+ entityLocation + ", Error: missing location path.");
+ isValid = false;
+ } else if (ArrayUtils.isNotEmpty(defaultServers) && currentUri.getAuthority() != null) {
+ String authority = currentUri.getAuthority();
+ boolean matchServer = false;
+ for(URI server : defaultServers) {
+ if (StringUtils.equalsIgnoreCase(server.getScheme(), scheme) &&
+ StringUtils.equalsIgnoreCase(server.getAuthority(), authority)) {
+ matchServer = true;
+ break;
+ }
+ }
+ if (!matchServer) {
+ System.err.println(entity + ", Location: " + entityLocation + ", Error: mismatched server.");
+ isValid = false;
+ }
+ }
+
+ // if there is no path element other than "/", report it but not fail
+ if (isValid && StringUtils.containsOnly(path, "/")) {
+ System.err.println(entity + ", Location: "+ entityLocation + ", Warn: location set to root, " +
+ "not a recommended config.");
+ }
+ } catch (Exception pe) {
+ System.err.println(entity + ", Error: invalid location - " + pe.getMessage());
+ isValid =false;
+ }
+ }
+
+ return isValid;
+ }
+
+ private String getNameOrID(ResultSet res, int nameInx, int idInx) throws SQLException {
+ String itemName = res.getString(nameInx);
+ return (itemName == null || itemName.isEmpty()) ? "ID: " + res.getString(idInx) : "Name: " + itemName;
+ }
+
+ private static final String QUERY_COLUMN_NULL_VALUES =
+ " select t.*" +
+ " from <q>TBLS<q> t" +
+ " where t.<q>SD_ID<q> IS NULL" +
+ " and (t.<q>TBL_TYPE<q> = '" + TableType.EXTERNAL_TABLE + "' or" +
+ " t.<q>TBL_TYPE<q> = '" + TableType.MANAGED_TABLE + "') " +
+ "order by t.<q>TBL_ID<q> ";
+
+ @VisibleForTesting
+ boolean validateColumnNullValues(Connection conn) throws HiveMetaException {
+ System.out.println("Validating columns for incorrect NULL values.");
+
+ boolean isValid = true;
+ String queryColumnNullValues = schemaTool.quote(QUERY_COLUMN_NULL_VALUES);
+
+ try (Statement stmt = conn.createStatement();
+ ResultSet res = stmt.executeQuery(queryColumnNullValues)) {
+ while (res.next()) {
+ long tableId = res.getLong("TBL_ID");
+ String tableName = res.getString("TBL_NAME");
+ String tableType = res.getString("TBL_TYPE");
+ isValid = false;
+ System.err.println("SD_ID in TBLS should not be NULL for Table Name=" + tableName + ", Table ID=" + tableId + ", Table Type=" + tableType);
+ }
+
+ System.out.println(isValid ? "[SUCCESS]\n" : "[FAIL]\n");
+ return isValid;
+ } catch(SQLException e) {
+ throw new HiveMetaException("Failed to validate columns for incorrect NULL values", e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/eae5225f/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/DbInstallBase.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/DbInstallBase.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/DbInstallBase.java
index 4722a56..2915720 100644
--- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/DbInstallBase.java
+++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/DbInstallBase.java
@@ -19,8 +19,6 @@ package org.apache.hadoop.hive.metastore.dbinstall;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.hive.metastore.HiveMetaException;
-import org.apache.hadoop.hive.metastore.MetaStoreSchemaInfoFactory;
-import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.hive.metastore.tools.MetastoreSchemaTool;
import org.junit.After;
import org.junit.Assert;
@@ -46,8 +44,6 @@ public abstract class DbInstallBase {
private static final String FIRST_VERSION = "1.2.0";
private static final int MAX_STARTUP_WAIT = 5 * 60 * 1000;
- private String metastoreHome;
-
protected abstract String getDockerContainerName();
protected abstract String getDockerImageName();
protected abstract String[] getDockerAdditionalArgs();
@@ -86,7 +82,7 @@ public abstract class DbInstallBase {
throw new RuntimeException("Container failed to be ready in " + MAX_STARTUP_WAIT/1000 +
" seconds");
}
- MetastoreSchemaTool.homeDir = metastoreHome = System.getProperty("test.tmp.dir", "target/tmp");
+ MetastoreSchemaTool.setHomeDirForTesting();
}
@After
@@ -145,7 +141,7 @@ public abstract class DbInstallBase {
}
private int createUser() {
- return MetastoreSchemaTool.run(buildArray(
+ return new MetastoreSchemaTool().run(buildArray(
"-createUser",
"-dbType",
getDbType(),
@@ -167,7 +163,7 @@ public abstract class DbInstallBase {
}
private int installLatest() {
- return MetastoreSchemaTool.run(buildArray(
+ return new MetastoreSchemaTool().run(buildArray(
"-initSchema",
"-dbType",
getDbType(),
@@ -183,7 +179,7 @@ public abstract class DbInstallBase {
}
private int installAVersion(String version) {
- return MetastoreSchemaTool.run(buildArray(
+ return new MetastoreSchemaTool().run(buildArray(
"-initSchemaTo",
version,
"-dbType",
@@ -200,7 +196,7 @@ public abstract class DbInstallBase {
}
private int upgradeToLatest() {
- return MetastoreSchemaTool.run(buildArray(
+ return new MetastoreSchemaTool().run(buildArray(
"-upgradeSchema",
"-dbType",
getDbType(),
@@ -219,17 +215,6 @@ public abstract class DbInstallBase {
return strs;
}
- private String getCurrentVersionMinusOne() throws HiveMetaException {
- List<String> scripts = MetaStoreSchemaInfoFactory.get(
- MetastoreConf.newMetastoreConf(), metastoreHome, getDbType()
- ).getUpgradeScripts(FIRST_VERSION);
- Assert.assertTrue(scripts.size() > 0);
- String lastUpgradePath = scripts.get(scripts.size() - 1);
- String version = lastUpgradePath.split("-")[1];
- LOG.info("Current version minus 1 is " + version);
- return version;
- }
-
@Test
public void install() {
Assert.assertEquals(0, createUser());
http://git-wip-us.apache.org/repos/asf/hive/blob/eae5225f/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/tools/TestSchemaToolForMetastore.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/tools/TestSchemaToolForMetastore.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/tools/TestSchemaToolForMetastore.java
index 9e425cf..0657ae5 100644
--- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/tools/TestSchemaToolForMetastore.java
+++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/tools/TestSchemaToolForMetastore.java
@@ -27,10 +27,13 @@ import java.io.OutputStream;
import java.io.PrintStream;
import java.net.URI;
import java.sql.Connection;
+import java.sql.DatabaseMetaData;
import java.sql.SQLException;
import java.util.Random;
+import org.apache.commons.dbcp.DelegatingConnection;
import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang.text.StrTokenizer;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.metastore.HiveMetaException;
import org.apache.hadoop.hive.metastore.IMetaStoreSchemaInfo;
@@ -56,6 +59,8 @@ public class TestSchemaToolForMetastore {
private String testMetastoreDB;
private PrintStream errStream;
private PrintStream outStream;
+ private String argsBase;
+ private SchemaToolTaskValidate validator;
@Before
public void setUp() throws HiveMetaException, IOException {
@@ -64,14 +69,21 @@ public class TestSchemaToolForMetastore {
System.setProperty(ConfVars.CONNECT_URL_KEY.toString(),
"jdbc:derby:" + testMetastoreDB + ";create=true");
conf = MetastoreConf.newMetastoreConf();
- schemaTool = new MetastoreSchemaTool(
- System.getProperty("test.tmp.dir", "target/tmp"), conf, "derby");
- schemaTool.setUserName(MetastoreConf.getVar(schemaTool.getConf(), ConfVars.CONNECTION_USER_NAME));
- schemaTool.setPassWord(MetastoreConf.getPassword(schemaTool.getConf(), ConfVars.PWD));
+ schemaTool = new MetastoreSchemaTool();
+ schemaTool.init(System.getProperty("test.tmp.dir", "target/tmp"),
+ new String[]{"-dbType", "derby", "--info"}, null, conf);
+ String userName = MetastoreConf.getVar(schemaTool.getConf(), ConfVars.CONNECTION_USER_NAME);
+ String passWord = MetastoreConf.getPassword(schemaTool.getConf(), ConfVars.PWD);
+ schemaTool.setUserName(userName);
+ schemaTool.setPassWord(passWord);
+ argsBase = "-dbType derby -userName " + userName + " -passWord " + passWord + " ";
System.setProperty("beeLine.system.exit", "true");
errStream = System.err;
outStream = System.out;
conn = schemaTool.getConnectionToMetastore(false);
+
+ validator = new SchemaToolTaskValidate();
+ validator.setHiveSchemaTool(schemaTool);
}
@After
@@ -87,24 +99,26 @@ public class TestSchemaToolForMetastore {
}
}
- // Test the sequence validation functionality
+ /*
+ * Test the sequence validation functionality
+ */
@Test
public void testValidateSequences() throws Exception {
- schemaTool.doInit();
+ execute(new SchemaToolTaskInit(), "-initSchema");
// Test empty database
- boolean isValid = schemaTool.validateSequences(conn);
+ boolean isValid = validator.validateSequences(conn);
Assert.assertTrue(isValid);
// Test valid case
String[] scripts = new String[] {
+ "insert into CTLGS values(99, 'test_cat_1', 'description', 'hdfs://myhost.com:8020/user/hive/warehouse/mydb');",
"insert into SEQUENCE_TABLE values('org.apache.hadoop.hive.metastore.model.MDatabase', 100);",
- "insert into CTLGS values(37, 'mycat', 'my description', 'hdfs://tmp');",
- "insert into DBS values(99, 'test db1', 'hdfs:///tmp', 'db1', 'test', 'test', 'mycat');"
+ "insert into DBS values(99, 'test db1', 'hdfs:///tmp', 'db1', 'test', 'test', 'test_cat_1');"
};
File scriptFile = generateTestScript(scripts);
- schemaTool.runSqlLine(scriptFile.getPath());
- isValid = schemaTool.validateSequences(conn);
+ schemaTool.execSql(scriptFile.getPath());
+ isValid = validator.validateSequences(conn);
Assert.assertTrue(isValid);
// Test invalid case
@@ -112,25 +126,27 @@ public class TestSchemaToolForMetastore {
"delete from SEQUENCE_TABLE;",
"delete from DBS;",
"insert into SEQUENCE_TABLE values('org.apache.hadoop.hive.metastore.model.MDatabase', 100);",
- "insert into DBS values(102, 'test db1', 'hdfs:///tmp', 'db1', 'test', 'test', 'mycat');"
+ "insert into DBS values(102, 'test db1', 'hdfs:///tmp', 'db1', 'test', 'test', 'test_cat_1');"
};
scriptFile = generateTestScript(scripts);
- schemaTool.runSqlLine(scriptFile.getPath());
- isValid = schemaTool.validateSequences(conn);
+ schemaTool.execSql(scriptFile.getPath());
+ isValid = validator.validateSequences(conn);
Assert.assertFalse(isValid);
}
- // Test to validate that all tables exist in the HMS metastore.
+ /*
+ * Test to validate that all tables exist in the HMS metastore.
+ */
@Test
public void testValidateSchemaTables() throws Exception {
- schemaTool.doInit("1.2.0");
+ execute(new SchemaToolTaskInit(), "-initSchemaTo 1.2.0");
- boolean isValid = schemaTool.validateSchemaTables(conn);
+ boolean isValid = validator.validateSchemaTables(conn);
Assert.assertTrue(isValid);
- // upgrade from 2.0.0 schema and re-validate
- schemaTool.doUpgrade("1.2.0");
- isValid = schemaTool.validateSchemaTables(conn);
+ // upgrade from 1.2.0 schema and re-validate
+ execute(new SchemaToolTaskUpgrade(), "-upgradeSchemaFrom 1.2.0");
+ isValid = validator.validateSchemaTables(conn);
Assert.assertTrue(isValid);
// Simulate a missing table scenario by renaming a couple of tables
@@ -140,8 +156,8 @@ public class TestSchemaToolForMetastore {
};
File scriptFile = generateTestScript(scripts);
- schemaTool.runSqlLine(scriptFile.getPath());
- isValid = schemaTool.validateSchemaTables(conn);
+ schemaTool.execSql(scriptFile.getPath());
+ isValid = validator.validateSchemaTables(conn);
Assert.assertFalse(isValid);
// Restored the renamed tables
@@ -151,31 +167,49 @@ public class TestSchemaToolForMetastore {
};
scriptFile = generateTestScript(scripts);
- schemaTool.runSqlLine(scriptFile.getPath());
- isValid = schemaTool.validateSchemaTables(conn);
+ schemaTool.execSql(scriptFile.getPath());
+ isValid = validator.validateSchemaTables(conn);
Assert.assertTrue(isValid);
- }
+
+ // Check that an exception from getMetaData() is reported correctly
+ try {
+ // Make a Connection object that will throw an exception
+ BadMetaDataConnection bad = new BadMetaDataConnection(conn);
+ validator.validateSchemaTables(bad);
+ Assert.fail("did not get expected exception");
+ } catch (HiveMetaException hme) {
+ String message = hme.getMessage();
+ Assert.assertTrue("Bad HiveMetaException message :" + message,
+ message.contains("Failed to retrieve schema tables from Hive Metastore DB"));
+ Throwable cause = hme.getCause();
+ Assert.assertNotNull("HiveMetaException did not contain a cause", cause);
+ String causeMessage = cause.getMessage();
+ Assert.assertTrue("Bad SQLException message: " + causeMessage, causeMessage.contains(
+ BadMetaDataConnection.FAILURE_TEXT));
+ }
+ }
+
// Test the validation of incorrect NULL values in the tables
@Test
public void testValidateNullValues() throws Exception {
- schemaTool.doInit();
+ execute(new SchemaToolTaskInit(), "-initSchema");
// Test empty database
- boolean isValid = schemaTool.validateColumnNullValues(conn);
+ boolean isValid = validator.validateColumnNullValues(conn);
Assert.assertTrue(isValid);
// Test valid case
createTestHiveTableSchemas();
- isValid = schemaTool.validateColumnNullValues(conn);
+ isValid = validator.validateColumnNullValues(conn);
// Test invalid case
String[] scripts = new String[] {
"update TBLS set SD_ID=null"
};
File scriptFile = generateTestScript(scripts);
- schemaTool.runSqlLine(scriptFile.getPath());
- isValid = schemaTool.validateColumnNullValues(conn);
+ schemaTool.execSql(scriptFile.getPath());
+ isValid = validator.validateColumnNullValues(conn);
Assert.assertFalse(isValid);
}
@@ -183,7 +217,7 @@ public class TestSchemaToolForMetastore {
@Test
public void testSchemaInitDryRun() throws Exception {
schemaTool.setDryRun(true);
- schemaTool.doInit("3.0.0");
+ execute(new SchemaToolTaskInit(), "-initSchemaTo 1.2.0");
schemaTool.setDryRun(false);
try {
schemaTool.verifySchemaVersion();
@@ -197,10 +231,10 @@ public class TestSchemaToolForMetastore {
// Test dryrun of schema upgrade
@Test
public void testSchemaUpgradeDryRun() throws Exception {
- schemaTool.doInit("1.2.0");
+ execute(new SchemaToolTaskInit(), "-initSchemaTo 1.2.0");
schemaTool.setDryRun(true);
- schemaTool.doUpgrade("1.2.0");
+ execute(new SchemaToolTaskUpgrade(), "-upgradeSchemaFrom 1.2.0");
schemaTool.setDryRun(false);
try {
schemaTool.verifySchemaVersion();
@@ -218,8 +252,7 @@ public class TestSchemaToolForMetastore {
public void testSchemaInit() throws Exception {
IMetaStoreSchemaInfo metastoreSchemaInfo = MetaStoreSchemaInfoFactory.get(conf,
System.getProperty("test.tmp.dir", "target/tmp"), "derby");
- LOG.info("Starting testSchemaInit");
- schemaTool.doInit(metastoreSchemaInfo.getHiveSchemaVersion());
+ execute(new SchemaToolTaskInit(), "-initSchemaTo " + metastoreSchemaInfo.getHiveSchemaVersion());
schemaTool.verifySchemaVersion();
}
@@ -227,35 +260,35 @@ public class TestSchemaToolForMetastore {
* Test validation for schema versions
*/
@Test
- public void testValidateSchemaVersions() throws Exception {
- schemaTool.doInit();
- boolean isValid = schemaTool.validateSchemaVersions();
- // Test an invalid case with multiple versions
- String[] scripts = new String[] {
- "insert into VERSION values(100, '2.2.0', 'Hive release version 2.2.0')"
- };
- File scriptFile = generateTestScript(scripts);
- schemaTool.runSqlLine(scriptFile.getPath());
- isValid = schemaTool.validateSchemaVersions();
- Assert.assertFalse(isValid);
-
- scripts = new String[] {
- "delete from VERSION where VER_ID = 100"
- };
- scriptFile = generateTestScript(scripts);
- schemaTool.runSqlLine(scriptFile.getPath());
- isValid = schemaTool.validateSchemaVersions();
- Assert.assertTrue(isValid);
-
- // Test an invalid case without version
- scripts = new String[] {
- "delete from VERSION"
- };
- scriptFile = generateTestScript(scripts);
- schemaTool.runSqlLine(scriptFile.getPath());
- isValid = schemaTool.validateSchemaVersions();
- Assert.assertFalse(isValid);
- }
+ public void testValidateSchemaVersions() throws Exception {
+ execute(new SchemaToolTaskInit(), "-initSchema");
+ boolean isValid = validator.validateSchemaVersions();
+ // Test an invalid case with multiple versions
+ String[] scripts = new String[] {
+ "insert into VERSION values(100, '2.2.0', 'Hive release version 2.2.0')"
+ };
+ File scriptFile = generateTestScript(scripts);
+ schemaTool.execSql(scriptFile.getPath());
+ isValid = validator.validateSchemaVersions();
+ Assert.assertFalse(isValid);
+
+ scripts = new String[] {
+ "delete from VERSION where VER_ID = 100"
+ };
+ scriptFile = generateTestScript(scripts);
+ schemaTool.execSql(scriptFile.getPath());
+ isValid = validator.validateSchemaVersions();
+ Assert.assertTrue(isValid);
+
+ // Test an invalid case without version
+ scripts = new String[] {
+ "delete from VERSION"
+ };
+ scriptFile = generateTestScript(scripts);
+ schemaTool.execSql(scriptFile.getPath());
+ isValid = validator.validateSchemaVersions();
+ Assert.assertFalse(isValid);
+ }
/**
* Test schema upgrade
@@ -264,7 +297,7 @@ public class TestSchemaToolForMetastore {
public void testSchemaUpgrade() throws Exception {
boolean foundException = false;
// Initialize 1.2.0 schema
- schemaTool.doInit("1.2.0");
+ execute(new SchemaToolTaskInit(), "-initSchemaTo 1.2.0");
// verify that driver fails due to older version schema
try {
schemaTool.verifySchemaVersion();
@@ -298,17 +331,7 @@ public class TestSchemaToolForMetastore {
System.setOut(outPrintStream);
// Upgrade schema from 0.7.0 to latest
- Exception caught = null;
- try {
- schemaTool.doUpgrade("1.2.0");
- } catch (Exception e) {
- caught = e;
- }
-
- LOG.info("stdout is " + stdout.toString());
- LOG.info("stderr is " + stderr.toString());
-
- if (caught != null) Assert.fail(caught.getMessage());
+ execute(new SchemaToolTaskUpgrade(), "-upgradeSchemaFrom 1.2.0");
// Verify that the schemaTool ran pre-upgrade scripts and ignored errors
Assert.assertTrue(stderr.toString().contains(invalidPreUpgradeScript));
@@ -327,38 +350,38 @@ public class TestSchemaToolForMetastore {
*/
@Test
public void testValidateLocations() throws Exception {
- schemaTool.doInit();
+ execute(new SchemaToolTaskInit(), "-initSchema");
URI defaultRoot = new URI("hdfs://myhost.com:8020");
URI defaultRoot2 = new URI("s3://myhost2.com:8888");
//check empty DB
- boolean isValid = schemaTool.validateLocations(conn, null);
+ boolean isValid = validator.validateLocations(conn, null);
Assert.assertTrue(isValid);
- isValid = schemaTool.validateLocations(conn, new URI[] {defaultRoot,defaultRoot2});
+ isValid = validator.validateLocations(conn, new URI[] {defaultRoot, defaultRoot2});
Assert.assertTrue(isValid);
- // Test valid case
+ // Test valid case
String[] scripts = new String[] {
- "insert into CTLGS values (1, 'mycat', 'mydescription', 'hdfs://myhost.com:8020/user/hive/warehouse');",
- "insert into DBS values(2, 'my db', 'hdfs://myhost.com:8020/user/hive/warehouse/mydb', 'mydb', 'public', 'role', 'mycat');",
- "insert into DBS values(7, 'db with bad port', 'hdfs://myhost.com:8020/', 'haDB', 'public', 'role', 'mycat');",
- "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (1,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020/user/hive/warehouse/mydb',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);",
- "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (2,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020/user/admin/2015_11_18',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);",
- "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (3,null,'org.apache.hadoop.mapred.TextInputFormat','N','N',null,-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);",
- "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (4000,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020/',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);",
- "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (2 ,1435255431,2,0 ,'hive',0,1,'mytal','MANAGED_TABLE',NULL,NULL,'n');",
- "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (3 ,1435255431,2,0 ,'hive',0,3,'myView','VIRTUAL_VIEW','select a.col1,a.col2 from foo','select * from foo','n');",
- "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (4012 ,1435255431,7,0 ,'hive',0,4000,'mytal4012','MANAGED_TABLE',NULL,NULL,'n');",
- "insert into PARTITIONS(PART_ID,CREATE_TIME,LAST_ACCESS_TIME, PART_NAME,SD_ID,TBL_ID) values(1, 1441402388,0, 'd1=1/d2=1',2,2);",
- "insert into SKEWED_STRING_LIST values(1);",
- "insert into SKEWED_STRING_LIST values(2);",
- "insert into SKEWED_COL_VALUE_LOC_MAP values(1,1,'hdfs://myhost.com:8020/user/hive/warehouse/mytal/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME/');",
- "insert into SKEWED_COL_VALUE_LOC_MAP values(2,2,'s3://myhost.com:8020/user/hive/warehouse/mytal/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME/');"
- };
+ "insert into CTLGS values(3, 'test_cat_2', 'description', 'hdfs://myhost.com:8020/user/hive/warehouse/mydb');",
+ "insert into DBS values(2, 'my db', 'hdfs://myhost.com:8020/user/hive/warehouse/mydb', 'mydb', 'public', 'role', 'test_cat_2');",
+ "insert into DBS values(7, 'db with bad port', 'hdfs://myhost.com:8020/', 'haDB', 'public', 'role', 'test_cat_2');",
+ "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (1,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020/user/hive/warehouse/mydb',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);",
+ "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (2,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020/user/admin/2015_11_18',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);",
+ "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (3,null,'org.apache.hadoop.mapred.TextInputFormat','N','N',null,-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);",
+ "insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (4000,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://myhost.com:8020/',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);",
+ "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (2 ,1435255431,2,0 ,'hive',0,1,'mytal','MANAGED_TABLE',NULL,NULL,'n');",
+ "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (3 ,1435255431,2,0 ,'hive',0,3,'myView','VIRTUAL_VIEW','select a.col1,a.col2 from foo','select * from foo','n');",
+ "insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (4012 ,1435255431,7,0 ,'hive',0,4000,'mytal4012','MANAGED_TABLE',NULL,NULL,'n');",
+ "insert into PARTITIONS(PART_ID,CREATE_TIME,LAST_ACCESS_TIME, PART_NAME,SD_ID,TBL_ID) values(1, 1441402388,0, 'd1=1/d2=1',2,2);",
+ "insert into SKEWED_STRING_LIST values(1);",
+ "insert into SKEWED_STRING_LIST values(2);",
+ "insert into SKEWED_COL_VALUE_LOC_MAP values(1,1,'hdfs://myhost.com:8020/user/hive/warehouse/mytal/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME/');",
+ "insert into SKEWED_COL_VALUE_LOC_MAP values(2,2,'s3://myhost.com:8020/user/hive/warehouse/mytal/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME/');"
+ };
File scriptFile = generateTestScript(scripts);
- schemaTool.runSqlLine(scriptFile.getPath());
- isValid = schemaTool.validateLocations(conn, null);
+ schemaTool.execSql(scriptFile.getPath());
+ isValid = validator.validateLocations(conn, null);
Assert.assertTrue(isValid);
- isValid = schemaTool.validateLocations(conn, new URI[] {defaultRoot, defaultRoot2});
+ isValid = validator.validateLocations(conn, new URI[] {defaultRoot, defaultRoot2});
Assert.assertTrue(isValid);
scripts = new String[] {
"delete from SKEWED_COL_VALUE_LOC_MAP;",
@@ -367,10 +390,10 @@ public class TestSchemaToolForMetastore {
"delete from TBLS;",
"delete from SDS;",
"delete from DBS;",
- "insert into DBS values(2, 'my db', '/user/hive/warehouse/mydb', 'mydb', 'public', 'role', 'mycat');",
- "insert into DBS values(4, 'my db2', 'hdfs://myhost.com:8020', '', 'public', 'role', 'mycat');",
- "insert into DBS values(6, 'db with bad port', 'hdfs://myhost.com:8020:', 'zDB', 'public', 'role', 'mycat');",
- "insert into DBS values(7, 'db with bad port', 'hdfs://mynameservice.com/', 'haDB', 'public', 'role', 'mycat');",
+ "insert into DBS values(2, 'my db', '/user/hive/warehouse/mydb', 'mydb', 'public', 'role', 'test_cat_2');",
+ "insert into DBS values(4, 'my db2', 'hdfs://myhost.com:8020', '', 'public', 'role', 'test_cat_2');",
+ "insert into DBS values(6, 'db with bad port', 'hdfs://myhost.com:8020:', 'zDB', 'public', 'role', 'test_cat_2');",
+ "insert into DBS values(7, 'db with bad port', 'hdfs://mynameservice.com/', 'haDB', 'public', 'role', 'test_cat_2');",
"insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (1,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','hdfs://yourhost.com:8020/user/hive/warehouse/mydb',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);",
"insert into SDS(SD_ID,CD_ID,INPUT_FORMAT,IS_COMPRESSED,IS_STOREDASSUBDIRECTORIES,LOCATION,NUM_BUCKETS,OUTPUT_FORMAT,SERDE_ID) values (2,null,'org.apache.hadoop.mapred.TextInputFormat','N','N','file:///user/admin/2015_11_18',-1,'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',null);",
"insert into TBLS(TBL_ID,CREATE_TIME,DB_ID,LAST_ACCESS_TIME,OWNER,RETENTION,SD_ID,TBL_NAME,TBL_TYPE,VIEW_EXPANDED_TEXT,VIEW_ORIGINAL_TEXT,IS_REWRITE_ENABLED) values (2 ,1435255431,2,0 ,'hive',0,1,'mytal','MANAGED_TABLE',NULL,NULL,'n');",
@@ -398,23 +421,23 @@ public class TestSchemaToolForMetastore {
"insert into SKEWED_COL_VALUE_LOC_MAP values(2,2,'file:///user/admin/warehouse/mytal/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME/');"
};
scriptFile = generateTestScript(scripts);
- schemaTool.runSqlLine(scriptFile.getPath());
- isValid = schemaTool.validateLocations(conn, null);
+ schemaTool.execSql(scriptFile.getPath());
+ isValid = validator.validateLocations(conn, null);
Assert.assertFalse(isValid);
- isValid = schemaTool.validateLocations(conn, new URI[] {defaultRoot, defaultRoot2});
+ isValid = validator.validateLocations(conn, new URI[] {defaultRoot, defaultRoot2});
Assert.assertFalse(isValid);
}
@Test
public void testHiveMetastoreDbPropertiesTable() throws HiveMetaException, IOException {
- schemaTool.doInit("3.0.0");
+ execute(new SchemaToolTaskInit(), "-initSchemaTo 3.0.0");
validateMetastoreDbPropertiesTable();
}
@Test
public void testMetastoreDbPropertiesAfterUpgrade() throws HiveMetaException, IOException {
- schemaTool.doInit("1.2.0");
- schemaTool.doUpgrade();
+ execute(new SchemaToolTaskInit(), "-initSchemaTo 1.2.0");
+ execute(new SchemaToolTaskUpgrade(), "-upgradeSchema");
validateMetastoreDbPropertiesTable();
}
@@ -432,7 +455,7 @@ public class TestSchemaToolForMetastore {
}
private void validateMetastoreDbPropertiesTable() throws HiveMetaException, IOException {
- boolean isValid = schemaTool.validateSchemaTables(conn);
+ boolean isValid = (boolean) validator.validateSchemaTables(conn);
Assert.assertTrue(isValid);
// adding same property key twice should throw unique key constraint violation exception
String[] scripts = new String[] {
@@ -441,12 +464,13 @@ public class TestSchemaToolForMetastore {
File scriptFile = generateTestScript(scripts);
Exception ex = null;
try {
- schemaTool.runSqlLine(scriptFile.getPath());
+ schemaTool.execSql(scriptFile.getPath());
} catch (Exception iox) {
ex = iox;
}
Assert.assertTrue(ex != null && ex instanceof IOException);
}
+
/**
* Write out a dummy pre-upgrade script with given SQL statement.
*/
@@ -476,6 +500,35 @@ public class TestSchemaToolForMetastore {
"insert into PARTITIONS(PART_ID,CREATE_TIME,LAST_ACCESS_TIME, PART_NAME,SD_ID,TBL_ID) values(1, 1441402388,0, 'd1=1/d2=1',2,2);"
};
File scriptFile = generateTestScript(scripts);
- schemaTool.runSqlLine(scriptFile.getPath());
+ schemaTool.execSql(scriptFile.getPath());
+ }
+
+ /**
+ * A mock Connection class that throws an exception out of getMetaData().
+ */
+ class BadMetaDataConnection extends DelegatingConnection {
+ static final String FAILURE_TEXT = "fault injected";
+
+ BadMetaDataConnection(Connection connection) {
+ super(connection);
+ }
+
+ @Override
+ public DatabaseMetaData getMetaData() throws SQLException {
+ throw new SQLException(FAILURE_TEXT);
+ }
+ }
+
+ private void execute(SchemaToolTask task, String taskArgs) throws HiveMetaException {
+ try {
+ StrTokenizer tokenizer = new StrTokenizer(argsBase + taskArgs, ' ', '\"');
+ SchemaToolCommandLine cl = new SchemaToolCommandLine(tokenizer.getTokenArray(), null);
+ task.setCommandLineArguments(cl);
+ } catch (Exception e) {
+ throw new IllegalStateException("Could not parse comman line \n" + argsBase + taskArgs, e);
+ }
+
+ task.setHiveSchemaTool(schemaTool);
+ task.execute();
}
}