You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2017/10/10 00:52:19 UTC
[45/61] [abbrv] hive git commit: HIVE-17706: Add a possibility to run
the BeeLine tests on the default database (Peter Vary,
reviewed by Barna Zsombor Klara)
HIVE-17706: Add a possibility to run the BeeLine tests on the default database (Peter Vary, reviewed by Barna Zsombor Klara)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/1dccdead
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/1dccdead
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/1dccdead
Branch: refs/heads/hive-14535
Commit: 1dccdeadaba35a0206b17d512ef1cf9aced374b8
Parents: 75c31e7
Author: Peter Vary <pv...@cloudera.com>
Authored: Mon Oct 9 12:09:06 2017 +0200
Committer: Peter Vary <pv...@cloudera.com>
Committed: Mon Oct 9 12:09:06 2017 +0200
----------------------------------------------------------------------
.../hive/cli/control/CoreBeeLineDriver.java | 6 +-
.../java/org/apache/hive/beeline/QFile.java | 65 ++++++---
.../apache/hive/beeline/QFileBeeLineClient.java | 138 +++++++++++++++----
3 files changed, 159 insertions(+), 50 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/1dccdead/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreBeeLineDriver.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreBeeLineDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreBeeLineDriver.java
index b44ffbd..66ccd6f 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreBeeLineDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreBeeLineDriver.java
@@ -49,7 +49,7 @@ public class CoreBeeLineDriver extends CliAdapter {
private final File testDataDirectory;
private final File testScriptDirectory;
private boolean overwrite = false;
- private boolean rewriteSourceTables = true;
+ private boolean useSharedDatabase = false;
private MiniHS2 miniHS2;
private QFileClientBuilder clientBuilder;
private QFileBuilder fileBuilder;
@@ -111,7 +111,7 @@ public class CoreBeeLineDriver extends CliAdapter {
public void beforeClass() throws Exception {
overwrite = getBooleanPropertyValue("test.output.overwrite", Boolean.FALSE);
- rewriteSourceTables = getBooleanPropertyValue("test.rewrite.source.tables", Boolean.TRUE);
+ useSharedDatabase = getBooleanPropertyValue("test.beeline.shared.database", Boolean.FALSE);
String beeLineUrl = System.getProperty("test.beeline.url");
if (StringUtils.isEmpty(beeLineUrl)) {
@@ -132,7 +132,7 @@ public class CoreBeeLineDriver extends CliAdapter {
.setLogDirectory(logDirectory)
.setQueryDirectory(queryDirectory)
.setResultsDirectory(resultsDirectory)
- .setRewriteSourceTables(rewriteSourceTables)
+ .setUseSharedDatabase(useSharedDatabase)
.setComparePortable(comparePortable);
runInfraScript(initScript, new File(logDirectory, "init.beeline"),
http://git-wip-us.apache.org/repos/asf/hive/blob/1dccdead/itests/util/src/main/java/org/apache/hive/beeline/QFile.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hive/beeline/QFile.java b/itests/util/src/main/java/org/apache/hive/beeline/QFile.java
index 38b0d91..21be8b0 100644
--- a/itests/util/src/main/java/org/apache/hive/beeline/QFile.java
+++ b/itests/util/src/main/java/org/apache/hive/beeline/QFile.java
@@ -81,7 +81,7 @@ public final class QFile {
private static RegexFilterSet staticFilterSet = getStaticFilterSet();
private static RegexFilterSet portableFilterSet = getPortableFilterSet();
private RegexFilterSet specificFilterSet;
- private boolean rewriteSourceTables;
+ private boolean useSharedDatabase;
private Converter converter;
private boolean comparePortable;
@@ -91,6 +91,7 @@ public final class QFile {
return name;
}
+
public String getDatabaseName() {
return databaseName;
}
@@ -127,6 +128,10 @@ public final class QFile {
return converter;
}
+ public boolean isUseSharedDatabase() {
+ return useSharedDatabase;
+ }
+
public String getDebugHint() {
return String.format(DEBUG_HINT, inputFile, rawOutputFile, outputFile, expectedOutputFile,
logFile, beforeExecuteLogFile, afterExecuteLogFile,
@@ -134,13 +139,13 @@ public final class QFile {
}
/**
- * Filters the sql commands if necessary.
+ * Filters the sql commands if necessary - eg. not using the shared database.
* @param commands The array of the sql commands before filtering
* @return The filtered array of the sql command strings
* @throws IOException File read error
*/
public String[] filterCommands(String[] commands) throws IOException {
- if (rewriteSourceTables) {
+ if (!useSharedDatabase) {
for (int i=0; i<commands.length; i++) {
if (USE_PATTERN.matcher(commands[i]).matches()) {
System.err.println(String.format(USE_COMMAND_WARNING, inputFile, commands[i]));
@@ -177,8 +182,8 @@ public final class QFile {
*/
private String revertReplaceTableNames(String source) {
for (String table : srcTables) {
- source = source.replaceAll("(?is)(?<!name:?|alias:?)(\\s+)default\\." + table
- + "([\\s;\\n\\)])", "$1" + table + "$2");
+ source = source.replaceAll("(?is)(?<!name:?|alias:?)(\\s+)default\\.(" + table
+ + ")([\\s;\\n\\),])", "$1$2$3");
}
return source;
}
@@ -199,18 +204,28 @@ public final class QFile {
return source;
}
+ /**
+ * Filters the generated output file
+ * @throws IOException
+ */
public void filterOutput() throws IOException {
String output = FileUtils.readFileToString(rawOutputFile, "UTF-8");
if (comparePortable) {
output = portableFilterSet.filter(output);
}
output = staticFilterSet.filter(specificFilterSet.filter(output));
- if (rewriteSourceTables) {
+ if (!useSharedDatabase) {
output = sortInputOutput(revertReplaceTableNames(output));
}
FileUtils.writeStringToFile(outputFile, output);
}
+ /**
+ * Compare the filtered file with the expected golden file
+ * @return The comparison data
+ * @throws IOException If there is a problem accessing the golden or generated file
+ * @throws InterruptedException If there is a problem running the diff command
+ */
public QTestProcessExecResult compareResults() throws IOException, InterruptedException {
if (!expectedOutputFile.exists()) {
throw new IOException("Expected results file does not exist: " + expectedOutputFile);
@@ -218,6 +233,10 @@ public final class QFile {
return executeDiff();
}
+ /**
+ * Overwrite the golden file with the generated output
+ * @throws IOException If there is a problem accessing the golden or generated file
+ */
public void overwriteResults() throws IOException {
FileUtils.copyFile(outputFile, expectedOutputFile);
}
@@ -316,8 +335,11 @@ public final class QFile {
.addFilter(".*/tmp/.*\n", MASK_PATTERN)
.addFilter(".*file:.*\n", MASK_PATTERN)
.addFilter(".*file\\..*\n", MASK_PATTERN)
+ .addFilter(".*Location.*\n", MASK_PATTERN)
+ .addFilter(".*LOCATION '.*\n", MASK_PATTERN)
.addFilter(".*Output:.*/data/files/.*\n", MASK_PATTERN)
.addFilter(".*CreateTime.*\n", MASK_PATTERN)
+ .addFilter(".*last_modified_.*\n", MASK_PATTERN)
.addFilter(".*transient_lastDdlTime.*\n", MASK_PATTERN)
.addFilter(".*lastUpdateTime.*\n", MASK_PATTERN)
.addFilter(".*lastAccessTime.*\n", MASK_PATTERN)
@@ -349,7 +371,7 @@ public final class QFile {
private File queryDirectory;
private File logDirectory;
private File resultsDirectory;
- private boolean rewriteSourceTables;
+ private boolean useSharedDatabase;
private boolean comparePortable;
public QFileBuilder() {
@@ -370,8 +392,8 @@ public final class QFile {
return this;
}
- public QFileBuilder setRewriteSourceTables(boolean rewriteSourceTables) {
- this.rewriteSourceTables = rewriteSourceTables;
+ public QFileBuilder setUseSharedDatabase(boolean useSharedDatabase) {
+ this.useSharedDatabase = useSharedDatabase;
return this;
}
@@ -383,22 +405,27 @@ public final class QFile {
public QFile getQFile(String name) throws IOException {
QFile result = new QFile();
result.name = name;
- result.databaseName = "test_db_" + name;
+ if (!useSharedDatabase) {
+ result.databaseName = "test_db_" + name.toLowerCase();
+ result.specificFilterSet = new RegexFilterSet()
+ .addFilter("(PREHOOK|POSTHOOK): (Output|Input): database:" + result.databaseName + "\n",
+ "$1: $2: database:default\n")
+ .addFilter("(PREHOOK|POSTHOOK): (Output|Input): " + result.databaseName + "@",
+ "$1: $2: default@")
+ .addFilter("name(:?) " + result.databaseName + "\\.(.*)\n", "name$1 default.$2\n")
+ .addFilter("alias(:?) " + result.databaseName + "\\.(.*)\n", "alias$1 default.$2\n")
+ .addFilter("/" + result.databaseName + ".db/", "/");
+ } else {
+ result.databaseName = "default";
+ result.specificFilterSet = new RegexFilterSet();
+ }
result.inputFile = new File(queryDirectory, name + ".q");
result.rawOutputFile = new File(logDirectory, name + ".q.out.raw");
result.outputFile = new File(logDirectory, name + ".q.out");
result.logFile = new File(logDirectory, name + ".q.beeline");
result.beforeExecuteLogFile = new File(logDirectory, name + ".q.beforeExecute.log");
result.afterExecuteLogFile = new File(logDirectory, name + ".q.afterExecute.log");
- result.rewriteSourceTables = rewriteSourceTables;
- result.specificFilterSet = new RegexFilterSet()
- .addFilter("(PREHOOK|POSTHOOK): (Output|Input): database:" + result.databaseName + "\n",
- "$1: $2: database:default\n")
- .addFilter("(PREHOOK|POSTHOOK): (Output|Input): " + result.databaseName + "@",
- "$1: $2: default@")
- .addFilter("name(:?) " + result.databaseName + "\\.(.*)\n", "name$1 default.$2\n")
- .addFilter("alias(:?) " + result.databaseName + "\\.(.*)\n", "alias$1 default.$2\n")
- .addFilter("/" + result.databaseName + ".db/", "/");
+ result.useSharedDatabase = useSharedDatabase;
result.converter = Converter.NONE;
String input = FileUtils.readFileToString(result.inputFile, "UTF-8");
if (input.contains("-- SORT_QUERY_RESULTS")) {
http://git-wip-us.apache.org/repos/asf/hive/blob/1dccdead/itests/util/src/main/java/org/apache/hive/beeline/QFileBeeLineClient.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hive/beeline/QFileBeeLineClient.java b/itests/util/src/main/java/org/apache/hive/beeline/QFileBeeLineClient.java
index 2f91834..d84ff39 100644
--- a/itests/util/src/main/java/org/apache/hive/beeline/QFileBeeLineClient.java
+++ b/itests/util/src/main/java/org/apache/hive/beeline/QFileBeeLineClient.java
@@ -18,12 +18,19 @@
package org.apache.hive.beeline;
+import org.apache.commons.lang3.ArrayUtils;
+import org.apache.hadoop.hive.ql.QTestUtil;
import org.apache.hive.beeline.ConvertedOutputFile.Converter;
import java.io.File;
import java.io.IOException;
import java.io.PrintStream;
+import java.sql.DatabaseMetaData;
+import java.sql.ResultSet;
import java.sql.SQLException;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.stream.Collectors;
/**
* QFile test client using BeeLine. It can be used to submit a list of command strings, or a QFile.
@@ -32,6 +39,26 @@ public class QFileBeeLineClient implements AutoCloseable {
private BeeLine beeLine;
private PrintStream beelineOutputStream;
private File logFile;
+ private String[] TEST_FIRST_COMMANDS = new String[] {
+ "!set outputformat tsv2",
+ "!set verbose false",
+ "!set silent true",
+ "!set showheader false",
+ "USE default;",
+ "SHOW TABLES;",
+ };
+ private String[] TEST_SET_LOG_COMMANDS = new String[] {
+ "set hive.testing.short.logs=true;",
+ "set hive.testing.remove.logs=false;",
+ };
+ private String[] TEST_RESET_COMMANDS = new String[] {
+ "set hive.testing.short.logs=false;",
+ "!set verbose true",
+ "!set silent false",
+ "!set showheader true",
+ "!set outputformat table",
+ "USE default;"
+ };
protected QFileBeeLineClient(String jdbcUrl, String jdbcDriver, String username, String password,
File log) throws IOException {
@@ -52,6 +79,47 @@ public class QFileBeeLineClient implements AutoCloseable {
});
}
+ private Set<String> getDatabases() throws SQLException {
+ Set<String> databases = new HashSet<String>();
+
+ DatabaseMetaData metaData = beeLine.getDatabaseMetaData();
+ // Get the databases
+ try (ResultSet schemasResultSet = metaData.getSchemas()) {
+ while (schemasResultSet.next()) {
+ databases.add(schemasResultSet.getString("TABLE_SCHEM"));
+ }
+ }
+ return databases;
+ }
+
+ private Set<String> getTables() throws SQLException {
+ Set<String> tables = new HashSet<String>();
+
+ DatabaseMetaData metaData = beeLine.getDatabaseMetaData();
+ // Get the tables in the default database
+ String[] types = new String[] {"TABLE"};
+ try (ResultSet tablesResultSet = metaData.getTables(null, "default", "%", types)) {
+ while (tablesResultSet.next()) {
+ tables.add(tablesResultSet.getString("TABLE_NAME"));
+ }
+ }
+ return tables;
+ }
+
+ private Set<String> getViews() throws SQLException {
+ Set<String> views = new HashSet<String>();
+
+ DatabaseMetaData metaData = beeLine.getDatabaseMetaData();
+ // Get the tables in the default database
+ String[] types = new String[] {"VIEW"};
+ try (ResultSet tablesResultSet = metaData.getTables(null, "default", "%", types)) {
+ while (tablesResultSet.next()) {
+ views.add(tablesResultSet.getString("TABLE_NAME"));
+ }
+ }
+ return views;
+ }
+
public void execute(String[] commands, File resultFile, Converter converter)
throws Exception {
beeLine.runCommands(
@@ -69,39 +137,53 @@ public class QFileBeeLineClient implements AutoCloseable {
}
private void beforeExecute(QFile qFile) throws Exception {
- execute(
- new String[] {
- "!set outputformat tsv2",
- "!set verbose false",
- "!set silent true",
- "!set showheader false",
- "USE default;",
- "SHOW TABLES;",
- "DROP DATABASE IF EXISTS `" + qFile.getDatabaseName() + "` CASCADE;",
- "CREATE DATABASE `" + qFile.getDatabaseName() + "`;",
- "USE `" + qFile.getDatabaseName() + "`;",
- "set hive.testing.short.logs=true;",
- "set hive.testing.remove.logs=false;",
- },
- qFile.getBeforeExecuteLogFile(),
- Converter.NONE);
+ String[] commands = TEST_FIRST_COMMANDS;
+
+ String[] extraCommands;
+ if (qFile.isUseSharedDatabase()) {
+ // If we are using a shared database, then remove not known databases, tables, views.
+ Set<String> dropCommands = getDatabases().stream()
+ .filter(database -> !database.equals("default"))
+ .map(database -> "DROP DATABASE `" + database + "` CASCADE;")
+ .collect(Collectors.toSet());
+
+ Set<String> srcTables = QTestUtil.getSrcTables();
+ dropCommands.addAll(getTables().stream()
+ .filter(table -> !srcTables.contains(table))
+ .map(table -> "DROP TABLE `" + table + "` PURGE;")
+ .collect(Collectors.toSet()));
+
+ dropCommands.addAll(getViews().stream()
+ .map(view -> "DROP VIEW `" + view + "`;")
+ .collect(Collectors.toSet()));
+ extraCommands = dropCommands.toArray(new String[]{});
+ } else {
+ // If we are using a test specific database, then we just drop the database, and recreate
+ extraCommands = new String[] {
+ "DROP DATABASE IF EXISTS `" + qFile.getDatabaseName() + "` CASCADE;",
+ "CREATE DATABASE `" + qFile.getDatabaseName() + "`;",
+ "USE `" + qFile.getDatabaseName() + "`;"
+ };
+ }
+ commands = ArrayUtils.addAll(commands, extraCommands);
+ commands = ArrayUtils.addAll(commands, TEST_SET_LOG_COMMANDS);
+ execute(commands, qFile.getBeforeExecuteLogFile(), Converter.NONE);
beeLine.setIsTestMode(true);
}
private void afterExecute(QFile qFile) throws Exception {
beeLine.setIsTestMode(false);
- execute(
- new String[] {
- "set hive.testing.short.logs=false;",
- "!set verbose true",
- "!set silent false",
- "!set showheader true",
- "!set outputformat table",
- "USE default;",
- "DROP DATABASE IF EXISTS `" + qFile.getDatabaseName() + "` CASCADE;",
- },
- qFile.getAfterExecuteLogFile(),
- Converter.NONE);
+ String[] commands = TEST_RESET_COMMANDS;
+
+ if (!qFile.isUseSharedDatabase()) {
+ // If we are using a test specific database, then we just drop the database
+ String[] extraCommands = new String[] {
+ "DROP DATABASE IF EXISTS `" + qFile.getDatabaseName() + "` CASCADE;"
+ };
+ commands = ArrayUtils.addAll(commands, extraCommands);
+ }
+
+ execute(commands, qFile.getAfterExecuteLogFile(), Converter.NONE);
}
public void execute(QFile qFile) throws Exception {