You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by kg...@apache.org on 2019/07/09 15:46:57 UTC
[hive] 01/04: HIVE-21938: Add database and table filter options to
PreUpgradeTool (Krisztian Kasa via Zoltan Haindrich)
This is an automated email from the ASF dual-hosted git repository.
kgyrtkirk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
commit b4a688b00738068bf27869791afbaaee4a05c396
Author: Krisztian Kasa <kk...@cloudera.com>
AuthorDate: Tue Jul 9 17:30:51 2019 +0200
HIVE-21938: Add database and table filter options to PreUpgradeTool (Krisztian Kasa via Zoltan Haindrich)
Signed-off-by: Zoltan Haindrich <ki...@rxd.hu>
---
.../hadoop/hive/upgrade/acid/PreUpgradeTool.java | 104 +++++++-----
.../hadoop/hive/upgrade/acid/RunOptions.java | 83 ++++++++++
.../hive/upgrade/acid/TestPreUpgradeTool.java | 181 +++++++++++++++++----
3 files changed, 302 insertions(+), 66 deletions(-)
diff --git a/upgrade-acid/pre-upgrade/src/main/java/org/apache/hadoop/hive/upgrade/acid/PreUpgradeTool.java b/upgrade-acid/pre-upgrade/src/main/java/org/apache/hadoop/hive/upgrade/acid/PreUpgradeTool.java
index 0e3e3e2..0a7354d 100644
--- a/upgrade-acid/pre-upgrade/src/main/java/org/apache/hadoop/hive/upgrade/acid/PreUpgradeTool.java
+++ b/upgrade-acid/pre-upgrade/src/main/java/org/apache/hadoop/hive/upgrade/acid/PreUpgradeTool.java
@@ -17,7 +17,23 @@
*/
package org.apache.hadoop.hive.upgrade.acid;
-import com.google.common.annotations.VisibleForTesting;
+import static org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.escapeSQLString;
+
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.nio.ByteBuffer;
+import java.nio.charset.CharacterCodingException;
+import java.nio.charset.Charset;
+import java.nio.charset.CharsetDecoder;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.GnuParser;
@@ -38,6 +54,7 @@ import org.apache.hadoop.hive.metastore.HiveMetaHookLoader;
import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
import org.apache.hadoop.hive.metastore.IMetaStoreClient;
import org.apache.hadoop.hive.metastore.RetryingMetaStoreClient;
+import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.Warehouse;
import org.apache.hadoop.hive.metastore.api.CompactionResponse;
import org.apache.hadoop.hive.metastore.api.MetaException;
@@ -49,10 +66,10 @@ import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
import org.apache.hadoop.hive.metastore.txn.TxnStore;
import org.apache.hadoop.hive.metastore.txn.TxnUtils;
import org.apache.hadoop.hive.ql.io.AcidUtils;
-import org.apache.hadoop.hive.ql.metadata.Hive;
-import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.io.orc.OrcFile;
import org.apache.hadoop.hive.ql.io.orc.Reader;
+import org.apache.hadoop.hive.ql.metadata.Hive;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.hive.shims.HadoopShims;
import org.apache.hadoop.security.AccessControlException;
@@ -62,20 +79,7 @@ import org.apache.thrift.TException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.PrintWriter;
-import java.nio.ByteBuffer;
-import java.nio.charset.CharacterCodingException;
-import java.nio.charset.Charset;
-import java.nio.charset.CharsetDecoder;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import static org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.escapeSQLString;
+import com.google.common.annotations.VisibleForTesting;
/**
* This utility is designed to help with upgrading Hive 2.x to Hive 3.0. On-disk layout for
@@ -122,8 +126,6 @@ public class PreUpgradeTool {
tool.init();
CommandLineParser parser = new GnuParser();
CommandLine line ;
- String outputDir = ".";
- boolean execute = false;
try {
line = parser.parse(tool.cmdLineOptions, args);
} catch (ParseException e) {
@@ -136,13 +138,8 @@ public class PreUpgradeTool {
formatter.printHelp("upgrade-acid", tool.cmdLineOptions);
return;
}
- if(line.hasOption("location")) {
- outputDir = line.getOptionValue("location");
- }
- if(line.hasOption("execute")) {
- execute = true;
- }
- LOG.info("Starting with execute=" + execute + ", location=" + outputDir);
+ RunOptions runOptions = RunOptions.fromCommandLine(line);
+ LOG.info("Starting with " + runOptions.toString());
try {
String hiveVer = HiveVersionInfo.getShortVersion();
@@ -151,7 +148,7 @@ public class PreUpgradeTool {
if(!hiveVer.startsWith("2.")) {
throw new IllegalStateException("preUpgrade requires Hive 2.x. Actual: " + hiveVer);
}
- tool.prepareAcidUpgradeInternal(outputDir, execute);
+ tool.prepareAcidUpgradeInternal(runOptions);
}
catch(Exception ex) {
LOG.error("PreUpgradeTool failed", ex);
@@ -172,8 +169,33 @@ public class PreUpgradeTool {
"Executes commands equivalent to generated scrips");
exec.setOptionalArg(true);
cmdLineOptions.addOption(exec);
- cmdLineOptions.addOption(new Option("location", true,
- "Location to write scripts to. Default is CWD."));
+ Option locationOption = new Option("location", true,
+ "Location to write scripts to. Default is CWD.");
+ locationOption.setArgName("path of directory");
+ cmdLineOptions.addOption(locationOption);
+
+ Option dbRegexOption = new Option("d",
+ "Regular expression to match database names on which this tool will be run. Default: all databases");
+ dbRegexOption.setLongOpt("dbRegex");
+ dbRegexOption.setArgs(1);
+ dbRegexOption.setArgName("regex");
+ cmdLineOptions.addOption(dbRegexOption);
+
+ Option tableRegexOption = new Option("t",
+ "Regular expression to match table names on which this tool will be run. Default: all tables");
+ tableRegexOption.setLongOpt("tableRegex");
+ tableRegexOption.setArgs(1);
+ tableRegexOption.setArgName("regex");
+ cmdLineOptions.addOption(tableRegexOption);
+
+ Option tableTypeOption = new Option("tt",
+ String.format("Table type to match tables on which this tool will be run. Possible values: %s " +
+ "Default: all tables",
+ Arrays.stream(TableType.values()).map(Enum::name).collect(Collectors.joining("|"))));
+ tableTypeOption.setLongOpt("tableType");
+ tableTypeOption.setArgs(1);
+ tableTypeOption.setArgName("table type");
+ cmdLineOptions.addOption(tableTypeOption);
}
catch(Exception ex) {
LOG.error("init()", ex);
@@ -219,7 +241,7 @@ public class PreUpgradeTool {
/**
* todo: change script comments to a preamble instead of a footer
*/
- private void prepareAcidUpgradeInternal(String scriptLocation, boolean execute)
+ private void prepareAcidUpgradeInternal(RunOptions runOptions)
throws HiveException, TException, IOException {
HiveConf conf = hiveConf != null ? hiveConf : new HiveConf();
boolean isAcidEnabled = isAcidEnabled(conf);
@@ -232,16 +254,22 @@ public class PreUpgradeTool {
ValidTxnList txns = null;
Hive db = null;
try {
- databases = hms.getAllDatabases();//TException
+ databases = hms.getDatabases(runOptions.getDbRegex()); //TException
LOG.debug("Found " + databases.size() + " databases to process");
- if (execute) {
+ if (runOptions.isExecute()) {
db = Hive.get(conf);
}
for (String dbName : databases) {
try {
- List<String> tables = hms.getAllTables(dbName);
- LOG.debug("found " + tables.size() + " tables in " + dbName);
+ List<String> tables;
+ if (runOptions.getTableType() == null) {
+ tables = hms.getTables(dbName, runOptions.getTableRegex());
+ LOG.debug("found {} tables in {}", tables.size(), dbName);
+ } else {
+ tables = hms.getTables(dbName, runOptions.getTableRegex(), runOptions.getTableType());
+ LOG.debug("found {} {} in {}", tables.size(), runOptions.getTableType().name(), dbName);
+ }
for (String tableName : tables) {
try {
Table t = hms.getTable(dbName, tableName);
@@ -257,7 +285,7 @@ public class PreUpgradeTool {
txns = TxnUtils.createValidCompactTxnList(txnHandler.getOpenTxnsInfo());
}
List<String> compactionCommands =
- getCompactionCommands(t, conf, hms, compactionMetaInfo, execute, db, txns);
+ getCompactionCommands(t, conf, hms, compactionMetaInfo, runOptions.isExecute(), db, txns);
compactions.addAll(compactionCommands);
}
/*todo: handle renaming files somewhere*/
@@ -293,9 +321,9 @@ public class PreUpgradeTool {
throw new HiveException(exceptionMsg, e);
}
- makeCompactionScript(compactions, scriptLocation, compactionMetaInfo);
+ makeCompactionScript(compactions, runOptions.getOutputDir(), compactionMetaInfo);
- if(execute) {
+ if(runOptions.isExecute()) {
while(compactionMetaInfo.compactionIds.size() > 0) {
LOG.debug("Will wait for " + compactionMetaInfo.compactionIds.size() +
" compactions to complete");
@@ -336,7 +364,7 @@ public class PreUpgradeTool {
}
Thread.sleep(pollIntervalMs);
} catch (InterruptedException ex) {
- ;//this only responds to ^C
+ //this only responds to ^C
}
}
}
diff --git a/upgrade-acid/pre-upgrade/src/main/java/org/apache/hadoop/hive/upgrade/acid/RunOptions.java b/upgrade-acid/pre-upgrade/src/main/java/org/apache/hadoop/hive/upgrade/acid/RunOptions.java
new file mode 100644
index 0000000..66213d4
--- /dev/null
+++ b/upgrade-acid/pre-upgrade/src/main/java/org/apache/hadoop/hive/upgrade/acid/RunOptions.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.upgrade.acid;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.hadoop.hive.metastore.TableType;
+
+/**
+ * This class's instance holds the option values were passed by the user via the command line.
+ */
+public class RunOptions {
+
+ public static RunOptions fromCommandLine(CommandLine commandLine) {
+ String tableTypeText = commandLine.getOptionValue("tableType");
+ return new RunOptions(
+ commandLine.getOptionValue("location", "."),
+ commandLine.hasOption("execute"),
+ commandLine.getOptionValue("dbRegex", ".*"),
+ commandLine.getOptionValue("tableRegex", ".*"),
+ tableTypeText == null ? null : TableType.valueOf(tableTypeText)
+ );
+ }
+
+ private final String outputDir;
+ private final boolean execute;
+ private final String dbRegex;
+ private final String tableRegex;
+ private final TableType tableType;
+
+ public RunOptions(String outputDir, boolean execute, String dbRegex, String tableRegex, TableType tableType) {
+ this.outputDir = outputDir;
+ this.execute = execute;
+ this.dbRegex = dbRegex;
+ this.tableRegex = tableRegex;
+ this.tableType = tableType;
+ }
+
+ public String getOutputDir() {
+ return outputDir;
+ }
+
+ public boolean isExecute() {
+ return execute;
+ }
+
+ public String getDbRegex() {
+ return dbRegex;
+ }
+
+ public String getTableRegex() {
+ return tableRegex;
+ }
+
+ public TableType getTableType() {
+ return tableType;
+ }
+
+ @Override
+ public String toString() {
+ return "RunOptions{" +
+ "outputDir='" + outputDir + '\'' +
+ ", execute=" + execute +
+ ", dbRegex='" + dbRegex + '\'' +
+ ", tableRegex='" + tableRegex + '\'' +
+ ", tableType=" + tableType +
+ '}';
+ }
+}
diff --git a/upgrade-acid/pre-upgrade/src/test/java/org/apache/hadoop/hive/upgrade/acid/TestPreUpgradeTool.java b/upgrade-acid/pre-upgrade/src/test/java/org/apache/hadoop/hive/upgrade/acid/TestPreUpgradeTool.java
index 90230d5d..e514e80 100644
--- a/upgrade-acid/pre-upgrade/src/test/java/org/apache/hadoop/hive/upgrade/acid/TestPreUpgradeTool.java
+++ b/upgrade-acid/pre-upgrade/src/test/java/org/apache/hadoop/hive/upgrade/acid/TestPreUpgradeTool.java
@@ -17,6 +17,25 @@
*/
package org.apache.hadoop.hive.upgrade.acid;
+import static org.hamcrest.CoreMatchers.allOf;
+import static org.hamcrest.CoreMatchers.hasItem;
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.CoreMatchers.not;
+import static org.hamcrest.CoreMatchers.nullValue;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.core.StringContains.containsString;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.attribute.PosixFilePermission;
+import java.nio.file.attribute.PosixFilePermissions;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.common.FileUtils;
@@ -30,12 +49,8 @@ import org.apache.hadoop.hive.metastore.txn.TxnStore;
import org.apache.hadoop.hive.metastore.txn.TxnUtils;
import org.apache.hadoop.hive.ql.Driver;
import org.apache.hadoop.hive.ql.QueryState;
-import org.apache.hadoop.hive.ql.exec.mr.MapRedTask;
-import org.apache.hadoop.hive.ql.io.AcidUtils;
import org.apache.hadoop.hive.ql.io.HiveInputFormat;
-import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.ql.txn.compactor.Worker;
@@ -46,16 +61,6 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TestName;
-import java.io.File;
-import java.nio.file.Files;
-import java.nio.file.attribute.FileAttribute;
-import java.nio.file.attribute.PosixFilePermission;
-import java.nio.file.attribute.PosixFilePermissions;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicBoolean;
-
public class TestPreUpgradeTool {
private static final String TEST_DATA_DIR = new File(System.getProperty("java.io.tmpdir") +
File.separator + TestPreUpgradeTool.class.getCanonicalName() + "-" + System.currentTimeMillis()
@@ -71,7 +76,7 @@ public class TestPreUpgradeTool {
*/
@Test
public void testUpgrade() throws Exception {
- int[][] data = {{1,2}, {3, 4}, {5, 6}};
+ int[][] data = {{1, 2}, {3, 4}, {5, 6}};
int[][] dataPart = {{1, 2, 10}, {3, 4, 11}, {5, 6, 12}};
runStatementOnDriver("drop table if exists TAcid");
runStatementOnDriver("drop table if exists TAcidPart");
@@ -117,13 +122,14 @@ public class TestPreUpgradeTool {
PreUpgradeTool.pollIntervalMs = 1;
PreUpgradeTool.hiveConf = hiveConf;
PreUpgradeTool.main(args);
- /*
- todo: parse
- target/tmp/org.apache.hadoop.hive.upgrade.acid.TestPreUpgradeTool-1527286256834/compacts_1527286277624.sql
- make sure it's the only 'compacts' file and contains
- ALTER TABLE default.tacid COMPACT 'major';
-ALTER TABLE default.tacidpart PARTITION(p=10Y) COMPACT 'major';
- * */
+
+ String[] scriptFiles = getScriptFiles();
+ assertThat(scriptFiles.length, is(1));
+
+ List<String> scriptContent = loadScriptContent(new File(getTestDataDir(), scriptFiles[0]));
+ assertThat(scriptContent.size(), is(2));
+ assertThat(scriptContent, hasItem(is("ALTER TABLE default.tacid COMPACT 'major';")));
+ assertThat(scriptContent, hasItem(is("ALTER TABLE default.tacidpart PARTITION(p=10Y) COMPACT 'major';")));
TxnStore txnHandler = TxnUtils.getTxnStore(hiveConf);
@@ -133,11 +139,20 @@ ALTER TABLE default.tacidpart PARTITION(p=10Y) COMPACT 'major';
Assert.assertEquals(e.toString(), TxnStore.CLEANING_RESPONSE, e.getState());
}
- String[] args2 = {"-location", getTestDataDir()};
+ // Check whether compaction was successful in the first run
+ File secondRunDataDir = new File(getTestDataDir(), "secondRun");
+ if (!secondRunDataDir.exists()) {
+ if (!secondRunDataDir.mkdir()) {
+ throw new IOException("Unable to create directory" + secondRunDataDir.getAbsolutePath());
+ }
+ }
+ String[] args2 = {"-location", secondRunDataDir.getAbsolutePath()};
PreUpgradeTool.main(args2);
- /*
- * todo: parse compacts script - make sure there is nothing in it
- * */
+
+ scriptFiles = secondRunDataDir.list();
+ assertThat(scriptFiles, is(not(nullValue())));
+ assertThat(scriptFiles.length, is(0));
+
} finally {
runStatementOnDriver("drop table if exists TAcid");
runStatementOnDriver("drop table if exists TAcidPart");
@@ -146,9 +161,119 @@ ALTER TABLE default.tacidpart PARTITION(p=10Y) COMPACT 'major';
}
}
+ private static final String INCLUDE_DATABASE_NAME ="DInclude";
+ private static final String EXCLUDE_DATABASE_NAME ="DExclude";
+
+ @Test
+ public void testOnlyFilteredDatabasesAreUpgradedWhenRegexIsGiven() throws Exception {
+ int[][] data = {{1, 2}, {3, 4}, {5, 6}};
+ runStatementOnDriver("drop database if exists " + INCLUDE_DATABASE_NAME + " cascade");
+ runStatementOnDriver("drop database if exists " + EXCLUDE_DATABASE_NAME + " cascade");
+
+ try {
+ runStatementOnDriver("create database " + INCLUDE_DATABASE_NAME);
+ runStatementOnDriver("use " + INCLUDE_DATABASE_NAME);
+ runStatementOnDriver("create table " + INCLUDE_TABLE_NAME + " (a int, b int) clustered by (b) " +
+ "into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true')");
+ runStatementOnDriver("insert into " + INCLUDE_TABLE_NAME + makeValuesClause(data));
+ runStatementOnDriver("update " + INCLUDE_TABLE_NAME + " set a = 1 where b = 2");
+
+ runStatementOnDriver("create database " + EXCLUDE_DATABASE_NAME);
+ runStatementOnDriver("use " + EXCLUDE_DATABASE_NAME);
+ runStatementOnDriver("create table " + EXCLUDE_DATABASE_NAME + " (a int, b int) clustered by (b) " +
+ "into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true')");
+ runStatementOnDriver("insert into " + EXCLUDE_DATABASE_NAME + makeValuesClause(data));
+ runStatementOnDriver("update " + EXCLUDE_DATABASE_NAME + " set a = 1 where b = 2");
+
+ String[] args = {"-location", getTestDataDir(), "-dbRegex", "*include*"};
+ PreUpgradeTool.callback = new PreUpgradeTool.Callback() {
+ @Override
+ void onWaitForCompaction() throws MetaException {
+ runWorker(hiveConf);
+ }
+ };
+ PreUpgradeTool.pollIntervalMs = 1;
+ PreUpgradeTool.hiveConf = hiveConf;
+ PreUpgradeTool.main(args);
+
+ String[] scriptFiles = getScriptFiles();
+ assertThat(scriptFiles.length, is(1));
+
+ List<String> scriptContent = loadScriptContent(new File(getTestDataDir(), scriptFiles[0]));
+ assertThat(scriptContent.size(), is(1));
+ assertThat(scriptContent.get(0), is("ALTER TABLE dinclude.tinclude COMPACT 'major';"));
+
+ } finally {
+ runStatementOnDriver("drop database if exists " + INCLUDE_DATABASE_NAME + " cascade");
+ runStatementOnDriver("drop database if exists " + EXCLUDE_DATABASE_NAME + " cascade");
+ }
+ }
+
+ private static final String INCLUDE_TABLE_NAME ="TInclude";
+ private static final String EXCLUDE_TABLE_NAME ="TExclude";
+
+ @Test
+ public void testOnlyFilteredTablesAreUpgradedWhenRegexIsGiven() throws Exception {
+ int[][] data = {{1, 2}, {3, 4}, {5, 6}};
+ runStatementOnDriver("drop table if exists " + INCLUDE_TABLE_NAME);
+ runStatementOnDriver("drop table if exists " + EXCLUDE_TABLE_NAME);
+
+ try {
+ runStatementOnDriver("create table " + INCLUDE_TABLE_NAME + " (a int, b int) clustered by (b) " +
+ "into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true')");
+ runStatementOnDriver("create table " + EXCLUDE_TABLE_NAME + " (a int, b int) clustered by (b) " +
+ "into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true')");
+
+ runStatementOnDriver("insert into " + INCLUDE_TABLE_NAME + makeValuesClause(data));
+ runStatementOnDriver("update " + INCLUDE_TABLE_NAME + " set a = 1 where b = 2");
+
+ runStatementOnDriver("insert into " + EXCLUDE_TABLE_NAME + makeValuesClause(data));
+ runStatementOnDriver("update " + EXCLUDE_TABLE_NAME + " set a = 1 where b = 2");
+
+ String[] args = {"-location", getTestDataDir(), "-tableRegex", "*include*"};
+ PreUpgradeTool.callback = new PreUpgradeTool.Callback() {
+ @Override
+ void onWaitForCompaction() throws MetaException {
+ runWorker(hiveConf);
+ }
+ };
+ PreUpgradeTool.pollIntervalMs = 1;
+ PreUpgradeTool.hiveConf = hiveConf;
+ PreUpgradeTool.main(args);
+
+ String[] scriptFiles = getScriptFiles();
+ assertThat(scriptFiles.length, is(1));
+
+ List<String> scriptContent = loadScriptContent(new File(getTestDataDir(), scriptFiles[0]));
+ assertThat(scriptContent.size(), is(1));
+ assertThat(scriptContent.get(0), allOf(
+ containsString("ALTER TABLE"),
+ containsString(INCLUDE_TABLE_NAME.toLowerCase()),
+ containsString("COMPACT")));
+
+ } finally {
+ runStatementOnDriver("drop table if exists " + INCLUDE_TABLE_NAME);
+ runStatementOnDriver("drop table if exists " + EXCLUDE_TABLE_NAME);
+ }
+ }
+
+ private String[] getScriptFiles() {
+ File testDataDir = new File(getTestDataDir());
+ String[] scriptFiles = testDataDir.list((dir, name) -> name.startsWith("compacts_") && name.endsWith(".sql"));
+ assertThat(scriptFiles, is(not(nullValue())));
+ return scriptFiles;
+ }
+
+ private List<String> loadScriptContent(File file) throws IOException {
+ List<String> content = org.apache.commons.io.FileUtils.readLines(file);
+ content.removeIf(line -> line.startsWith("--"));
+ content.removeIf(StringUtils::isBlank);
+ return content;
+ }
+
@Test
public void testUpgradeExternalTableNoReadPermissionForDatabase() throws Exception {
- int[][] data = {{1,2}, {3, 4}, {5, 6}};
+ int[][] data = {{1, 2}, {3, 4}, {5, 6}};
runStatementOnDriver("drop database if exists test cascade");
runStatementOnDriver("drop table if exists TExternal");
@@ -187,7 +312,7 @@ ALTER TABLE default.tacidpart PARTITION(p=10Y) COMPACT 'major';
@Test
public void testUpgradeExternalTableNoReadPermissionForTable() throws Exception {
- int[][] data = {{1,2}, {3, 4}, {5, 6}};
+ int[][] data = {{1, 2}, {3, 4}, {5, 6}};
runStatementOnDriver("drop table if exists TExternal");
runStatementOnDriver("create table TExternal (a int, b int) stored as orc tblproperties('transactional'='false')");