You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by we...@apache.org on 2017/05/08 20:43:23 UTC
[33/51] [partial] hive git commit: Revert "HIVE-14671 : merge master
into hive-14535 (Wei Zheng)"
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingLayout.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingLayout.java b/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingLayout.java
index 1a8337f..e344e0f 100644
--- a/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingLayout.java
+++ b/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingLayout.java
@@ -100,6 +100,22 @@ public class TestOperationLoggingLayout {
String row = iter.next()[0].toString();
Assert.assertEquals(true, row.matches("^.*(FATAL|ERROR|WARN|INFO|DEBUG|TRACE).*$"));
}
+
+ String queryString = "set hive.server2.logging.operation.level=verbose";
+ client.executeStatement(sessionHandle, queryString, null);
+ operationHandle = client.executeStatement(sessionHandle, sqlCntStar, null);
+ // just check for first few lines, some log lines are multi-line strings which can break format
+ // checks below
+ rowSetLog = client.fetchResults(operationHandle, FetchOrientation.FETCH_FIRST, 10,
+ FetchType.LOG);
+ iter = rowSetLog.iterator();
+ // verbose pattern is "%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n"
+ while (iter.hasNext()) {
+ String row = iter.next()[0].toString();
+ // just check if the log line starts with date
+ Assert.assertEquals(true,
+ row.matches("^\\d{2}[/](0[1-9]|1[012])[/](0[1-9]|[12][0-9]|3[01]).*$"));
+ }
}
private SessionHandle setupSession() throws Exception {
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/itests/pom.xml
----------------------------------------------------------------------
diff --git a/itests/pom.xml b/itests/pom.xml
index 400075b..e5b54bf 100644
--- a/itests/pom.xml
+++ b/itests/pom.xml
@@ -19,7 +19,7 @@
<parent>
<groupId>org.apache.hive</groupId>
<artifactId>hive</artifactId>
- <version>3.0.0-SNAPSHOT</version>
+ <version>2.2.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/itests/qtest-accumulo/pom.xml
----------------------------------------------------------------------
diff --git a/itests/qtest-accumulo/pom.xml b/itests/qtest-accumulo/pom.xml
index b7ce283..31cee36 100644
--- a/itests/qtest-accumulo/pom.xml
+++ b/itests/qtest-accumulo/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.hive</groupId>
<artifactId>hive-it</artifactId>
- <version>3.0.0-SNAPSHOT</version>
+ <version>2.2.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/itests/qtest-spark/pom.xml
----------------------------------------------------------------------
diff --git a/itests/qtest-spark/pom.xml b/itests/qtest-spark/pom.xml
index a506f7f..f301504 100644
--- a/itests/qtest-spark/pom.xml
+++ b/itests/qtest-spark/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.hive</groupId>
<artifactId>hive-it</artifactId>
- <version>3.0.0-SNAPSHOT</version>
+ <version>2.2.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/itests/qtest/pom.xml
----------------------------------------------------------------------
diff --git a/itests/qtest/pom.xml b/itests/qtest/pom.xml
index 02664f3..1c3b601 100644
--- a/itests/qtest/pom.xml
+++ b/itests/qtest/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.hive</groupId>
<artifactId>hive-it</artifactId>
- <version>3.0.0-SNAPSHOT</version>
+ <version>2.2.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestBeeLineDriver.java
----------------------------------------------------------------------
diff --git a/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestBeeLineDriver.java b/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestBeeLineDriver.java
index 4dd17c1e..24eeb9d 100644
--- a/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestBeeLineDriver.java
+++ b/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestBeeLineDriver.java
@@ -22,15 +22,15 @@ import java.util.List;
import org.apache.hadoop.hive.cli.control.CliAdapter;
import org.apache.hadoop.hive.cli.control.CliConfigs;
-import org.apache.hive.beeline.Parallelized;
import org.junit.ClassRule;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TestRule;
import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
-@RunWith(Parallelized.class)
+@RunWith(Parameterized.class)
public class TestBeeLineDriver {
static CliAdapter adapter = new CliConfigs.BeeLineConfig().getCliAdapter();
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index df947bb..dc02fd4 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -71,7 +71,6 @@ minillap.shared.query.files=insert_into1.q,\
orc_merge3.q,\
orc_merge4.q,\
orc_merge_diff_fs.q,\
- parallel_colstats.q,\
unionDistinct_1.q,\
union_type_chk.q,\
cte_2.q,\
@@ -353,8 +352,6 @@ minillaplocal.shared.query.files=alter_merge_2_orc.q,\
vector_reduce_groupby_decimal.q,\
vector_string_concat.q,\
vector_struct_in.q,\
- vector_udf_character_length.q,\
- vector_udf_octet_length.q,\
vector_varchar_4.q,\
vector_varchar_mapjoin1.q,\
vector_varchar_simple.q,\
@@ -408,7 +405,6 @@ minillaplocal.shared.query.files=alter_merge_2_orc.q,\
minillap.query.files=acid_bucket_pruning.q,\
bucket5.q,\
bucket6.q,\
- dynamic_semijoin_user_level.q,\
except_distinct.q,\
explainuser_2.q,\
empty_dir_in_table.q,\
@@ -503,7 +499,6 @@ minillaplocal.query.files=acid_globallimit.q,\
groupby2.q,\
hybridgrace_hashjoin_1.q,\
hybridgrace_hashjoin_2.q,\
- is_distinct_from.q,\
infer_bucket_sort_bucketed_table.q,\
input16_cc.q,\
insert_dir_distcp.q,\
@@ -515,14 +510,12 @@ minillaplocal.query.files=acid_globallimit.q,\
join_max_hashtable.q,\
join_nulls.q,\
join_nullsafe.q,\
- join_is_not_distinct_from.q,\
leftsemijoin_mr.q,\
limit_join_transpose.q,\
lineage2.q,\
lineage3.q,\
list_bucket_dml_10.q,\
llap_partitioned.q,\
- llap_vector_nohybridgrace.q,\
load_dyn_part5.q,\
lvj_mapjoin.q,\
mapjoin_decimal.q,\
@@ -574,7 +567,6 @@ minillaplocal.query.files=acid_globallimit.q,\
schema_evol_text_vecrow_table.q,\
selectDistinctStar.q,\
semijoin.q,\
- semijoin_hint.q,\
smb_cache.q,\
special_character_in_tabnames_1.q,\
sqlmerge.q,\
@@ -619,26 +611,11 @@ minillaplocal.query.files=acid_globallimit.q,\
vector_auto_smb_mapjoin_14.q,\
vector_decimal_2.q,\
vector_decimal_udf.q,\
- vector_groupby_cube1.q,\
- vector_groupby_grouping_id1.q,\
- vector_groupby_grouping_id2.q,\
- vector_groupby_grouping_id3.q,\
- vector_groupby_grouping_sets1.q,\
- vector_groupby_grouping_sets2.q,\
- vector_groupby_grouping_sets3.q,\
- vector_groupby_grouping_sets4.q,\
- vector_groupby_grouping_sets5.q,\
- vector_groupby_grouping_sets6.q,\
- vector_groupby_grouping_sets_grouping.q,\
- vector_groupby_grouping_sets_limit.q,\
- vector_groupby_grouping_window.q,\
- vector_groupby_rollup1.q,\
vector_join30.q,\
vector_join_filters.q,\
vector_leftsemi_mapjoin.q,\
vector_number_compare_projection.q,\
vector_partitioned_date_time.q,\
- vector_ptf_part_simple.q,\
vector_udf1.q,\
vectorization_short_regress.q,\
vectorized_dynamic_partition_pruning.q,\
@@ -748,22 +725,10 @@ encrypted.query.files=encryption_join_unencrypted_tbl.q,\
encryption_drop_view.q \
encryption_drop_partition.q \
encryption_with_trash.q \
- encryption_ctas.q \
- encryption_auto_purge_tables.q \
- encryption_drop_table_in_encrypted_db.q
+ encryption_ctas.q
beeline.positive.include=drop_with_concurrency.q,\
- escape_comments.q,\
- smb_mapjoin_1.q,\
- smb_mapjoin_10.q,\
- smb_mapjoin_11.q,\
- smb_mapjoin_12.q,\
- smb_mapjoin_13.q,\
- smb_mapjoin_16.q,\
- smb_mapjoin_2.q,\
- smb_mapjoin_3.q,\
- smb_mapjoin_7.q,\
- select_dummy_source.q
+ escape_comments.q
minimr.query.negative.files=cluster_tasklog_retrieval.q,\
file_with_header_footer_negative.q,\
@@ -1371,7 +1336,6 @@ spark.query.files=add_part_multiple.q, \
spark.only.query.files=spark_combine_equivalent_work.q,\
spark_dynamic_partition_pruning.q,\
spark_dynamic_partition_pruning_2.q,\
- spark_explainuser_1.q,\
spark_vectorized_dynamic_partition_pruning.q,\
spark_use_file_size_for_mapjoin.q,\
spark_use_op_stats.q
@@ -1446,6 +1410,4 @@ spark.query.negative.files=groupby2_map_skew_multi_distinct.q,\
groupby2_multi_distinct.q,\
groupby3_map_skew_multi_distinct.q,\
groupby3_multi_distinct.q,\
- groupby_grouping_sets7.q,\
- spark_job_max_tasks.q
-
+ groupby_grouping_sets7.q
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/itests/test-serde/pom.xml
----------------------------------------------------------------------
diff --git a/itests/test-serde/pom.xml b/itests/test-serde/pom.xml
index bf5f5d2..df0ce8a 100644
--- a/itests/test-serde/pom.xml
+++ b/itests/test-serde/pom.xml
@@ -19,7 +19,7 @@
<parent>
<groupId>org.apache.hive</groupId>
<artifactId>hive-it</artifactId>
- <version>3.0.0-SNAPSHOT</version>
+ <version>2.2.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/itests/util/pom.xml
----------------------------------------------------------------------
diff --git a/itests/util/pom.xml b/itests/util/pom.xml
index a44a55c..3740af8 100644
--- a/itests/util/pom.xml
+++ b/itests/util/pom.xml
@@ -19,7 +19,7 @@
<parent>
<groupId>org.apache.hive</groupId>
<artifactId>hive-it</artifactId>
- <version>3.0.0-SNAPSHOT</version>
+ <version>2.2.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java
index 02abe53..9c97c31 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java
@@ -29,7 +29,6 @@ import org.apache.hadoop.hive.cli.control.AbstractCliConfig.MetastoreType;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveVariableSource;
import org.apache.hadoop.hive.conf.VariableSubstitution;
-import org.apache.hadoop.hive.ql.QTestProcessExecResult;
import org.apache.hadoop.hive.ql.QTestUtil;
import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType;
import org.junit.After;
@@ -140,14 +139,12 @@ public abstract class AbstractCoreBlobstoreCliDriver extends CliAdapter {
if ((ecode == 0) ^ expectSuccess) {
qt.failed(ecode, fname, debugHint);
}
- QTestProcessExecResult result = qt.checkCliDriverResults(fname);
- if (result.getReturnCode() != 0) {
- String message = Strings.isNullOrEmpty(result.getCapturedOutput()) ?
- debugHint : "\r\n" + result.getCapturedOutput();
- qt.failedDiff(result.getReturnCode(), fname, message);
+ ecode = qt.checkCliDriverResults(fname);
+ if (ecode != 0) {
+ qt.failedDiff(ecode, fname, debugHint);
}
}
- catch (Exception e) {
+ catch (Throwable e) {
qt.failed(e, fname, debugHint);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java
index 1457db0..67064b8 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java
@@ -287,7 +287,6 @@ public class CliConfigs {
excludesFrom(testConfigProps, "minimr.query.negative.files");
excludeQuery("authorization_uri_import.q");
- excludeQuery("spark_job_max_tasks.q");
setResultsDir("ql/src/test/results/clientnegative");
setLogDir("itests/qtest/target/qfile-results/clientnegative");
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreAccumuloCliDriver.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreAccumuloCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreAccumuloCliDriver.java
index 73e5632..3e4b373 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreAccumuloCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreAccumuloCliDriver.java
@@ -18,10 +18,8 @@
package org.apache.hadoop.hive.cli.control;
import static org.junit.Assert.assertTrue;
-
import org.apache.hadoop.hive.accumulo.AccumuloQTestUtil;
import org.apache.hadoop.hive.accumulo.AccumuloTestSetup;
-import org.apache.hadoop.hive.ql.QTestProcessExecResult;
import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType;
import org.junit.After;
import org.junit.AfterClass;
@@ -94,13 +92,13 @@ public class CoreAccumuloCliDriver extends CliAdapter {
qt.failed(ecode, fname, null);
}
- QTestProcessExecResult result = qt.checkCliDriverResults(fname);
- if (result.getReturnCode() != 0) {
- qt.failedDiff(result.getReturnCode(), fname, result.getCapturedOutput());
+ ecode = qt.checkCliDriverResults(fname);
+ if (ecode != 0) {
+ qt.failedDiff(ecode, fname, null);
}
qt.clearPostTestEffects();
- } catch (Exception e) {
+ } catch (Throwable e) {
qt.failed(e, fname, null);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreBeeLineDriver.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreBeeLineDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreBeeLineDriver.java
index 8c7057c..acc02eb 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreBeeLineDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreBeeLineDriver.java
@@ -17,23 +17,21 @@
*/
package org.apache.hadoop.hive.cli.control;
+import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
-import com.google.common.base.Strings;
import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.ql.QTestProcessExecResult;
import org.apache.hadoop.hive.ql.hooks.PreExecutePrinter;
-import org.apache.hive.beeline.QFile;
-import org.apache.hive.beeline.QFile.QFileBuilder;
-import org.apache.hive.beeline.QFileBeeLineClient;
-import org.apache.hive.beeline.QFileBeeLineClient.QFileClientBuilder;
+import org.apache.hive.beeline.qfile.QFile;
+import org.apache.hive.beeline.qfile.QFile.QFileBuilder;
+import org.apache.hive.beeline.qfile.QFileBeeLineClient;
+import org.apache.hive.beeline.qfile.QFileBeeLineClient.QFileClientBuilder;
import org.apache.hive.jdbc.miniHS2.MiniHS2;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import java.io.File;
import java.io.IOException;
-import java.sql.SQLException;
import java.util.HashMap;
public class CoreBeeLineDriver extends CliAdapter {
@@ -46,7 +44,6 @@ public class CoreBeeLineDriver extends CliAdapter {
private final File testDataDirectory;
private final File testScriptDirectory;
private boolean overwrite = false;
- private boolean rewriteSourceTables = true;
private MiniHS2 miniHS2;
private QFileClientBuilder clientBuilder;
private QFileBuilder fileBuilder;
@@ -71,10 +68,6 @@ public class CoreBeeLineDriver extends CliAdapter {
if (testOutputOverwrite != null && "true".equalsIgnoreCase(testOutputOverwrite)) {
overwrite = true;
}
- String testRewriteSourceTables = System.getProperty("test.rewrite.source.tables");
- if (testRewriteSourceTables != null && "false".equalsIgnoreCase(testRewriteSourceTables)) {
- rewriteSourceTables = false;
- }
HiveConf hiveConf = new HiveConf();
// We do not need Zookeeper at the moment
@@ -99,17 +92,19 @@ public class CoreBeeLineDriver extends CliAdapter {
.setPassword("password");
fileBuilder = new QFileBuilder()
+ .setHiveRootDirectory(hiveRootDirectory)
.setLogDirectory(logDirectory)
.setQueryDirectory(queryDirectory)
.setResultsDirectory(resultsDirectory)
- .setRewriteSourceTables(rewriteSourceTables);
+ .setScratchDirectoryString(hiveConf.getVar(HiveConf.ConfVars.SCRATCHDIR))
+ .setWarehouseDirectoryString(hiveConf.getVar(HiveConf.ConfVars.METASTOREWAREHOUSE));
runInfraScript(initScript, new File(logDirectory, "init.beeline"),
new File(logDirectory, "init.raw"));
}
protected void runInfraScript(File script, File beeLineOutput, File log)
- throws IOException, SQLException {
+ throws IOException {
try (QFileBeeLineClient beeLineClient = clientBuilder.getClient(beeLineOutput)) {
beeLineClient.execute(
new String[]{
@@ -119,9 +114,6 @@ public class CoreBeeLineDriver extends CliAdapter {
"!run " + script,
},
log);
- } catch (Exception e) {
- throw new SQLException("Error running infra script: " + script
- + "\nCheck the following logs for details:\n - " + beeLineOutput + "\n - " + log, e);
}
}
@@ -142,41 +134,28 @@ public class CoreBeeLineDriver extends CliAdapter {
try (QFileBeeLineClient beeLineClient = clientBuilder.getClient(qFile.getLogFile())) {
long startTime = System.currentTimeMillis();
System.err.println(">>> STARTED " + qFile.getName());
+ assertTrue("QFile execution failed, see logs for details", beeLineClient.execute(qFile));
- beeLineClient.execute(qFile);
-
- long queryEndTime = System.currentTimeMillis();
- System.err.println(">>> EXECUTED " + qFile.getName() + ": " + (queryEndTime - startTime)
- + "ms");
+ long endTime = System.currentTimeMillis();
+ System.err.println(">>> EXECUTED " + qFile.getName() + ":" + (endTime - startTime) / 1000
+ + "s");
qFile.filterOutput();
long filterEndTime = System.currentTimeMillis();
- System.err.println(">>> FILTERED " + qFile.getName() + ": " + (filterEndTime - queryEndTime)
- + "ms");
+ System.err.println(">>> FILTERED " + qFile.getName() + ":" + (filterEndTime - endTime) / 1000
+ + "s");
if (!overwrite) {
- QTestProcessExecResult result = qFile.compareResults();
-
- long compareEndTime = System.currentTimeMillis();
- System.err.println(">>> COMPARED " + qFile.getName() + ": "
- + (compareEndTime - filterEndTime) + "ms");
- if (result.getReturnCode() == 0) {
+ if (qFile.compareResults()) {
System.err.println(">>> PASSED " + qFile.getName());
} else {
System.err.println(">>> FAILED " + qFile.getName());
- String messageText = "Client result comparison failed with error code = "
- + result.getReturnCode() + " while executing fname=" + qFile.getName() + "\n";
- String messageBody = Strings.isNullOrEmpty(result.getCapturedOutput()) ?
- qFile.getDebugHint() : result.getCapturedOutput();
- fail(messageText + messageBody);
+ fail("Failed diff");
}
} else {
qFile.overwriteResults();
System.err.println(">>> PASSED " + qFile.getName());
}
- } catch (Exception e) {
- throw new Exception("Exception running or analyzing the results of the query file: " + qFile
- + "\n" + qFile.getDebugHint(), e);
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java
index d59b650..a735346 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java
@@ -23,9 +23,7 @@ import static org.junit.Assert.fail;
import java.util.concurrent.TimeUnit;
import com.google.common.base.Stopwatch;
-import com.google.common.base.Strings;
import org.apache.hadoop.hive.cli.control.AbstractCliConfig.MetastoreType;
-import org.apache.hadoop.hive.ql.QTestProcessExecResult;
import org.apache.hadoop.hive.ql.QTestUtil;
import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType;
import org.apache.hadoop.hive.util.ElapsedTimeLoggingWrapper;
@@ -177,15 +175,13 @@ public class CoreCliDriver extends CliAdapter {
failed = true;
qt.failed(ecode, fname, debugHint);
}
- QTestProcessExecResult result = qt.checkCliDriverResults(fname);
- if (result.getReturnCode() != 0) {
+ ecode = qt.checkCliDriverResults(fname);
+ if (ecode != 0) {
failed = true;
- String message = Strings.isNullOrEmpty(result.getCapturedOutput()) ?
- debugHint : "\r\n" + result.getCapturedOutput();
- qt.failedDiff(result.getReturnCode(), fname, message);
+ qt.failedDiff(ecode, fname, debugHint);
}
}
- catch (Exception e) {
+ catch (Throwable e) {
failed = true;
qt.failed(e, fname, debugHint);
} finally {
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCompareCliDriver.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCompareCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCompareCliDriver.java
index bff81dd..71a02bc 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCompareCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCompareCliDriver.java
@@ -25,8 +25,6 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import com.google.common.base.Strings;
-import org.apache.hadoop.hive.ql.QTestProcessExecResult;
import org.apache.hadoop.hive.ql.QTestUtil;
import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType;
import org.junit.After;
@@ -145,14 +143,12 @@ public class CoreCompareCliDriver extends CliAdapter{
}
}
- QTestProcessExecResult result = qt.checkCompareCliDriverResults(fname, outputs);
- if (result.getReturnCode() != 0) {
- String message = Strings.isNullOrEmpty(result.getCapturedOutput()) ?
- debugHint : "\r\n" + result.getCapturedOutput();
- qt.failedDiff(result.getReturnCode(), fname, message);
+ ecode = qt.checkCompareCliDriverResults(fname, outputs);
+ if (ecode != 0) {
+ qt.failedDiff(ecode, fname, debugHint);
}
}
- catch (Exception e) {
+ catch (Throwable e) {
qt.failed(e, fname, debugHint);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseCliDriver.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseCliDriver.java
index aa0b071..956a42d 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseCliDriver.java
@@ -23,7 +23,6 @@ import static org.junit.Assert.fail;
import org.apache.hadoop.hive.hbase.HBaseQTestUtil;
import org.apache.hadoop.hive.hbase.HBaseTestSetup;
-import org.apache.hadoop.hive.ql.QTestProcessExecResult;
import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType;
import org.junit.After;
import org.junit.AfterClass;
@@ -121,12 +120,12 @@ public class CoreHBaseCliDriver extends CliAdapter {
qt.failed(ecode, fname, null);
}
- QTestProcessExecResult result = qt.checkCliDriverResults(fname);
- if (result.getReturnCode() != 0) {
- qt.failedDiff(result.getReturnCode(), fname, result.getCapturedOutput());
+ ecode = qt.checkCliDriverResults(fname);
+ if (ecode != 0) {
+ qt.failedDiff(ecode, fname, null);
}
- } catch (Exception e) {
+ } catch (Throwable e) {
qt.failed(e, fname, null);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseNegativeCliDriver.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseNegativeCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseNegativeCliDriver.java
index 8320a80..6225180 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseNegativeCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseNegativeCliDriver.java
@@ -23,7 +23,6 @@ import static org.junit.Assert.fail;
import org.apache.hadoop.hive.hbase.HBaseQTestUtil;
import org.apache.hadoop.hive.hbase.HBaseTestSetup;
-import org.apache.hadoop.hive.ql.QTestProcessExecResult;
import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType;
import org.junit.After;
import org.junit.AfterClass;
@@ -102,13 +101,13 @@ public class CoreHBaseNegativeCliDriver extends CliAdapter {
qt.failed(fname, null);
}
- QTestProcessExecResult result = qt.checkCliDriverResults(fname);
- if (result.getReturnCode() != 0) {
- qt.failedDiff(result.getReturnCode(), fname, result.getCapturedOutput());
+ ecode = qt.checkCliDriverResults(fname);
+ if (ecode != 0) {
+ qt.failedDiff(ecode, fname, null);
}
qt.clearPostTestEffects();
- } catch (Exception e) {
+ } catch (Throwable e) {
qt.failed(e, fname, null);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java
index 438a61e..65b2ce7 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java
@@ -20,8 +20,6 @@ package org.apache.hadoop.hive.cli.control;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
-import com.google.common.base.Strings;
-import org.apache.hadoop.hive.ql.QTestProcessExecResult;
import org.apache.hadoop.hive.ql.QTestUtil;
import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType;
import org.junit.After;
@@ -125,14 +123,12 @@ public class CoreNegativeCliDriver extends CliAdapter{
qt.failed(fname, debugHint);
}
- QTestProcessExecResult result = qt.checkCliDriverResults(fname);
- if (result.getReturnCode() != 0) {
- String message = Strings.isNullOrEmpty(result.getCapturedOutput()) ?
- debugHint : "\r\n" + result.getCapturedOutput();
- qt.failedDiff(result.getReturnCode(), fname, message);
+ ecode = qt.checkCliDriverResults(fname);
+ if (ecode != 0) {
+ qt.failedDiff(ecode, fname, debugHint);
}
}
- catch (Exception e) {
+ catch (Throwable e) {
qt.failed(e, fname, debugHint);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java
index 26b9ce1..8620cde 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java
@@ -22,8 +22,6 @@ package org.apache.hadoop.hive.cli.control;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
-import com.google.common.base.Strings;
-import org.apache.hadoop.hive.ql.QTestProcessExecResult;
import org.apache.hadoop.hive.ql.QTestUtil;
import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType;
import org.junit.After;
@@ -123,13 +121,11 @@ public class CorePerfCliDriver extends CliAdapter{
if (ecode != 0) {
qt.failed(ecode, fname, debugHint);
}
- QTestProcessExecResult result = qt.checkCliDriverResults(fname);
- if (result.getReturnCode() != 0) {
- String message = Strings.isNullOrEmpty(result.getCapturedOutput()) ?
- debugHint : "\r\n" + result.getCapturedOutput();
- qt.failedDiff(result.getReturnCode(), fname, message);
+ ecode = qt.checkCliDriverResults(fname);
+ if (ecode != 0) {
+ qt.failedDiff(ecode, fname, debugHint);
}
- } catch (Exception e) {
+ } catch (Throwable e) {
qt.failed(e, fname, debugHint);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestProcessExecResult.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestProcessExecResult.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestProcessExecResult.java
index 75cce14..6acc7a0 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestProcessExecResult.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestProcessExecResult.java
@@ -18,8 +18,12 @@
package org.apache.hadoop.hive.ql;
+
+import com.google.common.base.Strings;
+import org.apache.commons.lang3.StringUtils;
+
/**
- * Standard output and return code of a process executed during the qtests.
+ * Standard output and return code of a process executed during the qtests
*/
public class QTestProcessExecResult {
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
index b897ffa..e32ee74 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
@@ -39,10 +39,6 @@ import java.io.PrintStream;
import java.io.Serializable;
import java.io.StringWriter;
import java.net.URL;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.FileSystems;
-import java.nio.file.Files;
-import java.nio.file.StandardOpenOption;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
@@ -55,7 +51,6 @@ import java.util.Collection;
import java.util.Comparator;
import java.util.Deque;
import java.util.EnumSet;
-import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
@@ -64,22 +59,17 @@ import java.util.Properties;
import java.util.Set;
import java.util.SortedMap;
import java.util.TreeMap;
-import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import java.util.stream.Stream;
import com.google.common.base.Preconditions;
-import com.google.common.base.Throwables;
-import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
-import org.apache.commons.io.output.ByteArrayOutputStream;
import org.apache.commons.lang.StringUtils;
-import org.apache.commons.lang3.tuple.Pair;
import org.apache.commons.lang3.tuple.ImmutablePair;
+import org.apache.commons.lang3.tuple.Pair;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FileStatus;
@@ -128,6 +118,7 @@ import org.apache.hadoop.hive.ql.processors.HiveCommand;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.shims.HadoopShims;
import org.apache.hadoop.hive.shims.ShimLoader;
+import org.apache.hadoop.util.Shell;
import org.apache.hive.common.util.StreamPrinter;
import org.apache.logging.log4j.util.Strings;
import org.apache.tools.ant.BuildException;
@@ -176,7 +167,7 @@ public class QTestUtil {
private final Set<String> qNoSessionReuseQuerySet;
private final Set<String> qJavaVersionSpecificOutput;
private static final String SORT_SUFFIX = ".sorted";
- private final Set<String> srcTables;
+ private final HashSet<String> srcTables;
private final Set<String> srcUDFs;
private final MiniClusterType clusterType;
private final FsType fsType;
@@ -209,7 +200,7 @@ public class QTestUtil {
}
private HBaseTestingUtility utility;
- public static Set<String> getSrcTables() {
+ HashSet<String> getSrcTables() {
HashSet<String> srcTables = new HashSet<String>();
// FIXME: moved default value to here...for now
// i think this features is never really used from the command line
@@ -1404,8 +1395,7 @@ public class QTestUtil {
int rc = response.getResponseCode();
if (rc != 0) {
- SessionState.getConsole().printError(response.toString(), response.getException() != null ?
- Throwables.getStackTraceAsString(response.getException()) : "");
+ SessionState.get().out.println(response);
}
return rc;
@@ -1413,7 +1403,7 @@ public class QTestUtil {
throw new RuntimeException("Could not get CommandProcessor for command: " + commandName);
}
} catch (Exception e) {
- throw new RuntimeException("Could not execute test command", e);
+ throw new RuntimeException("Could not execute test command: " + e.getMessage());
}
}
@@ -1476,6 +1466,10 @@ public class QTestUtil {
// Create an instance of hive in order to create the tables
testWarehouse = conf.getVar(HiveConf.ConfVars.METASTOREWAREHOUSE);
db = Hive.get(conf);
+ // Create dest4 to replace dest4_sequencefile
+ LinkedList<String> cols = new LinkedList<String>();
+ cols.add("key");
+ cols.add("value");
// Move all data from dest4_sequencefile to dest4
drv
@@ -1486,7 +1480,7 @@ public class QTestUtil {
true, true);
}
- public QTestProcessExecResult checkNegativeResults(String tname, Exception e) throws Exception {
+ public int checkNegativeResults(String tname, Exception e) throws Exception {
String outFileExtension = getOutFileExtension(tname);
@@ -1509,17 +1503,16 @@ public class QTestUtil {
outfd.write(e.getMessage());
outfd.close();
- QTestProcessExecResult result = executeDiffCommand(outf.getPath(), expf, false,
+ int exitVal = executeDiffCommand(outf.getPath(), expf, false,
qSortSet.contains(qf.getName()));
- if (overWrite) {
- overwriteResults(outf.getPath(), expf);
- return QTestProcessExecResult.createWithoutOutput(0);
+ if (exitVal != 0 && overWrite) {
+ exitVal = overwriteResults(outf.getPath(), expf);
}
- return result;
+ return exitVal;
}
- public QTestProcessExecResult checkParseResults(String tname, ASTNode tree) throws Exception {
+ public int checkParseResults(String tname, ASTNode tree) throws Exception {
if (tree != null) {
String outFileExtension = getOutFileExtension(tname);
@@ -1535,11 +1528,10 @@ public class QTestUtil {
outfd.write(tree.toStringTree());
outfd.close();
- QTestProcessExecResult exitVal = executeDiffCommand(outf.getPath(), expf, false, false);
+ int exitVal = executeDiffCommand(outf.getPath(), expf, false, false);
- if (overWrite) {
- overwriteResults(outf.getPath(), expf);
- return QTestProcessExecResult.createWithoutOutput(0);
+ if (exitVal != 0 && overWrite) {
+ exitVal = overwriteResults(outf.getPath(), expf);
}
return exitVal;
@@ -1708,7 +1700,7 @@ public class QTestUtil {
});
/* This list may be modified by specific cli drivers to mask strings that change on every test */
- private final List<Pair<Pattern, String>> patternsWithMaskComments = new ArrayList<Pair<Pattern, String>>() {{
+ private List<Pair<Pattern, String>> patternsWithMaskComments = new ArrayList<Pair<Pattern, String>>() {{
add(toPatternPair("(pblob|s3.?|swift|wasb.?).*hive-staging.*","### BLOBSTORE_STAGING_PATH ###"));
}};
@@ -1720,7 +1712,7 @@ public class QTestUtil {
patternsWithMaskComments.add(toPatternPair(patternStr, maskComment));
}
- public QTestProcessExecResult checkCliDriverResults(String tname) throws Exception {
+ public int checkCliDriverResults(String tname) throws Exception {
assert(qMap.containsKey(tname));
String outFileExtension = getOutFileExtension(tname);
@@ -1729,71 +1721,69 @@ public class QTestUtil {
File f = new File(logDir, tname + outFileExtension);
maskPatterns(planMask, f.getPath());
- QTestProcessExecResult exitVal = executeDiffCommand(f.getPath(),
+ int exitVal = executeDiffCommand(f.getPath(),
outFileName, false,
qSortSet.contains(tname));
- if (overWrite) {
- overwriteResults(f.getPath(), outFileName);
- return QTestProcessExecResult.createWithoutOutput(0);
+ if (exitVal != 0 && overWrite) {
+ exitVal = overwriteResults(f.getPath(), outFileName);
}
return exitVal;
}
- public QTestProcessExecResult checkCompareCliDriverResults(String tname, List<String> outputs)
- throws Exception {
+ public int checkCompareCliDriverResults(String tname, List<String> outputs) throws Exception {
assert outputs.size() > 1;
maskPatterns(planMask, outputs.get(0));
for (int i = 1; i < outputs.size(); ++i) {
maskPatterns(planMask, outputs.get(i));
- QTestProcessExecResult result = executeDiffCommand(
+ int ecode = executeDiffCommand(
outputs.get(i - 1), outputs.get(i), false, qSortSet.contains(tname));
- if (result.getReturnCode() != 0) {
+ if (ecode != 0) {
System.out.println("Files don't match: " + outputs.get(i - 1) + " and " + outputs.get(i));
- return result;
+ return ecode;
}
}
- return QTestProcessExecResult.createWithoutOutput(0);
+ return 0;
}
- private static void overwriteResults(String inFileName, String outFileName) throws Exception {
+ private static int overwriteResults(String inFileName, String outFileName) throws Exception {
// This method can be replaced with Files.copy(source, target, REPLACE_EXISTING)
// once Hive uses JAVA 7.
System.out.println("Overwriting results " + inFileName + " to " + outFileName);
- int result = executeCmd(new String[]{
+ return executeCmd(new String[] {
"cp",
getQuotedString(inFileName),
getQuotedString(outFileName)
- }).getReturnCode();
- if (result != 0)
- throw new IllegalStateException("Unexpected error while overwriting " +
- inFileName + " with " + outFileName);
+ });
}
- private static QTestProcessExecResult executeDiffCommand(String inFileName,
+ private static int executeDiffCommand(String inFileName,
String outFileName,
boolean ignoreWhiteSpace,
boolean sortResults
) throws Exception {
- QTestProcessExecResult result;
+ int result = 0;
if (sortResults) {
// sort will try to open the output file in write mode on windows. We need to
// close it first.
SessionState ss = SessionState.get();
if (ss != null && ss.out != null && ss.out != System.out) {
- ss.out.close();
+ ss.out.close();
}
String inSorted = inFileName + SORT_SUFFIX;
String outSorted = outFileName + SORT_SUFFIX;
- sortFiles(inFileName, inSorted);
- sortFiles(outFileName, outSorted);
-
+ result = sortFiles(inFileName, inSorted);
+ result |= sortFiles(outFileName, outSorted);
+ if (result != 0) {
+ System.err.println("ERROR: Could not sort files before comparing");
+ return result;
+ }
inFileName = inSorted;
outFileName = outSorted;
}
@@ -1823,47 +1813,40 @@ public class QTestUtil {
return result;
}
- private static void sortFiles(String in, String out) throws Exception {
- int result = executeCmd(new String[]{
+ private static int sortFiles(String in, String out) throws Exception {
+ return executeCmd(new String[] {
"sort",
getQuotedString(in),
- }, out, null).getReturnCode();
- if (result != 0)
- throw new IllegalStateException("Unexpected error while sorting " + in);
+ }, out, null);
}
- private static QTestProcessExecResult executeCmd(Collection<String> args) throws Exception {
+ private static int executeCmd(Collection<String> args) throws Exception {
return executeCmd(args, null, null);
}
- private static QTestProcessExecResult executeCmd(String[] args) throws Exception {
+ private static int executeCmd(String[] args) throws Exception {
return executeCmd(args, null, null);
}
- private static QTestProcessExecResult executeCmd(Collection<String> args, String outFile,
- String errFile) throws Exception {
+ private static int executeCmd(Collection<String> args, String outFile, String errFile) throws Exception {
String[] cmdArray = args.toArray(new String[args.size()]);
return executeCmd(cmdArray, outFile, errFile);
}
- private static QTestProcessExecResult executeCmd(String[] args, String outFile,
- String errFile) throws Exception {
+ private static int executeCmd(String[] args, String outFile, String errFile) throws Exception {
System.out.println("Running: " + org.apache.commons.lang.StringUtils.join(args, ' '));
PrintStream out = outFile == null ?
SessionState.getConsole().getChildOutStream() :
- new PrintStream(new FileOutputStream(outFile), true, "UTF-8");
+ new PrintStream(new FileOutputStream(outFile), true);
PrintStream err = errFile == null ?
SessionState.getConsole().getChildErrStream() :
- new PrintStream(new FileOutputStream(errFile), true, "UTF-8");
+ new PrintStream(new FileOutputStream(errFile), true);
Process executor = Runtime.getRuntime().exec(args);
- ByteArrayOutputStream bos = new ByteArrayOutputStream();
- PrintStream str = new PrintStream(bos, true, "UTF-8");
-
StreamPrinter errPrinter = new StreamPrinter(executor.getErrorStream(), null, err);
- StreamPrinter outPrinter = new StreamPrinter(executor.getInputStream(), null, out, str);
+ StreamPrinter outPrinter = new StreamPrinter(executor.getInputStream(), null, out);
outPrinter.start();
errPrinter.start();
@@ -1881,8 +1864,7 @@ public class QTestUtil {
err.close();
}
- return QTestProcessExecResult.
- create(result, new String(bos.toByteArray(), StandardCharsets.UTF_8));
+ return result;
}
private static String getQuotedString(String str){
@@ -2058,18 +2040,11 @@ public class QTestUtil {
qt[i].clearTestSideEffects();
qt[i].cliInit(qfiles[i].getName(), false);
qt[i].executeClient(qfiles[i].getName());
- QTestProcessExecResult result = qt[i].checkCliDriverResults(qfiles[i].getName());
- if (result.getReturnCode() != 0) {
+ int ecode = qt[i].checkCliDriverResults(qfiles[i].getName());
+ if (ecode != 0) {
failed = true;
- StringBuilder builder = new StringBuilder();
- builder.append("Test ")
- .append(qfiles[i].getName())
- .append(" results check failed with error code ")
- .append(result.getReturnCode());
- if (Strings.isNotEmpty(result.getCapturedOutput())) {
- builder.append(" and diff value ").append(result.getCapturedOutput());
- }
- System.err.println(builder.toString());
+ System.err.println("Test " + qfiles[i].getName()
+ + " results check failed with error code " + ecode);
outputTestFailureHelpMessage();
}
qt[i].clearPostTestEffects();
@@ -2116,18 +2091,11 @@ public class QTestUtil {
for (int i = 0; i < qfiles.length; i++) {
qtThread[i].join();
- QTestProcessExecResult result = qt[i].checkCliDriverResults(qfiles[i].getName());
- if (result.getReturnCode() != 0) {
+ int ecode = qt[i].checkCliDriverResults(qfiles[i].getName());
+ if (ecode != 0) {
failed = true;
- StringBuilder builder = new StringBuilder();
- builder.append("Test ")
- .append(qfiles[i].getName())
- .append(" results check failed with error code ")
- .append(result.getReturnCode());
- if (Strings.isNotEmpty(result.getCapturedOutput())) {
- builder.append(" and diff value ").append(result.getCapturedOutput());
- }
- System.err.println(builder.toString());
+ System.err.println("Test " + qfiles[i].getName()
+ + " results check failed with error code " + ecode);
outputTestFailureHelpMessage();
}
}
@@ -2225,15 +2193,16 @@ public class QTestUtil {
public void failedDiff(int ecode, String fname, String debugHint) {
String message =
- "Client Execution succeeded but contained differences " +
- "(error code = " + ecode + ") after executing " +
+ "Client Execution results failed with error code = " + ecode + " while executing fname=" +
fname + (debugHint != null ? (" " + debugHint) : "");
LOG.error(message);
Assert.fail(message);
}
- public void failed(Exception e, String fname, String debugHint) {
+ public void failed(Throwable e, String fname, String debugHint) {
String command = SessionState.get() != null ? SessionState.get().getLastCommand() : null;
+ System.err.println("Exception: " + e.getMessage());
+ e.printStackTrace();
System.err.println("Failed query: " + fname);
System.err.flush();
Assert.fail("Unexpected exception " +
@@ -2273,6 +2242,9 @@ public class QTestUtil {
}
br.close();
} catch (Exception e) {
+ System.err.println("Exception: " + e.getMessage());
+ e.printStackTrace();
+ System.err.flush();
Assert.fail("Unexpected exception " + org.apache.hadoop.util.StringUtils.stringifyException(e));
}
}
@@ -2296,9 +2268,7 @@ public class QTestUtil {
String mdbPath = AbstractCliConfig.HIVE_ROOT + "/data/files/tpcds-perf/metastore_export/";
// Setup the table column stats
- BufferedReader br = new BufferedReader(
- new FileReader(
- new File(AbstractCliConfig.HIVE_ROOT + "/metastore/scripts/upgrade/derby/022-HIVE-11107.derby.sql")));
+ BufferedReader br = new BufferedReader(new FileReader(new File(AbstractCliConfig.HIVE_ROOT + "/metastore/scripts/upgrade/derby/022-HIVE-11107.derby.sql")));
String command;
s.execute("DROP TABLE APP.TABLE_PARAMS");
@@ -2324,13 +2294,15 @@ public class QTestUtil {
}
br.close();
- java.nio.file.Path tabColStatsCsv = FileSystems.getDefault().getPath(mdbPath, "csv" ,"TAB_COL_STATS.txt.bz2");
- java.nio.file.Path tabParamsCsv = FileSystems.getDefault().getPath(mdbPath, "csv", "TABLE_PARAMS.txt.bz2");
+ File tabColStatsCsv = new File(mdbPath+"csv/TAB_COL_STATS.txt");
+ File tabParamsCsv = new File(mdbPath+"csv/TABLE_PARAMS.txt");
// Set up the foreign key constraints properly in the TAB_COL_STATS data
String tmpBaseDir = System.getProperty(TEST_TMP_DIR_PROPERTY);
- java.nio.file.Path tmpFileLoc1 = FileSystems.getDefault().getPath(tmpBaseDir, "TAB_COL_STATS.txt");
- java.nio.file.Path tmpFileLoc2 = FileSystems.getDefault().getPath(tmpBaseDir, "TABLE_PARAMS.txt");
+ File tmpFileLoc1 = new File(tmpBaseDir+"/TAB_COL_STATS.txt");
+ File tmpFileLoc2 = new File(tmpBaseDir+"/TABLE_PARAMS.txt");
+ FileUtils.copyFile(tabColStatsCsv, tmpFileLoc1);
+ FileUtils.copyFile(tabParamsCsv, tmpFileLoc2);
class MyComp implements Comparator<String> {
@Override
@@ -2342,7 +2314,7 @@ public class QTestUtil {
}
}
- final SortedMap<String, Integer> tableNameToID = new TreeMap<String, Integer>(new MyComp());
+ SortedMap<String, Integer> tableNameToID = new TreeMap<String, Integer>(new MyComp());
rs = s.executeQuery("SELECT * FROM APP.TBLS");
while(rs.next()) {
@@ -2354,73 +2326,29 @@ public class QTestUtil {
LOG.debug("Resultset : " + tblName + " | " + tblId);
}
}
-
- final Map<String, Map<String, String>> data = new HashMap<>();
- rs = s.executeQuery("select TBLS.TBL_NAME, a.COLUMN_NAME, a.TYPE_NAME from "
- + "(select COLUMN_NAME, TYPE_NAME, SDS.SD_ID from APP.COLUMNS_V2 join APP.SDS on SDS.CD_ID = COLUMNS_V2.CD_ID) a"
- + " join APP.TBLS on TBLS.SD_ID = a.SD_ID");
- while (rs.next()) {
- String tblName = rs.getString(1);
- String colName = rs.getString(2);
- String typeName = rs.getString(3);
- Map<String, String> cols = data.get(tblName);
- if (null == cols) {
- cols = new HashMap<>();
+ for (Map.Entry<String, Integer> entry : tableNameToID.entrySet()) {
+ String toReplace1 = ",_" + entry.getKey() + "_" ;
+ String replacementString1 = ","+entry.getValue();
+ String toReplace2 = "_" + entry.getKey() + "_@" ;
+ String replacementString2 = ""+entry.getValue()+"@";
+ try {
+ String content1 = FileUtils.readFileToString(tmpFileLoc1, "UTF-8");
+ content1 = content1.replaceAll(toReplace1, replacementString1);
+ FileUtils.writeStringToFile(tmpFileLoc1, content1, "UTF-8");
+ String content2 = FileUtils.readFileToString(tmpFileLoc2, "UTF-8");
+ content2 = content2.replaceAll(toReplace2, replacementString2);
+ FileUtils.writeStringToFile(tmpFileLoc2, content2, "UTF-8");
+ } catch (IOException e) {
+ LOG.info("Generating file failed", e);
}
- cols.put(colName, typeName);
- data.put(tblName, cols);
}
- BufferedReader reader = new BufferedReader(new InputStreamReader(
- new BZip2CompressorInputStream(Files.newInputStream(tabColStatsCsv, StandardOpenOption.READ))));
-
- Stream<String> replaced = reader.lines().parallel().map(str-> {
- String[] splits = str.split(",");
- String tblName = splits[0];
- String colName = splits[1];
- Integer tblID = tableNameToID.get(tblName);
- StringBuilder sb = new StringBuilder("default@"+tblName + "@" + colName + "@" + data.get(tblName).get(colName)+"@");
- for (int i = 2; i < splits.length; i++) {
- sb.append(splits[i]+"@");
- }
- return sb.append(tblID).toString();
- });
-
- Files.write(tmpFileLoc1, (Iterable<String>)replaced::iterator);
- replaced.close();
- reader.close();
-
- BufferedReader reader2 = new BufferedReader(new InputStreamReader(
- new BZip2CompressorInputStream(Files.newInputStream(tabParamsCsv, StandardOpenOption.READ))));
- final Map<String,String> colStats = new ConcurrentHashMap<>();
- Stream<String> replacedStream = reader2.lines().parallel().map(str-> {
- String[] splits = str.split("_@");
- String tblName = splits[0];
- Integer tblId = tableNameToID.get(tblName);
- Map<String,String> cols = data.get(tblName);
- StringBuilder sb = new StringBuilder();
- sb.append("{\"COLUMN_STATS\":{");
- for (String colName : cols.keySet()) {
- sb.append("\""+colName+"\":\"true\",");
- }
- sb.append("},\"BASIC_STATS\":\"true\"}");
- colStats.put(tblId.toString(), sb.toString());
-
- return tblId.toString() + "@" + splits[1];
- });
-
- Files.write(tmpFileLoc2, (Iterable<String>)replacedStream::iterator);
- Files.write(tmpFileLoc2, (Iterable<String>)colStats.entrySet().stream()
- .map(map->map.getKey()+"@COLUMN_STATS_ACCURATE@"+map.getValue())::iterator, StandardOpenOption.APPEND);
-
- replacedStream.close();
- reader2.close();
// Load the column stats and table params with 30 TB scale
- String importStatement1 = "CALL SYSCS_UTIL.SYSCS_IMPORT_TABLE(null, '" + "TAB_COL_STATS" +
- "', '" + tmpFileLoc1.toAbsolutePath().toString() +
- "', '@', null, 'UTF-8', 1)";
- String importStatement2 = "CALL SYSCS_UTIL.SYSCS_IMPORT_TABLE(null, '" + "TABLE_PARAMS" +
- "', '" + tmpFileLoc2.toAbsolutePath().toString() +
+ String importStatement1 = "CALL SYSCS_UTIL.SYSCS_IMPORT_TABLE_LOBS_FROM_EXTFILE(null, '" + "TAB_COL_STATS" +
+ "', '" + tmpFileLoc1.getAbsolutePath() +
+ "', ',', null, 'UTF-8', 1)";
+ String importStatement2 = "CALL SYSCS_UTIL.SYSCS_IMPORT_TABLE_LOBS_FROM_EXTFILE(null, '" + "TABLE_PARAMS" +
+ "', '" + tmpFileLoc2.getAbsolutePath() +
"', '@', null, 'UTF-8', 1)";
try {
PreparedStatement psImport1 = conn.prepareStatement(importStatement1);
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java
index 31f69a3..8dba0bb 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java
@@ -21,12 +21,9 @@ import static org.junit.Assert.fail;
import java.io.Serializable;
import java.util.List;
-
-import com.google.common.base.Strings;
import org.apache.hadoop.hive.cli.control.AbstractCliConfig;
import org.apache.hadoop.hive.cli.control.CliAdapter;
import org.apache.hadoop.hive.cli.control.CliConfigs;
-import org.apache.hadoop.hive.ql.QTestProcessExecResult;
import org.apache.hadoop.hive.ql.QTestUtil;
import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType;
import org.apache.hadoop.hive.ql.exec.Task;
@@ -109,20 +106,18 @@ public class CoreParseNegative extends CliAdapter{
fail("Unexpected success for query: " + fname + debugHint);
}
catch (ParseException pe) {
- QTestProcessExecResult result = qt.checkNegativeResults(fname, pe);
- if (result.getReturnCode() != 0) {
- qt.failed(result.getReturnCode(), fname, result.getCapturedOutput() + "\r\n" + debugHint);
+ int ecode = qt.checkNegativeResults(fname, pe);
+ if (ecode != 0) {
+ qt.failed(ecode, fname, debugHint);
}
}
catch (SemanticException se) {
- QTestProcessExecResult result = qt.checkNegativeResults(fname, se);
- if (result.getReturnCode() != 0) {
- String message = Strings.isNullOrEmpty(result.getCapturedOutput()) ?
- debugHint : "\r\n" + result.getCapturedOutput();
- qt.failedDiff(result.getReturnCode(), fname, message);
+ int ecode = qt.checkNegativeResults(fname, se);
+ if (ecode != 0) {
+ qt.failedDiff(ecode, fname, debugHint);
}
}
- catch (Exception e) {
+ catch (Throwable e) {
qt.failed(e, fname, debugHint);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/itests/util/src/main/java/org/apache/hive/beeline/Parallelized.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hive/beeline/Parallelized.java b/itests/util/src/main/java/org/apache/hive/beeline/Parallelized.java
deleted file mode 100644
index a819e26..0000000
--- a/itests/util/src/main/java/org/apache/hive/beeline/Parallelized.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hive.beeline;
-
-import org.junit.runners.Parameterized;
-import org.junit.runners.model.RunnerScheduler;
-
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-
-/**
- * Class to run Parameterized test in parallel.
- * Source: http://hwellmann.blogspot.hu/2009/12/running-parameterized-junit-tests-in.html
- */
-public class Parallelized extends Parameterized {
- private static class ThreadPoolScheduler implements RunnerScheduler {
- private ExecutorService executor;
-
- public ThreadPoolScheduler() {
- String threads = System.getProperty("junit.parallel.threads");
- int numThreads = Runtime.getRuntime().availableProcessors();
- if (threads != null) {
- numThreads = Integer.parseInt(threads);
- }
- executor = Executors.newFixedThreadPool(numThreads);
- }
-
- @Override
- public void finished() {
- executor.shutdown();
- try {
- executor.awaitTermination(10, TimeUnit.MINUTES);
- } catch (InterruptedException exc) {
- throw new RuntimeException(exc);
- }
- }
-
- @Override
- public void schedule(Runnable childStatement) {
- executor.submit(childStatement);
- }
- }
-
- public Parallelized(Class klass) throws Throwable {
- super(klass);
- setScheduler(new ThreadPoolScheduler());
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/itests/util/src/main/java/org/apache/hive/beeline/QFile.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hive/beeline/QFile.java b/itests/util/src/main/java/org/apache/hive/beeline/QFile.java
deleted file mode 100644
index 0bde529..0000000
--- a/itests/util/src/main/java/org/apache/hive/beeline/QFile.java
+++ /dev/null
@@ -1,333 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.beeline;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.hive.ql.QTestProcessExecResult;
-import org.apache.hadoop.hive.ql.QTestUtil;
-import org.apache.hadoop.util.Shell;
-import org.apache.hive.common.util.StreamPrinter;
-
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.IOException;
-import java.io.PrintStream;
-import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Set;
-import java.util.regex.Pattern;
-
-/**
- * Class for representing a Query and the connected files. It provides accessors for the specific
- * input and output files, and provides methods for filtering the output of the runs.
- */
-public final class QFile {
- private static final Set<String> srcTables = QTestUtil.getSrcTables();
- private static final String DEBUG_HINT =
- "The following files can help you identifying the problem:%n"
- + " - Query file: %1s%n"
- + " - Raw output file: %2s%n"
- + " - Filtered output file: %3s%n"
- + " - Expected output file: %4s%n"
- + " - Client log file: %5s%n"
- + " - Client log files before the test: %6s%n"
- + " - Client log files after the test: %7s%n"
- + " - Hiveserver2 log file: %8s%n";
- private static final String USE_COMMAND_WARNING =
- "The query file %1s contains \"%2s\" command.%n"
- + "The source table name rewrite is turned on, so this might cause problems when the used "
- + "database contains tables named any of the following: " + srcTables + "%n"
- + "To turn off the table name rewrite use -Dtest.rewrite.source.tables=false%n";
-
- private static final Pattern USE_PATTERN =
- Pattern.compile("^\\s*use\\s.*", Pattern.CASE_INSENSITIVE);
-
- private static final String MASK_PATTERN = "#### A masked pattern was here ####\n";
-
- private String name;
- private File inputFile;
- private File rawOutputFile;
- private File outputFile;
- private File expectedOutputFile;
- private File logFile;
- private File beforeExecuteLogFile;
- private File afterExecuteLogFile;
- private static RegexFilterSet staticFilterSet = getStaticFilterSet();
- private RegexFilterSet specificFilterSet;
- private boolean rewriteSourceTables;
-
- private QFile() {}
-
- public String getName() {
- return name;
- }
-
- public File getInputFile() {
- return inputFile;
- }
-
- public File getRawOutputFile() {
- return rawOutputFile;
- }
-
- public File getOutputFile() {
- return outputFile;
- }
-
- public File getExpectedOutputFile() {
- return expectedOutputFile;
- }
-
- public File getLogFile() {
- return logFile;
- }
-
- public File getBeforeExecuteLogFile() {
- return beforeExecuteLogFile;
- }
-
- public File getAfterExecuteLogFile() {
- return afterExecuteLogFile;
- }
-
- public String getDebugHint() {
- return String.format(DEBUG_HINT, inputFile, rawOutputFile, outputFile, expectedOutputFile,
- logFile, beforeExecuteLogFile, afterExecuteLogFile,
- "./itests/qtest/target/tmp/log/hive.log");
- }
-
- /**
- * Filters the sql commands if necessary.
- * @param commands The array of the sql commands before filtering
- * @return The filtered array of the sql command strings
- * @throws IOException File read error
- */
- public String[] filterCommands(String[] commands) throws IOException {
- if (rewriteSourceTables) {
- for (int i=0; i<commands.length; i++) {
- if (USE_PATTERN.matcher(commands[i]).matches()) {
- System.err.println(String.format(USE_COMMAND_WARNING, inputFile, commands[i]));
- }
- commands[i] = replaceTableNames(commands[i]);
- }
- }
- return commands;
- }
-
- /**
- * Replace the default src database TABLE_NAMEs in the queries with default.TABLE_NAME, like
- * src->default.src, srcpart->default.srcpart, so the queries could be run even if the used
- * database is query specific. This change is only a best effort, since we do not want to parse
- * the queries, we could not be sure that we do not replace other strings which are not
- * tablenames. Like 'select src from othertable;'. The q files containing these commands should
- * be excluded. Only replace the tablenames, if rewriteSourceTables is set.
- * @param source The original query string
- * @return The query string where the tablenames are replaced
- */
- private String replaceTableNames(String source) {
- for (String table : srcTables) {
- source = source.replaceAll("(?is)(\\s+)" + table + "([\\s;\\n\\)])", "$1default." + table
- + "$2");
- }
- return source;
- }
-
- /**
- * The result contains the original queries. To revert them to the original form remove the
- * 'default' from every default.TABLE_NAME, like default.src->src, default.srcpart->srcpart.
- * @param source The original query output
- * @return The query output where the tablenames are replaced
- */
- private String revertReplaceTableNames(String source) {
- for (String table : srcTables) {
- source = source.replaceAll("(?is)(\\s+)default\\." + table + "([\\s;\\n\\)])", "$1" + table
- + "$2");
- }
- return source;
- }
-
- public void filterOutput() throws IOException {
- String rawOutput = FileUtils.readFileToString(rawOutputFile, "UTF-8");
- if (rewriteSourceTables) {
- rawOutput = revertReplaceTableNames(rawOutput);
- }
- String filteredOutput = staticFilterSet.filter(specificFilterSet.filter(rawOutput));
- FileUtils.writeStringToFile(outputFile, filteredOutput);
- }
-
- public QTestProcessExecResult compareResults() throws IOException, InterruptedException {
- if (!expectedOutputFile.exists()) {
- throw new IOException("Expected results file does not exist: " + expectedOutputFile);
- }
- return executeDiff();
- }
-
- public void overwriteResults() throws IOException {
- FileUtils.copyFile(outputFile, expectedOutputFile);
- }
-
- private QTestProcessExecResult executeDiff() throws IOException, InterruptedException {
- List<String> diffCommandArgs = new ArrayList<String>();
- diffCommandArgs.add("diff");
-
- // Text file comparison
- diffCommandArgs.add("-a");
-
- if (Shell.WINDOWS) {
- // Ignore changes in the amount of white space
- diffCommandArgs.add("-b");
-
- // Files created on Windows machines have different line endings
- // than files created on Unix/Linux. Windows uses carriage return and line feed
- // ("\r\n") as a line ending, whereas Unix uses just line feed ("\n").
- // Also StringBuilder.toString(), Stream to String conversions adds extra
- // spaces at the end of the line.
- diffCommandArgs.add("--strip-trailing-cr"); // Strip trailing carriage return on input
- diffCommandArgs.add("-B"); // Ignore changes whose lines are all blank
- }
-
- // Add files to compare to the arguments list
- diffCommandArgs.add(getQuotedString(expectedOutputFile));
- diffCommandArgs.add(getQuotedString(outputFile));
-
- System.out.println("Running: " + org.apache.commons.lang.StringUtils.join(diffCommandArgs,
- ' '));
- Process executor = Runtime.getRuntime().exec(diffCommandArgs.toArray(
- new String[diffCommandArgs.size()]));
-
- ByteArrayOutputStream bos = new ByteArrayOutputStream();
- PrintStream out = new PrintStream(bos, true, "UTF-8");
-
- StreamPrinter errPrinter = new StreamPrinter(executor.getErrorStream(), null, System.err);
- StreamPrinter outPrinter = new StreamPrinter(executor.getInputStream(), null, System.out, out);
-
- outPrinter.start();
- errPrinter.start();
-
- int result = executor.waitFor();
-
- outPrinter.join();
- errPrinter.join();
-
- executor.waitFor();
-
- return QTestProcessExecResult.create(result, new String(bos.toByteArray(),
- StandardCharsets.UTF_8));
- }
-
- private static String getQuotedString(File file) {
- return Shell.WINDOWS ? String.format("\"%s\"", file.getAbsolutePath()) : file.getAbsolutePath();
- }
-
- private static class Filter {
- private final Pattern pattern;
- private final String replacement;
-
- public Filter(Pattern pattern, String replacement) {
- this.pattern = pattern;
- this.replacement = replacement;
- }
- }
-
- private static class RegexFilterSet {
- private final List<Filter> regexFilters = new ArrayList<Filter>();
-
- public RegexFilterSet addFilter(String regex, String replacement) {
- regexFilters.add(new Filter(Pattern.compile(regex), replacement));
- return this;
- }
-
- public String filter(String input) {
- for (Filter filter : regexFilters) {
- input = filter.pattern.matcher(input).replaceAll(filter.replacement);
- }
- return input;
- }
- }
-
- // These are the filters which are common for every QTest.
- // Check specificFilterSet for QTest specific ones.
- private static RegexFilterSet getStaticFilterSet() {
- // Pattern to remove the timestamp and other infrastructural info from the out file
- return new RegexFilterSet()
- .addFilter("Reading log file: .*\n", "")
- .addFilter("INFO : ", "")
- .addFilter(".*/tmp/.*\n", MASK_PATTERN)
- .addFilter(".*file:.*\n", MASK_PATTERN)
- .addFilter(".*file\\..*\n", MASK_PATTERN)
- .addFilter(".*CreateTime.*\n", MASK_PATTERN)
- .addFilter(".*transient_lastDdlTime.*\n", MASK_PATTERN)
- .addFilter("(?s)(" + MASK_PATTERN + ")+", MASK_PATTERN);
- }
-
- /**
- * Builder to generate QFile objects. After initializing the builder it is possible the
- * generate the next QFile object using it's name only.
- */
- public static class QFileBuilder {
- private File queryDirectory;
- private File logDirectory;
- private File resultsDirectory;
- private boolean rewriteSourceTables;
-
- public QFileBuilder() {
- }
-
- public QFileBuilder setQueryDirectory(File queryDirectory) {
- this.queryDirectory = queryDirectory;
- return this;
- }
-
- public QFileBuilder setLogDirectory(File logDirectory) {
- this.logDirectory = logDirectory;
- return this;
- }
-
- public QFileBuilder setResultsDirectory(File resultsDirectory) {
- this.resultsDirectory = resultsDirectory;
- return this;
- }
-
- public QFileBuilder setRewriteSourceTables(boolean rewriteSourceTables) {
- this.rewriteSourceTables = rewriteSourceTables;
- return this;
- }
-
- public QFile getQFile(String name) throws IOException {
- QFile result = new QFile();
- result.name = name;
- result.inputFile = new File(queryDirectory, name + ".q");
- result.rawOutputFile = new File(logDirectory, name + ".q.out.raw");
- result.outputFile = new File(logDirectory, name + ".q.out");
- result.expectedOutputFile = new File(resultsDirectory, name + ".q.out");
- result.logFile = new File(logDirectory, name + ".q.beeline");
- result.beforeExecuteLogFile = new File(logDirectory, name + ".q.beforeExecute.log");
- result.afterExecuteLogFile = new File(logDirectory, name + ".q.afterExecute.log");
- result.rewriteSourceTables = rewriteSourceTables;
- result.specificFilterSet = new RegexFilterSet()
- .addFilter("(PREHOOK|POSTHOOK): (Output|Input): database:" + name + "\n",
- "$1: $2: database:default\n")
- .addFilter("(PREHOOK|POSTHOOK): (Output|Input): " + name + "@", "$1: $2: default@")
- .addFilter("name(:?) " + name + "\\.(.*)\n", "name$1 default.$2\n")
- .addFilter("/" + name + ".db/", "/");
- return result;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/itests/util/src/main/java/org/apache/hive/beeline/QFileBeeLineClient.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hive/beeline/QFileBeeLineClient.java b/itests/util/src/main/java/org/apache/hive/beeline/QFileBeeLineClient.java
deleted file mode 100644
index f1b53f7..0000000
--- a/itests/util/src/main/java/org/apache/hive/beeline/QFileBeeLineClient.java
+++ /dev/null
@@ -1,156 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.beeline;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.PrintStream;
-import java.sql.SQLException;
-
-/**
- * QFile test client using BeeLine. It can be used to submit a list of command strings, or a QFile.
- */
-public class QFileBeeLineClient implements AutoCloseable {
- private BeeLine beeLine;
- private PrintStream beelineOutputStream;
- private File logFile;
-
- protected QFileBeeLineClient(String jdbcUrl, String jdbcDriver, String username, String password,
- File log) throws IOException {
- logFile = log;
- beeLine = new BeeLine();
- beelineOutputStream = new PrintStream(logFile, "UTF-8");
- beeLine.setOutputStream(beelineOutputStream);
- beeLine.setErrorStream(beelineOutputStream);
- beeLine.runCommands(
- new String[] {
- "!set verbose true",
- "!set shownestederrs true",
- "!set showwarnings true",
- "!set showelapsedtime false",
- "!set trimscripts false",
- "!set maxwidth -1",
- "!connect " + jdbcUrl + " " + username + " " + password + " " + jdbcDriver
- });
- }
-
- public void execute(String[] commands, File resultFile) throws SQLException {
- beeLine.runCommands(
- new String[] {
- "!record " + resultFile.getAbsolutePath()
- });
-
- int lastSuccessfulCommand = beeLine.runCommands(commands);
- if (commands.length != lastSuccessfulCommand) {
- throw new SQLException("Error executing SQL command: " + commands[lastSuccessfulCommand]);
- }
-
- beeLine.runCommands(new String[] {"!record"});
- }
-
- private void beforeExecute(QFile qFile) throws SQLException {
- execute(
- new String[] {
- "!set outputformat tsv2",
- "!set verbose false",
- "!set silent true",
- "!set showheader false",
- "USE default;",
- "SHOW TABLES;",
- "DROP DATABASE IF EXISTS `" + qFile.getName() + "` CASCADE;",
- "CREATE DATABASE `" + qFile.getName() + "`;",
- "USE `" + qFile.getName() + "`;",
- "set hive.in.test.short.logs=true;",
- "set hive.in.test.remove.logs=false;",
- },
- qFile.getBeforeExecuteLogFile());
- beeLine.setIsTestMode(true);
- }
-
- private void afterExecute(QFile qFile) throws SQLException {
- beeLine.setIsTestMode(false);
- execute(
- new String[] {
- "set hive.in.test.short.logs=false;",
- "!set verbose true",
- "!set silent false",
- "!set showheader true",
- "!set outputformat table",
- "USE default;",
- "DROP DATABASE IF EXISTS `" + qFile.getName() + "` CASCADE;",
- },
- qFile.getAfterExecuteLogFile());
- }
-
- public void execute(QFile qFile) throws SQLException, IOException {
- beforeExecute(qFile);
- String[] commands = beeLine.getCommands(qFile.getInputFile());
- execute(qFile.filterCommands(commands), qFile.getRawOutputFile());
- afterExecute(qFile);
- }
-
- public void close() {
- if (beeLine != null) {
- beeLine.runCommands(new String[] {
- "!quit"
- });
- }
- if (beelineOutputStream != null) {
- beelineOutputStream.close();
- }
- }
-
- /**
- * Builder to generated QFileBeeLineClient objects. The after initializing the builder, it can be
- * used to create new clients without any parameters.
- */
- public static class QFileClientBuilder {
- private String username;
- private String password;
- private String jdbcUrl;
- private String jdbcDriver;
-
- public QFileClientBuilder() {
- }
-
- public QFileClientBuilder setUsername(String username) {
- this.username = username;
- return this;
- }
-
- public QFileClientBuilder setPassword(String password) {
- this.password = password;
- return this;
- }
-
- public QFileClientBuilder setJdbcUrl(String jdbcUrl) {
- this.jdbcUrl = jdbcUrl;
- return this;
- }
-
- public QFileClientBuilder setJdbcDriver(String jdbcDriver) {
- this.jdbcDriver = jdbcDriver;
- return this;
- }
-
- public QFileBeeLineClient getClient(File logFile) throws IOException {
- return new QFileBeeLineClient(jdbcUrl, jdbcDriver, username, password, logFile);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/itests/util/src/main/java/org/apache/hive/beeline/package-info.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hive/beeline/package-info.java b/itests/util/src/main/java/org/apache/hive/beeline/package-info.java
deleted file mode 100644
index e05ac0a..0000000
--- a/itests/util/src/main/java/org/apache/hive/beeline/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Package for the BeeLine specific QTest classes.
- */
-package org.apache.hive.beeline;