You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by kg...@apache.org on 2020/05/17 06:30:14 UTC
[hive] branch master updated (b53a62f -> 5c9fa2a)
This is an automated email from the ASF dual-hosted git repository.
kgyrtkirk pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git.
from b53a62f HIVE-23376 : Avoid repeated SHA computation in GenericUDTFGetSplits for hive-exec jar (Ramesh Kumar via Rajesh Balamohan)
new b829a26 HIVE-23460: Add qoption to disable qtests (Zoltan Haindrich reviewed by László Bodor, Miklos Gergely)
new d1286f2 HIVE-23396: Many fixes and improvements to stabilize tests (Zoltan Haindrich reviewed by Miklos Gergely)
new 5c9fa2a HIVE-23374: QueryDisplay must be threadsafe (Zoltan Haindrich reviewed by László Bodor)
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../src/test/queries/positive/accumulo_joins.q | 1 +
.../metrics/metrics2/TestCodahaleMetrics.java | 2 +-
.../test/resources/testconfiguration.properties | 15 -----------
.../hadoop/hive/cli/control/AbstractCliConfig.java | 2 +-
.../apache/hadoop/hive/cli/control/CliConfigs.java | 27 -------------------
.../hive/cli/control/CoreAccumuloCliDriver.java | 3 +++
.../hadoop/hive/cli/control/CoreCliDriver.java | 7 +++--
.../hive/cli/control/CoreNegativeCliDriver.java | 3 +++
.../hadoop/hive/cli/control/CorePerfCliDriver.java | 3 +++
.../java/org/apache/hadoop/hive/ql/QTestUtil.java | 2 ++
...ransactional.java => QTestDisabledHandler.java} | 31 ++++++++++------------
.../org/apache/hadoop/hive/ql/QueryDisplay.java | 7 +++--
.../hadoop/hive/metastore/txn/TestTxnHandler.java | 6 ++---
.../apache/hadoop/hive/ql/metadata/TestHive.java | 25 +++++++----------
.../hadoop/hive/ql/metadata/TestHiveRemote.java | 24 ++++++++++++-----
.../ql/parse/TestReplicationSemanticAnalyzer.java | 13 ++++-----
.../clientnegative/authorization_uri_import.q | 1 +
.../queries/clientpositive/bucket_map_join_tez1.q | 2 ++
ql/src/test/queries/clientpositive/cbo_rp_insert.q | 1 +
.../test/queries/clientpositive/cbo_rp_lineage2.q | 1 +
.../queries/clientpositive/cbo_rp_subq_exists.q | 1 +
.../test/queries/clientpositive/cbo_rp_subq_in.q | 1 +
.../queries/clientpositive/cbo_rp_subq_not_in.q | 1 +
.../test/queries/clientpositive/cbo_subq_not_in.q | 1 +
.../test/queries/clientpositive/constprog_cast.q | 2 ++
.../queries/clientpositive/druid_timestamptz.q | 2 ++
.../test/queries/clientpositive/druidmini_joins.q | 1 +
.../queries/clientpositive/druidmini_masking.q | 2 ++
.../test/queries/clientpositive/fouter_join_ppr.q | 1 +
ql/src/test/queries/clientpositive/input31.q | 6 +----
.../test/queries/clientpositive/load_dyn_part3.q | 1 +
.../clientpositive/multi_insert_partitioned.q | 1 +
.../test/queries/clientpositive/perf/cbo_query44.q | 1 +
.../test/queries/clientpositive/perf/cbo_query45.q | 1 +
.../test/queries/clientpositive/perf/cbo_query67.q | 1 +
.../test/queries/clientpositive/perf/cbo_query70.q | 1 +
.../test/queries/clientpositive/perf/cbo_query86.q | 1 +
ql/src/test/queries/clientpositive/rcfile_merge1.q | 1 +
.../clientpositive/rfc5424_parser_file_pruning.q | 1 +
.../clientpositive/root_dir_external_table.q | 1 +
ql/src/test/queries/clientpositive/sample2.q | 1 +
ql/src/test/queries/clientpositive/sample4.q | 1 +
.../clientpositive/schema_evol_orc_acidvec_part.q | 1 +
.../schema_evol_orc_vec_part_llap_io.q | 1 +
.../queries/clientpositive/stats_filemetadata.q | 1 +
ql/src/test/queries/clientpositive/tez_smb_1.q | 2 ++
.../queries/clientpositive/udaf_context_ngrams.q | 2 ++
ql/src/test/queries/clientpositive/udaf_corr.q | 2 ++
.../clientpositive/udaf_histogram_numeric.q | 2 ++
.../test/queries/clientpositive/union_fast_stats.q | 2 ++
ql/src/test/queries/clientpositive/union_stats.q | 1 +
.../queries/clientpositive/vector_groupby_reduce.q | 2 ++
.../cli/session/TestSessionManagerMetrics.java | 2 +-
standalone-metastore/metastore-server/pom.xml | 2 +-
.../hadoop/hive/metastore/HiveMetaStore.java | 6 +++--
.../hadoop/hive/metastore/MetaStoreTestUtils.java | 2 +-
.../hadoop/hive/metastore/TestMarkPartition.java | 2 +-
.../hive/metastore/client/MetaStoreClientTest.java | 13 +++------
58 files changed, 128 insertions(+), 121 deletions(-)
copy itests/util/src/main/java/org/apache/hadoop/hive/ql/qoption/{QTestTransactional.java => QTestDisabledHandler.java} (65%)
[hive] 03/03: HIVE-23374: QueryDisplay must be threadsafe (Zoltan Haindrich reviewed by László Bodor)
Posted by kg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kgyrtkirk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
commit 5c9fa2acd973c6d7cedeaf82e969714deeb601a6
Author: Zoltan Haindrich <ki...@rxd.hu>
AuthorDate: Sun May 17 06:15:33 2020 +0000
HIVE-23374: QueryDisplay must be threadsafe (Zoltan Haindrich reviewed by László Bodor)
Signed-off-by: Zoltan Haindrich <ki...@rxd.hu>
---
ql/src/java/org/apache/hadoop/hive/ql/QueryDisplay.java | 7 +++----
1 file changed, 3 insertions(+), 4 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/QueryDisplay.java b/ql/src/java/org/apache/hadoop/hive/ql/QueryDisplay.java
index 1aa5be3..0dafb00 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/QueryDisplay.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/QueryDisplay.java
@@ -23,7 +23,6 @@ import org.apache.hadoop.hive.ql.exec.TaskResult;
import org.apache.hadoop.hive.ql.plan.api.StageType;
import java.io.IOException;
-import java.io.Serializable;
import java.util.*;
import org.apache.hadoop.mapred.Counters;
@@ -54,7 +53,7 @@ public class QueryDisplay {
private final LinkedHashMap<String, TaskDisplay> tasks = new LinkedHashMap<String, TaskDisplay>();
- public void updateTaskStatus(Task<?> tTask) {
+ public synchronized void updateTaskStatus(Task<?> tTask) {
if (!tasks.containsKey(tTask.getId())) {
tasks.put(tTask.getId(), new TaskDisplay(tTask));
}
@@ -374,11 +373,11 @@ public class QueryDisplay {
this.queryId = queryId;
}
- private String returnStringOrUnknown(String s) {
+ private static String returnStringOrUnknown(String s) {
return s == null ? "UNKNOWN" : s;
}
- public long getQueryStartTime() {
+ public synchronized long getQueryStartTime() {
return queryStartTime;
}
}
[hive] 01/03: HIVE-23460: Add qoption to disable qtests (Zoltan Haindrich reviewed by László Bodor, Miklos Gergely)
Posted by kg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kgyrtkirk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
commit b829a26f98368fee39c750034b85feecd85d0e0a
Author: Zoltan Haindrich <ki...@rxd.hu>
AuthorDate: Sun May 17 06:14:42 2020 +0000
HIVE-23460: Add qoption to disable qtests (Zoltan Haindrich reviewed by László Bodor, Miklos Gergely)
Signed-off-by: Zoltan Haindrich <ki...@rxd.hu>
---
.../src/test/queries/positive/accumulo_joins.q | 1 +
.../test/resources/testconfiguration.properties | 15 ------
.../hadoop/hive/cli/control/AbstractCliConfig.java | 2 +-
.../apache/hadoop/hive/cli/control/CliConfigs.java | 27 -----------
.../hive/cli/control/CoreAccumuloCliDriver.java | 3 ++
.../hadoop/hive/cli/control/CoreCliDriver.java | 7 ++-
.../hive/cli/control/CoreNegativeCliDriver.java | 3 ++
.../hadoop/hive/cli/control/CorePerfCliDriver.java | 3 ++
.../java/org/apache/hadoop/hive/ql/QTestUtil.java | 2 +
.../hive/ql/qoption/QTestDisabledHandler.java | 54 ++++++++++++++++++++++
.../clientnegative/authorization_uri_import.q | 1 +
.../queries/clientpositive/bucket_map_join_tez1.q | 2 +
ql/src/test/queries/clientpositive/cbo_rp_insert.q | 1 +
.../test/queries/clientpositive/cbo_rp_lineage2.q | 1 +
.../queries/clientpositive/cbo_rp_subq_exists.q | 1 +
.../test/queries/clientpositive/cbo_rp_subq_in.q | 1 +
.../queries/clientpositive/cbo_rp_subq_not_in.q | 1 +
.../test/queries/clientpositive/cbo_subq_not_in.q | 1 +
.../test/queries/clientpositive/constprog_cast.q | 2 +
.../queries/clientpositive/druid_timestamptz.q | 2 +
.../test/queries/clientpositive/druidmini_joins.q | 1 +
.../queries/clientpositive/druidmini_masking.q | 2 +
.../test/queries/clientpositive/fouter_join_ppr.q | 1 +
ql/src/test/queries/clientpositive/input31.q | 6 +--
.../test/queries/clientpositive/load_dyn_part3.q | 1 +
.../clientpositive/multi_insert_partitioned.q | 1 +
.../test/queries/clientpositive/perf/cbo_query44.q | 1 +
.../test/queries/clientpositive/perf/cbo_query45.q | 1 +
.../test/queries/clientpositive/perf/cbo_query67.q | 1 +
.../test/queries/clientpositive/perf/cbo_query70.q | 1 +
.../test/queries/clientpositive/perf/cbo_query86.q | 1 +
ql/src/test/queries/clientpositive/rcfile_merge1.q | 1 +
.../clientpositive/rfc5424_parser_file_pruning.q | 1 +
.../clientpositive/root_dir_external_table.q | 1 +
ql/src/test/queries/clientpositive/sample2.q | 1 +
ql/src/test/queries/clientpositive/sample4.q | 1 +
.../clientpositive/schema_evol_orc_acidvec_part.q | 1 +
.../schema_evol_orc_vec_part_llap_io.q | 1 +
.../queries/clientpositive/stats_filemetadata.q | 1 +
ql/src/test/queries/clientpositive/tez_smb_1.q | 2 +
.../queries/clientpositive/udaf_context_ngrams.q | 2 +
ql/src/test/queries/clientpositive/udaf_corr.q | 2 +
.../clientpositive/udaf_histogram_numeric.q | 2 +
.../test/queries/clientpositive/union_fast_stats.q | 2 +
ql/src/test/queries/clientpositive/union_stats.q | 1 +
.../queries/clientpositive/vector_groupby_reduce.q | 2 +
46 files changed, 118 insertions(+), 50 deletions(-)
diff --git a/accumulo-handler/src/test/queries/positive/accumulo_joins.q b/accumulo-handler/src/test/queries/positive/accumulo_joins.q
index 9d93029..05f1b0b 100644
--- a/accumulo-handler/src/test/queries/positive/accumulo_joins.q
+++ b/accumulo-handler/src/test/queries/positive/accumulo_joins.q
@@ -1,3 +1,4 @@
+--! qt:disabled:disabled for a long time now...dont know why
--! qt:dataset:src
DROP TABLE users;
DROP TABLE states;
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index b48889e..2ad66a6 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -3,21 +3,6 @@
# DO NOT USE minimr, as MR is deprecated and MinimrCliDriver will be removed
minimr.query.files=doesnotexist.q\
-# Tests that are not enabled for CLI Driver
-disabled.query.files=cbo_rp_subq_in.q,\
- cbo_rp_subq_not_in.q,\
- cbo_rp_subq_exists.q,\
- rcfile_merge1.q,\
- stats_filemetadata.q,\
- cbo_rp_insert.q,\
- cbo_rp_lineage2.q,\
- union_stats.q,\
- sample2.q,\
- sample4.q,\
- root_dir_external_table.q,\
- input31.q
-
-
# NOTE: Add tests to minitez only if it is very
# specific to tez and cannot be added to minillap.
minitez.query.files.shared=delete_orig_table.q,\
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCliConfig.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCliConfig.java
index 353a4aa..060f9b7 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCliConfig.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCliConfig.java
@@ -130,7 +130,7 @@ public abstract class AbstractCliConfig {
}
}
- protected void excludeQuery(String qFile) {
+ private void excludeQuery(String qFile) {
excludedQueryFileNames.add(qFile);
}
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java
index 473a9f8..d7f519f 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java
@@ -56,7 +56,6 @@ public class CliConfigs {
excludesFrom(testConfigProps, "encrypted.query.files");
excludesFrom(testConfigProps, "spark.only.query.files");
excludesFrom(testConfigProps, "miniSparkOnYarn.only.query.files");
- excludesFrom(testConfigProps, "disabled.query.files");
excludesFrom(testConfigProps, "localSpark.only.query.files");
excludesFrom(testConfigProps, "druid.query.files");
excludesFrom(testConfigProps, "druid.kafka.query.files");
@@ -65,13 +64,6 @@ public class CliConfigs {
excludesFrom(testConfigProps, "erasurecoding.shared.query.files");
excludesFrom(testConfigProps, "beeline.positive.include");
- excludeQuery("fouter_join_ppr.q"); // Disabled in HIVE-19509
- excludeQuery("udaf_context_ngrams.q"); // disabled in HIVE-20741
- excludeQuery("udaf_corr.q"); // disabled in HIVE-20741
- excludeQuery("udaf_histogram_numeric.q"); // disabled in HIVE-20715
- excludeQuery("vector_groupby_reduce.q"); // Disabled in HIVE-21396
- excludeQuery("constprog_cast.q"); // TODO: Enable when we move to Calcite 1.23
-
setResultsDir("ql/src/test/results/clientpositive");
setLogDir("itests/qtest/target/qfile-results/clientpositive");
@@ -183,10 +175,6 @@ public class CliConfigs {
setQueryDir("ql/src/test/queries/clientpositive");
includesFrom(testConfigProps, "druid.query.files");
- excludeQuery("druid_timestamptz.q"); // Disabled in HIVE-20322
- excludeQuery("druidmini_joins.q"); // Disabled in HIVE-20322
- excludeQuery("druidmini_masking.q"); // Disabled in HIVE-20322
- //excludeQuery("druidmini_test1.q"); // Disabled in HIVE-20322
setResultsDir("ql/src/test/results/clientpositive/druid");
setLogDir("itests/qtest/target/tmp/log");
@@ -245,13 +233,6 @@ public class CliConfigs {
includesFrom(testConfigProps, "minillaplocal.query.files");
includesFrom(testConfigProps, "minillaplocal.shared.query.files");
- excludeQuery("bucket_map_join_tez1.q"); // Disabled in HIVE-19509
- excludeQuery("tez_smb_1.q"); // Disabled in HIVE-19509
- excludeQuery("union_fast_stats.q"); // Disabled in HIVE-19509
- excludeQuery("schema_evol_orc_acidvec_part.q"); // Disabled in HIVE-19509
- excludeQuery("schema_evol_orc_vec_part_llap_io.q"); // Disabled in HIVE-19509
- excludeQuery("load_dyn_part3.q"); // Disabled in HIVE-20662. Enable in HIVE-20663.
- excludeQuery("rfc5424_parser_file_pruning.q"); // Disabled in HIVE-21427
setResultsDir("ql/src/test/results/clientpositive/llap");
setLogDir("itests/qtest/target/qfile-results/clientpositive");
@@ -332,12 +313,6 @@ public class CliConfigs {
excludesFrom(testConfigProps, "encrypted.query.files");
excludesFrom(testConfigProps, "erasurecoding.only.query.files");
- excludeQuery("cbo_query44.q"); // TODO: Enable when we move to Calcite 1.18
- excludeQuery("cbo_query45.q"); // TODO: Enable when we move to Calcite 1.18
- excludeQuery("cbo_query67.q"); // TODO: Enable when we move to Calcite 1.18
- excludeQuery("cbo_query70.q"); // TODO: Enable when we move to Calcite 1.18
- excludeQuery("cbo_query86.q"); // TODO: Enable when we move to Calcite 1.18
-
setLogDir("itests/qtest/target/qfile-results/clientpositive/tez");
if (useConstraints) {
@@ -407,7 +382,6 @@ public class CliConfigs {
excludesFrom(testConfigProps, "minimr.query.negative.files");
excludesFrom(testConfigProps, "spark.only.query.negative.files");
- excludeQuery("authorization_uri_import.q");
setResultsDir("ql/src/test/results/clientnegative");
setLogDir("itests/qtest/target/qfile-results/clientnegative");
@@ -552,7 +526,6 @@ public class CliConfigs {
super(CoreAccumuloCliDriver.class);
try {
setQueryDir("accumulo-handler/src/test/queries/positive");
- excludeQuery("accumulo_joins.q");
setResultsDir("accumulo-handler/src/test/results/positive");
setLogDir("itests/qtest/target/qfile-results/accumulo-handler/positive");
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreAccumuloCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreAccumuloCliDriver.java
index 31f5cfa..c78348e 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreAccumuloCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreAccumuloCliDriver.java
@@ -32,6 +32,7 @@ import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
+import org.junit.internal.AssumptionViolatedException;
public class CoreAccumuloCliDriver extends CliAdapter {
@@ -119,6 +120,8 @@ public class CoreAccumuloCliDriver extends CliAdapter {
}
qt.clearPostTestEffects();
+ } catch (AssumptionViolatedException e) {
+ throw e;
} catch (Exception e) {
qt.failedWithException(e, fname, null);
}
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java
index 171e2d9..5e85f68 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java
@@ -35,6 +35,7 @@ import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
+import org.junit.internal.AssumptionViolatedException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -181,8 +182,10 @@ public class CoreCliDriver extends CliAdapter {
: "\r\n" + result.getCapturedOutput();
qt.failedDiff(result.getReturnCode(), fname, message);
}
- }
- catch (Exception e) {
+ } catch (AssumptionViolatedException e) {
+ skipped = true;
+ throw e;
+ } catch (Exception e) {
failed = true;
qt.failedWithException(e, fname, QTestUtil.DEBUG_HINT);
} finally {
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java
index 2536c1d..798d1f2 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.hive.ql.QTestMiniClusters.MiniClusterType;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
+import org.junit.internal.AssumptionViolatedException;
import com.google.common.base.Strings;
@@ -136,6 +137,8 @@ public class CoreNegativeCliDriver extends CliAdapter{
: "\r\n" + result.getCapturedOutput();
qt.failedDiff(result.getReturnCode(), fname, message);
}
+ } catch (AssumptionViolatedException e) {
+ throw e;
} catch (Error error) {
QTestProcessExecResult qTestProcessExecResult = qt.checkNegativeResults(fname, error);
if (qTestProcessExecResult.getReturnCode() != 0) {
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java
index d7faaf6..8b670c9 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.hive.ql.QTestMiniClusters.MiniClusterType;
import org.apache.hadoop.hive.ql.processors.CommandProcessorException;
import org.junit.After;
import org.junit.AfterClass;
+import org.junit.internal.AssumptionViolatedException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -148,6 +149,8 @@ public class CorePerfCliDriver extends CliAdapter {
: "\r\n" + result.getCapturedOutput();
qt.failedDiff(result.getReturnCode(), fname, message);
}
+ } catch (AssumptionViolatedException e) {
+ throw e;
} catch (Exception e) {
qt.failedWithException(e, fname, QTestUtil.DEBUG_HINT);
}
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
index 8d3856b..f7c21a0 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
@@ -77,6 +77,7 @@ import org.apache.hadoop.hive.ql.processors.CommandProcessorFactory;
import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
import org.apache.hadoop.hive.ql.processors.HiveCommand;
import org.apache.hadoop.hive.ql.qoption.QTestAuthorizerHandler;
+import org.apache.hadoop.hive.ql.qoption.QTestDisabledHandler;
import org.apache.hadoop.hive.ql.qoption.QTestOptionDispatcher;
import org.apache.hadoop.hive.ql.qoption.QTestReplaceHandler;
import org.apache.hadoop.hive.ql.qoption.QTestSysDbHandler;
@@ -225,6 +226,7 @@ public class QTestUtil {
dispatcher.register("scheduledqueryservice", new QTestScheduledQueryServiceProvider(conf));
dispatcher.register("scheduledquerycleaner", new QTestScheduledQueryCleaner());
dispatcher.register("authorizer", new QTestAuthorizerHandler());
+ dispatcher.register("disabled", new QTestDisabledHandler());
String scriptsDir = getScriptsDir();
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/qoption/QTestDisabledHandler.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/qoption/QTestDisabledHandler.java
new file mode 100644
index 0000000..4463a50
--- /dev/null
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/qoption/QTestDisabledHandler.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.qoption;
+
+import org.apache.hadoop.hive.ql.QTestUtil;
+import org.junit.Assume;
+
+import com.google.common.base.Strings;
+
+/**
+ * QTest disabled directive handler
+ *
+ * Example:
+ * --! qt:disabled:reason
+ *
+ */
+public class QTestDisabledHandler implements QTestOptionHandler {
+
+ private String message;
+
+ @Override
+ public void processArguments(String arguments) {
+ message = arguments;
+ if (Strings.isNullOrEmpty(message)) {
+ throw new RuntimeException("you have to give a reason why it was ignored");
+ }
+ }
+
+ @Override
+ public void beforeTest(QTestUtil qt) throws Exception {
+ Assume.assumeTrue(message, (message == null));
+ }
+
+ @Override
+ public void afterTest(QTestUtil qt) throws Exception {
+ message = null;
+ }
+
+}
diff --git a/ql/src/test/queries/clientnegative/authorization_uri_import.q b/ql/src/test/queries/clientnegative/authorization_uri_import.q
index a65e58e..c30acc6 100644
--- a/ql/src/test/queries/clientnegative/authorization_uri_import.q
+++ b/ql/src/test/queries/clientnegative/authorization_uri_import.q
@@ -1,3 +1,4 @@
+--! qt:disabled:disabled for a long time now...dont know why
--! qt:authorizer
set hive.mapred.mode=nonstrict;
diff --git a/ql/src/test/queries/clientpositive/bucket_map_join_tez1.q b/ql/src/test/queries/clientpositive/bucket_map_join_tez1.q
index 049a4d9..0230998 100644
--- a/ql/src/test/queries/clientpositive/bucket_map_join_tez1.q
+++ b/ql/src/test/queries/clientpositive/bucket_map_join_tez1.q
@@ -1,3 +1,5 @@
+--! qt:disabled:Disabled in HIVE-19509
+
set hive.strict.checks.bucketing=false;
set hive.mapred.mode=nonstrict;
diff --git a/ql/src/test/queries/clientpositive/cbo_rp_insert.q b/ql/src/test/queries/clientpositive/cbo_rp_insert.q
index 4c493db..ca72af2 100644
--- a/ql/src/test/queries/clientpositive/cbo_rp_insert.q
+++ b/ql/src/test/queries/clientpositive/cbo_rp_insert.q
@@ -1,3 +1,4 @@
+--! qt:disabled:disabled by 7e64114ddca5 in 2018
set hive.cbo.enable=true;
set hive.cbo.returnpath.hiveop=true;
diff --git a/ql/src/test/queries/clientpositive/cbo_rp_lineage2.q b/ql/src/test/queries/clientpositive/cbo_rp_lineage2.q
index fc22431..536d000 100644
--- a/ql/src/test/queries/clientpositive/cbo_rp_lineage2.q
+++ b/ql/src/test/queries/clientpositive/cbo_rp_lineage2.q
@@ -1,3 +1,4 @@
+--! qt:disabled:disabled by 6eaef86ea736 in 2018
--! qt_n16:dataset_n16:src1
--! qt_n16:dataset_n16:src
set_n16 hive.mapred.mode=nonstrict_n16;
diff --git a/ql/src/test/queries/clientpositive/cbo_rp_subq_exists.q b/ql/src/test/queries/clientpositive/cbo_rp_subq_exists.q
index b3e1af5..c60d4c2 100644
--- a/ql/src/test/queries/clientpositive/cbo_rp_subq_exists.q
+++ b/ql/src/test/queries/clientpositive/cbo_rp_subq_exists.q
@@ -1,3 +1,4 @@
+--! qt:disabled:disabled by 382dc2084224 in 2016
--! qt:dataset:src_cbo
set hive.cbo.enable=true;
set hive.cbo.returnpath.hiveop=true;
diff --git a/ql/src/test/queries/clientpositive/cbo_rp_subq_in.q b/ql/src/test/queries/clientpositive/cbo_rp_subq_in.q
index 67b1150..5f16a8f 100644
--- a/ql/src/test/queries/clientpositive/cbo_rp_subq_in.q
+++ b/ql/src/test/queries/clientpositive/cbo_rp_subq_in.q
@@ -1,3 +1,4 @@
+--! qt:disabled:disabled for a along time - along with the other cbo_rb_subq tests
--! qt:dataset:src_cbo
--! qt:dataset:part
--! qt:dataset:lineitem
diff --git a/ql/src/test/queries/clientpositive/cbo_rp_subq_not_in.q b/ql/src/test/queries/clientpositive/cbo_rp_subq_not_in.q
index fee5274..9719a47 100644
--- a/ql/src/test/queries/clientpositive/cbo_rp_subq_not_in.q
+++ b/ql/src/test/queries/clientpositive/cbo_rp_subq_not_in.q
@@ -1,3 +1,4 @@
+--! qt:disabled:disabled by 382dc2084224 in 2016
--! qt:dataset:src_cbo
--! qt:dataset:part
--! qt:dataset:lineitem
diff --git a/ql/src/test/queries/clientpositive/cbo_subq_not_in.q b/ql/src/test/queries/clientpositive/cbo_subq_not_in.q
index b3dd653..790a5db 100644
--- a/ql/src/test/queries/clientpositive/cbo_subq_not_in.q
+++ b/ql/src/test/queries/clientpositive/cbo_subq_not_in.q
@@ -1,3 +1,4 @@
+--! qt:disabled:disabled by 382dc2084224 in 2016
--! qt:dataset:src_cbo
--! qt:dataset:part
--! qt:dataset:lineitem
diff --git a/ql/src/test/queries/clientpositive/constprog_cast.q b/ql/src/test/queries/clientpositive/constprog_cast.q
index 765cd85..90831ef 100644
--- a/ql/src/test/queries/clientpositive/constprog_cast.q
+++ b/ql/src/test/queries/clientpositive/constprog_cast.q
@@ -1,3 +1,5 @@
+--! qt:disabled:Enable when we move to Calcite 1.23
+
set hive.optimize.constant.propagation=true;
CREATE TABLE constcasttest (id string);
diff --git a/ql/src/test/queries/clientpositive/druid_timestamptz.q b/ql/src/test/queries/clientpositive/druid_timestamptz.q
index b21ca9d..1c98594 100644
--- a/ql/src/test/queries/clientpositive/druid_timestamptz.q
+++ b/ql/src/test/queries/clientpositive/druid_timestamptz.q
@@ -1,3 +1,5 @@
+--! qt:disabled:Disabled in HIVE-20322
+
set hive.fetch.task.conversion=more;
SET hive.ctas.external.tables=true;
SET hive.external.table.purge.default = true;
diff --git a/ql/src/test/queries/clientpositive/druidmini_joins.q b/ql/src/test/queries/clientpositive/druidmini_joins.q
index b6876fe..9e8231d 100644
--- a/ql/src/test/queries/clientpositive/druidmini_joins.q
+++ b/ql/src/test/queries/clientpositive/druidmini_joins.q
@@ -1,3 +1,4 @@
+--! qt:disabled:Disabled in HIVE-20322
SET hive.explain.user=false;
SET hive.ctas.external.tables=true;
diff --git a/ql/src/test/queries/clientpositive/druidmini_masking.q b/ql/src/test/queries/clientpositive/druidmini_masking.q
index 824611f..da13c5a 100644
--- a/ql/src/test/queries/clientpositive/druidmini_masking.q
+++ b/ql/src/test/queries/clientpositive/druidmini_masking.q
@@ -1,3 +1,5 @@
+--! qt:disabled:Disabled in HIVE-20322
+
set hive.mapred.mode=nonstrict;
set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest;
SET hive.ctas.external.tables=true;
diff --git a/ql/src/test/queries/clientpositive/fouter_join_ppr.q b/ql/src/test/queries/clientpositive/fouter_join_ppr.q
index 735fba7..90018b8 100644
--- a/ql/src/test/queries/clientpositive/fouter_join_ppr.q
+++ b/ql/src/test/queries/clientpositive/fouter_join_ppr.q
@@ -1,3 +1,4 @@
+--! qt:disabled:Disabled in HIVE-19509
--! qt:dataset:srcpart
--! qt:dataset:src
set hive.mapred.mode=nonstrict;
diff --git a/ql/src/test/queries/clientpositive/input31.q b/ql/src/test/queries/clientpositive/input31.q
index eb48166..ac69ee6 100644
--- a/ql/src/test/queries/clientpositive/input31.q
+++ b/ql/src/test/queries/clientpositive/input31.q
@@ -1,3 +1,4 @@
+--! qt:disabled:disabled by 54b4b2d428bb in 2018
--! qt:dataset:srcbucket
set hive.mapred.mode=nonstrict;
@@ -19,8 +20,3 @@ select count(1) from srcbucket;
set hive.test.mode=false;
select * from tst_dest31;
-
-
-
-
-
diff --git a/ql/src/test/queries/clientpositive/load_dyn_part3.q b/ql/src/test/queries/clientpositive/load_dyn_part3.q
index 2427c84..6c7108d 100644
--- a/ql/src/test/queries/clientpositive/load_dyn_part3.q
+++ b/ql/src/test/queries/clientpositive/load_dyn_part3.q
@@ -1,3 +1,4 @@
+--! qt:disabled:Disabled in HIVE-20662. Enable in HIVE-20663
--! qt:dataset:srcpart
set hive.explain.user=false;
-- SORT_QUERY_RESULTS
diff --git a/ql/src/test/queries/clientpositive/multi_insert_partitioned.q b/ql/src/test/queries/clientpositive/multi_insert_partitioned.q
index 991dc8d..16f3996 100644
--- a/ql/src/test/queries/clientpositive/multi_insert_partitioned.q
+++ b/ql/src/test/queries/clientpositive/multi_insert_partitioned.q
@@ -1,3 +1,4 @@
+--! qt:disabled:multi insert is falky
--! qt:dataset:src
-- SORT_QUERY_RESULTS
set hive.stats.column.autogather=false;
diff --git a/ql/src/test/queries/clientpositive/perf/cbo_query44.q b/ql/src/test/queries/clientpositive/perf/cbo_query44.q
index 814a721..638d9d5 100644
--- a/ql/src/test/queries/clientpositive/perf/cbo_query44.q
+++ b/ql/src/test/queries/clientpositive/perf/cbo_query44.q
@@ -1,3 +1,4 @@
+--! qt:disabled:TODO: Enable when we move to Calcite 1.18
set hive.mapred.mode=nonstrict;
-- start query 1 in stream 0 using template query44.tpl and seed 1819994127
explain cbo
diff --git a/ql/src/test/queries/clientpositive/perf/cbo_query45.q b/ql/src/test/queries/clientpositive/perf/cbo_query45.q
index e1b4348..4548a43 100644
--- a/ql/src/test/queries/clientpositive/perf/cbo_query45.q
+++ b/ql/src/test/queries/clientpositive/perf/cbo_query45.q
@@ -1,3 +1,4 @@
+--! qt:disabled:TODO: Enable when we move to Calcite 1.18
set hive.mapred.mode=nonstrict;
-- start query 1 in stream 0 using template query45.tpl and seed 2031708268
explain cbo
diff --git a/ql/src/test/queries/clientpositive/perf/cbo_query67.q b/ql/src/test/queries/clientpositive/perf/cbo_query67.q
index 5781aac..f237344 100644
--- a/ql/src/test/queries/clientpositive/perf/cbo_query67.q
+++ b/ql/src/test/queries/clientpositive/perf/cbo_query67.q
@@ -1,3 +1,4 @@
+--! qt:disabled:TODO: Enable when we move to Calcite 1.18
set hive.mapred.mode=nonstrict;
-- start query 1 in stream 0 using template query67.tpl and seed 1819994127
explain cbo
diff --git a/ql/src/test/queries/clientpositive/perf/cbo_query70.q b/ql/src/test/queries/clientpositive/perf/cbo_query70.q
index 5b8b76e..b6b1ff5 100644
--- a/ql/src/test/queries/clientpositive/perf/cbo_query70.q
+++ b/ql/src/test/queries/clientpositive/perf/cbo_query70.q
@@ -1,3 +1,4 @@
+--! qt:disabled:TODO: Enable when we move to Calcite 1.18
set hive.mapred.mode=nonstrict;
-- start query 1 in stream 0 using template query70.tpl and seed 1819994127
explain cbo
diff --git a/ql/src/test/queries/clientpositive/perf/cbo_query86.q b/ql/src/test/queries/clientpositive/perf/cbo_query86.q
index a8e9941..df3ca41 100644
--- a/ql/src/test/queries/clientpositive/perf/cbo_query86.q
+++ b/ql/src/test/queries/clientpositive/perf/cbo_query86.q
@@ -1,3 +1,4 @@
+--! qt:disabled:TODO: Enable when we move to Calcite 1.18
set hive.mapred.mode=nonstrict;
-- start query 1 in stream 0 using template query86.tpl and seed 1819994127
explain cbo
diff --git a/ql/src/test/queries/clientpositive/rcfile_merge1.q b/ql/src/test/queries/clientpositive/rcfile_merge1.q
index d72cfa3..46fe03e 100644
--- a/ql/src/test/queries/clientpositive/rcfile_merge1.q
+++ b/ql/src/test/queries/clientpositive/rcfile_merge1.q
@@ -1,3 +1,4 @@
+--! qt:disabled:disabled by e1ce9a23a704 in 2015
--! qt:dataset:src
--! qt:dataset:part
set hive.merge.rcfile.block.level=false;
diff --git a/ql/src/test/queries/clientpositive/rfc5424_parser_file_pruning.q b/ql/src/test/queries/clientpositive/rfc5424_parser_file_pruning.q
index a9dd323..a65709c 100644
--- a/ql/src/test/queries/clientpositive/rfc5424_parser_file_pruning.q
+++ b/ql/src/test/queries/clientpositive/rfc5424_parser_file_pruning.q
@@ -1,3 +1,4 @@
+--! qt:disabled:Disabled in HIVE-21427
set hive.fetch.task.conversion=none;
set time zone UTC;
diff --git a/ql/src/test/queries/clientpositive/root_dir_external_table.q b/ql/src/test/queries/clientpositive/root_dir_external_table.q
index 66a50e6..cbe4aee 100644
--- a/ql/src/test/queries/clientpositive/root_dir_external_table.q
+++ b/ql/src/test/queries/clientpositive/root_dir_external_table.q
@@ -1,3 +1,4 @@
+--! qt:disabled:disabled by 54b4b2d428bb in 2018
--! qt:dataset:src
set hive.mapred.mode=nonstrict;
set hive.llap.io.enabled=false;
diff --git a/ql/src/test/queries/clientpositive/sample2.q b/ql/src/test/queries/clientpositive/sample2.q
index 3cd5521..c279af2 100644
--- a/ql/src/test/queries/clientpositive/sample2.q
+++ b/ql/src/test/queries/clientpositive/sample2.q
@@ -1,3 +1,4 @@
+--! qt:disabled:disabled by 38f7a7f3839e in 2018
--! qt:dataset:srcbucket
CREATE TABLE dest1_n29(key INT, value STRING) STORED AS TEXTFILE;
diff --git a/ql/src/test/queries/clientpositive/sample4.q b/ql/src/test/queries/clientpositive/sample4.q
index 756d05c..9afdbd5 100644
--- a/ql/src/test/queries/clientpositive/sample4.q
+++ b/ql/src/test/queries/clientpositive/sample4.q
@@ -1,3 +1,4 @@
+--! qt:disabled:disabled by 38f7a7f3839e in 2018
--! qt:dataset:srcbucket
CREATE TABLE dest1_n118(key INT, value STRING) STORED AS TEXTFILE;
diff --git a/ql/src/test/queries/clientpositive/schema_evol_orc_acidvec_part.q b/ql/src/test/queries/clientpositive/schema_evol_orc_acidvec_part.q
index 9466ea8..5e93ba1 100644
--- a/ql/src/test/queries/clientpositive/schema_evol_orc_acidvec_part.q
+++ b/ql/src/test/queries/clientpositive/schema_evol_orc_acidvec_part.q
@@ -1,3 +1,4 @@
+--! qt:disabled:Disabled in HIVE-19509
--! qt:dataset:part
set hive.explain.user=false;
set hive.fetch.task.conversion=none;
diff --git a/ql/src/test/queries/clientpositive/schema_evol_orc_vec_part_llap_io.q b/ql/src/test/queries/clientpositive/schema_evol_orc_vec_part_llap_io.q
index 1a25950..7dee814 100644
--- a/ql/src/test/queries/clientpositive/schema_evol_orc_vec_part_llap_io.q
+++ b/ql/src/test/queries/clientpositive/schema_evol_orc_vec_part_llap_io.q
@@ -1,3 +1,4 @@
+--! qt:disabled:Disabled in HIVE-19509
--! qt:dataset:part
set hive.explain.user=false;
set hive.fetch.task.conversion=none;
diff --git a/ql/src/test/queries/clientpositive/stats_filemetadata.q b/ql/src/test/queries/clientpositive/stats_filemetadata.q
index b0b9ef2..60e72a3 100644
--- a/ql/src/test/queries/clientpositive/stats_filemetadata.q
+++ b/ql/src/test/queries/clientpositive/stats_filemetadata.q
@@ -1,3 +1,4 @@
+--! qt:disabled:disabled by 98c5b637df2d in 2017
--! qt:dataset:src
set hive.mapred.mode=nonstrict;
diff --git a/ql/src/test/queries/clientpositive/tez_smb_1.q b/ql/src/test/queries/clientpositive/tez_smb_1.q
index e121d52..2649c68 100644
--- a/ql/src/test/queries/clientpositive/tez_smb_1.q
+++ b/ql/src/test/queries/clientpositive/tez_smb_1.q
@@ -1,3 +1,5 @@
+--! qt:disabled:Disabled in HIVE-19509
+
set hive.stats.column.autogather=false;
set hive.strict.checks.bucketing=false;
diff --git a/ql/src/test/queries/clientpositive/udaf_context_ngrams.q b/ql/src/test/queries/clientpositive/udaf_context_ngrams.q
index 45105bb..b993c67 100644
--- a/ql/src/test/queries/clientpositive/udaf_context_ngrams.q
+++ b/ql/src/test/queries/clientpositive/udaf_context_ngrams.q
@@ -1,3 +1,5 @@
+--! qt:disabled:disabled in HIVE-20741
+
CREATE TABLE kafka_n0 (contents STRING);
LOAD DATA LOCAL INPATH '../../data/files/text-en.txt' INTO TABLE kafka_n0;
set mapred.reduce.tasks=1;
diff --git a/ql/src/test/queries/clientpositive/udaf_corr.q b/ql/src/test/queries/clientpositive/udaf_corr.q
index 3578ac2..72ae886 100644
--- a/ql/src/test/queries/clientpositive/udaf_corr.q
+++ b/ql/src/test/queries/clientpositive/udaf_corr.q
@@ -1,3 +1,5 @@
+--! qt:disabled:disabled in HIVE-20741
+
set hive.mapred.mode=nonstrict;
DROP TABLE covar_tab_n0;
CREATE TABLE covar_tab_n0 (a INT, b INT, c INT)
diff --git a/ql/src/test/queries/clientpositive/udaf_histogram_numeric.q b/ql/src/test/queries/clientpositive/udaf_histogram_numeric.q
index c1f67fd..9fff207 100644
--- a/ql/src/test/queries/clientpositive/udaf_histogram_numeric.q
+++ b/ql/src/test/queries/clientpositive/udaf_histogram_numeric.q
@@ -1,3 +1,5 @@
+--! qt:disabled:disabled in HIVE-20715
+
--! qt:dataset:src
SELECT histogram_numeric(cast(substr(src.value,5) AS double), 2) FROM src;
diff --git a/ql/src/test/queries/clientpositive/union_fast_stats.q b/ql/src/test/queries/clientpositive/union_fast_stats.q
index 6359482..75bf00e 100644
--- a/ql/src/test/queries/clientpositive/union_fast_stats.q
+++ b/ql/src/test/queries/clientpositive/union_fast_stats.q
@@ -1,4 +1,6 @@
+--! qt:disabled:Disabled in HIVE-19509
--! qt:dataset:alltypesorc
+
set hive.stats.dbclass=fs;
set hive.merge.mapfiles=false;
set hive.merge.mapredfiles=false;
diff --git a/ql/src/test/queries/clientpositive/union_stats.q b/ql/src/test/queries/clientpositive/union_stats.q
index f36e15c..2cb6f80 100644
--- a/ql/src/test/queries/clientpositive/union_stats.q
+++ b/ql/src/test/queries/clientpositive/union_stats.q
@@ -1,3 +1,4 @@
+--! qt:disabled:disabled by 38f7a7f3839e in 2018
--! qt:dataset:src
explain extended create table t as select * from src union all select * from src;
diff --git a/ql/src/test/queries/clientpositive/vector_groupby_reduce.q b/ql/src/test/queries/clientpositive/vector_groupby_reduce.q
index d844366..55a0fe2 100644
--- a/ql/src/test/queries/clientpositive/vector_groupby_reduce.q
+++ b/ql/src/test/queries/clientpositive/vector_groupby_reduce.q
@@ -1,3 +1,5 @@
+--! qt:disabled:Disabled in HIVE-21396
+
set hive.mapred.mode=nonstrict;
set hive.explain.user=false;
SET hive.vectorized.execution.enabled=true;
[hive] 02/03: HIVE-23396: Many fixes and improvements to stabilize
tests (Zoltan Haindrich reviewed by Miklos Gergely)
Posted by kg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kgyrtkirk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
commit d1286f2f71f0da8fbede250a9dfc2a0a17c33f3f
Author: Zoltan Haindrich <ki...@rxd.hu>
AuthorDate: Sun May 17 06:15:16 2020 +0000
HIVE-23396: Many fixes and improvements to stabilize tests (Zoltan Haindrich reviewed by Miklos Gergely)
Signed-off-by: Zoltan Haindrich <ki...@rxd.hu>
---
.../metrics/metrics2/TestCodahaleMetrics.java | 2 +-
.../hadoop/hive/metastore/txn/TestTxnHandler.java | 6 +++---
.../apache/hadoop/hive/ql/metadata/TestHive.java | 25 ++++++++--------------
.../hadoop/hive/ql/metadata/TestHiveRemote.java | 24 +++++++++++++++------
.../ql/parse/TestReplicationSemanticAnalyzer.java | 13 +++++------
.../cli/session/TestSessionManagerMetrics.java | 2 +-
standalone-metastore/metastore-server/pom.xml | 2 +-
.../hadoop/hive/metastore/HiveMetaStore.java | 6 ++++--
.../hadoop/hive/metastore/MetaStoreTestUtils.java | 2 +-
.../hadoop/hive/metastore/TestMarkPartition.java | 2 +-
.../hive/metastore/client/MetaStoreClientTest.java | 13 ++++-------
11 files changed, 47 insertions(+), 50 deletions(-)
diff --git a/common/src/test/org/apache/hadoop/hive/common/metrics/metrics2/TestCodahaleMetrics.java b/common/src/test/org/apache/hadoop/hive/common/metrics/metrics2/TestCodahaleMetrics.java
index 9c4e475..85ded7e 100644
--- a/common/src/test/org/apache/hadoop/hive/common/metrics/metrics2/TestCodahaleMetrics.java
+++ b/common/src/test/org/apache/hadoop/hive/common/metrics/metrics2/TestCodahaleMetrics.java
@@ -55,7 +55,7 @@ public class TestCodahaleMetrics {
private static final Path tmpDir = Paths.get(System.getProperty("java.io.tmpdir"));
private static File jsonReportFile;
private static MetricRegistry metricRegistry;
- private static final long REPORT_INTERVAL_MS = 100;
+ private static final long REPORT_INTERVAL_MS = 2000;
@BeforeClass
public static void setUp() throws Exception {
diff --git a/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java b/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java
index 868da0c..f65619e 100644
--- a/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java
+++ b/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java
@@ -1203,7 +1203,7 @@ public class TestTxnHandler {
LockRequest req = new LockRequest(components, "me", "localhost");
LockResponse res = txnHandler.lock(req);
assertTrue(res.getState() == LockState.ACQUIRED);
- Thread.sleep(10);
+ Thread.sleep(1000);
txnHandler.performTimeOuts();
txnHandler.checkLock(new CheckLockRequest(res.getLockid()));
fail("Told there was a lock, when it should have timed out.");
@@ -1218,7 +1218,7 @@ public class TestTxnHandler {
long timeout = txnHandler.setTimeout(1);
try {
txnHandler.openTxns(new OpenTxnRequest(503, "me", "localhost"));
- Thread.sleep(10);
+ Thread.sleep(1000);
txnHandler.performTimeOuts();
GetOpenTxnsInfoResponse rsp = txnHandler.getOpenTxnsInfo();
int numAborted = 0;
@@ -1241,7 +1241,7 @@ public class TestTxnHandler {
request.setReplPolicy("default.*");
request.setReplSrcTxnIds(response.getTxn_ids());
OpenTxnsResponse responseRepl = txnHandler.openTxns(request);
- Thread.sleep(10);
+ Thread.sleep(1000);
txnHandler.performTimeOuts();
GetOpenTxnsInfoResponse rsp = txnHandler.getOpenTxnsInfo();
int numAborted = 0;
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java
index 5626dbe..49097a0 100755
--- a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java
@@ -38,9 +38,7 @@ import org.apache.hadoop.hive.metastore.PartitionDropOptions;
import org.apache.hadoop.hive.metastore.Warehouse;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.WMFullResourcePlan;
import org.apache.hadoop.hive.metastore.api.WMNullableResourcePlan;
import org.apache.hadoop.hive.metastore.api.WMPool;
import org.apache.hadoop.hive.metastore.api.WMResourcePlan;
@@ -67,20 +65,15 @@ import org.apache.logging.log4j.core.config.Configuration;
import org.apache.logging.log4j.core.config.LoggerConfig;
import org.apache.thrift.protocol.TBinaryProtocol;
import org.junit.Assert;
-import org.slf4j.LoggerFactory;
-
import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Lists;
-
-
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.fail;
-import org.junit.Before;
-import org.junit.After;
+import org.junit.BeforeClass;
+import org.junit.AfterClass;
import org.junit.Test;
/**
@@ -88,13 +81,13 @@ import org.junit.Test;
*
*/
public class TestHive {
- protected Hive hm;
- protected HiveConf hiveConf;
+ protected static Hive hm;
+ protected static HiveConf hiveConf;
- @Before
- public void setUp() throws Exception {
+ @BeforeClass
+ public static void setUp() throws Exception {
- hiveConf = new HiveConf(this.getClass());
+ hiveConf = new HiveConf(TestHive.class);
hm = setUpImpl(hiveConf);
}
@@ -116,8 +109,8 @@ public class TestHive {
}
}
- @After
- public void tearDown() throws Exception {
+ @AfterClass
+ public static void tearDown() throws Exception {
try {
// disable trash
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveRemote.java b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveRemote.java
index df3b2a5..5a695e9 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveRemote.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveRemote.java
@@ -23,9 +23,11 @@ import java.net.ServerSocket;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.MetaStoreTestUtils;
+import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.util.StringUtils;
-
+import org.junit.After;
import org.junit.Before;
+import org.junit.BeforeClass;
/**
*
@@ -38,15 +40,17 @@ public class TestHiveRemote extends TestHive {
/**
* Start a remote metastore and initialize a Hive object pointing at it.
*/
- @Before
- public void setUp() throws Exception {
- super.setUp();
- hiveConf = new HiveConf(this.getClass());
- hiveConf
- .setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER,
+ @BeforeClass
+ public static void setUp() throws Exception {
+ hiveConf = new HiveConf(TestHiveRemote.class);
+ hiveConf.setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER,
"org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory");
MetaStoreTestUtils.startMetaStoreWithRetry(hiveConf);
+ }
+ @Before
+ public void before() throws Exception {
+ SessionState.start(hiveConf);
try {
hm = Hive.get(hiveConf);
} catch (Exception e) {
@@ -58,6 +62,12 @@ public class TestHiveRemote extends TestHive {
}
}
+ @After
+ public void after() throws IOException {
+ SessionState.get().close();
+ hm.close(false);
+ }
+
/**
* Cannot control trash in remote metastore, so skip this test
*/
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestReplicationSemanticAnalyzer.java b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestReplicationSemanticAnalyzer.java
index e1c6eed..e52216c 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestReplicationSemanticAnalyzer.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestReplicationSemanticAnalyzer.java
@@ -31,15 +31,12 @@ import static org.junit.Assert.assertEquals;
@RunWith(Enclosed.class)
public class TestReplicationSemanticAnalyzer {
private static ParseDriver driver = new ParseDriver();
- private static HiveConf hiveConf;
+ private static HiveConf hiveConf = buildHiveConf();
- private TestReplicationSemanticAnalyzer() {
- }
-
- @BeforeClass
- public static void setUp() throws Exception {
- hiveConf = new HiveConf();
- hiveConf.setVar(HIVE_QUOTEDID_SUPPORT, Quotation.NONE.stringValue());
+ public static HiveConf buildHiveConf() {
+ HiveConf conf = new HiveConf();
+ conf.setVar(HIVE_QUOTEDID_SUPPORT, Quotation.NONE.stringValue());
+ return conf;
}
private static ASTNode parse(String command) throws Exception {
diff --git a/service/src/test/org/apache/hive/service/cli/session/TestSessionManagerMetrics.java b/service/src/test/org/apache/hive/service/cli/session/TestSessionManagerMetrics.java
index 2eb07ee..9658471 100644
--- a/service/src/test/org/apache/hive/service/cli/session/TestSessionManagerMetrics.java
+++ b/service/src/test/org/apache/hive/service/cli/session/TestSessionManagerMetrics.java
@@ -382,7 +382,7 @@ public class TestSessionManagerMetrics {
// We're going to wait for the session to be abandoned.
String currentValue;
- int count = 5; // how many times we'll sleep before giving up
+ int count = 10; // how many times we'll sleep before giving up
String expectedValue = "1";
do {
// HIVE_SERVER2_SESSION_CHECK_INTERVAL is set to 3 seconds, so we have to wait for at least
diff --git a/standalone-metastore/metastore-server/pom.xml b/standalone-metastore/metastore-server/pom.xml
index 021c25a..ea5f905 100644
--- a/standalone-metastore/metastore-server/pom.xml
+++ b/standalone-metastore/metastore-server/pom.xml
@@ -682,7 +682,7 @@
<configuration>
<sources>
<source>src/gen/thrift/gen-javabean</source>
- <source>${project.build.directory}/generated-sources</source>
+ <source>build/generated-sources</source>
<source>src/gen/version</source>
</sources>
</configuration>
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
index 915618c..57c006b 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
@@ -912,8 +912,10 @@ public class HiveMetaStore extends ThriftHiveMetastore {
} catch (JDOException e) {
LOG.warn("Retrying creating default database after error: " + e.getMessage(), e);
try {
- createDefaultDB_core(getMS());
- } catch (InvalidObjectException e1) {
+ RawStore ms = getMS();
+ createDefaultCatalog(ms, wh);
+ createDefaultDB_core(ms);
+ } catch (InvalidObjectException | InvalidOperationException e1) {
throw new MetaException(e1.getMessage());
}
} catch (InvalidObjectException|InvalidOperationException e) {
diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/MetaStoreTestUtils.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/MetaStoreTestUtils.java
index 2702e69..0e2c35a 100644
--- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/MetaStoreTestUtils.java
+++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/MetaStoreTestUtils.java
@@ -176,7 +176,7 @@ public class MetaStoreTestUtils {
String jdbcUrl = MetastoreConf.getVar(conf, ConfVars.CONNECT_URL_KEY);
if (!keepJdbcUri) {
// Setting metastore instance specific jdbc url postfixed with port
- jdbcUrl = "jdbc:derby:;databaseName=" + TMP_DIR + File.separator
+ jdbcUrl = "jdbc:derby:memory:" + TMP_DIR + File.separator
+ MetaStoreServerUtils.JUNIT_DATABASE_PREFIX + "_" + metaStorePort + ";create=true";
MetastoreConf.setVar(conf, ConfVars.CONNECT_URL_KEY, jdbcUrl);
}
diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestMarkPartition.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestMarkPartition.java
index 00fae25..811932f 100644
--- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestMarkPartition.java
+++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestMarkPartition.java
@@ -83,7 +83,7 @@ public class TestMarkPartition {
kvs.put("b", "'2011'");
msc.markPartitionForEvent(dbName, tableName, kvs, PartitionEventType.LOAD_DONE);
Assert.assertTrue(msc.isPartitionMarkedForEvent(dbName, tableName, kvs, PartitionEventType.LOAD_DONE));
- Thread.sleep(3000);
+ Thread.sleep(10000);
Assert.assertFalse(msc.isPartitionMarkedForEvent(dbName, tableName, kvs, PartitionEventType.LOAD_DONE));
kvs.put("b", "'2012'");
diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/MetaStoreClientTest.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/MetaStoreClientTest.java
index afe60b5..67fb749 100644
--- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/MetaStoreClientTest.java
+++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/MetaStoreClientTest.java
@@ -33,7 +33,6 @@ import org.slf4j.LoggerFactory;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.Set;
import java.util.stream.Collectors;
/**
@@ -47,7 +46,7 @@ public abstract class MetaStoreClientTest {
// Needed until there is no junit release with @BeforeParam, @AfterParam (junit 4.13)
// https://github.com/junit-team/junit4/commit/1bf8438b65858565dbb64736bfe13aae9cfc1b5a
// Then we should remove our own copy
- private static Set<AbstractMetaStoreService> metaStoreServices = null;
+ private static List<AbstractMetaStoreService> metaStoreServices = null;
@Rule
public TestRule ignoreRule;
@@ -57,7 +56,7 @@ public abstract class MetaStoreClientTest {
List<Object[]> result = MetaStoreFactoryForTests.getMetaStores();
metaStoreServices = result.stream()
.map(test -> (AbstractMetaStoreService)test[1])
- .collect(Collectors.toSet());
+ .collect(Collectors.toList());
return result;
}
@@ -78,9 +77,7 @@ public abstract class MetaStoreClientTest {
try {
metaStoreService.start(msConf, extraConf);
} catch(Exception e) {
- // Catch the exceptions, so every other metastore could be stopped as well
- // Log it, so at least there is a slight possibility we find out about this :)
- LOG.error("Error starting MetaStoreService", e);
+ throw new RuntimeException("Error starting MetaStoreService", e);
}
}
}
@@ -91,9 +88,7 @@ public abstract class MetaStoreClientTest {
try {
metaStoreService.stop();
} catch(Exception e) {
- // Catch the exceptions, so every other metastore could be stopped as well
- // Log it, so at least there is a slight possibility we find out about this :)
- LOG.error("Error stopping MetaStoreService", e);
+ throw new RuntimeException("Error stopping MetaStoreService", e);
}
}
}