You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by kg...@apache.org on 2021/11/15 08:55:06 UTC
[hive] branch master updated: HIVE-25681: Drop support for multi-threaded qtest execution via QTestRunnerUtils (#2769) (Stamatis Zampetakis reviewed by Zoltan Haindrich)
This is an automated email from the ASF dual-hosted git repository.
kgyrtkirk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 03a2aa8 HIVE-25681: Drop support for multi-threaded qtest execution via QTestRunnerUtils (#2769) (Stamatis Zampetakis reviewed by Zoltan Haindrich)
03a2aa8 is described below
commit 03a2aa85c0cc6dc47ae90db0694a8442b376d90c
Author: Stamatis Zampetakis <za...@gmail.com>
AuthorDate: Mon Nov 15 09:54:49 2021 +0100
HIVE-25681: Drop support for multi-threaded qtest execution via QTestRunnerUtils (#2769) (Stamatis Zampetakis reviewed by Zoltan Haindrich)
1. Remove QTestRunnerUtils#queryListRunnerMultiThreaded and related code
2. Remove (disabled) unit tests for this API (TestMTQueries).
The code is not being used by the current test infrastructure and this
is unlikely to change in the near future (especially after HIVE-22942).
Remove the code to facilitate code evolution and maintenance.
---
.../org/apache/hadoop/hive/ql/TestMTQueries.java | 63 -----------
.../apache/hadoop/hive/ql/QTestRunnerUtils.java | 115 +--------------------
2 files changed, 2 insertions(+), 176 deletions(-)
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestMTQueries.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestMTQueries.java
deleted file mode 100644
index f2c81fd..0000000
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestMTQueries.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql;
-
-import java.io.File;
-
-import org.junit.Ignore;
-import org.junit.Test;
-import static org.junit.Assert.fail;
-
-/**
- * Suite for testing running of queries in multi-threaded mode.
- */
-@Ignore("Ignore until HIVE-23138 is finished")
-public class TestMTQueries extends BaseTestQueries {
-
- public TestMTQueries() {
- File logDirFile = new File(logDir);
- if (!(logDirFile.exists() || logDirFile.mkdirs())) {
- fail("Could not create " + logDir);
- }
- }
-
- @Test
- public void testMTQueries1() throws Exception {
- String[] testNames = new String[] {"join2.q", "groupby1.q", "input1.q", "input19.q"};
-
- File[] qfiles = setupQFiles(testNames);
- QTestUtil[] qts = QTestRunnerUtils.queryListRunnerSetup(qfiles, resDir, logDir, "q_test_init_src_with_stats.sql",
- "q_test_cleanup_src_with_stats.sql");
- for (QTestUtil util : qts) {
- util.postInit();
- // derby fails creating multiple stats aggregator concurrently
- util.getConf().setBoolean("hive.exec.submitviachild", true);
- util.getConf().setBoolean("hive.exec.submit.local.task.via.child", true);
- util.getConf().setBoolean("hive.vectorized.execution.enabled", true);
- util.getConf().set("hive.stats.dbclass", "fs");
- util.getConf().set("hive.mapred.mode", "nonstrict");
- util.getConf().set("hive.stats.column.autogather", "false");
- util.newSession();
- }
- boolean success = QTestRunnerUtils.queryListRunnerMultiThreaded(qfiles, qts);
- if (!success) {
- fail("One or more queries failed");
- }
- }
-}
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestRunnerUtils.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestRunnerUtils.java
index ffd9952..6996ff7 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestRunnerUtils.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestRunnerUtils.java
@@ -21,66 +21,8 @@ package org.apache.hadoop.hive.ql;
import java.io.File;
import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.hive.ql.QTestMiniClusters.MiniClusterType;
public class QTestRunnerUtils {
- public static final String DEFAULT_INIT_SCRIPT = "q_test_init.sql";
- public static final String DEFAULT_CLEANUP_SCRIPT = "q_test_cleanup.sql";
-
- /**
- * QTRunner: Runnable class for running a single query file.
- **/
- public static class QTRunner implements Runnable {
- private final QTestUtil qt;
- private final File file;
-
- public QTRunner(QTestUtil qt, File file) {
- this.qt = qt;
- this.file = file;
- }
-
- @Override
- public void run() {
- try {
- qt.startSessionState(false);
- // assumption is that environment has already been cleaned once globally
- // hence each thread does not call cleanUp() and createSources() again
- qt.cliInit();
- qt.executeClient();
- } catch (Throwable e) {
- System.err
- .println("Query file " + file.getName() + " failed with exception " + e.getMessage());
- e.printStackTrace();
- outputTestFailureHelpMessage();
- }
- }
- }
-
- /**
- * Setup to execute a set of query files. Uses QTestUtil to do so.
- *
- * @param qfiles array of input query files containing arbitrary number of hive queries
- * @param resDir output directory
- * @param logDir log directory
- * @return one QTestUtil for each query file
- */
- public static QTestUtil[] queryListRunnerSetup(File[] qfiles, String resDir, String logDir,
- String initScript, String cleanupScript) throws Exception {
- QTestUtil[] qt = new QTestUtil[qfiles.length];
- for (int i = 0; i < qfiles.length; i++) {
-
- qt[i] = new QTestUtil(QTestArguments.QTestArgumentsBuilder.instance().withOutDir(resDir)
- .withLogDir(logDir).withClusterType(MiniClusterType.NONE).withConfDir(null)
- .withInitScript(initScript == null ? DEFAULT_INIT_SCRIPT : initScript)
- .withCleanupScript(cleanupScript == null ? DEFAULT_CLEANUP_SCRIPT : cleanupScript)
- .withLlapIo(false).build());
-
- qt[i].setInputFile(qfiles[i]);
- qt[i].clearTestSideEffects();
- }
-
- return qt;
- }
/**
* Executes a set of query files in sequence.
@@ -108,65 +50,12 @@ public class QTestRunnerUtils {
builder.append(" and diff value ").append(result.getCapturedOutput());
}
System.err.println(builder.toString());
- outputTestFailureHelpMessage();
+ System.err.println(QTestUtil.DEBUG_HINT);
+ System.err.flush();
}
qt[i].clearPostTestEffects();
}
return (!failed);
}
- /**
- * Executes a set of query files parallel.
- * <p>
- * Each query file is run in a separate thread. The caller has to arrange that different query
- * files do not collide (in terms of destination tables)
- *
- * @param qfiles array of input query files containing arbitrary number of hive queries
- * @param qt array of QTestUtils, one per qfile
- * @return true if all queries passed, false otw
- */
- public static boolean queryListRunnerMultiThreaded(File[] qfiles, QTestUtil[] qt)
- throws Exception {
- boolean failed = false;
-
- // in multithreaded mode - do cleanup/initialization just once
-
- qt[0].cleanUp();
- qt[0].createSources();
- qt[0].clearTestSideEffects();
-
- QTRunner[] qtRunners = new QTRunner[qfiles.length];
- Thread[] qtThread = new Thread[qfiles.length];
-
- for (int i = 0; i < qfiles.length; i++) {
- qtRunners[i] = new QTRunner(qt[i], qfiles[i]);
- qtThread[i] = new Thread(qtRunners[i]);
- }
-
- for (int i = 0; i < qfiles.length; i++) {
- qtThread[i].start();
- }
-
- for (int i = 0; i < qfiles.length; i++) {
- qtThread[i].join();
- QTestProcessExecResult result = qt[i].checkCliDriverResults();
- if (result.getReturnCode() != 0) {
- failed = true;
- StringBuilder builder = new StringBuilder();
- builder.append("Test ").append(qfiles[i].getName())
- .append(" results check failed with error code ").append(result.getReturnCode());
- if (StringUtils.isNotEmpty(result.getCapturedOutput())) {
- builder.append(" and diff value ").append(result.getCapturedOutput());
- }
- System.err.println(builder.toString());
- outputTestFailureHelpMessage();
- }
- }
- return (!failed);
- }
-
- public static void outputTestFailureHelpMessage() {
- System.err.println(QTestUtil.DEBUG_HINT);
- System.err.flush();
- }
}