You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by za...@apache.org on 2021/10/24 19:16:18 UTC
[hive] branch master updated: HIVE-25629: Drop support of multiple
qfiles in QTestUtil, output and result processors (Stamatis Zampetakis,
reviewed by Krisztian Kasa)
This is an automated email from the ASF dual-hosted git repository.
zabetak pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 8028907 HIVE-25629: Drop support of multiple qfiles in QTestUtil, output and result processors (Stamatis Zampetakis, reviewed by Krisztian Kasa)
8028907 is described below
commit 80289076599f1e73aa526d33172de2294754fe07
Author: Stamatis Zampetakis <za...@gmail.com>
AuthorDate: Wed Oct 20 11:14:30 2021 +0200
HIVE-25629: Drop support of multiple qfiles in QTestUtil, output and result processors (Stamatis Zampetakis, reviewed by Krisztian Kasa)
1. Refactor QTestUtil, QOutProcessor, QTestResultProcessor to handle
one file at a time.
In practice all the consumers of QTestUtil API so far need a single
file as input. Keeping every file in the map has certain drawbacks:
* the code becomes harder to follow;
* it is impossible to determine which test/file is actually executed at
the moment;
* memory is wasted by keeping all files encountered so far along with
their contents.
2. Remove "partial" argument from QTestUtil#addFile since it is always
false.
3. Remove useless parameter filename, and drop unused methods.
4. Rename shouldNotReuseSession to canReuseSession and adapt behavior;
excluding negation from method names makes the code more readable.
Closes #2735
---
.../apache/hadoop/hive/ql/TestLocationQueries.java | 5 +-
.../control/AbstractCoreBlobstoreCliDriver.java | 9 +-
.../hive/cli/control/CoreAccumuloCliDriver.java | 10 +--
.../hadoop/hive/cli/control/CoreCliDriver.java | 10 +--
.../hive/cli/control/CoreHBaseCliDriver.java | 10 +--
.../cli/control/CoreHBaseNegativeCliDriver.java | 10 +--
.../hadoop/hive/cli/control/CoreKuduCliDriver.java | 10 +--
.../cli/control/CoreKuduNegativeCliDriver.java | 10 +--
.../hive/cli/control/CoreNegativeCliDriver.java | 10 +--
.../hadoop/hive/cli/control/CorePerfCliDriver.java | 10 +--
.../apache/hadoop/hive/hbase/HBaseQTestUtil.java | 8 +-
.../org/apache/hadoop/hive/ql/QOutProcessor.java | 58 ++++++-------
.../hadoop/hive/ql/QTestResultProcessor.java | 87 ++++++++++----------
.../apache/hadoop/hive/ql/QTestRunnerUtils.java | 14 ++--
.../java/org/apache/hadoop/hive/ql/QTestUtil.java | 95 +++++++++-------------
.../hadoop/hive/ql/parse/CoreParseNegative.java | 8 +-
.../apache/hadoop/hive/ql/TestQOutProcessor.java | 2 +-
17 files changed, 165 insertions(+), 201 deletions(-)
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java
index 383e35e..66a8475 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java
@@ -56,7 +56,8 @@ public class TestLocationQueries extends BaseTestQueries {
* @return non-zero if it failed
*/
@Override
- public QTestProcessExecResult checkCliDriverResults(String tname) throws Exception {
+ public QTestProcessExecResult checkCliDriverResults() throws Exception {
+ String tname = getInputFile().getName();
File logFile = new File(logDir, tname + ".out");
int failedCount = 0;
@@ -124,7 +125,7 @@ public class TestLocationQueries extends BaseTestQueries {
qt[i] = new CheckResults(resDir, logDir, MiniClusterType.NONE, "parta");
qt[i].postInit();
qt[i].newSession();
- qt[i].addFile(qfiles[i], false);
+ qt[i].setInputFile(qfiles[i]);
qt[i].clearTestSideEffects();
}
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java
index 1cd90c4..aef0b07 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.hive.cli.control;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
-import java.io.File;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Map;
@@ -115,11 +114,11 @@ public abstract class AbstractCoreBlobstoreCliDriver extends CliAdapter {
try {
System.err.println("Begin query: " + fname);
- qt.addFile(fpath);
- qt.cliInit(new File(fpath));
+ qt.setInputFile(fpath);
+ qt.cliInit();
try {
- qt.executeClient(fname);
+ qt.executeClient();
if (!expectSuccess) {
qt.failedQuery(null, 0, fname, debugHint);
}
@@ -129,7 +128,7 @@ public abstract class AbstractCoreBlobstoreCliDriver extends CliAdapter {
}
}
- QTestProcessExecResult result = qt.checkCliDriverResults(fname);
+ QTestProcessExecResult result = qt.checkCliDriverResults();
if (result.getReturnCode() != 0) {
String message = Strings.isNullOrEmpty(result.getCapturedOutput()) ?
debugHint : "\r\n" + result.getCapturedOutput();
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreAccumuloCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreAccumuloCliDriver.java
index 8905ce7..1b634f2 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreAccumuloCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreAccumuloCliDriver.java
@@ -20,8 +20,6 @@ package org.apache.hadoop.hive.cli.control;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
-import java.io.File;
-
import org.apache.hadoop.hive.accumulo.AccumuloQTestUtil;
import org.apache.hadoop.hive.accumulo.AccumuloTestSetup;
import org.apache.hadoop.hive.ql.QTestProcessExecResult;
@@ -83,16 +81,16 @@ public class CoreAccumuloCliDriver extends CliAdapter {
try {
System.err.println("Begin query: " + fname);
- qt.addFile(fpath);
- qt.cliInit(new File(fpath));
+ qt.setInputFile(fpath);
+ qt.cliInit();
try {
- qt.executeClient(fname);
+ qt.executeClient();
} catch (CommandProcessorException e) {
qt.failedQuery(e.getCause(), e.getResponseCode(), fname, null);
}
- QTestProcessExecResult result = qt.checkCliDriverResults(fname);
+ QTestProcessExecResult result = qt.checkCliDriverResults();
if (result.getReturnCode() != 0) {
qt.failedDiff(result.getReturnCode(), fname, result.getCapturedOutput());
}
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java
index 9ded59b..c868ce2 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java
@@ -20,9 +20,7 @@ package org.apache.hadoop.hive.cli.control;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
-import java.io.File;
import java.util.Set;
-import java.util.LinkedHashSet;
import java.util.concurrent.TimeUnit;
import org.apache.hadoop.hive.conf.HiveConf;
@@ -137,18 +135,18 @@ public class CoreCliDriver extends CliAdapter {
LOG.info("Begin query: " + fname);
System.err.println("Begin query: " + fname);
- qt.addFile(fpath);
- qt.cliInit(new File(fpath));
+ qt.setInputFile(fpath);
+ qt.cliInit();
try {
- qt.executeClient(fname);
+ qt.executeClient();
} catch (CommandProcessorException e) {
failed = true;
qt.failedQuery(e.getCause(), e.getResponseCode(), fname, QTestUtil.DEBUG_HINT);
}
setupAdditionalPartialMasks();
- QTestProcessExecResult result = qt.checkCliDriverResults(fname);
+ QTestProcessExecResult result = qt.checkCliDriverResults();
resetAdditionalPartialMasks();
if (result.getReturnCode() != 0) {
failed = true;
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseCliDriver.java
index 9e153ea..db23405 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseCliDriver.java
@@ -20,8 +20,6 @@ package org.apache.hadoop.hive.cli.control;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
-import java.io.File;
-
import org.apache.hadoop.hive.hbase.HBaseQTestUtil;
import org.apache.hadoop.hive.hbase.HBaseTestSetup;
import org.apache.hadoop.hive.ql.QTestProcessExecResult;
@@ -81,17 +79,17 @@ public class CoreHBaseCliDriver extends CliAdapter {
try {
System.err.println("Begin query: " + fname);
- qt.addFile(fpath);
+ qt.setInputFile(fpath);
- qt.cliInit(new File(fpath));
+ qt.cliInit();
try {
- qt.executeClient(fname);
+ qt.executeClient();
} catch (CommandProcessorException e) {
qt.failedQuery(e.getCause(), e.getResponseCode(), fname, null);
}
- QTestProcessExecResult result = qt.checkCliDriverResults(fname);
+ QTestProcessExecResult result = qt.checkCliDriverResults();
if (result.getReturnCode() != 0) {
qt.failedDiff(result.getReturnCode(), fname, result.getCapturedOutput());
}
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseNegativeCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseNegativeCliDriver.java
index fac1766..0af6343 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseNegativeCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseNegativeCliDriver.java
@@ -21,8 +21,6 @@ package org.apache.hadoop.hive.cli.control;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
-import java.io.File;
-
import org.apache.hadoop.hive.hbase.HBaseQTestUtil;
import org.apache.hadoop.hive.hbase.HBaseTestSetup;
import org.apache.hadoop.hive.ql.QTestProcessExecResult;
@@ -80,16 +78,16 @@ public class CoreHBaseNegativeCliDriver extends CliAdapter {
long startTime = System.currentTimeMillis();
try {
System.err.println("Begin query: " + fname);
- qt.addFile(fpath);
- qt.cliInit(new File(fpath));
+ qt.setInputFile(fpath);
+ qt.cliInit();
try {
- qt.executeClient(fname);
+ qt.executeClient();
qt.failed(fname, null);
} catch (CommandProcessorException e) {
// this is the expected result
}
- QTestProcessExecResult result = qt.checkCliDriverResults(fname);
+ QTestProcessExecResult result = qt.checkCliDriverResults();
if (result.getReturnCode() != 0) {
qt.failedDiff(result.getReturnCode(), fname, result.getCapturedOutput());
}
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreKuduCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreKuduCliDriver.java
index 081d154..370a54d 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreKuduCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreKuduCliDriver.java
@@ -27,8 +27,6 @@ import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
-import java.io.File;
-
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -88,16 +86,16 @@ public class CoreKuduCliDriver extends CliAdapter {
try {
System.err.println("Begin query: " + fname);
- qt.addFile(fpath);
- qt.cliInit(new File(fpath));
+ qt.setInputFile(fpath);
+ qt.cliInit();
try {
- qt.executeClient(fname);
+ qt.executeClient();
} catch (CommandProcessorException e) {
qt.failedQuery(e.getCause(), e.getResponseCode(), fname, null);
}
- QTestProcessExecResult result = qt.checkCliDriverResults(fname);
+ QTestProcessExecResult result = qt.checkCliDriverResults();
if (result.getReturnCode() != 0) {
qt.failedDiff(result.getReturnCode(), fname, result.getCapturedOutput());
}
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreKuduNegativeCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreKuduNegativeCliDriver.java
index 92f4c64..18e565a 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreKuduNegativeCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreKuduNegativeCliDriver.java
@@ -27,8 +27,6 @@ import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
-import java.io.File;
-
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -87,16 +85,16 @@ public class CoreKuduNegativeCliDriver extends CliAdapter {
long startTime = System.currentTimeMillis();
try {
System.err.println("Begin query: " + fname);
- qt.addFile(fpath);
- qt.cliInit(new File(fpath));
+ qt.setInputFile(fpath);
+ qt.cliInit();
try {
- qt.executeClient(fname);
+ qt.executeClient();
qt.failed(fname, null);
} catch (CommandProcessorException e) {
// this is the expected behaviour
}
- QTestProcessExecResult result = qt.checkCliDriverResults(fname);
+ QTestProcessExecResult result = qt.checkCliDriverResults();
if (result.getReturnCode() != 0) {
qt.failedDiff(result.getReturnCode(), fname, result.getCapturedOutput());
}
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java
index 05c2a62..b52885a 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java
@@ -20,8 +20,6 @@ package org.apache.hadoop.hive.cli.control;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
-import java.io.File;
-
import org.apache.hadoop.hive.ql.QTestArguments;
import org.apache.hadoop.hive.ql.QTestProcessExecResult;
import org.apache.hadoop.hive.ql.QTestUtil;
@@ -98,17 +96,17 @@ public class CoreNegativeCliDriver extends CliAdapter{
try {
System.err.println("Begin query: " + fname);
- qt.addFile(fpath);
- qt.cliInit(new File(fpath));
+ qt.setInputFile(fpath);
+ qt.cliInit();
try {
- qt.executeClient(fname);
+ qt.executeClient();
qt.failed(fname, QTestUtil.DEBUG_HINT);
} catch (CommandProcessorException e) {
// this is the expected outcome
}
- QTestProcessExecResult result = qt.checkCliDriverResults(fname);
+ QTestProcessExecResult result = qt.checkCliDriverResults();
if (result.getReturnCode() != 0) {
String message = Strings.isNullOrEmpty(result.getCapturedOutput()) ? QTestUtil.DEBUG_HINT
: "\r\n" + result.getCapturedOutput();
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java
index 28a46a4..e45385d 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java
@@ -18,8 +18,6 @@
package org.apache.hadoop.hive.cli.control;
-import java.io.File;
-
import org.apache.hadoop.hive.ql.QTestArguments;
import org.apache.hadoop.hive.ql.QTestProcessExecResult;
import org.apache.hadoop.hive.ql.QTestUtil;
@@ -83,16 +81,16 @@ public class CorePerfCliDriver extends CliAdapter {
try {
LOG.info("Begin query: " + fname);
- qt.addFile(fpath);
- qt.cliInit(new File(fpath));
+ qt.setInputFile(fpath);
+ qt.cliInit();
try {
- qt.executeClient(fname);
+ qt.executeClient();
} catch (CommandProcessorException e) {
qt.failedQuery(e.getCause(), e.getResponseCode(), fname, QTestUtil.DEBUG_HINT);
}
- QTestProcessExecResult result = qt.checkCliDriverResults(fname);
+ QTestProcessExecResult result = qt.checkCliDriverResults();
if (result.getReturnCode() != 0) {
String message = Strings.isNullOrEmpty(result.getCapturedOutput()) ? QTestUtil.DEBUG_HINT :
"\r\n" + result.getCapturedOutput();
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java
index a327167..94b72f7 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java
@@ -59,8 +59,8 @@ public class HBaseQTestUtil extends QTestUtil {
}
@Override
- public void createSources(String tname) throws Exception {
- super.createSources(tname);
+ public void createSources() throws Exception {
+ super.createSources();
conf.setBoolean("hive.test.init.phase", true);
datasetHandler.initDataset(HBASE_SRC_NAME, getCliDriver());
@@ -79,8 +79,8 @@ public class HBaseQTestUtil extends QTestUtil {
}
@Override
- public void cleanUp(String tname) throws Exception {
- super.cleanUp(tname);
+ public void cleanUp() throws Exception {
+ super.cleanUp();
Admin admin = null;
try {
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QOutProcessor.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QOutProcessor.java
index 840276e..22aad8e 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QOutProcessor.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QOutProcessor.java
@@ -54,11 +54,6 @@ public class QOutProcessor {
public static final String MASK_PATTERN = "#### A masked pattern was here ####";
public static final String PARTIAL_MASK_PATTERN = "#### A PARTIAL masked pattern was here ####";
-
- private final Set<String> qMaskStatsQuerySet = new HashSet<String>();
- private final Set<String> qMaskDataSizeQuerySet = new HashSet<String>();
- private final Set<String> qMaskLineageQuerySet = new HashSet<String>();
-
private static final PatternReplacementPair MASK_STATS = new PatternReplacementPair(
Pattern.compile(" Num rows: [1-9][0-9]* Data size: [1-9][0-9]*"),
" Num rows: ###Masked### Data size: ###Masked###");
@@ -69,10 +64,6 @@ public class QOutProcessor {
Pattern.compile("POSTHOOK: Lineage: .*"),
"POSTHOOK: Lineage: ###Masked###");
- private static final Pattern PATTERN_MASK_STATS = Pattern.compile("-- MASK_STATS");
- private static final Pattern PATTERN_MASK_DATA_SIZE = Pattern.compile("-- MASK_DATA_SIZE");
- private static final Pattern PATTERN_MASK_LINEAGE = Pattern.compile("-- MASK_LINEAGE");
-
private FsType fsType = FsType.LOCAL;
public static class LineProcessingResult {
@@ -151,7 +142,21 @@ public class QOutProcessor {
{"Output:", "/data/files/"}
};
+ private enum Mask {
+ STATS("-- MASK_STATS"), DATASIZE("-- MASK_DATA_SIZE"), LINEAGE("-- MASK_LINEAGE");
+ private Pattern pattern;
+
+ Mask(String pattern) {
+ this.pattern = Pattern.compile(pattern);
+ }
+
+ boolean existsIn(String query) {
+ return pattern.matcher(query).find();
+ }
+ }
+
private final QTestReplaceHandler replaceHandler;
+ private final Set<Mask> queryMasks = new HashSet<>();
public QOutProcessor(FsType fsType, QTestReplaceHandler replaceHandler) {
this.fsType = fsType;
@@ -166,7 +171,7 @@ public class QOutProcessor {
return patterns;
}
- public void maskPatterns(String fname, String tname) throws Exception {
+ public void maskPatterns(String fname) throws Exception {
String line;
BufferedReader in;
@@ -182,7 +187,7 @@ public class QOutProcessor {
boolean lastWasMasked = false;
while (null != (line = in.readLine())) {
- LineProcessingResult result = processLine(line, tname);
+ LineProcessingResult result = processLine(line);
if (result.line.equals(MASK_PATTERN)) {
// We're folding multiple masked lines into one.
@@ -204,7 +209,7 @@ public class QOutProcessor {
out.close();
}
- public LineProcessingResult processLine(String line, String tname) {
+ LineProcessingResult processLine(String line) {
LineProcessingResult result = new LineProcessingResult(line);
Matcher matcher = null;
@@ -242,7 +247,7 @@ public class QOutProcessor {
}
}
- if (!result.partialMaskWasMatched && qMaskStatsQuerySet.contains(tname)) {
+ if (!result.partialMaskWasMatched && queryMasks.contains(Mask.STATS)) {
matcher = MASK_STATS.pattern.matcher(result.line);
if (matcher.find()) {
result.line = result.line.replaceAll(MASK_STATS.pattern.pattern(), MASK_STATS.replacement);
@@ -250,7 +255,7 @@ public class QOutProcessor {
}
}
- if (!result.partialMaskWasMatched && qMaskDataSizeQuerySet.contains(tname)) {
+ if (!result.partialMaskWasMatched && queryMasks.contains(Mask.DATASIZE)) {
matcher = MASK_DATA_SIZE.pattern.matcher(result.line);
if (matcher.find()) {
result.line = result.line.replaceAll(MASK_DATA_SIZE.pattern.pattern(), MASK_DATA_SIZE.replacement);
@@ -258,7 +263,7 @@ public class QOutProcessor {
}
}
- if (!result.partialMaskWasMatched && qMaskLineageQuerySet.contains(tname)) {
+ if (!result.partialMaskWasMatched && queryMasks.contains(Mask.LINEAGE)) {
matcher = MASK_LINEAGE.pattern.matcher(result.line);
if (matcher.find()) {
result.line = result.line.replaceAll(MASK_LINEAGE.pattern.pattern(), MASK_LINEAGE.replacement);
@@ -370,24 +375,13 @@ public class QOutProcessor {
patternsWithMaskComments.add(toPatternPair(patternStr, maskComment));
}
- public void initMasks(File qf, String query) {
- if (matches(PATTERN_MASK_STATS, query)) {
- qMaskStatsQuerySet.add(qf.getName());
- }
- if (matches(PATTERN_MASK_DATA_SIZE, query)) {
- qMaskDataSizeQuerySet.add(qf.getName());
- }
- if (matches(PATTERN_MASK_LINEAGE, query)) {
- qMaskLineageQuerySet.add(qf.getName());
- }
- }
-
- private boolean matches(Pattern pattern, String query) {
- Matcher matcher = pattern.matcher(query);
- if (matcher.find()) {
- return true;
+ public void initMasks(String query) {
+ queryMasks.clear();
+ for (Mask m : Mask.values()) {
+ if (m.existsIn(query)) {
+ queryMasks.add(m);
+ }
}
- return false;
}
public void resetPatternwithMaskComments() {
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestResultProcessor.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestResultProcessor.java
index 204a969..eb7615b 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestResultProcessor.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestResultProcessor.java
@@ -27,7 +27,6 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
import java.util.Set;
-import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.io.output.ByteArrayOutputStream;
@@ -46,71 +45,75 @@ import org.apache.hive.common.util.StreamPrinter;
*
*/
public class QTestResultProcessor {
- private static final Pattern SORT_BEFORE_DIFF = Pattern.compile("-- SORT_BEFORE_DIFF");
- private static final Pattern SORT_QUERY_RESULTS = Pattern.compile("-- SORT_QUERY_RESULTS");
- private static final Pattern HASH_QUERY_RESULTS = Pattern.compile("-- HASH_QUERY_RESULTS");
- private static final Pattern SORT_AND_HASH_QUERY_RESULTS =
- Pattern.compile("-- SORT_AND_HASH_QUERY_RESULTS");
- private static final Pattern NO_SESSION_REUSE = Pattern.compile("-- NO_SESSION_REUSE");
-
private static final String SORT_SUFFIX = ".sorted";
- private final Set<String> qSortSet = new HashSet<String>();
- private final Set<String> qSortQuerySet = new HashSet<String>();
- private final Set<String> qHashQuerySet = new HashSet<String>();
- private final Set<String> qSortNHashQuerySet = new HashSet<String>();
- private final Set<String> qNoSessionReuseQuerySet = new HashSet<String>();
-
- public void add(File qf, String query) {
- if (matches(SORT_BEFORE_DIFF, query)) {
- qSortSet.add(qf.getName());
- } else if (matches(SORT_QUERY_RESULTS, query)) {
- qSortQuerySet.add(qf.getName());
- } else if (matches(HASH_QUERY_RESULTS, query)) {
- qHashQuerySet.add(qf.getName());
- } else if (matches(SORT_AND_HASH_QUERY_RESULTS, query)) {
- qSortNHashQuerySet.add(qf.getName());
+ private enum Operation {
+ /***/
+ PRESORT("-- SORT_BEFORE_DIFF"),
+ /***/
+ SORT("-- SORT_QUERY_RESULTS"),
+ /***/
+ HASH("-- HASH_QUERY_RESULTS"),
+ /***/
+ SORT_N_HASH("-- SORT_AND_HASH_QUERY_RESULTS"),
+ /***/
+ NEW_SESSION("-- NO_SESSION_REUSE");
+ private final Pattern pattern;
+
+ Operation(String pattern) {
+ this.pattern = Pattern.compile(pattern);
}
- if (matches(NO_SESSION_REUSE, query)) {
- qNoSessionReuseQuerySet.add(qf.getName());
+ boolean existsIn(String query) {
+ return pattern.matcher(query).find();
}
}
- private boolean matches(Pattern pattern, String query) {
- Matcher matcher = pattern.matcher(query);
- if (matcher.find()) {
- return true;
+ /**
+ * Operations present in a given file/test.
+ */
+ private final Set<Operation> operations = new HashSet<>();
+
+ public void init(String query) {
+ operations.clear();
+ for (Operation op : Operation.values()) {
+ if (op.existsIn(query)) {
+ operations.add(op);
+ }
}
- return false;
}
- public boolean shouldSort(String fileName) {
- return qSortSet.contains(fileName);
+ private boolean shouldSort() {
+ return operations.contains(Operation.PRESORT);
}
- public void setOutputs(CliSessionState ss, OutputStream fo, String fileName) throws Exception {
- if (qSortQuerySet.contains(fileName)) {
+ public void setOutputs(CliSessionState ss, OutputStream fo) throws Exception {
+ // Normally, only one of PRESORT, SORT, HASH, SORT_N_HASH, should be present
+ // in a file. If there are multiple then the code will pick one in the order
+ // specified below. This ensures the behavior remains the same as before this
+ // refactoring.
+ // It would be better to throw an error than silently pick one and ignore the
+ // rest but it is out of the scope of the current change.
+ if (operations.contains(Operation.SORT)) {
ss.out = new SortPrintStream(fo, "UTF-8");
- } else if (qHashQuerySet.contains(fileName)) {
+ } else if (operations.contains(Operation.HASH)) {
ss.out = new DigestPrintStream(fo, "UTF-8");
- } else if (qSortNHashQuerySet.contains(fileName)) {
+ } else if (operations.contains(Operation.SORT_N_HASH)) {
ss.out = new SortAndDigestPrintStream(fo, "UTF-8");
} else {
ss.out = new SessionStream(fo, true, "UTF-8");
}
}
- public boolean shouldNotReuseSession(String fileName) {
- return qNoSessionReuseQuerySet.contains(fileName);
+ public boolean canReuseSession() {
+ return !operations.contains(Operation.NEW_SESSION);
}
- public QTestProcessExecResult executeDiffCommand(String inFileName, String outFileName,
- boolean ignoreWhiteSpace, String tname) throws Exception {
+ public QTestProcessExecResult executeDiffCommand(String inFileName, String outFileName, boolean ignoreWhiteSpace) throws Exception {
QTestProcessExecResult result;
- if (shouldSort(tname)) {
+ if (shouldSort()) {
String inSorted = inFileName + SORT_SUFFIX;
String outSorted = outFileName + SORT_SUFFIX;
@@ -137,7 +140,7 @@ public class QTestResultProcessor {
result = executeCmd(diffCommandArgs);
- if (shouldSort(tname)) {
+ if (shouldSort()) {
new File(inFileName).delete();
new File(outFileName).delete();
}
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestRunnerUtils.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestRunnerUtils.java
index 5fb138d..ffd9952 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestRunnerUtils.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestRunnerUtils.java
@@ -45,8 +45,8 @@ public class QTestRunnerUtils {
qt.startSessionState(false);
// assumption is that environment has already been cleaned once globally
// hence each thread does not call cleanUp() and createSources() again
- qt.cliInit(file);
- qt.executeClient(file.getName());
+ qt.cliInit();
+ qt.executeClient();
} catch (Throwable e) {
System.err
.println("Query file " + file.getName() + " failed with exception " + e.getMessage());
@@ -75,7 +75,7 @@ public class QTestRunnerUtils {
.withCleanupScript(cleanupScript == null ? DEFAULT_CLEANUP_SCRIPT : cleanupScript)
.withLlapIo(false).build());
- qt[i].addFile(qfiles[i], false);
+ qt[i].setInputFile(qfiles[i]);
qt[i].clearTestSideEffects();
}
@@ -96,9 +96,9 @@ public class QTestRunnerUtils {
qt[0].createSources();
for (int i = 0; i < qfiles.length && !failed; i++) {
qt[i].clearTestSideEffects();
- qt[i].cliInit(qfiles[i]);
- qt[i].executeClient(qfiles[i].getName());
- QTestProcessExecResult result = qt[i].checkCliDriverResults(qfiles[i].getName());
+ qt[i].cliInit();
+ qt[i].executeClient();
+ QTestProcessExecResult result = qt[i].checkCliDriverResults();
if (result.getReturnCode() != 0) {
failed = true;
StringBuilder builder = new StringBuilder();
@@ -149,7 +149,7 @@ public class QTestRunnerUtils {
for (int i = 0; i < qfiles.length; i++) {
qtThread[i].join();
- QTestProcessExecResult result = qt[i].checkCliDriverResults(qfiles[i].getName());
+ QTestProcessExecResult result = qt[i].checkCliDriverResults();
if (result.getReturnCode() != 0) {
failed = true;
StringBuilder builder = new StringBuilder();
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
index f86037f..a9e6b05 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
@@ -36,8 +36,8 @@ import java.util.Deque;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
+import java.util.Objects;
import java.util.Set;
-import java.util.TreeMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -55,7 +55,6 @@ import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.metastore.HiveMetaStoreClientWithLocalCache;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
-import org.apache.hadoop.hive.ql.QTestExternalDB;
import org.apache.hadoop.hive.ql.QTestMiniClusters.FsType;
import org.apache.hadoop.hive.ql.cache.results.QueryResultsCache;
import org.apache.hadoop.hive.ql.dataset.QTestDatasetHandler;
@@ -114,7 +113,8 @@ public class QTestUtil {
@Deprecated private final String testFiles;
private final String outDir;
protected final String logDir;
- private final TreeMap<String, String> qMap = new TreeMap<String, String>();
+ private File inputFile;
+ private String inputContent;
private final Set<String> srcUDFs;
private final FsType fsType;
private ParseDriver pd;
@@ -297,19 +297,20 @@ public class QTestUtil {
Hive.closeCurrent();
}
- public void addFile(String queryFile) throws IOException {
- addFile(new File(queryFile), false);
+ public void setInputFile(String queryFile) throws IOException {
+ setInputFile(new File(queryFile));
}
- public void addFile(File qf, boolean partial) throws IOException {
+ public void setInputFile(File qf) throws IOException {
String query = FileUtils.readFileToString(qf);
- qMap.put(qf.getName(), query);
- if (partial) {
- return;
- }
+ inputFile = qf;
+ inputContent = query;
+ qTestResultProcessor.init(query);
+ qOutProcessor.initMasks(query);
+ }
- qTestResultProcessor.add(qf, query);
- qOutProcessor.initMasks(qf, query);
+ public final File getInputFile() {
+ return inputFile;
}
/**
@@ -454,7 +455,7 @@ public class QTestUtil {
cliDriver = new CliDriver();
File outf = new File(logDir, "initialize.log");
- setSessionOutputs("that_shouldnt_happen_there", ss, outf);
+ setSessionOutputs(ss, outf);
}
@@ -484,13 +485,8 @@ public class QTestUtil {
}
public void cleanUp() throws Exception {
- cleanUp(null);
- }
-
- public void cleanUp(String fileName) throws Exception {
- boolean canReuseSession = (fileName == null) || !qTestResultProcessor.shouldNotReuseSession(fileName);
if (!isSessionStateStarted) {
- startSessionState(canReuseSession);
+ startSessionState(qTestResultProcessor.canReuseSession());
}
if (System.getenv(QTEST_LEAVE_FILES) != null) {
return;
@@ -562,13 +558,8 @@ public class QTestUtil {
}
public void createSources() throws Exception {
- createSources(null);
- }
-
- public void createSources(String fileName) throws Exception {
- boolean canReuseSession = (fileName == null) || !qTestResultProcessor.shouldNotReuseSession(fileName);
if (!isSessionStateStarted) {
- startSessionState(canReuseSession);
+ startSessionState(qTestResultProcessor.canReuseSession());
}
// connect to externalDB if size is not zero
@@ -658,26 +649,20 @@ public class QTestUtil {
conf.set("hive.execution.engine", execEngine);
}
- public void init(String fileName) throws Exception {
- cleanUp(fileName);
- createSources(fileName);
- cliDriver.processCmd("set hive.cli.print.header=true;");
- }
-
- public String cliInit(File file) throws Exception {
- String fileName = file.getName();
+ public String cliInit() throws Exception {
+ File file = Objects.requireNonNull(inputFile);
+ String fileName = inputFile.getName();
dispatcher.process(file);
dispatcher.beforeTest(this);
-
- if (qTestResultProcessor.shouldNotReuseSession(fileName)) {
+ if (!qTestResultProcessor.canReuseSession()) {
newSession(false);
}
CliSessionState ss = (CliSessionState) SessionState.get();
- String outFileExtension = getOutFileExtension(fileName);
+ String outFileExtension = getOutFileExtension();
String stdoutName = null;
if (outDir != null) {
@@ -688,7 +673,7 @@ public class QTestUtil {
stdoutName = fileName + outFileExtension;
}
File outf = new File(logDir, stdoutName);
- setSessionOutputs(fileName, ss, outf);
+ setSessionOutputs(ss, outf);
ss.setIsQtestLogging(true);
if (fileName.equals("init_file.q")) {
@@ -699,7 +684,7 @@ public class QTestUtil {
return outf.getAbsolutePath();
}
- private void setSessionOutputs(String fileName, CliSessionState ss, File outf) throws Exception {
+ private void setSessionOutputs(CliSessionState ss, File outf) throws Exception {
OutputStream fo = new BufferedOutputStream(new FileOutputStream(outf));
if (ss.out != null) {
ss.out.flush();
@@ -708,7 +693,7 @@ public class QTestUtil {
ss.err.flush();
}
- qTestResultProcessor.setOutputs(ss, fo, fileName);
+ qTestResultProcessor.setOutputs(ss, fo);
ss.err = new CachingPrintStream(fo, true, "UTF-8");
ss.setIsSilent(true);
@@ -764,8 +749,8 @@ public class QTestUtil {
return 0;
}
- public CommandProcessorResponse executeClient(String fileName) throws CommandProcessorException {
- return executeClientInternal(getCommand(fileName));
+ public CommandProcessorResponse executeClient() throws CommandProcessorException {
+ return executeClientInternal(getCommand());
}
private CommandProcessorResponse executeClientInternal(String commands) throws CommandProcessorException {
@@ -895,8 +880,8 @@ public class QTestUtil {
return testCommand != null;
}
- private String getCommand(String fileName) {
- String commands = qMap.get(fileName);
+ private String getCommand() {
+ String commands = inputContent;
StringBuilder newCommands = new StringBuilder(commands.length());
int lastMatchEnd = 0;
Matcher commentMatcher = Pattern.compile("^--.*$", Pattern.MULTILINE).matcher(commands);
@@ -910,13 +895,13 @@ public class QTestUtil {
return commands;
}
- private String getOutFileExtension(String fname) {
+ private String getOutFileExtension() {
return ".out";
}
public QTestProcessExecResult checkNegativeResults(String tname, Exception e) throws Exception {
- String outFileExtension = getOutFileExtension(tname);
+ String outFileExtension = getOutFileExtension();
File qf = new File(outDir, tname);
String expf = outPath(outDir.toString(), tname.concat(outFileExtension));
@@ -938,7 +923,7 @@ public class QTestUtil {
outfd.write(e.getMessage());
outfd.close();
- QTestProcessExecResult result = qTestResultProcessor.executeDiffCommand(outf.getPath(), expf, false, qf.getName());
+ QTestProcessExecResult result = qTestResultProcessor.executeDiffCommand(outf.getPath(), expf, false);
if (QTestSystemProperties.shouldOverwriteResults()) {
qTestResultProcessor.overwriteResults(outf.getPath(), expf);
return QTestProcessExecResult.createWithoutOutput(0);
@@ -949,7 +934,7 @@ public class QTestUtil {
public QTestProcessExecResult checkNegativeResults(String tname, Error e) throws Exception {
- String outFileExtension = getOutFileExtension(tname);
+ String outFileExtension = getOutFileExtension();
File qf = new File(outDir, tname);
String expf = outPath(outDir.toString(), tname.concat(outFileExtension));
@@ -969,7 +954,7 @@ public class QTestUtil {
+ "\n");
outfd.close();
- QTestProcessExecResult result = qTestResultProcessor.executeDiffCommand(outf.getPath(), expf, false, qf.getName());
+ QTestProcessExecResult result = qTestResultProcessor.executeDiffCommand(outf.getPath(), expf, false);
if (QTestSystemProperties.shouldOverwriteResults()) {
qTestResultProcessor.overwriteResults(outf.getPath(), expf);
return QTestProcessExecResult.createWithoutOutput(0);
@@ -1014,25 +999,25 @@ public class QTestUtil {
return ret;
}
- public QTestProcessExecResult checkCliDriverResults(String tname) throws Exception {
- assert (qMap.containsKey(tname));
+ public QTestProcessExecResult checkCliDriverResults() throws Exception {
+ String tname = inputFile.getName();
- String outFileExtension = getOutFileExtension(tname);
+ String outFileExtension = getOutFileExtension();
String outFileName = outPath(outDir, tname + outFileExtension);
File f = new File(logDir, tname + outFileExtension);
- qOutProcessor.maskPatterns(f.getPath(), tname);
+ qOutProcessor.maskPatterns(f.getPath());
if (QTestSystemProperties.shouldOverwriteResults()) {
qTestResultProcessor.overwriteResults(f.getPath(), outFileName);
return QTestProcessExecResult.createWithoutOutput(0);
} else {
- return qTestResultProcessor.executeDiffCommand(f.getPath(), outFileName, false, tname);
+ return qTestResultProcessor.executeDiffCommand(f.getPath(), outFileName, false);
}
}
- public ASTNode parseQuery(String tname) throws Exception {
- return pd.parse(qMap.get(tname)).getTree();
+ public ASTNode parseQuery() throws Exception {
+ return pd.parse(inputContent).getTree();
}
public List<Task<?>> analyzeAST(ASTNode ast) throws Exception {
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java
index 03061b5..d8d35bf 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java
@@ -20,8 +20,6 @@ package org.apache.hadoop.hive.ql.parse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
-import java.io.File;
-
import org.apache.hadoop.hive.cli.control.AbstractCliConfig;
import org.apache.hadoop.hive.cli.control.CliAdapter;
import org.apache.hadoop.hive.cli.control.CliConfigs;
@@ -95,10 +93,10 @@ public class CoreParseNegative extends CliAdapter{
try {
System.err.println("Begin query: " + fname);
- qt.addFile(fpath);
- qt.cliInit(new File(fpath));
+ qt.setInputFile(fpath);
+ qt.cliInit();
- ASTNode tree = qt.parseQuery(fname);
+ ASTNode tree = qt.parseQuery();
qt.analyzeAST(tree);
fail("Unexpected success for query: " + fname + QTestUtil.DEBUG_HINT);
} catch (ParseException pe) {
diff --git a/itests/util/src/test/java/org/apache/hadoop/hive/ql/TestQOutProcessor.java b/itests/util/src/test/java/org/apache/hadoop/hive/ql/TestQOutProcessor.java
index 70d9ef5..00c4203 100644
--- a/itests/util/src/test/java/org/apache/hadoop/hive/ql/TestQOutProcessor.java
+++ b/itests/util/src/test/java/org/apache/hadoop/hive/ql/TestQOutProcessor.java
@@ -69,6 +69,6 @@ public class TestQOutProcessor {
}
private String processLine(String line) {
- return qOutProcessor.processLine(line, null).get();
+ return qOutProcessor.processLine(line).get();
}
}
\ No newline at end of file