You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2017/04/07 01:16:11 UTC
hive git commit: HIVE-15616 : Improve contents of qfile test output
Repository: hive
Updated Branches:
refs/heads/master bc248642a -> a01a6a348
HIVE-15616 : Improve contents of qfile test output
Signed-off-by: Ashutosh Chauhan <ha...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a01a6a34
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a01a6a34
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a01a6a34
Branch: refs/heads/master
Commit: a01a6a348825152c8f5495017e41016fc2ce4ac4
Parents: bc24864
Author: Barna Zsombor Klara <zs...@cloudera.com>
Authored: Fri Jan 13 06:44:00 2017 -0800
Committer: Ashutosh Chauhan <ha...@apache.org>
Committed: Thu Apr 6 18:15:14 2017 -0700
----------------------------------------------------------------------
.../hadoop/hive/ql/TestLocationQueries.java | 8 +-
.../control/AbstractCoreBlobstoreCliDriver.java | 11 +-
.../hive/cli/control/CoreAccumuloCliDriver.java | 10 +-
.../hadoop/hive/cli/control/CoreCliDriver.java | 12 +-
.../hive/cli/control/CoreCompareCliDriver.java | 12 +-
.../hive/cli/control/CoreHBaseCliDriver.java | 9 +-
.../cli/control/CoreHBaseNegativeCliDriver.java | 9 +-
.../hive/cli/control/CoreNegativeCliDriver.java | 12 +-
.../hive/cli/control/CorePerfCliDriver.java | 12 +-
.../hadoop/hive/ql/QTestProcessExecResult.java | 6 +-
.../org/apache/hadoop/hive/ql/QTestUtil.java | 146 +++++++++++--------
.../hadoop/hive/ql/parse/CoreParseNegative.java | 19 ++-
12 files changed, 157 insertions(+), 109 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/a01a6a34/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java
index 0688846..c17ca10 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java
@@ -53,10 +53,11 @@ public class TestLocationQueries extends BaseTestQueries {
* @return non-zero if it failed
*/
@Override
- public int checkCliDriverResults(String tname) throws Exception {
+ public QTestProcessExecResult checkCliDriverResults(String tname) throws Exception {
File logFile = new File(logDir, tname + ".out");
int failedCount = 0;
+ StringBuilder fileNames = new StringBuilder("Files failing the location check:");
FileReader fr = new FileReader(logFile);
BufferedReader in = new BufferedReader(fr);
try {
@@ -69,19 +70,20 @@ public class TestLocationQueries extends BaseTestQueries {
File f = new File(m.group(1));
if (!f.getName().equals(locationSubdir)) {
failedCount++;
+ fileNames.append(f.getName()).append("\r\n");
}
locationCount++;
}
}
// we always have to find at least one location, otw the test is useless
if (locationCount == 0) {
- return Integer.MAX_VALUE;
+ return QTestProcessExecResult.create(Integer.MAX_VALUE, "0 locations tested");
}
} finally {
in.close();
}
- return failedCount;
+ return QTestProcessExecResult.create(failedCount, fileNames.toString());
}
public CheckResults(String outDir, String logDir, MiniClusterType miniMr,
http://git-wip-us.apache.org/repos/asf/hive/blob/a01a6a34/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java
index 9c97c31..02abe53 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.hive.cli.control.AbstractCliConfig.MetastoreType;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveVariableSource;
import org.apache.hadoop.hive.conf.VariableSubstitution;
+import org.apache.hadoop.hive.ql.QTestProcessExecResult;
import org.apache.hadoop.hive.ql.QTestUtil;
import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType;
import org.junit.After;
@@ -139,12 +140,14 @@ public abstract class AbstractCoreBlobstoreCliDriver extends CliAdapter {
if ((ecode == 0) ^ expectSuccess) {
qt.failed(ecode, fname, debugHint);
}
- ecode = qt.checkCliDriverResults(fname);
- if (ecode != 0) {
- qt.failedDiff(ecode, fname, debugHint);
+ QTestProcessExecResult result = qt.checkCliDriverResults(fname);
+ if (result.getReturnCode() != 0) {
+ String message = Strings.isNullOrEmpty(result.getCapturedOutput()) ?
+ debugHint : "\r\n" + result.getCapturedOutput();
+ qt.failedDiff(result.getReturnCode(), fname, message);
}
}
- catch (Throwable e) {
+ catch (Exception e) {
qt.failed(e, fname, debugHint);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/a01a6a34/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreAccumuloCliDriver.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreAccumuloCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreAccumuloCliDriver.java
index 3e4b373..73e5632 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreAccumuloCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreAccumuloCliDriver.java
@@ -18,8 +18,10 @@
package org.apache.hadoop.hive.cli.control;
import static org.junit.Assert.assertTrue;
+
import org.apache.hadoop.hive.accumulo.AccumuloQTestUtil;
import org.apache.hadoop.hive.accumulo.AccumuloTestSetup;
+import org.apache.hadoop.hive.ql.QTestProcessExecResult;
import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType;
import org.junit.After;
import org.junit.AfterClass;
@@ -92,13 +94,13 @@ public class CoreAccumuloCliDriver extends CliAdapter {
qt.failed(ecode, fname, null);
}
- ecode = qt.checkCliDriverResults(fname);
- if (ecode != 0) {
- qt.failedDiff(ecode, fname, null);
+ QTestProcessExecResult result = qt.checkCliDriverResults(fname);
+ if (result.getReturnCode() != 0) {
+ qt.failedDiff(result.getReturnCode(), fname, result.getCapturedOutput());
}
qt.clearPostTestEffects();
- } catch (Throwable e) {
+ } catch (Exception e) {
qt.failed(e, fname, null);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/a01a6a34/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java
index a735346..d59b650 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java
@@ -23,7 +23,9 @@ import static org.junit.Assert.fail;
import java.util.concurrent.TimeUnit;
import com.google.common.base.Stopwatch;
+import com.google.common.base.Strings;
import org.apache.hadoop.hive.cli.control.AbstractCliConfig.MetastoreType;
+import org.apache.hadoop.hive.ql.QTestProcessExecResult;
import org.apache.hadoop.hive.ql.QTestUtil;
import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType;
import org.apache.hadoop.hive.util.ElapsedTimeLoggingWrapper;
@@ -175,13 +177,15 @@ public class CoreCliDriver extends CliAdapter {
failed = true;
qt.failed(ecode, fname, debugHint);
}
- ecode = qt.checkCliDriverResults(fname);
- if (ecode != 0) {
+ QTestProcessExecResult result = qt.checkCliDriverResults(fname);
+ if (result.getReturnCode() != 0) {
failed = true;
- qt.failedDiff(ecode, fname, debugHint);
+ String message = Strings.isNullOrEmpty(result.getCapturedOutput()) ?
+ debugHint : "\r\n" + result.getCapturedOutput();
+ qt.failedDiff(result.getReturnCode(), fname, message);
}
}
- catch (Throwable e) {
+ catch (Exception e) {
failed = true;
qt.failed(e, fname, debugHint);
} finally {
http://git-wip-us.apache.org/repos/asf/hive/blob/a01a6a34/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCompareCliDriver.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCompareCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCompareCliDriver.java
index 71a02bc..bff81dd 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCompareCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCompareCliDriver.java
@@ -25,6 +25,8 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import com.google.common.base.Strings;
+import org.apache.hadoop.hive.ql.QTestProcessExecResult;
import org.apache.hadoop.hive.ql.QTestUtil;
import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType;
import org.junit.After;
@@ -143,12 +145,14 @@ public class CoreCompareCliDriver extends CliAdapter{
}
}
- ecode = qt.checkCompareCliDriverResults(fname, outputs);
- if (ecode != 0) {
- qt.failedDiff(ecode, fname, debugHint);
+ QTestProcessExecResult result = qt.checkCompareCliDriverResults(fname, outputs);
+ if (result.getReturnCode() != 0) {
+ String message = Strings.isNullOrEmpty(result.getCapturedOutput()) ?
+ debugHint : "\r\n" + result.getCapturedOutput();
+ qt.failedDiff(result.getReturnCode(), fname, message);
}
}
- catch (Throwable e) {
+ catch (Exception e) {
qt.failed(e, fname, debugHint);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/a01a6a34/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseCliDriver.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseCliDriver.java
index 956a42d..aa0b071 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseCliDriver.java
@@ -23,6 +23,7 @@ import static org.junit.Assert.fail;
import org.apache.hadoop.hive.hbase.HBaseQTestUtil;
import org.apache.hadoop.hive.hbase.HBaseTestSetup;
+import org.apache.hadoop.hive.ql.QTestProcessExecResult;
import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType;
import org.junit.After;
import org.junit.AfterClass;
@@ -120,12 +121,12 @@ public class CoreHBaseCliDriver extends CliAdapter {
qt.failed(ecode, fname, null);
}
- ecode = qt.checkCliDriverResults(fname);
- if (ecode != 0) {
- qt.failedDiff(ecode, fname, null);
+ QTestProcessExecResult result = qt.checkCliDriverResults(fname);
+ if (result.getReturnCode() != 0) {
+ qt.failedDiff(result.getReturnCode(), fname, result.getCapturedOutput());
}
- } catch (Throwable e) {
+ } catch (Exception e) {
qt.failed(e, fname, null);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/a01a6a34/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseNegativeCliDriver.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseNegativeCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseNegativeCliDriver.java
index 6225180..8320a80 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseNegativeCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseNegativeCliDriver.java
@@ -23,6 +23,7 @@ import static org.junit.Assert.fail;
import org.apache.hadoop.hive.hbase.HBaseQTestUtil;
import org.apache.hadoop.hive.hbase.HBaseTestSetup;
+import org.apache.hadoop.hive.ql.QTestProcessExecResult;
import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType;
import org.junit.After;
import org.junit.AfterClass;
@@ -101,13 +102,13 @@ public class CoreHBaseNegativeCliDriver extends CliAdapter {
qt.failed(fname, null);
}
- ecode = qt.checkCliDriverResults(fname);
- if (ecode != 0) {
- qt.failedDiff(ecode, fname, null);
+ QTestProcessExecResult result = qt.checkCliDriverResults(fname);
+ if (result.getReturnCode() != 0) {
+ qt.failedDiff(result.getReturnCode(), fname, result.getCapturedOutput());
}
qt.clearPostTestEffects();
- } catch (Throwable e) {
+ } catch (Exception e) {
qt.failed(e, fname, null);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/a01a6a34/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java
index 65b2ce7..438a61e 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java
@@ -20,6 +20,8 @@ package org.apache.hadoop.hive.cli.control;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import com.google.common.base.Strings;
+import org.apache.hadoop.hive.ql.QTestProcessExecResult;
import org.apache.hadoop.hive.ql.QTestUtil;
import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType;
import org.junit.After;
@@ -123,12 +125,14 @@ public class CoreNegativeCliDriver extends CliAdapter{
qt.failed(fname, debugHint);
}
- ecode = qt.checkCliDriverResults(fname);
- if (ecode != 0) {
- qt.failedDiff(ecode, fname, debugHint);
+ QTestProcessExecResult result = qt.checkCliDriverResults(fname);
+ if (result.getReturnCode() != 0) {
+ String message = Strings.isNullOrEmpty(result.getCapturedOutput()) ?
+ debugHint : "\r\n" + result.getCapturedOutput();
+ qt.failedDiff(result.getReturnCode(), fname, message);
}
}
- catch (Throwable e) {
+ catch (Exception e) {
qt.failed(e, fname, debugHint);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/a01a6a34/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java
index 8620cde..26b9ce1 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java
@@ -22,6 +22,8 @@ package org.apache.hadoop.hive.cli.control;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import com.google.common.base.Strings;
+import org.apache.hadoop.hive.ql.QTestProcessExecResult;
import org.apache.hadoop.hive.ql.QTestUtil;
import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType;
import org.junit.After;
@@ -121,11 +123,13 @@ public class CorePerfCliDriver extends CliAdapter{
if (ecode != 0) {
qt.failed(ecode, fname, debugHint);
}
- ecode = qt.checkCliDriverResults(fname);
- if (ecode != 0) {
- qt.failedDiff(ecode, fname, debugHint);
+ QTestProcessExecResult result = qt.checkCliDriverResults(fname);
+ if (result.getReturnCode() != 0) {
+ String message = Strings.isNullOrEmpty(result.getCapturedOutput()) ?
+ debugHint : "\r\n" + result.getCapturedOutput();
+ qt.failedDiff(result.getReturnCode(), fname, message);
}
- } catch (Throwable e) {
+ } catch (Exception e) {
qt.failed(e, fname, debugHint);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/a01a6a34/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestProcessExecResult.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestProcessExecResult.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestProcessExecResult.java
index 6acc7a0..75cce14 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestProcessExecResult.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestProcessExecResult.java
@@ -18,12 +18,8 @@
package org.apache.hadoop.hive.ql;
-
-import com.google.common.base.Strings;
-import org.apache.commons.lang3.StringUtils;
-
/**
- * Standard output and return code of a process executed during the qtests
+ * Standard output and return code of a process executed during the qtests.
*/
public class QTestProcessExecResult {
http://git-wip-us.apache.org/repos/asf/hive/blob/a01a6a34/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
index 2abf252..c820dc7 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
@@ -39,6 +39,7 @@ import java.io.PrintStream;
import java.io.Serializable;
import java.io.StringWriter;
import java.net.URL;
+import java.nio.charset.StandardCharsets;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
@@ -68,9 +69,10 @@ import com.google.common.base.Throwables;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
+import org.apache.commons.io.output.ByteArrayOutputStream;
import org.apache.commons.lang.StringUtils;
-import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.commons.lang3.tuple.Pair;
+import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FileStatus;
@@ -1468,10 +1470,6 @@ public class QTestUtil {
// Create an instance of hive in order to create the tables
testWarehouse = conf.getVar(HiveConf.ConfVars.METASTOREWAREHOUSE);
db = Hive.get(conf);
- // Create dest4 to replace dest4_sequencefile
- LinkedList<String> cols = new LinkedList<String>();
- cols.add("key");
- cols.add("value");
// Move all data from dest4_sequencefile to dest4
drv
@@ -1482,7 +1480,7 @@ public class QTestUtil {
true, true);
}
- public int checkNegativeResults(String tname, Exception e) throws Exception {
+ public QTestProcessExecResult checkNegativeResults(String tname, Exception e) throws Exception {
String outFileExtension = getOutFileExtension(tname);
@@ -1505,16 +1503,17 @@ public class QTestUtil {
outfd.write(e.getMessage());
outfd.close();
- int exitVal = executeDiffCommand(outf.getPath(), expf, false,
+ QTestProcessExecResult result = executeDiffCommand(outf.getPath(), expf, false,
qSortSet.contains(qf.getName()));
- if (exitVal != 0 && overWrite) {
- exitVal = overwriteResults(outf.getPath(), expf);
+ if (overWrite) {
+ overwriteResults(outf.getPath(), expf);
+ return QTestProcessExecResult.createWithoutOutput(0);
}
- return exitVal;
+ return result;
}
- public int checkParseResults(String tname, ASTNode tree) throws Exception {
+ public QTestProcessExecResult checkParseResults(String tname, ASTNode tree) throws Exception {
if (tree != null) {
String outFileExtension = getOutFileExtension(tname);
@@ -1530,10 +1529,11 @@ public class QTestUtil {
outfd.write(tree.toStringTree());
outfd.close();
- int exitVal = executeDiffCommand(outf.getPath(), expf, false, false);
+ QTestProcessExecResult exitVal = executeDiffCommand(outf.getPath(), expf, false, false);
- if (exitVal != 0 && overWrite) {
- exitVal = overwriteResults(outf.getPath(), expf);
+ if (overWrite) {
+ overwriteResults(outf.getPath(), expf);
+ return QTestProcessExecResult.createWithoutOutput(0);
}
return exitVal;
@@ -1714,7 +1714,7 @@ public class QTestUtil {
patternsWithMaskComments.add(toPatternPair(patternStr, maskComment));
}
- public int checkCliDriverResults(String tname) throws Exception {
+ public QTestProcessExecResult checkCliDriverResults(String tname) throws Exception {
assert(qMap.containsKey(tname));
String outFileExtension = getOutFileExtension(tname);
@@ -1723,69 +1723,71 @@ public class QTestUtil {
File f = new File(logDir, tname + outFileExtension);
maskPatterns(planMask, f.getPath());
- int exitVal = executeDiffCommand(f.getPath(),
+ QTestProcessExecResult exitVal = executeDiffCommand(f.getPath(),
outFileName, false,
qSortSet.contains(tname));
- if (exitVal != 0 && overWrite) {
- exitVal = overwriteResults(f.getPath(), outFileName);
+ if (overWrite) {
+ overwriteResults(f.getPath(), outFileName);
+ return QTestProcessExecResult.createWithoutOutput(0);
}
return exitVal;
}
- public int checkCompareCliDriverResults(String tname, List<String> outputs) throws Exception {
+ public QTestProcessExecResult checkCompareCliDriverResults(String tname, List<String> outputs)
+ throws Exception {
assert outputs.size() > 1;
maskPatterns(planMask, outputs.get(0));
for (int i = 1; i < outputs.size(); ++i) {
maskPatterns(planMask, outputs.get(i));
- int ecode = executeDiffCommand(
+ QTestProcessExecResult result = executeDiffCommand(
outputs.get(i - 1), outputs.get(i), false, qSortSet.contains(tname));
- if (ecode != 0) {
+ if (result.getReturnCode() != 0) {
System.out.println("Files don't match: " + outputs.get(i - 1) + " and " + outputs.get(i));
- return ecode;
+ return result;
}
}
- return 0;
+ return QTestProcessExecResult.createWithoutOutput(0);
}
- private static int overwriteResults(String inFileName, String outFileName) throws Exception {
+ private static void overwriteResults(String inFileName, String outFileName) throws Exception {
// This method can be replaced with Files.copy(source, target, REPLACE_EXISTING)
// once Hive uses JAVA 7.
System.out.println("Overwriting results " + inFileName + " to " + outFileName);
- return executeCmd(new String[] {
+ int result = executeCmd(new String[]{
"cp",
getQuotedString(inFileName),
getQuotedString(outFileName)
- });
+ }).getReturnCode();
+ if (result != 0)
+ throw new IllegalStateException("Unexpected error while overwriting " +
+ inFileName + " with " + outFileName);
}
- private static int executeDiffCommand(String inFileName,
+ private static QTestProcessExecResult executeDiffCommand(String inFileName,
String outFileName,
boolean ignoreWhiteSpace,
boolean sortResults
) throws Exception {
- int result = 0;
+ QTestProcessExecResult result;
if (sortResults) {
// sort will try to open the output file in write mode on windows. We need to
// close it first.
SessionState ss = SessionState.get();
if (ss != null && ss.out != null && ss.out != System.out) {
- ss.out.close();
+ ss.out.close();
}
String inSorted = inFileName + SORT_SUFFIX;
String outSorted = outFileName + SORT_SUFFIX;
- result = sortFiles(inFileName, inSorted);
- result |= sortFiles(outFileName, outSorted);
- if (result != 0) {
- System.err.println("ERROR: Could not sort files before comparing");
- return result;
- }
+ sortFiles(inFileName, inSorted);
+ sortFiles(outFileName, outSorted);
+
inFileName = inSorted;
outFileName = outSorted;
}
@@ -1815,40 +1817,47 @@ public class QTestUtil {
return result;
}
- private static int sortFiles(String in, String out) throws Exception {
- return executeCmd(new String[] {
+ private static void sortFiles(String in, String out) throws Exception {
+ int result = executeCmd(new String[]{
"sort",
getQuotedString(in),
- }, out, null);
+ }, out, null).getReturnCode();
+ if (result != 0)
+ throw new IllegalStateException("Unexpected error while sorting " + in);
}
- private static int executeCmd(Collection<String> args) throws Exception {
+ private static QTestProcessExecResult executeCmd(Collection<String> args) throws Exception {
return executeCmd(args, null, null);
}
- private static int executeCmd(String[] args) throws Exception {
+ private static QTestProcessExecResult executeCmd(String[] args) throws Exception {
return executeCmd(args, null, null);
}
- private static int executeCmd(Collection<String> args, String outFile, String errFile) throws Exception {
+ private static QTestProcessExecResult executeCmd(Collection<String> args, String outFile,
+ String errFile) throws Exception {
String[] cmdArray = args.toArray(new String[args.size()]);
return executeCmd(cmdArray, outFile, errFile);
}
- private static int executeCmd(String[] args, String outFile, String errFile) throws Exception {
+ private static QTestProcessExecResult executeCmd(String[] args, String outFile,
+ String errFile) throws Exception {
System.out.println("Running: " + org.apache.commons.lang.StringUtils.join(args, ' '));
PrintStream out = outFile == null ?
SessionState.getConsole().getChildOutStream() :
- new PrintStream(new FileOutputStream(outFile), true);
+ new PrintStream(new FileOutputStream(outFile), true, "UTF-8");
PrintStream err = errFile == null ?
SessionState.getConsole().getChildErrStream() :
- new PrintStream(new FileOutputStream(errFile), true);
+ new PrintStream(new FileOutputStream(errFile), true, "UTF-8");
Process executor = Runtime.getRuntime().exec(args);
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ PrintStream str = new PrintStream(bos, true, "UTF-8");
+
StreamPrinter errPrinter = new StreamPrinter(executor.getErrorStream(), null, err);
- StreamPrinter outPrinter = new StreamPrinter(executor.getInputStream(), null, out);
+ StreamPrinter outPrinter = new StreamPrinter(executor.getInputStream(), null, out, str);
outPrinter.start();
errPrinter.start();
@@ -1866,7 +1875,8 @@ public class QTestUtil {
err.close();
}
- return result;
+ return QTestProcessExecResult.
+ create(result, new String(bos.toByteArray(), StandardCharsets.UTF_8));
}
private static String getQuotedString(String str){
@@ -2042,11 +2052,18 @@ public class QTestUtil {
qt[i].clearTestSideEffects();
qt[i].cliInit(qfiles[i].getName(), false);
qt[i].executeClient(qfiles[i].getName());
- int ecode = qt[i].checkCliDriverResults(qfiles[i].getName());
- if (ecode != 0) {
+ QTestProcessExecResult result = qt[i].checkCliDriverResults(qfiles[i].getName());
+ if (result.getReturnCode() != 0) {
failed = true;
- System.err.println("Test " + qfiles[i].getName()
- + " results check failed with error code " + ecode);
+ StringBuilder builder = new StringBuilder();
+ builder.append("Test ")
+ .append(qfiles[i].getName())
+ .append(" results check failed with error code ")
+ .append(result.getReturnCode());
+ if (Strings.isNotEmpty(result.getCapturedOutput())) {
+ builder.append(" and diff value ").append(result.getCapturedOutput());
+ }
+ System.err.println(builder.toString());
outputTestFailureHelpMessage();
}
qt[i].clearPostTestEffects();
@@ -2093,11 +2110,18 @@ public class QTestUtil {
for (int i = 0; i < qfiles.length; i++) {
qtThread[i].join();
- int ecode = qt[i].checkCliDriverResults(qfiles[i].getName());
- if (ecode != 0) {
+ QTestProcessExecResult result = qt[i].checkCliDriverResults(qfiles[i].getName());
+ if (result.getReturnCode() != 0) {
failed = true;
- System.err.println("Test " + qfiles[i].getName()
- + " results check failed with error code " + ecode);
+ StringBuilder builder = new StringBuilder();
+ builder.append("Test ")
+ .append(qfiles[i].getName())
+ .append(" results check failed with error code ")
+ .append(result.getReturnCode());
+ if (Strings.isNotEmpty(result.getCapturedOutput())) {
+ builder.append(" and diff value ").append(result.getCapturedOutput());
+ }
+ System.err.println(builder.toString());
outputTestFailureHelpMessage();
}
}
@@ -2195,16 +2219,15 @@ public class QTestUtil {
public void failedDiff(int ecode, String fname, String debugHint) {
String message =
- "Client Execution results failed with error code = " + ecode + " while executing fname=" +
+ "Client Execution succeeded but contained differences " +
+ "(error code = " + ecode + ") after executing " +
fname + (debugHint != null ? (" " + debugHint) : "");
LOG.error(message);
Assert.fail(message);
}
- public void failed(Throwable e, String fname, String debugHint) {
+ public void failed(Exception e, String fname, String debugHint) {
String command = SessionState.get() != null ? SessionState.get().getLastCommand() : null;
- System.err.println("Exception: " + e.getMessage());
- e.printStackTrace();
System.err.println("Failed query: " + fname);
System.err.flush();
Assert.fail("Unexpected exception " +
@@ -2244,9 +2267,6 @@ public class QTestUtil {
}
br.close();
} catch (Exception e) {
- System.err.println("Exception: " + e.getMessage());
- e.printStackTrace();
- System.err.flush();
Assert.fail("Unexpected exception " + org.apache.hadoop.util.StringUtils.stringifyException(e));
}
}
@@ -2270,7 +2290,9 @@ public class QTestUtil {
String mdbPath = AbstractCliConfig.HIVE_ROOT + "/data/files/tpcds-perf/metastore_export/";
// Setup the table column stats
- BufferedReader br = new BufferedReader(new FileReader(new File(AbstractCliConfig.HIVE_ROOT + "/metastore/scripts/upgrade/derby/022-HIVE-11107.derby.sql")));
+ BufferedReader br = new BufferedReader(
+ new FileReader(
+ new File(AbstractCliConfig.HIVE_ROOT + "/metastore/scripts/upgrade/derby/022-HIVE-11107.derby.sql")));
String command;
s.execute("DROP TABLE APP.TABLE_PARAMS");
http://git-wip-us.apache.org/repos/asf/hive/blob/a01a6a34/itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java
index 8dba0bb..31f69a3 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java
@@ -21,9 +21,12 @@ import static org.junit.Assert.fail;
import java.io.Serializable;
import java.util.List;
+
+import com.google.common.base.Strings;
import org.apache.hadoop.hive.cli.control.AbstractCliConfig;
import org.apache.hadoop.hive.cli.control.CliAdapter;
import org.apache.hadoop.hive.cli.control.CliConfigs;
+import org.apache.hadoop.hive.ql.QTestProcessExecResult;
import org.apache.hadoop.hive.ql.QTestUtil;
import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType;
import org.apache.hadoop.hive.ql.exec.Task;
@@ -106,18 +109,20 @@ public class CoreParseNegative extends CliAdapter{
fail("Unexpected success for query: " + fname + debugHint);
}
catch (ParseException pe) {
- int ecode = qt.checkNegativeResults(fname, pe);
- if (ecode != 0) {
- qt.failed(ecode, fname, debugHint);
+ QTestProcessExecResult result = qt.checkNegativeResults(fname, pe);
+ if (result.getReturnCode() != 0) {
+ qt.failed(result.getReturnCode(), fname, result.getCapturedOutput() + "\r\n" + debugHint);
}
}
catch (SemanticException se) {
- int ecode = qt.checkNegativeResults(fname, se);
- if (ecode != 0) {
- qt.failedDiff(ecode, fname, debugHint);
+ QTestProcessExecResult result = qt.checkNegativeResults(fname, se);
+ if (result.getReturnCode() != 0) {
+ String message = Strings.isNullOrEmpty(result.getCapturedOutput()) ?
+ debugHint : "\r\n" + result.getCapturedOutput();
+ qt.failedDiff(result.getReturnCode(), fname, message);
}
}
- catch (Throwable e) {
+ catch (Exception e) {
qt.failed(e, fname, debugHint);
}