You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by kh...@apache.org on 2015/05/01 10:47:09 UTC
hive git commit: HIVE-10514 : Fix MiniCliDriver tests failure (Hari
Subramaniyan, reviewed by Szehon Ho, Sushanth Sowmyan)
Repository: hive
Updated Branches:
refs/heads/master aa1321aaa -> 283466374
HIVE-10514 : Fix MiniCliDriver tests failure (Hari Subramaniyan, reviewed by Szehon Ho, Sushanth Sowmyan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/28346637
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/28346637
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/28346637
Branch: refs/heads/master
Commit: 2834663742b42d2bb82e30714a3b96d1faaa3ff5
Parents: aa1321a
Author: Sushanth Sowmyan <kh...@gmail.com>
Authored: Fri May 1 01:46:25 2015 -0700
Committer: Sushanth Sowmyan <kh...@gmail.com>
Committed: Fri May 1 01:46:25 2015 -0700
----------------------------------------------------------------------
.../src/test/templates/TestAccumuloCliDriver.vm | 19 +++++----
.../apache/hadoop/hive/ant/QTestGenTask.java | 25 ++++++++++++
.../src/test/templates/TestHBaseCliDriver.vm | 18 ++++----
.../templates/TestHBaseNegativeCliDriver.vm | 19 +++++----
.../org/apache/hadoop/hive/ql/QTestUtil.java | 43 ++++++++++++++++++++
ql/src/test/templates/TestCliDriver.vm | 18 ++++----
ql/src/test/templates/TestCompareCliDriver.vm | 21 +++++-----
ql/src/test/templates/TestNegativeCliDriver.vm | 18 ++++----
ql/src/test/templates/TestParseNegative.vm | 17 ++++----
9 files changed, 139 insertions(+), 59 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/28346637/accumulo-handler/src/test/templates/TestAccumuloCliDriver.vm
----------------------------------------------------------------------
diff --git a/accumulo-handler/src/test/templates/TestAccumuloCliDriver.vm b/accumulo-handler/src/test/templates/TestAccumuloCliDriver.vm
index dd1ee64..9872291 100644
--- a/accumulo-handler/src/test/templates/TestAccumuloCliDriver.vm
+++ b/accumulo-handler/src/test/templates/TestAccumuloCliDriver.vm
@@ -26,6 +26,7 @@ import java.util.*;
import org.apache.hadoop.hive.accumulo.AccumuloQTestUtil;
import org.apache.hadoop.hive.accumulo.AccumuloTestSetup;
+import org.apache.hadoop.hive.ql.QTestUtil;
import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType;
import org.apache.hadoop.hive.ql.session.SessionState;
@@ -35,6 +36,12 @@ public class $className extends TestCase {
private AccumuloQTestUtil qt;
private AccumuloTestSetup setup;
+ public static class TestAccumuloCliDriverAddTestFromQFiles implements QTestUtil.SuiteAddTestFunctor {
+ public void addTestToSuite(TestSuite suite, Object setup, String tName) {
+ suite.addTest(new $className("testCliDriver_"+tName, (AccumuloTestSetup)setup));
+ }
+ }
+
public $className(String name, AccumuloTestSetup setup) {
super(name);
qt = null;
@@ -83,16 +90,12 @@ public class $className extends TestCase {
}
}
}
+
TestSuite suite = new TestSuite();
AccumuloTestSetup setup = new AccumuloTestSetup(suite);
-#foreach ($qf in $qfiles)
- #set ($fname = $qf.getName())
- #set ($eidx = $fname.indexOf('.'))
- #set ($tname = $fname.substring(0, $eidx))
- if(qFilesToExecute.isEmpty() || qFilesToExecute.contains("$fname")) {
- suite.addTest(new $className("testCliDriver_$tname", setup));
- }
-#end
+
+ QTestUtil.addTestsToSuiteFromQfileNames("$qFileNamesFile", qFilesToExecute,
+ suite, setup, new TestAccumuloCliDriverAddTestFromQFiles());
return setup;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/28346637/ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java
----------------------------------------------------------------------
diff --git a/ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java b/ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java
index ca959c0..60c7764 100644
--- a/ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java
+++ b/ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.hive.ant;
+import java.io.BufferedWriter;
import java.io.File;
import java.io.FileFilter;
import java.io.FileWriter;
@@ -453,6 +454,29 @@ public class QTestGenTask extends Task {
hadoopVersion = "";
}
+ File qFileNames = new File(outputDirectory, className + "QFileNames.txt");
+ String qFileNamesFile = qFileNames.getCanonicalPath();
+
+ if (qFileNames.exists()) {
+ if (!qFileNames.delete()) {
+ throw new Exception("Could not delete old query file names containing file " +
+ qFileNamesFile);
+ }
+ }
+ if (!qFileNames.createNewFile()) {
+ throw new Exception("Could not create query file names containing file " +
+ qFileNamesFile);
+ }
+
+ FileWriter fw = new FileWriter(qFileNames.getCanonicalFile());
+ BufferedWriter bw = new BufferedWriter(fw);
+
+ for (File qFile: qFiles) {
+ bw.write(qFile.getName());
+ bw.newLine();
+ }
+ bw.close();
+
// For each of the qFiles generate the test
System.out.println("hiveRootDir = " + hiveRootDir);
VelocityContext ctx = new VelocityContext();
@@ -464,6 +488,7 @@ public class QTestGenTask extends Task {
System.out.println("queryDir = " + strQueryDir);
ctx.put("queryDir", strQueryDir);
ctx.put("qfiles", qFiles);
+ ctx.put("qFileNamesFile", qFileNamesFile);
ctx.put("qfilesMap", qFilesMap);
if (resultsDir != null) {
ctx.put("resultsDir", relativePath(hiveRootDir, resultsDir));
http://git-wip-us.apache.org/repos/asf/hive/blob/28346637/hbase-handler/src/test/templates/TestHBaseCliDriver.vm
----------------------------------------------------------------------
diff --git a/hbase-handler/src/test/templates/TestHBaseCliDriver.vm b/hbase-handler/src/test/templates/TestHBaseCliDriver.vm
index 69fad97..de0be32 100644
--- a/hbase-handler/src/test/templates/TestHBaseCliDriver.vm
+++ b/hbase-handler/src/test/templates/TestHBaseCliDriver.vm
@@ -24,6 +24,7 @@ import junit.framework.TestSuite;
import java.io.*;
import java.util.*;
+import org.apache.hadoop.hive.ql.QTestUtil;
import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType;
import org.apache.hadoop.hive.hbase.HBaseQTestUtil;
import org.apache.hadoop.hive.hbase.HBaseTestSetup;
@@ -34,6 +35,12 @@ public class $className extends TestCase {
private HBaseQTestUtil qt;
private HBaseTestSetup setup;
+ public static class TestHBaseCliDriverAddTestFromQFiles implements QTestUtil.SuiteAddTestFunctor {
+ public void addTestToSuite(TestSuite suite, Object setup, String tName) {
+ suite.addTest(new $className("testCliDriver_"+tName, (HBaseTestSetup)setup));
+ }
+ }
+
public $className(String name, HBaseTestSetup setup) {
super(name);
qt = null;
@@ -84,14 +91,9 @@ public class $className extends TestCase {
}
TestSuite suite = new TestSuite();
HBaseTestSetup setup = new HBaseTestSetup(suite);
-#foreach ($qf in $qfiles)
- #set ($fname = $qf.getName())
- #set ($eidx = $fname.indexOf('.'))
- #set ($tname = $fname.substring(0, $eidx))
- if(qFilesToExecute.isEmpty() || qFilesToExecute.contains("$fname")) {
- suite.addTest(new $className("testCliDriver_$tname", setup));
- }
-#end
+
+ QTestUtil.addTestsToSuiteFromQfileNames("$qFileNamesFile", qFilesToExecute,
+ suite, setup, new TestHBaseCliDriverAddTestFromQFiles());
return setup;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/28346637/hbase-handler/src/test/templates/TestHBaseNegativeCliDriver.vm
----------------------------------------------------------------------
diff --git a/hbase-handler/src/test/templates/TestHBaseNegativeCliDriver.vm b/hbase-handler/src/test/templates/TestHBaseNegativeCliDriver.vm
index 82f0584..b402585 100644
--- a/hbase-handler/src/test/templates/TestHBaseNegativeCliDriver.vm
+++ b/hbase-handler/src/test/templates/TestHBaseNegativeCliDriver.vm
@@ -25,6 +25,7 @@ import junit.framework.TestSuite;
import java.io.*;
import java.util.*;
+import org.apache.hadoop.hive.ql.QTestUtil;
import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType;
import org.apache.hadoop.hive.hbase.HBaseQTestUtil;
import org.apache.hadoop.hive.hbase.HBaseTestSetup;
@@ -35,6 +36,12 @@ public class $className extends TestCase {
private HBaseQTestUtil qt;
private HBaseTestSetup setup;
+ public static class TestHBaseNegativeCliDriverAddTestFromQFiles implements QTestUtil.SuiteAddTestFunctor {
+ public void addTestToSuite(TestSuite suite, Object setup, String tName) {
+ suite.addTest(new $className("testCliDriver_"+tName, (HBaseTestSetup)setup));
+ }
+ }
+
public $className(String name, HBaseTestSetup setup) {
super(name);
qt = null;
@@ -83,16 +90,12 @@ public class $className extends TestCase {
}
}
}
+
TestSuite suite = new TestSuite();
HBaseTestSetup setup = new HBaseTestSetup(suite);
-#foreach ($qf in $qfiles)
- #set ($fname = $qf.getName())
- #set ($eidx = $fname.indexOf('.'))
- #set ($tname = $fname.substring(0, $eidx))
- if(qFilesToExecute.isEmpty() || qFilesToExecute.contains("$fname")) {
- suite.addTest(new $className("testCliDriver_$tname", setup));
- }
-#end
+
+ QTestUtil.addTestsToSuiteFromQfileNames("$qFileNamesFile", qFilesToExecute,
+ suite, setup, new TestHBaseNegativeCliDriverAddTestFromQFiles());
return setup;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/28346637/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
index f6081f4..d1104b3 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
@@ -55,6 +55,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import junit.framework.Assert;
+import junit.framework.TestSuite;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.io.FileUtils;
@@ -158,6 +159,10 @@ public class QTestUtil {
private final String initScript;
private final String cleanupScript;
+ public interface SuiteAddTestFunctor {
+ public void addTestToSuite(TestSuite suite, Object setup, String tName);
+ }
+
static {
for (String srcTable : System.getProperty("test.src.tables", "").trim().split(",")) {
srcTable = srcTable.trim();
@@ -1992,4 +1997,42 @@ public class QTestUtil {
(command != null ? " running " + command : "") +
(debugHint != null ? debugHint : ""));
}
+
+ public static void addTestsToSuiteFromQfileNames(
+ String qFileNamesFile,
+ Set<String> qFilesToExecute,
+ TestSuite suite,
+ Object setup,
+ SuiteAddTestFunctor suiteAddTestCallback) {
+ try {
+ File qFileNames = new File(qFileNamesFile);
+ FileReader fr = new FileReader(qFileNames.getCanonicalFile());
+ BufferedReader br = new BufferedReader(fr);
+ String fName = null;
+
+ while ((fName = br.readLine()) != null) {
+ if (fName.isEmpty() || fName.trim().equals("")) {
+ continue;
+ }
+
+ int eIdx = fName.indexOf('.');
+
+ if (eIdx == -1) {
+ continue;
+ }
+
+ String tName = fName.substring(0, eIdx);
+
+ if (qFilesToExecute.isEmpty() || qFilesToExecute.contains(fName)) {
+ suiteAddTestCallback.addTestToSuite(suite, setup, tName);
+ }
+ }
+ br.close();
+ } catch (Exception e) {
+ System.err.println("Exception: " + e.getMessage());
+ e.printStackTrace();
+ System.err.flush();
+ Assert.fail("Unexpected exception " + org.apache.hadoop.util.StringUtils.stringifyException(e));
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/28346637/ql/src/test/templates/TestCliDriver.vm
----------------------------------------------------------------------
diff --git a/ql/src/test/templates/TestCliDriver.vm b/ql/src/test/templates/TestCliDriver.vm
index 8827f12..ae449c5 100644
--- a/ql/src/test/templates/TestCliDriver.vm
+++ b/ql/src/test/templates/TestCliDriver.vm
@@ -33,6 +33,12 @@ public class $className extends TestCase {
private static final String HIVE_ROOT = QTestUtil.ensurePathEndsInSlash(System.getProperty("hive.root"));
private static QTestUtil qt;
+ public static class TestCliDriverAddTestFromQFiles implements QTestUtil.SuiteAddTestFunctor {
+ public void addTestToSuite(TestSuite suite, Object setup, String tName) {
+ suite.addTest(new $className("testCliDriver_"+tName));
+ }
+ }
+
static {
MiniClusterType miniMR = MiniClusterType.valueForString("$clusterMode");
@@ -108,15 +114,11 @@ public class $className extends TestCase {
}
}
}
+
TestSuite suite = new TestSuite();
-#foreach ($qf in $qfiles)
- #set ($fname = $qf.getName())
- #set ($eidx = $fname.indexOf('.'))
- #set ($tname = $fname.substring(0, $eidx))
- if(qFilesToExecute.isEmpty() || qFilesToExecute.contains("$fname")) {
- suite.addTest(new $className("testCliDriver_$tname"));
- }
-#end
+
+ QTestUtil.addTestsToSuiteFromQfileNames("$qFileNamesFile", qFilesToExecute,
+ suite, null, new TestCliDriverAddTestFromQFiles());
suite.addTest(new $className("testCliDriver_shutdown"));
return suite;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/28346637/ql/src/test/templates/TestCompareCliDriver.vm
----------------------------------------------------------------------
diff --git a/ql/src/test/templates/TestCompareCliDriver.vm b/ql/src/test/templates/TestCompareCliDriver.vm
index 6b367d6..4712708 100644
--- a/ql/src/test/templates/TestCompareCliDriver.vm
+++ b/ql/src/test/templates/TestCompareCliDriver.vm
@@ -34,6 +34,12 @@ public class $className extends TestCase {
private static final String HIVE_ROOT = QTestUtil.ensurePathEndsInSlash(System.getProperty("hive.root"));
private static QTestUtil qt;
+ public static class TestCompareCliDriverAddTestFromQFiles implements QTestUtil.SuiteAddTestFunctor {
+ public void addTestToSuite(TestSuite suite, Object setup, String tName) {
+ suite.addTest(new $className("testCompareCliDriver_"+tName));
+ }
+ }
+
static {
MiniClusterType miniMR = MiniClusterType.valueForString("$clusterMode");
@@ -109,18 +115,11 @@ public class $className extends TestCase {
}
}
}
+
TestSuite suite = new TestSuite();
-
-
-
-#foreach ($qf in $qfiles)
- #set ($fname = $qf.getName())
- #set ($eidx = $fname.indexOf('.'))
- #set ($tname = $fname.substring(0, $eidx))
- if (qFilesToExecute.isEmpty() || qFilesToExecute.contains("$fname")) {
- suite.addTest(new $className("testCompareCliDriver_$tname"));
- }
-#end
+
+ QTestUtil.addTestsToSuiteFromQfileNames("$qFileNamesFile", qFilesToExecute,
+ suite, null, new TestCompareCliDriverAddTestFromQFiles());
suite.addTest(new $className("testCompareCliDriver_shutdown"));
return suite;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/28346637/ql/src/test/templates/TestNegativeCliDriver.vm
----------------------------------------------------------------------
diff --git a/ql/src/test/templates/TestNegativeCliDriver.vm b/ql/src/test/templates/TestNegativeCliDriver.vm
index 742044a..5f8ee8e 100644
--- a/ql/src/test/templates/TestNegativeCliDriver.vm
+++ b/ql/src/test/templates/TestNegativeCliDriver.vm
@@ -33,6 +33,12 @@ public class $className extends TestCase {
private static final String HIVE_ROOT = QTestUtil.ensurePathEndsInSlash(System.getProperty("hive.root"));
private static QTestUtil qt;
+ public static class TestNegativeCliDriverAddTestFromQFiles implements QTestUtil.SuiteAddTestFunctor {
+ public void addTestToSuite(TestSuite suite, Object setup, String tName) {
+ suite.addTest(new $className("testNegativeCliDriver_"+tName));
+ }
+ }
+
static {
MiniClusterType miniMR = MiniClusterType.valueForString("$clusterMode");
String initScript = "$initScript";
@@ -95,15 +101,11 @@ public class $className extends TestCase {
}
}
}
+
TestSuite suite = new TestSuite();
-#foreach ($qf in $qfiles)
- #set ($fname = $qf.getName())
- #set ($eidx = $fname.indexOf('.'))
- #set ($tname = $fname.substring(0, $eidx))
- if(qFilesToExecute.isEmpty() || qFilesToExecute.contains("$fname")) {
- suite.addTest(new $className("testNegativeCliDriver_$tname"));
- }
-#end
+
+ QTestUtil.addTestsToSuiteFromQfileNames("$qFileNamesFile", qFilesToExecute,
+ suite, null, new TestNegativeCliDriverAddTestFromQFiles());
suite.addTest(new $className("testNegativeCliDriver_shutdown"));
return suite;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/28346637/ql/src/test/templates/TestParseNegative.vm
----------------------------------------------------------------------
diff --git a/ql/src/test/templates/TestParseNegative.vm b/ql/src/test/templates/TestParseNegative.vm
index 33b238e..c5e7bdf 100755
--- a/ql/src/test/templates/TestParseNegative.vm
+++ b/ql/src/test/templates/TestParseNegative.vm
@@ -32,6 +32,12 @@ public class $className extends TestCase {
private static final String HIVE_ROOT = QTestUtil.ensurePathEndsInSlash(System.getProperty("hive.root"));
private static QTestUtil qt;
+
+ public static class TestParseNegativeAddTestFromQFiles implements QTestUtil.SuiteAddTestFunctor {
+ public void addTestToSuite(TestSuite suite, Object setup, String tName) {
+ suite.addTest(new $className("testParseNegative_"+tName));
+ }
+ }
static {
@@ -88,16 +94,11 @@ public class $className extends TestCase {
}
}
}
+
TestSuite suite = new TestSuite();
-#foreach ($qf in $qfiles)
- #set ($fname = $qf.getName())
- #set ($eidx = $fname.indexOf('.'))
- #set ($tname = $fname.substring(0, $eidx))
- if(qFilesToExecute.isEmpty() || qFilesToExecute.contains("$fname")) {
- suite.addTest(new $className("testParseNegative_$tname"));
- }
-#end
+ QTestUtil.addTestsToSuiteFromQfileNames("$qFileNamesFile", qFilesToExecute,
+ suite, null, new TestParseNegativeAddTestFromQFiles());
suite.addTest(new $className("testParseNegative_shutdown"));
return suite;
}