You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by sp...@apache.org on 2016/05/28 13:37:02 UTC
[2/2] hive git commit: HIVE-13511. Run clidriver tests from within
the qtest dir for the precommit tests. (Siddharth Seth,
reviewed by Ashutosh Chauhan)
HIVE-13511. Run clidriver tests from within the qtest dir for the precommit tests. (Siddharth Seth, reviewed by Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e1e68b29
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e1e68b29
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e1e68b29
Branch: refs/heads/master
Commit: e1e68b29a5151c6b829d5ddecd9f03fd84504b31
Parents: 9906cd2
Author: Siddharth Seth <ss...@apache.org>
Authored: Sat May 28 08:35:55 2016 -0500
Committer: Sergio Pena <se...@cloudera.com>
Committed: Sat May 28 08:35:55 2016 -0500
----------------------------------------------------------------------
.../hive/ptest/execution/HostExecutor.java | 4 ++++
.../org/apache/hive/ptest/execution/PTest.java | 1 +
.../ptest/execution/conf/QFileTestBatch.java | 12 ++++++++++--
.../hive/ptest/execution/conf/TestBatch.java | 1 +
.../hive/ptest/execution/conf/TestParser.java | 20 +++++++++++++++++---
.../ptest/execution/conf/UnitTestBatch.java | 6 ++++++
.../ptest/execution/TestExecutionPhase.java | 4 +++-
.../execution/conf/TestQFileTestBatch.java | 12 +++++++++---
8 files changed, 51 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/e1e68b29/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/HostExecutor.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/HostExecutor.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/HostExecutor.java
index 735b261..2c9100e 100644
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/HostExecutor.java
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/HostExecutor.java
@@ -40,6 +40,7 @@ import org.apache.hive.ptest.execution.ssh.SSHCommand;
import org.apache.hive.ptest.execution.ssh.SSHCommandExecutor;
import org.apache.hive.ptest.execution.ssh.SSHExecutionException;
import org.apache.hive.ptest.execution.ssh.SSHResult;
+import org.apache.logging.log4j.util.Strings;
import org.slf4j.Logger;
import com.google.common.annotations.VisibleForTesting;
@@ -243,6 +244,9 @@ class HostExecutor {
templateVariables.put("testArguments", batch.getTestArguments());
templateVariables.put("localDir", drone.getLocalDirectory());
templateVariables.put("logDir", drone.getLocalLogDirectory());
+ if (!Strings.isEmpty(batch.getTestModule())) {
+ templateVariables.put("testModule", batch.getTestModule());
+ }
String command = Templates.getTemplateResult("bash $localDir/$instanceName/scratch/" + script.getName(),
templateVariables);
Templates.writeTemplateResult("batch-exec.vm", script, templateVariables);
http://git-wip-us.apache.org/repos/asf/hive/blob/e1e68b29/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java
index 35cc752..de5c322 100644
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java
@@ -164,6 +164,7 @@ public class PTest {
Map<String, Long> elapsedTimes = Maps.newTreeMap();
try {
mLogger.info("Running tests with " + mConfiguration);
+ mLogger.info("Running tests with configuration context=[{}]", mConfiguration.getContext());
for(Phase phase : mPhases) {
String msg = "Executing " + phase.getClass().getName();
mLogger.info(msg);
http://git-wip-us.apache.org/repos/asf/hive/blob/e1e68b29/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/QFileTestBatch.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/QFileTestBatch.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/QFileTestBatch.java
index 61ecc88..fa213db 100644
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/QFileTestBatch.java
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/QFileTestBatch.java
@@ -29,10 +29,11 @@ public class QFileTestBatch implements TestBatch {
private final String driver;
private final String queryFilesProperty;
private final String name;
+ private final String moduleName;
private final Set<String> tests;
private final boolean isParallel;
public QFileTestBatch(String testCasePropertyName, String driver,
- String queryFilesProperty, Set<String> tests, boolean isParallel) {
+ String queryFilesProperty, Set<String> tests, boolean isParallel, String moduleName) {
this.testCasePropertyName = testCasePropertyName;
this.driver = driver;
this.queryFilesProperty = queryFilesProperty;
@@ -44,6 +45,7 @@ public class QFileTestBatch implements TestBatch {
}
this.name = name;
this.isParallel = isParallel;
+ this.moduleName = moduleName;
}
public String getDriver() {
return driver;
@@ -66,12 +68,18 @@ public class QFileTestBatch implements TestBatch {
public String toString() {
return "QFileTestBatch [driver=" + driver + ", queryFilesProperty="
+ queryFilesProperty + ", name=" + name + ", tests=" + tests
- + ", isParallel=" + isParallel + "]";
+ + ", isParallel=" + isParallel + ", moduleName=" + moduleName + "]";
}
@Override
public boolean isParallel() {
return isParallel;
}
+
+ @Override
+ public String getTestModule() {
+ return moduleName;
+ }
+
@Override
public int hashCode() {
final int prime = 31;
http://git-wip-us.apache.org/repos/asf/hive/blob/e1e68b29/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestBatch.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestBatch.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestBatch.java
index fc5a7c5..4ebb670 100644
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestBatch.java
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestBatch.java
@@ -28,4 +28,5 @@ public interface TestBatch {
public boolean isParallel();
+ public String getTestModule();
}
http://git-wip-us.apache.org/repos/asf/hive/blob/e1e68b29/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestParser.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestParser.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestParser.java
index aef6ac1..c51ef3c 100644
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestParser.java
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestParser.java
@@ -50,6 +50,9 @@ public class TestParser {
private static final Splitter TEST_SPLITTER = Splitter.onPattern("[, ]")
.trimResults().omitEmptyStrings();
+ private static final String QTEST_MODULE_NAME = "itests/qtest";
+ private static final String QTEST_SPARK_MODULE_NAME = "itests/qtest-spark";
+
private final Context context;
private final String testCasePropertyName;
private final File sourceDirectory;
@@ -177,17 +180,20 @@ public class TestParser {
logger.info("Exlcuding test " + driver + " " + test);
} else if(isolated.contains(test)) {
logger.info("Executing isolated test " + driver + " " + test);
- testBatches.add(new QFileTestBatch(testCasePropertyName, driver, queryFilesProperty, Sets.newHashSet(test), isParallel));
+ testBatches.add(new QFileTestBatch(testCasePropertyName, driver, queryFilesProperty,
+ Sets.newHashSet(test), isParallel, getModuleName(driver)));
} else {
if(testBatch.size() >= batchSize) {
- testBatches.add(new QFileTestBatch(testCasePropertyName, driver, queryFilesProperty, Sets.newHashSet(testBatch), isParallel));
+ testBatches.add(new QFileTestBatch(testCasePropertyName, driver, queryFilesProperty,
+ Sets.newHashSet(testBatch), isParallel, getModuleName(driver)));
testBatch = Lists.newArrayList();
}
testBatch.add(test);
}
}
if(!testBatch.isEmpty()) {
- testBatches.add(new QFileTestBatch(testCasePropertyName, driver, queryFilesProperty, Sets.newHashSet(testBatch), isParallel));
+ testBatches.add(new QFileTestBatch(testCasePropertyName, driver, queryFilesProperty,
+ Sets.newHashSet(testBatch), isParallel, getModuleName(driver)));
}
return testBatches;
}
@@ -259,6 +265,14 @@ public class TestParser {
tests.addAll(toAdd);
}
+ private String getModuleName(String driverName) {
+ if (driverName.toLowerCase().contains("spark")) {
+ return QTEST_SPARK_MODULE_NAME;
+ } else {
+ return QTEST_MODULE_NAME;
+ }
+ }
+
public Supplier<List<TestBatch>> parse() {
return new Supplier<List<TestBatch>>() {
@Override
http://git-wip-us.apache.org/repos/asf/hive/blob/e1e68b29/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/UnitTestBatch.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/UnitTestBatch.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/UnitTestBatch.java
index ca1adec..51f7f90 100644
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/UnitTestBatch.java
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/UnitTestBatch.java
@@ -51,6 +51,12 @@ public class UnitTestBatch implements TestBatch {
public boolean isParallel() {
return isParallel;
}
+
+ @Override
+ public String getTestModule() {
+ return null;
+ }
+
@Override
public int hashCode() {
final int prime = 31;
http://git-wip-us.apache.org/repos/asf/hive/blob/e1e68b29/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestExecutionPhase.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestExecutionPhase.java b/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestExecutionPhase.java
index 09f237e..29d148b 100644
--- a/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestExecutionPhase.java
+++ b/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestExecutionPhase.java
@@ -69,7 +69,9 @@ public class TestExecutionPhase extends AbstractTestPhase {
private void setupQFile(boolean isParallel) throws Exception {
testDir = Dirs.create( new File(baseDir, "test"));
Assert.assertTrue(new File(testDir, QFILENAME).createNewFile());
- testBatch = new QFileTestBatch("testcase", DRIVER, "qfile", Sets.newHashSet(QFILENAME), isParallel);
+ testBatch =
+ new QFileTestBatch("testcase", DRIVER, "qfile", Sets.newHashSet(QFILENAME), isParallel,
+ "testModule");
testBatches = Collections.singletonList(testBatch);
}
private void setupUnitTest() throws Exception {
http://git-wip-us.apache.org/repos/asf/hive/blob/e1e68b29/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/conf/TestQFileTestBatch.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/conf/TestQFileTestBatch.java b/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/conf/TestQFileTestBatch.java
index 67349fc..0257591 100644
--- a/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/conf/TestQFileTestBatch.java
+++ b/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/conf/TestQFileTestBatch.java
@@ -32,6 +32,7 @@ public class TestQFileTestBatch {
private static final String DRIVER = "driver";
private static final String QUERY_FILES_PROPERTY = "qfile";
+ private static final String TEST_MODULE_NAME = "testModule";
private Set<String> tests;
@@ -42,23 +43,28 @@ public class TestQFileTestBatch {
@Test
public void testParallel() throws Exception {
- QFileTestBatch batch = new QFileTestBatch("testcase", DRIVER, QUERY_FILES_PROPERTY, tests, true);
+ QFileTestBatch batch =
+ new QFileTestBatch("testcase", DRIVER, QUERY_FILES_PROPERTY, tests, true, TEST_MODULE_NAME);
Assert.assertTrue(batch.isParallel());
Assert.assertEquals(DRIVER, batch.getDriver());
Assert.assertEquals(Joiner.on("-").join(DRIVER, "a", "b", "c"), batch.getName());
Assert.assertEquals(String.format("-Dtestcase=%s -D%s=a,b,c", DRIVER,
QUERY_FILES_PROPERTY), batch.getTestArguments());
+ Assert.assertEquals(TEST_MODULE_NAME, batch.getTestModule());
}
@Test
public void testMoreThanThreeTests() throws Exception {
Assert.assertTrue(tests.add("d"));
- QFileTestBatch batch = new QFileTestBatch("testcase", DRIVER, QUERY_FILES_PROPERTY, tests, true);
+ QFileTestBatch batch =
+ new QFileTestBatch("testcase", DRIVER, QUERY_FILES_PROPERTY, tests, true, TEST_MODULE_NAME);
Assert.assertEquals(Joiner.on("-").join(DRIVER, "a", "b", "c", "and", "1", "more"),
batch.getName());
}
@Test
public void testNotParallel() throws Exception {
- QFileTestBatch batch = new QFileTestBatch("testcase", DRIVER, QUERY_FILES_PROPERTY, tests, false);
+ QFileTestBatch batch =
+ new QFileTestBatch("testcase", DRIVER, QUERY_FILES_PROPERTY, tests, false,
+ TEST_MODULE_NAME);
Assert.assertFalse(batch.isParallel());
}
}