You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ss...@apache.org on 2016/05/27 21:10:41 UTC
hive git commit: HIVE-13511. Run clidriver tests from within the
qtest dir for the precommit tests. (Siddharth Seth,
reviewed by Ashutosh Chauhan) (cherry picked from commit
02b2fb5a991b3dada90b442dbb013d466c3a50b0)
Repository: hive
Updated Branches:
refs/heads/branch-2.1 f547a6420 -> 4b553358b
HIVE-13511. Run clidriver tests from within the qtest dir for the precommit tests. (Siddharth Seth, reviewed by Ashutosh Chauhan)
(cherry picked from commit 02b2fb5a991b3dada90b442dbb013d466c3a50b0)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/4b553358
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/4b553358
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/4b553358
Branch: refs/heads/branch-2.1
Commit: 4b553358b0cfa19a1f1eb7eebf1b82f20f9b3e81
Parents: f547a64
Author: Siddharth Seth <ss...@apache.org>
Authored: Fri May 27 14:04:55 2016 -0700
Committer: Siddharth Seth <ss...@apache.org>
Committed: Fri May 27 14:10:56 2016 -0700
----------------------------------------------------------------------
.../hive/ptest/execution/HostExecutor.java | 4 ++++
.../org/apache/hive/ptest/execution/PTest.java | 1 +
.../ptest/execution/conf/QFileTestBatch.java | 12 ++++++++++--
.../hive/ptest/execution/conf/TestBatch.java | 1 +
.../hive/ptest/execution/conf/TestParser.java | 20 +++++++++++++++++---
.../ptest/execution/conf/UnitTestBatch.java | 6 ++++++
.../ptest/execution/TestExecutionPhase.java | 4 +++-
.../execution/conf/TestQFileTestBatch.java | 12 +++++++++---
8 files changed, 51 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/4b553358/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/HostExecutor.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/HostExecutor.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/HostExecutor.java
index 735b261..2c9100e 100644
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/HostExecutor.java
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/HostExecutor.java
@@ -40,6 +40,7 @@ import org.apache.hive.ptest.execution.ssh.SSHCommand;
import org.apache.hive.ptest.execution.ssh.SSHCommandExecutor;
import org.apache.hive.ptest.execution.ssh.SSHExecutionException;
import org.apache.hive.ptest.execution.ssh.SSHResult;
+import org.apache.logging.log4j.util.Strings;
import org.slf4j.Logger;
import com.google.common.annotations.VisibleForTesting;
@@ -243,6 +244,9 @@ class HostExecutor {
templateVariables.put("testArguments", batch.getTestArguments());
templateVariables.put("localDir", drone.getLocalDirectory());
templateVariables.put("logDir", drone.getLocalLogDirectory());
+ if (!Strings.isEmpty(batch.getTestModule())) {
+ templateVariables.put("testModule", batch.getTestModule());
+ }
String command = Templates.getTemplateResult("bash $localDir/$instanceName/scratch/" + script.getName(),
templateVariables);
Templates.writeTemplateResult("batch-exec.vm", script, templateVariables);
http://git-wip-us.apache.org/repos/asf/hive/blob/4b553358/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java
index 35cc752..de5c322 100644
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java
@@ -164,6 +164,7 @@ public class PTest {
Map<String, Long> elapsedTimes = Maps.newTreeMap();
try {
mLogger.info("Running tests with " + mConfiguration);
+ mLogger.info("Running tests with configuration context=[{}]", mConfiguration.getContext());
for(Phase phase : mPhases) {
String msg = "Executing " + phase.getClass().getName();
mLogger.info(msg);
http://git-wip-us.apache.org/repos/asf/hive/blob/4b553358/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/QFileTestBatch.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/QFileTestBatch.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/QFileTestBatch.java
index 61ecc88..fa213db 100644
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/QFileTestBatch.java
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/QFileTestBatch.java
@@ -29,10 +29,11 @@ public class QFileTestBatch implements TestBatch {
private final String driver;
private final String queryFilesProperty;
private final String name;
+ private final String moduleName;
private final Set<String> tests;
private final boolean isParallel;
public QFileTestBatch(String testCasePropertyName, String driver,
- String queryFilesProperty, Set<String> tests, boolean isParallel) {
+ String queryFilesProperty, Set<String> tests, boolean isParallel, String moduleName) {
this.testCasePropertyName = testCasePropertyName;
this.driver = driver;
this.queryFilesProperty = queryFilesProperty;
@@ -44,6 +45,7 @@ public class QFileTestBatch implements TestBatch {
}
this.name = name;
this.isParallel = isParallel;
+ this.moduleName = moduleName;
}
public String getDriver() {
return driver;
@@ -66,12 +68,18 @@ public class QFileTestBatch implements TestBatch {
public String toString() {
return "QFileTestBatch [driver=" + driver + ", queryFilesProperty="
+ queryFilesProperty + ", name=" + name + ", tests=" + tests
- + ", isParallel=" + isParallel + "]";
+ + ", isParallel=" + isParallel + ", moduleName=" + moduleName + "]";
}
@Override
public boolean isParallel() {
return isParallel;
}
+
+ @Override
+ public String getTestModule() {
+ return moduleName;
+ }
+
@Override
public int hashCode() {
final int prime = 31;
http://git-wip-us.apache.org/repos/asf/hive/blob/4b553358/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestBatch.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestBatch.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestBatch.java
index fc5a7c5..4ebb670 100644
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestBatch.java
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestBatch.java
@@ -28,4 +28,5 @@ public interface TestBatch {
public boolean isParallel();
+ public String getTestModule();
}
http://git-wip-us.apache.org/repos/asf/hive/blob/4b553358/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestParser.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestParser.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestParser.java
index aef6ac1..6d100de 100644
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestParser.java
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestParser.java
@@ -50,6 +50,9 @@ public class TestParser {
private static final Splitter TEST_SPLITTER = Splitter.onPattern("[, ]")
.trimResults().omitEmptyStrings();
+ private static final String QTEST_MODULE_NAME = "qtest";
+ private static final String QTEST_SPARK_MODULE_NAME = "qtest-spark";
+
private final Context context;
private final String testCasePropertyName;
private final File sourceDirectory;
@@ -177,17 +180,20 @@ public class TestParser {
logger.info("Exlcuding test " + driver + " " + test);
} else if(isolated.contains(test)) {
logger.info("Executing isolated test " + driver + " " + test);
- testBatches.add(new QFileTestBatch(testCasePropertyName, driver, queryFilesProperty, Sets.newHashSet(test), isParallel));
+ testBatches.add(new QFileTestBatch(testCasePropertyName, driver, queryFilesProperty,
+ Sets.newHashSet(test), isParallel, getModuleName(driver)));
} else {
if(testBatch.size() >= batchSize) {
- testBatches.add(new QFileTestBatch(testCasePropertyName, driver, queryFilesProperty, Sets.newHashSet(testBatch), isParallel));
+ testBatches.add(new QFileTestBatch(testCasePropertyName, driver, queryFilesProperty,
+ Sets.newHashSet(testBatch), isParallel, getModuleName(driver)));
testBatch = Lists.newArrayList();
}
testBatch.add(test);
}
}
if(!testBatch.isEmpty()) {
- testBatches.add(new QFileTestBatch(testCasePropertyName, driver, queryFilesProperty, Sets.newHashSet(testBatch), isParallel));
+ testBatches.add(new QFileTestBatch(testCasePropertyName, driver, queryFilesProperty,
+ Sets.newHashSet(testBatch), isParallel, getModuleName(driver)));
}
return testBatches;
}
@@ -259,6 +265,14 @@ public class TestParser {
tests.addAll(toAdd);
}
+ private String getModuleName(String driverName) {
+ if (driverName.toLowerCase().contains("spark")) {
+ return QTEST_SPARK_MODULE_NAME;
+ } else {
+ return QTEST_MODULE_NAME;
+ }
+ }
+
public Supplier<List<TestBatch>> parse() {
return new Supplier<List<TestBatch>>() {
@Override
http://git-wip-us.apache.org/repos/asf/hive/blob/4b553358/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/UnitTestBatch.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/UnitTestBatch.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/UnitTestBatch.java
index ca1adec..51f7f90 100644
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/UnitTestBatch.java
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/UnitTestBatch.java
@@ -51,6 +51,12 @@ public class UnitTestBatch implements TestBatch {
public boolean isParallel() {
return isParallel;
}
+
+ @Override
+ public String getTestModule() {
+ return null;
+ }
+
@Override
public int hashCode() {
final int prime = 31;
http://git-wip-us.apache.org/repos/asf/hive/blob/4b553358/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestExecutionPhase.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestExecutionPhase.java b/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestExecutionPhase.java
index 09f237e..29d148b 100644
--- a/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestExecutionPhase.java
+++ b/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestExecutionPhase.java
@@ -69,7 +69,9 @@ public class TestExecutionPhase extends AbstractTestPhase {
private void setupQFile(boolean isParallel) throws Exception {
testDir = Dirs.create( new File(baseDir, "test"));
Assert.assertTrue(new File(testDir, QFILENAME).createNewFile());
- testBatch = new QFileTestBatch("testcase", DRIVER, "qfile", Sets.newHashSet(QFILENAME), isParallel);
+ testBatch =
+ new QFileTestBatch("testcase", DRIVER, "qfile", Sets.newHashSet(QFILENAME), isParallel,
+ "testModule");
testBatches = Collections.singletonList(testBatch);
}
private void setupUnitTest() throws Exception {
http://git-wip-us.apache.org/repos/asf/hive/blob/4b553358/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/conf/TestQFileTestBatch.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/conf/TestQFileTestBatch.java b/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/conf/TestQFileTestBatch.java
index 67349fc..0257591 100644
--- a/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/conf/TestQFileTestBatch.java
+++ b/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/conf/TestQFileTestBatch.java
@@ -32,6 +32,7 @@ public class TestQFileTestBatch {
private static final String DRIVER = "driver";
private static final String QUERY_FILES_PROPERTY = "qfile";
+ private static final String TEST_MODULE_NAME = "testModule";
private Set<String> tests;
@@ -42,23 +43,28 @@ public class TestQFileTestBatch {
@Test
public void testParallel() throws Exception {
- QFileTestBatch batch = new QFileTestBatch("testcase", DRIVER, QUERY_FILES_PROPERTY, tests, true);
+ QFileTestBatch batch =
+ new QFileTestBatch("testcase", DRIVER, QUERY_FILES_PROPERTY, tests, true, TEST_MODULE_NAME);
Assert.assertTrue(batch.isParallel());
Assert.assertEquals(DRIVER, batch.getDriver());
Assert.assertEquals(Joiner.on("-").join(DRIVER, "a", "b", "c"), batch.getName());
Assert.assertEquals(String.format("-Dtestcase=%s -D%s=a,b,c", DRIVER,
QUERY_FILES_PROPERTY), batch.getTestArguments());
+ Assert.assertEquals(TEST_MODULE_NAME, batch.getTestModule());
}
@Test
public void testMoreThanThreeTests() throws Exception {
Assert.assertTrue(tests.add("d"));
- QFileTestBatch batch = new QFileTestBatch("testcase", DRIVER, QUERY_FILES_PROPERTY, tests, true);
+ QFileTestBatch batch =
+ new QFileTestBatch("testcase", DRIVER, QUERY_FILES_PROPERTY, tests, true, TEST_MODULE_NAME);
Assert.assertEquals(Joiner.on("-").join(DRIVER, "a", "b", "c", "and", "1", "more"),
batch.getName());
}
@Test
public void testNotParallel() throws Exception {
- QFileTestBatch batch = new QFileTestBatch("testcase", DRIVER, QUERY_FILES_PROPERTY, tests, false);
+ QFileTestBatch batch =
+ new QFileTestBatch("testcase", DRIVER, QUERY_FILES_PROPERTY, tests, false,
+ TEST_MODULE_NAME);
Assert.assertFalse(batch.isParallel());
}
}