You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ss...@apache.org on 2016/09/14 21:42:20 UTC
[2/2] hive git commit: Revert "HIVE-14540. Add support in ptest to
create batches for non qfile tests."
Revert "HIVE-14540. Add support in ptest to create batches for non qfile tests."
This reverts commit 94dcf4abb360a0571eb54d220a1fb0b3130211cf.
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/0e91e28a
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/0e91e28a
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/0e91e28a
Branch: refs/heads/master
Commit: 0e91e28a9b81916259b8ec8d82445e398d33d4c3
Parents: 94dcf4a
Author: Siddharth Seth <ss...@apache.org>
Authored: Wed Sep 14 14:42:14 2016 -0700
Committer: Siddharth Seth <ss...@apache.org>
Committed: Wed Sep 14 14:42:14 2016 -0700
----------------------------------------------------------------------
.../hive/ptest/execution/ExecutionPhase.java | 8 -
.../hive/ptest/execution/HostExecutor.java | 7 +-
.../ptest/execution/conf/FileListProvider.java | 31 -
.../ptest/execution/conf/QFileTestBatch.java | 11 +-
.../hive/ptest/execution/conf/TestBatch.java | 38 +-
.../ptest/execution/conf/TestConfiguration.java | 4 -
.../hive/ptest/execution/conf/TestParser.java | 54 +-
.../ptest/execution/conf/UnitTestBatch.java | 88 +--
.../conf/UnitTestPropertiesParser.java | 662 ------------------
.../ptest2/src/main/resources/batch-exec.vm | 10 +-
.../ptest2/src/main/resources/source-prep.vm | 20 +-
.../execution/MockRSyncCommandExecutor.java | 7 -
.../ptest/execution/MockSSHCommandExecutor.java | 7 -
.../ptest/execution/TestExecutionPhase.java | 9 +-
...cutionPhase.testFailingUnitTest.approved.txt | 6 +-
...cutionPhase.testPassingUnitTest.approved.txt | 6 +-
.../hive/ptest/execution/TestHostExecutor.java | 51 +-
.../TestHostExecutor.testBasic.approved.txt | 24 +-
...xecutor.testIsolatedFailsOnExec.approved.txt | 10 +-
...tor.testIsolatedFailsOnRsyncOne.approved.txt | 8 +-
...testIsolatedFailsOnRsyncUnknown.approved.txt | 8 +-
...xecutor.testParallelFailsOnExec.approved.txt | 10 +-
...ecutor.testParallelFailsOnRsync.approved.txt | 8 +-
...tScripts.testAlternativeTestJVM.approved.txt | 10 +-
.../TestScripts.testBatch.approved.txt | 10 +-
.../TestScripts.testPrepGit.approved.txt | 13 +-
.../TestScripts.testPrepHadoop1.approved.txt | 20 +-
.../TestScripts.testPrepNone.approved.txt | 13 +-
.../TestScripts.testPrepSvn.approved.txt | 13 +-
.../execution/conf/TestQFileTestBatch.java | 2 +-
.../ptest/execution/conf/TestTestParser.java | 3 -
.../conf/TestUnitTestPropertiesParser.java | 671 -------------------
.../ptest2/src/test/resources/log4j2.properties | 62 --
.../resources/test-configuration2.properties | 135 ----
34 files changed, 166 insertions(+), 1873 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/0e91e28a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ExecutionPhase.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ExecutionPhase.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ExecutionPhase.java
index 65af6fa..6063afc 100644
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ExecutionPhase.java
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ExecutionPhase.java
@@ -88,14 +88,6 @@ public class ExecutionPhase extends Phase {
}
logger.info("ParallelWorkQueueSize={}, IsolatedWorkQueueSize={}", parallelWorkQueue.size(),
isolatedWorkQueue.size());
- if (logger.isDebugEnabled()) {
- for (TestBatch testBatch : parallelWorkQueue) {
- logger.debug("PBatch: {}", testBatch);
- }
- for (TestBatch testBatch : isolatedWorkQueue) {
- logger.debug("IBatch: {}", testBatch);
- }
- }
try {
int expectedNumHosts = hostExecutors.size();
initalizeHosts();
http://git-wip-us.apache.org/repos/asf/hive/blob/0e91e28a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/HostExecutor.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/HostExecutor.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/HostExecutor.java
index 123e310..5f84f00 100644
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/HostExecutor.java
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/HostExecutor.java
@@ -30,7 +30,6 @@ import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.TimeUnit;
import com.google.common.base.Stopwatch;
-import org.apache.commons.lang.StringUtils;
import org.apache.hive.ptest.execution.conf.Host;
import org.apache.hive.ptest.execution.conf.TestBatch;
import org.apache.hive.ptest.execution.ssh.RSyncCommand;
@@ -41,6 +40,7 @@ import org.apache.hive.ptest.execution.ssh.SSHCommand;
import org.apache.hive.ptest.execution.ssh.SSHCommandExecutor;
import org.apache.hive.ptest.execution.ssh.SSHExecutionException;
import org.apache.hive.ptest.execution.ssh.SSHResult;
+import org.apache.logging.log4j.util.Strings;
import org.slf4j.Logger;
import com.google.common.annotations.VisibleForTesting;
@@ -244,8 +244,9 @@ class HostExecutor {
templateVariables.put("testArguments", batch.getTestArguments());
templateVariables.put("localDir", drone.getLocalDirectory());
templateVariables.put("logDir", drone.getLocalLogDirectory());
- Preconditions.checkArgument(StringUtils.isNotBlank(batch.getTestModuleRelativeDir()));
- templateVariables.put("testModule", batch.getTestModuleRelativeDir());
+ if (!Strings.isEmpty(batch.getTestModule())) {
+ templateVariables.put("testModule", batch.getTestModule());
+ }
String command = Templates.getTemplateResult("bash $localDir/$instanceName/scratch/" + script.getName(),
templateVariables);
Templates.writeTemplateResult("batch-exec.vm", script, templateVariables);
http://git-wip-us.apache.org/repos/asf/hive/blob/0e91e28a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/FileListProvider.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/FileListProvider.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/FileListProvider.java
deleted file mode 100644
index b1eb66f..0000000
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/FileListProvider.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hive.ptest.execution.conf;
-
-import java.io.File;
-import java.util.Collection;
-
-// Exists primarily to allow for easier unit tests.
-public interface FileListProvider {
-
- Collection<File> listFiles(
- File directory, String[] extensions, boolean recursive);
-
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/0e91e28a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/QFileTestBatch.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/QFileTestBatch.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/QFileTestBatch.java
index eecdf7e..fa213db 100644
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/QFileTestBatch.java
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/QFileTestBatch.java
@@ -23,7 +23,7 @@ import java.util.Set;
import com.google.common.base.Joiner;
import com.google.common.collect.Iterators;
-public class QFileTestBatch extends TestBatch {
+public class QFileTestBatch implements TestBatch {
private final String testCasePropertyName;
private final String driver;
@@ -66,7 +66,7 @@ public class QFileTestBatch extends TestBatch {
@Override
public String toString() {
- return "QFileTestBatch [batchId=" + getBatchId() + ", driver=" + driver + ", queryFilesProperty="
+ return "QFileTestBatch [driver=" + driver + ", queryFilesProperty="
+ queryFilesProperty + ", name=" + name + ", tests=" + tests
+ ", isParallel=" + isParallel + ", moduleName=" + moduleName + "]";
}
@@ -76,16 +76,11 @@ public class QFileTestBatch extends TestBatch {
}
@Override
- public String getTestModuleRelativeDir() {
+ public String getTestModule() {
return moduleName;
}
@Override
- public int getNumTestsInBatch() {
- return tests.size();
- }
-
- @Override
public int hashCode() {
final int prime = 31;
int result = 1;
http://git-wip-us.apache.org/repos/asf/hive/blob/0e91e28a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestBatch.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestBatch.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestBatch.java
index 799c399..4ebb670 100644
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestBatch.java
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestBatch.java
@@ -18,39 +18,15 @@
*/
package org.apache.hive.ptest.execution.conf;
-import java.util.concurrent.atomic.AtomicInteger;
+public interface TestBatch {
-import com.google.common.annotations.VisibleForTesting;
+ public String getTestArguments();
+
+ public String getTestClass();
-public abstract class TestBatch {
+ public String getName();
- private static final AtomicInteger BATCH_ID_GEN = new AtomicInteger(1);
+ public boolean isParallel();
- private final int batchId;
-
- public TestBatch() {
- this.batchId = BATCH_ID_GEN.getAndIncrement();
- }
-
- public abstract String getTestArguments();
-
- // TODO Get rid of this.
- public abstract String getTestClass();
-
- public abstract String getName();
-
- public abstract boolean isParallel();
-
- public abstract String getTestModuleRelativeDir();
-
- public abstract int getNumTestsInBatch();
-
- public final int getBatchId() {
- return batchId;
- }
-
- @VisibleForTesting
- public static void resetBatchCoutner() {
- BATCH_ID_GEN.set(1);
- }
+ public String getTestModule();
}
http://git-wip-us.apache.org/repos/asf/hive/blob/0e91e28a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestConfiguration.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestConfiguration.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestConfiguration.java
index f14026c..2c5bd3a 100644
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestConfiguration.java
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestConfiguration.java
@@ -58,7 +58,6 @@ public class TestConfiguration {
private static final String JENKINS_URL = "jenkinsURL";
private static final String SSH_OPTS = "sshOpts";
private static final String LOGS_URL = "logsURL";
- // This ends up being set to "test" | mvn ${testCasePropertyName} for instance
private static final String TEST_CASE_PROPERTY_NAME = "testCasePropertyName";
private static final String BUILD_TOOL = "buildTool";
// The following parameters are not supported yet. TODO Add support
@@ -227,17 +226,14 @@ public class TestConfiguration {
return testCasePropertyName;
}
- // TODO Make sure this method is eventually used to find the prep / batch scripts.
public String getApplyPathScriptPath() {
return applyPathScriptPath;
}
- // TODO Make sure this method is eventually used to find the prep / batch scripts.
public String getPrepTemplatePath() {
return prepTemplatePath;
}
- // TODO Make sure this method is eventually used to find the prep / batch scripts.
public String getBatchExecTemplatePath() {
return batchExecTemplatePath;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/0e91e28a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestParser.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestParser.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestParser.java
index 28a08ca..5da804f 100644
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestParser.java
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestParser.java
@@ -24,7 +24,6 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.Arrays;
-import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -34,6 +33,7 @@ import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import org.apache.commons.io.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -44,9 +44,8 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
public class TestParser {
-
private static final Splitter TEST_SPLITTER = Splitter.onPattern("[, ]")
- .trimResults().omitEmptyStrings();
+ .trimResults().omitEmptyStrings();
private static final String QTEST_MODULE_NAME = "itests/qtest";
private static final String QTEST_SPARK_MODULE_NAME = "itests/qtest-spark";
@@ -64,21 +63,50 @@ public class TestParser {
this.logger = logger;
}
private List<TestBatch> parseTests() {
-
- Set<String> excluded = new HashSet<String>();
-
-
+ Context unitContext = new Context(context.getSubProperties(
+ Joiner.on(".").join("unitTests", "")));
+ Set<String> excluded = Sets.newHashSet(TEST_SPLITTER.split(unitContext.getString("exclude", "")));
+ Set<String> isolated = Sets.newHashSet(TEST_SPLITTER.split(unitContext.getString("isolate", "")));
+ Set<String> included = Sets.newHashSet(TEST_SPLITTER.split(unitContext.getString("include", "")));
+ if(!included.isEmpty() && !excluded.isEmpty()) {
+ throw new IllegalArgumentException(String.format("Included and excluded mutally exclusive." +
+ " Included = %s, excluded = %s", included.toString(), excluded.toString()));
+ }
+ List<File> unitTestsDirs = Lists.newArrayList();
+ for(String unitTestDir : TEST_SPLITTER
+ .split(checkNotNull(unitContext.getString("directories"), "directories"))) {
+ File unitTestParent = new File(sourceDirectory, unitTestDir);
+ if(unitTestParent.isDirectory()) {
+ unitTestsDirs.add(unitTestParent);
+ } else {
+ logger.warn("Unit test directory " + unitTestParent + " does not exist.");
+ }
+ }
List<TestBatch> result = Lists.newArrayList();
for(QFileTestBatch test : parseQFileTests()) {
result.add(test);
excluded.add(test.getDriver());
}
-
- Collection<TestBatch> unitTestBatches =
- new UnitTestPropertiesParser(context, testCasePropertyName, sourceDirectory, logger,
- excluded).generateTestBatches();
- result.addAll(unitTestBatches);
-
+ for(File unitTestDir : unitTestsDirs) {
+ for(File classFile : FileUtils.listFiles(unitTestDir, new String[]{"class"}, true)) {
+ String className = classFile.getName();
+ logger.debug("In " + unitTestDir + ", found " + className);
+ if(className.startsWith("Test") && !className.contains("$")) {
+ String testName = className.replaceAll("\\.class$", "");
+ if(excluded.contains(testName)) {
+ logger.info("Exlcuding unit test " + testName);
+ } else if(included.isEmpty() || included.contains(testName)) {
+ if(isolated.contains(testName)) {
+ logger.info("Executing isolated unit test " + testName);
+ result.add(new UnitTestBatch(testCasePropertyName, testName, false));
+ } else {
+ logger.info("Executing parallel unit test " + testName);
+ result.add(new UnitTestBatch(testCasePropertyName, testName, true));
+ }
+ }
+ }
+ }
+ }
return result;
}
private List<QFileTestBatch> parseQFileTests() {
http://git-wip-us.apache.org/repos/asf/hive/blob/0e91e28a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/UnitTestBatch.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/UnitTestBatch.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/UnitTestBatch.java
index 3d57ed7..51f7f90 100644
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/UnitTestBatch.java
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/UnitTestBatch.java
@@ -18,55 +18,34 @@
*/
package org.apache.hive.ptest.execution.conf;
-import java.util.List;
-
-import com.google.common.base.Joiner;
-import com.google.common.base.Preconditions;
-
-public class UnitTestBatch extends TestBatch {
+public class UnitTestBatch implements TestBatch {
private final String testCasePropertyName;
- private final List<String> testList;
+ private final String testName;
private final boolean isParallel;
- private final String moduleName;
- private final String batchName;
- public UnitTestBatch(String testCasePropertyName, List<String> tests, String moduleName, boolean isParallel) {
- super();
- Preconditions.checkNotNull(testCasePropertyName);
- Preconditions.checkArgument(tests!= null && !tests.isEmpty());
+ public UnitTestBatch(String testCasePropertyName, String testName, boolean isParallel) {
this.testCasePropertyName = testCasePropertyName;
- this.testList = tests;
+ this.testName = testName;
this.isParallel = isParallel;
- this.moduleName = moduleName;
- if (tests.size() == 1) {
- batchName = String.format("%d_%s", getBatchId(), tests.get(0));
- } else {
- batchName = String.format("%d_UTBatch_%s_%d_tests", getBatchId(),
- (moduleName.replace("/", "__").replace(".", "__")), tests.size());
- }
}
@Override
public String getTestArguments() {
- String testArg = Joiner.on(",").join(testList);
- return String.format("-D%s=%s", testCasePropertyName, testArg);
+ return String.format("-D%s=%s", testCasePropertyName, testName);
}
@Override
public String getName() {
- // Used for logDir, failure messages etc.
- return batchName;
+ return testName;
}
@Override
public String getTestClass() {
- // Used to identify the module name. Return any.
- return testList.get(0);
+ return testName;
}
-
@Override
public String toString() {
- return "UnitTestBatch [name=" + batchName + ", id=" + getBatchId() + ", moduleName=" + moduleName +
- ", isParallel=" + isParallel + ", testList=" + testList + "]";
+ return "UnitTestBatch [testName=" + testName + ", isParallel=" + isParallel
+ + "]";
}
@Override
public boolean isParallel() {
@@ -74,45 +53,34 @@ public class UnitTestBatch extends TestBatch {
}
@Override
- public String getTestModuleRelativeDir() {
- return moduleName;
+ public String getTestModule() {
+ return null;
}
@Override
- public int getNumTestsInBatch() {
- return testList.size();
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + (isParallel ? 1231 : 1237);
+ result = prime * result + ((testName == null) ? 0 : testName.hashCode());
+ return result;
}
-
@Override
- public boolean equals(Object o) {
- if (this == o) {
+ public boolean equals(Object obj) {
+ if (this == obj)
return true;
- }
- if (o == null || getClass() != o.getClass()) {
+ if (obj == null)
return false;
- }
-
- UnitTestBatch that = (UnitTestBatch) o;
-
- if (isParallel != that.isParallel) {
+ if (getClass() != obj.getClass())
return false;
- }
- if (testList != null ? !testList.equals(that.testList) : that.testList != null) {
+ UnitTestBatch other = (UnitTestBatch) obj;
+ if (isParallel != other.isParallel)
return false;
- }
- if (moduleName != null ? !moduleName.equals(that.moduleName) : that.moduleName != null) {
+ if (testName == null) {
+ if (other.testName != null)
+ return false;
+ } else if (!testName.equals(other.testName))
return false;
- }
- return batchName != null ? batchName.equals(that.batchName) : that.batchName == null;
-
- }
-
- @Override
- public int hashCode() {
- int result = testList != null ? testList.hashCode() : 0;
- result = 31 * result + (isParallel ? 1 : 0);
- result = 31 * result + (moduleName != null ? moduleName.hashCode() : 0);
- result = 31 * result + (batchName != null ? batchName.hashCode() : 0);
- return result;
+ return true;
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/0e91e28a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/UnitTestPropertiesParser.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/UnitTestPropertiesParser.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/UnitTestPropertiesParser.java
deleted file mode 100644
index 395c5ab..0000000
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/UnitTestPropertiesParser.java
+++ /dev/null
@@ -1,662 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hive.ptest.execution.conf;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedHashMap;
-import java.util.LinkedHashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Joiner;
-import com.google.common.base.Preconditions;
-import com.google.common.base.Splitter;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang.StringUtils;
-import org.slf4j.Logger;
-
-class UnitTestPropertiesParser {
-
- private static final Splitter VALUE_SPLITTER = Splitter.onPattern("[, ]")
- .trimResults().omitEmptyStrings();
-
- // Prefix for top level properties.
- static final String PROP_PREFIX_ROOT = "unitTests";
- // Prefix used to specify module specific properties. Mainly to avoid conflicts with older unitTests properties
- static final String PROP_PREFIX_MODULE = "ut";
-
- static final String PROP_DIRECTORIES = "directories";
- static final String PROP_INCLUDE = "include";
- static final String PROP_EXCLUDE = "exclude";
- static final String PROP_ISOLATE = "isolate";
- static final String PROP_SKIP_BATCHING = "skipBatching";
- static final String PROP_BATCH_SIZE = "batchSize";
- static final String PROP_SUBDIR_FOR_PREFIX = "subdirForPrefix";
-
- static final String PROP_ONE_MODULE = "module";
- static final String PROP_MODULE_LIST = "modules";
-
-
- static final int DEFAULT_PROP_BATCH_SIZE = 1;
- static final int DEFAULT_PROP_BATCH_SIZE_NOT_SPECIFIED = -1;
- static final int DEFAULT_PROP_BATCH_SIZE_INCLUDE_ALL = 0;
- static final String DEFAULT_PROP_DIRECTORIES = ".";
- static final String DEFAULT_PROP_SUBDIR_FOR_PREFIX = "target";
-
- static final String MODULE_NAME_TOP_LEVEL = "_root_"; // Special module for tests in the rootDir.
- static final String PREFIX_TOP_LEVEL = ".";
-
- private final Context unitRootContext; // Everything prefixed by ^unitTests.
- private final Context unitModuleContext; // Everything prefixed by ^ut.
- private final String testCasePropertyName;
- private final Logger logger;
- private final File sourceDirectory;
- private final FileListProvider fileListProvider;
- private final Set<String> excludedProvided; // excludedProvidedBy Framework vs excludedConfigured
- private final boolean inTest;
-
-
- @VisibleForTesting
- UnitTestPropertiesParser(Context testContext, String testCasePropertyName,
- File sourceDirectory, Logger logger,
- FileListProvider fileListProvider,
- Set<String> excludedProvided, boolean inTest) {
- logger.info("{} created with sourceDirectory={}, testCasePropertyName={}, excludedProvide={}",
- "fileListProvider={}, inTest={}",
- UnitTestPropertiesParser.class.getSimpleName(), sourceDirectory, testCasePropertyName,
- excludedProvided,
- (fileListProvider == null ? "null" : fileListProvider.getClass().getSimpleName()), inTest);
- Preconditions.checkNotNull(testContext, "testContext cannot be null");
- Preconditions.checkNotNull(testCasePropertyName, "testCasePropertyName cannot be null");
- Preconditions.checkNotNull(sourceDirectory, "sourceDirectory cannot be null");
- Preconditions.checkNotNull(logger, "logger must be specified");
- this.unitRootContext =
- new Context(testContext.getSubProperties(Joiner.on(".").join(PROP_PREFIX_ROOT, "")));
- this.unitModuleContext =
- new Context(testContext.getSubProperties(Joiner.on(".").join(PROP_PREFIX_MODULE, "")));
- this.sourceDirectory = sourceDirectory;
- this.testCasePropertyName = testCasePropertyName;
- this.logger = logger;
- if (excludedProvided != null) {
- this.excludedProvided = excludedProvided;
- } else {
- this.excludedProvided = new HashSet<>();
- }
- if (fileListProvider != null) {
- this.fileListProvider = fileListProvider;
- } else {
- this.fileListProvider = new DefaultFileListProvider();
- }
- this.inTest = inTest;
-
- }
-
- UnitTestPropertiesParser(Context testContext, String testCasePropertyName,
- File sourceDirectory, Logger logger,
- Set<String> excludedProvided) {
- this(testContext, testCasePropertyName, sourceDirectory, logger, null, excludedProvided, false);
- }
-
-
- Collection<TestBatch> generateTestBatches() {
- try {
- return parse();
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
-
- private Collection<TestBatch> parse() throws IOException {
-
- RootConfig rootConfig = getRootConfig(unitRootContext);
- logger.info("RootConfig: " + rootConfig);
-
- // TODO: Set this up as a tree, instead of a flat list.
- Map<String, ModuleConfig> moduleConfigs = extractModuleConfigs();
- logger.info("ModuleConfigs: {} ", moduleConfigs);
-
- List<TestDir> unitTestsDirs = processPropertyDirectories();
-
- validateConfigs(rootConfig, moduleConfigs, unitTestsDirs);
-
- LinkedHashMap<String, LinkedHashSet<TestInfo>> allTests =
- generateFullTestSet(rootConfig, moduleConfigs, unitTestsDirs);
-
-
- return createTestBatches(allTests, rootConfig, moduleConfigs);
- }
-
- private Collection<TestBatch> createTestBatches(
- LinkedHashMap<String, LinkedHashSet<TestInfo>> allTests, RootConfig rootConfig,
- Map<String, ModuleConfig> moduleConfigs) {
- List<TestBatch> testBatches = new LinkedList<>();
- for (Map.Entry<String, LinkedHashSet<TestInfo>> entry : allTests.entrySet()) {
- logger.info("Creating test batches for module={}, numTests={}", entry.getKey(),
- entry.getValue().size());
- String currentModule = entry.getKey();
- String currentPathPrefix = getPathPrefixFromModuleName(currentModule);
- int batchSize = rootConfig.batchSize;
- if (moduleConfigs.containsKey(currentModule)) {
- ModuleConfig moduleConfig = moduleConfigs.get(currentModule);
- int batchSizeModule = moduleConfig.batchSize;
- if (batchSizeModule != DEFAULT_PROP_BATCH_SIZE_NOT_SPECIFIED) {
- batchSize = batchSizeModule;
- }
- }
-
- if (batchSize == DEFAULT_PROP_BATCH_SIZE_INCLUDE_ALL) {
- batchSize = Integer.MAX_VALUE;
- }
- logger.info("batchSize determined to be {} for module={}", batchSize, currentModule);
-
- // TODO Even out the batch sizes (i.e. 20/20/1 should be replaced by 14/14/13)
- List<String> currentList = new LinkedList<>();
- for (TestInfo testInfo : entry.getValue()) {
- if (testInfo.isIsolated || testInfo.skipBatching) {
- UnitTestBatch unitTestBatch =
- new UnitTestBatch(testCasePropertyName, Collections.singletonList(testInfo.testName),
- currentPathPrefix, !testInfo.isIsolated);
- testBatches.add(unitTestBatch);
- } else {
- currentList.add(testInfo.testName);
- if (currentList.size() == batchSize) {
- UnitTestBatch unitTestBatch =
- new UnitTestBatch(testCasePropertyName, Collections.unmodifiableList(currentList),
- currentPathPrefix, true);
- testBatches.add(unitTestBatch);
- currentList = new LinkedList<>();
- }
- }
- }
- if (!currentList.isEmpty()) {
- UnitTestBatch unitTestBatch =
- new UnitTestBatch(testCasePropertyName, Collections.unmodifiableList(currentList),
- currentPathPrefix, true);
- testBatches.add(unitTestBatch);
- }
- }
- return testBatches;
- }
-
-
- private RootConfig getRootConfig(Context context) {
- ModuleConfig moduleConfig =
- getModuleConfig(context, "irrelevant", DEFAULT_PROP_BATCH_SIZE);
-
- String subDirForPrefix =
- context.getString(PROP_SUBDIR_FOR_PREFIX, DEFAULT_PROP_SUBDIR_FOR_PREFIX);
- Preconditions
- .checkArgument(StringUtils.isNotBlank(subDirForPrefix) && !subDirForPrefix.contains("/"));
-
- Context modulesContext =
- new Context(context.getSubProperties(Joiner.on(".").join(PROP_MODULE_LIST, "")));
- Set<String> includedModules = getProperty(modulesContext, PROP_INCLUDE);
- Set<String> excludedModules = getProperty(modulesContext, PROP_EXCLUDE);
- if (!includedModules.isEmpty() && !excludedModules.isEmpty()) {
- throw new IllegalArgumentException(String.format(
- "%s and %s are mutually exclusive for property %s. Provided values: included=%s, excluded=%s",
- PROP_INCLUDE, PROP_EXCLUDE, PROP_MODULE_LIST, includedModules, excludedModules));
- }
-
- return new RootConfig(includedModules, excludedModules, moduleConfig.include,
- moduleConfig.exclude, moduleConfig.skipBatching, moduleConfig.isolate,
- moduleConfig.batchSize, subDirForPrefix);
- }
-
- private ModuleConfig getModuleConfig(Context context, String moduleName, int defaultBatchSize) {
- Set<String> excluded = getProperty(context, PROP_EXCLUDE);
- Set<String> isolated = getProperty(context, PROP_ISOLATE);
- Set<String> included = getProperty(context, PROP_INCLUDE);
- Set<String> skipBatching = getProperty(context, PROP_SKIP_BATCHING);
- if (!included.isEmpty() && !excluded.isEmpty()) {
- throw new IllegalArgumentException(String.format("Included and excluded mutually exclusive." +
- " Included = %s, excluded = %s", included.toString(), excluded.toString()) +
- " for module: " + moduleName);
- }
- int batchSize = context.getInteger(PROP_BATCH_SIZE, defaultBatchSize);
-
- String pathPrefix = getPathPrefixFromModuleName(moduleName);
-
- return new ModuleConfig(moduleName, included, excluded, skipBatching, isolated, batchSize,
- pathPrefix);
- }
-
- private Set<String> getProperty(Context context, String propertyName) {
- return Sets.newHashSet(VALUE_SPLITTER.split(context.getString(propertyName, "")));
- }
-
- private String getPathPrefixFromModuleName(String moduleName) {
- String pathPrefix;
- if (moduleName.equals(MODULE_NAME_TOP_LEVEL)) {
- pathPrefix = PREFIX_TOP_LEVEL;
- } else {
- pathPrefix = moduleName.replace(".", "/");
- }
- return pathPrefix;
- }
-
- private String getModuleNameFromPathPrefix(String pathPrefix) {
- if (pathPrefix.equals(PREFIX_TOP_LEVEL)) {
- return MODULE_NAME_TOP_LEVEL;
- } else {
- pathPrefix = stripEndAndStart(pathPrefix, "/");
- pathPrefix = pathPrefix.replace("/", ".");
- // Example handling of dirs with a .
- // shims/hadoop-2.6
- // -> moduleName=shims.hadoop-.2.6
- return pathPrefix;
- }
- }
-
- private String stripEndAndStart(String srcString, String stripChars) {
- srcString = StringUtils.stripEnd(srcString, stripChars);
- srcString = StringUtils.stripStart(srcString, stripChars);
- return srcString;
- }
-
- private Map<String, ModuleConfig> extractModuleConfigs() {
- Collection<String> modules = extractConfiguredModules();
- Map<String, ModuleConfig> result = new HashMap<>();
-
- for (String moduleName : modules) {
- Context moduleContext =
- new Context(unitModuleContext.getSubProperties(Joiner.on(".").join(moduleName, "")));
- ModuleConfig moduleConfig =
- getModuleConfig(moduleContext, moduleName, DEFAULT_PROP_BATCH_SIZE_NOT_SPECIFIED);
- logger.info("Adding moduleConfig={}", moduleConfig);
- result.put(moduleName, moduleConfig);
- }
- return result;
- }
-
- private Collection<String> extractConfiguredModules() {
- List<String> configuredModules = new LinkedList<>();
-
- Map<String, String> modulesMap = unitRootContext.getSubProperties(Joiner.on(".").join(
- PROP_ONE_MODULE, ""));
- for (Map.Entry<String, String> module : modulesMap.entrySet()) {
- // This is an unnecessary check, and forced configuration in the property file. Maybe
- // replace with an enforced empty value string.
- Preconditions.checkArgument(module.getKey().equals(module.getValue()));
- String moduleName = module.getKey();
- configuredModules.add(moduleName);
- }
- return configuredModules;
- }
-
- private List<TestDir> processPropertyDirectories() throws IOException {
- String srcDirString = sourceDirectory.getCanonicalPath();
- List<TestDir> unitTestsDirs = Lists.newArrayList();
- String propDirectoriies = unitRootContext.getString(PROP_DIRECTORIES, DEFAULT_PROP_DIRECTORIES);
- Iterable<String> propDirectoriesIterable = VALUE_SPLITTER.split(propDirectoriies);
-
- for (String unitTestDir : propDirectoriesIterable) {
- File unitTestParent = new File(sourceDirectory, unitTestDir);
- if (unitTestParent.isDirectory() || inTest) {
- String absUnitTestDir = unitTestParent.getCanonicalPath();
-
- Preconditions.checkState(absUnitTestDir.startsWith(srcDirString),
- "Unit test dir: " + absUnitTestDir + " is not under provided src dir: " + srcDirString);
- String modulePath = absUnitTestDir.substring(srcDirString.length());
-
- modulePath = stripEndAndStart(modulePath, "/");
-
- Preconditions.checkState(!modulePath.startsWith("/"),
- String.format("Illegal module path: [%s]", modulePath));
- if (StringUtils.isEmpty(modulePath)) {
- modulePath = PREFIX_TOP_LEVEL;
- }
- String moduleName = getModuleNameFromPathPrefix(modulePath);
- logger.info("modulePath determined as {} for testdir={}, DerivedModuleName={}", modulePath,
- absUnitTestDir, moduleName);
-
-
- logger.info("Adding unitTests dir [{}],[{}]", unitTestParent, moduleName);
- unitTestsDirs.add(new TestDir(unitTestParent, moduleName));
- } else {
- logger.warn("Unit test directory " + unitTestParent + " does not exist, or is a file.");
- }
- }
-
- return unitTestsDirs;
- }
-
- private void validateConfigs(RootConfig rootConfig,
- Map<String, ModuleConfig> moduleConfigs,
- List<TestDir> unitTestDir) {
-
- if (rootConfig.include.isEmpty() & rootConfig.exclude.isEmpty()) {
- // No conflicts. Module configuration is what will be used.
- // We've already verified that includes and excludes are not present at the same time for
- // individual modules.
- return;
- }
-
- // Validate mainly for includes / excludes working as they should.
- for (Map.Entry<String, ModuleConfig> entry : moduleConfigs.entrySet()) {
- if (rootConfig.excludedModules.contains(entry.getKey())) {
- // Don't bother validating.
- continue;
- }
-
- if (!rootConfig.includedModules.isEmpty() &&
- !rootConfig.includedModules.contains(entry.getKey())) {
- // Include specified, but this module is not in the set.
- continue;
- }
-
- // If global contains includes, individual modules can only contain additional includes.
- if (!rootConfig.include.isEmpty() && !entry.getValue().exclude.isEmpty()) {
- throw new IllegalStateException(String.format(
- "Global config specified includes, while module config for %s specified excludes",
- entry.getKey()));
- }
- // If global contains excludes, individual modules can only contain additional excludes.
- if (!rootConfig.exclude.isEmpty() && !entry.getValue().include.isEmpty()) {
- throw new IllegalStateException(String.format(
- "Global config specified excludes, while module config for %s specified includes",
- entry.getKey()));
- }
- }
- }
-
- private LinkedHashMap<String, LinkedHashSet<TestInfo>> generateFullTestSet(RootConfig rootConfig,
- Map<String, ModuleConfig> moduleConfigs,
- List<TestDir> unitTestDirs) throws
- IOException {
- LinkedHashMap<String, LinkedHashSet<TestInfo>> result = new LinkedHashMap<>();
-
- for (TestDir unitTestDir : unitTestDirs) {
- for (File classFile : fileListProvider
- .listFiles(unitTestDir.path, new String[]{"class"}, true)) {
- String className = classFile.getName();
-
- if (className.startsWith("Test") && !className.contains("$")) {
- String testName = className.replaceAll("\\.class$", "");
- String pathPrefix = getPathPrefix(classFile, rootConfig.subDirForPrefix);
- String moduleName = getModuleNameFromPathPrefix(pathPrefix);
- logger.debug("In {}, found class {} with pathPrefix={}, moduleName={}", unitTestDir.path,
- className,
- pathPrefix, moduleName);
-
-
- ModuleConfig moduleConfig = moduleConfigs.get(moduleName);
- if (moduleConfig == null) {
- moduleConfig = FAKE_MODULE_CONFIG;
- }
- TestInfo testInfo = checkAndGetTestInfo(moduleName, pathPrefix, testName, rootConfig, moduleConfig);
- if (testInfo != null) {
- logger.info("Adding test: " + testInfo);
- addTestToResult(result, testInfo);
- }
- } else {
- logger.trace("In {}, found class {} with pathPrefix={}. Not a test", unitTestDir.path,
- className);
- }
- }
- }
- return result;
- }
-
- private void addTestToResult(Map<String, LinkedHashSet<TestInfo>> result, TestInfo testInfo) {
- LinkedHashSet<TestInfo> moduleSet = result.get(testInfo.moduleName);
- if (moduleSet == null) {
- moduleSet = new LinkedHashSet<>();
- result.put(testInfo.moduleName, moduleSet);
- }
- moduleSet.add(testInfo);
- }
-
- private String getPathPrefix(File file, String subDirPrefix) throws IOException {
- String fname = file.getCanonicalPath();
- Preconditions.checkState(fname.startsWith(sourceDirectory.getCanonicalPath()));
- fname = fname.substring(sourceDirectory.getCanonicalPath().length(), fname.length());
- if (fname.contains(subDirPrefix)) {
- fname = fname.substring(0, fname.indexOf(subDirPrefix));
- fname = StringUtils.stripStart(fname, "/");
- if (StringUtils.isEmpty(fname)) {
- fname = PREFIX_TOP_LEVEL;
- }
- return fname;
- } else {
- logger.error("Could not find subDirPrefix {} in path: {}", subDirPrefix, fname);
- return PREFIX_TOP_LEVEL;
- }
- }
-
- private TestInfo checkAndGetTestInfo(String moduleName, String moduleRelDir, String testName,
- RootConfig rootConfig, ModuleConfig moduleConfig) {
- Preconditions.checkNotNull(moduleConfig);
- TestInfo testInfo;
- String rejectReason = null;
- try {
- if (rootConfig.excludedModules.contains(moduleName)) {
- rejectReason = "root level module exclude";
- return null;
- }
- if (!rootConfig.includedModules.isEmpty() &&
- !rootConfig.includedModules.contains(moduleName)) {
- rejectReason = "root level include, but not for module";
- return null;
- }
- if (rootConfig.exclude.contains(testName)) {
- rejectReason = "root excludes test";
- return null;
- }
- if (moduleConfig.exclude.contains(testName)) {
- rejectReason = "module excludes test";
- return null;
- }
- boolean containsInclude = !rootConfig.include.isEmpty() || !moduleConfig.include.isEmpty();
- if (containsInclude) {
- if (!(rootConfig.include.contains(testName) || moduleConfig.include.contains(testName))) {
- rejectReason = "test missing from include list";
- return null;
- }
- }
- if (excludedProvided.contains(testName)) {
- // All qfiles handled via this...
- rejectReason = "test present in provided exclude list";
- return null;
- }
-
- // Add the test.
- testInfo = new TestInfo(moduleName, moduleRelDir, testName, rootConfig.skipBatching.contains(testName) ||
- moduleConfig.skipBatching.contains(testName),
- rootConfig.isolate.contains(testName) || moduleConfig.isolate.contains(testName));
- return testInfo;
-
- } finally {
- if (rejectReason != null) {
- logger.debug("excluding {} due to {}", testName, rejectReason);
- }
- }
- }
-
- private static final class RootConfig {
- private final Set<String> includedModules;
- private final Set<String> excludedModules;
- private final Set<String> include;
- private final Set<String> exclude;
- private final Set<String> skipBatching;
- private final Set<String> isolate;
- private final int batchSize;
- private final String subDirForPrefix;
-
- RootConfig(Set<String> includedModules, Set<String> excludedModules, Set<String> include,
- Set<String> exclude, Set<String> skipBatching, Set<String> isolate,
- int batchSize, String subDirForPrefix) {
- this.includedModules = includedModules;
- this.excludedModules = excludedModules;
- this.include = include;
- this.exclude = exclude;
- this.skipBatching = skipBatching;
- this.isolate = isolate;
- this.batchSize = batchSize;
- this.subDirForPrefix = subDirForPrefix;
- }
-
- @Override
- public String toString() {
- return "RootConfig{" +
- "includedModules=" + includedModules +
- ", excludedModules=" + excludedModules +
- ", include=" + include +
- ", exclude=" + exclude +
- ", skipBatching=" + skipBatching +
- ", isolate=" + isolate +
- ", batchSize=" + batchSize +
- ", subDirForPrefix='" + subDirForPrefix + '\'' +
- '}';
- }
- }
-
- private static final ModuleConfig FAKE_MODULE_CONFIG =
- new ModuleConfig("_FAKE_", new HashSet<String>(), new HashSet<String>(),
- new HashSet<String>(), new HashSet<String>(), DEFAULT_PROP_BATCH_SIZE_NOT_SPECIFIED,
- "_fake_");
-
- private static final class ModuleConfig {
- private final String name;
- private final Set<String> include;
- private final Set<String> exclude;
- private final Set<String> skipBatching;
- private final Set<String> isolate;
- private final String pathPrefix;
- private final int batchSize;
-
- ModuleConfig(String name, Set<String> include, Set<String> exclude,
- Set<String> skipBatching, Set<String> isolate, int batchSize,
- String pathPrefix) {
- this.name = name;
- this.include = include;
- this.exclude = exclude;
- this.skipBatching = skipBatching;
- this.isolate = isolate;
- this.batchSize = batchSize;
- this.pathPrefix = pathPrefix;
- }
-
- @Override
- public String toString() {
- return "ModuleConfig{" +
- "name='" + name + '\'' +
- ", include=" + include +
- ", exclude=" + exclude +
- ", skipBatching=" + skipBatching +
- ", isolate=" + isolate +
- ", pathPrefix='" + pathPrefix + '\'' +
- ", batchSize=" + batchSize +
- '}';
- }
- }
-
- private static class TestDir {
- final File path;
- final String module;
-
- TestDir(File path, String module) {
- this.path = path;
- this.module = module;
- }
-
- @Override
- public String toString() {
- return "TestDir{" +
- "path=" + path +
- ", module='" + module + '\'' +
- '}';
- }
- }
-
- private static class TestInfo {
- final String moduleName;
- final String moduleRelativeDir;
- final String testName;
- final boolean skipBatching;
- final boolean isIsolated;
-
- TestInfo(String moduleName, String moduleRelativeDir, String testName, boolean skipBatching, boolean isIsolated) {
- this.moduleName = moduleName;
- this.moduleRelativeDir = moduleRelativeDir;
- this.testName = testName;
- this.skipBatching = skipBatching;
- this.isIsolated = isIsolated;
- }
-
- @Override
- public String toString() {
- return "TestInfo{" +
- "moduleName='" + moduleName + '\'' +
- ", moduleRelativeDir='" + moduleRelativeDir + '\'' +
- ", testName='" + testName + '\'' +
- ", skipBatching=" + skipBatching +
- ", isIsolated=" + isIsolated +
- '}';
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
-
- TestInfo testInfo = (TestInfo) o;
-
- return skipBatching == testInfo.skipBatching && isIsolated == testInfo.isIsolated &&
- moduleName.equals(testInfo.moduleName) &&
- moduleRelativeDir.equals(testInfo.moduleRelativeDir) &&
- testName.equals(testInfo.testName);
-
- }
-
- @Override
- public int hashCode() {
- int result = moduleName.hashCode();
- result = 31 * result + moduleRelativeDir.hashCode();
- result = 31 * result + testName.hashCode();
- result = 31 * result + (skipBatching ? 1 : 0);
- result = 31 * result + (isIsolated ? 1 : 0);
- return result;
- }
- }
-
- private static final class DefaultFileListProvider implements FileListProvider {
-
- @Override
- public Collection<File> listFiles(File directory, String[] extensions, boolean recursive) {
- return FileUtils.listFiles(directory, extensions, recursive);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/0e91e28a/testutils/ptest2/src/main/resources/batch-exec.vm
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/resources/batch-exec.vm b/testutils/ptest2/src/main/resources/batch-exec.vm
index 2cc56ea..d8141b0 100644
--- a/testutils/ptest2/src/main/resources/batch-exec.vm
+++ b/testutils/ptest2/src/main/resources/batch-exec.vm
@@ -16,7 +16,6 @@
##### Remember, this is a velocity template
set -x
-date +"%Y-%m-%d %T.%3N"
umask 0022
echo $$
ps -e -o pid,pgrp,user,args
@@ -39,8 +38,6 @@ export M2_OPTS="-Xmx1g -XX:MaxPermSize=256m -Djava.io.tmpdir=$logDir/tmp ${maven
export HADOOP_ROOT_LOGGER=INFO,console
export HADOOP_OPTS="-Dhive.log.dir=$logDir -Dhive.query.id=hadoop -Djava.io.tmpdir=$logDir/tmp"
cd $localDir/$instanceName/${repositoryName}-source || exit 1
-date +"%Y-%m-%d %T.%3N"
-echo "Pre test cleanup"
if [[ -s batch.pid ]]
then
while read pid
@@ -56,11 +53,10 @@ echo "$$" > batch.pid
find ./ -name 'TEST-*.xml' -delete
find ./ -name 'hive.log' -delete
find ./ -name junit_metastore_db | xargs -r rm -rf
-date +"%Y-%m-%d %T.%3N"
-echo "Pre test cleanup done"
ret=0
if [[ "${buildTool}" == "maven" ]]
then
+ testModule=$(find ./ -name '${testClass}.java' | awk -F'/' '{print $2}')
if [[ -z "$testModule" ]]
then
testModule=./
@@ -68,7 +64,6 @@ then
pushd $testModule
timeout 40m mvn -B test -Dmaven.repo.local=$localDir/$instanceName/maven \
$mavenArgs $mavenTestArgs $testArguments 1>$logDir/maven-test.txt 2>&1 </dev/null &
- date +"%Y-%m-%d %T.%3N"
#[[
pid=$!
]]#
@@ -91,7 +86,6 @@ fi
echo $pid >> batch.pid
wait $pid
ret=$?
-date +"%Y-%m-%d %T.%3N"
find ./ -type f -name hive.log -o -name spark.log -o -name derby.log | \
xargs -I {} sh -c 'f=$(basename {}); test -f ${logDir}/$f && f=$f-$(uuidgen); mv {} ${logDir}/$f'
find ./ -type f -name 'TEST-*.xml' | \
@@ -100,7 +94,7 @@ find ./ -path "*/spark/work" | \
xargs -I {} sh -c 'mv {} ${logDir}/spark-log'
find ./ -type f -name 'syslog*' | \
xargs -I {} sh -c 'mkdir -p ${logDir}/syslogs; mv {} ${logDir}/syslogs'
-date +"%Y-%m-%d %T.%3N"
+
if [[ -f $logDir/.log ]]
then
http://git-wip-us.apache.org/repos/asf/hive/blob/0e91e28a/testutils/ptest2/src/main/resources/source-prep.vm
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/resources/source-prep.vm b/testutils/ptest2/src/main/resources/source-prep.vm
index 67e6a95..9c83a14 100644
--- a/testutils/ptest2/src/main/resources/source-prep.vm
+++ b/testutils/ptest2/src/main/resources/source-prep.vm
@@ -16,7 +16,6 @@
##### Remember this is a velocity template
set -e
set -x
-date +"%Y-%m-%d %T.%3N"
if [[ -n "${javaHome}" ]]
then
export JAVA_HOME=$javaHome
@@ -65,15 +64,13 @@ cd $workingDir/
then
git clone $repository ${repositoryName}-source
fi
- date +"%Y-%m-%d %T.%3N"
cd ${repositoryName}-source
git fetch origin
git reset --hard HEAD && git clean -f -d
git checkout $branch || git checkout -b $branch origin/$branch
git reset --hard origin/$branch
git merge --ff-only origin/$branch
- #git gc
- date +"%Y-%m-%d %T.%3N"
+ git gc
else
echo "Unknown repository type '${repositoryType}'"
exit 1
@@ -94,20 +91,14 @@ cd $workingDir/
for i in $(echo $ADDITIONAL_PROFILES | tr "," "\n")
do
mvn clean install -DskipTests -P$i;
- if [[ "-d itests" ]]
- then
- cd itests
- mvn clean install -DskipTests -P$i;
+ cd itests
+ mvn clean install -DskipTests -P$i;
cd ..
- fi
done
#end
mvn -B clean install -DskipTests -Dmaven.repo.local=$workingDir/maven $mavenArgs $mavenBuildArgs
- if [[ -d "itests" ]]
- then
- cd itests
- mvn -B clean install -DskipTests -Dmaven.repo.local=$workingDir/maven $mavenArgs $mavenBuildArgs
- fi
+ cd itests
+ mvn -B clean install -DskipTests -Dmaven.repo.local=$workingDir/maven $mavenArgs $mavenBuildArgs
elif [[ "${buildTool}" == "ant" ]]
then
ant $antArgs -Divy.default.ivy.user.dir=$workingDir/ivy \
@@ -117,6 +108,5 @@ cd $workingDir/
echo "Unknown build tool ${buildTool}"
exit 127
fi
- date +"%Y-%m-%d %T.%3N"
) 2>&1 | tee $logDir/source-prep.txt
exit ${PIPESTATUS[0]}
http://git-wip-us.apache.org/repos/asf/hive/blob/0e91e28a/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/MockRSyncCommandExecutor.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/MockRSyncCommandExecutor.java b/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/MockRSyncCommandExecutor.java
index 3906435..6347ce5 100644
--- a/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/MockRSyncCommandExecutor.java
+++ b/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/MockRSyncCommandExecutor.java
@@ -22,8 +22,6 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Queue;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hive.ptest.execution.ssh.RSyncCommand;
import org.apache.hive.ptest.execution.ssh.RSyncCommandExecutor;
@@ -35,7 +33,6 @@ import com.google.common.collect.Maps;
public class MockRSyncCommandExecutor extends RSyncCommandExecutor {
private final List<String> mCommands;
private final Map<String, Queue<Integer>> mFailures;
- private final AtomicInteger matchCount = new AtomicInteger(0);
public MockRSyncCommandExecutor(Logger logger) {
super(logger, 0, null);
mCommands = Lists.newArrayList();
@@ -65,13 +62,9 @@ public class MockRSyncCommandExecutor extends RSyncCommandExecutor {
if(queue == null || queue.isEmpty()) {
command.setExitCode(0);
} else {
- matchCount.incrementAndGet();
command.setExitCode(queue.remove());
}
}
- public int getMatchCount() {
- return matchCount.get();
- }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/0e91e28a/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/MockSSHCommandExecutor.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/MockSSHCommandExecutor.java b/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/MockSSHCommandExecutor.java
index 1f3db12..e4cd807 100644
--- a/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/MockSSHCommandExecutor.java
+++ b/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/MockSSHCommandExecutor.java
@@ -22,7 +22,6 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Queue;
-import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hive.ptest.execution.ssh.SSHCommand;
import org.apache.hive.ptest.execution.ssh.SSHCommandExecutor;
@@ -34,7 +33,6 @@ import com.google.common.collect.Maps;
public class MockSSHCommandExecutor extends SSHCommandExecutor {
private final List<String> mCommands;
private final Map<String, Queue<Integer>> mFailures;
- private final AtomicInteger matchCount = new AtomicInteger(0);
public MockSSHCommandExecutor(Logger logger) {
super(logger);
mCommands = Lists.newArrayList();
@@ -63,12 +61,7 @@ public class MockSSHCommandExecutor extends SSHCommandExecutor {
if(queue == null || queue.isEmpty()) {
command.setExitCode(0);
} else {
- matchCount.incrementAndGet();
command.setExitCode(queue.remove());
}
}
-
- public int getMatchCount() {
- return matchCount.get();
- }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/0e91e28a/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestExecutionPhase.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestExecutionPhase.java b/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestExecutionPhase.java
index 74f4204..29d148b 100644
--- a/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestExecutionPhase.java
+++ b/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestExecutionPhase.java
@@ -20,7 +20,6 @@ package org.apache.hive.ptest.execution;
import java.io.File;
import java.io.IOException;
-import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Set;
@@ -68,7 +67,6 @@ public class TestExecutionPhase extends AbstractTestPhase {
return phase;
}
private void setupQFile(boolean isParallel) throws Exception {
- TestBatch.resetBatchCoutner();
testDir = Dirs.create( new File(baseDir, "test"));
Assert.assertTrue(new File(testDir, QFILENAME).createNewFile());
testBatch =
@@ -77,8 +75,7 @@ public class TestExecutionPhase extends AbstractTestPhase {
testBatches = Collections.singletonList(testBatch);
}
private void setupUnitTest() throws Exception {
- TestBatch.resetBatchCoutner();
- testBatch = new UnitTestBatch("testcase", Arrays.asList(DRIVER), "fakemodule", false);
+ testBatch = new UnitTestBatch("testcase", DRIVER, false);
testBatches = Collections.singletonList(testBatch);
}
private void copyTestOutput(String resource, File directory, String name) throws Exception {
@@ -107,7 +104,6 @@ public class TestExecutionPhase extends AbstractTestPhase {
"-0/scratch/hiveptest-" + DRIVER + "-" + QFILENAME + ".sh", 1);
copyTestOutput("SomeTest-failure.xml", failedLogDir, testBatch.getName());
getPhase().execute();
- Assert.assertEquals(1, sshCommandExecutor.getMatchCount());
Approvals.verify(getExecutedCommands());
Assert.assertEquals(Sets.newHashSet("SomeTest." + QFILENAME), executedTests);
Assert.assertEquals(Sets.newHashSet("SomeTest." + QFILENAME), failedTests);
@@ -125,10 +121,9 @@ public class TestExecutionPhase extends AbstractTestPhase {
public void testFailingUnitTest() throws Throwable {
setupUnitTest();
sshCommandExecutor.putFailure("bash " + LOCAL_DIR + "/" + HOST + "-" + USER +
- "-0/scratch/hiveptest-" + testBatch.getBatchId() + "_" + DRIVER + ".sh", 1);
+ "-0/scratch/hiveptest-" + DRIVER + ".sh", 1);
copyTestOutput("SomeTest-failure.xml", failedLogDir, testBatch.getName());
getPhase().execute();
- Assert.assertEquals(1, sshCommandExecutor.getMatchCount());
Approvals.verify(getExecutedCommands());
Assert.assertEquals(Sets.newHashSet("SomeTest." + QFILENAME), executedTests);
Assert.assertEquals(Sets.newHashSet("SomeTest." + QFILENAME), failedTests);
http://git-wip-us.apache.org/repos/asf/hive/blob/0e91e28a/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestExecutionPhase.testFailingUnitTest.approved.txt
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestExecutionPhase.testFailingUnitTest.approved.txt b/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestExecutionPhase.testFailingUnitTest.approved.txt
index 97b66af..0727830 100644
--- a/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestExecutionPhase.testFailingUnitTest.approved.txt
+++ b/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestExecutionPhase.testFailingUnitTest.approved.txt
@@ -1,9 +1,9 @@
/some/working/dir/ivy /some/local/dir/somehost-someuser-0
/some/working/dir/maven /some/local/dir/somehost-someuser-0
/some/working/dir/repositoryName-source /some/local/dir/somehost-someuser-0
-/tmp/hive-ptest-units/TestExecutionPhase/logs/failed/1_driver /some/local/dir/somehost-someuser-0/logs/
-/tmp/hive-ptest-units/TestExecutionPhase/scratch/hiveptest-1_driver.sh /some/local/dir/somehost-someuser-0/scratch/hiveptest-1_driver.sh
-bash /some/local/dir/somehost-someuser-0/scratch/hiveptest-1_driver.sh
+/tmp/hive-ptest-units/TestExecutionPhase/logs/failed/driver /some/local/dir/somehost-someuser-0/logs/
+/tmp/hive-ptest-units/TestExecutionPhase/scratch/hiveptest-driver.sh /some/local/dir/somehost-someuser-0/scratch/hiveptest-driver.sh
+bash /some/local/dir/somehost-someuser-0/scratch/hiveptest-driver.sh
killall -q -9 -f java || true
mkdir -p /some/local/dir/somehost-someuser-0/logs /some/local/dir/somehost-someuser-0/maven /some/local/dir/somehost-someuser-0/scratch /some/local/dir/somehost-someuser-0/ivy /some/local/dir/somehost-someuser-0/repositoryName-source
mkdir -p /some/local/dir/somehost-someuser-1/logs /some/local/dir/somehost-someuser-1/maven /some/local/dir/somehost-someuser-1/scratch /some/local/dir/somehost-someuser-1/ivy /some/local/dir/somehost-someuser-1/repositoryName-source
http://git-wip-us.apache.org/repos/asf/hive/blob/0e91e28a/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestExecutionPhase.testPassingUnitTest.approved.txt
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestExecutionPhase.testPassingUnitTest.approved.txt b/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestExecutionPhase.testPassingUnitTest.approved.txt
index 9cb4715..3ce10b1 100644
--- a/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestExecutionPhase.testPassingUnitTest.approved.txt
+++ b/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestExecutionPhase.testPassingUnitTest.approved.txt
@@ -1,9 +1,9 @@
/some/working/dir/ivy /some/local/dir/somehost-someuser-0
/some/working/dir/maven /some/local/dir/somehost-someuser-0
/some/working/dir/repositoryName-source /some/local/dir/somehost-someuser-0
-/tmp/hive-ptest-units/TestExecutionPhase/logs/succeeded/1_driver /some/local/dir/somehost-someuser-0/logs/
-/tmp/hive-ptest-units/TestExecutionPhase/scratch/hiveptest-1_driver.sh /some/local/dir/somehost-someuser-0/scratch/hiveptest-1_driver.sh
-bash /some/local/dir/somehost-someuser-0/scratch/hiveptest-1_driver.sh
+/tmp/hive-ptest-units/TestExecutionPhase/logs/succeeded/driver /some/local/dir/somehost-someuser-0/logs/
+/tmp/hive-ptest-units/TestExecutionPhase/scratch/hiveptest-driver.sh /some/local/dir/somehost-someuser-0/scratch/hiveptest-driver.sh
+bash /some/local/dir/somehost-someuser-0/scratch/hiveptest-driver.sh
killall -q -9 -f java || true
mkdir -p /some/local/dir/somehost-someuser-0/logs /some/local/dir/somehost-someuser-0/maven /some/local/dir/somehost-someuser-0/scratch /some/local/dir/somehost-someuser-0/ivy /some/local/dir/somehost-someuser-0/repositoryName-source
mkdir -p /some/local/dir/somehost-someuser-1/logs /some/local/dir/somehost-someuser-1/maven /some/local/dir/somehost-someuser-1/scratch /some/local/dir/somehost-someuser-1/ivy /some/local/dir/somehost-someuser-1/repositoryName-source
http://git-wip-us.apache.org/repos/asf/hive/blob/0e91e28a/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestHostExecutor.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestHostExecutor.java b/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestHostExecutor.java
index 2a33395..0acebb9 100644
--- a/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestHostExecutor.java
+++ b/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestHostExecutor.java
@@ -22,7 +22,6 @@ import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
import java.io.File;
-import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Set;
@@ -105,11 +104,10 @@ public class TestHostExecutor {
parallelWorkQueue = new LinkedBlockingQueue<TestBatch>();
isolatedWorkQueue = new LinkedBlockingQueue<TestBatch>();
failedTestResults = Sets.newHashSet();
- UnitTestBatch.resetBatchCoutner();
- testBatchParallel1 = new UnitTestBatch("testcase", Arrays.asList(DRIVER_PARALLEL_1), "fakeModule1", true);
- testBatchParallel2 = new UnitTestBatch("testcase", Arrays.asList(DRIVER_PARALLEL_2), "fakeModule2", true);
- testBatchIsolated1 = new UnitTestBatch("testcase", Arrays.asList(DRIVER_ISOLATED_1), "fakeModule3", false);
- testBatchIsolated2 = new UnitTestBatch("testcase", Arrays.asList(DRIVER_ISOLATED_2), "fakeModule4", false);
+ testBatchParallel1 = new UnitTestBatch("testcase", DRIVER_PARALLEL_1, true);
+ testBatchParallel2 = new UnitTestBatch("testcase", DRIVER_PARALLEL_2, true);
+ testBatchIsolated1 = new UnitTestBatch("testcase", DRIVER_ISOLATED_1, false);
+ testBatchIsolated2 = new UnitTestBatch("testcase", DRIVER_ISOLATED_2, false);
executor = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(2));
localCommandFactory = new MockLocalCommandFactory(LOG);
localCommand = mock(LocalCommand.class);
@@ -161,55 +159,44 @@ public class TestHostExecutor {
@Test
public void testParallelFailsOnExec()
throws Exception {
- sshCommandExecutor.putFailure("bash /some/local/dir/somehost-someuser-0/scratch/hiveptest-" +
- testBatchParallel1.getBatchId() + "_driver-parallel-1.sh",
+ sshCommandExecutor.putFailure("bash /some/local/dir/somehost-someuser-0/scratch/hiveptest-driver-parallel-1.sh",
Constants.EXIT_CODE_UNKNOWN);
HostExecutor executor = createHostExecutor();
parallelWorkQueue.addAll(Lists.newArrayList(testBatchParallel1));
executor.submitTests(parallelWorkQueue, isolatedWorkQueue, failedTestResults).get();
Assert.assertEquals(Collections.emptySet(), failedTestResults);
Assert.assertTrue(parallelWorkQueue.toString(), parallelWorkQueue.isEmpty());
- Assert.assertEquals(1, sshCommandExecutor.getMatchCount());
Approvals.verify(getExecutedCommands());
}
@Test
public void testIsolatedFailsOnExec()
throws Exception {
- sshCommandExecutor.putFailure("bash /some/local/dir/somehost-someuser-0/scratch/hiveptest-" +
- testBatchIsolated1.getBatchId() + "_driver-isolated-1.sh",
+ sshCommandExecutor.putFailure("bash /some/local/dir/somehost-someuser-0/scratch/hiveptest-driver-isolated-1.sh",
Constants.EXIT_CODE_UNKNOWN);
HostExecutor executor = createHostExecutor();
isolatedWorkQueue.addAll(Lists.newArrayList(testBatchIsolated1));
executor.submitTests(parallelWorkQueue, isolatedWorkQueue, failedTestResults).get();
Assert.assertEquals(Collections.emptySet(), failedTestResults);
Assert.assertTrue(isolatedWorkQueue.toString(), parallelWorkQueue.isEmpty());
- Assert.assertEquals(1, sshCommandExecutor.getMatchCount());
Approvals.verify(getExecutedCommands());
}
@Test
public void testParallelFailsOnRsync()
throws Exception {
- rsyncCommandExecutor.putFailure("/tmp/hive-ptest-units/TestHostExecutor/scratch/hiveptest-" +
- testBatchParallel1.getBatchId() + "_driver-parallel-1.sh "
- + "/some/local/dir/somehost-someuser-0/scratch/hiveptest-" +
- testBatchParallel1.getBatchId() + "_driver-parallel-1.sh",
- Constants.EXIT_CODE_UNKNOWN);
+ rsyncCommandExecutor.putFailure("/tmp/hive-ptest-units/TestHostExecutor/scratch/hiveptest-driver-parallel-1.sh "
+ + "/some/local/dir/somehost-someuser-0/scratch/hiveptest-driver-parallel-1.sh", Constants.EXIT_CODE_UNKNOWN);
HostExecutor executor = createHostExecutor();
parallelWorkQueue.addAll(Lists.newArrayList(testBatchParallel1));
executor.submitTests(parallelWorkQueue, isolatedWorkQueue, failedTestResults).get();
Assert.assertEquals(Collections.emptySet(), failedTestResults);
Assert.assertTrue(parallelWorkQueue.toString(), parallelWorkQueue.isEmpty());
- Assert.assertEquals(1, rsyncCommandExecutor.getMatchCount());
Approvals.verify(getExecutedCommands());
}
@Test
public void testShutdownBeforeExec()
throws Exception {
- rsyncCommandExecutor.putFailure("/tmp/hive-ptest-units/TestHostExecutor/scratch/hiveptest-" +
- testBatchParallel1.getBatchId() + "_driver-parallel-1.sh "
- + "/some/local/dir/somehost-someuser-0/scratch/hiveptest-" +
- testBatchParallel1.getBatchId() + "_driver-parallel-1.sh",
- Constants.EXIT_CODE_UNKNOWN);
+ rsyncCommandExecutor.putFailure("/tmp/hive-ptest-units/TestHostExecutor/scratch/hiveptest-driver-parallel-1.sh "
+ + "/some/local/dir/somehost-someuser-0/scratch/hiveptest-driver-parallel-1.sh", Constants.EXIT_CODE_UNKNOWN);
HostExecutor executor = createHostExecutor();
parallelWorkQueue.addAll(Lists.newArrayList(testBatchParallel1));
executor.shutdownNow();
@@ -217,38 +204,30 @@ public class TestHostExecutor {
Assert.assertEquals(Collections.emptySet(), failedTestResults);
Assert.assertEquals(parallelWorkQueue.toString(), 1, parallelWorkQueue.size());
Approvals.verify("EMPTY\n" + getExecutedCommands());
- Assert.assertEquals(0, rsyncCommandExecutor.getMatchCount());
Assert.assertTrue(executor.isShutdown());
}
@Test
public void testIsolatedFailsOnRsyncUnknown()
throws Exception {
- rsyncCommandExecutor.putFailure("/tmp/hive-ptest-units/TestHostExecutor/scratch/hiveptest-" +
- testBatchIsolated1.getBatchId() + "_driver-isolated-1.sh " +
- "/some/local/dir/somehost-someuser-0/scratch/hiveptest-" +
- testBatchIsolated1.getBatchId() + "_driver-isolated-1.sh",
- Constants.EXIT_CODE_UNKNOWN);
+ rsyncCommandExecutor.putFailure("/tmp/hive-ptest-units/TestHostExecutor/scratch/hiveptest-driver-isolated-1.sh "+
+ "/some/local/dir/somehost-someuser-0/scratch/hiveptest-driver-isolated-1.sh", Constants.EXIT_CODE_UNKNOWN);
HostExecutor executor = createHostExecutor();
isolatedWorkQueue.addAll(Lists.newArrayList(testBatchIsolated1));
executor.submitTests(parallelWorkQueue, isolatedWorkQueue, failedTestResults).get();
Assert.assertEquals(Collections.emptySet(), failedTestResults);
Assert.assertTrue(isolatedWorkQueue.toString(), isolatedWorkQueue.isEmpty());
- Assert.assertEquals(1, rsyncCommandExecutor.getMatchCount());
Approvals.verify(getExecutedCommands());
}
@Test
public void testIsolatedFailsOnRsyncOne()
throws Exception {
- rsyncCommandExecutor.putFailure("/tmp/hive-ptest-units/TestHostExecutor/scratch/hiveptest-" +
- testBatchIsolated1.getBatchId() + "_driver-isolated-1.sh " +
- "/some/local/dir/somehost-someuser-0/scratch/hiveptest-" + testBatchIsolated1.getBatchId() +
- "_driver-isolated-1.sh", 1);
+ rsyncCommandExecutor.putFailure("/tmp/hive-ptest-units/TestHostExecutor/scratch/hiveptest-driver-isolated-1.sh "+
+ "/some/local/dir/somehost-someuser-0/scratch/hiveptest-driver-isolated-1.sh", 1);
HostExecutor executor = createHostExecutor();
isolatedWorkQueue.addAll(Lists.newArrayList(testBatchIsolated1));
executor.submitTests(parallelWorkQueue, isolatedWorkQueue, failedTestResults).get();
Assert.assertEquals(Collections.emptySet(), failedTestResults);
Assert.assertTrue(isolatedWorkQueue.toString(), parallelWorkQueue.isEmpty());
- Assert.assertEquals(1, rsyncCommandExecutor.getMatchCount());
Approvals.verify(getExecutedCommands());
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/0e91e28a/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestHostExecutor.testBasic.approved.txt
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestHostExecutor.testBasic.approved.txt b/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestHostExecutor.testBasic.approved.txt
index c4cc9f6..c2a702c 100644
--- a/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestHostExecutor.testBasic.approved.txt
+++ b/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestHostExecutor.testBasic.approved.txt
@@ -1,12 +1,12 @@
-/tmp/hive-ptest-units/TestHostExecutor/logs/succeeded/1_driver-parallel-1 /some/local/dir/somehost-someuser-0/logs/
-/tmp/hive-ptest-units/TestHostExecutor/logs/succeeded/2_driver-parallel-2 /some/local/dir/somehost-someuser-0/logs/
-/tmp/hive-ptest-units/TestHostExecutor/logs/succeeded/3_driver-isolated-1 /some/local/dir/somehost-someuser-0/logs/
-/tmp/hive-ptest-units/TestHostExecutor/logs/succeeded/4_driver-isolated-2 /some/local/dir/somehost-someuser-0/logs/
-/tmp/hive-ptest-units/TestHostExecutor/scratch/hiveptest-1_driver-parallel-1.sh /some/local/dir/somehost-someuser-0/scratch/hiveptest-1_driver-parallel-1.sh
-/tmp/hive-ptest-units/TestHostExecutor/scratch/hiveptest-2_driver-parallel-2.sh /some/local/dir/somehost-someuser-0/scratch/hiveptest-2_driver-parallel-2.sh
-/tmp/hive-ptest-units/TestHostExecutor/scratch/hiveptest-3_driver-isolated-1.sh /some/local/dir/somehost-someuser-0/scratch/hiveptest-3_driver-isolated-1.sh
-/tmp/hive-ptest-units/TestHostExecutor/scratch/hiveptest-4_driver-isolated-2.sh /some/local/dir/somehost-someuser-0/scratch/hiveptest-4_driver-isolated-2.sh
-bash /some/local/dir/somehost-someuser-0/scratch/hiveptest-1_driver-parallel-1.sh
-bash /some/local/dir/somehost-someuser-0/scratch/hiveptest-2_driver-parallel-2.sh
-bash /some/local/dir/somehost-someuser-0/scratch/hiveptest-3_driver-isolated-1.sh
-bash /some/local/dir/somehost-someuser-0/scratch/hiveptest-4_driver-isolated-2.sh
\ No newline at end of file
+/tmp/hive-ptest-units/TestHostExecutor/logs/succeeded/driver-isolated-1 /some/local/dir/somehost-someuser-0/logs/
+/tmp/hive-ptest-units/TestHostExecutor/logs/succeeded/driver-isolated-2 /some/local/dir/somehost-someuser-0/logs/
+/tmp/hive-ptest-units/TestHostExecutor/logs/succeeded/driver-parallel-1 /some/local/dir/somehost-someuser-0/logs/
+/tmp/hive-ptest-units/TestHostExecutor/logs/succeeded/driver-parallel-2 /some/local/dir/somehost-someuser-0/logs/
+/tmp/hive-ptest-units/TestHostExecutor/scratch/hiveptest-driver-isolated-1.sh /some/local/dir/somehost-someuser-0/scratch/hiveptest-driver-isolated-1.sh
+/tmp/hive-ptest-units/TestHostExecutor/scratch/hiveptest-driver-isolated-2.sh /some/local/dir/somehost-someuser-0/scratch/hiveptest-driver-isolated-2.sh
+/tmp/hive-ptest-units/TestHostExecutor/scratch/hiveptest-driver-parallel-1.sh /some/local/dir/somehost-someuser-0/scratch/hiveptest-driver-parallel-1.sh
+/tmp/hive-ptest-units/TestHostExecutor/scratch/hiveptest-driver-parallel-2.sh /some/local/dir/somehost-someuser-0/scratch/hiveptest-driver-parallel-2.sh
+bash /some/local/dir/somehost-someuser-0/scratch/hiveptest-driver-isolated-1.sh
+bash /some/local/dir/somehost-someuser-0/scratch/hiveptest-driver-isolated-2.sh
+bash /some/local/dir/somehost-someuser-0/scratch/hiveptest-driver-parallel-1.sh
+bash /some/local/dir/somehost-someuser-0/scratch/hiveptest-driver-parallel-2.sh
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/0e91e28a/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestHostExecutor.testIsolatedFailsOnExec.approved.txt
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestHostExecutor.testIsolatedFailsOnExec.approved.txt b/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestHostExecutor.testIsolatedFailsOnExec.approved.txt
index 506b2e0..2a3a4a6 100644
--- a/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestHostExecutor.testIsolatedFailsOnExec.approved.txt
+++ b/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestHostExecutor.testIsolatedFailsOnExec.approved.txt
@@ -1,5 +1,5 @@
-/tmp/hive-ptest-units/TestHostExecutor/logs/succeeded/3_driver-isolated-1 /some/local/dir/somehost-someuser-1/logs/
-/tmp/hive-ptest-units/TestHostExecutor/scratch/hiveptest-3_driver-isolated-1.sh /some/local/dir/somehost-someuser-0/scratch/hiveptest-3_driver-isolated-1.sh
-/tmp/hive-ptest-units/TestHostExecutor/scratch/hiveptest-3_driver-isolated-1.sh /some/local/dir/somehost-someuser-1/scratch/hiveptest-3_driver-isolated-1.sh
-bash /some/local/dir/somehost-someuser-0/scratch/hiveptest-3_driver-isolated-1.sh
-bash /some/local/dir/somehost-someuser-1/scratch/hiveptest-3_driver-isolated-1.sh
\ No newline at end of file
+/tmp/hive-ptest-units/TestHostExecutor/logs/succeeded/driver-isolated-1 /some/local/dir/somehost-someuser-1/logs/
+/tmp/hive-ptest-units/TestHostExecutor/scratch/hiveptest-driver-isolated-1.sh /some/local/dir/somehost-someuser-0/scratch/hiveptest-driver-isolated-1.sh
+/tmp/hive-ptest-units/TestHostExecutor/scratch/hiveptest-driver-isolated-1.sh /some/local/dir/somehost-someuser-1/scratch/hiveptest-driver-isolated-1.sh
+bash /some/local/dir/somehost-someuser-0/scratch/hiveptest-driver-isolated-1.sh
+bash /some/local/dir/somehost-someuser-1/scratch/hiveptest-driver-isolated-1.sh
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/0e91e28a/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestHostExecutor.testIsolatedFailsOnRsyncOne.approved.txt
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestHostExecutor.testIsolatedFailsOnRsyncOne.approved.txt b/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestHostExecutor.testIsolatedFailsOnRsyncOne.approved.txt
index a460175..13945bf 100644
--- a/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestHostExecutor.testIsolatedFailsOnRsyncOne.approved.txt
+++ b/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestHostExecutor.testIsolatedFailsOnRsyncOne.approved.txt
@@ -1,4 +1,4 @@
-/tmp/hive-ptest-units/TestHostExecutor/logs/succeeded/3_driver-isolated-1 /some/local/dir/somehost-someuser-1/logs/
-/tmp/hive-ptest-units/TestHostExecutor/scratch/hiveptest-3_driver-isolated-1.sh /some/local/dir/somehost-someuser-0/scratch/hiveptest-3_driver-isolated-1.sh
-/tmp/hive-ptest-units/TestHostExecutor/scratch/hiveptest-3_driver-isolated-1.sh /some/local/dir/somehost-someuser-1/scratch/hiveptest-3_driver-isolated-1.sh
-bash /some/local/dir/somehost-someuser-1/scratch/hiveptest-3_driver-isolated-1.sh
\ No newline at end of file
+/tmp/hive-ptest-units/TestHostExecutor/logs/succeeded/driver-isolated-1 /some/local/dir/somehost-someuser-1/logs/
+/tmp/hive-ptest-units/TestHostExecutor/scratch/hiveptest-driver-isolated-1.sh /some/local/dir/somehost-someuser-0/scratch/hiveptest-driver-isolated-1.sh
+/tmp/hive-ptest-units/TestHostExecutor/scratch/hiveptest-driver-isolated-1.sh /some/local/dir/somehost-someuser-1/scratch/hiveptest-driver-isolated-1.sh
+bash /some/local/dir/somehost-someuser-1/scratch/hiveptest-driver-isolated-1.sh
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/0e91e28a/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestHostExecutor.testIsolatedFailsOnRsyncUnknown.approved.txt
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestHostExecutor.testIsolatedFailsOnRsyncUnknown.approved.txt b/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestHostExecutor.testIsolatedFailsOnRsyncUnknown.approved.txt
index a460175..13945bf 100644
--- a/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestHostExecutor.testIsolatedFailsOnRsyncUnknown.approved.txt
+++ b/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestHostExecutor.testIsolatedFailsOnRsyncUnknown.approved.txt
@@ -1,4 +1,4 @@
-/tmp/hive-ptest-units/TestHostExecutor/logs/succeeded/3_driver-isolated-1 /some/local/dir/somehost-someuser-1/logs/
-/tmp/hive-ptest-units/TestHostExecutor/scratch/hiveptest-3_driver-isolated-1.sh /some/local/dir/somehost-someuser-0/scratch/hiveptest-3_driver-isolated-1.sh
-/tmp/hive-ptest-units/TestHostExecutor/scratch/hiveptest-3_driver-isolated-1.sh /some/local/dir/somehost-someuser-1/scratch/hiveptest-3_driver-isolated-1.sh
-bash /some/local/dir/somehost-someuser-1/scratch/hiveptest-3_driver-isolated-1.sh
\ No newline at end of file
+/tmp/hive-ptest-units/TestHostExecutor/logs/succeeded/driver-isolated-1 /some/local/dir/somehost-someuser-1/logs/
+/tmp/hive-ptest-units/TestHostExecutor/scratch/hiveptest-driver-isolated-1.sh /some/local/dir/somehost-someuser-0/scratch/hiveptest-driver-isolated-1.sh
+/tmp/hive-ptest-units/TestHostExecutor/scratch/hiveptest-driver-isolated-1.sh /some/local/dir/somehost-someuser-1/scratch/hiveptest-driver-isolated-1.sh
+bash /some/local/dir/somehost-someuser-1/scratch/hiveptest-driver-isolated-1.sh
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/0e91e28a/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestHostExecutor.testParallelFailsOnExec.approved.txt
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestHostExecutor.testParallelFailsOnExec.approved.txt b/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestHostExecutor.testParallelFailsOnExec.approved.txt
index 2ae7fa1..df86b02 100644
--- a/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestHostExecutor.testParallelFailsOnExec.approved.txt
+++ b/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestHostExecutor.testParallelFailsOnExec.approved.txt
@@ -1,5 +1,5 @@
-/tmp/hive-ptest-units/TestHostExecutor/logs/succeeded/1_driver-parallel-1 /some/local/dir/somehost-someuser-1/logs/
-/tmp/hive-ptest-units/TestHostExecutor/scratch/hiveptest-1_driver-parallel-1.sh /some/local/dir/somehost-someuser-0/scratch/hiveptest-1_driver-parallel-1.sh
-/tmp/hive-ptest-units/TestHostExecutor/scratch/hiveptest-1_driver-parallel-1.sh /some/local/dir/somehost-someuser-1/scratch/hiveptest-1_driver-parallel-1.sh
-bash /some/local/dir/somehost-someuser-0/scratch/hiveptest-1_driver-parallel-1.sh
-bash /some/local/dir/somehost-someuser-1/scratch/hiveptest-1_driver-parallel-1.sh
\ No newline at end of file
+/tmp/hive-ptest-units/TestHostExecutor/logs/succeeded/driver-parallel-1 /some/local/dir/somehost-someuser-1/logs/
+/tmp/hive-ptest-units/TestHostExecutor/scratch/hiveptest-driver-parallel-1.sh /some/local/dir/somehost-someuser-0/scratch/hiveptest-driver-parallel-1.sh
+/tmp/hive-ptest-units/TestHostExecutor/scratch/hiveptest-driver-parallel-1.sh /some/local/dir/somehost-someuser-1/scratch/hiveptest-driver-parallel-1.sh
+bash /some/local/dir/somehost-someuser-0/scratch/hiveptest-driver-parallel-1.sh
+bash /some/local/dir/somehost-someuser-1/scratch/hiveptest-driver-parallel-1.sh
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/0e91e28a/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestHostExecutor.testParallelFailsOnRsync.approved.txt
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestHostExecutor.testParallelFailsOnRsync.approved.txt b/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestHostExecutor.testParallelFailsOnRsync.approved.txt
index 76ab9e3..4f32a9f 100644
--- a/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestHostExecutor.testParallelFailsOnRsync.approved.txt
+++ b/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestHostExecutor.testParallelFailsOnRsync.approved.txt
@@ -1,4 +1,4 @@
-/tmp/hive-ptest-units/TestHostExecutor/logs/succeeded/1_driver-parallel-1 /some/local/dir/somehost-someuser-1/logs/
-/tmp/hive-ptest-units/TestHostExecutor/scratch/hiveptest-1_driver-parallel-1.sh /some/local/dir/somehost-someuser-0/scratch/hiveptest-1_driver-parallel-1.sh
-/tmp/hive-ptest-units/TestHostExecutor/scratch/hiveptest-1_driver-parallel-1.sh /some/local/dir/somehost-someuser-1/scratch/hiveptest-1_driver-parallel-1.sh
-bash /some/local/dir/somehost-someuser-1/scratch/hiveptest-1_driver-parallel-1.sh
\ No newline at end of file
+/tmp/hive-ptest-units/TestHostExecutor/logs/succeeded/driver-parallel-1 /some/local/dir/somehost-someuser-1/logs/
+/tmp/hive-ptest-units/TestHostExecutor/scratch/hiveptest-driver-parallel-1.sh /some/local/dir/somehost-someuser-0/scratch/hiveptest-driver-parallel-1.sh
+/tmp/hive-ptest-units/TestHostExecutor/scratch/hiveptest-driver-parallel-1.sh /some/local/dir/somehost-someuser-1/scratch/hiveptest-driver-parallel-1.sh
+bash /some/local/dir/somehost-someuser-1/scratch/hiveptest-driver-parallel-1.sh
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/0e91e28a/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testAlternativeTestJVM.approved.txt
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testAlternativeTestJVM.approved.txt b/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testAlternativeTestJVM.approved.txt
index 5318a83..092461b 100644
--- a/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testAlternativeTestJVM.approved.txt
+++ b/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testAlternativeTestJVM.approved.txt
@@ -15,7 +15,6 @@
# limitations under the License.
set -x
-date +"%Y-%m-%d %T.%3N"
umask 0022
echo $$
ps -e -o pid,pgrp,user,args
@@ -38,8 +37,6 @@ export M2_OPTS="-Xmx1g -XX:MaxPermSize=256m -Djava.io.tmpdir=/some/log/dir/tmp $
export HADOOP_ROOT_LOGGER=INFO,console
export HADOOP_OPTS="-Dhive.log.dir=/some/log/dir -Dhive.query.id=hadoop -Djava.io.tmpdir=/some/log/dir/tmp"
cd /some/local/dir/instance-1/apache-source || exit 1
-date +"%Y-%m-%d %T.%3N"
-echo "Pre test cleanup"
if [[ -s batch.pid ]]
then
while read pid
@@ -55,11 +52,10 @@ echo "$$" > batch.pid
find ./ -name 'TEST-*.xml' -delete
find ./ -name 'hive.log' -delete
find ./ -name junit_metastore_db | xargs -r rm -rf
-date +"%Y-%m-%d %T.%3N"
-echo "Pre test cleanup done"
ret=0
if [[ "ant" == "maven" ]]
then
+ testModule=$(find ./ -name 'TestCliDriver.java' | awk -F'/' '{print $2}')
if [[ -z "$testModule" ]]
then
testModule=./
@@ -67,7 +63,6 @@ then
pushd $testModule
timeout 40m mvn -B test -Dmaven.repo.local=/some/local/dir/instance-1/maven \
$mavenArgs $mavenTestArgs -Dtest=arg1 1>/some/log/dir/maven-test.txt 2>&1 </dev/null &
- date +"%Y-%m-%d %T.%3N"
pid=$!
@@ -90,7 +85,6 @@ fi
echo $pid >> batch.pid
wait $pid
ret=$?
-date +"%Y-%m-%d %T.%3N"
find ./ -type f -name hive.log -o -name spark.log -o -name derby.log | \
xargs -I {} sh -c 'f=$(basename {}); test -f /some/log/dir/$f && f=$f-$(uuidgen); mv {} /some/log/dir/$f'
find ./ -type f -name 'TEST-*.xml' | \
@@ -99,7 +93,7 @@ find ./ -path "*/spark/work" | \
xargs -I {} sh -c 'mv {} /some/log/dir/spark-log'
find ./ -type f -name 'syslog*' | \
xargs -I {} sh -c 'mkdir -p /some/log/dir/syslogs; mv {} /some/log/dir/syslogs'
-date +"%Y-%m-%d %T.%3N"
+
if [[ -f /some/log/dir/.log ]]
then