You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by br...@apache.org on 2013/09/27 17:00:31 UTC
svn commit: r1526925 - in /hive/trunk/testutils/ptest2/src:
main/java/org/apache/hive/ptest/execution/
main/java/org/apache/hive/ptest/execution/conf/
main/java/org/apache/hive/ptest/execution/context/ main/resources/
test/java/org/apache/hive/ptest/ex...
Author: brock
Date: Fri Sep 27 15:00:31 2013
New Revision: 1526925
URL: http://svn.apache.org/r1526925
Log:
HIVE-5361 - PTest2 should allow a different JVM for compilation versus execution
Added:
hive/trunk/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testAlternativeTestJVM.approved.txt
Removed:
hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/CleanupPhase.java
hive/trunk/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestCleanupPhase.java
Modified:
hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java
hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestConfiguration.java
hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/CloudExecutionContextProvider.java
hive/trunk/testutils/ptest2/src/main/resources/batch-exec.vm
hive/trunk/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.java
hive/trunk/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testBatch.approved.txt
Modified: hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java
URL: http://svn.apache.org/viewvc/hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java?rev=1526925&r1=1526924&r2=1526925&view=diff
==============================================================================
--- hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java (original)
+++ hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java Fri Sep 27 15:00:31 2013
@@ -110,6 +110,7 @@ public class PTest {
put("buildTag", buildTag).
put("logDir", logDir.getAbsolutePath()).
put("javaHome", configuration.getJavaHome()).
+ put("javaHomeForTests", configuration.getJavaHomeForTests()).
put("antEnvOpts", configuration.getAntEnvOpts());
final ImmutableMap<String, String> templateDefaults = templateDefaultsBuilder.build();
TestParser testParser = new TestParser(configuration.getContext(),
@@ -221,12 +222,13 @@ public class PTest {
}
private static final String PROPERTIES = "properties";
- private static final String REPOSITORY = "repository";
- private static final String REPOSITORY_NAME = "repositoryName";
- private static final String BRANCH = "branch";
+ private static final String REPOSITORY = TestConfiguration.REPOSITORY;
+ private static final String REPOSITORY_NAME = TestConfiguration.REPOSITORY_NAME;
+ private static final String BRANCH = TestConfiguration.BRANCH;
private static final String PATCH = "patch";
- private static final String JAVA_HOME = "javaHome";
- private static final String ANT_ENV_OPTS = "antEnvOpts";
+ private static final String JAVA_HOME = TestConfiguration.JAVA_HOME;
+ private static final String JAVA_HOME_TEST = TestConfiguration.JAVA_HOME_TEST;
+ private static final String ANT_ENV_OPTS = TestConfiguration.ANT_ENV_OPTS;
/**
* All args override properties file settings except
* for this one which is additive.
@@ -243,7 +245,8 @@ public class PTest {
options.addOption(null, BRANCH, true, "Overrides git branch in properties file");
options.addOption(null, PATCH, true, "URI to patch, either file:/// or http(s)://");
options.addOption(ANT_ARG, null, true, "Supplemntal ant arguments");
- options.addOption(null, JAVA_HOME, true, "Java Home for compiling and running tests");
+ options.addOption(null, JAVA_HOME, true, "Java Home for compiling and running tests (unless " + JAVA_HOME_TEST + " is specified)");
+ options.addOption(null, JAVA_HOME_TEST, true, "Java Home for running tests (optional)");
options.addOption(null, ANT_ENV_OPTS, true, "ANT_OPTS environemnt variable setting");
CommandLine commandLine = parser.parse(options, args);
if(!commandLine.hasOption(PROPERTIES)) {
@@ -282,6 +285,10 @@ public class PTest {
if(!javaHome.isEmpty()) {
conf.setJavaHome(javaHome);
}
+ String javaHomeForTests = Strings.nullToEmpty(commandLine.getOptionValue(JAVA_HOME_TEST)).trim();
+ if(!javaHomeForTests.isEmpty()) {
+ conf.setJavaHomeForTests(javaHomeForTests);
+ }
String antEnvOpts = Strings.nullToEmpty(commandLine.getOptionValue(ANT_ENV_OPTS)).trim();
if(!antEnvOpts.isEmpty()) {
conf.setAntEnvOpts(antEnvOpts);
Modified: hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestConfiguration.java
URL: http://svn.apache.org/viewvc/hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestConfiguration.java?rev=1526925&r1=1526924&r2=1526925&view=diff
==============================================================================
--- hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestConfiguration.java (original)
+++ hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/TestConfiguration.java Fri Sep 27 15:00:31 2013
@@ -32,19 +32,21 @@ import com.google.common.base.Strings;
import com.google.common.collect.Maps;
public class TestConfiguration {
+ public static final String REPOSITORY = "repository";
+ public static final String REPOSITORY_NAME = "repositoryName";
+ public static final String BRANCH = "branch";
+ public static final String JAVA_HOME = "javaHome";
+ public static final String JAVA_HOME_TEST = "javaHomeForTests";
+ public static final String ANT_ENV_OPTS = "antEnvOpts";
+
private static final String REPOSITORY_TYPE = "repositoryType";
private static final String GIT = "git";
private static final String SVN = "svn";
- private static final String REPOSITORY = "repository";
- private static final String REPOSITORY_NAME = "repositoryName";
- private static final String BRANCH = "branch";
private static final String ANT_ARGS = "antArgs";
- private static final String ANT_ENV_OPTS = "antEnvOpts";
- private static final String JAVA_HOME = "javaHome";
private static final String JIRA_URL = "jiraUrl";
private static final String JIRA_USER = "jiraUser";
private static final String JIRA_PASSWORD = "jiraPassword";
- public static final String JENKINS_URL = "jenkinsURL";
+ private static final String JENKINS_URL = "jenkinsURL";
private final Context context;
private String antArgs;
@@ -54,6 +56,7 @@ public class TestConfiguration {
private String repositoryName;
private String patch;
private String javaHome;
+ private String javaHomeForTests;
private String branch;
private final String jenkinsURL;
private final String jiraUrl;
@@ -79,6 +82,7 @@ public class TestConfiguration {
antArgs = Preconditions.checkNotNull(context.getString(ANT_ARGS), ANT_ARGS).trim();
antEnvOpts = context.getString(ANT_ENV_OPTS, "").trim();
javaHome = context.getString(JAVA_HOME, "").trim();
+ javaHomeForTests = context.getString(JAVA_HOME_TEST, "").trim();
patch = Strings.nullToEmpty(null);
jiraName = Strings.nullToEmpty(null);
jiraUrl = context.getString(JIRA_URL, "").trim();
@@ -132,6 +136,9 @@ public class TestConfiguration {
public String getJavaHome() {
return javaHome;
}
+ public String getJavaHomeForTests() {
+ return javaHomeForTests;
+ }
public String getPatch() {
return patch;
}
@@ -150,6 +157,9 @@ public class TestConfiguration {
public void setJavaHome(String javaHome) {
this.javaHome = Strings.nullToEmpty(javaHome);
}
+ public void setJavaHomeForTests(String javaHomeForTests) {
+ this.javaHomeForTests = javaHomeForTests;
+ }
public void setAntArgs(String antArgs) {
this.antArgs = Strings.nullToEmpty(antArgs);
}
@@ -161,10 +171,14 @@ public class TestConfiguration {
}
@Override
public String toString() {
- return "Configuration [context=" + context + ", antArgs=" + antArgs
- + ", antEnvOpts=" + antEnvOpts + ", repository=" + repository
- + ", repositoryName=" + repositoryName + ", patch=" + patch
- + ", javaHome=" + javaHome + ", branch=" + branch + "]";
+ return "TestConfiguration [antArgs=" + antArgs + ", antEnvOpts="
+ + antEnvOpts + ", repositoryType=" + repositoryType + ", repository="
+ + repository + ", repositoryName=" + repositoryName + ", patch="
+ + patch + ", javaHome=" + javaHome + ", javaHomeForTests="
+ + javaHomeForTests + ", branch=" + branch + ", jenkinsURL="
+ + jenkinsURL + ", jiraUrl=" + jiraUrl + ", jiraUser=" + jiraUser
+ + ", jiraName=" + jiraName + ", clearLibraryCache=" + clearLibraryCache
+ + "]";
}
public static TestConfiguration fromInputStream(InputStream inputStream, Logger logger)
throws IOException {
Modified: hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/CloudExecutionContextProvider.java
URL: http://svn.apache.org/viewvc/hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/CloudExecutionContextProvider.java?rev=1526925&r1=1526924&r2=1526925&view=diff
==============================================================================
--- hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/CloudExecutionContextProvider.java (original)
+++ hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/CloudExecutionContextProvider.java Fri Sep 27 15:00:31 2013
@@ -106,7 +106,7 @@ public class CloudExecutionContextProvid
return size() > 100;
}
});
- mTerminationExecutor = Executors.newCachedThreadPool();
+ mTerminationExecutor = Executors.newSingleThreadExecutor();
mHostLog = new RandomAccessFile(new File(dataDir, "hosts"), "rw");
initialize();
}
Modified: hive/trunk/testutils/ptest2/src/main/resources/batch-exec.vm
URL: http://svn.apache.org/viewvc/hive/trunk/testutils/ptest2/src/main/resources/batch-exec.vm?rev=1526925&r1=1526924&r2=1526925&view=diff
==============================================================================
--- hive/trunk/testutils/ptest2/src/main/resources/batch-exec.vm (original)
+++ hive/trunk/testutils/ptest2/src/main/resources/batch-exec.vm Fri Sep 27 15:00:31 2013
@@ -23,7 +23,11 @@ chmod -R u+w $logDir
rm -rf $logDir
# makes $logDir and $logDir/tmp
mkdir -p $logDir/tmp
-if [[ -n "${javaHome}" ]]
+if [[ -n "${javaHomeForTests}" ]]
+then
+ export JAVA_HOME=$javaHomeForTests
+ export PATH=$JAVA_HOME/bin/:$PATH
+elif [[ -n "${javaHome}" ]]
then
export JAVA_HOME=$javaHome
export PATH=$JAVA_HOME/bin/:$PATH
Modified: hive/trunk/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.java
URL: http://svn.apache.org/viewvc/hive/trunk/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.java?rev=1526925&r1=1526924&r2=1526925&view=diff
==============================================================================
--- hive/trunk/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.java (original)
+++ hive/trunk/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.java Fri Sep 27 15:00:31 2013
@@ -78,6 +78,30 @@ public class TestScripts {
Approvals.verify(actual);
}
@Test
+ public void testAlternativeTestJVM() throws Throwable {
+ Map<String, String> templateVariables = Maps.newHashMap();
+ templateVariables.put("repository", "git:///repo1");
+ templateVariables.put("repositoryName", "apache");
+ templateVariables.put("branch", "branch-1");
+ templateVariables.put("localDir", "/some/local/dir");
+ templateVariables.put("workingDir", "/some/working/dir");
+ templateVariables.put("antArgs", "-Dant=arg1");
+ templateVariables.put("buildTag", "build-1");
+ templateVariables.put("logDir", "/some/log/dir");
+ templateVariables.put("instanceName", "instance-1");
+ templateVariables.put("batchName","batch-1");
+ templateVariables.put("numOfFailedTests", "20");
+ templateVariables.put("maxSourceDirs", String.valueOf(5));
+ templateVariables.put("testArguments", "-Dtest=arg1");
+ templateVariables.put("clearLibraryCache", "true");
+ templateVariables.put("javaHome", "/usr/java/jdk1.7");
+ templateVariables.put("javaHomeForTests", "/usr/java/jdk1.7-other");
+ templateVariables.put("antEnvOpts", "-Dhttp.proxyHost=somehost -Dhttp.proxyPort=3128");
+ String template = readResource("batch-exec.vm");
+ String actual = getTemplateResult(template, templateVariables);
+ Approvals.verify(actual);
+ }
+ @Test
public void testPrepNone() throws Throwable {
Map<String, String> templateVariables = Maps.newHashMap();
templateVariables.put("repository", "git:///repo1");
Added: hive/trunk/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testAlternativeTestJVM.approved.txt
URL: http://svn.apache.org/viewvc/hive/trunk/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testAlternativeTestJVM.approved.txt?rev=1526925&view=auto
==============================================================================
--- hive/trunk/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testAlternativeTestJVM.approved.txt (added)
+++ hive/trunk/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testAlternativeTestJVM.approved.txt Fri Sep 27 15:00:31 2013
@@ -0,0 +1,69 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -x
+umask 0022
+find /some/local/dir/instance-1/apache-source/build -name 'TEST-*.xml' -delete
+find /some/local/dir/instance-1/apache-source/build -name 'hive.log' -delete
+chmod -R u+w /some/log/dir
+rm -rf /some/log/dir
+# makes /some/log/dir and /some/log/dir/tmp
+mkdir -p /some/log/dir/tmp
+if [[ -n "/usr/java/jdk1.7-other" ]]
+then
+ export JAVA_HOME=/usr/java/jdk1.7-other
+ export PATH=$JAVA_HOME/bin/:$PATH
+elif [[ -n "/usr/java/jdk1.7" ]]
+then
+ export JAVA_HOME=/usr/java/jdk1.7
+ export PATH=$JAVA_HOME/bin/:$PATH
+fi
+export ANT_OPTS="-Xmx1g -XX:MaxPermSize=256m -Djava.io.tmpdir=/some/log/dir/tmp -Dhttp.proxyHost=somehost -Dhttp.proxyPort=3128"
+export HADOOP_ROOT_LOGGER=INFO,console
+export HADOOP_OPTS="-Dhive.log.dir=/some/log/dir -Dhive.query.id=hadoop -Djava.io.tmpdir=/some/log/dir/tmp"
+cd /some/local/dir/instance-1/apache-source && \
+ timeout 2h ant test -Dtest.junit.output.format=xml -Dant=arg1 \
+ -Divy.default.ivy.user.dir=/some/local/dir/instance-1/ivy \
+ -Divy.default.always.check.exact.revision=false \
+ -Divy.cache.ttl.default=eternal -Divy.checkmodified=false \
+ -Dmvn.local.repo=/some/local/dir/instance-1/maven \
+ -Dant=arg1 -Dtest=arg1 1>/some/log/dir/ant-test.txt 2>&1
+ret=$?
+if [[ $ret -ne 0 ]]
+then
+ if [[ 20 -lt 5 ]]
+ then
+ cp -R /some/local/dir/instance-1/apache-source /some/log/dir/source
+ else
+ echo "Number of failed tests 20 exceeded threshold, not copying source"
+ fi
+fi
+if [[ -f /some/local/dir/instance-1/apache-source/build/ql/tmp/hive.log ]]
+then
+ mv /some/local/dir/instance-1/apache-source/build/ql/tmp/hive.log /some/log/dir/hive.log
+fi
+batchName=batch-1
+for testOutputFile in $(find /some/local/dir/instance-1/apache-source/ -name 'TEST-*.xml')
+do
+ mv "$testOutputFile" "/some/log/dir/TEST-$batchName-$(basename $testOutputFile)"
+done
+if [[ -f /some/log/dir/.log ]]
+then
+ mv /some/log/dir/.log /some/log/dir/dot.log
+else
+ echo "/some/log/dir/.log does not exist"
+fi
+exit $ret
Modified: hive/trunk/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testBatch.approved.txt
URL: http://svn.apache.org/viewvc/hive/trunk/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testBatch.approved.txt?rev=1526925&r1=1526924&r2=1526925&view=diff
==============================================================================
--- hive/trunk/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testBatch.approved.txt (original)
+++ hive/trunk/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testBatch.approved.txt Fri Sep 27 15:00:31 2013
@@ -22,7 +22,11 @@ chmod -R u+w /some/log/dir
rm -rf /some/log/dir
# makes /some/log/dir and /some/log/dir/tmp
mkdir -p /some/log/dir/tmp
-if [[ -n "/usr/java/jdk1.7" ]]
+if [[ -n "${javaHomeForTests}" ]]
+then
+ export JAVA_HOME=$javaHomeForTests
+ export PATH=$JAVA_HOME/bin/:$PATH
+elif [[ -n "/usr/java/jdk1.7" ]]
then
export JAVA_HOME=/usr/java/jdk1.7
export PATH=$JAVA_HOME/bin/:$PATH