You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by gu...@apache.org on 2013/11/05 08:01:58 UTC
svn commit: r1538880 [46/46] - in /hive/branches/tez: ./ ant/
ant/src/org/apache/hadoop/hive/ant/ beeline/
beeline/src/java/org/apache/hive/beeline/ beeline/src/main/
beeline/src/test/org/apache/hive/beeline/src/test/ cli/ common/
common/src/java/conf/...
Modified: hive/branches/tez/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java?rev=1538880&r1=1538879&r2=1538880&view=diff
==============================================================================
--- hive/branches/tez/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java (original)
+++ hive/branches/tez/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java Tue Nov 5 07:01:32 2013
@@ -77,6 +77,8 @@ public class PTest {
private final Logger mLogger;
private final List<HostExecutor> mHostExecutors;
private final String mBuildTag;
+ private final SSHCommandExecutor mSshCommandExecutor;
+ private final RSyncCommandExecutor mRsyncCommandExecutor;
public PTest(final TestConfiguration configuration, final ExecutionContext executionContext,
final String buildTag, final File logDir, final LocalCommandFactory localCommandFactory,
@@ -88,6 +90,8 @@ public class PTest {
mExecutedTests = Collections.newSetFromMap(new ConcurrentHashMap<String, Boolean>());
mFailedTests = Collections.newSetFromMap(new ConcurrentHashMap<String, Boolean>());
mExecutionContext = executionContext;
+ mSshCommandExecutor = sshCommandExecutor;
+ mRsyncCommandExecutor = rsyncCommandExecutor;
mExecutor = MoreExecutors.listeningDecorator(Executors.newCachedThreadPool());
final File failedLogDir = Dirs.create(new File(logDir, "failed"));
final File succeededLogDir = Dirs.create(new File(logDir, "succeeded"));
@@ -171,10 +175,13 @@ public class PTest {
error = true;
} finally {
for(HostExecutor hostExecutor : mHostExecutors) {
+ hostExecutor.shutdownNow();
if(hostExecutor.isBad()) {
mExecutionContext.addBadHost(hostExecutor.getHost());
}
}
+ mSshCommandExecutor.shutdownNow();
+ mRsyncCommandExecutor.shutdownNow();
mExecutor.shutdownNow();
SortedSet<String> failedTests = new TreeSet<String>(mFailedTests);
if(failedTests.isEmpty()) {
Modified: hive/branches/tez/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ssh/RSyncCommandExecutor.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ssh/RSyncCommandExecutor.java?rev=1538880&r1=1538879&r2=1538880&view=diff
==============================================================================
--- hive/branches/tez/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ssh/RSyncCommandExecutor.java (original)
+++ hive/branches/tez/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ssh/RSyncCommandExecutor.java Tue Nov 5 07:01:32 2013
@@ -22,7 +22,9 @@ import java.io.IOException;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
+import org.apache.hive.ptest.execution.Constants;
import org.apache.hive.ptest.execution.LocalCommand;
+import org.apache.hive.ptest.execution.LocalCommandFactory;
import org.apache.hive.ptest.execution.LocalCommand.CollectPolicy;
import org.slf4j.Logger;
@@ -30,12 +32,20 @@ import org.slf4j.Logger;
public class RSyncCommandExecutor {
private final Logger mLogger;
+ private final LocalCommandFactory mLocalCommandFactory;
private final Semaphore mSemaphore;
+ private volatile boolean mShutdown;
- public RSyncCommandExecutor(Logger logger) {
+ public RSyncCommandExecutor(Logger logger, LocalCommandFactory localCommandFactory) {
mLogger = logger;
- mSemaphore = new Semaphore(5);
+ mLocalCommandFactory = localCommandFactory;
+ mSemaphore = new Semaphore(Math.min(Runtime.getRuntime().availableProcessors() * 5, 10));
+ mShutdown = false;
+ }
+ public RSyncCommandExecutor(Logger logger) {
+ this(logger, new LocalCommandFactory(logger));
}
+
/**
* Execute the given RSync. If the command exits with a non-zero
* exit status the command will be retried up to three times.
@@ -52,25 +62,31 @@ public class RSyncCommandExecutor {
do {
retry = false;
if(command.getType() == RSyncCommand.Type.TO_LOCAL) {
- cmd = new LocalCommand(mLogger, collector,
+ cmd = mLocalCommandFactory.create(collector,
String.format("timeout 1h rsync -vaPe \"ssh -i %s\" --timeout 600 %s@%s:%s %s",
command.getPrivateKey(), command.getUser(), command.getHost(),
command.getRemoteFile(), command.getLocalFile()));
} else if(command.getType() == RSyncCommand.Type.FROM_LOCAL) {
- cmd = new LocalCommand(mLogger, collector,
+ cmd = mLocalCommandFactory.create(collector,
String.format("timeout 1h rsync -vaPe \"ssh -i %s\" --timeout 600 --delete --delete-during --force %s %s@%s:%s",
command.getPrivateKey(), command.getLocalFile(), command.getUser(), command.getHost(),
command.getRemoteFile()));
} else {
throw new UnsupportedOperationException(String.valueOf(command.getType()));
}
+ if(mShutdown) {
+ mLogger.warn("Shutting down command " + command);
+ cmd.kill();
+ command.setExitCode(Constants.EXIT_CODE_UNKNOWN);
+ return;
+ }
// 12 is timeout and 255 is unspecified error
if(attempts++ <= 3 && cmd.getExitCode() != 0) {
mLogger.warn("Command exited with " + cmd.getExitCode() + ", will retry: " + command);
retry = true;
TimeUnit.SECONDS.sleep(20);
}
- } while (retry); // an error occurred, re-try
+ } while (!mShutdown && retry); // an error occurred, re-try
command.setExitCode(cmd.getExitCode());
} catch (IOException e) {
command.setException(e);
@@ -83,4 +99,10 @@ public class RSyncCommandExecutor {
command.setOutput(collector.getOutput());
}
}
-}
+ boolean isShutdown() {
+ return mShutdown;
+ }
+ public void shutdownNow() {
+ this.mShutdown = true;
+ }
+}
\ No newline at end of file
Modified: hive/branches/tez/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ssh/SSHCommandExecutor.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ssh/SSHCommandExecutor.java?rev=1538880&r1=1538879&r2=1538880&view=diff
==============================================================================
--- hive/branches/tez/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ssh/SSHCommandExecutor.java (original)
+++ hive/branches/tez/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ssh/SSHCommandExecutor.java Tue Nov 5 07:01:32 2013
@@ -23,14 +23,22 @@ import java.util.concurrent.TimeUnit;
import org.apache.hive.ptest.execution.Constants;
import org.apache.hive.ptest.execution.LocalCommand;
import org.apache.hive.ptest.execution.LocalCommand.CollectPolicy;
+import org.apache.hive.ptest.execution.LocalCommandFactory;
import org.slf4j.Logger;
public class SSHCommandExecutor {
private final Logger mLogger;
-
- public SSHCommandExecutor(Logger logger) {
+ private final LocalCommandFactory mLocalCommandFactory;
+ private volatile boolean mShutdown;
+
+ public SSHCommandExecutor(Logger logger, LocalCommandFactory localCommandFactory) {
mLogger = logger;
+ mShutdown = false;
+ mLocalCommandFactory = localCommandFactory;
+ }
+ public SSHCommandExecutor(Logger logger) {
+ this(logger, new LocalCommandFactory(logger));
}
/**
* Execute the given command via the ssh command line tool. If the command
@@ -46,13 +54,19 @@ public class SSHCommandExecutor {
LocalCommand cmd;
do {
retry = false;
- cmd = new LocalCommand(mLogger, collector, commandText);
+ cmd = mLocalCommandFactory.create(collector, commandText);
+ if(mShutdown) {
+ mLogger.warn("Shutting down command " + command);
+ cmd.kill();
+ command.setExitCode(Constants.EXIT_CODE_UNKNOWN);
+ return;
+ }
if(attempts++ <= 3 && cmd.getExitCode() == Constants.EXIT_CODE_UNKNOWN) {
mLogger.warn("Command exited with " + cmd.getExitCode() + ", will retry: " + command);
retry = true;
TimeUnit.SECONDS.sleep(5);
}
- } while (retry); // an error occurred, re-try
+ } while (!mShutdown && retry); // an error occurred, re-try
command.setExitCode(cmd.getExitCode());
} catch (Exception e) {
if(command.getExitCode() == Constants.EXIT_CODE_SUCCESS) {
@@ -63,4 +77,10 @@ public class SSHCommandExecutor {
command.setOutput(collector.getOutput());
}
}
-}
+ boolean isShutdown() {
+ return mShutdown;
+ }
+ public void shutdownNow() {
+ this.mShutdown = true;
+ }
+}
\ No newline at end of file
Modified: hive/branches/tez/testutils/ptest2/src/main/resources/log4j.properties
URL: http://svn.apache.org/viewvc/hive/branches/tez/testutils/ptest2/src/main/resources/log4j.properties?rev=1538880&r1=1538879&r2=1538880&view=diff
==============================================================================
--- hive/branches/tez/testutils/ptest2/src/main/resources/log4j.properties (original)
+++ hive/branches/tez/testutils/ptest2/src/main/resources/log4j.properties Tue Nov 5 07:01:32 2013
@@ -31,3 +31,7 @@ log4j.logger.org.jclouds=INFO
log4j.logger.jclouds=INFO
log4j.logger.org.apache.hive=DEBUG
log4j.logger.org.apache.http=TRACE
+
+# Silence useless ZK logs
+log4j.logger.org.apache.zookeeper.server.NIOServerCnxn=WARN
+log4j.logger.org.apache.zookeeper.ClientCnxnSocketNIO=WARN
Modified: hive/branches/tez/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestHostExecutor.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestHostExecutor.java?rev=1538880&r1=1538879&r2=1538880&view=diff
==============================================================================
--- hive/branches/tez/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestHostExecutor.java (original)
+++ hive/branches/tez/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestHostExecutor.java Tue Nov 5 07:01:32 2013
@@ -193,6 +193,20 @@ public class TestHostExecutor {
Approvals.verify(getExecutedCommands());
}
@Test
+ public void testShutdownBeforeExec()
+ throws Exception {
+ rsyncCommandExecutor.putFailure("/tmp/hive-ptest-units/TestHostExecutor/scratch/hiveptest-driver-parallel-1.sh "
+ + "/some/local/dir/somehost-someuser-0/scratch/hiveptest-driver-parallel-1.sh", Constants.EXIT_CODE_UNKNOWN);
+ HostExecutor executor = createHostExecutor();
+ parallelWorkQueue.addAll(Lists.newArrayList(testBatchParallel1));
+ executor.shutdownNow();
+ executor.submitTests(parallelWorkQueue, isolatedWorkQueue, failedTestResults).get();
+ Assert.assertEquals(Collections.emptySet(), failedTestResults);
+ Assert.assertEquals(parallelWorkQueue.toString(), 1, parallelWorkQueue.size());
+ Approvals.verify(getExecutedCommands());
+ Assert.assertTrue(executor.isShutdown());
+ }
+ @Test
public void testIsolatedFailsOnRsyncUnknown()
throws Exception {
rsyncCommandExecutor.putFailure("/tmp/hive-ptest-units/TestHostExecutor/scratch/hiveptest-driver-isolated-1.sh "+
@@ -201,7 +215,7 @@ public class TestHostExecutor {
isolatedWorkQueue.addAll(Lists.newArrayList(testBatchIsolated1));
executor.submitTests(parallelWorkQueue, isolatedWorkQueue, failedTestResults).get();
Assert.assertEquals(Collections.emptySet(), failedTestResults);
- Assert.assertTrue(isolatedWorkQueue.toString(), parallelWorkQueue.isEmpty());
+ Assert.assertTrue(isolatedWorkQueue.toString(), isolatedWorkQueue.isEmpty());
Approvals.verify(getExecutedCommands());
}
@Test
Modified: hive/branches/tez/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestJIRAService.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestJIRAService.java?rev=1538880&r1=1538879&r2=1538880&view=diff
==============================================================================
--- hive/branches/tez/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestJIRAService.java (original)
+++ hive/branches/tez/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestJIRAService.java Tue Nov 5 07:01:32 2013
@@ -18,9 +18,10 @@
*/
package org.apache.hive.ptest.execution;
+import java.util.List;
import junit.framework.Assert;
-
import org.junit.Test;
+import com.google.common.collect.Lists;
public class TestJIRAService {
@@ -41,4 +42,30 @@ public class TestJIRAService {
public void testFormatBuildTagNoDashNone() throws Throwable {
JIRAService.formatBuildTag("abc123");
}
+ @Test
+ public void testTrimMesssagesBoundry() {
+ List<String> messages = Lists.newArrayList();
+ Assert.assertEquals(messages, JIRAService.trimMessages(messages));
+ messages.clear();
+ for (int i = 0; i < JIRAService.MAX_MESSAGES; i++) {
+ messages.add(String.valueOf(i));
+ }
+ Assert.assertEquals(messages, JIRAService.trimMessages(messages));
+ }
+ @Test
+ public void testTrimMesssagesNotTrimmed() {
+ List<String> messages = Lists.newArrayList("a", "b", "c");
+ Assert.assertEquals(messages, JIRAService.trimMessages(messages));
+ }
+ @Test
+ public void testTrimMesssagesTrimmed() {
+ List<String> messages = Lists.newArrayList();
+ for (int i = 0; i < JIRAService.MAX_MESSAGES + 1; i++) {
+ messages.add(String.valueOf(i));
+ }
+ List<String> expected = Lists.newArrayList(messages);
+ expected.remove(0);
+ expected.add(0, JIRAService.TRIMMED_MESSAGE);
+ Assert.assertEquals(expected, JIRAService.trimMessages(messages));
+ }
}
\ No newline at end of file
Modified: hive/branches/tez/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testAlternativeTestJVM.approved.txt
URL: http://svn.apache.org/viewvc/hive/branches/tez/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testAlternativeTestJVM.approved.txt?rev=1538880&r1=1538879&r2=1538880&view=diff
==============================================================================
--- hive/branches/tez/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testAlternativeTestJVM.approved.txt (original)
+++ hive/branches/tez/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testAlternativeTestJVM.approved.txt Tue Nov 5 07:01:32 2013
@@ -61,7 +61,7 @@ then
testModule=./
fi
pushd $testModule
- timeout 2h mvn -o test -Dmaven.repo.local=/some/local/dir/instance-1/maven \
+ timeout 2h mvn -B -o test -Dmaven.repo.local=/some/local/dir/instance-1/maven \
$mavenArgs $mavenTestArgs -Dtest=arg1 1>/some/log/dir/maven-test.txt 2>&1 </dev/null &
pid=$!
Modified: hive/branches/tez/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testBatch.approved.txt
URL: http://svn.apache.org/viewvc/hive/branches/tez/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testBatch.approved.txt?rev=1538880&r1=1538879&r2=1538880&view=diff
==============================================================================
--- hive/branches/tez/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testBatch.approved.txt (original)
+++ hive/branches/tez/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testBatch.approved.txt Tue Nov 5 07:01:32 2013
@@ -61,7 +61,7 @@ then
testModule=./
fi
pushd $testModule
- timeout 2h mvn -o test -Dmaven.repo.local=/some/local/dir/instance-1/maven \
+ timeout 2h mvn -B -o test -Dmaven.repo.local=/some/local/dir/instance-1/maven \
$mavenArgs $mavenTestArgs -Dtest=arg1 1>/some/log/dir/maven-test.txt 2>&1 </dev/null &
pid=$!
Modified: hive/branches/tez/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testPrepGit.approved.txt
URL: http://svn.apache.org/viewvc/hive/branches/tez/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testPrepGit.approved.txt?rev=1538880&r1=1538879&r2=1538880&view=diff
==============================================================================
--- hive/branches/tez/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testPrepGit.approved.txt (original)
+++ hive/branches/tez/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testPrepGit.approved.txt Tue Nov 5 07:01:32 2013
@@ -84,11 +84,11 @@ cd /some/working/dir/
if [[ "${buildTool}" == "maven" ]]
then
rm -rf /some/working/dir/maven/org/apache/hive
- mvn clean install -DskipTests -Dmaven.repo.local=/some/working/dir/maven
- mvn test -Dmaven.repo.local=/some/working/dir/maven -Dtest=nothing
+ mvn -B clean install -DskipTests -Dmaven.repo.local=/some/working/dir/maven
+ mvn -B test -Dmaven.repo.local=/some/working/dir/maven -Dtest=TestDummy
cd itests
- mvn clean install -DskipTests -Dmaven.repo.local=/some/working/dir/maven
- mvn test -Dmaven.repo.local=/some/working/dir/maven -Dtest=nothing
+ mvn -B clean install -DskipTests -Dmaven.repo.local=/some/working/dir/maven
+ mvn -B test -Dmaven.repo.local=/some/working/dir/maven -Dtest=TestDummy
elif [[ "${buildTool}" == "ant" ]]
then
ant -Dant=arg1 -Divy.default.ivy.user.dir=/some/working/dir/ivy \
Modified: hive/branches/tez/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testPrepNone.approved.txt
URL: http://svn.apache.org/viewvc/hive/branches/tez/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testPrepNone.approved.txt?rev=1538880&r1=1538879&r2=1538880&view=diff
==============================================================================
--- hive/branches/tez/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testPrepNone.approved.txt (original)
+++ hive/branches/tez/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testPrepNone.approved.txt Tue Nov 5 07:01:32 2013
@@ -84,11 +84,11 @@ cd /some/working/dir/
if [[ "ant" == "maven" ]]
then
rm -rf /some/working/dir/maven/org/apache/hive
- mvn clean install -DskipTests -Dmaven.repo.local=/some/working/dir/maven
- mvn test -Dmaven.repo.local=/some/working/dir/maven -Dtest=nothing
+ mvn -B clean install -DskipTests -Dmaven.repo.local=/some/working/dir/maven
+ mvn -B test -Dmaven.repo.local=/some/working/dir/maven -Dtest=TestDummy
cd itests
- mvn clean install -DskipTests -Dmaven.repo.local=/some/working/dir/maven
- mvn test -Dmaven.repo.local=/some/working/dir/maven -Dtest=nothing
+ mvn -B clean install -DskipTests -Dmaven.repo.local=/some/working/dir/maven
+ mvn -B test -Dmaven.repo.local=/some/working/dir/maven -Dtest=TestDummy
elif [[ "ant" == "ant" ]]
then
ant -Dant=arg1 -Divy.default.ivy.user.dir=/some/working/dir/ivy \
Modified: hive/branches/tez/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testPrepSvn.approved.txt
URL: http://svn.apache.org/viewvc/hive/branches/tez/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testPrepSvn.approved.txt?rev=1538880&r1=1538879&r2=1538880&view=diff
==============================================================================
--- hive/branches/tez/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testPrepSvn.approved.txt (original)
+++ hive/branches/tez/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testPrepSvn.approved.txt Tue Nov 5 07:01:32 2013
@@ -84,11 +84,11 @@ cd /some/working/dir/
if [[ "maven" == "maven" ]]
then
rm -rf /some/working/dir/maven/org/apache/hive
- mvn clean install -DskipTests -Dmaven.repo.local=/some/working/dir/maven
- mvn test -Dmaven.repo.local=/some/working/dir/maven -Dtest=nothing
+ mvn -B clean install -DskipTests -Dmaven.repo.local=/some/working/dir/maven
+ mvn -B test -Dmaven.repo.local=/some/working/dir/maven -Dtest=TestDummy
cd itests
- mvn clean install -DskipTests -Dmaven.repo.local=/some/working/dir/maven
- mvn test -Dmaven.repo.local=/some/working/dir/maven -Dtest=nothing
+ mvn -B clean install -DskipTests -Dmaven.repo.local=/some/working/dir/maven
+ mvn -B test -Dmaven.repo.local=/some/working/dir/maven -Dtest=TestDummy
elif [[ "maven" == "ant" ]]
then
ant -Dant=arg1 -Divy.default.ivy.user.dir=/some/working/dir/ivy \