You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jd...@apache.org on 2016/04/20 22:05:46 UTC
[29/30] hive git commit: Revert "HIVE-13507. Improved logging for
ptest. (Siddharth Seth, reviewed by Thejas M Nair)"
Revert "HIVE-13507. Improved logging for ptest. (Siddharth Seth, reviewed by Thejas M Nair)"
This patch is causing Hive ptest to stay in a loop creating nodes due to an error exception.
This reverts commit fcc2e79511bb1a0db70f4a17ede6ade5e73b1f42.
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/9b5eb45b
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/9b5eb45b
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/9b5eb45b
Branch: refs/heads/llap
Commit: 9b5eb45b83d29adf0eb83f65b8821384d9254985
Parents: 6c3d3b4
Author: Sergio Pena <se...@cloudera.com>
Authored: Wed Apr 20 13:36:31 2016 -0500
Committer: Sergio Pena <se...@cloudera.com>
Committed: Wed Apr 20 13:36:31 2016 -0500
----------------------------------------------------------------------
.../hive/ptest/execution/ExecutionPhase.java | 2 -
.../hive/ptest/execution/HostExecutor.java | 48 ++------------------
.../hive/ptest/execution/LocalCommand.java | 18 +-------
.../apache/hive/ptest/execution/PrepPhase.java | 1 -
.../apache/hive/ptest/execution/conf/Host.java | 3 --
5 files changed, 7 insertions(+), 65 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/9b5eb45b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ExecutionPhase.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ExecutionPhase.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ExecutionPhase.java
index 6063afc..3026ea0 100644
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ExecutionPhase.java
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ExecutionPhase.java
@@ -86,8 +86,6 @@ public class ExecutionPhase extends Phase {
isolatedWorkQueue.add(batch);
}
}
- logger.info("ParallelWorkQueueSize={}, IsolatedWorkQueueSize={}", parallelWorkQueue.size(),
- isolatedWorkQueue.size());
try {
int expectedNumHosts = hostExecutors.size();
initalizeHosts();
http://git-wip-us.apache.org/repos/asf/hive/blob/9b5eb45b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/HostExecutor.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/HostExecutor.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/HostExecutor.java
index 735b261..b05d2c2 100644
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/HostExecutor.java
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/HostExecutor.java
@@ -29,7 +29,6 @@ import java.util.concurrent.Callable;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.TimeUnit;
-import com.google.common.base.Stopwatch;
import org.apache.hive.ptest.execution.conf.Host;
import org.apache.hive.ptest.execution.conf.TestBatch;
import org.apache.hive.ptest.execution.ssh.RSyncCommand;
@@ -66,8 +65,6 @@ class HostExecutor {
private final File mFailedTestLogDir;
private final long mNumPollSeconds;
private volatile boolean mShutdown;
- private int numParallelBatchesProcessed = 0;
- private int numIsolatedBatchesProcessed = 0;
HostExecutor(Host host, String privateKey, ListeningExecutorService executor,
SSHCommandExecutor sshCommandExecutor,
@@ -103,18 +100,7 @@ class HostExecutor {
return mExecutor.submit(new Callable<Void>() {
@Override
public Void call() throws Exception {
- Stopwatch stopwatch = Stopwatch.createStarted();
- mLogger.info("Starting SubmitTests on host {}", getHost());
- try {
- executeTests(parallelWorkQueue, isolatedWorkQueue, failedTestResults);
- } finally {
- stopwatch.stop();
- mLogger.info("Finishing submitTests on host: {}. ElapsedTime(seconds)={}," +
- " NumParallelBatchesProcessed={}, NumIsolatedBatchesProcessed={}",
- new Object[]{getHost().toString(),
- stopwatch.elapsed(TimeUnit.SECONDS), numParallelBatchesProcessed,
- numIsolatedBatchesProcessed});
- }
+ executeTests(parallelWorkQueue, isolatedWorkQueue, failedTestResults);
return null;
}
@@ -157,7 +143,6 @@ class HostExecutor {
@Override
public Void call() throws Exception {
TestBatch batch = null;
- Stopwatch sw = Stopwatch.createUnstarted();
try {
do {
batch = parallelWorkQueue.poll(mNumPollSeconds, TimeUnit.SECONDS);
@@ -166,16 +151,8 @@ class HostExecutor {
return null;
}
if(batch != null) {
- numParallelBatchesProcessed++;
- sw.reset().start();
- try {
- if (!executeTestBatch(drone, batch, failedTestResults)) {
- failedTestResults.add(batch);
- }
- } finally {
- sw.stop();
- mLogger.info("Finished processing parallel batch [{}] on host {}. ElapsedTime(seconds)={}",
- new Object[]{batch.getName(), getHost().toShortString(), sw.elapsed(TimeUnit.SECONDS)});
+ if(!executeTestBatch(drone, batch, failedTestResults)) {
+ failedTestResults.add(batch);
}
}
} while(!mShutdown && !parallelWorkQueue.isEmpty());
@@ -199,22 +176,12 @@ class HostExecutor {
mLogger.info("Starting isolated execution on " + mHost.getName());
for(Drone drone : ImmutableList.copyOf(mDrones)) {
TestBatch batch = null;
- Stopwatch sw = Stopwatch.createUnstarted();
try {
do {
-
batch = isolatedWorkQueue.poll(mNumPollSeconds, TimeUnit.SECONDS);
if(batch != null) {
- numIsolatedBatchesProcessed++;
- sw.reset().start();
- try {
- if (!executeTestBatch(drone, batch, failedTestResults)) {
- failedTestResults.add(batch);
- }
- } finally {
- sw.stop();
- mLogger.info("Finished processing isolated batch [{}] on host {}. ElapsedTime(seconds)={}",
- new Object[]{batch.getName(), getHost().toShortString(), sw.elapsed(TimeUnit.SECONDS)});
+ if(!executeTestBatch(drone, batch, failedTestResults)) {
+ failedTestResults.add(batch);
}
}
} while(!mShutdown && !isolatedWorkQueue.isEmpty());
@@ -248,15 +215,10 @@ class HostExecutor {
Templates.writeTemplateResult("batch-exec.vm", script, templateVariables);
copyToDroneFromLocal(drone, script.getAbsolutePath(), "$localDir/$instanceName/scratch/" + scriptName);
script.delete();
- Stopwatch sw = Stopwatch.createStarted();
mLogger.info(drone + " executing " + batch + " with " + command);
RemoteCommandResult sshResult = new SSHCommand(mSSHCommandExecutor, drone.getPrivateKey(), drone.getUser(),
drone.getHost(), drone.getInstance(), command, true).
call();
- sw.stop();
- mLogger.info("Completed executing tests for batch [{}] on host {}. ElapsedTime(seconds)={}",
- new Object[]{batch.getName(),
- getHost().toShortString(), sw.elapsed(TimeUnit.SECONDS)});
File batchLogDir = null;
if(sshResult.getExitCode() == Constants.EXIT_CODE_UNKNOWN) {
throw new AbortDroneException("Drone " + drone.toString() + " exited with " +
http://git-wip-us.apache.org/repos/asf/hive/blob/9b5eb45b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/LocalCommand.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/LocalCommand.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/LocalCommand.java
index 8e2c5c7..ec99656 100644
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/LocalCommand.java
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/LocalCommand.java
@@ -22,28 +22,17 @@ import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-import com.google.common.base.Stopwatch;
import org.slf4j.Logger;
public class LocalCommand {
- private static final AtomicInteger localCommandCounter = new AtomicInteger(0);
-
- private final Logger logger;
private final Process process;
private final StreamReader streamReader;
private Integer exitCode;
- private final int commandId;
- private final Stopwatch stopwatch = Stopwatch.createUnstarted();
public LocalCommand(Logger logger, OutputPolicy outputPolicy, String command) throws IOException {
- this.commandId = localCommandCounter.incrementAndGet();
- this.logger = logger;
- logger.info("Starting LocalCommandId={}: {}" + commandId, command);
- stopwatch.start();
+ logger.info("Starting " + command);
process = new ProcessBuilder().command(new String[] {"bash", "-c", command}).redirectErrorStream(true).start();
streamReader = new StreamReader(outputPolicy, process.getInputStream());
streamReader.setName("StreamReader-[" + command + "]");
@@ -56,13 +45,10 @@ public class LocalCommand {
if(exitCode == null) {
exitCode = process.waitFor();
}
- stopwatch.stop();
- logger.info("Finished LocalCommandId={}. ElapsedTime(seconds)={}", commandId, stopwatch.elapsed(
- TimeUnit.SECONDS));
return exitCode;
}
}
-
+
public void kill() {
synchronized (process) {
process.destroy();
http://git-wip-us.apache.org/repos/asf/hive/blob/9b5eb45b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PrepPhase.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PrepPhase.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PrepPhase.java
index 8fef413..825f0c0 100644
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PrepPhase.java
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PrepPhase.java
@@ -62,7 +62,6 @@ public class PrepPhase extends Phase {
// source prep
start = System.currentTimeMillis();
File sourcePrepScript = new File(mScratchDir, "source-prep.sh");
- logger.info("Writing {} from template", sourcePrepScript);
Templates.writeTemplateResult("source-prep.vm", sourcePrepScript, getTemplateDefaults());
execLocally("bash " + sourcePrepScript.getPath());
logger.debug("Deleting " + sourcePrepScript + ": " + sourcePrepScript.delete());
http://git-wip-us.apache.org/repos/asf/hive/blob/9b5eb45b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/Host.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/Host.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/Host.java
index a56824c..c1216c1 100644
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/Host.java
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/Host.java
@@ -47,9 +47,6 @@ public class Host {
public String[] getLocalDirectories() {
return localDirectories;
}
- public String toShortString() {
- return name;
- }
@Override
public String toString() {
return "Host [name=" + name + ", user=" + user + ", threads=" + threads