You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by vi...@apache.org on 2018/06/26 05:17:42 UTC
[6/8] hive git commit: temp work on docker exec phase testing
temp work on docker exec phase testing
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ebb40fe0
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ebb40fe0
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ebb40fe0
Branch: refs/heads/HIVE-19429
Commit: ebb40fe0e71944dde5dc2ce3f48a925cc636f894
Parents: 519b62e
Author: Vihang Karajgaonkar <vi...@cloudera.com>
Authored: Fri Jun 22 00:14:48 2018 -0700
Committer: Vihang Karajgaonkar <vi...@cloudera.com>
Committed: Fri Jun 22 00:14:48 2018 -0700
----------------------------------------------------------------------
.../hive/ptest/execution/ContainerClient.java | 2 +-
.../execution/containers/DockerClient.java | 3 +-
.../containers/DockerHostExecutor.java | 4 +-
.../ptest/execution/TestPtestOnDockers.java | 42 ++++++++++++++++----
4 files changed, 40 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/ebb40fe0/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ContainerClient.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ContainerClient.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ContainerClient.java
index 59da132..6d3b112 100644
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ContainerClient.java
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ContainerClient.java
@@ -27,7 +27,7 @@ public interface ContainerClient {
void defineImage(String dir) throws Exception;
String getBuildCommand(String dir, long toWait, TimeUnit unit)
throws Exception;
- String getRunContainerCommand(String containerName, TestBatch testBatch);
+ String getRunContainerCommand(String containerName, final String imageTag, TestBatch testBatch);
String getCopyTestLogsCommand(String containerName, String dir);
http://git-wip-us.apache.org/repos/asf/hive/blob/ebb40fe0/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerClient.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerClient.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerClient.java
index da54348..5d107a1 100644
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerClient.java
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerClient.java
@@ -85,8 +85,9 @@ public class DockerClient implements ContainerClient {
}
@Override
- public String getRunContainerCommand(String containerName, TestBatch batch) {
+ public String getRunContainerCommand(String containerName, final String imageTag, TestBatch batch) {
return new StringBuilder("docker run")
+ .append(" -t " + imageTag)
.append(" --name " + containerName)
.append(" " + imageName())
.append(" /bin/bash")
http://git-wip-us.apache.org/repos/asf/hive/blob/ebb40fe0/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerHostExecutor.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerHostExecutor.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerHostExecutor.java
index 01a1280..4aa03d2 100644
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerHostExecutor.java
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerHostExecutor.java
@@ -67,7 +67,7 @@ public class DockerHostExecutor extends HostExecutor {
private final int numParallelContainersPerHost;
private AtomicInteger containerNameId = new AtomicInteger(0);
- DockerHostExecutor(Host host, String privateKey, ListeningExecutorService executor,
+ public DockerHostExecutor(Host host, String privateKey, ListeningExecutorService executor,
SSHCommandExecutor sshCommandExecutor, RSyncCommandExecutor rsyncCommandExecutor,
ImmutableMap<String, String> templateDefaults, File scratchDir, File succeededLogDir,
File failedLogDir, long numPollSeconds, boolean fetchLogsForSuccessfulTests, Logger logger)
@@ -141,7 +141,7 @@ public class DockerHostExecutor extends HostExecutor {
throws AbortContainerException, IOException, SSHExecutionException {
final int containerInstanceId = containerNameId.getAndIncrement();
final String containerName = getContainerName(containerInstanceId);
- String runCommand = dockerClient.getRunContainerCommand(containerName, batch);
+ String runCommand = dockerClient.getRunContainerCommand(containerName, mTemplateDefaults.get("buildTag"), batch);
Stopwatch sw = Stopwatch.createStarted();
mLogger.info("Executing " + batch + " with " + runCommand);
RemoteCommandResult sshResult = new SSHCommand(mSSHCommandExecutor, mPrivateKey, mHost.getUser(),
http://git-wip-us.apache.org/repos/asf/hive/blob/ebb40fe0/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestPtestOnDockers.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestPtestOnDockers.java b/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestPtestOnDockers.java
index 76716c4..c5e82c4 100644
--- a/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestPtestOnDockers.java
+++ b/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestPtestOnDockers.java
@@ -19,14 +19,19 @@
package org.apache.hive.ptest.execution;
+import com.google.common.base.Suppliers;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Sets;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
import org.apache.hive.ptest.execution.LocalCommand.CollectLogPolicy;
import org.apache.hive.ptest.execution.conf.Host;
+import org.apache.hive.ptest.execution.conf.QFileTestBatch;
import org.apache.hive.ptest.execution.conf.TestBatch;
+import org.apache.hive.ptest.execution.conf.UnitTestBatch;
import org.apache.hive.ptest.execution.containers.DockerExecutionPhase;
+import org.apache.hive.ptest.execution.containers.DockerHostExecutor;
import org.apache.hive.ptest.execution.containers.DockerPrepPhase;
import org.apache.hive.ptest.execution.containers.TestDockerPrepPhase;
import org.apache.hive.ptest.execution.context.ExecutionContext;
@@ -41,7 +46,14 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.net.URL;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicInteger;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
@@ -55,7 +67,7 @@ public class TestPtestOnDockers {
private DockerPrepPhase prepPhase;
private DockerExecutionPhase execPhase;
private static File dummyPatchFile;
- private static final Logger logger = LoggerFactory.getLogger(TestDockerPrepPhase.class);
+ private static final Logger logger = LoggerFactory.getLogger(TestPtestOnDockers.class);
private File baseDir;
private File scratchDir;
@@ -85,6 +97,9 @@ public class TestPtestOnDockers {
private MockSSHCommandExecutor sshCommandExecutor;
private MockRSyncCommandExecutor rsyncCommandExecutor;
private static final String BUILD_TAG = "docker-ptest-tag";
+ private final Set<String> executedTests = Collections.newSetFromMap(new ConcurrentHashMap<String, Boolean>());
+ private final Set<String> failedTests = Collections.newSetFromMap(new ConcurrentHashMap<String, Boolean>());
+ private List<TestBatch> testBatches;
public void initialize(String name) throws Exception {
baseDir = AbstractTestPhase.createBaseDir(name);
@@ -128,15 +143,23 @@ public class TestPtestOnDockers {
createHostExecutor();
prepPhase = new DockerPrepPhase(hostExecutors, localCommandFactory,
templateDefaults, baseDir, dummyPatchFile, logger);
- /*execPhase = new DockerExecutionPhase(hostExecutors, executionContext,
+ createTestBatches();
+ execPhase = new DockerExecutionPhase(hostExecutors, executionContext,
hostExecutorBuilder, localCommandFactory,
templateDefaults, succeededLogDir, failedLogDir,
- testBatchSupplier, executedTests,
- failedTests, logger);*/
+ Suppliers.ofInstance(testBatches), executedTests,
+ failedTests, logger);
}
- private void createHostExecutor() {
- hostExecutor = new HostExecutor(host, PRIVATE_KEY, executor, sshCommandExecutor,
+ private void createTestBatches() throws Exception {
+ testBatches = new ArrayList<>();
+ TestBatch qfileTestBatch = new QFileTestBatch(new AtomicInteger(1), "", "TestCliDriver", "",
+ Sets.newHashSet("insert0.q"), true, "itests/qtest");
+ testBatches.add(qfileTestBatch);
+ }
+
+ private void createHostExecutor() throws Exception {
+ hostExecutor = new DockerHostExecutor(host, PRIVATE_KEY, executor, sshCommandExecutor,
rsyncCommandExecutor, templateDefaults, scratchDir, succeededLogDir, failedLogDir, 1, true,
logger);
hostExecutors = ImmutableList.of(hostExecutor);
@@ -153,10 +176,15 @@ public class TestPtestOnDockers {
* @throws Exception
*/
@Test
- public void testDockerFile() throws Exception {
+ public void testDockerFile() throws Throwable {
prepPhase.execute();
Assert.assertNotNull("Scratch directory needs to be set", prepPhase.getLocalScratchDir());
File dockerFile = new File(prepPhase.getLocalScratchDir(), "Dockerfile");
Assert.assertTrue("Docker file not found", dockerFile.exists());
}
+
+ @Test
+ public void testDockerExecutionPhase() throws Throwable {
+ execPhase.execute();
+ }
}