You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by vi...@apache.org on 2018/06/26 05:17:37 UTC
[1/8] hive git commit: first commit dummy
Repository: hive
Updated Branches:
refs/heads/HIVE-19429 5b694b7be -> a2b2e125e
first commit dummy
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/9765a43d
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/9765a43d
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/9765a43d
Branch: refs/heads/HIVE-19429
Commit: 9765a43d0caa72c2a52eebb4b7d48a7b17034282
Parents: 5b694b7
Author: Vihang Karajgaonkar <vi...@cloudera.com>
Authored: Sat Jun 2 09:21:46 2018 -0700
Committer: Vihang Karajgaonkar <vi...@cloudera.com>
Committed: Mon Jun 4 11:33:12 2018 -0700
----------------------------------------------------------------------
.../execution/DockerBasedExecutionPhase.java | 119 +++++++++++++++++++
.../ptest/execution/containers/DockerPTest.java | 40 +++++++
.../ptest/api/server/TestTestStartRequest.java | 40 +++++++
3 files changed, 199 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/9765a43d/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/DockerBasedExecutionPhase.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/DockerBasedExecutionPhase.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/DockerBasedExecutionPhase.java
new file mode 100644
index 0000000..4b19ccc
--- /dev/null
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/DockerBasedExecutionPhase.java
@@ -0,0 +1,119 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hive.ptest.execution;
+
+import com.google.common.base.Supplier;
+import com.google.common.collect.ImmutableMap;
+import org.apache.hive.ptest.execution.conf.TestBatch;
+import org.apache.hive.ptest.execution.context.ExecutionContext;
+import org.slf4j.Logger;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+public class DockerBasedExecutionPhase extends ExecutionPhase {
+
+ public DockerBasedExecutionPhase(List<HostExecutor> hostExecutors,
+ ExecutionContext executionContext, HostExecutorBuilder hostExecutorBuilder,
+ LocalCommandFactory localCommandFactory, ImmutableMap<String, String> templateDefaults,
+ File succeededLogDir, File failedLogDir, Supplier<List<TestBatch>> testBatchSupplier,
+ Set<String> executedTests, Set<String> failedTests, Logger logger) throws IOException {
+ super(hostExecutors, executionContext, hostExecutorBuilder, localCommandFactory,
+ templateDefaults, succeededLogDir, failedLogDir, testBatchSupplier, executedTests,
+ failedTests, logger);
+ }
+
+ @Override
+ public void execute() throws Throwable {
+ long start = System.currentTimeMillis();
+ /*List<TestBatch> testBatches = Lists.newArrayList();
+ for(TestBatch batch : testBatchSupplier.get()) {
+ testBatches.add(batch);
+ if(batch.isParallel()) {
+ parallelWorkQueue.add(batch);
+ } else {
+ isolatedWorkQueue.add(batch);
+ }
+ }
+ logger.info("ParallelWorkQueueSize={}, IsolatedWorkQueueSize={}", parallelWorkQueue.size(),
+ isolatedWorkQueue.size());
+ if (logger.isDebugEnabled()) {
+ for (TestBatch testBatch : parallelWorkQueue) {
+ logger.debug("PBatch: {}", testBatch);
+ }
+ for (TestBatch testBatch : isolatedWorkQueue) {
+ logger.debug("IBatch: {}", testBatch);
+ }
+ }*/
+ try {
+ int expectedNumHosts = hostExecutors.size();
+ initalizeHosts();
+ resetPerfMetrics();
+ /*do {
+ //replaceBadHosts(expectedNumHosts);
+ List<ListenableFuture<Void>> results = Lists.newArrayList();
+ for(HostExecutor hostExecutor : ImmutableList.copyOf(hostExecutors)) {
+ results.add(hostExecutor.submitTests(parallelWorkQueue, isolatedWorkQueue, failedTestResults));
+ }
+ Futures.allAsList(results).get();
+ } while(!(parallelWorkQueue.isEmpty() && isolatedWorkQueue.isEmpty()));
+ for(TestBatch batch : testBatches) {
+ File batchLogDir;
+ if(failedTestResults.contains(batch)) {
+ batchLogDir = new File(failedLogDir, batch.getName());
+ } else {
+ batchLogDir = new File(succeededLogDir, batch.getName());
+ }
+ JUnitReportParser parser = new JUnitReportParser(logger, batchLogDir);
+ executedTests.addAll(parser.getAllExecutedTests());
+ for (String failedTest : parser.getAllFailedTests()) {
+ failedTests.add(failedTest + " (batchId=" + batch.getBatchId() + ")");
+ }
+
+ // if the TEST*.xml was not generated or was corrupt, let someone know
+ if (parser.getTestClassesWithReportAvailable().size() < batch.getTestClasses().size()) {
+ Set<String> expTestClasses = new HashSet<>(batch.getTestClasses());
+ expTestClasses.removeAll(parser.getTestClassesWithReportAvailable());
+ for (String testClass : expTestClasses) {
+ StringBuilder messageBuilder = new StringBuilder();
+ messageBuilder.append(testClass).append(" - did not produce a TEST-*.xml file (likely timed out)")
+ .append(" (batchId=").append(batch.getBatchId()).append(")");
+ if (batch instanceof QFileTestBatch) {
+ Collection<String> tests = ((QFileTestBatch)batch).getTests();
+ if (tests.size() != 0) {
+ messageBuilder.append("\n\t[");
+ messageBuilder.append(Joiner.on(",").join(tests));
+ messageBuilder.append("]");
+ }
+ }
+ failedTests.add(messageBuilder.toString());
+ }
+ }
+ }*/
+ } finally {
+ long elapsed = System.currentTimeMillis() - start;
+ //addAggregatePerfMetrics();
+ logger.info("PERF: exec phase " + TimeUnit.MINUTES.convert(elapsed, TimeUnit.MILLISECONDS) + " minutes");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9765a43d/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerPTest.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerPTest.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerPTest.java
new file mode 100644
index 0000000..81867cc
--- /dev/null
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerPTest.java
@@ -0,0 +1,40 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hive.ptest.execution.containers;
+
+import org.apache.hive.ptest.execution.LocalCommandFactory;
+import org.apache.hive.ptest.execution.PTest;
+import org.apache.hive.ptest.execution.conf.TestConfiguration;
+import org.apache.hive.ptest.execution.context.ExecutionContext;
+import org.apache.hive.ptest.execution.ssh.RSyncCommandExecutor;
+import org.apache.hive.ptest.execution.ssh.SSHCommandExecutor;
+import org.slf4j.Logger;
+
+import java.io.File;
+
+public class DockerPTest extends PTest {
+ public DockerPTest(TestConfiguration configuration, ExecutionContext executionContext,
+ String buildTag, File logDir, LocalCommandFactory localCommandFactory,
+ SSHCommandExecutor sshCommandExecutor, RSyncCommandExecutor rsyncCommandExecutor,
+ Logger logger) throws Exception {
+ super(configuration, executionContext, buildTag, logDir, localCommandFactory,
+ sshCommandExecutor, rsyncCommandExecutor, logger);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9765a43d/testutils/ptest2/src/test/java/org/apache/hive/ptest/api/server/TestTestStartRequest.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/test/java/org/apache/hive/ptest/api/server/TestTestStartRequest.java b/testutils/ptest2/src/test/java/org/apache/hive/ptest/api/server/TestTestStartRequest.java
new file mode 100644
index 0000000..d5282c9
--- /dev/null
+++ b/testutils/ptest2/src/test/java/org/apache/hive/ptest/api/server/TestTestStartRequest.java
@@ -0,0 +1,40 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hive.ptest.api.server;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.hive.ptest.api.request.TestStartRequest;
+import org.junit.Test;
+
+public class TestTestStartRequest {
+ ObjectMapper mMapper = new ObjectMapper();
+ @Test
+ public void testJson() throws JsonProcessingException {
+ String profile = "ptest";
+ String testHandle = "Dummy-Precommit-Test-1";
+ String jira = "HIVE-19425";
+ String patch = "https://issues.apache.org/jira/secure/attachment/12923111/HIVE-19429.01-ptest.patch";
+ boolean clearLibraryCache = false;
+ TestStartRequest startRequest = new TestStartRequest(profile, testHandle, jira, patch, clearLibraryCache);
+ String payloadString = mMapper.writeValueAsString(startRequest);
+ System.out.println(payloadString);
+ }
+}
[5/8] hive git commit: Docker prep phase working
Posted by vi...@apache.org.
Docker prep phase working
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/519b62e3
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/519b62e3
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/519b62e3
Branch: refs/heads/HIVE-19429
Commit: 519b62e3e28f26d8eb52c7dba486f8999f06b443
Parents: 5787e77
Author: Vihang Karajgaonkar <vi...@cloudera.com>
Authored: Thu Jun 21 23:21:43 2018 -0700
Committer: Vihang Karajgaonkar <vi...@cloudera.com>
Committed: Thu Jun 21 23:21:43 2018 -0700
----------------------------------------------------------------------
.../execution/containers/DockerClient.java | 2 ++
.../execution/containers/DockerPrepPhase.java | 4 +--
.../src/main/resources/dockerfile-template.vm | 20 ++++++-----
.../ptest/execution/TestPtestOnDockers.java | 34 +++++++++---------
.../ptest2/src/test/resources/DUMMY-001.patch | 38 ++++++++++++++------
5 files changed, 59 insertions(+), 39 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/519b62e3/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerClient.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerClient.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerClient.java
index 8e6e0cf..da54348 100644
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerClient.java
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerClient.java
@@ -64,6 +64,8 @@ public class DockerClient implements ContainerClient {
logger.info("Building image");
String dockerBuildCommand =
new StringBuilder("docker build")
+ .append(" --memory " + "2g")
+ .append(" --memory-swap " + "-1")
.append(" --tag " + imageName())
.append(" --build-arg ")
.append(" workingDir=$workingDir")
http://git-wip-us.apache.org/repos/asf/hive/blob/519b62e3/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerPrepPhase.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerPrepPhase.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerPrepPhase.java
index f5455b5..80c1aef 100644
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerPrepPhase.java
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerPrepPhase.java
@@ -58,11 +58,11 @@ public class DockerPrepPhase extends PrepPhase {
execLocally(getDockerBuildCommand());
elapsedTime = TimeUnit.MINUTES.convert((System.currentTimeMillis() - start),
TimeUnit.MILLISECONDS);
- logger.info("PERF: Docker source prep took " + elapsedTime + " minutes");
+ logger.info("PERF: Docker build image took " + elapsedTime + " minutes");
}
@VisibleForTesting
- String getLocalScratchDir() {
+ public String getLocalScratchDir() {
return mScratchDir.getAbsolutePath();
}
http://git-wip-us.apache.org/repos/asf/hive/blob/519b62e3/testutils/ptest2/src/main/resources/dockerfile-template.vm
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/resources/dockerfile-template.vm b/testutils/ptest2/src/main/resources/dockerfile-template.vm
index fe421f3..4f96ea7 100644
--- a/testutils/ptest2/src/main/resources/dockerfile-template.vm
+++ b/testutils/ptest2/src/main/resources/dockerfile-template.vm
@@ -35,16 +35,18 @@ ARG repository
ARG branch
ARG buildTag
-RUN export MAVEN_OPTS="$mavenEnvOpts"
+ENV MAVEN_OPTS="$mavenEnvOpts"
+RUN /usr/bin/mvn -version
+RUN echo $MAVEN_OPTS
RUN /usr/bin/git clone $repository
RUN cd hive
WORKDIR /home/ptestuser/hive
-RUN /usr/bin/git checkout $branch || checkout -b $branch origin/$branch
-RUN /usr/bin/git reset --hard origin/$branch
-RUN /usr/bin/git merge --ff-only origin/$branch
-RUN /home/ptestuser/scratch/smart-apply-patch.sh /home/ptestuser/scratch/build.patch
-RUN /usr/bin/mvn -B -T 4 -q install -Dtest=TestMetastoreConf
-RUN cd itests
-RUN /usr/bin/mvn -B -T 4 -q install -DskipSparkTests -DskipTests
-RUN echo This build is labeled $buildTag
+RUN /usr/bin/git checkout $branch || checkout -b $branch origin/$branch &&\
+ /usr/bin/git reset --hard origin/$branch &&\
+ /usr/bin/git merge --ff-only origin/$branch &&\
+ /home/ptestuser/scratch/smart-apply-patch.sh /home/ptestuser/scratch/build.patch &&\
+ /usr/bin/mvn -q -B install -Dtest=TestMetastoreConf &&\
+ cd itests &&\
+ /usr/bin/mvn -q -B install -DskipSparkTests -DskipTests &&\
+ echo This build is labeled $buildTag
http://git-wip-us.apache.org/repos/asf/hive/blob/519b62e3/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestPtestOnDockers.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestPtestOnDockers.java b/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestPtestOnDockers.java
index 1109a8a..76716c4 100644
--- a/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestPtestOnDockers.java
+++ b/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestPtestOnDockers.java
@@ -23,9 +23,10 @@ import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
-import org.apache.commons.io.FileUtils;
import org.apache.hive.ptest.execution.LocalCommand.CollectLogPolicy;
import org.apache.hive.ptest.execution.conf.Host;
+import org.apache.hive.ptest.execution.conf.TestBatch;
+import org.apache.hive.ptest.execution.containers.DockerExecutionPhase;
import org.apache.hive.ptest.execution.containers.DockerPrepPhase;
import org.apache.hive.ptest.execution.containers.TestDockerPrepPhase;
import org.apache.hive.ptest.execution.context.ExecutionContext;
@@ -51,7 +52,8 @@ import static org.mockito.Mockito.spy;
public class TestPtestOnDockers {
//TODO add logic to ignore this test if docker is not found on the machine
- private DockerPrepPhase phase;
+ private DockerPrepPhase prepPhase;
+ private DockerExecutionPhase execPhase;
private static File dummyPatchFile;
private static final Logger logger = LoggerFactory.getLogger(TestDockerPrepPhase.class);
@@ -79,7 +81,7 @@ public class TestPtestOnDockers {
private static final String REAL_BRANCH = "master";
private static final String REAL_REPOSITORY = "https://github.com/apache/hive.git";
private static final String REAL_REPOSITORY_NAME = "apache-hive";
- private static final String REAL_MAVEN_OPTS = "-Xmx1g";
+ private static final String REAL_MAVEN_OPTS = "-Xmx2048m";
private MockSSHCommandExecutor sshCommandExecutor;
private MockRSyncCommandExecutor rsyncCommandExecutor;
private static final String BUILD_TAG = "docker-ptest-tag";
@@ -124,8 +126,13 @@ public class TestPtestOnDockers {
public void setup() throws Exception {
initialize(getClass().getSimpleName());
createHostExecutor();
- phase = new DockerPrepPhase(hostExecutors, localCommandFactory,
+ prepPhase = new DockerPrepPhase(hostExecutors, localCommandFactory,
templateDefaults, baseDir, dummyPatchFile, logger);
+ /*execPhase = new DockerExecutionPhase(hostExecutors, executionContext,
+ hostExecutorBuilder, localCommandFactory,
+ templateDefaults, succeededLogDir, failedLogDir,
+ testBatchSupplier, executedTests,
+ failedTests, logger);*/
}
private void createHostExecutor() {
@@ -137,7 +144,7 @@ public class TestPtestOnDockers {
@After
public void teardown() {
- phase = null;
+ prepPhase = null;
//FileUtils.deleteQuietly(baseDir);
}
@@ -147,18 +154,9 @@ public class TestPtestOnDockers {
*/
@Test
public void testDockerFile() throws Exception {
- phase.execute();
- File dockerFile = new File(scratchDir, "Dockerfile");
- Assert.assertTrue("Docker file was not found" , dockerFile.exists());
- Assert.assertTrue("patch file not found", new File(scratchDir, "build.patch").exists());
- LocalCommandFactory localCommandFactory = new LocalCommandFactory(logger);
- CollectLogPolicy localCollector = new CollectLogPolicy(logger);
- localCommandFactory.create(localCollector, "cp -f " + dockerFile.getAbsolutePath() + " /tmp/myDockerFile");
- LocalCommand localCmd = localCommandFactory.create(localCollector, phase.getDockerBuildCommand());
- if(localCmd.getExitCode() != 0) {
- throw new NonZeroExitCodeException(String.format(
- "Command '%s' failed with exit status %d and output '%s'",
- localCmd, localCmd.getExitCode(), localCollector.getOutput()));
- }
+ prepPhase.execute();
+ Assert.assertNotNull("Scratch directory needs to be set", prepPhase.getLocalScratchDir());
+ File dockerFile = new File(prepPhase.getLocalScratchDir(), "Dockerfile");
+ Assert.assertTrue("Docker file not found", dockerFile.exists());
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/519b62e3/testutils/ptest2/src/test/resources/DUMMY-001.patch
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/test/resources/DUMMY-001.patch b/testutils/ptest2/src/test/resources/DUMMY-001.patch
index 6277ca9..97d7feb 100644
--- a/testutils/ptest2/src/test/resources/DUMMY-001.patch
+++ b/testutils/ptest2/src/test/resources/DUMMY-001.patch
@@ -1,12 +1,30 @@
-diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java
-index 2868ff0..6a6a5ff 100644
---- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java
-+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java
-@@ -69,6 +69,7 @@
+diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java
+index 1d7eceb1ef380fdb6f62d8b831b1e2bd23333c67..b78e7b2bef78b99a3cab9cc89bd8551649285417 100644
+--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java
++++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java
+@@ -68,9 +68,7 @@
- public class PTest {
+ import com.google.common.annotations.VisibleForTesting;
+ import com.google.protobuf.CodedInputStream;
+-
+-import sun.misc.Cleaner;
+-
++//import sun.misc.Cleaner;
-+ // test patching
- static {
- Velocity.init();
- }
+ /**
+ * Encoded reader implementation.
+@@ -1654,12 +1652,12 @@ private void releaseBuffer(ByteBuffer bb, boolean isFromDataReader) {
+ Field localCf = cleanerField;
+ if (!bb.isDirect() || localCf == null) return;
+ try {
+- Cleaner cleaner = (Cleaner) localCf.get(bb);
++ /*Cleaner cleaner = (Cleaner) localCf.get(bb);
+ if (cleaner != null) {
+ cleaner.clean();
+ } else {
+ LOG.debug("Unable to clean a buffer using cleaner - no cleaner");
+- }
++ }*/
+ } catch (Exception e) {
+ // leave it for GC to clean up
+ LOG.warn("Unable to clean direct buffers using Cleaner.");
[6/8] hive git commit: temp work on docker exec phase testing
Posted by vi...@apache.org.
temp work on docker exec phase testing
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ebb40fe0
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ebb40fe0
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ebb40fe0
Branch: refs/heads/HIVE-19429
Commit: ebb40fe0e71944dde5dc2ce3f48a925cc636f894
Parents: 519b62e
Author: Vihang Karajgaonkar <vi...@cloudera.com>
Authored: Fri Jun 22 00:14:48 2018 -0700
Committer: Vihang Karajgaonkar <vi...@cloudera.com>
Committed: Fri Jun 22 00:14:48 2018 -0700
----------------------------------------------------------------------
.../hive/ptest/execution/ContainerClient.java | 2 +-
.../execution/containers/DockerClient.java | 3 +-
.../containers/DockerHostExecutor.java | 4 +-
.../ptest/execution/TestPtestOnDockers.java | 42 ++++++++++++++++----
4 files changed, 40 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/ebb40fe0/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ContainerClient.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ContainerClient.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ContainerClient.java
index 59da132..6d3b112 100644
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ContainerClient.java
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ContainerClient.java
@@ -27,7 +27,7 @@ public interface ContainerClient {
void defineImage(String dir) throws Exception;
String getBuildCommand(String dir, long toWait, TimeUnit unit)
throws Exception;
- String getRunContainerCommand(String containerName, TestBatch testBatch);
+ String getRunContainerCommand(String containerName, final String imageTag, TestBatch testBatch);
String getCopyTestLogsCommand(String containerName, String dir);
http://git-wip-us.apache.org/repos/asf/hive/blob/ebb40fe0/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerClient.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerClient.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerClient.java
index da54348..5d107a1 100644
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerClient.java
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerClient.java
@@ -85,8 +85,9 @@ public class DockerClient implements ContainerClient {
}
@Override
- public String getRunContainerCommand(String containerName, TestBatch batch) {
+ public String getRunContainerCommand(String containerName, final String imageTag, TestBatch batch) {
return new StringBuilder("docker run")
+ .append(" -t " + imageTag)
.append(" --name " + containerName)
.append(" " + imageName())
.append(" /bin/bash")
http://git-wip-us.apache.org/repos/asf/hive/blob/ebb40fe0/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerHostExecutor.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerHostExecutor.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerHostExecutor.java
index 01a1280..4aa03d2 100644
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerHostExecutor.java
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerHostExecutor.java
@@ -67,7 +67,7 @@ public class DockerHostExecutor extends HostExecutor {
private final int numParallelContainersPerHost;
private AtomicInteger containerNameId = new AtomicInteger(0);
- DockerHostExecutor(Host host, String privateKey, ListeningExecutorService executor,
+ public DockerHostExecutor(Host host, String privateKey, ListeningExecutorService executor,
SSHCommandExecutor sshCommandExecutor, RSyncCommandExecutor rsyncCommandExecutor,
ImmutableMap<String, String> templateDefaults, File scratchDir, File succeededLogDir,
File failedLogDir, long numPollSeconds, boolean fetchLogsForSuccessfulTests, Logger logger)
@@ -141,7 +141,7 @@ public class DockerHostExecutor extends HostExecutor {
throws AbortContainerException, IOException, SSHExecutionException {
final int containerInstanceId = containerNameId.getAndIncrement();
final String containerName = getContainerName(containerInstanceId);
- String runCommand = dockerClient.getRunContainerCommand(containerName, batch);
+ String runCommand = dockerClient.getRunContainerCommand(containerName, mTemplateDefaults.get("buildTag"), batch);
Stopwatch sw = Stopwatch.createStarted();
mLogger.info("Executing " + batch + " with " + runCommand);
RemoteCommandResult sshResult = new SSHCommand(mSSHCommandExecutor, mPrivateKey, mHost.getUser(),
http://git-wip-us.apache.org/repos/asf/hive/blob/ebb40fe0/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestPtestOnDockers.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestPtestOnDockers.java b/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestPtestOnDockers.java
index 76716c4..c5e82c4 100644
--- a/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestPtestOnDockers.java
+++ b/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestPtestOnDockers.java
@@ -19,14 +19,19 @@
package org.apache.hive.ptest.execution;
+import com.google.common.base.Suppliers;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Sets;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
import org.apache.hive.ptest.execution.LocalCommand.CollectLogPolicy;
import org.apache.hive.ptest.execution.conf.Host;
+import org.apache.hive.ptest.execution.conf.QFileTestBatch;
import org.apache.hive.ptest.execution.conf.TestBatch;
+import org.apache.hive.ptest.execution.conf.UnitTestBatch;
import org.apache.hive.ptest.execution.containers.DockerExecutionPhase;
+import org.apache.hive.ptest.execution.containers.DockerHostExecutor;
import org.apache.hive.ptest.execution.containers.DockerPrepPhase;
import org.apache.hive.ptest.execution.containers.TestDockerPrepPhase;
import org.apache.hive.ptest.execution.context.ExecutionContext;
@@ -41,7 +46,14 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.net.URL;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicInteger;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
@@ -55,7 +67,7 @@ public class TestPtestOnDockers {
private DockerPrepPhase prepPhase;
private DockerExecutionPhase execPhase;
private static File dummyPatchFile;
- private static final Logger logger = LoggerFactory.getLogger(TestDockerPrepPhase.class);
+ private static final Logger logger = LoggerFactory.getLogger(TestPtestOnDockers.class);
private File baseDir;
private File scratchDir;
@@ -85,6 +97,9 @@ public class TestPtestOnDockers {
private MockSSHCommandExecutor sshCommandExecutor;
private MockRSyncCommandExecutor rsyncCommandExecutor;
private static final String BUILD_TAG = "docker-ptest-tag";
+ private final Set<String> executedTests = Collections.newSetFromMap(new ConcurrentHashMap<String, Boolean>());
+ private final Set<String> failedTests = Collections.newSetFromMap(new ConcurrentHashMap<String, Boolean>());
+ private List<TestBatch> testBatches;
public void initialize(String name) throws Exception {
baseDir = AbstractTestPhase.createBaseDir(name);
@@ -128,15 +143,23 @@ public class TestPtestOnDockers {
createHostExecutor();
prepPhase = new DockerPrepPhase(hostExecutors, localCommandFactory,
templateDefaults, baseDir, dummyPatchFile, logger);
- /*execPhase = new DockerExecutionPhase(hostExecutors, executionContext,
+ createTestBatches();
+ execPhase = new DockerExecutionPhase(hostExecutors, executionContext,
hostExecutorBuilder, localCommandFactory,
templateDefaults, succeededLogDir, failedLogDir,
- testBatchSupplier, executedTests,
- failedTests, logger);*/
+ Suppliers.ofInstance(testBatches), executedTests,
+ failedTests, logger);
}
- private void createHostExecutor() {
- hostExecutor = new HostExecutor(host, PRIVATE_KEY, executor, sshCommandExecutor,
+ private void createTestBatches() throws Exception {
+ testBatches = new ArrayList<>();
+ TestBatch qfileTestBatch = new QFileTestBatch(new AtomicInteger(1), "", "TestCliDriver", "",
+ Sets.newHashSet("insert0.q"), true, "itests/qtest");
+ testBatches.add(qfileTestBatch);
+ }
+
+ private void createHostExecutor() throws Exception {
+ hostExecutor = new DockerHostExecutor(host, PRIVATE_KEY, executor, sshCommandExecutor,
rsyncCommandExecutor, templateDefaults, scratchDir, succeededLogDir, failedLogDir, 1, true,
logger);
hostExecutors = ImmutableList.of(hostExecutor);
@@ -153,10 +176,15 @@ public class TestPtestOnDockers {
* @throws Exception
*/
@Test
- public void testDockerFile() throws Exception {
+ public void testDockerFile() throws Throwable {
prepPhase.execute();
Assert.assertNotNull("Scratch directory needs to be set", prepPhase.getLocalScratchDir());
File dockerFile = new File(prepPhase.getLocalScratchDir(), "Dockerfile");
Assert.assertTrue("Docker file not found", dockerFile.exists());
}
+
+ @Test
+ public void testDockerExecutionPhase() throws Throwable {
+ execPhase.execute();
+ }
}
[3/8] hive git commit: Temp work related to docker execution phase.
code may not compile
Posted by vi...@apache.org.
Temp work related to docker execution phase. code may not compile
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ea53020e
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ea53020e
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ea53020e
Branch: refs/heads/HIVE-19429
Commit: ea53020e84c51f35810b0977e160c78b1376f07e
Parents: 4192812
Author: Vihang Karajgaonkar <vi...@cloudera.com>
Authored: Mon Jun 4 11:31:48 2018 -0700
Committer: Vihang Karajgaonkar <vi...@cloudera.com>
Committed: Mon Jun 4 11:34:21 2018 -0700
----------------------------------------------------------------------
.../hive/ptest/execution/ContainerClient.java | 11 ++
.../hive/ptest/execution/ExecutionPhase.java | 101 ++++++-----
.../hive/ptest/execution/HostExecutor.java | 30 ++--
.../containers/AbortContainerException.java | 23 +++
.../execution/containers/DockerClient.java | 38 ++++
.../containers/DockerExecutionPhase.java | 73 ++++++++
.../containers/DockerHostExectutor.java | 177 +++++++++++++++++++
.../execution/containers/DockerPrepPhase.java | 4 +
.../src/main/resources/dockerfile-template.vm | 8 +
9 files changed, 402 insertions(+), 63 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/ea53020e/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ContainerClient.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ContainerClient.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ContainerClient.java
index a9a8ead..d4f887c 100644
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ContainerClient.java
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ContainerClient.java
@@ -20,12 +20,23 @@
package org.apache.hive.ptest.execution;
import org.apache.hive.ptest.execution.ContainerClientFactory.ContainerClientContext;
+<<<<<<< HEAD
+=======
+import org.apache.hive.ptest.execution.conf.TestBatch;
+>>>>>>> a1fe94a... Temp work related to docker execution phase. code may not compile
import java.io.IOException;
import java.util.concurrent.TimeUnit;
public interface ContainerClient {
+<<<<<<< HEAD
void defineImage(String dir, String label) throws IOException;
String getBuildCommand(String dir, long toWait, TimeUnit unit)
throws Exception;
+=======
+ void defineImage(String dir) throws Exception;
+ String getBuildCommand(String dir, long toWait, TimeUnit unit)
+ throws Exception;
+ String getRunContainerCommand(String containerName, TestBatch testBatch);
+>>>>>>> a1fe94a... Temp work related to docker execution phase. code may not compile
}
http://git-wip-us.apache.org/repos/asf/hive/blob/ea53020e/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ExecutionPhase.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ExecutionPhase.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ExecutionPhase.java
index 7ab98f6..1f7d2fd 100644
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ExecutionPhase.java
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ExecutionPhase.java
@@ -22,10 +22,8 @@ import java.io.File;
import java.io.IOException;
import java.util.Collection;
import java.util.Collections;
-import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
-import java.util.Map;
import java.util.Set;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
@@ -52,11 +50,11 @@ public class ExecutionPhase extends Phase {
private final HostExecutorBuilder hostExecutorBuilder;
private final File succeededLogDir;
private final File failedLogDir;
- private final BlockingQueue<TestBatch> parallelWorkQueue;
- private final BlockingQueue<TestBatch> isolatedWorkQueue;
+ protected final BlockingQueue<TestBatch> parallelWorkQueue;
+ protected final BlockingQueue<TestBatch> isolatedWorkQueue;
private final Set<String> executedTests;
private final Set<String> failedTests;
- private final Supplier<List<TestBatch>> testBatchSupplier;
+ protected final Supplier<List<TestBatch>> testBatchSupplier;
private final Set<TestBatch> failedTestResults;
public ExecutionPhase(List<HostExecutor> hostExecutors, ExecutionContext executionContext,
@@ -82,15 +80,7 @@ public class ExecutionPhase extends Phase {
@Override
public void execute() throws Throwable {
long start = System.currentTimeMillis();
- List<TestBatch> testBatches = Lists.newArrayList();
- for(TestBatch batch : testBatchSupplier.get()) {
- testBatches.add(batch);
- if(batch.isParallel()) {
- parallelWorkQueue.add(batch);
- } else {
- isolatedWorkQueue.add(batch);
- }
- }
+ List<TestBatch> testBatches = getTestBatches();
logger.info("ParallelWorkQueueSize={}, IsolatedWorkQueueSize={}", parallelWorkQueue.size(),
isolatedWorkQueue.size());
if (logger.isDebugEnabled()) {
@@ -113,39 +103,7 @@ public class ExecutionPhase extends Phase {
}
Futures.allAsList(results).get();
} while(!(parallelWorkQueue.isEmpty() && isolatedWorkQueue.isEmpty()));
- for(TestBatch batch : testBatches) {
- File batchLogDir;
- if(failedTestResults.contains(batch)) {
- batchLogDir = new File(failedLogDir, batch.getName());
- } else {
- batchLogDir = new File(succeededLogDir, batch.getName());
- }
- JUnitReportParser parser = new JUnitReportParser(logger, batchLogDir);
- executedTests.addAll(parser.getAllExecutedTests());
- for (String failedTest : parser.getAllFailedTests()) {
- failedTests.add(failedTest + " (batchId=" + batch.getBatchId() + ")");
- }
-
- // if the TEST*.xml was not generated or was corrupt, let someone know
- if (parser.getTestClassesWithReportAvailable().size() < batch.getTestClasses().size()) {
- Set<String> expTestClasses = new HashSet<>(batch.getTestClasses());
- expTestClasses.removeAll(parser.getTestClassesWithReportAvailable());
- for (String testClass : expTestClasses) {
- StringBuilder messageBuilder = new StringBuilder();
- messageBuilder.append(testClass).append(" - did not produce a TEST-*.xml file (likely timed out)")
- .append(" (batchId=").append(batch.getBatchId()).append(")");
- if (batch instanceof QFileTestBatch) {
- Collection<String> tests = ((QFileTestBatch)batch).getTests();
- if (tests.size() != 0) {
- messageBuilder.append("\n\t[");
- messageBuilder.append(Joiner.on(",").join(tests));
- messageBuilder.append("]");
- }
- }
- failedTests.add(messageBuilder.toString());
- }
- }
- }
+ checkResults(testBatches);
} finally {
long elapsed = System.currentTimeMillis() - start;
addAggregatePerfMetrics();
@@ -154,6 +112,55 @@ public class ExecutionPhase extends Phase {
}
}
+ protected void checkResults(List<TestBatch> testBatches) throws Exception {
+ for(TestBatch batch : testBatches) {
+ File batchLogDir;
+ if(failedTestResults.contains(batch)) {
+ batchLogDir = new File(failedLogDir, batch.getName());
+ } else {
+ batchLogDir = new File(succeededLogDir, batch.getName());
+ }
+ JUnitReportParser parser = new JUnitReportParser(logger, batchLogDir);
+ executedTests.addAll(parser.getAllExecutedTests());
+ for (String failedTest : parser.getAllFailedTests()) {
+ failedTests.add(failedTest + " (batchId=" + batch.getBatchId() + ")");
+ }
+
+ // if the TEST*.xml was not generated or was corrupt, let someone know
+ if (parser.getTestClassesWithReportAvailable().size() < batch.getTestClasses().size()) {
+ Set<String> expTestClasses = new HashSet<>(batch.getTestClasses());
+ expTestClasses.removeAll(parser.getTestClassesWithReportAvailable());
+ for (String testClass : expTestClasses) {
+ StringBuilder messageBuilder = new StringBuilder();
+ messageBuilder.append(testClass).append(" - did not produce a TEST-*.xml file (likely timed out)")
+ .append(" (batchId=").append(batch.getBatchId()).append(")");
+ if (batch instanceof QFileTestBatch) {
+ Collection<String> tests = ((QFileTestBatch)batch).getTests();
+ if (tests.size() != 0) {
+ messageBuilder.append("\n\t[");
+ messageBuilder.append(Joiner.on(",").join(tests));
+ messageBuilder.append("]");
+ }
+ }
+ failedTests.add(messageBuilder.toString());
+ }
+ }
+ }
+ }
+
+ protected List<TestBatch> getTestBatches() {
+ List<TestBatch> testBatches = Lists.newArrayList();
+ for(TestBatch batch : testBatchSupplier.get()) {
+ testBatches.add(batch);
+ if(batch.isParallel()) {
+ parallelWorkQueue.add(batch);
+ } else {
+ isolatedWorkQueue.add(batch);
+ }
+ }
+ return testBatches;
+ }
+
public static final String TOTAL_RSYNC_TIME = "TotalRsyncElapsedTime";
private void addAggregatePerfMetrics() {
long totalRsycTime = 0L;
http://git-wip-us.apache.org/repos/asf/hive/blob/ea53020e/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/HostExecutor.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/HostExecutor.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/HostExecutor.java
index 24f0bf8..74be1e0 100644
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/HostExecutor.java
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/HostExecutor.java
@@ -56,29 +56,27 @@ import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListeningExecutorService;
public class HostExecutor {
- private final Host mHost;
+ protected final Host mHost;
private final List<Drone> mDrones;
- private final ListeningExecutorService mExecutor;
+ protected final ListeningExecutorService mExecutor;
private final SSHCommandExecutor mSSHCommandExecutor;
private final RSyncCommandExecutor mRSyncCommandExecutor;
- private final ImmutableMap<String, String> mTemplateDefaults;
- private final Logger mLogger;
+ protected final ImmutableMap<String, String> mTemplateDefaults;
+ protected final Logger mLogger;
private final File mLocalScratchDirectory;
private final File mSuccessfulTestLogDir;
private final File mFailedTestLogDir;
- private final long mNumPollSeconds;
+ protected final long mNumPollSeconds;
private final boolean fetchLogsForSuccessfulTests;
- private volatile boolean mShutdown;
- private int numParallelBatchesProcessed = 0;
- private int numIsolatedBatchesProcessed = 0;
+ protected volatile boolean mShutdown;
+ protected int numParallelBatchesProcessed = 0;
+ protected int numIsolatedBatchesProcessed = 0;
private AtomicLong totalElapsedTimeInRsync = new AtomicLong(0L);
- HostExecutor(Host host, String privateKey, ListeningExecutorService executor,
- SSHCommandExecutor sshCommandExecutor,
- RSyncCommandExecutor rsyncCommandExecutor,
- ImmutableMap<String, String> templateDefaults, File scratchDir,
- File succeededLogDir, File failedLogDir, long numPollSeconds,
- boolean fetchLogsForSuccessfulTests, Logger logger) {
+ protected HostExecutor(Host host, String privateKey, ListeningExecutorService executor,
+ SSHCommandExecutor sshCommandExecutor, RSyncCommandExecutor rsyncCommandExecutor,
+ ImmutableMap<String, String> templateDefaults, File scratchDir, File succeededLogDir,
+ File failedLogDir, long numPollSeconds, boolean fetchLogsForSuccessfulTests, Logger logger) {
List<Drone> drones = Lists.newArrayList();
String[] localDirs = host.getLocalDirectories();
for (int index = 0; index < host.getThreads(); index++) {
@@ -103,7 +101,7 @@ public class HostExecutor {
/**
* @return failed tests
*/
- ListenableFuture<Void> submitTests(final BlockingQueue<TestBatch> parallelWorkQueue,
+ protected ListenableFuture<Void> submitTests(final BlockingQueue<TestBatch> parallelWorkQueue,
final BlockingQueue<TestBatch> isolatedWorkQueue, final Set<TestBatch> failedTestResults) {
return mExecutor.submit(new Callable<Void>() {
@Override
@@ -152,7 +150,7 @@ public class HostExecutor {
* leaving this host with zero functioning drones. If all drones
* are removed the host will be replaced before the next run.
*/
- private void executeTests(final BlockingQueue<TestBatch> parallelWorkQueue,
+ protected void executeTests(final BlockingQueue<TestBatch> parallelWorkQueue,
final BlockingQueue<TestBatch> isolatedWorkQueue, final Set<TestBatch> failedTestResults)
throws Exception {
if(mShutdown) {
http://git-wip-us.apache.org/repos/asf/hive/blob/ea53020e/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/AbortContainerException.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/AbortContainerException.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/AbortContainerException.java
new file mode 100644
index 0000000..efc0b66
--- /dev/null
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/AbortContainerException.java
@@ -0,0 +1,23 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hive.ptest.execution.containers;
+
+public class AbortContainerException extends Exception {
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/ea53020e/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerClient.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerClient.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerClient.java
index d2428a9..767459c 100644
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerClient.java
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerClient.java
@@ -23,6 +23,10 @@ import org.apache.hive.ptest.api.server.TestLogger;
import org.apache.hive.ptest.execution.ContainerClient;
import org.apache.hive.ptest.execution.ContainerClientFactory.ContainerClientContext;
import org.apache.hive.ptest.execution.Templates;
+<<<<<<< HEAD
+=======
+import org.apache.hive.ptest.execution.conf.TestBatch;
+>>>>>>> a1fe94a... Temp work related to docker execution phase. code may not compile
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -47,9 +51,17 @@ public class DockerClient implements ContainerClient {
}
@Override
+<<<<<<< HEAD
public void defineImage(String dir, String label) throws IOException {
if (label == null)
label = UUID.randomUUID().toString();
+=======
+ public void defineImage(String dir) throws Exception {
+ final String label = context.getTemplateDefaults().get("buildTag");
+ if (label == null) {
+ throw new Exception("buildTag not found");
+ }
+>>>>>>> a1fe94a... Temp work related to docker execution phase. code may not compile
File dockerfile = new File(dir, "Dockerfile");
logger.info("Writing {} from template", dockerfile);
Map<String, String> templateDefaults = context.getTemplateDefaults();
@@ -68,8 +80,12 @@ public class DockerClient implements ContainerClient {
logger.info("Building image");
String dockerBuildCommand =
new StringBuilder("docker build")
+<<<<<<< HEAD
//TODO do we need --tag?
//.append(" --tag " + imageName())
+=======
+ .append(" --tag " + imageName())
+>>>>>>> a1fe94a... Temp work related to docker execution phase. code may not compile
.append(" --build-arg ")
.append(" workingDir=$workingDir")
.append(" --build-arg ")
@@ -87,8 +103,30 @@ public class DockerClient implements ContainerClient {
return dockerBuildCommand;
}
+<<<<<<< HEAD
private String imageName() {
//TODO add a proper image name using patch number
return "Ptest-dummy-build";
+=======
+ @Override
+ public String getRunContainerCommand(String containerName, TestBatch batch) {
+ return new StringBuilder("docker run")
+ .append(" --name " + containerName)
+ .append(" " + imageName())
+ .append(" /bin/bash")
+ .append(" -c")
+ .append("( cd " + batch.getTestModuleRelativeDir() + "; ")
+ .append("/usr/bin/mvn")
+ .append(" -Dsurefire.timeout=40m")
+ .append(" -B test")
+ .append(" " + batch.getTestArguments())
+ .append(" 1>$workingDir/logs" + File.separatorChar + "maven.txt")
+ .append(" 2>&1")
+ .toString();
+ }
+
+ private String imageName() {
+ return context.getTemplateDefaults().get("buildTag");
+>>>>>>> a1fe94a... Temp work related to docker execution phase. code may not compile
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/ea53020e/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerExecutionPhase.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerExecutionPhase.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerExecutionPhase.java
new file mode 100644
index 0000000..4774933
--- /dev/null
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerExecutionPhase.java
@@ -0,0 +1,73 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hive.ptest.execution.containers;
+
+import com.google.common.base.Supplier;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
+import org.apache.hive.ptest.execution.ContainerClient;
+import org.apache.hive.ptest.execution.ContainerClientFactory;
+import org.apache.hive.ptest.execution.ContainerClientFactory.ContainerClientContext;
+import org.apache.hive.ptest.execution.ContainerClientFactory.ContainerType;
+import org.apache.hive.ptest.execution.ExecutionPhase;
+import org.apache.hive.ptest.execution.HostExecutor;
+import org.apache.hive.ptest.execution.HostExecutorBuilder;
+import org.apache.hive.ptest.execution.LocalCommandFactory;
+import org.apache.hive.ptest.execution.conf.TestBatch;
+import org.apache.hive.ptest.execution.context.ExecutionContext;
+import org.apache.hive.ptest.execution.ssh.RemoteCommandResult;
+import org.slf4j.Logger;
+
+import java.io.File;
+import java.util.List;
+import java.util.Set;
+
+public class DockerExecutionPhase extends ExecutionPhase {
+ private final ContainerClient dockerClient;
+ public DockerExecutionPhase(List<HostExecutor> hostExecutors, ExecutionContext executionContext,
+ HostExecutorBuilder hostExecutorBuilder, LocalCommandFactory localCommandFactory,
+ ImmutableMap<String, String> templateDefaults, File succeededLogDir, File failedLogDir,
+ Supplier<List<TestBatch>> testBatchSupplier, Set<String> executedTests,
+ Set<String> failedTests, Logger logger) throws Exception {
+ super(hostExecutors, executionContext, hostExecutorBuilder, localCommandFactory,
+ templateDefaults, succeededLogDir, failedLogDir, testBatchSupplier, executedTests,
+ failedTests, logger);
+ ContainerClientContext context = new ContainerClientContext(logger, templateDefaults);
+ dockerClient = ContainerClientFactory.get(ContainerType.DOCKER, context);
+ }
+
+ @Override
+ public List<TestBatch> getTestBatches() {
+ List<TestBatch> testBatches = Lists.newArrayList();
+ //Docker containers should be able to run all the batches in parallel
+ for (TestBatch batch : testBatchSupplier.get()) {
+ testBatches.add(batch);
+ parallelWorkQueue.add(batch);
+ }
+ return testBatches;
+ }
+
+ @Override
+ protected List<RemoteCommandResult> initalizeHosts()
+ throws Exception {
+ //TODO kill docker containers in case they are running here
+ return Lists.newArrayList();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/ea53020e/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerHostExectutor.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerHostExectutor.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerHostExectutor.java
new file mode 100644
index 0000000..133d54a
--- /dev/null
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerHostExectutor.java
@@ -0,0 +1,177 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hive.ptest.execution.containers;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Stopwatch;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.ListeningExecutorService;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hive.ptest.execution.AbortDroneException;
+import org.apache.hive.ptest.execution.Constants;
+import org.apache.hive.ptest.execution.ContainerClient;
+import org.apache.hive.ptest.execution.ContainerClientFactory;
+import org.apache.hive.ptest.execution.ContainerClientFactory.ContainerClientContext;
+import org.apache.hive.ptest.execution.ContainerClientFactory.ContainerType;
+import org.apache.hive.ptest.execution.Dirs;
+import org.apache.hive.ptest.execution.HostExecutor;
+import org.apache.hive.ptest.execution.Templates;
+import org.apache.hive.ptest.execution.conf.Host;
+import org.apache.hive.ptest.execution.conf.TestBatch;
+import org.apache.hive.ptest.execution.ssh.RSyncCommandExecutor;
+import org.apache.hive.ptest.execution.ssh.RemoteCommandResult;
+import org.apache.hive.ptest.execution.ssh.SSHCommand;
+import org.apache.hive.ptest.execution.ssh.SSHCommandExecutor;
+import org.slf4j.Logger;
+
+import java.io.File;
+import java.io.PrintWriter;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.Callable;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+public class DockerHostExectutor extends HostExecutor {
+ private final ContainerClientContext containerClientContext;
+ private final ContainerClient dockerClient;
+ private final int numParallelContainersPerHost;
+ private String containerName;
+
+ DockerHostExectutor(Host host, String privateKey, ListeningExecutorService executor,
+ SSHCommandExecutor sshCommandExecutor, RSyncCommandExecutor rsyncCommandExecutor,
+ ImmutableMap<String, String> templateDefaults, File scratchDir, File succeededLogDir,
+ File failedLogDir, long numPollSeconds, boolean fetchLogsForSuccessfulTests, Logger logger)
+ throws Exception {
+ super(host, privateKey, executor, sshCommandExecutor, rsyncCommandExecutor, templateDefaults,
+ scratchDir, succeededLogDir, failedLogDir, numPollSeconds, fetchLogsForSuccessfulTests,
+ logger);
+ containerClientContext = new ContainerClientContext(logger, templateDefaults);
+ dockerClient = ContainerClientFactory.get(ContainerType.DOCKER, containerClientContext);
+ //TODO get this value from executionContext
+ numParallelContainersPerHost = 3;
+ }
+
+ @Override
+ protected void executeTests(final BlockingQueue<TestBatch> parallelWorkQueue,
+ final BlockingQueue<TestBatch> isolatedWorkQueue, final Set<TestBatch> failedTestResults)
+ throws Exception {
+ if(mShutdown) {
+ mLogger.warn("Shutting down host " + mHost.getName());
+ return;
+ }
+ mLogger.info("Starting parallel execution on " + mHost.getName() + " using dockers");
+ List<ListenableFuture<Void>> containerResults = Lists.newArrayList();
+ for(int containerId = 0; containerId < numParallelContainersPerHost; containerId++) {
+ containerResults.add(mExecutor.submit(new Callable<Void>() {
+ @Override
+ public Void call() throws Exception {
+ TestBatch batch = null;
+ Stopwatch sw = Stopwatch.createUnstarted();
+ try {
+ do {
+ batch = parallelWorkQueue.poll(mNumPollSeconds, TimeUnit.SECONDS);
+ if(mShutdown) {
+ mLogger.warn("Shutting down host " + mHost.getName());
+ return null;
+ }
+ if(batch != null) {
+ numParallelBatchesProcessed++;
+ sw.reset().start();
+ try {
+ if (!executeTestBatch(batch, failedTestResults)) {
+ failedTestResults.add(batch);
+ }
+ } finally {
+ sw.stop();
+ mLogger.info("Finished processing parallel batch [{}] on host {}. ElapsedTime(ms)={}",
+ new Object[]{batch.getName(),mHost.toShortString(), sw.elapsed(TimeUnit.MILLISECONDS)});
+ }
+ }
+ } while(!mShutdown && !parallelWorkQueue.isEmpty());
+ } catch(AbortContainerException ex) {
+ mLogger.error("Aborting container during parallel execution", ex);
+ if(batch != null) {
+ Preconditions.checkState(parallelWorkQueue.add(batch),
+ "Could not add batch to parallel queue " + batch);
+ }
+ }
+ return null;
+ }
+ }));
+ }
+ if(mShutdown) {
+ mLogger.warn("Shutting down host " + mHost.getName());
+ return;
+ }
+ Futures.allAsList(containerResults).get();
+ }
+
+ private boolean executeTestBatch(TestBatch batch, Set<TestBatch> failedTestResults)
+ throws AbortContainerException {
+ String runCommand = dockerClient.getRunContainerCommand(getContainerName(), batch);
+ Stopwatch sw = Stopwatch.createStarted();
+ mLogger.info("Executing " + batch + " with " + runCommand);
+ sw.stop();
+ /*mLogger.info("Completed executing tests for batch [{}] on host {}. ElapsedTime(ms)={}",
+ new Object[] { batch.getName(), getHost().toShortString(),
+ sw.elapsed(TimeUnit.MILLISECONDS) });
+ File batchLogDir = null;
+ if (sshResult.getExitCode() == Constants.EXIT_CODE_UNKNOWN) {
+ throw new AbortDroneException(
+ "Drone " + drone.toString() + " exited with " + Constants.EXIT_CODE_UNKNOWN + ": " + sshResult);
+ }
+ if (mShutdown) {
+ mLogger.warn("Shutting down host " + mHost.getName());
+ return false;
+ }
+ boolean result;
+ if (sshResult.getExitCode() != 0 || sshResult.getException() != null) {
+ result = false;
+ batchLogDir = Dirs.create(new File(mFailedTestLogDir, batch.getName()));
+ } else {
+ result = true;
+ batchLogDir = Dirs.create(new File(mSuccessfulTestLogDir, batch.getName()));
+ }
+ copyFromDroneToLocal(drone, batchLogDir.getAbsolutePath(), drone.getLocalLogDirectory() + "/",
+ fetchLogsForSuccessfulTests || !result);
+ File logFile = new File(batchLogDir, String.format("%s.txt", batch.getName()));
+ PrintWriter writer = new PrintWriter(logFile);
+ writer.write(String.format("result = '%s'\n", sshResult.toString()));
+ writer.write(String.format("output = '%s'\n", sshResult.getOutput()));
+ if (sshResult.getException() != null) {
+ sshResult.getException().printStackTrace(writer);
+ }
+ writer.close();*/
+ return false;
+ }
+
+ AtomicLong containerNameId = new AtomicLong(1);
+ public String getContainerName() {
+ return mHost.getName() + "-" + mTemplateDefaults.get("buildTag") + "-" + String
+ .valueOf(containerNameId.getAndIncrement());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/ea53020e/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerPrepPhase.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerPrepPhase.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerPrepPhase.java
index b1d9ad6..0e0a706 100644
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerPrepPhase.java
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerPrepPhase.java
@@ -54,7 +54,11 @@ public class DockerPrepPhase extends PrepPhase {
long elapsedTime;
start = System.currentTimeMillis();
//TODO give a proper label to the build
+<<<<<<< HEAD
containerClient.defineImage(getLocalScratchDir(), null);
+=======
+ containerClient.defineImage(getLocalScratchDir());
+>>>>>>> a1fe94a... Temp work related to docker execution phase. code may not compile
execLocally(getDockerBuildCommand());
elapsedTime = TimeUnit.MINUTES.convert((System.currentTimeMillis() - start),
TimeUnit.MILLISECONDS);
http://git-wip-us.apache.org/repos/asf/hive/blob/ea53020e/testutils/ptest2/src/main/resources/dockerfile-template.vm
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/resources/dockerfile-template.vm b/testutils/ptest2/src/main/resources/dockerfile-template.vm
index 3652720..9383e96 100644
--- a/testutils/ptest2/src/main/resources/dockerfile-template.vm
+++ b/testutils/ptest2/src/main/resources/dockerfile-template.vm
@@ -17,7 +17,11 @@ ARG workingDir
ARG mavenEnvOpts
ARG repository
ARG branch
+<<<<<<< HEAD
ARG label
+=======
+ARG buildTag
+>>>>>>> a1fe94a... Temp work related to docker execution phase. code may not compile
RUN export MAVEN_OPTS="$mavenEnvOpts"
RUN /usr/bin/git clone $repository
@@ -30,4 +34,8 @@ RUN /home/ptestuser/scratch/smart-apply-patch.sh /home/ptestuser/scratch/build.p
RUN /usr/bin/mvn -B -T 4 -q install -Dtest=TestMetastoreConf
RUN cd itests
RUN /usr/bin/mvn -B -T 4 -q install -DskipSparkTests -DskipTests
+<<<<<<< HEAD
RUN echo This build is labeled $label
+=======
+RUN echo This build is labeled $buildTag
+>>>>>>> a1fe94a... Temp work related to docker execution phase. code may not compile
[8/8] hive git commit: Running multiple test batches on one host
using containers
Posted by vi...@apache.org.
Running multiple test batches on one host using containers
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a2b2e125
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a2b2e125
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a2b2e125
Branch: refs/heads/HIVE-19429
Commit: a2b2e125e547c2fafff0daa001e1de249ebb916d
Parents: 47f76b3
Author: Vihang Karajgaonkar <vi...@cloudera.com>
Authored: Mon Jun 25 22:17:15 2018 -0700
Committer: Vihang Karajgaonkar <vi...@cloudera.com>
Committed: Mon Jun 25 22:17:15 2018 -0700
----------------------------------------------------------------------
.../hive/ptest/execution/ContainerClient.java | 4 +-
.../execution/containers/DockerClient.java | 47 ++++----
.../containers/DockerHostExecutor.java | 34 ++++--
.../ptest2/src/main/resources/copy-test-logs.vm | 14 +--
.../ptest/execution/TestPtestOnDockers.java | 107 +++++++++++++++++--
.../ptest2/src/test/resources/DUMMY-002.patch | 51 +++++++++
6 files changed, 206 insertions(+), 51 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/a2b2e125/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ContainerClient.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ContainerClient.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ContainerClient.java
index 6d3b112..02fda33 100644
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ContainerClient.java
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ContainerClient.java
@@ -27,9 +27,9 @@ public interface ContainerClient {
void defineImage(String dir) throws Exception;
String getBuildCommand(String dir, long toWait, TimeUnit unit)
throws Exception;
- String getRunContainerCommand(String containerName, final String imageTag, TestBatch testBatch);
+ String getRunContainerCommand(String containerName, TestBatch testBatch);
String getCopyTestLogsCommand(String containerName, String dir);
- String getStopContainerCommand(String containerName);
+ String getStopContainerCommand(String containerName, boolean forceRemove);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/a2b2e125/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerClient.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerClient.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerClient.java
index 5d107a1..fe1a547 100644
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerClient.java
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerClient.java
@@ -85,41 +85,46 @@ public class DockerClient implements ContainerClient {
}
@Override
- public String getRunContainerCommand(String containerName, final String imageTag, TestBatch batch) {
- return new StringBuilder("docker run")
- .append(" -t " + imageTag)
+ public String getRunContainerCommand(String containerName, TestBatch batch) {
+ return new StringBuilder("/usr/local/bin/docker run")
+ .append(" --memory " + "2G")
.append(" --name " + containerName)
- .append(" " + imageName())
+ //.append(" -d")
+ .append(" -t " + imageName())
.append(" /bin/bash")
.append(" -c")
- .append("( cd " + batch.getTestModuleRelativeDir() + "; ")
- .append("/usr/bin/mvn")
- .append(" -Dsurefire.timeout=40m")
+ .append(" \"cd " + batch.getTestModuleRelativeDir() + ";")
+ .append(" /usr/bin/mvn")
+ .append(" -Dsurefire.timeout=2400")
.append(" -B test")
.append(" " + batch.getTestArguments())
- .append(" 1>$workingDir/logs" + File.separatorChar + "maven.txt")
- .append(" 2>&1")
+ .append(
+ " 1>" + File.separatorChar + "/tmp" + File.separatorChar + "maven.txt")
+ .append(" 2>&1 ;")
+ .append(
+ " /bin/bash /home/ptestuser/scratch/copy-test-logs.sh 1>" + File.separatorChar + "tmp"
+ + File.separatorChar + "copy-test-logs.txt 2>&1; \"")
.toString();
}
@Override
public String getCopyTestLogsCommand(String containerName, String dir) {
- return new StringBuilder("docker run")
- .append(" --name " + containerName)
- .append(" " + imageName())
- .append(" /bin/bash")
- .append(" -c")
- .append("( cd " + dir + "; ")
- .append("bash")
- .append(" copy-test-logs.sh")
- .append(" 1>$workingDir/logs" + File.separatorChar + "copy-test-logs.txt")
- .append(" 2>&1")
+ String containerLogDir = context.getTemplateDefaults().get("containerLogDir");
+ //TODO get path for docker executable from context
+ return new StringBuilder("/usr/local/bin/docker cp")
+ .append(" " + containerName + ":" + containerLogDir)
+ .append(" " + dir)
.toString();
}
@Override
- public String getStopContainerCommand(String containerName) {
- return new StringBuilder("docker stop " + containerName).toString();
+ public String getStopContainerCommand(String containerName, boolean forceRemove) {
+ //TODO get path for docker executable from context
+ StringBuilder ret = new StringBuilder("/usr/local/bin/docker stop " + containerName);
+ if (forceRemove) {
+ ret.append("; /usr/local/bin/docker rm " + containerName);
+ }
+ return ret.toString();
}
private String imageName() {
http://git-wip-us.apache.org/repos/asf/hive/blob/a2b2e125/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerHostExecutor.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerHostExecutor.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerHostExecutor.java
index 4aa03d2..51b8fea 100644
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerHostExecutor.java
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerHostExecutor.java
@@ -141,7 +141,7 @@ public class DockerHostExecutor extends HostExecutor {
throws AbortContainerException, IOException, SSHExecutionException {
final int containerInstanceId = containerNameId.getAndIncrement();
final String containerName = getContainerName(containerInstanceId);
- String runCommand = dockerClient.getRunContainerCommand(containerName, mTemplateDefaults.get("buildTag"), batch);
+ String runCommand = dockerClient.getRunContainerCommand(containerName, batch);
Stopwatch sw = Stopwatch.createStarted();
mLogger.info("Executing " + batch + " with " + runCommand);
RemoteCommandResult sshResult = new SSHCommand(mSSHCommandExecutor, mPrivateKey, mHost.getUser(),
@@ -165,27 +165,33 @@ public class DockerHostExecutor extends HostExecutor {
}
boolean result;
if (sshResult.getExitCode() != 0 || sshResult.getException() != null) {
+ mLogger.debug(sshResult.getOutput());
result = false;
batchLogDir = Dirs.create(new File(mFailedTestLogDir, batch.getName()));
} else {
result = true;
batchLogDir = Dirs.create(new File(mSuccessfulTestLogDir, batch.getName()));
}
- String copyLogsCommand = dockerClient.getCopyTestLogsCommand(containerName,"/home/ptestuser/scratch/");
+ String copyLogsCommand = dockerClient.getCopyTestLogsCommand(containerName, batchLogDir.getAbsolutePath());
sw = Stopwatch.createStarted();
- mLogger.info("Copying logs for the " + batch + " with " + runCommand);
+ mLogger.info("Copying logs for the " + batch + " with " + copyLogsCommand);
sshResult = new SSHCommand(mSSHCommandExecutor, mPrivateKey, mHost.getUser(),
mHost.getName(), containerInstanceId, copyLogsCommand, true).
call();
sw.stop();
- mLogger.info(
- "Completed copying logs for batch [{}] on host {} using container instance {}. ElapsedTime(ms)={}",
- new Object[] { batch.getName(), mHost.toShortString(), containerInstanceId,
- sw.elapsed(TimeUnit.MILLISECONDS) });
+ if (sshResult.getExitCode() != 0 || sshResult.getException() != null) {
+ mLogger.error("Could not copy logs for batch [{}] on host {} using container instance {}. ElapsedTime(ms)={}",
+ new Object[] { batch.getName(), mHost.toShortString(), containerInstanceId,
+ sw.elapsed(TimeUnit.MILLISECONDS) });
+ //TODO do we need to throw error?
+ //throw new AbortContainerException("Could not stop container after test execution");
+ } else {
+ mLogger.info(
+ "Completed copying logs for batch [{}] on host {} using container instance {}. ElapsedTime(ms)={}",
+ new Object[] { batch.getName(), mHost.toShortString(), containerInstanceId,
+ sw.elapsed(TimeUnit.MILLISECONDS) });
+ }
- //Copy log files from the container to Ptest server
- copyFromContainerHostToLocal(containerInstanceId, batchLogDir.getAbsolutePath(),
- mHost.getLocalDirectories()[containerSlotId] + "/", fetchLogsForSuccessfulTests || !result);
File logFile = new File(batchLogDir, String.format("%s.txt", batch.getName()));
PrintWriter writer = new PrintWriter(logFile);
writer.write(String.format("result = '%s'\n", sshResult.toString()));
@@ -194,8 +200,14 @@ public class DockerHostExecutor extends HostExecutor {
sshResult.getException().printStackTrace(writer);
}
writer.close();
+
+ //Copy log files from the container to Ptest server
+ //TODO original code had String[] for localDirectories
+ copyFromContainerHostToLocal(containerInstanceId, batchLogDir.getAbsolutePath(),
+ mHost.getLocalDirectories()[0] + "/", fetchLogsForSuccessfulTests || !result);
+
//TODO add code to shutdown the container and delete it
- String stopContainerCommand = dockerClient.getStopContainerCommand(containerName);
+ String stopContainerCommand = dockerClient.getStopContainerCommand(containerName, true);
sw = Stopwatch.createStarted();
mLogger.info("Stopping container " + containerName + " with " + stopContainerCommand);
sshResult = new SSHCommand(mSSHCommandExecutor, mPrivateKey, mHost.getUser(),
http://git-wip-us.apache.org/repos/asf/hive/blob/a2b2e125/testutils/ptest2/src/main/resources/copy-test-logs.vm
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/resources/copy-test-logs.vm b/testutils/ptest2/src/main/resources/copy-test-logs.vm
index d218e90..4cf110d 100644
--- a/testutils/ptest2/src/main/resources/copy-test-logs.vm
+++ b/testutils/ptest2/src/main/resources/copy-test-logs.vm
@@ -22,19 +22,19 @@ ps -e -o pid,pgrp,user,args
ps x -o "%p %r %y %x %c "
date +"%Y-%m-%d %T.%3N"
-mkdir ${logDir}/logs
+mkdir -p ${containerLogDir}
cd /home/ptestuser/hive
find ./ -type f -name 'TEST-*.xml' | \
-xargs -I {} sh -c 'f=TEST-${batchName}-$(basename {}); test -f ${logDir}/$f && f=$f-$(uuidgen); mv {} ${logDir}/$f'
+xargs -I {} sh -c 'f=TEST-${batchName}-$(basename {}); test -f ${containerLogDir}/$f && f=$f-$(uuidgen); mv {} ${containerLogDir}/$f'
find ./ -type f -name hive.log -o -name spark.log -o -name derby.log | \
-xargs -I {} sh -c 'f=$(basename {}); test -f ${logDir}/logs/$f && f=$f-$(uuidgen); mv {} ${logDir}/logs/$f'
+xargs -I {} sh -c 'f=$(basename {}); test -f ${containerLogDir}/logs/$f && f=$f-$(uuidgen); mv {} ${containerLogDir}/logs/$f'
find ./ -path "*/spark/work" | \
-xargs -I {} sh -c 'mv {} ${logDir}/logs/spark-log'
+xargs -I {} sh -c 'mv {} ${containerLogDir}/logs/spark-log'
find ./ -type f -name 'syslog*' | \
-xargs -I {} sh -c 'mkdir -p ${logDir}/logs/syslogs; mv {} ${logDir}/logs/syslogs'
+xargs -I {} sh -c 'mkdir -p ${containerLogDir}/logs/syslogs; mv {} ${containerLogDir}/logs/syslogs'
date +"%Y-%m-%d %T.%3N"
-if [[ -f $logDir/.log ]]
+if [[ -f $containerLogDir/.log ]]
then
-mv $logDir/.log $logDir/logs/dot.log
+mv $containerLogDir/.log $containerLogDir/logs/dot.log
fi
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/a2b2e125/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestPtestOnDockers.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestPtestOnDockers.java b/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestPtestOnDockers.java
index c5e82c4..4f9445e 100644
--- a/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestPtestOnDockers.java
+++ b/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestPtestOnDockers.java
@@ -19,12 +19,16 @@
package org.apache.hive.ptest.execution;
+import com.google.common.base.Splitter;
import com.google.common.base.Suppliers;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
import org.apache.hive.ptest.execution.LocalCommand.CollectLogPolicy;
import org.apache.hive.ptest.execution.conf.Host;
import org.apache.hive.ptest.execution.conf.QFileTestBatch;
@@ -36,6 +40,9 @@ import org.apache.hive.ptest.execution.containers.DockerPrepPhase;
import org.apache.hive.ptest.execution.containers.TestDockerPrepPhase;
import org.apache.hive.ptest.execution.context.ExecutionContext;
import org.apache.hive.ptest.execution.ssh.NonZeroExitCodeException;
+import org.apache.hive.ptest.execution.ssh.SSHCommand;
+import org.apache.hive.ptest.execution.ssh.SSHCommandExecutor;
+import org.apache.hive.ptest.execution.ssh.TestSSHCommandExecutor;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -49,7 +56,10 @@ import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
+import java.util.LinkedList;
import java.util.List;
+import java.util.Map;
+import java.util.Queue;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executors;
@@ -64,6 +74,47 @@ import static org.mockito.Mockito.spy;
public class TestPtestOnDockers {
//TODO add logic to ignore this test if docker is not found on the machine
+ private static class TestSSHCommandExecutor extends SSHCommandExecutor {
+ private final List<String> mCommands;
+ private final Map<String, Queue<Integer>> mFailures;
+ private final AtomicInteger matchCount = new AtomicInteger(0);
+ public TestSSHCommandExecutor(Logger logger) {
+ super(logger);
+ mCommands = Lists.newArrayList();
+ mFailures = Maps.newHashMap();
+ }
+ public synchronized List<String> getCommands() {
+ return mCommands;
+ }
+ public synchronized void putFailure(String command, Integer... exitCodes) {
+ Queue<Integer> queue = mFailures.get(command);
+ if(queue == null) {
+ queue = new LinkedList<Integer>();
+ mFailures.put(command, queue);
+ } else {
+ queue = mFailures.get(command);
+ }
+ for(Integer exitCode : exitCodes) {
+ queue.add(exitCode);
+ }
+ }
+ @Override
+ public synchronized void execute(SSHCommand command) {
+ mCommands.add(command.getCommand());
+ command.setOutput("");
+ Queue<Integer> queue = mFailures.get(command.getCommand());
+ if(queue == null || queue.isEmpty()) {
+ command.setExitCode(0);
+ } else {
+ matchCount.incrementAndGet();
+ command.setExitCode(queue.remove());
+ }
+ }
+
+ public int getMatchCount() {
+ return matchCount.get();
+ }
+ }
private DockerPrepPhase prepPhase;
private DockerExecutionPhase execPhase;
private static File dummyPatchFile;
@@ -85,16 +136,16 @@ public class TestPtestOnDockers {
private Host host;
private static final String LOCAL_DIR = "/some/local/dir";
- private static final String PRIVATE_KEY = "some.private.key";
- private static final String USER = "someuser";
- private static final String HOST = "somehost";
+ private static final String PRIVATE_KEY = "~/.ssh/id_rsa";
+ private static final String USER = "vihang";
+ private static final String HOST = "localhost";
private static final int INSTANCE = 13;
private static final String INSTANCE_NAME = HOST + "-" + USER + "-" + INSTANCE;
private static final String REAL_BRANCH = "master";
private static final String REAL_REPOSITORY = "https://github.com/apache/hive.git";
private static final String REAL_REPOSITORY_NAME = "apache-hive";
private static final String REAL_MAVEN_OPTS = "-Xmx2048m";
- private MockSSHCommandExecutor sshCommandExecutor;
+ private SSHCommandExecutor sshCommandExecutor;
private MockRSyncCommandExecutor rsyncCommandExecutor;
private static final String BUILD_TAG = "docker-ptest-tag";
private final Set<String> executedTests = Collections.newSetFromMap(new ConcurrentHashMap<String, Boolean>());
@@ -107,12 +158,13 @@ public class TestPtestOnDockers {
scratchDir = Dirs.create(new File(baseDir, "scratch"));
succeededLogDir = Dirs.create(new File(logDir, "succeeded"));
failedLogDir = Dirs.create(new File(logDir, "failed"));
- executor = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(2));
+ executor = MoreExecutors.listeningDecorator(Executors.newCachedThreadPool(
+ new ThreadFactoryBuilder().setDaemon(true).setNameFormat("DockerHostExecutor %d").build()));
executionContext = mock(ExecutionContext.class);
hostExecutorBuilder = mock(HostExecutorBuilder.class);
//use real localCommandFactory
localCommandFactory = new LocalCommandFactory(logger);
- sshCommandExecutor = spy(new MockSSHCommandExecutor(logger));
+ sshCommandExecutor = new SSHCommandExecutor(logger);
rsyncCommandExecutor = spy(new MockRSyncCommandExecutor(logger));
templateDefaults = ImmutableMap.<String, String>builder()
.put("localDir", LOCAL_DIR)
@@ -125,13 +177,14 @@ public class TestPtestOnDockers {
.put("repository", REAL_REPOSITORY)
.put("repositoryName", REAL_REPOSITORY_NAME)
.put("mavenEnvOpts", REAL_MAVEN_OPTS)
+ .put("containerLogDir", "/tmp/testlogs")
.build();
host = new Host(HOST, USER, new String[] { LOCAL_DIR }, 2);
}
@BeforeClass
public static void beforeClass() throws Exception {
- URL url = TestPtestOnDockers.class.getResource("/DUMMY-001.patch");
+ URL url = TestPtestOnDockers.class.getResource("/DUMMY-002.patch");
dummyPatchFile = new File(url.getFile());
Assert.assertTrue("Could not find dummy patch file " + dummyPatchFile.getAbsolutePath(),
dummyPatchFile.exists());
@@ -153,9 +206,43 @@ public class TestPtestOnDockers {
private void createTestBatches() throws Exception {
testBatches = new ArrayList<>();
- TestBatch qfileTestBatch = new QFileTestBatch(new AtomicInteger(1), "", "TestCliDriver", "",
- Sets.newHashSet("insert0.q"), true, "itests/qtest");
+ AtomicInteger batchIdCounter = new AtomicInteger(1);
+ TestBatch qfileTestBatch = new QFileTestBatch(batchIdCounter, "test", "TestCliDriver", "qfile",
+ Sets.newHashSet(Splitter.on(", ").split(
+ "ppd_join3.q, auto_join23.q, list_bucket_dml_11.q, join10.q, udf_lower.q, "
+ + "avro_type_evolution.q, constprog_dp.q, create_struct_table.q, "
+ + "skewjoin_mapjoin9.q, check_constraint.q, hook_context_cs.q, "
+ + "vector_parquet_nested_two_level_complex.q, exim_22_import_exist_authsuccess.q,"
+ + "groupby1.q, create_func1.q, cbo_rp_udf_udaf.q, vector_decimal_aggregate.q,"
+ + "create_skewed_table1.q, partition_wise_fileformat.q, union_ppr.q,"
+ + "spark_combine_equivalent_work.q, stats_partial_size.q, join32.q,"
+ + "list_bucket_dml_14.q, input34.q, udf_parse_url.q, "
+ + "schema_evol_text_nonvec_part.q, enforce_constraint_notnull.q, "
+ + "zero_rows_single_insert.q, ctas_char.q")),
+ true, "itests/qtest");
+
+ /*TestBatch unitTestBatch = new UnitTestBatch(new AtomicInteger(1), "test", Lists.newArrayList(
+ Splitter.on(", ").split("TestCommands, TestUserHS2ConnectionFileParser, TestBufferedRows, "
+ + "TestBeeLineOpts, TestHiveCli, TestClientCommandHookFactory, "
+ + "TestBeeLineExceptionHandling, TestBeelineArgParsing, TestIncrementalRows, "
+ + "TestShutdownHook, TestBeeLineHistory, TestHiveSchemaTool, TestTableOutputFormat")),
+ "beeline", true);*/
+ TestBatch unitTestBatch = new UnitTestBatch(batchIdCounter, "test", Lists.newArrayList(
+ Splitter.on(", ").split("TestCommands, TestUserHS2ConnectionFileParser, TestBufferedRows, "
+ + "TestBeeLineOpts")),
+ "beeline", true);
+
+ TestBatch failingQueryTestBatch =
+ new QFileTestBatch(batchIdCounter, "test", "TestCliDriver", "qfile",
+ Sets.newHashSet(Splitter.on(", ").split("dummy_failing_test.q")), true, "itests/qtest");
+
+ TestBatch failingUnitTestBatch =
+ new UnitTestBatch(batchIdCounter, "test", Lists.newArrayList("TestFakeFailure"), "service",
+ true);
testBatches.add(qfileTestBatch);
+ testBatches.add(unitTestBatch);
+ testBatches.add(failingQueryTestBatch);
+ testBatches.add(failingUnitTestBatch);
}
private void createHostExecutor() throws Exception {
@@ -176,7 +263,7 @@ public class TestPtestOnDockers {
* @throws Exception
*/
@Test
- public void testDockerFile() throws Throwable {
+ public void testPrepPhase() throws Throwable {
prepPhase.execute();
Assert.assertNotNull("Scratch directory needs to be set", prepPhase.getLocalScratchDir());
File dockerFile = new File(prepPhase.getLocalScratchDir(), "Dockerfile");
http://git-wip-us.apache.org/repos/asf/hive/blob/a2b2e125/testutils/ptest2/src/test/resources/DUMMY-002.patch
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/test/resources/DUMMY-002.patch b/testutils/ptest2/src/test/resources/DUMMY-002.patch
new file mode 100644
index 0000000..4bba56c
--- /dev/null
+++ b/testutils/ptest2/src/test/resources/DUMMY-002.patch
@@ -0,0 +1,51 @@
+commit 47f76b3dac1b21d106051ab145f85932a3e0aa78
+Author: Vihang Karajgaonkar <vi...@cloudera.com>
+Date: Mon Jun 25 17:27:55 2018 -0700
+
+ fake tests
+
+diff --git a/ql/src/test/queries/clientpositive/dummy_failing_test.q b/ql/src/test/queries/clientpositive/dummy_failing_test.q
+new file mode 100644
+index 0000000000000000000000000000000000000000..8d5b53aacb6fc1339164104f0123cbaabc4a06e0
+--- /dev/null
++++ b/ql/src/test/queries/clientpositive/dummy_failing_test.q
+@@ -0,0 +1,2 @@
++-! qt:dataset:src
++select * from table_not_exists limit 1;
+diff --git a/service/src/test/org/apache/hive/service/TestFakeFailure.java b/service/src/test/org/apache/hive/service/TestFakeFailure.java
+new file mode 100644
+index 0000000000000000000000000000000000000000..7b5a7ba9cd680a31dbb60adb7823e032881c5120
+--- /dev/null
++++ b/service/src/test/org/apache/hive/service/TestFakeFailure.java
+@@ -0,0 +1,31 @@
++/*
++ *
++ * Licensed to the Apache Software Foundation (ASF) under one
++ * or more contributor license agreements. See the NOTICE file
++ * distributed with this work for additional information
++ * regarding copyright ownership. The ASF licenses this file
++ * to you under the Apache License, Version 2.0 (the
++ * "License"); you may not use this file except in compliance
++ * with the License. You may obtain a copy of the License at
++ *
++ * http://www.apache.org/licenses/LICENSE-2.0
++ *
++ * Unless required by applicable law or agreed to in writing, software
++ * distributed under the License is distributed on an "AS IS" BASIS,
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ * See the License for the specific language governing permissions and
++ * limitations under the License.
++ */
++
++package org.apache.hive.service;
++
++import org.junit.Assert;
++import org.junit.Test;
++
++public class TestFakeFailure {
++ @Test
++ public void fakeTest() {
++ Assert.assertTrue("Something gotta be seriously wrong if this test didn't work",
++ 1 == 2);
++ }
++}
[7/8] hive git commit: fake tests
Posted by vi...@apache.org.
fake tests
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/47f76b3d
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/47f76b3d
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/47f76b3d
Branch: refs/heads/HIVE-19429
Commit: 47f76b3dac1b21d106051ab145f85932a3e0aa78
Parents: ebb40fe
Author: Vihang Karajgaonkar <vi...@cloudera.com>
Authored: Mon Jun 25 17:27:55 2018 -0700
Committer: Vihang Karajgaonkar <vi...@cloudera.com>
Committed: Mon Jun 25 17:27:55 2018 -0700
----------------------------------------------------------------------
.../queries/clientpositive/dummy_failing_test.q | 2 ++
.../apache/hive/service/TestFakeFailure.java | 31 ++++++++++++++++++++
2 files changed, 33 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/47f76b3d/ql/src/test/queries/clientpositive/dummy_failing_test.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/dummy_failing_test.q b/ql/src/test/queries/clientpositive/dummy_failing_test.q
new file mode 100644
index 0000000..8d5b53a
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/dummy_failing_test.q
@@ -0,0 +1,2 @@
+-! qt:dataset:src
+select * from table_not_exists limit 1;
http://git-wip-us.apache.org/repos/asf/hive/blob/47f76b3d/service/src/test/org/apache/hive/service/TestFakeFailure.java
----------------------------------------------------------------------
diff --git a/service/src/test/org/apache/hive/service/TestFakeFailure.java b/service/src/test/org/apache/hive/service/TestFakeFailure.java
new file mode 100644
index 0000000..7b5a7ba
--- /dev/null
+++ b/service/src/test/org/apache/hive/service/TestFakeFailure.java
@@ -0,0 +1,31 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hive.service;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestFakeFailure {
+ @Test
+ public void fakeTest() {
+ Assert.assertTrue("Something gotta be seriously wrong if this test didn't work",
+ 1 == 2);
+ }
+}
[2/8] hive git commit: Implements the logic to generate and build a
docker image in DockerTestPrepPhase
Posted by vi...@apache.org.
Implements the logic to generate and build a docker image in DockerTestPrepPhase
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/41928126
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/41928126
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/41928126
Branch: refs/heads/HIVE-19429
Commit: 41928126427f3b9f0a5732816712041211675ac0
Parents: 9765a43
Author: Vihang Karajgaonkar <vi...@cloudera.com>
Authored: Sat Jun 2 17:36:51 2018 -0700
Committer: Vihang Karajgaonkar <vi...@cloudera.com>
Committed: Mon Jun 4 11:33:21 2018 -0700
----------------------------------------------------------------------
.../hive/ptest/execution/ContainerClient.java | 31 ++++
.../ptest/execution/ContainerClientFactory.java | 58 +++++++
.../execution/DockerBasedExecutionPhase.java | 3 +
.../hive/ptest/execution/HostExecutor.java | 2 +-
.../org/apache/hive/ptest/execution/Phase.java | 1 +
.../apache/hive/ptest/execution/PrepPhase.java | 37 +++--
.../execution/containers/DockerClient.java | 94 +++++++++++
.../execution/containers/DockerPrepPhase.java | 73 +++++++++
.../src/main/resources/dockerfile-template.vm | 33 ++++
.../hive/ptest/execution/AbstractTestPhase.java | 4 +-
.../ptest/execution/TestPtestOnDockers.java | 161 +++++++++++++++++++
.../containers/TestDockerPrepPhase.java | 73 +++++++++
12 files changed, 553 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/41928126/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ContainerClient.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ContainerClient.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ContainerClient.java
new file mode 100644
index 0000000..a9a8ead
--- /dev/null
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ContainerClient.java
@@ -0,0 +1,31 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hive.ptest.execution;
+
+import org.apache.hive.ptest.execution.ContainerClientFactory.ContainerClientContext;
+
+import java.io.IOException;
+import java.util.concurrent.TimeUnit;
+
+public interface ContainerClient {
+ void defineImage(String dir, String label) throws IOException;
+ String getBuildCommand(String dir, long toWait, TimeUnit unit)
+ throws Exception;
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/41928126/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ContainerClientFactory.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ContainerClientFactory.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ContainerClientFactory.java
new file mode 100644
index 0000000..d14176b
--- /dev/null
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ContainerClientFactory.java
@@ -0,0 +1,58 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hive.ptest.execution;
+
+import com.google.common.collect.ImmutableMap;
+import org.apache.hive.ptest.api.server.TestLogger;
+import org.apache.hive.ptest.execution.containers.DockerClient;
+import org.slf4j.Logger;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public abstract class ContainerClientFactory {
+ public enum ContainerType {
+ DOCKER
+ }
+ public static ContainerClient get(ContainerType containerType, ContainerClientContext context) throws Exception {
+ switch(containerType) {
+ case DOCKER:
+ return new DockerClient(context);
+ default:
+ throw new Exception("Unknown container type");
+ }
+ }
+
+ public static class ContainerClientContext {
+ final Logger logger;
+ final Map<String, String> templateDefaults;
+ public ContainerClientContext(Logger logger, Map<String, String> templateDefaults) {
+ this.logger = logger;
+ this.templateDefaults = new HashMap<>(templateDefaults);
+ }
+ public Logger getLogger() {
+ return logger;
+ }
+
+ public Map<String, String> getTemplateDefaults() {
+ return templateDefaults;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/41928126/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/DockerBasedExecutionPhase.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/DockerBasedExecutionPhase.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/DockerBasedExecutionPhase.java
index 4b19ccc..d7f14f0 100644
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/DockerBasedExecutionPhase.java
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/DockerBasedExecutionPhase.java
@@ -23,12 +23,15 @@ import com.google.common.base.Supplier;
import com.google.common.collect.ImmutableMap;
import org.apache.hive.ptest.execution.conf.TestBatch;
import org.apache.hive.ptest.execution.context.ExecutionContext;
+import org.apache.hive.ptest.execution.ssh.RemoteCommandResult;
import org.slf4j.Logger;
import java.io.File;
+import java.io.FileWriter;
import java.io.IOException;
import java.util.List;
import java.util.Set;
+import java.util.UUID;
import java.util.concurrent.TimeUnit;
public class DockerBasedExecutionPhase extends ExecutionPhase {
http://git-wip-us.apache.org/repos/asf/hive/blob/41928126/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/HostExecutor.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/HostExecutor.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/HostExecutor.java
index 47347eb..24f0bf8 100644
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/HostExecutor.java
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/HostExecutor.java
@@ -55,7 +55,7 @@ import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListeningExecutorService;
-class HostExecutor {
+public class HostExecutor {
private final Host mHost;
private final List<Drone> mDrones;
private final ListeningExecutorService mExecutor;
http://git-wip-us.apache.org/repos/asf/hive/blob/41928126/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/Phase.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/Phase.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/Phase.java
index 34c66ce..45007f0 100644
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/Phase.java
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/Phase.java
@@ -160,6 +160,7 @@ public abstract class Phase {
results.addAll(flatten(futures));
return results;
}
+
private <T extends RemoteCommandResult> List<T> flatten(List<ListenableFuture<List<ListenableFuture<T>>>> futures)
throws Exception {
List<T> results = Lists.newArrayList();
http://git-wip-us.apache.org/repos/asf/hive/blob/41928126/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PrepPhase.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PrepPhase.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PrepPhase.java
index 8fef413..1f57e3a 100644
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PrepPhase.java
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PrepPhase.java
@@ -24,13 +24,14 @@ import java.io.PrintWriter;
import java.util.List;
import java.util.concurrent.TimeUnit;
+import org.apache.hive.ptest.execution.ssh.NonZeroExitCodeException;
import org.slf4j.Logger;
import com.google.common.collect.ImmutableMap;
public class PrepPhase extends Phase {
- private final File mScratchDir;
- private final File mPatchFile;
+ protected final File mScratchDir;
+ protected final File mPatchFile;
public PrepPhase(List<HostExecutor> hostExecutors,
LocalCommandFactory localCommandFactory,
@@ -40,23 +41,12 @@ public class PrepPhase extends Phase {
this.mScratchDir = scratchDir;
this.mPatchFile = patchFile;
}
+
@Override
public void execute() throws Exception {
execLocally("rm -rf $workingDir/scratch");
execLocally("mkdir -p $workingDir/scratch");
- if(mPatchFile != null) {
- File smartApplyPatch = new File(mScratchDir, "smart-apply-patch.sh");
- PrintWriter writer = new PrintWriter(smartApplyPatch);
- try {
- writer.write(Templates.readResource("smart-apply-patch.sh"));
- if(writer.checkError()) {
- throw new IOException("Error writing to " + smartApplyPatch);
- }
- } finally {
- writer.close();
- }
- execLocally("cp -f " + mPatchFile.getPath() + " $workingDir/scratch/build.patch");
- }
+ createPatchFiles();
long start;
long elapsedTime;
// source prep
@@ -72,4 +62,21 @@ public class PrepPhase extends Phase {
TimeUnit.MILLISECONDS);
logger.info("PERF: source prep took " + elapsedTime + " minutes");
}
+
+ protected void createPatchFiles()
+ throws IOException, InterruptedException, NonZeroExitCodeException {
+ if(mPatchFile != null) {
+ File smartApplyPatch = new File(mScratchDir, "smart-apply-patch.sh");
+ PrintWriter writer = new PrintWriter(smartApplyPatch);
+ try {
+ writer.write(Templates.readResource("smart-apply-patch.sh"));
+ if(writer.checkError()) {
+ throw new IOException("Error writing to " + smartApplyPatch);
+ }
+ } finally {
+ writer.close();
+ }
+ execLocally("cp -f " + mPatchFile.getPath() + " $workingDir/scratch/build.patch");
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/41928126/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerClient.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerClient.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerClient.java
new file mode 100644
index 0000000..d2428a9
--- /dev/null
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerClient.java
@@ -0,0 +1,94 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hive.ptest.execution.containers;
+
+import org.apache.hive.ptest.api.server.TestLogger;
+import org.apache.hive.ptest.execution.ContainerClient;
+import org.apache.hive.ptest.execution.ContainerClientFactory.ContainerClientContext;
+import org.apache.hive.ptest.execution.Templates;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+
+public class DockerClient implements ContainerClient {
+
+ private static final String USER = "ptestuser";
+ private static final String HOME_DIR = File.separatorChar + "home" + File.separatorChar + USER;
+ private static final String DOCKER_SCRATCH_DIR = HOME_DIR + File.separatorChar + "scratch" + File.separatorChar;
+ private static Logger logger;
+ private final ContainerClientContext context;
+ public DockerClient(ContainerClientContext context) {
+ logger = context.getLogger();
+ this.context = context;
+ }
+
+ @Override
+ public void defineImage(String dir, String label) throws IOException {
+ if (label == null)
+ label = UUID.randomUUID().toString();
+ File dockerfile = new File(dir, "Dockerfile");
+ logger.info("Writing {} from template", dockerfile);
+ Map<String, String> templateDefaults = context.getTemplateDefaults();
+ if (!templateDefaults.containsKey("label")) {
+ templateDefaults.put("label", label);
+ } else {
+ //TODO throw?
+ }
+ Templates.writeTemplateResult("dockerfile-template.vm", dockerfile, templateDefaults);
+ }
+
+ @Override
+ public String getBuildCommand(String dir, long toWait, TimeUnit unit) throws Exception {
+ //TODO timeout not used currently
+ long seconds = TimeUnit.SECONDS.convert(toWait, unit);
+ logger.info("Building image");
+ String dockerBuildCommand =
+ new StringBuilder("docker build")
+ //TODO do we need --tag?
+ //.append(" --tag " + imageName())
+ .append(" --build-arg ")
+ .append(" workingDir=$workingDir")
+ .append(" --build-arg ")
+ .append(" patchFilePath=$workingDir/scratch/build.patch")
+ //TODO confirm if we need antEnvOpts
+ //.append(" antEnvOpts=$antEnvOpts")
+ .append(" --build-arg ")
+ .append(" mavenEnvOpts=$mavenEnvOpts")
+ .append(" --build-arg ")
+ .append(" repository=$repository")
+ .append(" --build-arg ")
+ .append(" branch=$branch")
+ .append(" " + dir)
+ .toString();
+ return dockerBuildCommand;
+ }
+
+ private String imageName() {
+ //TODO add a proper image name using patch number
+ return "Ptest-dummy-build";
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/41928126/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerPrepPhase.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerPrepPhase.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerPrepPhase.java
new file mode 100644
index 0000000..b1d9ad6
--- /dev/null
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerPrepPhase.java
@@ -0,0 +1,73 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hive.ptest.execution.containers;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableMap;
+import org.apache.hive.ptest.execution.ContainerClient;
+import org.apache.hive.ptest.execution.ContainerClientFactory;
+import org.apache.hive.ptest.execution.ContainerClientFactory.ContainerClientContext;
+import org.apache.hive.ptest.execution.ContainerClientFactory.ContainerType;
+import org.apache.hive.ptest.execution.HostExecutor;
+import org.apache.hive.ptest.execution.LocalCommandFactory;
+import org.apache.hive.ptest.execution.PrepPhase;
+import org.slf4j.Logger;
+
+import java.io.File;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+public class DockerPrepPhase extends PrepPhase {
+ private final ContainerClient containerClient;
+ public DockerPrepPhase(List<HostExecutor> hostExecutors, LocalCommandFactory localCommandFactory,
+ ImmutableMap<String, String> templateDefaults, File scratchDir, File patchFile,
+ Logger logger) throws Exception {
+ super(hostExecutors, localCommandFactory, templateDefaults, scratchDir, patchFile, logger);
+ //TODO create context builder and pass the environment variables to the context
+ ContainerClientContext context = new ContainerClientContext(logger, templateDefaults);
+ containerClient = ContainerClientFactory.get(ContainerType.DOCKER, context);
+ }
+
+ @Override
+ public void execute() throws Exception {
+ execLocally("rm -rf $workingDir/scratch");
+ execLocally("mkdir -p $workingDir/scratch");
+ createPatchFiles();
+ long start;
+ long elapsedTime;
+ start = System.currentTimeMillis();
+ //TODO give a proper label to the build
+ containerClient.defineImage(getLocalScratchDir(), null);
+ execLocally(getDockerBuildCommand());
+ elapsedTime = TimeUnit.MINUTES.convert((System.currentTimeMillis() - start),
+ TimeUnit.MILLISECONDS);
+ logger.info("PERF: Docker source prep took " + elapsedTime + " minutes");
+ }
+
+ @VisibleForTesting
+ String getLocalScratchDir() {
+ return mScratchDir.getAbsolutePath();
+ }
+
+ @VisibleForTesting
+ public String getDockerBuildCommand() throws Exception {
+ return containerClient.getBuildCommand(getLocalScratchDir(), 30, TimeUnit.MINUTES);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/41928126/testutils/ptest2/src/main/resources/dockerfile-template.vm
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/resources/dockerfile-template.vm b/testutils/ptest2/src/main/resources/dockerfile-template.vm
new file mode 100644
index 0000000..3652720
--- /dev/null
+++ b/testutils/ptest2/src/main/resources/dockerfile-template.vm
@@ -0,0 +1,33 @@
+FROM centos
+
+RUN yum upgrade -y && \
+ yum update -y && \
+ yum install -y java-1.8.0-openjdk-devel unzip git maven
+
+RUN useradd -m ptestuser
+RUN mkdir -p /home/ptestuser
+COPY smart-apply-patch.sh scratch/build.patch /home/ptestuser/scratch/
+RUN chown -R ptestuser /home/ptestuser
+RUN chmod -R 755 /home/ptestuser
+RUN ls -l /home
+USER ptestuser
+RUN cd /home/ptestuser
+WORKDIR /home/ptestuser
+ARG workingDir
+ARG mavenEnvOpts
+ARG repository
+ARG branch
+ARG label
+
+RUN export MAVEN_OPTS="$mavenEnvOpts"
+RUN /usr/bin/git clone $repository
+RUN cd hive
+WORKDIR /home/ptestuser/hive
+RUN /usr/bin/git checkout $branch || checkout -b $branch origin/$branch
+RUN /usr/bin/git reset --hard origin/$branch
+RUN /usr/bin/git merge --ff-only origin/$branch
+RUN /home/ptestuser/scratch/smart-apply-patch.sh /home/ptestuser/scratch/build.patch
+RUN /usr/bin/mvn -B -T 4 -q install -Dtest=TestMetastoreConf
+RUN cd itests
+RUN /usr/bin/mvn -B -T 4 -q install -DskipSparkTests -DskipTests
+RUN echo This build is labeled $label
http://git-wip-us.apache.org/repos/asf/hive/blob/41928126/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/AbstractTestPhase.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/AbstractTestPhase.java b/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/AbstractTestPhase.java
index 71e1a64..01c65a6 100644
--- a/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/AbstractTestPhase.java
+++ b/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/AbstractTestPhase.java
@@ -58,6 +58,7 @@ public abstract class AbstractTestPhase {
protected static final String BRANCH = "branch";
protected static final String REPOSITORY = "repository";
protected static final String REPOSITORY_NAME = "repositoryName";
+ protected static final String MAVEN_OPTS = "-Xmx128m";
protected Host host;
protected File baseDir;
@@ -100,6 +101,7 @@ public abstract class AbstractTestPhase {
.put("logDir", logDir.getAbsolutePath())
.put("repository", REPOSITORY)
.put("repositoryName", REPOSITORY_NAME)
+ .put("mavenEnvOpts", MAVEN_OPTS)
.build();
host = new Host(HOST, USER, new String[] { LOCAL_DIR }, 2);
}
@@ -115,7 +117,7 @@ public abstract class AbstractTestPhase {
String osName = System.getProperty("os.name");
return osName.contains("OS X");
}
- static File createBaseDir(String name) throws IOException {
+ public static File createBaseDir(String name) throws IOException {
File baseDir;
if(isOSX()) {
// else osx gives ugly temp path which screws up approvals
http://git-wip-us.apache.org/repos/asf/hive/blob/41928126/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestPtestOnDockers.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestPtestOnDockers.java b/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestPtestOnDockers.java
new file mode 100644
index 0000000..516a12e
--- /dev/null
+++ b/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestPtestOnDockers.java
@@ -0,0 +1,161 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hive.ptest.execution;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.util.concurrent.ListeningExecutorService;
+import com.google.common.util.concurrent.MoreExecutors;
+import org.apache.commons.io.FileUtils;
+import org.apache.hive.ptest.execution.LocalCommand.CollectLogPolicy;
+import org.apache.hive.ptest.execution.conf.Host;
+import org.apache.hive.ptest.execution.containers.DockerPrepPhase;
+import org.apache.hive.ptest.execution.containers.TestDockerPrepPhase;
+import org.apache.hive.ptest.execution.context.ExecutionContext;
+import org.apache.hive.ptest.execution.ssh.NonZeroExitCodeException;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.util.concurrent.Executors;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+
+/**
+ * This test assumes that the host where this test is being run has docker installed
+ */
+public class TestPtestOnDockers {
+ //TODO add logic to ignore this test if docker is not found on the machine
+
+ private DockerPrepPhase phase;
+ private static File dummyPatchFile;
+ private static final Logger logger = LoggerFactory.getLogger(TestDockerPrepPhase.class);
+
+ private File baseDir;
+ private File scratchDir;
+ private File logDir;
+ private File succeededLogDir;
+ private File failedLogDir;
+ private ListeningExecutorService executor;
+ private LocalCommandFactory localCommandFactory;
+ private LocalCommand localCommand;
+ private ImmutableMap<String, String> templateDefaults;
+ private ImmutableList<HostExecutor> hostExecutors;
+ private HostExecutor hostExecutor;
+ private ExecutionContext executionContext;
+ private HostExecutorBuilder hostExecutorBuilder;
+ private Host host;
+
+ private static final String LOCAL_DIR = "/some/local/dir";
+ private static final String PRIVATE_KEY = "some.private.key";
+ private static final String USER = "someuser";
+ private static final String HOST = "somehost";
+ private static final int INSTANCE = 13;
+ private static final String INSTANCE_NAME = HOST + "-" + USER + "-" + INSTANCE;
+ private static final String REAL_BRANCH = "master";
+ private static final String REAL_REPOSITORY = "https://git-wip-us.apache.org/repos/asf/hive.git";
+ private static final String REAL_REPOSITORY_NAME = "apache-hive";
+ private static final String REAL_MAVEN_OPTS = "-Xmx1g";
+ private MockSSHCommandExecutor sshCommandExecutor;
+ private MockRSyncCommandExecutor rsyncCommandExecutor;
+
+ public void initialize(String name) throws Exception {
+ baseDir = AbstractTestPhase.createBaseDir(name);
+ logDir = Dirs.create(new File(baseDir, "logs"));
+ scratchDir = Dirs.create(new File(baseDir, "scratch"));
+ succeededLogDir = Dirs.create(new File(logDir, "succeeded"));
+ failedLogDir = Dirs.create(new File(logDir, "failed"));
+ executor = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(2));
+ executionContext = mock(ExecutionContext.class);
+ hostExecutorBuilder = mock(HostExecutorBuilder.class);
+ //use real localCommandFactory
+ localCommandFactory = new LocalCommandFactory(logger);
+ sshCommandExecutor = spy(new MockSSHCommandExecutor(logger));
+ rsyncCommandExecutor = spy(new MockRSyncCommandExecutor(logger));
+ templateDefaults = ImmutableMap.<String, String>builder()
+ .put("localDir", LOCAL_DIR)
+ //use baseDir as working directory
+ .put("workingDir", baseDir.getAbsolutePath())
+ .put("instanceName", INSTANCE_NAME)
+ .put("branch", REAL_BRANCH)
+ .put("logDir", logDir.getAbsolutePath())
+ .put("repository", REAL_REPOSITORY)
+ .put("repositoryName", REAL_REPOSITORY_NAME)
+ .put("mavenEnvOpts", REAL_MAVEN_OPTS)
+ .build();
+ host = new Host(HOST, USER, new String[] { LOCAL_DIR }, 2);
+ }
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ dummyPatchFile = File.createTempFile("dummy", "patch");
+ dummyPatchFile.deleteOnExit();
+ Assert.assertTrue("Could not create dummy patch file " + dummyPatchFile.getAbsolutePath(),
+ dummyPatchFile.exists());
+ }
+
+ @Before
+ public void setup() throws Exception {
+ initialize(getClass().getSimpleName());
+ createHostExecutor();
+ phase = new DockerPrepPhase(hostExecutors, localCommandFactory,
+ templateDefaults, baseDir, dummyPatchFile, logger);
+ }
+
+ private void createHostExecutor() {
+ hostExecutor = new HostExecutor(host, PRIVATE_KEY, executor, sshCommandExecutor,
+ rsyncCommandExecutor, templateDefaults, scratchDir, succeededLogDir, failedLogDir, 1, true,
+ logger);
+ hostExecutors = ImmutableList.of(hostExecutor);
+ }
+
+ @After
+ public void teardown() {
+ phase = null;
+ FileUtils.deleteQuietly(baseDir);
+ }
+
+ /**
+ * This test requires docker to be installed to test on local machine
+ * @throws Exception
+ */
+ @Test
+ public void testDockerFile() throws Exception {
+ phase.execute();
+ File dockerFile = new File(scratchDir, "Dockerfile");
+ Assert.assertTrue("Docker file was not found" , dockerFile.exists());
+ Assert.assertTrue("patch file not found", new File(scratchDir, "build.patch").exists());
+ LocalCommandFactory localCommandFactory = new LocalCommandFactory(logger);
+ CollectLogPolicy localCollector = new CollectLogPolicy(logger);
+ localCommandFactory.create(localCollector, "cp -f " + dockerFile.getAbsolutePath() + " /tmp/myDockerFile");
+ LocalCommand localCmd = localCommandFactory.create(localCollector, phase.getDockerBuildCommand());
+ if(localCmd.getExitCode() != 0) {
+ throw new NonZeroExitCodeException(String.format(
+ "Command '%s' failed with exit status %d and output '%s'",
+ localCmd, localCmd.getExitCode(), localCollector.getOutput()));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/41928126/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/containers/TestDockerPrepPhase.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/containers/TestDockerPrepPhase.java b/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/containers/TestDockerPrepPhase.java
new file mode 100644
index 0000000..8efc5e0
--- /dev/null
+++ b/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/containers/TestDockerPrepPhase.java
@@ -0,0 +1,73 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hive.ptest.execution.containers;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hive.ptest.execution.AbstractTestPhase;
+
+import org.apache.hive.ptest.execution.LocalCommand;
+import org.apache.hive.ptest.execution.LocalCommand.CollectLogPolicy;
+import org.apache.hive.ptest.execution.LocalCommandFactory;
+import org.apache.hive.ptest.execution.ssh.NonZeroExitCodeException;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+
+public class TestDockerPrepPhase extends AbstractTestPhase {
+ private DockerPrepPhase phase;
+ private static File dummyPatchFile;
+ private static final Logger logger = LoggerFactory.getLogger(TestDockerPrepPhase.class);
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ dummyPatchFile = File.createTempFile("dummy", "patch");
+ dummyPatchFile.deleteOnExit();
+ Assert.assertTrue("Could not create dummy patch file " + dummyPatchFile.getAbsolutePath(),
+ dummyPatchFile.exists());
+ }
+
+ @Before
+ public void setup() throws Exception {
+ initialize(getClass().getSimpleName());
+ createHostExecutor();
+ phase = new DockerPrepPhase(hostExecutors, localCommandFactory,
+ templateDefaults, baseDir, dummyPatchFile, logger);
+ }
+
+ @After
+ public void teardown() {
+ phase = null;
+ FileUtils.deleteQuietly(baseDir);
+ }
+
+ @Test
+ public void testExecute() throws Exception {
+ phase.execute();
+ Assert.assertNotNull("Scratch directory needs to be set", phase.getLocalScratchDir());
+ File dockerFile = new File(phase.getLocalScratchDir(), "Dockerfile");
+ Assert.assertTrue("Docker file not found", dockerFile.exists());
+ }
+}
[4/8] hive git commit: temp work related to dockers
Posted by vi...@apache.org.
temp work related to dockers
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/5787e77f
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/5787e77f
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/5787e77f
Branch: refs/heads/HIVE-19429
Commit: 5787e77f4c6b4303737f9388724a2c2fb5737292
Parents: ea53020
Author: Vihang Karajgaonkar <vi...@cloudera.com>
Authored: Tue Jun 5 11:16:58 2018 -0700
Committer: Vihang Karajgaonkar <vi...@cloudera.com>
Committed: Tue Jun 5 11:16:58 2018 -0700
----------------------------------------------------------------------
.../hive/ptest/execution/ContainerClient.java | 15 +-
.../hive/ptest/execution/HostExecutor.java | 16 +-
.../org/apache/hive/ptest/execution/PTest.java | 1 +
.../containers/AbortContainerException.java | 3 +
.../execution/containers/DockerClient.java | 55 ++---
.../containers/DockerHostExectutor.java | 177 --------------
.../containers/DockerHostExecutor.java | 233 +++++++++++++++++++
.../execution/containers/DockerPrepPhase.java | 4 -
.../ptest2/src/main/resources/copy-test-logs.vm | 40 ++++
.../src/main/resources/dockerfile-template.vm | 29 ++-
.../ptest/execution/TestPtestOnDockers.java | 13 +-
.../ptest2/src/test/resources/DUMMY-001.patch | 12 +
12 files changed, 353 insertions(+), 245 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/5787e77f/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ContainerClient.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ContainerClient.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ContainerClient.java
index d4f887c..59da132 100644
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ContainerClient.java
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ContainerClient.java
@@ -19,24 +19,17 @@
package org.apache.hive.ptest.execution;
-import org.apache.hive.ptest.execution.ContainerClientFactory.ContainerClientContext;
-<<<<<<< HEAD
-=======
import org.apache.hive.ptest.execution.conf.TestBatch;
->>>>>>> a1fe94a... Temp work related to docker execution phase. code may not compile
-import java.io.IOException;
import java.util.concurrent.TimeUnit;
public interface ContainerClient {
-<<<<<<< HEAD
- void defineImage(String dir, String label) throws IOException;
- String getBuildCommand(String dir, long toWait, TimeUnit unit)
- throws Exception;
-=======
void defineImage(String dir) throws Exception;
String getBuildCommand(String dir, long toWait, TimeUnit unit)
throws Exception;
String getRunContainerCommand(String containerName, TestBatch testBatch);
->>>>>>> a1fe94a... Temp work related to docker execution phase. code may not compile
+
+ String getCopyTestLogsCommand(String containerName, String dir);
+
+ String getStopContainerCommand(String containerName);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/5787e77f/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/HostExecutor.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/HostExecutor.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/HostExecutor.java
index 74be1e0..ee153f8 100644
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/HostExecutor.java
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/HostExecutor.java
@@ -59,19 +59,20 @@ public class HostExecutor {
protected final Host mHost;
private final List<Drone> mDrones;
protected final ListeningExecutorService mExecutor;
- private final SSHCommandExecutor mSSHCommandExecutor;
- private final RSyncCommandExecutor mRSyncCommandExecutor;
+ protected final SSHCommandExecutor mSSHCommandExecutor;
+ protected final RSyncCommandExecutor mRSyncCommandExecutor;
protected final ImmutableMap<String, String> mTemplateDefaults;
+ protected final String mPrivateKey;
protected final Logger mLogger;
- private final File mLocalScratchDirectory;
- private final File mSuccessfulTestLogDir;
- private final File mFailedTestLogDir;
+ protected final File mLocalScratchDirectory;
+ protected final File mSuccessfulTestLogDir;
+ protected final File mFailedTestLogDir;
protected final long mNumPollSeconds;
- private final boolean fetchLogsForSuccessfulTests;
+ protected final boolean fetchLogsForSuccessfulTests;
protected volatile boolean mShutdown;
protected int numParallelBatchesProcessed = 0;
protected int numIsolatedBatchesProcessed = 0;
- private AtomicLong totalElapsedTimeInRsync = new AtomicLong(0L);
+ protected AtomicLong totalElapsedTimeInRsync = new AtomicLong(0L);
protected HostExecutor(Host host, String privateKey, ListeningExecutorService executor,
SSHCommandExecutor sshCommandExecutor, RSyncCommandExecutor rsyncCommandExecutor,
@@ -85,6 +86,7 @@ public class HostExecutor {
}
mShutdown = false;
mHost = host;
+ mPrivateKey = privateKey;
mDrones = new CopyOnWriteArrayList<Drone>(drones);
mExecutor = executor;
mSSHCommandExecutor = sshCommandExecutor;
http://git-wip-us.apache.org/repos/asf/hive/blob/5787e77f/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java
index 2868ff0..6a6a5ff 100644
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java
@@ -69,6 +69,7 @@ import com.google.common.util.concurrent.MoreExecutors;
public class PTest {
+ // test patching
static {
Velocity.init();
}
http://git-wip-us.apache.org/repos/asf/hive/blob/5787e77f/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/AbortContainerException.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/AbortContainerException.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/AbortContainerException.java
index efc0b66..9b5a10b 100644
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/AbortContainerException.java
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/AbortContainerException.java
@@ -20,4 +20,7 @@
package org.apache.hive.ptest.execution.containers;
public class AbortContainerException extends Exception {
+ public AbortContainerException(String msg) {
+ super(msg);
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/5787e77f/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerClient.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerClient.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerClient.java
index 767459c..8e6e0cf 100644
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerClient.java
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerClient.java
@@ -19,25 +19,17 @@
package org.apache.hive.ptest.execution.containers;
-import org.apache.hive.ptest.api.server.TestLogger;
import org.apache.hive.ptest.execution.ContainerClient;
import org.apache.hive.ptest.execution.ContainerClientFactory.ContainerClientContext;
import org.apache.hive.ptest.execution.Templates;
-<<<<<<< HEAD
-=======
import org.apache.hive.ptest.execution.conf.TestBatch;
->>>>>>> a1fe94a... Temp work related to docker execution phase. code may not compile
import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import java.io.BufferedReader;
import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
import java.util.Map;
-import java.util.UUID;
import java.util.concurrent.TimeUnit;
+//TODO this implementation to actually run the commands instead of returning string commands
public class DockerClient implements ContainerClient {
private static final String USER = "ptestuser";
@@ -51,26 +43,18 @@ public class DockerClient implements ContainerClient {
}
@Override
-<<<<<<< HEAD
- public void defineImage(String dir, String label) throws IOException {
- if (label == null)
- label = UUID.randomUUID().toString();
-=======
public void defineImage(String dir) throws Exception {
final String label = context.getTemplateDefaults().get("buildTag");
if (label == null) {
throw new Exception("buildTag not found");
}
->>>>>>> a1fe94a... Temp work related to docker execution phase. code may not compile
File dockerfile = new File(dir, "Dockerfile");
+ File copyTestLogs = new File(dir, "copy-test-logs.sh");
logger.info("Writing {} from template", dockerfile);
Map<String, String> templateDefaults = context.getTemplateDefaults();
- if (!templateDefaults.containsKey("label")) {
- templateDefaults.put("label", label);
- } else {
- //TODO throw?
- }
Templates.writeTemplateResult("dockerfile-template.vm", dockerfile, templateDefaults);
+ //generate the script to copy test logs
+ Templates.writeTemplateResult("copy-test-logs.vm", copyTestLogs, templateDefaults);
}
@Override
@@ -80,12 +64,7 @@ public class DockerClient implements ContainerClient {
logger.info("Building image");
String dockerBuildCommand =
new StringBuilder("docker build")
-<<<<<<< HEAD
- //TODO do we need --tag?
- //.append(" --tag " + imageName())
-=======
.append(" --tag " + imageName())
->>>>>>> a1fe94a... Temp work related to docker execution phase. code may not compile
.append(" --build-arg ")
.append(" workingDir=$workingDir")
.append(" --build-arg ")
@@ -103,11 +82,6 @@ public class DockerClient implements ContainerClient {
return dockerBuildCommand;
}
-<<<<<<< HEAD
- private String imageName() {
- //TODO add a proper image name using patch number
- return "Ptest-dummy-build";
-=======
@Override
public String getRunContainerCommand(String containerName, TestBatch batch) {
return new StringBuilder("docker run")
@@ -125,8 +99,27 @@ public class DockerClient implements ContainerClient {
.toString();
}
+ @Override
+ public String getCopyTestLogsCommand(String containerName, String dir) {
+ return new StringBuilder("docker run")
+ .append(" --name " + containerName)
+ .append(" " + imageName())
+ .append(" /bin/bash")
+ .append(" -c")
+ .append("( cd " + dir + "; ")
+ .append("bash")
+ .append(" copy-test-logs.sh")
+ .append(" 1>$workingDir/logs" + File.separatorChar + "copy-test-logs.txt")
+ .append(" 2>&1")
+ .toString();
+ }
+
+ @Override
+ public String getStopContainerCommand(String containerName) {
+ return new StringBuilder("docker stop " + containerName).toString();
+ }
+
private String imageName() {
return context.getTemplateDefaults().get("buildTag");
->>>>>>> a1fe94a... Temp work related to docker execution phase. code may not compile
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/5787e77f/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerHostExectutor.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerHostExectutor.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerHostExectutor.java
deleted file mode 100644
index 133d54a..0000000
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerHostExectutor.java
+++ /dev/null
@@ -1,177 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.ptest.execution.containers;
-
-import com.google.common.base.Preconditions;
-import com.google.common.base.Stopwatch;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import com.google.common.util.concurrent.Futures;
-import com.google.common.util.concurrent.ListenableFuture;
-import com.google.common.util.concurrent.ListeningExecutorService;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hive.ptest.execution.AbortDroneException;
-import org.apache.hive.ptest.execution.Constants;
-import org.apache.hive.ptest.execution.ContainerClient;
-import org.apache.hive.ptest.execution.ContainerClientFactory;
-import org.apache.hive.ptest.execution.ContainerClientFactory.ContainerClientContext;
-import org.apache.hive.ptest.execution.ContainerClientFactory.ContainerType;
-import org.apache.hive.ptest.execution.Dirs;
-import org.apache.hive.ptest.execution.HostExecutor;
-import org.apache.hive.ptest.execution.Templates;
-import org.apache.hive.ptest.execution.conf.Host;
-import org.apache.hive.ptest.execution.conf.TestBatch;
-import org.apache.hive.ptest.execution.ssh.RSyncCommandExecutor;
-import org.apache.hive.ptest.execution.ssh.RemoteCommandResult;
-import org.apache.hive.ptest.execution.ssh.SSHCommand;
-import org.apache.hive.ptest.execution.ssh.SSHCommandExecutor;
-import org.slf4j.Logger;
-
-import java.io.File;
-import java.io.PrintWriter;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.Callable;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicLong;
-
-public class DockerHostExectutor extends HostExecutor {
- private final ContainerClientContext containerClientContext;
- private final ContainerClient dockerClient;
- private final int numParallelContainersPerHost;
- private String containerName;
-
- DockerHostExectutor(Host host, String privateKey, ListeningExecutorService executor,
- SSHCommandExecutor sshCommandExecutor, RSyncCommandExecutor rsyncCommandExecutor,
- ImmutableMap<String, String> templateDefaults, File scratchDir, File succeededLogDir,
- File failedLogDir, long numPollSeconds, boolean fetchLogsForSuccessfulTests, Logger logger)
- throws Exception {
- super(host, privateKey, executor, sshCommandExecutor, rsyncCommandExecutor, templateDefaults,
- scratchDir, succeededLogDir, failedLogDir, numPollSeconds, fetchLogsForSuccessfulTests,
- logger);
- containerClientContext = new ContainerClientContext(logger, templateDefaults);
- dockerClient = ContainerClientFactory.get(ContainerType.DOCKER, containerClientContext);
- //TODO get this value from executionContext
- numParallelContainersPerHost = 3;
- }
-
- @Override
- protected void executeTests(final BlockingQueue<TestBatch> parallelWorkQueue,
- final BlockingQueue<TestBatch> isolatedWorkQueue, final Set<TestBatch> failedTestResults)
- throws Exception {
- if(mShutdown) {
- mLogger.warn("Shutting down host " + mHost.getName());
- return;
- }
- mLogger.info("Starting parallel execution on " + mHost.getName() + " using dockers");
- List<ListenableFuture<Void>> containerResults = Lists.newArrayList();
- for(int containerId = 0; containerId < numParallelContainersPerHost; containerId++) {
- containerResults.add(mExecutor.submit(new Callable<Void>() {
- @Override
- public Void call() throws Exception {
- TestBatch batch = null;
- Stopwatch sw = Stopwatch.createUnstarted();
- try {
- do {
- batch = parallelWorkQueue.poll(mNumPollSeconds, TimeUnit.SECONDS);
- if(mShutdown) {
- mLogger.warn("Shutting down host " + mHost.getName());
- return null;
- }
- if(batch != null) {
- numParallelBatchesProcessed++;
- sw.reset().start();
- try {
- if (!executeTestBatch(batch, failedTestResults)) {
- failedTestResults.add(batch);
- }
- } finally {
- sw.stop();
- mLogger.info("Finished processing parallel batch [{}] on host {}. ElapsedTime(ms)={}",
- new Object[]{batch.getName(),mHost.toShortString(), sw.elapsed(TimeUnit.MILLISECONDS)});
- }
- }
- } while(!mShutdown && !parallelWorkQueue.isEmpty());
- } catch(AbortContainerException ex) {
- mLogger.error("Aborting container during parallel execution", ex);
- if(batch != null) {
- Preconditions.checkState(parallelWorkQueue.add(batch),
- "Could not add batch to parallel queue " + batch);
- }
- }
- return null;
- }
- }));
- }
- if(mShutdown) {
- mLogger.warn("Shutting down host " + mHost.getName());
- return;
- }
- Futures.allAsList(containerResults).get();
- }
-
- private boolean executeTestBatch(TestBatch batch, Set<TestBatch> failedTestResults)
- throws AbortContainerException {
- String runCommand = dockerClient.getRunContainerCommand(getContainerName(), batch);
- Stopwatch sw = Stopwatch.createStarted();
- mLogger.info("Executing " + batch + " with " + runCommand);
- sw.stop();
- /*mLogger.info("Completed executing tests for batch [{}] on host {}. ElapsedTime(ms)={}",
- new Object[] { batch.getName(), getHost().toShortString(),
- sw.elapsed(TimeUnit.MILLISECONDS) });
- File batchLogDir = null;
- if (sshResult.getExitCode() == Constants.EXIT_CODE_UNKNOWN) {
- throw new AbortDroneException(
- "Drone " + drone.toString() + " exited with " + Constants.EXIT_CODE_UNKNOWN + ": " + sshResult);
- }
- if (mShutdown) {
- mLogger.warn("Shutting down host " + mHost.getName());
- return false;
- }
- boolean result;
- if (sshResult.getExitCode() != 0 || sshResult.getException() != null) {
- result = false;
- batchLogDir = Dirs.create(new File(mFailedTestLogDir, batch.getName()));
- } else {
- result = true;
- batchLogDir = Dirs.create(new File(mSuccessfulTestLogDir, batch.getName()));
- }
- copyFromDroneToLocal(drone, batchLogDir.getAbsolutePath(), drone.getLocalLogDirectory() + "/",
- fetchLogsForSuccessfulTests || !result);
- File logFile = new File(batchLogDir, String.format("%s.txt", batch.getName()));
- PrintWriter writer = new PrintWriter(logFile);
- writer.write(String.format("result = '%s'\n", sshResult.toString()));
- writer.write(String.format("output = '%s'\n", sshResult.getOutput()));
- if (sshResult.getException() != null) {
- sshResult.getException().printStackTrace(writer);
- }
- writer.close();*/
- return false;
- }
-
- AtomicLong containerNameId = new AtomicLong(1);
- public String getContainerName() {
- return mHost.getName() + "-" + mTemplateDefaults.get("buildTag") + "-" + String
- .valueOf(containerNameId.getAndIncrement());
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/5787e77f/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerHostExecutor.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerHostExecutor.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerHostExecutor.java
new file mode 100644
index 0000000..01a1280
--- /dev/null
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerHostExecutor.java
@@ -0,0 +1,233 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hive.ptest.execution.containers;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Stopwatch;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.ListeningExecutorService;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hive.ptest.execution.AbortDroneException;
+import org.apache.hive.ptest.execution.Constants;
+import org.apache.hive.ptest.execution.ContainerClient;
+import org.apache.hive.ptest.execution.ContainerClientFactory;
+import org.apache.hive.ptest.execution.ContainerClientFactory.ContainerClientContext;
+import org.apache.hive.ptest.execution.ContainerClientFactory.ContainerType;
+import org.apache.hive.ptest.execution.Dirs;
+import org.apache.hive.ptest.execution.Drone;
+import org.apache.hive.ptest.execution.HostExecutor;
+import org.apache.hive.ptest.execution.Templates;
+import org.apache.hive.ptest.execution.conf.Host;
+import org.apache.hive.ptest.execution.conf.TestBatch;
+import org.apache.hive.ptest.execution.ssh.RSyncCommand;
+import org.apache.hive.ptest.execution.ssh.RSyncCommandExecutor;
+import org.apache.hive.ptest.execution.ssh.RSyncResult;
+import org.apache.hive.ptest.execution.ssh.RemoteCommandResult;
+import org.apache.hive.ptest.execution.ssh.SSHCommand;
+import org.apache.hive.ptest.execution.ssh.SSHCommandExecutor;
+import org.apache.hive.ptest.execution.ssh.SSHExecutionException;
+import org.slf4j.Logger;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.Callable;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+
+public class DockerHostExecutor extends HostExecutor {
+ private final ContainerClientContext containerClientContext;
+ private final ContainerClient dockerClient;
+ private final int numParallelContainersPerHost;
+ private AtomicInteger containerNameId = new AtomicInteger(0);
+
+ DockerHostExecutor(Host host, String privateKey, ListeningExecutorService executor,
+ SSHCommandExecutor sshCommandExecutor, RSyncCommandExecutor rsyncCommandExecutor,
+ ImmutableMap<String, String> templateDefaults, File scratchDir, File succeededLogDir,
+ File failedLogDir, long numPollSeconds, boolean fetchLogsForSuccessfulTests, Logger logger)
+ throws Exception {
+ super(host, privateKey, executor, sshCommandExecutor, rsyncCommandExecutor, templateDefaults,
+ scratchDir, succeededLogDir, failedLogDir, numPollSeconds, fetchLogsForSuccessfulTests,
+ logger);
+ containerClientContext = new ContainerClientContext(logger, templateDefaults);
+ dockerClient = ContainerClientFactory.get(ContainerType.DOCKER, containerClientContext);
+ //TODO get this value from executionContext
+ numParallelContainersPerHost = 2;
+ }
+
+ @Override
+ protected void executeTests(final BlockingQueue<TestBatch> parallelWorkQueue,
+ final BlockingQueue<TestBatch> isolatedWorkQueue, final Set<TestBatch> failedTestResults)
+ throws Exception {
+ if(mShutdown) {
+ mLogger.warn("Shutting down host " + mHost.getName());
+ return;
+ }
+ mLogger.info("Starting parallel execution on " + mHost.getName() + " using dockers");
+ List<ListenableFuture<Void>> containerResults = Lists.newArrayList();
+ for(int containerSlotId = 0; containerSlotId < numParallelContainersPerHost; containerSlotId++) {
+ final int finalContainerSlotId = containerSlotId;
+ containerResults.add(mExecutor.submit(new Callable<Void>() {
+ @Override
+ public Void call() throws Exception {
+ TestBatch batch = null;
+ Stopwatch sw = Stopwatch.createUnstarted();
+ try {
+ do {
+ batch = parallelWorkQueue.poll(mNumPollSeconds, TimeUnit.SECONDS);
+ if(mShutdown) {
+ mLogger.warn("Shutting down host " + mHost.getName());
+ return null;
+ }
+ if(batch != null) {
+ numParallelBatchesProcessed++;
+ sw.reset().start();
+ try {
+ if (!executeTestBatch(batch, finalContainerSlotId, failedTestResults)) {
+ failedTestResults.add(batch);
+ }
+ } finally {
+ sw.stop();
+ mLogger.info("Finished processing parallel batch [{}] on host {}. ElapsedTime(ms)={}",
+ new Object[]{batch.getName(),mHost.toShortString(), sw.elapsed(TimeUnit.MILLISECONDS)});
+ }
+ }
+ } while(!mShutdown && !parallelWorkQueue.isEmpty());
+ } catch(AbortContainerException ex) {
+ mLogger.error("Aborting container during parallel execution", ex);
+ if(batch != null) {
+ Preconditions.checkState(parallelWorkQueue.add(batch),
+ "Could not add batch to parallel queue " + batch);
+ }
+ }
+ return null;
+ }
+ }));
+ }
+ if(mShutdown) {
+ mLogger.warn("Shutting down host " + mHost.getName());
+ return;
+ }
+ Futures.allAsList(containerResults).get();
+ }
+
+ private boolean executeTestBatch(TestBatch batch, int containerSlotId, Set<TestBatch> failedTestResults)
+ throws AbortContainerException, IOException, SSHExecutionException {
+ final int containerInstanceId = containerNameId.getAndIncrement();
+ final String containerName = getContainerName(containerInstanceId);
+ String runCommand = dockerClient.getRunContainerCommand(containerName, batch);
+ Stopwatch sw = Stopwatch.createStarted();
+ mLogger.info("Executing " + batch + " with " + runCommand);
+ RemoteCommandResult sshResult = new SSHCommand(mSSHCommandExecutor, mPrivateKey, mHost.getUser(),
+ mHost.getName(), containerInstanceId, runCommand, true).
+ call();
+ sw.stop();
+ mLogger.info(
+ "Completed executing tests for batch [{}] on host {} using container instance {}. ElapsedTime(ms)={}",
+ new Object[] { batch.getName(), mHost.toShortString(), containerInstanceId,
+ sw.elapsed(TimeUnit.MILLISECONDS) });
+
+ File batchLogDir = null;
+ if (sshResult.getExitCode() == Constants.EXIT_CODE_UNKNOWN) {
+ throw new AbortContainerException(
+ "Container " + containerInstanceId + " exited with " + Constants.EXIT_CODE_UNKNOWN + ": "
+ + sshResult);
+ }
+ if (mShutdown) {
+ mLogger.warn("Shutting down host " + mHost.getName());
+ return false;
+ }
+ boolean result;
+ if (sshResult.getExitCode() != 0 || sshResult.getException() != null) {
+ result = false;
+ batchLogDir = Dirs.create(new File(mFailedTestLogDir, batch.getName()));
+ } else {
+ result = true;
+ batchLogDir = Dirs.create(new File(mSuccessfulTestLogDir, batch.getName()));
+ }
+ String copyLogsCommand = dockerClient.getCopyTestLogsCommand(containerName,"/home/ptestuser/scratch/");
+ sw = Stopwatch.createStarted();
+ mLogger.info("Copying logs for the " + batch + " with " + runCommand);
+ sshResult = new SSHCommand(mSSHCommandExecutor, mPrivateKey, mHost.getUser(),
+ mHost.getName(), containerInstanceId, copyLogsCommand, true).
+ call();
+ sw.stop();
+ mLogger.info(
+ "Completed copying logs for batch [{}] on host {} using container instance {}. ElapsedTime(ms)={}",
+ new Object[] { batch.getName(), mHost.toShortString(), containerInstanceId,
+ sw.elapsed(TimeUnit.MILLISECONDS) });
+
+ //Copy log files from the container to Ptest server
+ copyFromContainerHostToLocal(containerInstanceId, batchLogDir.getAbsolutePath(),
+ mHost.getLocalDirectories()[containerSlotId] + "/", fetchLogsForSuccessfulTests || !result);
+ File logFile = new File(batchLogDir, String.format("%s.txt", batch.getName()));
+ PrintWriter writer = new PrintWriter(logFile);
+ writer.write(String.format("result = '%s'\n", sshResult.toString()));
+ writer.write(String.format("output = '%s'\n", sshResult.getOutput()));
+ if (sshResult.getException() != null) {
+ sshResult.getException().printStackTrace(writer);
+ }
+ writer.close();
+ //TODO add code to shutdown the container and delete it
+ String stopContainerCommand = dockerClient.getStopContainerCommand(containerName);
+ sw = Stopwatch.createStarted();
+ mLogger.info("Stopping container " + containerName + " with " + stopContainerCommand);
+ sshResult = new SSHCommand(mSSHCommandExecutor, mPrivateKey, mHost.getUser(),
+ mHost.getName(), containerInstanceId, stopContainerCommand, true).
+ call();
+ sw.stop();
+ if (sshResult.getExitCode() != 0 || sshResult.getException() != null) {
+ throw new AbortContainerException("Could not stop container after test execution");
+ }
+ return true;
+ }
+
+ public String getContainerName(int containerInstanceId) {
+ return mHost.getName() + "-" + mTemplateDefaults.get("buildTag") + "-" + String
+ .valueOf(containerInstanceId);
+ }
+
+ RSyncResult copyFromContainerHostToLocal(int containerInstanceId, String localFile, String remoteFile, boolean fetchAllLogs)
+ throws SSHExecutionException, IOException {
+ Map<String, String> templateVariables = Maps.newHashMap(mTemplateDefaults);
+ //TODO do we need this here?
+ //templateVariables.put("instanceName", drone.getInstanceName());
+ //templateVariables.put("localDir", drone.getLocalDirectory());
+ RSyncResult result = new RSyncCommand(mRSyncCommandExecutor, mPrivateKey, mHost.getUser(),
+ mHost.getName(), containerInstanceId,
+ Templates.getTemplateResult(localFile, templateVariables),
+ Templates.getTemplateResult(remoteFile, templateVariables),
+ fetchAllLogs ? RSyncCommand.Type.TO_LOCAL : RSyncCommand.Type.TO_LOCAL_NON_RECURSIVE).call();
+ if(result.getException() != null || result.getExitCode() != Constants.EXIT_CODE_SUCCESS) {
+ throw new SSHExecutionException(result);
+ }
+ totalElapsedTimeInRsync.getAndAdd(result.getElapsedTimeInMs());
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/5787e77f/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerPrepPhase.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerPrepPhase.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerPrepPhase.java
index 0e0a706..f5455b5 100644
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerPrepPhase.java
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerPrepPhase.java
@@ -54,11 +54,7 @@ public class DockerPrepPhase extends PrepPhase {
long elapsedTime;
start = System.currentTimeMillis();
//TODO give a proper label to the build
-<<<<<<< HEAD
- containerClient.defineImage(getLocalScratchDir(), null);
-=======
containerClient.defineImage(getLocalScratchDir());
->>>>>>> a1fe94a... Temp work related to docker execution phase. code may not compile
execLocally(getDockerBuildCommand());
elapsedTime = TimeUnit.MINUTES.convert((System.currentTimeMillis() - start),
TimeUnit.MILLISECONDS);
http://git-wip-us.apache.org/repos/asf/hive/blob/5787e77f/testutils/ptest2/src/main/resources/copy-test-logs.vm
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/resources/copy-test-logs.vm b/testutils/ptest2/src/main/resources/copy-test-logs.vm
new file mode 100644
index 0000000..d218e90
--- /dev/null
+++ b/testutils/ptest2/src/main/resources/copy-test-logs.vm
@@ -0,0 +1,40 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -x
+date +"%Y-%m-%d %T.%3N"
+umask 0022
+echo $$
+ps -e -o pid,pgrp,user,args
+ps x -o "%p %r %y %x %c "
+
+date +"%Y-%m-%d %T.%3N"
+mkdir ${logDir}/logs
+cd /home/ptestuser/hive
+find ./ -type f -name 'TEST-*.xml' | \
+xargs -I {} sh -c 'f=TEST-${batchName}-$(basename {}); test -f ${logDir}/$f && f=$f-$(uuidgen); mv {} ${logDir}/$f'
+find ./ -type f -name hive.log -o -name spark.log -o -name derby.log | \
+xargs -I {} sh -c 'f=$(basename {}); test -f ${logDir}/logs/$f && f=$f-$(uuidgen); mv {} ${logDir}/logs/$f'
+find ./ -path "*/spark/work" | \
+xargs -I {} sh -c 'mv {} ${logDir}/logs/spark-log'
+find ./ -type f -name 'syslog*' | \
+xargs -I {} sh -c 'mkdir -p ${logDir}/logs/syslogs; mv {} ${logDir}/logs/syslogs'
+date +"%Y-%m-%d %T.%3N"
+
+if [[ -f $logDir/.log ]]
+then
+mv $logDir/.log $logDir/logs/dot.log
+fi
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/5787e77f/testutils/ptest2/src/main/resources/dockerfile-template.vm
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/resources/dockerfile-template.vm b/testutils/ptest2/src/main/resources/dockerfile-template.vm
index 9383e96..fe421f3 100644
--- a/testutils/ptest2/src/main/resources/dockerfile-template.vm
+++ b/testutils/ptest2/src/main/resources/dockerfile-template.vm
@@ -1,3 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
FROM centos
RUN yum upgrade -y && \
@@ -6,10 +21,11 @@ RUN yum upgrade -y && \
RUN useradd -m ptestuser
RUN mkdir -p /home/ptestuser
-COPY smart-apply-patch.sh scratch/build.patch /home/ptestuser/scratch/
+COPY smart-apply-patch.sh copy-test-logs.sh scratch/build.patch /home/ptestuser/scratch/
RUN chown -R ptestuser /home/ptestuser
RUN chmod -R 755 /home/ptestuser
RUN ls -l /home
+RUN cat /home/ptestuser/scratch/build.patch
USER ptestuser
RUN cd /home/ptestuser
WORKDIR /home/ptestuser
@@ -17,15 +33,12 @@ ARG workingDir
ARG mavenEnvOpts
ARG repository
ARG branch
-<<<<<<< HEAD
-ARG label
-=======
ARG buildTag
->>>>>>> a1fe94a... Temp work related to docker execution phase. code may not compile
RUN export MAVEN_OPTS="$mavenEnvOpts"
RUN /usr/bin/git clone $repository
RUN cd hive
+
WORKDIR /home/ptestuser/hive
RUN /usr/bin/git checkout $branch || checkout -b $branch origin/$branch
RUN /usr/bin/git reset --hard origin/$branch
@@ -33,9 +46,5 @@ RUN /usr/bin/git merge --ff-only origin/$branch
RUN /home/ptestuser/scratch/smart-apply-patch.sh /home/ptestuser/scratch/build.patch
RUN /usr/bin/mvn -B -T 4 -q install -Dtest=TestMetastoreConf
RUN cd itests
-RUN /usr/bin/mvn -B -T 4 -q install -DskipSparkTests -DskipTests
-<<<<<<< HEAD
-RUN echo This build is labeled $label
-=======
+RUN /usr/bin/mvn -B -T 4 -q install -DskipSparkTests -DskipTests
RUN echo This build is labeled $buildTag
->>>>>>> a1fe94a... Temp work related to docker execution phase. code may not compile
http://git-wip-us.apache.org/repos/asf/hive/blob/5787e77f/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestPtestOnDockers.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestPtestOnDockers.java b/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestPtestOnDockers.java
index 516a12e..1109a8a 100644
--- a/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestPtestOnDockers.java
+++ b/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestPtestOnDockers.java
@@ -39,6 +39,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
+import java.net.URL;
import java.util.concurrent.Executors;
import static org.mockito.Mockito.mock;
@@ -76,11 +77,12 @@ public class TestPtestOnDockers {
private static final int INSTANCE = 13;
private static final String INSTANCE_NAME = HOST + "-" + USER + "-" + INSTANCE;
private static final String REAL_BRANCH = "master";
- private static final String REAL_REPOSITORY = "https://git-wip-us.apache.org/repos/asf/hive.git";
+ private static final String REAL_REPOSITORY = "https://github.com/apache/hive.git";
private static final String REAL_REPOSITORY_NAME = "apache-hive";
private static final String REAL_MAVEN_OPTS = "-Xmx1g";
private MockSSHCommandExecutor sshCommandExecutor;
private MockRSyncCommandExecutor rsyncCommandExecutor;
+ private static final String BUILD_TAG = "docker-ptest-tag";
public void initialize(String name) throws Exception {
baseDir = AbstractTestPhase.createBaseDir(name);
@@ -97,6 +99,7 @@ public class TestPtestOnDockers {
rsyncCommandExecutor = spy(new MockRSyncCommandExecutor(logger));
templateDefaults = ImmutableMap.<String, String>builder()
.put("localDir", LOCAL_DIR)
+ .put("buildTag", BUILD_TAG)
//use baseDir as working directory
.put("workingDir", baseDir.getAbsolutePath())
.put("instanceName", INSTANCE_NAME)
@@ -111,9 +114,9 @@ public class TestPtestOnDockers {
@BeforeClass
public static void beforeClass() throws Exception {
- dummyPatchFile = File.createTempFile("dummy", "patch");
- dummyPatchFile.deleteOnExit();
- Assert.assertTrue("Could not create dummy patch file " + dummyPatchFile.getAbsolutePath(),
+ URL url = TestPtestOnDockers.class.getResource("/DUMMY-001.patch");
+ dummyPatchFile = new File(url.getFile());
+ Assert.assertTrue("Could not find dummy patch file " + dummyPatchFile.getAbsolutePath(),
dummyPatchFile.exists());
}
@@ -135,7 +138,7 @@ public class TestPtestOnDockers {
@After
public void teardown() {
phase = null;
- FileUtils.deleteQuietly(baseDir);
+ //FileUtils.deleteQuietly(baseDir);
}
/**
http://git-wip-us.apache.org/repos/asf/hive/blob/5787e77f/testutils/ptest2/src/test/resources/DUMMY-001.patch
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/test/resources/DUMMY-001.patch b/testutils/ptest2/src/test/resources/DUMMY-001.patch
new file mode 100644
index 0000000..6277ca9
--- /dev/null
+++ b/testutils/ptest2/src/test/resources/DUMMY-001.patch
@@ -0,0 +1,12 @@
+diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java
+index 2868ff0..6a6a5ff 100644
+--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java
++++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java
+@@ -69,6 +69,7 @@
+
+ public class PTest {
+
++ // test patching
+ static {
+ Velocity.init();
+ }