You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by vi...@apache.org on 2018/06/26 05:17:41 UTC
[5/8] hive git commit: Docker prep phase working
Docker prep phase working
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/519b62e3
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/519b62e3
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/519b62e3
Branch: refs/heads/HIVE-19429
Commit: 519b62e3e28f26d8eb52c7dba486f8999f06b443
Parents: 5787e77
Author: Vihang Karajgaonkar <vi...@cloudera.com>
Authored: Thu Jun 21 23:21:43 2018 -0700
Committer: Vihang Karajgaonkar <vi...@cloudera.com>
Committed: Thu Jun 21 23:21:43 2018 -0700
----------------------------------------------------------------------
.../execution/containers/DockerClient.java | 2 ++
.../execution/containers/DockerPrepPhase.java | 4 +--
.../src/main/resources/dockerfile-template.vm | 20 ++++++-----
.../ptest/execution/TestPtestOnDockers.java | 34 +++++++++---------
.../ptest2/src/test/resources/DUMMY-001.patch | 38 ++++++++++++++------
5 files changed, 59 insertions(+), 39 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/519b62e3/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerClient.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerClient.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerClient.java
index 8e6e0cf..da54348 100644
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerClient.java
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerClient.java
@@ -64,6 +64,8 @@ public class DockerClient implements ContainerClient {
logger.info("Building image");
String dockerBuildCommand =
new StringBuilder("docker build")
+ .append(" --memory " + "2g")
+ .append(" --memory-swap " + "-1")
.append(" --tag " + imageName())
.append(" --build-arg ")
.append(" workingDir=$workingDir")
http://git-wip-us.apache.org/repos/asf/hive/blob/519b62e3/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerPrepPhase.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerPrepPhase.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerPrepPhase.java
index f5455b5..80c1aef 100644
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerPrepPhase.java
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerPrepPhase.java
@@ -58,11 +58,11 @@ public class DockerPrepPhase extends PrepPhase {
execLocally(getDockerBuildCommand());
elapsedTime = TimeUnit.MINUTES.convert((System.currentTimeMillis() - start),
TimeUnit.MILLISECONDS);
- logger.info("PERF: Docker source prep took " + elapsedTime + " minutes");
+ logger.info("PERF: Docker build image took " + elapsedTime + " minutes");
}
@VisibleForTesting
- String getLocalScratchDir() {
+ public String getLocalScratchDir() {
return mScratchDir.getAbsolutePath();
}
http://git-wip-us.apache.org/repos/asf/hive/blob/519b62e3/testutils/ptest2/src/main/resources/dockerfile-template.vm
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/resources/dockerfile-template.vm b/testutils/ptest2/src/main/resources/dockerfile-template.vm
index fe421f3..4f96ea7 100644
--- a/testutils/ptest2/src/main/resources/dockerfile-template.vm
+++ b/testutils/ptest2/src/main/resources/dockerfile-template.vm
@@ -35,16 +35,18 @@ ARG repository
ARG branch
ARG buildTag
-RUN export MAVEN_OPTS="$mavenEnvOpts"
+ENV MAVEN_OPTS="$mavenEnvOpts"
+RUN /usr/bin/mvn -version
+RUN echo $MAVEN_OPTS
RUN /usr/bin/git clone $repository
RUN cd hive
WORKDIR /home/ptestuser/hive
-RUN /usr/bin/git checkout $branch || checkout -b $branch origin/$branch
-RUN /usr/bin/git reset --hard origin/$branch
-RUN /usr/bin/git merge --ff-only origin/$branch
-RUN /home/ptestuser/scratch/smart-apply-patch.sh /home/ptestuser/scratch/build.patch
-RUN /usr/bin/mvn -B -T 4 -q install -Dtest=TestMetastoreConf
-RUN cd itests
-RUN /usr/bin/mvn -B -T 4 -q install -DskipSparkTests -DskipTests
-RUN echo This build is labeled $buildTag
+RUN /usr/bin/git checkout $branch || checkout -b $branch origin/$branch &&\
+ /usr/bin/git reset --hard origin/$branch &&\
+ /usr/bin/git merge --ff-only origin/$branch &&\
+ /home/ptestuser/scratch/smart-apply-patch.sh /home/ptestuser/scratch/build.patch &&\
+ /usr/bin/mvn -q -B install -Dtest=TestMetastoreConf &&\
+ cd itests &&\
+ /usr/bin/mvn -q -B install -DskipSparkTests -DskipTests &&\
+ echo This build is labeled $buildTag
http://git-wip-us.apache.org/repos/asf/hive/blob/519b62e3/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestPtestOnDockers.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestPtestOnDockers.java b/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestPtestOnDockers.java
index 1109a8a..76716c4 100644
--- a/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestPtestOnDockers.java
+++ b/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestPtestOnDockers.java
@@ -23,9 +23,10 @@ import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
-import org.apache.commons.io.FileUtils;
import org.apache.hive.ptest.execution.LocalCommand.CollectLogPolicy;
import org.apache.hive.ptest.execution.conf.Host;
+import org.apache.hive.ptest.execution.conf.TestBatch;
+import org.apache.hive.ptest.execution.containers.DockerExecutionPhase;
import org.apache.hive.ptest.execution.containers.DockerPrepPhase;
import org.apache.hive.ptest.execution.containers.TestDockerPrepPhase;
import org.apache.hive.ptest.execution.context.ExecutionContext;
@@ -51,7 +52,8 @@ import static org.mockito.Mockito.spy;
public class TestPtestOnDockers {
//TODO add logic to ignore this test if docker is not found on the machine
- private DockerPrepPhase phase;
+ private DockerPrepPhase prepPhase;
+ private DockerExecutionPhase execPhase;
private static File dummyPatchFile;
private static final Logger logger = LoggerFactory.getLogger(TestDockerPrepPhase.class);
@@ -79,7 +81,7 @@ public class TestPtestOnDockers {
private static final String REAL_BRANCH = "master";
private static final String REAL_REPOSITORY = "https://github.com/apache/hive.git";
private static final String REAL_REPOSITORY_NAME = "apache-hive";
- private static final String REAL_MAVEN_OPTS = "-Xmx1g";
+ private static final String REAL_MAVEN_OPTS = "-Xmx2048m";
private MockSSHCommandExecutor sshCommandExecutor;
private MockRSyncCommandExecutor rsyncCommandExecutor;
private static final String BUILD_TAG = "docker-ptest-tag";
@@ -124,8 +126,13 @@ public class TestPtestOnDockers {
public void setup() throws Exception {
initialize(getClass().getSimpleName());
createHostExecutor();
- phase = new DockerPrepPhase(hostExecutors, localCommandFactory,
+ prepPhase = new DockerPrepPhase(hostExecutors, localCommandFactory,
templateDefaults, baseDir, dummyPatchFile, logger);
+ /*execPhase = new DockerExecutionPhase(hostExecutors, executionContext,
+ hostExecutorBuilder, localCommandFactory,
+ templateDefaults, succeededLogDir, failedLogDir,
+ testBatchSupplier, executedTests,
+ failedTests, logger);*/
}
private void createHostExecutor() {
@@ -137,7 +144,7 @@ public class TestPtestOnDockers {
@After
public void teardown() {
- phase = null;
+ prepPhase = null;
//FileUtils.deleteQuietly(baseDir);
}
@@ -147,18 +154,9 @@ public class TestPtestOnDockers {
*/
@Test
public void testDockerFile() throws Exception {
- phase.execute();
- File dockerFile = new File(scratchDir, "Dockerfile");
- Assert.assertTrue("Docker file was not found" , dockerFile.exists());
- Assert.assertTrue("patch file not found", new File(scratchDir, "build.patch").exists());
- LocalCommandFactory localCommandFactory = new LocalCommandFactory(logger);
- CollectLogPolicy localCollector = new CollectLogPolicy(logger);
- localCommandFactory.create(localCollector, "cp -f " + dockerFile.getAbsolutePath() + " /tmp/myDockerFile");
- LocalCommand localCmd = localCommandFactory.create(localCollector, phase.getDockerBuildCommand());
- if(localCmd.getExitCode() != 0) {
- throw new NonZeroExitCodeException(String.format(
- "Command '%s' failed with exit status %d and output '%s'",
- localCmd, localCmd.getExitCode(), localCollector.getOutput()));
- }
+ prepPhase.execute();
+ Assert.assertNotNull("Scratch directory needs to be set", prepPhase.getLocalScratchDir());
+ File dockerFile = new File(prepPhase.getLocalScratchDir(), "Dockerfile");
+ Assert.assertTrue("Docker file not found", dockerFile.exists());
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/519b62e3/testutils/ptest2/src/test/resources/DUMMY-001.patch
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/test/resources/DUMMY-001.patch b/testutils/ptest2/src/test/resources/DUMMY-001.patch
index 6277ca9..97d7feb 100644
--- a/testutils/ptest2/src/test/resources/DUMMY-001.patch
+++ b/testutils/ptest2/src/test/resources/DUMMY-001.patch
@@ -1,12 +1,30 @@
-diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java
-index 2868ff0..6a6a5ff 100644
---- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java
-+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java
-@@ -69,6 +69,7 @@
+diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java
+index 1d7eceb1ef380fdb6f62d8b831b1e2bd23333c67..b78e7b2bef78b99a3cab9cc89bd8551649285417 100644
+--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java
++++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java
+@@ -68,9 +68,7 @@
- public class PTest {
+ import com.google.common.annotations.VisibleForTesting;
+ import com.google.protobuf.CodedInputStream;
+-
+-import sun.misc.Cleaner;
+-
++//import sun.misc.Cleaner;
-+ // test patching
- static {
- Velocity.init();
- }
+ /**
+ * Encoded reader implementation.
+@@ -1654,12 +1652,12 @@ private void releaseBuffer(ByteBuffer bb, boolean isFromDataReader) {
+ Field localCf = cleanerField;
+ if (!bb.isDirect() || localCf == null) return;
+ try {
+- Cleaner cleaner = (Cleaner) localCf.get(bb);
++ /*Cleaner cleaner = (Cleaner) localCf.get(bb);
+ if (cleaner != null) {
+ cleaner.clean();
+ } else {
+ LOG.debug("Unable to clean a buffer using cleaner - no cleaner");
+- }
++ }*/
+ } catch (Exception e) {
+ // leave it for GC to clean up
+ LOG.warn("Unable to clean direct buffers using Cleaner.");