You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by pv...@apache.org on 2017/11/22 09:05:06 UTC
hive git commit: HIVE-16748: Integreate YETUS to Pre-Commit (Adam
Szita, reviewed by Barna Zsombor Klara, Peter Vary)
Repository: hive
Updated Branches:
refs/heads/master d22784a9e -> c7aa7b64a
HIVE-16748: Integreate YETUS to Pre-Commit (Adam Szita, reviewed by Barna Zsombor Klara, Peter Vary)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c7aa7b64
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c7aa7b64
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c7aa7b64
Branch: refs/heads/master
Commit: c7aa7b64add25f864c3e61756b54f7ada57edfa3
Parents: d22784a
Author: Peter Vary <pv...@cloudera.com>
Authored: Wed Nov 22 10:04:35 2017 +0100
Committer: Peter Vary <pv...@cloudera.com>
Committed: Wed Nov 22 10:04:35 2017 +0100
----------------------------------------------------------------------
dev-support/hive-personality.sh | 2 +-
.../org/apache/hive/ptest/execution/PTest.java | 2 +
.../apache/hive/ptest/execution/YetusPhase.java | 138 +++++++++++++++++++
.../ptest2/src/main/resources/source-prep.vm | 4 +
.../ptest2/src/main/resources/yetus-exec.vm | 28 ++++
5 files changed, 173 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/c7aa7b64/dev-support/hive-personality.sh
----------------------------------------------------------------------
diff --git a/dev-support/hive-personality.sh b/dev-support/hive-personality.sh
index f3247aa..c6570d4 100644
--- a/dev-support/hive-personality.sh
+++ b/dev-support/hive-personality.sh
@@ -16,7 +16,7 @@
# Override these to match Apache Hadoop's requirements
-personality_plugins "maven,asflicense,author,checkstyle,findbugs,javac,compile,javadoc,whitespace,xml"
+personality_plugins "maven,asflicense,author,checkstyle,findbugs,javac,compile,javadoc,whitespace,xml,jira"
## @description Globals specific to this personality
## @audience private
http://git-wip-us.apache.org/repos/asf/hive/blob/c7aa7b64/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java
index 65a8216..8df5162 100644
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java
@@ -158,6 +158,8 @@ public class PTest {
mPhases = Lists.newArrayList();
mPhases.add(new TestCheckPhase(mHostExecutors, localCommandFactory, templateDefaults, patchFile, logger, mAddedTests));
mPhases.add(new PrepPhase(mHostExecutors, localCommandFactory, templateDefaults, scratchDir, patchFile, logger));
+ mPhases.add(new YetusPhase(configuration, mHostExecutors, localCommandFactory, templateDefaults,
+ mExecutionContext.getLocalWorkingDirectory(), scratchDir, logger, logDir, patchFile));
mPhases.add(new ExecutionPhase(mHostExecutors, mExecutionContext, hostExecutorBuilder, localCommandFactory, templateDefaults,
succeededLogDir, failedLogDir, testParser.parse(), mExecutedTests, mFailedTests, logger));
mPhases.add(new ReportingPhase(mHostExecutors, localCommandFactory, templateDefaults, logger));
http://git-wip-us.apache.org/repos/asf/hive/blob/c7aa7b64/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/YetusPhase.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/YetusPhase.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/YetusPhase.java
new file mode 100644
index 0000000..021646b
--- /dev/null
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/YetusPhase.java
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hive.ptest.execution;
+
+import java.io.File;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hive.ptest.execution.conf.TestConfiguration;
+
+import com.google.common.base.Strings;
+import com.google.common.collect.ImmutableMap;
+
+import org.slf4j.Logger;
+
+/**
+ * Wrapper phase for Yetus check execution kick-off. It will invoke ./dev-support/test-patch.sh
+ * with the proper arguments and run the check itself in a asynchronous fashion.
+ */
+public class YetusPhase extends Phase {
+
+ private static final String YETUS_LOG_FILE = "yetus.txt";
+ private static final String YETUS_OUTPUT_FOLDER = "yetus";
+ private static final String YETUS_EXEC_SCRIPT = "yetus-exec.sh";
+ private static final String YETUS_EXEC_VM = "yetus-exec.vm";
+
+ private final File mPatchFile;
+ private final File mWorkingDir;
+ private final File mLogFile;
+ private final File mOutputDir;
+ private final File mScratchDir;
+ private final String buildUrl;
+ private final TestConfiguration conf;
+
+
+ public YetusPhase(TestConfiguration configuration, List<HostExecutor> hostExecutors,
+ LocalCommandFactory localCommandFactory, ImmutableMap<String, String> templateDefaults,
+ String workingDir, File scratchDir, Logger logger, File logDir, File patchFile) {
+
+ super(hostExecutors, localCommandFactory, templateDefaults, logger);
+ this.mPatchFile = patchFile;
+ this.mWorkingDir = new File(workingDir, YETUS_OUTPUT_FOLDER);
+ this.mLogFile = new File(logDir, YETUS_LOG_FILE);
+ this.mOutputDir = new File(logDir, YETUS_OUTPUT_FOLDER);
+ this.mScratchDir = scratchDir;
+ this.conf = configuration;
+ this.buildUrl = conf.getLogsURL() + "/" + templateDefaults.get("buildTag") + "/";
+ }
+
+ @Override
+ public void execute() throws Exception {
+
+ Thread t = new Thread(new Runnable() {
+ @Override
+ public void run() {
+
+ if (!checkDependencies()) {
+ return;
+ }
+
+ File yetusExecScript = new File(mScratchDir, YETUS_EXEC_SCRIPT);
+ Map<String, String> templateVars = new HashMap<>();
+ templateVars.putAll(getTemplateDefaults());
+ templateVars.put("workingDir", mWorkingDir.getAbsolutePath());
+ templateVars.put("jiraName", conf.getJiraName());
+ templateVars.put("patchFile", mPatchFile.getAbsolutePath());
+ templateVars.put("jiraUrl", conf.getJiraUrl());
+ templateVars.put("jiraUser", conf.getJiraUser());
+ templateVars.put("jiraPass", conf.getJiraPassword());
+ templateVars.put("outputDir", mOutputDir.getAbsolutePath());
+ templateVars.put("buildUrl", buildUrl);
+ templateVars.put("buildUrlLog", YETUS_LOG_FILE);
+ templateVars.put("buildUrlOutputDir", YETUS_OUTPUT_FOLDER);
+ templateVars.put("logFile", mLogFile.getAbsolutePath());
+
+ try {
+ logger.info("Writing {} from template", yetusExecScript);
+
+ Templates.writeTemplateResult(YETUS_EXEC_VM, yetusExecScript, templateVars);
+ Process proc = new ProcessBuilder().command("bash", yetusExecScript.getPath()).start();
+ int exitCode = proc.waitFor();
+
+ if (exitCode == 0) {
+ logger.info("Finished processing Yetus check successfully");
+ }
+ } catch (Exception e) {
+ logger.error("Error processing Yetus check", e);
+ } finally {
+ logger.debug("Deleting " + yetusExecScript + ": " + yetusExecScript.delete());
+ }
+ }
+ });
+ t.start();
+ logger.info("Started Yetus check..");
+ }
+
+ private boolean checkDependencies(){
+
+ if (mPatchFile == null || !mPatchFile.canRead()) {
+ logger.error("Cannot run Yetus check - patch file is null or not readable.");
+ return false;
+ }
+
+ if (!((mWorkingDir.isDirectory() && mWorkingDir.canWrite()) &&
+ (mOutputDir.isDirectory() && mOutputDir.canWrite()))) {
+ logger.error("Cannot run Yetus check - output directories not present and writable: " +
+ "workingDir:%s, outputDir:%s", mWorkingDir.getAbsolutePath(), mOutputDir.getAbsolutePath());
+ return false;
+ }
+
+ if (Strings.isNullOrEmpty(conf.getJiraUrl()) ||
+ Strings.isNullOrEmpty(conf.getJiraName()) ||
+ Strings.isNullOrEmpty(conf.getJiraPassword()) ||
+ Strings.isNullOrEmpty(conf.getJiraUser())) {
+ logger.error("Cannot run Yetus check - credentials for Jira not provided.");
+ return false;
+ }
+
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/c7aa7b64/testutils/ptest2/src/main/resources/source-prep.vm
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/resources/source-prep.vm b/testutils/ptest2/src/main/resources/source-prep.vm
index 7ad5024..5820731 100644
--- a/testutils/ptest2/src/main/resources/source-prep.vm
+++ b/testutils/ptest2/src/main/resources/source-prep.vm
@@ -78,6 +78,10 @@ cd $workingDir/
echo "Unknown repository type '${repositoryType}'"
exit 1
fi
+ rm -rf ../yetus
+ mkdir ../yetus
+ cp -R . ../yetus
+ mkdir $logDir/yetus
patchCommandPath=$workingDir/scratch/smart-apply-patch.sh
patchFilePath=$workingDir/scratch/build.patch
if [[ -f $patchFilePath ]]
http://git-wip-us.apache.org/repos/asf/hive/blob/c7aa7b64/testutils/ptest2/src/main/resources/yetus-exec.vm
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/resources/yetus-exec.vm b/testutils/ptest2/src/main/resources/yetus-exec.vm
new file mode 100644
index 0000000..9dea59e
--- /dev/null
+++ b/testutils/ptest2/src/main/resources/yetus-exec.vm
@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+##### Remember this is a velocity template
+set -e
+set -x
+date +"%Y-%m-%d %T.%3N"
+
+pushd ${workingDir}
+export JIRA_ISSUE=${jiraName}
+export JAVA_HOME=${javaHome}
+./dev-support/test-patch.sh ${patchFile} --jenkins --jira-base-url=${jiraUrl} --jira-user=${jiraUser} \
+ --jira-password=${jiraPass} --patch-dir=${outputDir} --build-url=${buildUrl} --build-url-console=${buildUrlLog} \
+ --build-url-artifacts=${buildUrlOutputDir} 2>&1 | tee ${logFile}
+popd
+