You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ec...@apache.org on 2013/07/19 05:47:39 UTC
svn commit: r1504759 [2/6] - in /hive/trunk: ./ testutils/ptest2/
testutils/ptest2/conf/ testutils/ptest2/src/ testutils/ptest2/src/main/
testutils/ptest2/src/main/java/ testutils/ptest2/src/main/java/org/
testutils/ptest2/src/main/java/org/apache/ tes...
Added: hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/server/TestExecutor.java
URL: http://svn.apache.org/viewvc/hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/server/TestExecutor.java?rev=1504759&view=auto
==============================================================================
--- hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/server/TestExecutor.java (added)
+++ hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/server/TestExecutor.java Fri Jul 19 03:47:36 2013
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hive.ptest.api.server;
+
+import java.io.File;
+import java.io.PrintStream;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hive.ptest.api.Status;
+import org.apache.hive.ptest.api.request.TestStartRequest;
+import org.apache.hive.ptest.execution.Constants;
+import org.apache.hive.ptest.execution.Dirs;
+import org.apache.hive.ptest.execution.LocalCommandFactory;
+import org.apache.hive.ptest.execution.LogDirectoryCleaner;
+import org.apache.hive.ptest.execution.PTest;
+import org.apache.hive.ptest.execution.conf.ExecutionContextConfiguration;
+import org.apache.hive.ptest.execution.conf.TestConfiguration;
+import org.apache.hive.ptest.execution.context.CreateHostsFailedException;
+import org.apache.hive.ptest.execution.context.ExecutionContext;
+import org.apache.hive.ptest.execution.context.ExecutionContextProvider;
+import org.apache.hive.ptest.execution.context.ServiceNotAvailableException;
+import org.apache.hive.ptest.execution.ssh.RSyncCommandExecutor;
+import org.apache.hive.ptest.execution.ssh.SSHCommandExecutor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Executes parallel test in a single thread since the slaves
+ * will be fully utilized by the test environment.
+ */
+public class TestExecutor extends Thread {
+ private static final Logger LOG = LoggerFactory
+ .getLogger(TestExecutor.class);
+ private final ExecutionContextConfiguration mExecutionContextConfiguration;
+ private final ExecutionContextProvider mExecutionContextProvider;
+ private final BlockingQueue<Test> mTestQueue;
+ private final PTest.Builder mPTestBuilder;
+ private ExecutionContext mExecutionContext;
+
+ private boolean execute;
+ public TestExecutor(ExecutionContextConfiguration executionContextConfiguration,
+ ExecutionContextProvider executionContextProvider,
+ BlockingQueue<Test> testQueue, PTest.Builder pTestBuilder) {
+ mExecutionContextConfiguration = executionContextConfiguration;
+ mExecutionContextProvider = executionContextProvider;
+ mTestQueue = testQueue;
+ mPTestBuilder = pTestBuilder;
+ execute = true;
+ }
+
+ @Override
+public void run() {
+ while(execute) {
+ Test test = null;
+ PrintStream logStream = null;
+ Logger logger = null;
+ try {
+ // start a log cleaner at the start of each test
+ LogDirectoryCleaner cleaner = new LogDirectoryCleaner(new File(mExecutionContextConfiguration.
+ getGlobalLogDirectory()), mExecutionContextConfiguration.getMaxLogDirectoriesPerProfile());
+ cleaner.setName("LogCleaner-" + mExecutionContextConfiguration.
+ getGlobalLogDirectory());
+ cleaner.setDaemon(true);
+ cleaner.start();
+ test = mTestQueue.poll(30, TimeUnit.MINUTES);
+ if(!execute) {
+ terminateExecutionContext();
+ break;
+ }
+ if(test == null) {
+ terminateExecutionContext();
+ } else {
+ test.setStatus(Status.inProgress());
+ test.setDequeueTime(System.currentTimeMillis());
+ if(mExecutionContext == null) {
+ mExecutionContext = createExceutionContext();
+ }
+ test.setExecutionStartTime(System.currentTimeMillis());
+ TestStartRequest startRequest = test.getStartRequest();
+ String profile = startRequest.getProfile();
+ File profileConfFile = new File(mExecutionContextConfiguration.getProfileDirectory(),
+ String.format("%s.properties", profile));
+ if(!profileConfFile.isFile()) {
+ test.setStatus(Status.illegalArgument("Profile " + profile + " not found"));
+ test.setExecutionFinishTime(System.currentTimeMillis());
+ } else {
+ File logDir = Dirs.create(new File(mExecutionContextConfiguration.
+ getGlobalLogDirectory(), test.getStartRequest().getTestHandle()));
+ File logFile = new File(logDir, "execution.txt");
+ test.setOutputFile(logFile);
+ logStream = new PrintStream(logFile);
+ logger = new TestLogger(logStream, TestLogger.LEVEL.DEBUG);
+ TestConfiguration testConfiguration = TestConfiguration.fromFile(profileConfFile, logger);
+ testConfiguration.setPatch(startRequest.getPatchURL());
+ testConfiguration.setJiraName(startRequest.getJiraName());
+ PTest ptest = mPTestBuilder.build(testConfiguration, mExecutionContext,
+ test.getStartRequest().getTestHandle(), logDir,
+ new LocalCommandFactory(logger), new SSHCommandExecutor(logger),
+ new RSyncCommandExecutor(logger), logger);
+ int result = ptest.run();
+ if(result == Constants.EXIT_CODE_SUCCESS) {
+ test.setStatus(Status.ok());
+ } else {
+ test.setStatus(Status.failed("Tests failed with exit code " + result));
+ }
+ logStream.flush();
+ // if all drones where abandoned on a host, replace it
+ mExecutionContext.replaceBadHosts();
+ }
+ }
+ } catch (Exception e) {
+ LOG.error("Unxpected Error", e);
+ if(test != null) {
+ test.setStatus(Status.failed("Tests failed with exception " +
+ e.getClass().getName() + ": " + e.getMessage()));
+ if(logger != null) {
+ String msg = "Error executing " + test.getStartRequest().getTestHandle();
+ logger.error(msg, e);
+ }
+ }
+ // if we died for any reason lets get a new set of hosts
+ terminateExecutionContext();
+ } finally {
+ if(test != null) {
+ test.setExecutionFinishTime(System.currentTimeMillis());
+ }
+ if(logStream != null) {
+ logStream.flush();
+ logStream.close();
+ }
+ }
+ }
+ }
+
+ private void terminateExecutionContext() {
+ if(mExecutionContext != null) {
+ mExecutionContext.terminate();
+ mExecutionContext = null;
+ }
+ }
+ private ExecutionContext createExceutionContext()
+ throws ServiceNotAvailableException, InterruptedException, CreateHostsFailedException {
+ long start = System.currentTimeMillis();
+ LOG.info("Attempting to create a new execution context");
+ ExecutionContext result = mExecutionContextProvider.createExecutionContext();
+ long elapsedTime = System.currentTimeMillis() - start;
+ LOG.info("Context Creation time: " + TimeUnit.SECONDS.
+ convert(elapsedTime, TimeUnit.MILLISECONDS) + " seconds");
+ return result;
+ }
+
+ public void shutdown() {
+ execute = false;
+ this.interrupt();
+ }
+}
Added: hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/server/TestLogger.java
URL: http://svn.apache.org/viewvc/hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/server/TestLogger.java?rev=1504759&view=auto
==============================================================================
--- hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/server/TestLogger.java (added)
+++ hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/server/TestLogger.java Fri Jul 19 03:47:36 2013
@@ -0,0 +1,256 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hive.ptest.api.server;
+
+import java.io.PrintStream;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+
+import org.slf4j.helpers.FormattingTuple;
+import org.slf4j.helpers.MarkerIgnoringBase;
+import org.slf4j.helpers.MessageFormatter;
+
+/**
+ * Simple logger which allows each test to have it's own log file.
+ */
+public class TestLogger extends MarkerIgnoringBase {
+
+ private static final long serialVersionUID = -1711679924980202258L;
+ private final LEVEL mLevel;
+ private final PrintStream mLog;
+ private SimpleDateFormat mDateFormatter;
+
+ public TestLogger(PrintStream logFile, LEVEL level) {
+ mLog = logFile;
+ mLevel = level;
+ mDateFormatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss,SSS");
+ }
+
+ public static enum LEVEL {
+ TRACE(1),
+ DEBUG(2),
+ INFO(3),
+ WARN(4),
+ ERROR(5);
+ private int index;
+ private LEVEL(int index) {
+ this.index = index;
+ }
+ }
+ @Override
+ public boolean isTraceEnabled() {
+ return mLevel.index >= LEVEL.TRACE.index;
+ }
+
+ @Override
+ public void trace(String msg) {
+ log(LEVEL.TRACE, msg, null);
+ }
+
+ @Override
+ public void trace(String format, Object arg) {
+ FormattingTuple ft = MessageFormatter.format(format, arg);
+ log(LEVEL.TRACE, ft.getMessage(), ft.getThrowable());
+ }
+
+ @Override
+ public void trace(String format, Object arg1, Object arg2) {
+ FormattingTuple ft = MessageFormatter.format(format, arg1, arg2);
+ log(LEVEL.TRACE, ft.getMessage(), ft.getThrowable());
+ }
+
+ @Override
+ public void trace(String format, Object[] argArray) {
+ FormattingTuple ft = MessageFormatter.arrayFormat(format, argArray);
+ log(LEVEL.TRACE, ft.getMessage(), ft.getThrowable());
+ }
+
+ @Override
+ public void trace(String msg, Throwable t) {
+ log(LEVEL.TRACE, msg, t);
+ }
+
+ @Override
+ public boolean isDebugEnabled() {
+ return mLevel.index >= LEVEL.DEBUG.index;
+ }
+
+ @Override
+ public void debug(String msg) {
+ log(LEVEL.DEBUG, msg, null);
+ }
+
+ @Override
+ public void debug(String format, Object arg) {
+ FormattingTuple ft = MessageFormatter.format(format, arg);
+ log(LEVEL.DEBUG, ft.getMessage(), ft.getThrowable());
+ }
+
+ @Override
+ public void debug(String format, Object arg1, Object arg2) {
+ FormattingTuple ft = MessageFormatter.format(format, arg1, arg2);
+ log(LEVEL.DEBUG, ft.getMessage(), ft.getThrowable());
+ }
+
+ @Override
+ public void debug(String format, Object[] argArray) {
+ FormattingTuple ft = MessageFormatter.arrayFormat(format, argArray);
+ log(LEVEL.DEBUG, ft.getMessage(), ft.getThrowable());
+ }
+
+ @Override
+ public void debug(String msg, Throwable t) {
+ log(LEVEL.DEBUG, msg, t);
+ }
+
+ @Override
+ public boolean isInfoEnabled() {
+ return mLevel.index >= LEVEL.INFO.index;
+ }
+
+ @Override
+ public void info(String msg) {
+ log(LEVEL.INFO, msg, null);
+ }
+
+ @Override
+ public void info(String format, Object arg) {
+ FormattingTuple ft = MessageFormatter.format(format, arg);
+ log(LEVEL.INFO, ft.getMessage(), ft.getThrowable());
+ }
+
+ @Override
+ public void info(String format, Object arg1, Object arg2) {
+ FormattingTuple ft = MessageFormatter.format(format, arg1, arg2);
+ log(LEVEL.INFO, ft.getMessage(), ft.getThrowable());
+ }
+
+ @Override
+ public void info(String format, Object[] argArray) {
+ FormattingTuple ft = MessageFormatter.arrayFormat(format, argArray);
+ log(LEVEL.INFO, ft.getMessage(), ft.getThrowable());
+ }
+
+ @Override
+ public void info(String msg, Throwable t) {
+ log(LEVEL.INFO, msg, t);
+ }
+
+ @Override
+ public boolean isWarnEnabled() {
+ return mLevel.index >= LEVEL.WARN.index;
+ }
+ @Override
+ public void warn(String msg) {
+ log(LEVEL.WARN, msg, null);
+ }
+
+ @Override
+ public void warn(String format, Object arg) {
+ FormattingTuple ft = MessageFormatter.format(format, arg);
+ log(LEVEL.WARN, ft.getMessage(), ft.getThrowable());
+ }
+
+ @Override
+ public void warn(String format, Object arg1, Object arg2) {
+ FormattingTuple ft = MessageFormatter.format(format, arg1, arg2);
+ log(LEVEL.WARN, ft.getMessage(), ft.getThrowable());
+ }
+
+ @Override
+ public void warn(String format, Object[] argArray) {
+ FormattingTuple ft = MessageFormatter.arrayFormat(format, argArray);
+ log(LEVEL.WARN, ft.getMessage(), ft.getThrowable());
+ }
+
+ @Override
+ public void warn(String msg, Throwable t) {
+ log(LEVEL.WARN, msg, t);
+ }
+
+ @Override
+ public boolean isErrorEnabled() {
+ return mLevel.index >= LEVEL.ERROR.index;
+ }
+
+ @Override
+ public void error(String msg) {
+ log(LEVEL.ERROR, msg, null);
+ }
+
+ @Override
+ public void error(String format, Object arg) {
+ FormattingTuple ft = MessageFormatter.format(format, arg);
+ log(LEVEL.ERROR, ft.getMessage(), ft.getThrowable());
+ }
+
+ @Override
+ public void error(String format, Object arg1, Object arg2) {
+ FormattingTuple ft = MessageFormatter.format(format, arg1, arg2);
+ log(LEVEL.ERROR, ft.getMessage(), ft.getThrowable());
+ }
+
+ @Override
+ public void error(String format, Object[] argArray) {
+ FormattingTuple ft = MessageFormatter.arrayFormat(format, argArray);
+ log(LEVEL.ERROR, ft.getMessage(), ft.getThrowable());
+ }
+
+ @Override
+ public void error(String msg, Throwable t) {
+ log(LEVEL.ERROR, msg, t);
+ }
+
+ private String getCaller() {
+ StackTraceElement[] stack = new Exception().getStackTrace();
+ if(stack.length > 3) {
+ return getCallerShortName(stack[3]);
+ }
+ return "<unknown>";
+ }
+
+ private String getCallerShortName(StackTraceElement frame) {
+ String className = frame.getClassName();
+ String methodName = frame.getMethodName();
+ int lineNumber = frame.getLineNumber();
+ int pos = className.lastIndexOf(".");
+ if(pos > 0) {
+ className = className.substring(pos + 1);
+ }
+ return String.format("%s.%s:%d", className, methodName, lineNumber);
+ }
+
+ private synchronized void log(LEVEL level, String msg, Throwable t) {
+ if(level.index >= mLevel.index) {
+ mLog.print(mDateFormatter.format(new Date()));
+ mLog.print(" ");
+ mLog.print(String.format("%5s", level.name()));
+ mLog.print(" ");
+ mLog.print(getCaller());
+ mLog.print(" ");
+ mLog.print(msg);
+ if(t != null) {
+ mLog.print(" ");
+ t.printStackTrace(mLog);
+ }
+ mLog.print("\n");
+ mLog.flush();
+ }
+ }
+}
Added: hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/AbortDroneException.java
URL: http://svn.apache.org/viewvc/hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/AbortDroneException.java?rev=1504759&view=auto
==============================================================================
--- hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/AbortDroneException.java (added)
+++ hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/AbortDroneException.java Fri Jul 19 03:47:36 2013
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hive.ptest.execution;
+
+public class AbortDroneException extends Exception {
+ private static final long serialVersionUID = 6673699997331155666L;
+ public AbortDroneException(String msg) {
+ this(msg, null);
+ }
+ public AbortDroneException(String msg, Throwable throwable) {
+ super(msg, throwable);
+ }
+}
Added: hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/CleanupPhase.java
URL: http://svn.apache.org/viewvc/hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/CleanupPhase.java?rev=1504759&view=auto
==============================================================================
--- hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/CleanupPhase.java (added)
+++ hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/CleanupPhase.java Fri Jul 19 03:47:36 2013
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hive.ptest.execution;
+
+import java.util.concurrent.TimeUnit;
+
+import org.slf4j.Logger;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+
+public class CleanupPhase extends Phase {
+
+ public CleanupPhase(ImmutableList<HostExecutor> hostExecutors,
+ LocalCommandFactory localCommandFactory,
+ ImmutableMap<String, String> templateDefaults, Logger logger) {
+ super(hostExecutors, localCommandFactory, templateDefaults, logger);
+ }
+ @Override
+public void execute() throws Exception {
+ execHosts("killall -q -9 -f java || true");
+ TimeUnit.SECONDS.sleep(1);
+ execLocally("rm -rf $workingDir/scratch");
+ execInstances("rm -rf $localDir/$instanceName/scratch $localDir/$instanceName/logs");
+ }
+}
\ No newline at end of file
Added: hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/Constants.java
URL: http://svn.apache.org/viewvc/hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/Constants.java?rev=1504759&view=auto
==============================================================================
--- hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/Constants.java (added)
+++ hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/Constants.java Fri Jul 19 03:47:36 2013
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hive.ptest.execution;
+
+
+public class Constants {
+
+ public static final int EXIT_CODE_SUCCESS = 0;
+ public static final int EXIT_CODE_UNKNOWN = 255;
+}
Added: hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/Dirs.java
URL: http://svn.apache.org/viewvc/hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/Dirs.java?rev=1504759&view=auto
==============================================================================
--- hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/Dirs.java (added)
+++ hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/Dirs.java Fri Jul 19 03:47:36 2013
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hive.ptest.execution;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.commons.io.FileUtils;
+
+public class Dirs {
+
+ public static File createEmpty(File dir) throws IOException {
+ FileUtils.deleteQuietly(dir);
+ return create(dir);
+ }
+ public static File create(File dir) throws IOException {
+ if(dir.isDirectory()) {
+ return dir;
+ }
+ if(dir.mkdirs()) {
+ return dir;
+ }
+ throw new IOException("Could not create " + dir);
+ }
+}
Added: hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/Drone.java
URL: http://svn.apache.org/viewvc/hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/Drone.java?rev=1504759&view=auto
==============================================================================
--- hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/Drone.java (added)
+++ hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/Drone.java Fri Jul 19 03:47:36 2013
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hive.ptest.execution;
+
+import java.io.File;
+
+public class Drone {
+
+ private final String privateKey;
+ private final String user;
+ private final String host;
+ private final int instance;
+ private final String localDir;
+
+ public Drone(String privateKey, String user, String host, int instance, String localDir) {
+ this.privateKey = privateKey;
+ this.user = user;
+ this.host = host;
+ this.instance = instance;
+ this.localDir = localDir;
+ }
+
+ public String getPrivateKey() {
+ return privateKey;
+ }
+
+ public String getUser() {
+ return user;
+ }
+
+ public String getHost() {
+ return host;
+ }
+
+ @Override
+ public String toString() {
+ return "Drone [user=" + user + ", host=" + host + ", instance=" + instance
+ + "]";
+ }
+ public String getLocalDirectory() {
+ return localDir;
+ }
+ public String getLocalLogDirectory() {
+ return (new File(new File(localDir, getInstanceName()), "logs")).getAbsolutePath();
+ }
+ public String getInstanceName() {
+ return String.format("%s-%s-%d", host, user, instance);
+ }
+ public int getInstance() {
+ return instance;
+ }
+
+}
Added: hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ExecutionPhase.java
URL: http://svn.apache.org/viewvc/hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ExecutionPhase.java?rev=1504759&view=auto
==============================================================================
--- hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ExecutionPhase.java (added)
+++ hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ExecutionPhase.java Fri Jul 19 03:47:36 2013
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hive.ptest.execution;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hive.ptest.execution.conf.TestBatch;
+import org.slf4j.Logger;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Supplier;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+
+public class ExecutionPhase extends Phase {
+
+ private final File failedLogDir;
+ private final BlockingQueue<TestBatch> parallelWorkQueue;
+ private final BlockingQueue<TestBatch> isolatedWorkQueue;
+ private final Set<String> failedTests;
+ private final Supplier<List<TestBatch>> testBatchSupplier;
+ private final List<TestBatch> failedTestResults;
+
+ public ExecutionPhase(ImmutableList<HostExecutor> hostExecutors,
+ LocalCommandFactory localCommandFactory,
+ ImmutableMap<String, String> templateDefaults,
+ File failedLogDir, Supplier<List<TestBatch>> testBatchSupplier,
+ Set<String> failedTests, Logger logger) throws IOException {
+ super(hostExecutors, localCommandFactory, templateDefaults, logger);
+ this.failedLogDir = failedLogDir;
+ this.testBatchSupplier = testBatchSupplier;
+ this.failedTests = failedTests;
+ this.parallelWorkQueue = new LinkedBlockingQueue<TestBatch>();
+ this.isolatedWorkQueue = new LinkedBlockingQueue<TestBatch>();
+ this.failedTestResults = Collections.
+ synchronizedList(new ArrayList<TestBatch>());
+ }
+ @Override
+public void execute() throws Throwable {
+ long start = System.currentTimeMillis();
+ for(TestBatch batch : testBatchSupplier.get()) {
+ if(batch.isParallel()) {
+ parallelWorkQueue.add(batch);
+ } else {
+ isolatedWorkQueue.add(batch);
+ }
+ }
+ try {
+ do {
+ double numberBadHosts = 0d;
+ for(HostExecutor hostExecutor : hostExecutors) {
+ if(hostExecutor.remainingDrones() == 0) {
+ numberBadHosts++;
+ }
+ }
+ Preconditions.checkState(hostExecutors.size() > 0, "Host executors cannot be empty");
+ if((numberBadHosts / (double)hostExecutors.size()) > 0.30d) {
+ throw new IllegalStateException("Too many bad hosts: " + (int)numberBadHosts +
+ " bad hosts out of " + hostExecutors.size() + " is greater than threshold of 30%");
+ }
+ List<ListenableFuture<Void>> results = Lists.newArrayList();
+ for(HostExecutor hostExecutor : getHostExecutors()) {
+ results.add(hostExecutor.submitTests(parallelWorkQueue, isolatedWorkQueue, failedTestResults));
+ }
+ Futures.allAsList(results).get();
+ } while(!(parallelWorkQueue.isEmpty() && isolatedWorkQueue.isEmpty()));
+ Preconditions.checkState(parallelWorkQueue.isEmpty(), "Parallel work queue is not empty. All drones must have aborted.");
+ Preconditions.checkState(isolatedWorkQueue.isEmpty(), "Isolated work queue is not empty. All drones must have aborted.");
+ if(!failedTestResults.isEmpty()) {
+ for(TestBatch failure : failedTestResults) {
+ File batchLogDir = new File(failedLogDir, failure.getName());
+ JUnitReportParser parser = new JUnitReportParser(logger, batchLogDir);
+ for(String failedTest : parser.getFailedTests()) {
+ failedTests.add(failedTest);
+ }
+ }
+ }
+ } finally {
+ long elapsed = System.currentTimeMillis() - start;
+ logger.info("PERF: exec phase " +
+ TimeUnit.MINUTES.convert(elapsed, TimeUnit.MILLISECONDS) + " minutes");
+ }
+ }
+
+}
Added: hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/HostExecutor.java
URL: http://svn.apache.org/viewvc/hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/HostExecutor.java?rev=1504759&view=auto
==============================================================================
--- hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/HostExecutor.java (added)
+++ hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/HostExecutor.java Fri Jul 19 03:47:36 2013
@@ -0,0 +1,350 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hive.ptest.execution;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hive.ptest.execution.conf.Host;
+import org.apache.hive.ptest.execution.conf.TestBatch;
+import org.apache.hive.ptest.execution.ssh.RSyncCommand;
+import org.apache.hive.ptest.execution.ssh.RSyncCommandExecutor;
+import org.apache.hive.ptest.execution.ssh.RSyncResult;
+import org.apache.hive.ptest.execution.ssh.RemoteCommandResult;
+import org.apache.hive.ptest.execution.ssh.SSHCommand;
+import org.apache.hive.ptest.execution.ssh.SSHCommandExecutor;
+import org.apache.hive.ptest.execution.ssh.SSHExecutionException;
+import org.apache.hive.ptest.execution.ssh.SSHResult;
+import org.slf4j.Logger;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.ListeningExecutorService;
+
+class HostExecutor {
+ private final Host mHost;
+ private final List<Drone> mDrones;
+ private final ListeningExecutorService mExecutor;
+ private final SSHCommandExecutor mSSHCommandExecutor;
+ private final RSyncCommandExecutor mRSyncCommandExecutor;
+ private final ImmutableMap<String, String> mTemplateDefaults;
+ private final Logger mLogger;
+ private final File mLocalScratchDirectory;
+ private final File mSuccessfulTestLogDir;
+ private final File mFailedTestLogDir;
+ private final long mNumPollSeconds;
+
+ HostExecutor(Host host, String privateKey, ListeningExecutorService executor,
+ SSHCommandExecutor sshCommandExecutor,
+ RSyncCommandExecutor rsyncCommandExecutor,
+ ImmutableMap<String, String> templateDefaults, File scratchDir,
+ File succeededLogDir, File failedLogDir, long numPollSeconds,
+ Logger logger) {
+ List<Drone> drones = Lists.newArrayList();
+ String[] localDirs = host.getLocalDirectories();
+ for (int index = 0; index < host.getThreads(); index++) {
+ drones.add(new Drone(privateKey, host.getUser(), host.getName(),
+ index, localDirs[index % localDirs.length]));
+ }
+ mHost = host;
+ mDrones = new CopyOnWriteArrayList<Drone>(drones);
+ mExecutor = executor;
+ mSSHCommandExecutor = sshCommandExecutor;
+ mRSyncCommandExecutor = rsyncCommandExecutor;
+ mTemplateDefaults = templateDefaults;
+ mLocalScratchDirectory = scratchDir;
+ mSuccessfulTestLogDir = succeededLogDir;
+ mFailedTestLogDir = failedLogDir;
+ mNumPollSeconds = numPollSeconds;
+ mLogger = logger;
+ }
+
+ /**
+ * @return failed tests
+ */
+ ListenableFuture<Void> submitTests(final BlockingQueue<TestBatch> parallelWorkQueue,
+ final BlockingQueue<TestBatch> isolatedWorkQueue, final List<TestBatch> failedTestResults) {
+ return mExecutor.submit(new Callable<Void>() {
+ @Override
+ public Void call() throws Exception {
+ executeTests(parallelWorkQueue, isolatedWorkQueue, failedTestResults);
+ return null;
+ }
+
+ });
+ }
+
+ int remainingDrones() {
+ return mDrones.size();
+ }
+ Host getHost() {
+ return mHost;
+ }
+ /**
+ * Executes parallel test until the parallel work queue is empty. Then
+ * executes the isolated tests on the host. During each phase if a
+ * AbortDroneException is thrown the drone is removed possibly
+ * leaving this host with zero functioning drones. If all drones
+ * are removed the host will be replaced before the next run.
+ */
+ private void executeTests(final BlockingQueue<TestBatch> parallelWorkQueue,
+ final BlockingQueue<TestBatch> isolatedWorkQueue, final List<TestBatch> failedTestResults)
+ throws Exception {
+ mLogger.info("Starting parallel execution on " + mHost.getName());
+ List<ListenableFuture<Void>> droneResults = Lists.newArrayList();
+ for(final Drone drone : ImmutableList.copyOf(mDrones)) {
+ droneResults.add(mExecutor.submit(new Callable<Void>() {
+ @Override
+ public Void call() throws Exception {
+ TestBatch batch = null;
+ try {
+ do {
+ batch = parallelWorkQueue.poll(mNumPollSeconds, TimeUnit.SECONDS);
+ if(batch != null) {
+ if(!executeTestBatch(drone, batch)) {
+ failedTestResults.add(batch);
+ }
+ }
+ } while(!parallelWorkQueue.isEmpty());
+ } catch(AbortDroneException ex) {
+ mDrones.remove(drone); // return value not checked due to concurrent access
+ mLogger.error("Aborting drone during parallel execution", ex);
+ if(batch != null) {
+ Preconditions.checkState(parallelWorkQueue.add(batch),
+ "Could not add batch to parallel queue " + batch);
+ }
+ }
+ return null;
+ }
+ }));
+ }
+ Futures.allAsList(droneResults).get();
+ mLogger.info("Starting isolated execution on " + mHost.getName());
+ for(Drone drone : ImmutableList.copyOf(mDrones)) {
+ TestBatch batch = null;
+ try {
+ do {
+ batch = isolatedWorkQueue.poll(mNumPollSeconds, TimeUnit.SECONDS);
+ if(batch != null) {
+ if(!executeTestBatch(drone, batch)) {
+ failedTestResults.add(batch);
+ }
+ }
+ } while(!isolatedWorkQueue.isEmpty());
+ } catch(AbortDroneException ex) {
+ mDrones.remove(drone); // return value not checked due to concurrent access
+ mLogger.error("Aborting drone during isolated execution", ex);
+ if(batch != null) {
+ Preconditions.checkState(isolatedWorkQueue.add(batch),
+ "Could not add batch to isolated queue " + batch);
+ }
+ }
+ }
+ }
+ /**
+ * Executes the test batch on the drone in question. If the command
+ * exits with a status code of 255 throw an AbortDroneException.
+ */
+ private boolean executeTestBatch(Drone drone, TestBatch batch)
+ throws IOException, SSHExecutionException, AbortDroneException {
+ String scriptName = "hiveptest-" + batch.getName() + ".sh";
+ File script = new File(mLocalScratchDirectory, scriptName);
+ Map<String, String> templateVariables = Maps.newHashMap(mTemplateDefaults);
+ templateVariables.put("instanceName", drone.getInstanceName());
+ templateVariables.put("batchName",batch.getName());
+ templateVariables.put("testArguments", batch.getTestArguments());
+ templateVariables.put("localDir", drone.getLocalDirectory());
+ templateVariables.put("logDir", drone.getLocalLogDirectory());
+ String command = Templates.getTemplateResult("bash $localDir/$instanceName/scratch/" + script.getName(),
+ templateVariables);
+ Templates.writeTemplateResult("batch-exec.vm", script, templateVariables);
+ copyToDroneFromLocal(drone, script.getAbsolutePath(), "$localDir/$instanceName/scratch/" + scriptName);
+ script.delete();
+ mLogger.info(drone + " executing " + batch + " with " + command);
+ RemoteCommandResult sshResult = new SSHCommand(mSSHCommandExecutor, drone.getPrivateKey(), drone.getUser(),
+ drone.getHost(), drone.getInstance(), command).
+ call();
+ File batchLogDir = null;
+ if(sshResult.getExitCode() == Constants.EXIT_CODE_UNKNOWN) {
+ throw new AbortDroneException("Drone " + drone.toString() + " exited with " +
+ Constants.EXIT_CODE_UNKNOWN + ": " + sshResult);
+ }
+ boolean result;
+ if(sshResult.getExitCode() != 0 || sshResult.getException() != null) {
+ result = false;
+ batchLogDir = Dirs.create(new File(mFailedTestLogDir, batch.getName()));
+ } else {
+ result = true;
+ batchLogDir = Dirs.create(new File(mSuccessfulTestLogDir, batch.getName()));
+ }
+ copyFromDroneToLocal(drone, batchLogDir.getAbsolutePath(),
+ drone.getLocalLogDirectory() + "/");
+ File logFile = new File(batchLogDir, String.format("%s.txt", batch.getName()));
+ PrintWriter writer = new PrintWriter(logFile);
+ writer.write(String.format("result = '%s'\n", sshResult.toString()));
+ writer.write(String.format("output = '%s'\n", sshResult.getOutput()));
+ if(sshResult.getException() != null) {
+ sshResult.getException().printStackTrace(writer);
+ }
+ writer.close();
+ return result;
+ }
+ /**
+ * RSync from a single drone. If the command exits with a status of not 0
+ * throw an AbortDroneException.
+ */
+ RSyncResult copyToDroneFromLocal(Drone drone, String localFile, String remoteFile)
+ throws AbortDroneException, SSHExecutionException, IOException {
+ Map<String, String> templateVariables = Maps.newHashMap(mTemplateDefaults);
+ templateVariables.put("instanceName", drone.getInstanceName());
+ templateVariables.put("localDir", drone.getLocalDirectory());
+ RSyncResult result = new RSyncCommand(mRSyncCommandExecutor, drone.getPrivateKey(), drone.getUser(),
+ drone.getHost(), drone.getInstance(),
+ Templates.getTemplateResult(localFile, templateVariables),
+ Templates.getTemplateResult(remoteFile, templateVariables),
+ RSyncCommand.Type.FROM_LOCAL).call();
+ if(result.getExitCode() != Constants.EXIT_CODE_SUCCESS) {
+ throw new AbortDroneException("Drone " + drone + " exited with " +
+ result.getExitCode() + ": " + result);
+ }
+ if(result.getException() != null || result.getExitCode() != 0) {
+ throw new SSHExecutionException(result);
+ }
+ return result;
+ }
+ /**
+ * RSync file to all drones. If any drones exit with a status of not 0
+ * they will be removed from use possibly leaving this host with zero
+ * functioning drones.
+ */
+ List<ListenableFuture<RSyncResult>> rsyncFromLocalToRemoteInstances(final String localFile, final String remoteFile)
+ throws InterruptedException, IOException {
+ List<ListenableFuture<RSyncResult>> result = Lists.newArrayList();
+ for(final Drone drone : ImmutableList.copyOf(mDrones)) {
+ final Map<String, String> templateVariables = Maps.newHashMap(mTemplateDefaults);
+ templateVariables.put("instanceName", drone.getInstanceName());
+ templateVariables.put("localDir", drone.getLocalDirectory());
+ result.add(mExecutor.submit(new Callable<RSyncResult>() {
+ @Override
+ public RSyncResult call() throws Exception {
+ RSyncResult result = new RSyncCommand(mRSyncCommandExecutor, drone.getPrivateKey(), drone.getUser(),
+ drone.getHost(), drone.getInstance(),
+ Templates.getTemplateResult(localFile, templateVariables),
+ Templates.getTemplateResult(remoteFile, templateVariables),
+ RSyncCommand.Type.FROM_LOCAL).call();
+ if(result.getExitCode() != Constants.EXIT_CODE_SUCCESS) {
+ mDrones.remove(drone);
+ mLogger.error("Aborting drone during rsync",
+ new AbortDroneException("Drone " + drone + " exited with "
+ + result.getExitCode() + ": " + result));
+ return null;
+ } else {
+ return result;
+ }
+ }
+ }));
+ }
+ return result;
+ }
+ RSyncResult copyFromDroneToLocal(Drone drone, String localFile, String remoteFile)
+ throws SSHExecutionException, IOException {
+ Map<String, String> templateVariables = Maps.newHashMap(mTemplateDefaults);
+ templateVariables.put("instanceName", drone.getInstanceName());
+ templateVariables.put("localDir", drone.getLocalDirectory());
+ RSyncResult result = new RSyncCommand(mRSyncCommandExecutor, drone.getPrivateKey(), drone.getUser(),
+ drone.getHost(), drone.getInstance(),
+ Templates.getTemplateResult(localFile, templateVariables),
+ Templates.getTemplateResult(remoteFile, templateVariables),
+ RSyncCommand.Type.TO_LOCAL).call();
+ if(result.getException() != null || result.getExitCode() != Constants.EXIT_CODE_SUCCESS) {
+ throw new SSHExecutionException(result);
+ }
+ return result;
+ }
+ /**
+ * Execute command on at least one drone. The method will retry when the command
+ * exits with a status code of 255 until all drones have been utilized, possibly
+ * excluding the host from future use.
+ */
+ ListenableFuture<SSHResult> exec(final String cmd)
+ throws Exception {
+ return mExecutor.submit(new Callable<SSHResult>() {
+ @Override
+ public SSHResult call() throws Exception {
+ for(final Drone drone : ImmutableList.copyOf(mDrones)) {
+ Map<String, String> templateVariables = Maps.newHashMap(mTemplateDefaults);
+ templateVariables.put("instanceName", drone.getInstanceName());
+ templateVariables.put("localDir", drone.getLocalDirectory());
+ String command = Templates.getTemplateResult(cmd, templateVariables);
+ SSHResult result = new SSHCommand(mSSHCommandExecutor, drone.getPrivateKey(), drone.getUser(),
+ drone.getHost(), drone.getInstance(), command).call();
+ if(result.getExitCode() == Constants.EXIT_CODE_UNKNOWN) {
+ mDrones.remove(drone); // return value not checked due to concurrent access
+ mLogger.error("Aborting drone during exec " + command,
+ new AbortDroneException("Drone " + drone + " exited with "
+ + Constants.EXIT_CODE_UNKNOWN + ": " + result));
+ } else {
+ return result;
+ }
+ }
+ return null;
+ }
+ });
+
+ }
+ List<ListenableFuture<SSHResult>> execInstances(final String cmd)
+ throws SSHExecutionException, InterruptedException, IOException {
+ List<ListenableFuture<SSHResult>> result = Lists.newArrayList();
+ for(final Drone drone : ImmutableList.copyOf(mDrones)) {
+ result.add(mExecutor.submit(new Callable<SSHResult>() {
+ @Override
+ public SSHResult call() throws Exception {
+ Map<String, String> templateVariables = Maps.newHashMap(mTemplateDefaults);
+ templateVariables.put("instanceName", drone.getInstanceName());
+ templateVariables.put("localDir", drone.getLocalDirectory());
+ String command = Templates.getTemplateResult(cmd, templateVariables);
+ SSHResult result = new SSHCommand(mSSHCommandExecutor, drone.getPrivateKey(), drone.getUser(),
+ drone.getHost(), drone.getInstance(), command).call();
+ if(result.getExitCode() == Constants.EXIT_CODE_UNKNOWN) {
+ mDrones.remove(drone); // return value not checked due to concurrent access
+ mLogger.error("Aborting drone during exec " + command,
+ new AbortDroneException("Drone " + drone + " exited with "
+ + Constants.EXIT_CODE_UNKNOWN + ": " + result));
+ return null;
+ } else {
+ return result;
+ }
+ }
+ }));
+ }
+ return result;
+ }
+}
\ No newline at end of file
Added: hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/JIRAService.java
URL: http://svn.apache.org/viewvc/hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/JIRAService.java?rev=1504759&view=auto
==============================================================================
--- hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/JIRAService.java (added)
+++ hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/JIRAService.java Fri Jul 19 03:47:36 2013
@@ -0,0 +1,185 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hive.ptest.execution;
+
+import java.io.IOException;
+import java.net.URL;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.hive.ptest.execution.conf.TestConfiguration;
+import org.apache.http.HttpException;
+import org.apache.http.HttpHost;
+import org.apache.http.HttpRequest;
+import org.apache.http.HttpRequestInterceptor;
+import org.apache.http.HttpResponse;
+import org.apache.http.StatusLine;
+import org.apache.http.auth.AuthScheme;
+import org.apache.http.auth.AuthScope;
+import org.apache.http.auth.AuthState;
+import org.apache.http.auth.Credentials;
+import org.apache.http.auth.UsernamePasswordCredentials;
+import org.apache.http.client.CredentialsProvider;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.protocol.ClientContext;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.impl.auth.BasicScheme;
+import org.apache.http.impl.client.DefaultHttpClient;
+import org.apache.http.protocol.BasicHttpContext;
+import org.apache.http.protocol.ExecutionContext;
+import org.apache.http.protocol.HttpContext;
+import org.slf4j.Logger;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Joiner;
+import com.google.common.collect.Lists;
+
+class JIRAService {
+ private final Logger mLogger;
+ private final String mName;
+ private final String mBuildTag;
+ private final String mPatch;
+ private final String mUrl;
+ private final String mUser;
+ private final String mPassword;
+ private final String mJenkinsURL;
+
+ public JIRAService(Logger logger, TestConfiguration configuration, String buildTag) {
+ mLogger = logger;
+ mName = configuration.getJiraName();
+ mBuildTag = buildTag;
+ mPatch = configuration.getPatch();
+ mUrl = configuration.getJiraUrl();
+ mUser = configuration.getJiraUser();
+ mPassword = configuration.getJiraPassword();
+ mJenkinsURL = configuration.getJenkinsURL();
+ }
+
+ void postComment(boolean error, Set<String> failedTests,
+ List<String> messages) {
+ DefaultHttpClient httpClient = new DefaultHttpClient();
+ try {
+ String buildTag = formatBuildTag(mBuildTag);
+ List<String> comments = Lists.newArrayList();
+ comments.add("");
+ comments.add("");
+ if (error || !failedTests.isEmpty()) {
+ comments.add("{color:red}Overall{color}: -1 build exited with an error");
+ } else {
+ comments.add("{color:green}Overall{color}: +1 all checks pass");
+ }
+ comments.add("");
+ if(!mPatch.isEmpty()) {
+ comments.add("Here are the results of testing the latest attachment:");
+ comments.add(mPatch);
+ }
+ comments.add("");
+ if (failedTests.isEmpty()) {
+ comments.add(formatSuccess("+1 all tests passed"));
+ } else {
+ comments.add(formatError("-1 due to " + failedTests.size()
+ + " failed/errored test(s)"));
+ comments.add("Failed tests:");
+ comments.addAll(failedTests);
+ }
+ comments.add("");
+ comments.add("Test results: " + mJenkinsURL + "/" + buildTag + "/testReport");
+ comments.add("Console output: " + mJenkinsURL + "/" + buildTag + "/console");
+ comments.add("");
+ comments.add("Messages:");
+ for (String message : messages) {
+ comments.add(message.replaceAll("\n", "\\n"));
+ }
+ comments.add("");
+ comments.add("This message is automatically generated.");
+ mLogger.info("Comment: " + Joiner.on("\n").join(comments));
+ String body = Joiner.on("\\n").join(comments);
+ String url = String.format("%s/rest/api/2/issue/%s/comment", mUrl, mName);
+ URL apiURL = new URL(mUrl);
+ httpClient.getCredentialsProvider()
+ .setCredentials(
+ new AuthScope(apiURL.getHost(), apiURL.getPort(),
+ AuthScope.ANY_REALM),
+ new UsernamePasswordCredentials(mUser, mPassword));
+ BasicHttpContext localcontext = new BasicHttpContext();
+ localcontext.setAttribute("preemptive-auth", new BasicScheme());
+ httpClient.addRequestInterceptor(new PreemptiveAuth(), 0);
+ HttpPost request = new HttpPost(url);
+ StringEntity params = new StringEntity(String.format(
+ "{\"body\": \"%s\"}", body));
+ request.addHeader("Content-Type", "application/json");
+ request.setEntity(params);
+ HttpResponse httpResponse = httpClient.execute(request, localcontext);
+ StatusLine statusLine = httpResponse.getStatusLine();
+ if (statusLine.getStatusCode() != 201) {
+ throw new RuntimeException(statusLine.getStatusCode() + " "
+ + statusLine.getReasonPhrase());
+ }
+ mLogger.info("JIRA Response Metadata: " + httpResponse);
+ } catch (Exception e) {
+ mLogger.error("Encountered error attempting to post comment to " + mName,
+ e);
+ } finally {
+ httpClient.getConnectionManager().shutdown();
+ }
+ }
+
+ /**
+ * Hive-Build-123 to Hive-Build/123
+ */
+ @VisibleForTesting
+ static String formatBuildTag(String buildTag) {
+ if(buildTag.contains("-")) {
+ int lastDashIndex = buildTag.lastIndexOf("-");
+ String buildName = buildTag.substring(0, lastDashIndex);
+ String buildId = buildTag.substring(lastDashIndex + 1);
+ return buildName + "/" + buildId;
+ }
+ throw new IllegalArgumentException("Build tag '" + buildTag + "' must contain a -");
+ }
+ private static String formatError(String msg) {
+ return String.format("{color:red}ERROR:{color} %s", msg);
+ }
+
+ private static String formatSuccess(String msg) {
+ return String.format("{color:green}SUCCESS:{color} %s", msg);
+ }
+
+ static class PreemptiveAuth implements HttpRequestInterceptor {
+
+ public void process(final HttpRequest request, final HttpContext context)
+ throws HttpException, IOException {
+ AuthState authState = (AuthState) context.getAttribute(ClientContext.TARGET_AUTH_STATE);
+ if (authState.getAuthScheme() == null) {
+ AuthScheme authScheme = (AuthScheme) context.getAttribute("preemptive-auth");
+ CredentialsProvider credsProvider = (CredentialsProvider) context.getAttribute(ClientContext.CREDS_PROVIDER);
+ HttpHost targetHost = (HttpHost) context.getAttribute(ExecutionContext.HTTP_TARGET_HOST);
+ if (authScheme != null) {
+ Credentials creds = credsProvider.getCredentials(new AuthScope(
+ targetHost.getHostName(), targetHost.getPort()));
+ if (creds == null) {
+ throw new HttpException(
+ "No credentials for preemptive authentication");
+ }
+ authState.update(authScheme, creds);
+ }
+ }
+ }
+ }
+}
\ No newline at end of file
Added: hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/JUnitReportParser.java
URL: http://svn.apache.org/viewvc/hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/JUnitReportParser.java?rev=1504759&view=auto
==============================================================================
--- hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/JUnitReportParser.java (added)
+++ hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/JUnitReportParser.java Fri Jul 19 03:47:36 2013
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hive.ptest.execution;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.util.Set;
+
+import javax.xml.parsers.SAXParser;
+import javax.xml.parsers.SAXParserFactory;
+
+import org.slf4j.Logger;
+import org.xml.sax.Attributes;
+import org.xml.sax.InputSource;
+import org.xml.sax.helpers.DefaultHandler;
+
+import com.google.common.collect.Sets;
+
+
+public class JUnitReportParser {
+ private final File directory;
+ private final Logger logger;
+ public JUnitReportParser(Logger logger, File directory) throws Exception {
+ this.logger = logger;
+ this.directory = directory;
+ }
+
+ private Set<File> getFiles(File directory) {
+ Set<File> result = Sets.newHashSet();
+ File[] files = directory.listFiles();
+ if(files != null) {
+ for(File file : files) {
+ if(file.isFile()) {
+ String name = file.getName();
+ if(name.startsWith("TEST-") && name.endsWith(".xml")) {
+ result.add(file);
+ }
+ } else if(file.isDirectory()) {
+ result.addAll(getFiles(file));
+ }
+ }
+ }
+ return result;
+ }
+ public Set<String> getFailedTests() {
+ final Set<String> failedTests = Sets.newHashSet();
+ for(File file : getFiles(directory)) {
+ FileInputStream stream = null;
+ try {
+ stream = new FileInputStream(file);
+ SAXParserFactory factory = SAXParserFactory.newInstance();
+ SAXParser saxParser = factory.newSAXParser();
+ saxParser.parse(new InputSource( stream ), new DefaultHandler() {
+ private String name;
+ private boolean failedOrErrored;
+ @Override
+ public void startElement(String uri, String localName, String qName, Attributes attributes) {
+ if ("testcase".equals(qName)) {
+ name = attributes.getValue("classname");
+ failedOrErrored = false;
+ if(name == null) {
+ name = attributes.getValue("name");
+ } else {
+ name = name + "." + attributes.getValue("name");
+ }
+ } else if (name != null && ("failure".equals(qName) || "error".equals(qName))) {
+ failedOrErrored = true;
+ }
+ }
+ @Override
+ public void endElement(String uri, String localName, String qName) {
+ if ("testcase".equals(qName)) {
+ if(failedOrErrored && name != null) {
+ failedTests.add(name);
+ }
+ }
+ }
+ });
+ } catch (Exception e) {
+ logger.error("Error parsing file " + file, e);
+ } finally {
+ if(stream != null) {
+ try {
+ stream.close();
+ } catch (IOException e) {
+ logger.warn("Error closing file " + file, e);
+ }
+ }
+ }
+ }
+ return failedTests;
+ }
+}
\ No newline at end of file
Added: hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/LocalCommand.java
URL: http://svn.apache.org/viewvc/hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/LocalCommand.java?rev=1504759&view=auto
==============================================================================
--- hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/LocalCommand.java (added)
+++ hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/LocalCommand.java Fri Jul 19 03:47:36 2013
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hive.ptest.execution;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+
+import org.slf4j.Logger;
+
+public class LocalCommand {
+
+ private final Process process;
+ private final StreamReader streamReader;
+ private Integer exitCode;
+
+ public LocalCommand(Logger logger, OutputPolicy outputPolicy, String command) throws IOException {
+ logger.info("Starting " + command);
+ process = new ProcessBuilder().command(new String[] {"bash", "-c", command}).redirectErrorStream(true).start();
+ streamReader = new StreamReader(outputPolicy, process.getInputStream());
+ streamReader.setName("StreamReader-[" + command + "]");
+ streamReader.setDaemon(true);
+ streamReader.start();
+ }
+
+ public int getExitCode() throws InterruptedException {
+ synchronized (process) {
+ if(exitCode == null) {
+ exitCode = process.waitFor();
+ }
+ return exitCode;
+ }
+ }
+
+ public static interface OutputPolicy {
+ public void handleOutput(String line);
+ public void handleThrowable(Throwable throwable);
+ }
+ public static class CollectLogPolicy extends CollectPolicy {
+ private final Logger logger;
+ public CollectLogPolicy(Logger logger) {
+ this.logger = logger;
+ }
+ @Override
+ public void handleOutput(String line) {
+ logger.info(line);
+ output.append(line).append("\n");
+ }
+ }
+ public static class CollectPolicy implements OutputPolicy {
+ protected final StringBuilder output = new StringBuilder();
+ protected Throwable throwable;
+ @Override
+ public void handleOutput(String line) {
+ output.append(line).append("\n");
+ }
+ @Override
+ public void handleThrowable(Throwable throwable) {
+ if(throwable instanceof IOException &&
+ "Stream closed".equals(throwable.getMessage())) {
+ return;
+ }
+ this.throwable = throwable;
+ }
+ public String getOutput() {
+ String result = output.toString();
+ if(throwable != null) {
+ throw new RuntimeException(result, throwable);
+ }
+ return result;
+ }
+ }
+
+ private static class StreamReader extends Thread {
+ private final BufferedReader input;
+ private final OutputPolicy outputPolicy;
+ public StreamReader(OutputPolicy outputPolicy, InputStream in) {
+ this.outputPolicy = outputPolicy;
+ this.input = new BufferedReader(new InputStreamReader(in));
+ }
+ @Override
+ public void run() {
+ try {
+ String line;
+ while((line = input.readLine()) != null) {
+ outputPolicy.handleOutput(line);
+ }
+ } catch(Exception e) {
+ outputPolicy.handleThrowable(e);
+ } finally {
+ try {
+ input.close();
+ } catch (IOException ignored) {
+
+ }
+ }
+ }
+ }
+}
\ No newline at end of file
Added: hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/LocalCommandFactory.java
URL: http://svn.apache.org/viewvc/hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/LocalCommandFactory.java?rev=1504759&view=auto
==============================================================================
--- hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/LocalCommandFactory.java (added)
+++ hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/LocalCommandFactory.java Fri Jul 19 03:47:36 2013
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hive.ptest.execution;
+
+import java.io.IOException;
+
+import org.slf4j.Logger;
+
+public class LocalCommandFactory {
+
+
+ private final Logger mLogger;
+
+ public LocalCommandFactory(Logger logger) {
+ mLogger = logger;
+ }
+ public LocalCommand create(LocalCommand.CollectPolicy policy, String command)
+ throws IOException {
+ return new LocalCommand(mLogger, policy, command);
+ }
+}
Added: hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/LogDirectoryCleaner.java
URL: http://svn.apache.org/viewvc/hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/LogDirectoryCleaner.java?rev=1504759&view=auto
==============================================================================
--- hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/LogDirectoryCleaner.java (added)
+++ hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/LogDirectoryCleaner.java Fri Jul 19 03:47:36 2013
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hive.ptest.execution;
+
+import java.io.File;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.io.FileUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+
+public class LogDirectoryCleaner extends Thread {
+ private static final Logger LOG = LoggerFactory
+ .getLogger(LogDirectoryCleaner.class);
+ private final File mLogDir;
+ private final int mMaxDirectoriesPerProfile;
+ public LogDirectoryCleaner(File logDir, int maxDirectoriesPerProfile) {
+ mLogDir = logDir;
+ mMaxDirectoriesPerProfile = maxDirectoriesPerProfile;
+ }
+
+ @Override
+public void run() {
+ try {
+ File[] logDirs = mLogDir.listFiles();
+ if(logDirs != null && logDirs.length > 0) {
+ Map<String, ProfileLogs> profiles = Maps.newHashMap();
+ for(File logDir : logDirs) {
+ String name = logDir.getName();
+ if(name.contains("-")) {
+ String profile = name.substring(0, name.lastIndexOf("-"));
+ ProfileLogs logs = profiles.get(profile);
+ if(logs == null) {
+ logs = new ProfileLogs(profile);
+ profiles.put(profile, logs);
+ }
+ logs.dirs.add(logDir);
+ }
+ }
+ for(String profile : profiles.keySet()) {
+ ProfileLogs logs = profiles.get(profile);
+ if(logs.dirs.size() > mMaxDirectoriesPerProfile) {
+ File oldest = logs.getOldest();
+ LOG.info("Deleting " + oldest + " from " + logs.dirs);
+ FileUtils.deleteQuietly(oldest);
+ }
+ }
+ }
+ } catch(Throwable t) {
+ LOG.error("Unexpected error cleaning " + mLogDir, t);
+ }
+ }
+
+ private static class ProfileLogs {
+ String name;
+ List<File> dirs = Lists.newArrayList();
+ ProfileLogs(String name) {
+ this.name = name;
+ }
+ File getOldest() {
+ Preconditions.checkState(!dirs.isEmpty(), "Cannot be called unless dirs.size() >= 1");
+ File eldestDir = null;
+ int eldestId = Integer.MAX_VALUE;
+ for(File dir : dirs) {
+ try {
+ int id = Integer.parseInt(dir.getName().substring(name.length() + 1));
+ if(id < eldestId) {
+ eldestId = id;
+ eldestDir = dir;
+ }
+ } catch (NumberFormatException e) {
+ LOG.warn("Error parsing " + dir.getName(), e);
+ }
+ }
+ return Preconditions.checkNotNull(eldestDir, "eldestDir");
+ }
+ }
+}
Added: hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java
URL: http://svn.apache.org/viewvc/hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java?rev=1504759&view=auto
==============================================================================
--- hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java (added)
+++ hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java Fri Jul 19 03:47:36 2013
@@ -0,0 +1,305 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hive.ptest.execution;
+
+import java.io.File;
+import java.net.URL;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.Options;
+import org.apache.hive.ptest.execution.conf.ExecutionContextConfiguration;
+import org.apache.hive.ptest.execution.conf.Host;
+import org.apache.hive.ptest.execution.conf.TestConfiguration;
+import org.apache.hive.ptest.execution.conf.TestParser;
+import org.apache.hive.ptest.execution.context.ExecutionContext;
+import org.apache.hive.ptest.execution.context.ExecutionContextProvider;
+import org.apache.hive.ptest.execution.ssh.RSyncCommandExecutor;
+import org.apache.hive.ptest.execution.ssh.SSHCommandExecutor;
+import org.apache.velocity.app.Velocity;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Strings;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.io.Files;
+import com.google.common.io.Resources;
+import com.google.common.util.concurrent.ListeningExecutorService;
+import com.google.common.util.concurrent.MoreExecutors;
+
+public class PTest {
+
+ static {
+ Velocity.init();
+ }
+ private static final Logger LOG = LoggerFactory
+ .getLogger(PTest.class);
+
+
+ private final TestConfiguration mConfiguration;
+ private final ListeningExecutorService mExecutor;
+ private final Set<String> mFailedTests;
+ private final List<Phase> mPhases;
+ private final ExecutionContext mExecutionContext;
+ private final Logger mLogger;
+ private final ImmutableList<HostExecutor> mHostExecutors;
+ private final String mBuildTag;
+
+ public PTest(final TestConfiguration configuration, ExecutionContext executionContext,
+ String buildTag, File logDir, LocalCommandFactory localCommandFactory, SSHCommandExecutor sshCommandExecutor,
+ RSyncCommandExecutor rsyncCommandExecutor, Logger logger)
+ throws Exception {
+ mConfiguration = configuration;
+ mLogger = logger;
+ mBuildTag = buildTag;
+ mFailedTests = Collections.newSetFromMap(new ConcurrentHashMap<String, Boolean>());
+ mExecutionContext = executionContext;
+ mExecutor = MoreExecutors.listeningDecorator(Executors.newCachedThreadPool());
+ File failedLogDir = Dirs.create(new File(logDir, "failed"));
+ File succeededLogDir = Dirs.create(new File(logDir, "succeeded"));
+ File scratchDir = Dirs.createEmpty(new File(mExecutionContext.getLocalWorkingDirectory(), "scratch"));
+ File patchDir = Dirs.createEmpty(new File(logDir, "patches"));
+ File patchFile = null;
+ if(!configuration.getPatch().isEmpty()) {
+ patchFile = new File(patchDir, buildTag + ".patch");
+ Files.write(Resources.toByteArray(new URL(configuration.getPatch())), patchFile);
+ }
+ ImmutableMap.Builder<String, String> templateDefaultsBuilder = ImmutableMap.builder();
+ templateDefaultsBuilder.
+ put("repository", configuration.getRepository()).
+ put("repositoryName", configuration.getRepositoryName()).
+ put("repositoryType", configuration.getRepositoryType()).
+ put("branch", configuration.getBranch()).
+ put("workingDir", mExecutionContext.getLocalWorkingDirectory()).
+ put("antArgs", configuration.getAntArgs()).
+ put("buildTag", buildTag).
+ put("logDir", logDir.getAbsolutePath()).
+ put("javaHome", configuration.getJavaHome()).
+ put("antEnvOpts", configuration.getAntEnvOpts());
+ ImmutableMap<String, String> templateDefaults = templateDefaultsBuilder.build();
+ TestParser testParser = new TestParser(configuration.getContext(),
+ new File(mExecutionContext.getLocalWorkingDirectory(), configuration.getRepositoryName() + "-source"),
+ logger);
+
+ ImmutableList.Builder<HostExecutor> hostExecutorsBuilder = ImmutableList.builder();
+ for(Host host : mExecutionContext.getHosts()) {
+ hostExecutorsBuilder.add(new HostExecutor(host, executionContext.getPrivateKey(), mExecutor, sshCommandExecutor,
+ rsyncCommandExecutor, templateDefaults, scratchDir, succeededLogDir, failedLogDir, 10, logger));
+ }
+ mHostExecutors = hostExecutorsBuilder.build();
+ mPhases = Lists.newArrayList();
+ mPhases.add(new CleanupPhase(mHostExecutors, localCommandFactory, templateDefaults, logger));
+ mPhases.add(new PrepPhase(mHostExecutors, localCommandFactory, templateDefaults, scratchDir, patchFile, logger));
+ mPhases.add(new ExecutionPhase(mHostExecutors, localCommandFactory, templateDefaults,
+ failedLogDir, testParser.parse(), mFailedTests, logger));
+ mPhases.add(new ReportingPhase(mHostExecutors, localCommandFactory, templateDefaults, logger));
+ }
+ public int run() {
+ int result = 0;
+ boolean error = false;
+ List<String> messages = Lists.newArrayList();
+ Map<String, Long> elapsedTimes = Maps.newTreeMap();
+ try {
+ mLogger.info("Running tests with " + mConfiguration);
+ for(Phase phase : mPhases) {
+ String msg = "Executing " + phase.getClass().getName();
+ mLogger.info(msg);
+ messages.add(msg);
+ long start = System.currentTimeMillis();
+ try {
+ phase.execute();
+ } finally {
+ long elapsedTime = TimeUnit.MINUTES.convert((System.currentTimeMillis() - start),
+ TimeUnit.MILLISECONDS);
+ elapsedTimes.put(phase.getClass().getSimpleName(), elapsedTime);
+ }
+ }
+ for(HostExecutor hostExecutor : mHostExecutors) {
+ if(hostExecutor.remainingDrones() == 0) {
+ mExecutionContext.addBadHost(hostExecutor.getHost());
+ }
+ }
+ if(!mFailedTests.isEmpty()) {
+ throw new TestsFailedException(mFailedTests.size() + " tests failed");
+ }
+ } catch(Throwable throwable) {
+ mLogger.error("Test run exited with an unexpected error", throwable);
+ messages.add("Tests failed with: " + throwable.getClass().getSimpleName() + ": " + throwable.getMessage());
+ error = true;
+ } finally {
+ mExecutor.shutdownNow();
+ if(mFailedTests.isEmpty()) {
+ mLogger.info(String.format("%d failed tests", mFailedTests.size()));
+ } else {
+ mLogger.warn(String.format("%d failed tests", mFailedTests.size()));
+ }
+ for(String failingTestName : mFailedTests) {
+ mLogger.warn(failingTestName);
+ }
+ for(Map.Entry<String, Long> entry : elapsedTimes.entrySet()) {
+ mLogger.info(String.format("PERF: Phase %s took %d minutes", entry.getKey(), entry.getValue()));
+ }
+ publishJiraComment(error, messages);
+ if(error || !mFailedTests.isEmpty()) {
+ result = 1;
+ }
+ }
+ return result;
+ }
+
+ private void publishJiraComment(boolean error, List<String> messages) {
+ if(mConfiguration.getJiraName().isEmpty()) {
+ mLogger.info("Skipping JIRA comment as name is empty.");
+ return;
+ }
+ if(mConfiguration.getJiraUrl().isEmpty()) {
+ mLogger.info("Skipping JIRA comment as URL is empty.");
+ return;
+ }
+ if(mConfiguration.getJiraUser().isEmpty()) {
+ mLogger.info("Skipping JIRA comment as user is empty.");
+ return;
+ }
+ if(mConfiguration.getJiraPassword().isEmpty()) {
+ mLogger.info("Skipping JIRA comment as password is empty.");
+ return;
+ }
+ JIRAService jira = new JIRAService(mLogger, mConfiguration, mBuildTag);
+ jira.postComment(error, mFailedTests, messages);
+ }
+
+ public static class Builder {
+ public PTest build(TestConfiguration configuration, ExecutionContext executionContext,
+ String buildTag, File logDir, LocalCommandFactory localCommandFactory, SSHCommandExecutor sshCommandExecutor,
+ RSyncCommandExecutor rsyncCommandExecutor, Logger logger) throws Exception {
+ return new PTest(configuration, executionContext, buildTag, logDir, localCommandFactory, sshCommandExecutor,
+ rsyncCommandExecutor, logger);
+ }
+ }
+
+ private static final String PROPERTIES = "properties";
+ private static final String REPOSITORY = "repository";
+ private static final String REPOSITORY_NAME = "repositoryName";
+ private static final String BRANCH = "branch";
+ private static final String PATCH = "patch";
+ private static final String JAVA_HOME = "javaHome";
+ private static final String ANT_ENV_OPTS = "antEnvOpts";
+ /**
+ * All args override properties file settings except
+ * for this one which is additive.
+ */
+ private static final String ANT_ARG = "D";
+
+ public static void main(String[] args) throws Exception {
+ LOG.info("Args " + Arrays.toString(args));
+ CommandLineParser parser = new GnuParser();
+ Options options = new Options();
+ options.addOption(null, PROPERTIES, true, "properties file");
+ options.addOption(null, REPOSITORY, true, "Overrides git repository in properties file");
+ options.addOption(null, REPOSITORY_NAME, true, "Overrides git repository *name* in properties file");
+ options.addOption(null, BRANCH, true, "Overrides git branch in properties file");
+ options.addOption(null, PATCH, true, "URI to patch, either file:/// or http(s)://");
+ options.addOption(ANT_ARG, null, true, "Supplemntal ant arguments");
+ options.addOption(null, JAVA_HOME, true, "Java Home for compiling and running tests");
+ options.addOption(null, ANT_ENV_OPTS, true, "ANT_OPTS environemnt variable setting");
+ CommandLine commandLine = parser.parse(options, args);
+ if(!commandLine.hasOption(PROPERTIES)) {
+ throw new IllegalArgumentException(Joiner.on(" ").
+ join(PTest.class.getName(), "--" + PROPERTIES,"config.properties"));
+ }
+ String testConfigurationFile = commandLine.getOptionValue(PROPERTIES);
+ ExecutionContextConfiguration executionContextConfiguration = ExecutionContextConfiguration.
+ fromFile(testConfigurationFile);
+ String buildTag = System.getenv("BUILD_TAG") == null ? "undefined-"
+ + System.currentTimeMillis() : System.getenv("BUILD_TAG");
+ File logDir = Dirs.create(new File(executionContextConfiguration.getGlobalLogDirectory(), buildTag));
+ LogDirectoryCleaner cleaner = new LogDirectoryCleaner(new File(executionContextConfiguration.
+ getGlobalLogDirectory()), 5);
+ cleaner.setName("LogCleaner-" + executionContextConfiguration.getGlobalLogDirectory());
+ cleaner.setDaemon(true);
+ cleaner.start();
+ TestConfiguration conf = TestConfiguration.fromFile(testConfigurationFile, LOG);
+ String repository = Strings.nullToEmpty(commandLine.getOptionValue(REPOSITORY)).trim();
+ if(!repository.isEmpty()) {
+ conf.setRepository(repository);
+ }
+ String repositoryName = Strings.nullToEmpty(commandLine.getOptionValue(REPOSITORY_NAME)).trim();
+ if(!repositoryName.isEmpty()) {
+ conf.setRepositoryName(repositoryName);
+ }
+ String branch = Strings.nullToEmpty(commandLine.getOptionValue(BRANCH)).trim();
+ if(!branch.isEmpty()) {
+ conf.setBranch(branch);
+ }
+ String patch = Strings.nullToEmpty(commandLine.getOptionValue(PATCH)).trim();
+ if(!patch.isEmpty()) {
+ conf.setPatch(patch);
+ }
+ String javaHome = Strings.nullToEmpty(commandLine.getOptionValue(JAVA_HOME)).trim();
+ if(!javaHome.isEmpty()) {
+ conf.setJavaHome(javaHome);
+ }
+ String antEnvOpts = Strings.nullToEmpty(commandLine.getOptionValue(ANT_ENV_OPTS)).trim();
+ if(!antEnvOpts.isEmpty()) {
+ conf.setAntEnvOpts(antEnvOpts);
+ }
+ String[] supplementalAntArgs = commandLine.getOptionValues(ANT_ARG);
+ if(supplementalAntArgs != null && supplementalAntArgs.length > 0) {
+ String antArgs = Strings.nullToEmpty(conf.getAntArgs());
+ if(!(antArgs.isEmpty() || antArgs.endsWith(" "))) {
+ antArgs += " ";
+ }
+ antArgs += "-" + ANT_ARG + Joiner.on(" -" + ANT_ARG).join(supplementalAntArgs);
+ conf.setAntArgs(antArgs);
+ }
+ ExecutionContextProvider executionContextProvider = null;
+ ExecutionContext executionContext = null;
+ int exitCode = 0;
+ try {
+ executionContextProvider = executionContextConfiguration
+ .getExecutionContextProvider();
+ executionContext = executionContextProvider.createExecutionContext();
+ PTest ptest = new PTest(conf, executionContext, buildTag, logDir,
+ new LocalCommandFactory(LOG), new SSHCommandExecutor(LOG),
+ new RSyncCommandExecutor(LOG), LOG);
+ exitCode = ptest.run();
+ } finally {
+ if(executionContext != null) {
+ executionContext.terminate();
+ }
+ if(executionContextProvider != null) {
+ executionContextProvider.close();
+ }
+ }
+ System.exit(exitCode);
+ }
+}
\ No newline at end of file