You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by br...@apache.org on 2013/11/15 22:02:19 UTC
svn commit: r1542400 - in /hive/trunk/testutils/ptest2/src:
main/java/org/apache/hive/ptest/execution/context/
test/java/org/apache/hive/ptest/execution/context/
Author: brock
Date: Fri Nov 15 21:02:19 2013
New Revision: 1542400
URL: http://svn.apache.org/r1542400
Log:
HIVE-5782 - PTest2 should be able to ride out price spikes
Added:
hive/trunk/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/context/TestCloudComputeService.java
Modified:
hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/CloudComputeService.java
hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/CloudExecutionContextProvider.java
hive/trunk/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/context/TestCloudExecutionContextProvider.java
Modified: hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/CloudComputeService.java
URL: http://svn.apache.org/viewvc/hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/CloudComputeService.java?rev=1542400&r1=1542399&r2=1542400&view=diff
==============================================================================
--- hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/CloudComputeService.java (original)
+++ hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/CloudComputeService.java Fri Nov 15 21:02:19 2013
@@ -35,6 +35,7 @@ import org.jclouds.compute.domain.Templa
import org.jclouds.logging.log4j.config.Log4JLoggingModule;
import com.google.common.base.Predicate;
+import com.google.common.base.Strings;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Sets;
@@ -77,18 +78,32 @@ public class CloudComputeService {
result.addAll(mComputeService.createNodesInGroup(mGroupName, count, template));
return result;
}
+ static Predicate<ComputeMetadata> createFilterPTestPredicate(final String groupName,
+ final String groupTag) {
+ return new Predicate<ComputeMetadata>() {
+ @Override
+ public boolean apply(ComputeMetadata computeMetadata) {
+ NodeMetadata nodeMetadata = (NodeMetadata) computeMetadata;
+ return nodeMetadata.getStatus() == Status.RUNNING && isPTestHost(nodeMetadata);
+ }
+ private boolean isPTestHost(NodeMetadata node) {
+ if(groupName.equalsIgnoreCase(node.getGroup())) {
+ return true;
+ }
+ if(Strings.nullToEmpty(node.getName()).startsWith(groupName)) {
+ return true;
+ }
+ if(node.getTags().contains(groupTag)) {
+ return true;
+ }
+ return false;
+ }
+ };
+ }
public Set<NodeMetadata> listRunningNodes(){
Set<NodeMetadata> result = Sets.newHashSet();
- result.addAll(mComputeService
- .listNodesDetailsMatching(new Predicate<ComputeMetadata>() {
- @Override
- public boolean apply(ComputeMetadata computeMetadata) {
- NodeMetadata nodeMetadata = (NodeMetadata) computeMetadata;
- return nodeMetadata.getStatus() == Status.RUNNING
- && (mGroupName.equalsIgnoreCase(nodeMetadata.getGroup()) ||
- nodeMetadata.getTags().contains(mGroupTag));
- }
- }));
+ result.addAll(mComputeService.listNodesDetailsMatching(
+ createFilterPTestPredicate(mGroupName, mGroupTag)));
return result;
}
public void destroyNode(String nodeId) {
Modified: hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/CloudExecutionContextProvider.java
URL: http://svn.apache.org/viewvc/hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/CloudExecutionContextProvider.java?rev=1542400&r1=1542399&r2=1542400&view=diff
==============================================================================
--- hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/CloudExecutionContextProvider.java (original)
+++ hive/trunk/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/CloudExecutionContextProvider.java Fri Nov 15 21:02:19 2013
@@ -74,7 +74,6 @@ public class CloudExecutionContextProvid
private final String[] mSlaveLocalDirs;
private final int mNumThreads;
private final int mNumHosts;
- private final int mNumRetries;
private final long mRetrySleepInterval;
private final CloudComputeService mCloudComputeService;
private final Map<String, Long> mTerminatedHosts;
@@ -86,14 +85,13 @@ public class CloudExecutionContextProvid
CloudExecutionContextProvider(String dataDir,
int numHosts, CloudComputeService cloudComputeService, SSHCommandExecutor sshCommandExecutor,
String workingDirectory, String privateKey, String user, String[] slaveLocalDirs, int numThreads,
- int numRetries, long retrySleepInterval) throws IOException {
+ long retrySleepInterval) throws IOException {
mNumHosts = numHosts;
mCloudComputeService = cloudComputeService;
mPrivateKey = privateKey;
mUser = user;
mSlaveLocalDirs = slaveLocalDirs;
mNumThreads = numThreads;
- mNumRetries = numRetries;
mRetrySleepInterval = retrySleepInterval;
mSSHCommandExecutor = sshCommandExecutor;
mWorkingDir = Dirs.create(new File(workingDirectory, "working"));
@@ -191,35 +189,33 @@ public class CloudExecutionContextProvid
}
}
- private Set<NodeMetadata> createNodes(int numHosts)
+ private Set<NodeMetadata> createNodes(final int numHosts)
throws CreateHostsFailedException {
Set<NodeMetadata> result = Sets.newHashSet();
int attempts = 0;
int numRequired = numHosts;
- RunNodesException exception = null;
do {
LOG.info("Attempting to create " + numRequired + " nodes");
try {
- result.addAll(verifyHosts(mCloudComputeService.createNodes(numRequired)));
+ result.addAll(mCloudComputeService.createNodes(numRequired));
} catch (RunNodesException e) {
- exception = e;
LOG.warn("Error creating nodes", e);
terminateInternal(e.getNodeErrors().keySet());
- result.addAll(verifyHosts(e.getSuccessfulNodes()));
+ result.addAll(e.getSuccessfulNodes());
}
+ result = verifyHosts(result);
LOG.info("Successfully created " + result.size() + " nodes");
numRequired = numHosts - result.size();
if(numRequired > 0) {
try {
- TimeUnit.SECONDS.sleep(mRetrySleepInterval);
+ TimeUnit.SECONDS.sleep(++attempts * mRetrySleepInterval);
} catch(InterruptedException e) {
throw new CreateHostsFailedException("Interrupted while trying to create hosts", e);
}
}
- } while(attempts++ < mNumRetries && numRequired > 0);
- if(result.size() < numHosts) {
- throw new CreateHostsFailedException("Error creating nodes", exception);
- }
+ } while(numRequired > 0);
+ Preconditions.checkState(result.size() >= numHosts,
+ "Results should always be >= numHosts " + numHosts + " => " + result.size());
return result;
}
@@ -240,39 +236,37 @@ public class CloudExecutionContextProvid
private Set<NodeMetadata> verifyHosts(Set<? extends NodeMetadata> hosts)
throws CreateHostsFailedException {
- persistHostnamesToLog(hosts);
final Set<NodeMetadata> result = Collections.synchronizedSet(new HashSet<NodeMetadata>());
- ExecutorService executorService = Executors.newFixedThreadPool(Math.min(hosts.size(), 25));
- try {
- for(final NodeMetadata node : hosts) {
- executorService.submit(new Runnable() {
- @Override
- public void run() {
- SSHCommand command = new SSHCommand(mSSHCommandExecutor, mPrivateKey, mUser, node.getHostname(), 0, "pkill -f java");
- mSSHCommandExecutor.execute(command);
- if(command.getExitCode() == Constants.EXIT_CODE_UNKNOWN ||
- command.getException() != null) {
- if(command.getException() == null) {
- LOG.error("Node " + node.getHostname() + " is bad on startup");
+ if(!hosts.isEmpty()) {
+ persistHostnamesToLog(hosts);
+ ExecutorService executorService = Executors.newFixedThreadPool(Math.min(hosts.size(), 25));
+ try {
+ for(final NodeMetadata node : hosts) {
+ executorService.submit(new Runnable() {
+ @Override
+ public void run() {
+ SSHCommand command = new SSHCommand(mSSHCommandExecutor, mPrivateKey, mUser, node.getHostname(), 0, "pkill -f java");
+ mSSHCommandExecutor.execute(command);
+ if(command.getExitCode() == Constants.EXIT_CODE_UNKNOWN ||
+ command.getException() != null) {
+ LOG.error("Node " + node + " is bad on startup", command.getException());
+ terminateInternal(node);
} else {
- LOG.error("Node " + node.getHostname() + " is bad on startup", command.getException());
+ result.add(node);
}
- terminateInternal(node);
- } else {
- result.add(node);
}
- }
- });
- }
- executorService.shutdown();
- if(!executorService.awaitTermination(10, TimeUnit.MINUTES)) {
- LOG.error("Verify command still executing on a host after 10 minutes");
- }
- } catch (InterruptedException e) {
- throw new CreateHostsFailedException("Interrupted while trying to create hosts", e);
- } finally {
- if(!executorService.isShutdown()) {
- executorService.shutdownNow();
+ });
+ }
+ executorService.shutdown();
+ if(!executorService.awaitTermination(10, TimeUnit.MINUTES)) {
+ LOG.error("Verify command still executing on a host after 10 minutes");
+ }
+ } catch (InterruptedException e) {
+ throw new CreateHostsFailedException("Interrupted while trying to create hosts", e);
+ } finally {
+ if(!executorService.isShutdown()) {
+ executorService.shutdownNow();
+ }
}
}
return result;
@@ -310,7 +304,7 @@ public class CloudExecutionContextProvid
}
private void terminateInternal(final NodeMetadata node) {
- LOG.info("Submitting termination for " + node.getHostname());
+ LOG.info("Submitting termination for " + node);
mTerminationExecutor.submit(new Runnable() {
@Override
public void run() {
@@ -404,7 +398,7 @@ public class CloudExecutionContextProvid
instanceType, groupName, imageId, keyPair, securityGroup, maxBid);
CloudExecutionContextProvider service = new CloudExecutionContextProvider(
dataDir, numHosts, cloudComputeService, new SSHCommandExecutor(LOG), workingDirectory,
- privateKey, user, localDirs, numThreads, 10, 10);
+ privateKey, user, localDirs, numThreads, 60);
return service;
}
}
Added: hive/trunk/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/context/TestCloudComputeService.java
URL: http://svn.apache.org/viewvc/hive/trunk/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/context/TestCloudComputeService.java?rev=1542400&view=auto
==============================================================================
--- hive/trunk/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/context/TestCloudComputeService.java (added)
+++ hive/trunk/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/context/TestCloudComputeService.java Fri Nov 15 21:02:19 2013
@@ -0,0 +1,54 @@
+package org.apache.hive.ptest.execution.context;
+
+import java.util.Set;
+
+import org.jclouds.compute.domain.NodeMetadata;
+import org.jclouds.compute.domain.NodeMetadata.Status;
+import org.junit.Assert;
+import org.junit.Before;
+
+import com.google.common.collect.Sets;
+
+import static org.mockito.Mockito.*;
+
+public class TestCloudComputeService {
+ private static final String GROUP_NAME = "grp";
+ private static final String GROUP_TAG = "group=" + GROUP_NAME;
+ private NodeMetadata node;
+ private Set<String> tags;
+
+ @Before
+ public void setup() {
+ node = mock(NodeMetadata.class);
+ tags = Sets.newHashSet(GROUP_TAG);
+ when(node.getStatus()).thenReturn(Status.RUNNING);
+ when(node.getName()).thenReturn(GROUP_NAME + "-1");
+ when(node.getGroup()).thenReturn(GROUP_NAME + "-1");
+ when(node.getTags()).thenReturn(tags);
+ }
+
+ @org.junit.Test
+ public void testNotStarted() throws Exception {
+ when(node.getStatus()).thenReturn(Status.ERROR);
+ Assert.assertFalse("Node is not running, should be filtered out", CloudComputeService.
+ createFilterPTestPredicate(GROUP_NAME, GROUP_TAG).apply(node));
+ }
+ @org.junit.Test
+ public void testBadName() throws Exception {
+ when(node.getName()).thenReturn(null);
+ Assert.assertTrue("Node should be filtered in by group or tag", CloudComputeService.
+ createFilterPTestPredicate(GROUP_NAME, GROUP_TAG).apply(node));
+ }
+ @org.junit.Test
+ public void testBadGroup() throws Exception {
+ when(node.getGroup()).thenReturn(null);
+ Assert.assertTrue("Node should be filtered in by name or tag", CloudComputeService.
+ createFilterPTestPredicate(GROUP_NAME, GROUP_TAG).apply(node));
+ }
+ @org.junit.Test
+ public void testBadTag() throws Exception {
+ tags.clear();
+ Assert.assertTrue("Node should be filtered in by name or group", CloudComputeService.
+ createFilterPTestPredicate(GROUP_NAME, GROUP_TAG).apply(node));
+ }
+}
Modified: hive/trunk/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/context/TestCloudExecutionContextProvider.java
URL: http://svn.apache.org/viewvc/hive/trunk/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/context/TestCloudExecutionContextProvider.java?rev=1542400&r1=1542399&r2=1542400&view=diff
==============================================================================
--- hive/trunk/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/context/TestCloudExecutionContextProvider.java (original)
+++ hive/trunk/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/context/TestCloudExecutionContextProvider.java Fri Nov 15 21:02:19 2013
@@ -92,7 +92,7 @@ public class TestCloudExecutionContextPr
}
@org.junit.Test
- public void testRetrySucceeds() throws Exception {
+ public void testRetry() throws Exception {
when(cloudComputeService.createNodes(anyInt())).then(new Answer<Set<NodeMetadata>>() {
int count = 0;
@Override
@@ -105,7 +105,7 @@ public class TestCloudExecutionContextPr
}
});
CloudExecutionContextProvider provider = new CloudExecutionContextProvider(dataDir, NUM_NODES,
- cloudComputeService, sshCommandExecutor, workingDir, PRIVATE_KEY, USER, SLAVE_DIRS, 1, 1, 0);
+ cloudComputeService, sshCommandExecutor, workingDir, PRIVATE_KEY, USER, SLAVE_DIRS, 1, 0);
ExecutionContext executionContext = provider.createExecutionContext();
Set<String> hosts = Sets.newHashSet();
for(Host host : executionContext.getHosts()) {
@@ -113,11 +113,4 @@ public class TestCloudExecutionContextPr
}
Assert.assertEquals(Sets.newHashSet("node1", "node3"), hosts);
}
- @org.junit.Test(expected=CreateHostsFailedException.class)
- public void testRetryFails() throws Exception {
- when(cloudComputeService.createNodes(anyInt())).thenThrow(runNodesException);
- CloudExecutionContextProvider provider = new CloudExecutionContextProvider(dataDir, NUM_NODES,
- cloudComputeService, sshCommandExecutor, workingDir, PRIVATE_KEY, USER, SLAVE_DIRS, 1, 1, 0);
- provider.createExecutionContext();
- }
}