You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by sp...@apache.org on 2016/09/29 02:33:38 UTC
hive git commit: HIVE-14849: Support google-compute-engine provider
on Hive ptest framework (Sergio Pena, reviewed by Prasanth Jayachandran)
Repository: hive
Updated Branches:
refs/heads/master cf72a7370 -> 291f3d503
HIVE-14849: Support google-compute-engine provider on Hive ptest framework (Sergio Pena, reviewed by Prasanth Jayachandran)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/291f3d50
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/291f3d50
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/291f3d50
Branch: refs/heads/master
Commit: 291f3d503d5a8627f86ef5f7fdd7880d8da4760c
Parents: cf72a73
Author: Sergio Pena <se...@cloudera.com>
Authored: Wed Sep 28 21:33:00 2016 -0500
Committer: Sergio Pena <se...@cloudera.com>
Committed: Wed Sep 28 21:33:00 2016 -0500
----------------------------------------------------------------------
.../ptest2/conf/cloudhost.properties.example | 37 +++
testutils/ptest2/pom.xml | 5 +
.../execution/context/CloudComputeService.java | 224 +++++++++++++++----
.../context/CloudExecutionContextProvider.java | 105 +++++++--
4 files changed, 311 insertions(+), 60 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/291f3d50/testutils/ptest2/conf/cloudhost.properties.example
----------------------------------------------------------------------
diff --git a/testutils/ptest2/conf/cloudhost.properties.example b/testutils/ptest2/conf/cloudhost.properties.example
new file mode 100644
index 0000000..c336052
--- /dev/null
+++ b/testutils/ptest2/conf/cloudhost.properties.example
@@ -0,0 +1,37 @@
+#
+# This is just an example of different cloudhost providers
+#
+
+# This context provides configurations for AWS EC2 and GCE (google compute engine)
+executionContextProvider = org.apache.hive.ptest.execution.context.CloudExecutionContextProvider$Builder
+
+# Option: GCE
+cloudProvider = google-compute-engine
+gceJsonFile = # GCE JSON KEY FILE
+instanceType = https://www.googleapis.com/compute/v1/projects/<PROJECT_ID>/zones/us-central1-a/machineTypes/n1-standard-8
+imageId = https://www.googleapis.com/compute/v1/projects/<PROJECT_ID>/global/images/hive-ptest-debian-8-20160927
+# keyPair = # UNUSED
+securityGroup = hive-ptest
+
+# Option: AWS
+cloudProvider = aws-ec2
+apiKey = # AWS ACCESS KEY
+accessKey = # AWS SECRET ACCESS KEY
+instanceType = c3.2xlarge
+imageId = us-west-1/ami-1fa1445b
+keyPair = hive-ptest
+securityGroup = hive-ptest
+
+# Generic options
+workingDirectory = /data/hive-ptest
+profileDirectory = /usr/local/hiveptest/etc/public/
+privateKey = /home/hiveptest/.ssh/hive-ptest-user-key
+dataDir = /data/hive-ptest/data/
+numHosts = 12
+groupName = hive-ptest-slaves
+localDirs = /home/hiveptest/
+user = hiveptest
+numThreads = 2
+maxLogDirectoriesPerProfile = 30
+userMetadata.owner = # USER
+maxHostsPerCreateRequest = 12
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/291f3d50/testutils/ptest2/pom.xml
----------------------------------------------------------------------
diff --git a/testutils/ptest2/pom.xml b/testutils/ptest2/pom.xml
index cea29b6..97981fb 100644
--- a/testutils/ptest2/pom.xml
+++ b/testutils/ptest2/pom.xml
@@ -107,6 +107,11 @@ limitations under the License.
<version>${jclouds.version}</version>
</dependency>
<dependency>
+ <groupId>org.apache.jclouds.labs</groupId>
+ <artifactId>google-compute-engine</artifactId>
+ <version>${jclouds.version}</version>
+ </dependency>
+ <dependency>
<groupId>org.apache.jclouds.driver</groupId>
<artifactId>jclouds-sshj</artifactId>
<version>${jclouds.version}</version>
http://git-wip-us.apache.org/repos/asf/hive/blob/291f3d50/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/CloudComputeService.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/CloudComputeService.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/CloudComputeService.java
index 64ee68e..e26c5ca 100644
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/CloudComputeService.java
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/CloudComputeService.java
@@ -18,11 +18,13 @@
*/
package org.apache.hive.ptest.execution.context;
-import java.util.Collections;
-import java.util.Properties;
-import java.util.Map;
-import java.util.Set;
+import java.io.File;
+import java.io.IOException;
+import java.util.*;
+import com.google.common.base.Supplier;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.io.Files;
import org.jclouds.Constants;
import org.jclouds.ContextBuilder;
import org.jclouds.aws.ec2.compute.AWSEC2TemplateOptions;
@@ -34,7 +36,12 @@ import org.jclouds.compute.domain.ComputeMetadata;
import org.jclouds.compute.domain.NodeMetadata;
import org.jclouds.compute.domain.NodeMetadata.Status;
import org.jclouds.compute.domain.Template;
+import org.jclouds.compute.options.TemplateOptions;
+import org.jclouds.domain.Credentials;
+import org.jclouds.googlecloud.GoogleCredentialsFromJson;
+import org.jclouds.googlecomputeengine.compute.options.GoogleComputeEngineTemplateOptions;
import org.jclouds.logging.log4j.config.Log4JLoggingModule;
+import org.jclouds.sshj.config.SshjSshClientModule;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -43,55 +50,76 @@ import com.google.common.base.Strings;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Sets;
+import static com.google.common.base.Charsets.UTF_8;
+
public class CloudComputeService {
private static final Logger LOG = LoggerFactory
.getLogger(CloudComputeService.class);
+ private final CloudComputeConfig mConfig;
private final ComputeServiceContext mComputeServiceContext;
private final ComputeService mComputeService;
- private final String mInstanceType;
- private final String mGroupName;
+ private final Template mTemplate;
private final String mGroupTag;
- private final String mImageId;
- private final String mkeyPair;
- private final String mSecurityGroup;
- private final Map<String, String> mUserMetadata;
- /**
- * JClouds requests on-demand instances when null
- */
- private final Float mMaxBid;
- public CloudComputeService(String apiKey, String accessKey, String instanceType, String groupName,
- String imageId, String keyPair, String securityGroup, Float maxBid, Map<String,String> userMetadata) {
- mInstanceType = instanceType;
- mGroupName = groupName;
- mImageId = imageId;
- mkeyPair = keyPair;
- mSecurityGroup = securityGroup;
- mMaxBid = maxBid;
- mGroupTag = "group=" + mGroupName;
- mUserMetadata = userMetadata;
+
+ public CloudComputeService(final CloudComputeConfig config) {
+ mConfig = config;
+
+ mComputeServiceContext = initComputeServiceContext(config.getmProvider(), config.getIdentity(), config.getCredential());
+ mComputeService = mComputeServiceContext.getComputeService();
+ mTemplate = mComputeService.templateBuilder().hardwareId(config.getInstanceType()).imageId(config.getImageId()).build();
+
+ TemplateOptions options = mTemplate.getOptions();
+
+ // Set generic options
+ options.blockOnPort(22, 60);
+ options.userMetadata(config.getUserMetaData());
+
+ // Set provider options
+ switch (config.getmProvider()) {
+ case AWS:
+ mGroupTag = String.format("group=%s", config.getGroupName());
+
+ options.as(AWSEC2TemplateOptions.class)
+ .keyPair(config.getKeyPairName())
+ .securityGroupIds(config.getSecurityGroup())
+ .spotPrice(config.getMaxBid())
+ .tags(Collections.singletonList(mGroupTag));
+ break;
+ case GCE:
+ mGroupTag = config.getGroupName();
+
+ options.as(GoogleComputeEngineTemplateOptions.class)
+ .tags(Arrays.asList(config.getGroupName(), config.getSecurityGroup())); // GCE firewall is set through instance tags
+ break;
+ default:
+ mGroupTag = "";
+ }
+ }
+
+ private ComputeServiceContext initComputeServiceContext(CloudComputeConfig.CloudComputeProvider provider, String identity, String credential) {
Properties overrides = new Properties();
+
overrides.put(ComputeServiceProperties.POLL_INITIAL_PERIOD, String.valueOf(60L * 1000L));
overrides.put(ComputeServiceProperties.POLL_MAX_PERIOD, String.valueOf(600L * 1000L));
overrides.put(Constants.PROPERTY_MAX_RETRIES, String.valueOf(60));
- mComputeServiceContext = ContextBuilder.newBuilder("aws-ec2")
- .credentials(apiKey, accessKey)
- .modules(ImmutableSet.of(new Log4JLoggingModule()))
- .overrides(overrides)
- .buildView(ComputeServiceContext.class);
- mComputeService = mComputeServiceContext.getComputeService();
+
+ return ContextBuilder.newBuilder(provider.getmJcloudsId())
+ .credentials(identity, credential)
+ .modules(ImmutableSet.of(
+ new SshjSshClientModule(),
+ new Log4JLoggingModule()
+ ))
+ .overrides(overrides)
+ .buildView(ComputeServiceContext.class);
}
+
public Set<NodeMetadata> createNodes(int count)
throws RunNodesException {
Set<NodeMetadata> result = Sets.newHashSet();
- Template template = mComputeService.templateBuilder()
- .hardwareId(mInstanceType).imageId(mImageId).build();
- template.getOptions().as(AWSEC2TemplateOptions.class).keyPair(mkeyPair)
- .securityGroupIds(mSecurityGroup).blockOnPort(22, 60)
- .spotPrice(mMaxBid).tags(Collections.singletonList(mGroupTag))
- .userMetadata(mUserMetadata);
- result.addAll(mComputeService.createNodesInGroup(mGroupName, count, template));
+ result.addAll(mComputeService.createNodesInGroup(mConfig.getGroupName(), count, mTemplate));
return result;
}
+
static Predicate<ComputeMetadata> createFilterPTestPredicate(final String groupName,
final String groupTag) {
return new Predicate<ComputeMetadata>() {
@@ -122,7 +150,7 @@ public class CloudComputeService {
public Set<NodeMetadata> listRunningNodes(){
Set<NodeMetadata> result = Sets.newHashSet();
result.addAll(mComputeService.listNodesDetailsMatching(
- createFilterPTestPredicate(mGroupName, mGroupTag)));
+ createFilterPTestPredicate(mConfig.getGroupName(), mGroupTag)));
return result;
}
public void destroyNode(String nodeId) {
@@ -131,4 +159,124 @@ public class CloudComputeService {
public void close() {
mComputeServiceContext.close();
}
+
+ public static class CloudComputeConfig {
+ public enum CloudComputeProvider {
+ AWS("aws-ec2"),
+ GCE("google-compute-engine");
+
+ private final String mJcloudsId;
+
+ CloudComputeProvider(String jcloudsId) {
+ mJcloudsId = jcloudsId;
+ }
+
+ public String getmJcloudsId() {
+ return mJcloudsId;
+ }
+ };
+
+ private final CloudComputeProvider mProvider;
+
+ private String mIdentity;
+ private String mCredential;
+ private String mInstanceType;
+ private String mImageId;
+ private String mGroupName;
+ private String mSecurityGroup;
+ private String mKeyPairName;
+ private Map<String, String> mUserMetaData;
+
+ /**
+ * JClouds requests on-demand instances when null
+ */
+ private Float mMaxBid;
+
+ public CloudComputeConfig(CloudComputeProvider provider) {
+ mProvider = provider;
+ }
+
+ public void setCredentials(String identity, String credential) {
+ mIdentity = identity;
+ mCredential = credential;
+ }
+
+ public void setInstanceType(String instanceType) {
+ mInstanceType = instanceType;
+ }
+
+ public void setImageId(String imageId) {
+ mImageId = imageId;
+ }
+
+ public void setGroupName(String groupName) {
+ mGroupName = groupName;
+ }
+
+ public void setSecurityGroup(String securityGroup) {
+ mSecurityGroup = securityGroup;
+ }
+
+ public void setMaxBid(Float maxBid) {
+ mMaxBid = maxBid;
+ }
+
+ public void setmKeyPairName(String keyPairName) {
+ mKeyPairName = keyPairName;
+ }
+
+ public void setUserMetaData(Map<String, String> userMetaData) {
+ mUserMetaData = userMetaData;
+ }
+
+ public CloudComputeProvider getmProvider() {
+ return mProvider;
+ }
+
+ public String getIdentity() {
+ return mIdentity;
+ }
+
+ public String getCredential() {
+ return mCredential;
+ }
+
+ public String getInstanceType() {
+ return mInstanceType;
+ }
+
+ public String getImageId() {
+ return mImageId;
+ }
+
+ public String getGroupName() {
+ return mGroupName;
+ }
+
+ public String getSecurityGroup() {
+ return mSecurityGroup;
+ }
+
+ public Float getMaxBid() {
+ return mMaxBid;
+ }
+
+ public String getKeyPairName() {
+ return mKeyPairName;
+ }
+
+ public Map<String, String> getUserMetaData() {
+ if (mUserMetaData == null) {
+ return ImmutableMap.of();
+ }
+
+ return mUserMetaData;
+ }
+ }
+
+ public static Credentials getCredentialsFromJsonKeyFile(String filename) throws IOException {
+ String fileContents = Files.toString(new File(filename), UTF_8);
+ Supplier<Credentials> credentialSupplier = new GoogleCredentialsFromJson(fileContents);
+ return credentialSupplier.get();
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/291f3d50/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/CloudExecutionContextProvider.java
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/CloudExecutionContextProvider.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/CloudExecutionContextProvider.java
index 343efde..8b82497 100644
--- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/CloudExecutionContextProvider.java
+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/context/CloudExecutionContextProvider.java
@@ -34,12 +34,14 @@ import java.util.concurrent.TimeUnit;
import org.apache.hive.ptest.execution.Constants;
import org.apache.hive.ptest.execution.Dirs;
+import org.apache.hive.ptest.execution.LocalCommandFactory;
import org.apache.hive.ptest.execution.conf.Context;
import org.apache.hive.ptest.execution.conf.Host;
import org.apache.hive.ptest.execution.ssh.SSHCommand;
import org.apache.hive.ptest.execution.ssh.SSHCommandExecutor;
import org.jclouds.compute.RunNodesException;
import org.jclouds.compute.domain.NodeMetadata;
+import org.jclouds.domain.Credentials;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -55,16 +57,24 @@ import com.google.common.collect.Sets;
public class CloudExecutionContextProvider implements ExecutionContextProvider {
private static final Logger LOG = LoggerFactory
.getLogger(CloudExecutionContextProvider.class);
- public static final String DATA_DIR = "dataDir";
+ public static final String CLOUD_PROVIDER = "cloudProvider";
+
+ // GCE settings
+ public static final String GCE_JSON_CREDS_FILE = "gceJsonFile";
+
+ // AWS settings
public static final String API_KEY = "apiKey";
public static final String ACCESS_KEY = "accessKey";
+ public static final String KEY_PAIR = "keyPair";
+ public static final String MAX_BID = "maxBid";
+
+ // Generic settings
+ public static final String DATA_DIR = "dataDir";
public static final String NUM_HOSTS = "numHosts";
public static final String MAX_HOSTS_PER_CREATE_REQUEST = "maxHostsPerCreateRequest";
public static final String GROUP_NAME = "groupName";
public static final String IMAGE_ID = "imageId";
- public static final String KEY_PAIR = "keyPair";
public static final String SECURITY_GROUP = "securityGroup";
- public static final String MAX_BID = "maxBid";
public static final String SLAVE_LOCAL_DIRECTORIES = "localDirs";
public static final String USERNAME = "user";
public static final String INSTANCE_TYPE = "instanceType";
@@ -304,6 +314,7 @@ public class CloudExecutionContextProvider implements ExecutionContextProvider {
LOG.error("Verify command still executing on a host after 10 minutes");
}
} catch (InterruptedException e) {
+ terminateInternal(result);
throw new CreateHostsFailedException("Interrupted while trying to create hosts", e);
} finally {
if(!executorService.isShutdown()) {
@@ -418,29 +429,79 @@ public class CloudExecutionContextProvider implements ExecutionContextProvider {
return create(context, workingDirectory);
}
}
+
+ private static CloudComputeService createAwsService(final Context context) {
+ String apiKey = Preconditions.checkNotNull(context.getString(API_KEY), API_KEY + " is required");
+ String accessKey = Preconditions.checkNotNull(context.getString(ACCESS_KEY), ACCESS_KEY + " is required");
+ String imageId = Preconditions.checkNotNull(context.getString(IMAGE_ID), IMAGE_ID + " is required");
+ String keyPair = Preconditions.checkNotNull(context.getString(KEY_PAIR), KEY_PAIR + " is required");
+ String securityGroup = Preconditions.checkNotNull(context.getString(SECURITY_GROUP), SECURITY_GROUP + " is required");
+
+ Float maxBid = context.getFloat(MAX_BID);
+ Preconditions.checkArgument(maxBid == null || maxBid > 0, MAX_BID + " must be null or greater than zero");
+
+ String instanceType = context.getString(INSTANCE_TYPE, "c1.xlarge");
+ String groupName = context.getString(GROUP_NAME, "hive-ptest-slaves");
+
+ CloudComputeService.CloudComputeConfig config =
+ new CloudComputeService.CloudComputeConfig(CloudComputeService.CloudComputeConfig.CloudComputeProvider.AWS);
+
+ config.setCredentials(apiKey, accessKey);
+ config.setInstanceType(instanceType);
+ config.setGroupName(groupName);
+ config.setImageId(imageId);
+ config.setmKeyPairName(keyPair);
+ config.setSecurityGroup(securityGroup);
+ config.setMaxBid(maxBid);
+ config.setUserMetaData(context.getSubProperties(USER_METADATA + "."));
+
+ return new CloudComputeService(config);
+ }
+
+ private static CloudComputeService createGceService(final Context context) throws IOException {
+ String gceJsonFile = Preconditions.checkNotNull(context.getString(GCE_JSON_CREDS_FILE), GCE_JSON_CREDS_FILE + " is required");
+ String imageId = Preconditions.checkNotNull(context.getString(IMAGE_ID), IMAGE_ID + " is required");
+ String securityGroup = Preconditions.checkNotNull(context.getString(SECURITY_GROUP), SECURITY_GROUP + " is required");
+ String instanceType = Preconditions.checkNotNull(context.getString(INSTANCE_TYPE, ""), INSTANCE_TYPE + " is required");
+
+ String groupName = context.getString(GROUP_NAME, "hive-ptest-slaves");
+
+ CloudComputeService.CloudComputeConfig config =
+ new CloudComputeService.CloudComputeConfig(CloudComputeService.CloudComputeConfig.CloudComputeProvider.GCE);
+
+ Credentials creds =
+ CloudComputeService.getCredentialsFromJsonKeyFile(gceJsonFile);
+
+ config.setCredentials(creds.identity, creds.credential);
+ config.setInstanceType(instanceType);
+ config.setGroupName(groupName);
+ config.setImageId(imageId);
+ config.setSecurityGroup(securityGroup);
+ config.setUserMetaData(context.getSubProperties(USER_METADATA + "."));
+
+ return new CloudComputeService(config);
+ }
+
+ private static CloudComputeService createService(final Context context) throws IOException {
+ String cloudProvider = context.getString(CLOUD_PROVIDER, "aws-ec2");
+
+ if (cloudProvider.equalsIgnoreCase("aws-ec2")) {
+ return createAwsService(context);
+ } else if (cloudProvider.equalsIgnoreCase("google-compute-engine")) {
+ return createGceService(context);
+ } else {
+ throw new IllegalArgumentException("Unknown cloud provider name: " + cloudProvider);
+ }
+ }
private static CloudExecutionContextProvider create(Context context,
String workingDirectory) throws IOException {
String dataDir = Preconditions.checkNotNull(context.getString(DATA_DIR),
DATA_DIR + " is required");
- String apiKey = Preconditions.checkNotNull(context.getString(API_KEY),
- API_KEY + " is required");
- String accessKey = Preconditions.checkNotNull(
- context.getString(ACCESS_KEY), ACCESS_KEY + " is required");
int maxHostsPerCreateRequest = context.getInteger(MAX_HOSTS_PER_CREATE_REQUEST, 2);
Integer numHosts = context.getInteger(NUM_HOSTS, 8);
Preconditions.checkArgument(numHosts > 0, NUM_HOSTS
+ " must be greater than zero");
- String groupName = context.getString(GROUP_NAME, "hive-ptest-slaves");
- String imageId = Preconditions.checkNotNull(context.getString(IMAGE_ID),
- IMAGE_ID + " is required");
- String keyPair = Preconditions.checkNotNull(context.getString(KEY_PAIR),
- KEY_PAIR + " is required");
- String securityGroup = Preconditions.checkNotNull(
- context.getString(SECURITY_GROUP), SECURITY_GROUP + " is required");
- Float maxBid = context.getFloat(MAX_BID);
- Preconditions.checkArgument(maxBid == null || maxBid > 0, MAX_BID
- + " must be null or greater than zero");
String privateKey = Preconditions.checkNotNull(
context.getString(PRIVATE_KEY), PRIVATE_KEY + " is required");
String user = context.getString(USERNAME, "hiveptest");
@@ -448,12 +509,12 @@ public class CloudExecutionContextProvider implements ExecutionContextProvider {
.split(context.getString(SLAVE_LOCAL_DIRECTORIES, "/home/hiveptest/")),
String.class);
Integer numThreads = context.getInteger(NUM_THREADS, 3);
- String instanceType = context.getString(INSTANCE_TYPE, "c1.xlarge");
- CloudComputeService cloudComputeService = new CloudComputeService(apiKey, accessKey,
- instanceType, groupName, imageId, keyPair, securityGroup, maxBid, context.getSubProperties(USER_METADATA + "."));
+
+ CloudComputeService cloudComputeService = createService(context);
CloudExecutionContextProvider service = new CloudExecutionContextProvider(
- dataDir, numHosts, cloudComputeService, new SSHCommandExecutor(LOG), workingDirectory,
- privateKey, user, localDirs, numThreads, 60, maxHostsPerCreateRequest);
+ dataDir, numHosts, cloudComputeService,
+ new SSHCommandExecutor(LOG, new LocalCommandFactory(LOG), "-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no"),
+ workingDirectory, privateKey, user, localDirs, numThreads, 60, maxHostsPerCreateRequest);
return service;
}
}