You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@brooklyn.apache.org by an...@apache.org on 2017/06/09 10:54:16 UTC
[3/4] brooklyn-server git commit: extract JCloudsPropertiesBuilder
into own class
extract JCloudsPropertiesBuilder into own class
Project: http://git-wip-us.apache.org/repos/asf/brooklyn-server/repo
Commit: http://git-wip-us.apache.org/repos/asf/brooklyn-server/commit/14e844d8
Tree: http://git-wip-us.apache.org/repos/asf/brooklyn-server/tree/14e844d8
Diff: http://git-wip-us.apache.org/repos/asf/brooklyn-server/diff/14e844d8
Branch: refs/heads/master
Commit: 14e844d8e24b1092cd0ddf940711e50cd4ff87ca
Parents: a4d8a95
Author: Robert Moss <ro...@cloudsoftcorp.com>
Authored: Thu Jun 8 17:59:08 2017 +0100
Committer: Robert Moss <ro...@cloudsoftcorp.com>
Committed: Thu Jun 8 17:59:08 2017 +0100
----------------------------------------------------------------------
.../jclouds/AbstractComputeServiceRegistry.java | 138 +--------------
...wsEc2SessionAwareComputeServiceRegistry.java | 18 +-
.../jclouds/JCloudsPropertiesBuilder.java | 166 +++++++++++++++++++
3 files changed, 186 insertions(+), 136 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/14e844d8/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/AbstractComputeServiceRegistry.java
----------------------------------------------------------------------
diff --git a/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/AbstractComputeServiceRegistry.java b/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/AbstractComputeServiceRegistry.java
index 8bc3388..04951e0 100644
--- a/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/AbstractComputeServiceRegistry.java
+++ b/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/AbstractComputeServiceRegistry.java
@@ -19,51 +19,37 @@
package org.apache.brooklyn.location.jclouds;
import static com.google.common.base.Preconditions.checkNotNull;
-import static org.apache.brooklyn.util.JavaGroovyEquivalents.groovyTruth;
-import static org.jclouds.aws.ec2.reference.AWSEC2Constants.PROPERTY_EC2_AMI_QUERY;
-import static org.jclouds.aws.ec2.reference.AWSEC2Constants.PROPERTY_EC2_CC_AMI_QUERY;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.ConcurrentHashMap;
-import org.apache.brooklyn.core.config.Sanitizer;
import org.apache.brooklyn.core.location.cloud.CloudLocationConfig;
import org.apache.brooklyn.core.mgmt.persist.DeserializingJcloudsRenamesProvider;
import org.apache.brooklyn.util.collections.MutableMap;
import org.apache.brooklyn.util.core.config.ConfigBag;
-import org.apache.brooklyn.util.text.Strings;
-import org.apache.brooklyn.util.time.Duration;
import org.jclouds.Constants;
import org.jclouds.ContextBuilder;
import org.jclouds.azurecompute.arm.config.AzureComputeRateLimitModule;
import org.jclouds.compute.ComputeService;
import org.jclouds.compute.ComputeServiceContext;
import org.jclouds.domain.Credentials;
-import org.jclouds.ec2.reference.EC2Constants;
import org.jclouds.encryption.bouncycastle.config.BouncyCastleCryptoModule;
-import org.jclouds.location.reference.LocationConstants;
import org.jclouds.logging.slf4j.config.SLF4JLoggingModule;
import org.jclouds.sshj.config.SshjSshClientModule;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import com.google.common.base.Predicates;
import com.google.common.base.Supplier;
import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Maps;
import com.google.inject.Module;
public abstract class AbstractComputeServiceRegistry implements ComputeServiceRegistry, JcloudsLocationConfig {
- private static final Logger LOG = LoggerFactory.getLogger(AbstractComputeServiceRegistry.class);
-
private final Map<Map<?, ?>, ComputeService> cachedComputeServices = new ConcurrentHashMap<>();
@Override
public ComputeService findComputeService(ConfigBag conf, boolean allowReuse) {
- PropertiesBuilder propertiesBuilder = new PropertiesBuilder(conf)
- .setCommonProperties();
+ JCloudsPropertiesBuilder propertiesBuilder = new JCloudsPropertiesBuilder(conf)
+ .setCommonJcloudsProperties();
Iterable<Module> modules = getCommonModules();
@@ -84,7 +70,7 @@ public abstract class AbstractComputeServiceRegistry implements ComputeServiceRe
}
Properties properties = propertiesBuilder
- .setJCloudsProperties()
+ .setCustomJcloudsProperties()
.setEndpointProperty()
.build();
@@ -110,115 +96,6 @@ public abstract class AbstractComputeServiceRegistry implements ComputeServiceRe
.asUnmodifiable();
}
- public class PropertiesBuilder {
- private ConfigBag conf;
- private Properties properties = new Properties();
-
- public PropertiesBuilder(ConfigBag conf) {
- this.conf = conf;
- }
-
- public PropertiesBuilder setCommonProperties() {
- properties.setProperty(Constants.PROPERTY_TRUST_ALL_CERTS, Boolean.toString(true));
- properties.setProperty(Constants.PROPERTY_RELAX_HOSTNAME, Boolean.toString(true));
- properties.setProperty("jclouds.ssh.max-retries", conf.getStringKey("jclouds.ssh.max-retries") != null ?
- conf.getStringKey("jclouds.ssh.max-retries").toString() : "50");
-
- if (conf.get(OAUTH_ENDPOINT) != null)
- properties.setProperty(OAUTH_ENDPOINT.getName(), conf.get(OAUTH_ENDPOINT));
-
- // See https://issues.apache.org/jira/browse/BROOKLYN-394
- // For retries, the backoff times are:
- // Math.min(2^failureCount * retryDelayStart, retryDelayStart * 10) + random(10%)
- // Therefore the backoff times will be: 500ms, 1s, 2s, 4s, 5s, 5s.
- // The defaults (if not overridden here) are 50ms and 5 retires. This gives backoff
- // times of 50ms, 100ms, 200ms, 400ms, 500ms (so a total backoff time of 1.25s),
- // which is not long when you're being rate-limited and there are multiple thread all
- // retrying their API calls.
- properties.setProperty(Constants.PROPERTY_RETRY_DELAY_START, "500");
- properties.setProperty(Constants.PROPERTY_MAX_RETRIES, "6");
- return this;
- }
-
- public PropertiesBuilder setAWSEC2Properties() {
- // TODO convert AWS-only flags to config keys
- if (groovyTruth(conf.get(IMAGE_ID))) {
- properties.setProperty(PROPERTY_EC2_AMI_QUERY, "");
- properties.setProperty(PROPERTY_EC2_CC_AMI_QUERY, "");
- } else if (groovyTruth(conf.getStringKey("imageOwner"))) {
- properties.setProperty(PROPERTY_EC2_AMI_QUERY, "owner-id=" + conf.getStringKey("imageOwner") + ";state=available;image-type=machine");
- } else if (groovyTruth(conf.getStringKey("anyOwner"))) {
- // set `anyOwner: true` to override the default query (which is restricted to certain owners as per below),
- // allowing the AMI query to bind to any machine
- // (note however, we sometimes pick defaults in JcloudsLocationFactory);
- // (and be careful, this can give a LOT of data back, taking several minutes,
- // and requiring extra memory allocated on the command-line)
- properties.setProperty(PROPERTY_EC2_AMI_QUERY, "state=available;image-type=machine");
- /*
- * by default the following filters are applied:
- * Filter.1.Name=owner-id&Filter.1.Value.1=137112412989&
- * Filter.1.Value.2=063491364108&
- * Filter.1.Value.3=099720109477&
- * Filter.1.Value.4=411009282317&
- * Filter.2.Name=state&Filter.2.Value.1=available&
- * Filter.3.Name=image-type&Filter.3.Value.1=machine&
- */
- }
-
- // See https://issues.apache.org/jira/browse/BROOKLYN-399
- String region = conf.get(CLOUD_REGION_ID);
- if (Strings.isNonBlank(region)) {
- /*
- * Drop availability zone suffixes. Without this deployments to regions like us-east-1b fail
- * because jclouds throws an IllegalStateException complaining that: location id us-east-1b
- * not found in: [{scope=PROVIDER, id=aws-ec2, description=https://ec2.us-east-1.amazonaws.com,
- * iso3166Codes=[US-VA, US-CA, US-OR, BR-SP, IE, DE-HE, SG, AU-NSW, JP-13]}]. The exception is
- * thrown by org.jclouds.compute.domain.internal.TemplateBuilderImpl#locationId(String).
- */
- if (Character.isLetter(region.charAt(region.length() - 1))) {
- region = region.substring(0, region.length() - 1);
- }
- properties.setProperty(LocationConstants.PROPERTY_REGIONS, region);
- }
-
- // occasionally can get com.google.common.util.concurrent.UncheckedExecutionException: java.lang.RuntimeException:
- // security group eu-central-1/jclouds#brooklyn-bxza-alex-eu-central-shoul-u2jy-nginx-ielm is not available after creating
- // the default timeout was 500ms so let's raise it in case that helps
- properties.setProperty(EC2Constants.PROPERTY_EC2_TIMEOUT_SECURITYGROUP_PRESENT, "" + Duration.seconds(30).toMilliseconds());
- return this;
- }
-
- private PropertiesBuilder setAzureComputeArmProperties() {
- String region = conf.get(CLOUD_REGION_ID);
- if (Strings.isNonBlank(region)) {
- properties.setProperty(LocationConstants.PROPERTY_REGIONS, region);
- }
- return this;
- }
-
- private PropertiesBuilder setJCloudsProperties() {
- // Add extra jclouds-specific configuration
- Map<String, Object> extra = Maps.filterKeys(conf.getAllConfig(), Predicates.containsPattern("^jclouds\\."));
- if (extra.size() > 0) {
- String provider = getProviderFromConfig(conf);
- LOG.debug("Configuring custom jclouds property overrides for {}: {}", provider, Sanitizer.sanitize(extra));
- }
- properties.putAll(Maps.filterValues(extra, Predicates.notNull()));
- return this;
- }
-
- private PropertiesBuilder setEndpointProperty() {
- String endpoint = conf.get(CloudLocationConfig.CLOUD_ENDPOINT);
- if (!groovyTruth(endpoint)) endpoint = getDeprecatedProperty(conf, Constants.PROPERTY_ENDPOINT);
- if (groovyTruth(endpoint)) properties.setProperty(Constants.PROPERTY_ENDPOINT, endpoint);
- return this;
- }
-
- public Properties build() {
- return properties;
- }
- }
-
public class ComputeServiceSupplier implements Supplier<ComputeService> {
private final String provider;
@@ -266,15 +143,6 @@ public abstract class AbstractComputeServiceRegistry implements ComputeServiceRe
new BouncyCastleCryptoModule());
}
- protected String getDeprecatedProperty(ConfigBag conf, String key) {
- if (conf.containsKey(key)) {
- LOG.warn("Jclouds using deprecated brooklyn-jclouds property " + key + ": " + Sanitizer.sanitize(conf.getAllConfig()));
- return (String) conf.getStringKey(key);
- } else {
- return null;
- }
- }
-
@Override
public String toString() {
return getClass().getName();
http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/14e844d8/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/AwsEc2SessionAwareComputeServiceRegistry.java
----------------------------------------------------------------------
diff --git a/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/AwsEc2SessionAwareComputeServiceRegistry.java b/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/AwsEc2SessionAwareComputeServiceRegistry.java
index 1b3c8fe..436b823 100644
--- a/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/AwsEc2SessionAwareComputeServiceRegistry.java
+++ b/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/AwsEc2SessionAwareComputeServiceRegistry.java
@@ -36,6 +36,20 @@ import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Supplier;
+/**
+ * A ComputeServiceRegistry that can infer credentials from the AWS EC2 instance profile.
+ *
+ * This only works if Brooklyn is running on an EC2 instance that has an IAM Role attached with e.g the AmazonEC2FullAccess
+ * policy set.
+ *
+ * usage:
+ *
+ * {@code
+ * jclouds.computeServiceRegistry:
+ * $brooklyn:object:
+ * type: org.apache.brooklyn.location.jclouds.AwsEc2SessionAwareComputeServiceRegistry
+ * }
+ */
public class AwsEc2SessionAwareComputeServiceRegistry extends AbstractComputeServiceRegistry implements ComputeServiceRegistry, AwsEc2SessionAwareLocationConfig {
public static final String ACCESS_KEY_ID = "AccessKeyId";
@@ -45,7 +59,9 @@ public class AwsEc2SessionAwareComputeServiceRegistry extends AbstractComputeSer
public static final String AWS_SECURITY_CREDENTIAL_URL = "http://169.254.169.254/latest/meta-data/iam/security-credentials";
public static final String AWS_EXPIRATION_DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ss'Z'";
- public AwsEc2SessionAwareComputeServiceRegistry(){}
+ public AwsEc2SessionAwareComputeServiceRegistry(){
+ // empty constructor required
+ }
@Override
public ComputeService findComputeService(ConfigBag conf, boolean allowReuse) {
http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/14e844d8/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/JCloudsPropertiesBuilder.java
----------------------------------------------------------------------
diff --git a/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/JCloudsPropertiesBuilder.java b/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/JCloudsPropertiesBuilder.java
new file mode 100644
index 0000000..8178134
--- /dev/null
+++ b/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/JCloudsPropertiesBuilder.java
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.brooklyn.location.jclouds;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static org.apache.brooklyn.util.JavaGroovyEquivalents.groovyTruth;
+import static org.jclouds.aws.ec2.reference.AWSEC2Constants.PROPERTY_EC2_AMI_QUERY;
+import static org.jclouds.aws.ec2.reference.AWSEC2Constants.PROPERTY_EC2_CC_AMI_QUERY;
+
+import java.util.Map;
+import java.util.Properties;
+
+import org.apache.brooklyn.core.config.Sanitizer;
+import org.apache.brooklyn.core.location.cloud.CloudLocationConfig;
+import org.apache.brooklyn.core.mgmt.persist.DeserializingJcloudsRenamesProvider;
+import org.apache.brooklyn.util.core.config.ConfigBag;
+import org.apache.brooklyn.util.text.Strings;
+import org.apache.brooklyn.util.time.Duration;
+import org.jclouds.Constants;
+import org.jclouds.ec2.reference.EC2Constants;
+import org.jclouds.location.reference.LocationConstants;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Predicates;
+import com.google.common.collect.Maps;
+
+public class JCloudsPropertiesBuilder implements JcloudsLocationConfig{
+ private ConfigBag conf;
+ private Properties properties = new Properties();
+
+ private static final Logger LOG = LoggerFactory.getLogger(JCloudsPropertiesBuilder.class);
+
+ public JCloudsPropertiesBuilder(ConfigBag conf) {
+ this.conf = conf;
+ }
+
+ public JCloudsPropertiesBuilder setCommonJcloudsProperties() {
+ properties.setProperty(Constants.PROPERTY_TRUST_ALL_CERTS, Boolean.toString(true));
+ properties.setProperty(Constants.PROPERTY_RELAX_HOSTNAME, Boolean.toString(true));
+ properties.setProperty("jclouds.ssh.max-retries", conf.getStringKey("jclouds.ssh.max-retries") != null ?
+ conf.getStringKey("jclouds.ssh.max-retries").toString() : "50");
+
+ if (conf.get(OAUTH_ENDPOINT) != null)
+ properties.setProperty(OAUTH_ENDPOINT.getName(), conf.get(OAUTH_ENDPOINT));
+
+ // See https://issues.apache.org/jira/browse/BROOKLYN-394
+ // For retries, the backoff times are:
+ // Math.min(2^failureCount * retryDelayStart, retryDelayStart * 10) + random(10%)
+ // Therefore the backoff times will be: 500ms, 1s, 2s, 4s, 5s, 5s.
+ // The defaults (if not overridden here) are 50ms and 5 retires. This gives backoff
+ // times of 50ms, 100ms, 200ms, 400ms, 500ms (so a total backoff time of 1.25s),
+ // which is not long when you're being rate-limited and there are multiple thread all
+ // retrying their API calls.
+ properties.setProperty(Constants.PROPERTY_RETRY_DELAY_START, "500");
+ properties.setProperty(Constants.PROPERTY_MAX_RETRIES, "6");
+ return this;
+ }
+
+ public JCloudsPropertiesBuilder setAWSEC2Properties() {
+ // TODO convert AWS-only flags to config keys
+ if (groovyTruth(conf.get(IMAGE_ID))) {
+ properties.setProperty(PROPERTY_EC2_AMI_QUERY, "");
+ properties.setProperty(PROPERTY_EC2_CC_AMI_QUERY, "");
+ } else if (groovyTruth(conf.getStringKey("imageOwner"))) {
+ properties.setProperty(PROPERTY_EC2_AMI_QUERY, "owner-id=" + conf.getStringKey("imageOwner") + ";state=available;image-type=machine");
+ } else if (groovyTruth(conf.getStringKey("anyOwner"))) {
+ // set `anyOwner: true` to override the default query (which is restricted to certain owners as per below),
+ // allowing the AMI query to bind to any machine
+ // (note however, we sometimes pick defaults in JcloudsLocationFactory);
+ // (and be careful, this can give a LOT of data back, taking several minutes,
+ // and requiring extra memory allocated on the command-line)
+ properties.setProperty(PROPERTY_EC2_AMI_QUERY, "state=available;image-type=machine");
+ /*
+ * by default the following filters are applied:
+ * Filter.1.Name=owner-id&Filter.1.Value.1=137112412989&
+ * Filter.1.Value.2=063491364108&
+ * Filter.1.Value.3=099720109477&
+ * Filter.1.Value.4=411009282317&
+ * Filter.2.Name=state&Filter.2.Value.1=available&
+ * Filter.3.Name=image-type&Filter.3.Value.1=machine&
+ */
+ }
+
+ // See https://issues.apache.org/jira/browse/BROOKLYN-399
+ String region = conf.get(CLOUD_REGION_ID);
+ if (Strings.isNonBlank(region)) {
+ /*
+ * Drop availability zone suffixes. Without this deployments to regions like us-east-1b fail
+ * because jclouds throws an IllegalStateException complaining that: location id us-east-1b
+ * not found in: [{scope=PROVIDER, id=aws-ec2, description=https://ec2.us-east-1.amazonaws.com,
+ * iso3166Codes=[US-VA, US-CA, US-OR, BR-SP, IE, DE-HE, SG, AU-NSW, JP-13]}]. The exception is
+ * thrown by org.jclouds.compute.domain.internal.TemplateBuilderImpl#locationId(String).
+ */
+ if (Character.isLetter(region.charAt(region.length() - 1))) {
+ region = region.substring(0, region.length() - 1);
+ }
+ properties.setProperty(LocationConstants.PROPERTY_REGIONS, region);
+ }
+
+ // occasionally can get com.google.common.util.concurrent.UncheckedExecutionException: java.lang.RuntimeException:
+ // security group eu-central-1/jclouds#brooklyn-bxza-alex-eu-central-shoul-u2jy-nginx-ielm is not available after creating
+ // the default timeout was 500ms so let's raise it in case that helps
+ properties.setProperty(EC2Constants.PROPERTY_EC2_TIMEOUT_SECURITYGROUP_PRESENT, "" + Duration.seconds(30).toMilliseconds());
+ return this;
+ }
+
+ public JCloudsPropertiesBuilder setAzureComputeArmProperties() {
+ String region = conf.get(CloudLocationConfig.CLOUD_REGION_ID);
+ if (Strings.isNonBlank(region)) {
+ properties.setProperty(LocationConstants.PROPERTY_REGIONS, region);
+ }
+ return this;
+ }
+
+ public JCloudsPropertiesBuilder setCustomJcloudsProperties() {
+ Map<String, Object> extra = Maps.filterKeys(conf.getAllConfig(), Predicates.containsPattern("^jclouds\\."));
+ if (extra.size() > 0) {
+ String provider = getProviderFromConfig(conf);
+ LOG.debug("Configuring custom jclouds property overrides for {}: {}", provider, Sanitizer.sanitize(extra));
+ }
+ properties.putAll(Maps.filterValues(extra, Predicates.notNull()));
+ return this;
+ }
+
+ public JCloudsPropertiesBuilder setEndpointProperty() {
+ String endpoint = conf.get(CloudLocationConfig.CLOUD_ENDPOINT);
+ if (!groovyTruth(endpoint)) endpoint = getDeprecatedProperty(conf, Constants.PROPERTY_ENDPOINT);
+ if (groovyTruth(endpoint)) properties.setProperty(Constants.PROPERTY_ENDPOINT, endpoint);
+ return this;
+ }
+
+ public Properties build() {
+ return properties;
+ }
+
+ private String getDeprecatedProperty(ConfigBag conf, String key) {
+ if (conf.containsKey(key)) {
+ LOG.warn("Jclouds using deprecated brooklyn-jclouds property " + key + ": " + Sanitizer.sanitize(conf.getAllConfig()));
+ return (String) conf.getStringKey(key);
+ } else {
+ return null;
+ }
+ }
+
+ private String getProviderFromConfig(ConfigBag conf) {
+ String rawProvider = checkNotNull(conf.get(CLOUD_PROVIDER), "provider must not be null");
+ return DeserializingJcloudsRenamesProvider.INSTANCE.applyJcloudsRenames(rawProvider);
+ }
+}