You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@brooklyn.apache.org by an...@apache.org on 2017/06/09 10:54:14 UTC
[1/4] brooklyn-server git commit: adds a new ComputeServiceRegistry
to handle when Brooklyn runs on an ec2 instance and want to pick up
credentials from an IAM role
Repository: brooklyn-server
Updated Branches:
refs/heads/master 2a94137b7 -> 78f41f8d6
adds a new ComputeServiceRegistry to handle when Brooklyn runs on an ec2 instance and want to pick up credentials from an IAM role
Project: http://git-wip-us.apache.org/repos/asf/brooklyn-server/repo
Commit: http://git-wip-us.apache.org/repos/asf/brooklyn-server/commit/e6ca01c9
Tree: http://git-wip-us.apache.org/repos/asf/brooklyn-server/tree/e6ca01c9
Diff: http://git-wip-us.apache.org/repos/asf/brooklyn-server/diff/e6ca01c9
Branch: refs/heads/master
Commit: e6ca01c9ae56d8ca0065793f4fbd33bdf8d409ec
Parents: 30292c9
Author: Robert Moss <ro...@cloudsoftcorp.com>
Authored: Wed May 24 16:41:53 2017 +0100
Committer: Robert Moss <ro...@cloudsoftcorp.com>
Committed: Wed Jun 7 20:42:55 2017 +0100
----------------------------------------------------------------------
.../jclouds/AbstractComputeServiceRegistry.java | 312 +++++++++++++++++++
...wsEc2SessionAwareComputeServiceRegistry.java | 99 ++++++
.../AwsEc2SessionAwareLocationConfig.java | 28 ++
.../jclouds/ComputeServiceRegistryImpl.java | 202 +-----------
4 files changed, 447 insertions(+), 194 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/e6ca01c9/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/AbstractComputeServiceRegistry.java
----------------------------------------------------------------------
diff --git a/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/AbstractComputeServiceRegistry.java b/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/AbstractComputeServiceRegistry.java
new file mode 100644
index 0000000..076d442
--- /dev/null
+++ b/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/AbstractComputeServiceRegistry.java
@@ -0,0 +1,312 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.brooklyn.location.jclouds;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static org.apache.brooklyn.util.JavaGroovyEquivalents.groovyTruth;
+import static org.jclouds.aws.ec2.reference.AWSEC2Constants.PROPERTY_EC2_AMI_QUERY;
+import static org.jclouds.aws.ec2.reference.AWSEC2Constants.PROPERTY_EC2_CC_AMI_QUERY;
+
+import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.brooklyn.core.config.Sanitizer;
+import org.apache.brooklyn.core.location.cloud.CloudLocationConfig;
+import org.apache.brooklyn.core.mgmt.persist.DeserializingJcloudsRenamesProvider;
+import org.apache.brooklyn.util.collections.MutableMap;
+import org.apache.brooklyn.util.core.config.ConfigBag;
+import org.apache.brooklyn.util.text.Strings;
+import org.apache.brooklyn.util.time.Duration;
+import org.jclouds.Constants;
+import org.jclouds.ContextBuilder;
+import org.jclouds.azurecompute.arm.config.AzureComputeRateLimitModule;
+import org.jclouds.compute.ComputeService;
+import org.jclouds.compute.ComputeServiceContext;
+import org.jclouds.domain.Credentials;
+import org.jclouds.ec2.reference.EC2Constants;
+import org.jclouds.encryption.bouncycastle.config.BouncyCastleCryptoModule;
+import org.jclouds.location.reference.LocationConstants;
+import org.jclouds.logging.slf4j.config.SLF4JLoggingModule;
+import org.jclouds.sshj.config.SshjSshClientModule;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Predicates;
+import com.google.common.base.Supplier;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Maps;
+import com.google.inject.Module;
+
+public abstract class AbstractComputeServiceRegistry implements ComputeServiceRegistry, JcloudsLocationConfig {
+
+ private static final Logger LOG = LoggerFactory.getLogger(AbstractComputeServiceRegistry.class);
+
+ private final Map<Map<?, ?>, ComputeService> cachedComputeServices = new ConcurrentHashMap<>();
+
+ @Override
+ public ComputeService findComputeService(ConfigBag conf, boolean allowReuse) {
+ Properties properties = new Properties();
+ setCommonProperties(conf, properties);
+
+ Iterable<Module> modules = getCommonModules();
+
+ // Enable aws-ec2 lazy image fetching, if given a specific imageId; otherwise customize for specific owners; or all as a last resort
+ // See https://issues.apache.org/jira/browse/WHIRR-416
+ String provider = getProviderFromConfig(conf);
+ if ("aws-ec2".equals(provider)) {
+ setAWSEC2Properties(conf, properties);
+ } else if ("azurecompute-arm".equals(provider)) {
+ setAzureComputeArmProperties(conf, properties);
+ // jclouds 2.0.0 does not include the rate limit module for Azure ARM. This quick fix enables this which will
+ // avoid provisioning to fail due to rate limit exceeded
+ // See https://issues.apache.org/jira/browse/JCLOUDS-1229
+ modules = ImmutableSet.<Module>builder()
+ .addAll(modules)
+ .add(new AzureComputeRateLimitModule())
+ .build();
+ }
+
+ addJCloudsProperties(conf, properties);
+ addEndpointProperty(conf, properties);
+
+ Supplier<ComputeService> computeServiceSupplier = allowReuse
+ ? new ReusableComputeServiceSupplier(conf, modules, properties)
+ : new ComputeServiceSupplierImpl(conf, modules, properties);
+
+ return computeServiceSupplier.get();
+ }
+
+ public abstract class ComputeServiceSupplier implements Supplier<ComputeService> {
+
+ private final String provider;
+ private final ConfigBag conf;
+ private final Iterable<? extends Module> modules;
+ private final Properties properties;
+
+ private final Object createComputeServicesMutex = new Object();
+
+ public ComputeServiceSupplier(ConfigBag conf, Iterable<? extends Module> modules, Properties properties) {
+ this.provider = getProviderFromConfig(conf);
+ this.conf = conf;
+ this.modules = modules;
+ this.properties = properties;
+ }
+
+ public ComputeService get() {
+ // Synchronizing to avoid deadlock from sun.reflect.annotation.AnnotationType.
+ // See https://github.com/brooklyncentral/brooklyn/issues/974
+ synchronized (createComputeServicesMutex) {
+ ComputeServiceContext computeServiceContext = ContextBuilder.newBuilder(provider)
+ .modules(modules)
+ .credentialsSupplier(AbstractComputeServiceRegistry.this.makeCredentials(conf))
+ .overrides(properties)
+ .build(ComputeServiceContext.class);
+ return computeServiceContext.getComputeService();
+ }
+ }
+
+ protected ConfigBag getConf() {
+ return conf;
+ }
+
+ protected Properties getProperties() {
+ return properties;
+ }
+ }
+
+ public class ComputeServiceSupplierImpl extends ComputeServiceSupplier {
+
+ public ComputeServiceSupplierImpl(ConfigBag conf, Iterable<? extends Module> modules, Properties properties) {
+ super(conf, modules, properties);
+ }
+ }
+
+ public class ReusableComputeServiceSupplier extends ComputeServiceSupplier {
+
+ private Map<?, ?> cacheKey;
+
+ public ReusableComputeServiceSupplier(ConfigBag conf, Iterable<? extends Module> modules, Properties properties) {
+ super(conf, modules, properties);
+ this.cacheKey = makeCacheKey();
+ }
+
+ @Override
+ public ComputeService get() {
+ ComputeService result = cachedComputeServices.get(cacheKey);
+ if (result != null) {
+ LOG.trace("jclouds ComputeService cache hit for compute service, for " + Sanitizer.sanitize(getProperties()));
+ return result;
+ }
+ LOG.debug("jclouds ComputeService cache miss for compute service, creating, for " + Sanitizer.sanitize(getProperties()));
+ final ComputeService computeService = super.get();
+ synchronized (cachedComputeServices) {
+ result = cachedComputeServices.get(cacheKey);
+ if (result != null) {
+ LOG.debug("jclouds ComputeService cache recovery for compute service, for " + Sanitizer.sanitize(cacheKey));
+ //keep the old one, discard the new one
+ computeService.getContext().close();
+ return result;
+ }
+ LOG.debug("jclouds ComputeService created " + computeService + ", adding to cache, for " + Sanitizer.sanitize(getProperties()));
+ cachedComputeServices.put(cacheKey, computeService);
+ }
+ return result;
+ }
+
+ private Map<?, ?> makeCacheKey() {
+ String provider = getProviderFromConfig(getConf());
+ String identity = checkNotNull(getConf().get(CloudLocationConfig.ACCESS_IDENTITY), "identity must not be null");
+ String credential = checkNotNull(getConf().get(CloudLocationConfig.ACCESS_CREDENTIAL), "credential must not be null");
+ String endpoint = getProperties().getProperty(Constants.PROPERTY_ENDPOINT);
+ return MutableMap.builder()
+ .putAll(getProperties())
+ .put("provider", provider)
+ .put("identity", identity)
+ .put("credential", credential)
+ .putIfNotNull("endpoint", endpoint)
+ .build()
+ .asUnmodifiable();
+ }
+ }
+
+ protected String getProviderFromConfig(ConfigBag conf) {
+ String rawProvider = checkNotNull(conf.get(CLOUD_PROVIDER), "provider must not be null");
+ return DeserializingJcloudsRenamesProvider.INSTANCE.applyJcloudsRenames(rawProvider);
+ }
+
+ private String addEndpointProperty(ConfigBag conf, Properties properties) {
+ String endpoint = conf.get(CloudLocationConfig.CLOUD_ENDPOINT);
+ if (!groovyTruth(endpoint)) endpoint = getDeprecatedProperty(conf, Constants.PROPERTY_ENDPOINT);
+ if (groovyTruth(endpoint)) properties.setProperty(Constants.PROPERTY_ENDPOINT, endpoint);
+ return endpoint;
+ }
+
+ private void setCommonProperties(ConfigBag conf, Properties properties) {
+ properties.setProperty(Constants.PROPERTY_TRUST_ALL_CERTS, Boolean.toString(true));
+ properties.setProperty(Constants.PROPERTY_RELAX_HOSTNAME, Boolean.toString(true));
+ properties.setProperty("jclouds.ssh.max-retries", conf.getStringKey("jclouds.ssh.max-retries") != null ?
+ conf.getStringKey("jclouds.ssh.max-retries").toString() : "50");
+
+ if (conf.get(OAUTH_ENDPOINT) != null)
+ properties.setProperty(OAUTH_ENDPOINT.getName(), conf.get(OAUTH_ENDPOINT));
+
+ // See https://issues.apache.org/jira/browse/BROOKLYN-394
+ // For retries, the backoff times are:
+ // Math.min(2^failureCount * retryDelayStart, retryDelayStart * 10) + random(10%)
+ // Therefore the backoff times will be: 500ms, 1s, 2s, 4s, 5s, 5s.
+ // The defaults (if not overridden here) are 50ms and 5 retires. This gives backoff
+ // times of 50ms, 100ms, 200ms, 400ms, 500ms (so a total backoff time of 1.25s),
+ // which is not long when you're being rate-limited and there are multiple thread all
+ // retrying their API calls.
+ properties.setProperty(Constants.PROPERTY_RETRY_DELAY_START, "500");
+ properties.setProperty(Constants.PROPERTY_MAX_RETRIES, "6");
+ }
+
+ private void addJCloudsProperties(ConfigBag conf, Properties properties) {
+ // Add extra jclouds-specific configuration
+ Map<String, Object> extra = Maps.filterKeys(conf.getAllConfig(), Predicates.containsPattern("^jclouds\\."));
+ if (extra.size() > 0) {
+ String provider = getProviderFromConfig(conf);
+ LOG.debug("Configuring custom jclouds property overrides for {}: {}", provider, Sanitizer.sanitize(extra));
+ }
+ properties.putAll(Maps.filterValues(extra, Predicates.notNull()));
+ }
+
+ private void setAzureComputeArmProperties(ConfigBag conf, Properties properties) {
+ String region = conf.get(CLOUD_REGION_ID);
+ if (Strings.isNonBlank(region)) {
+ properties.setProperty(LocationConstants.PROPERTY_REGIONS, region);
+ }
+ }
+
+ private void setAWSEC2Properties(ConfigBag conf, Properties properties) {
+ // TODO convert AWS-only flags to config keys
+ if (groovyTruth(conf.get(IMAGE_ID))) {
+ properties.setProperty(PROPERTY_EC2_AMI_QUERY, "");
+ properties.setProperty(PROPERTY_EC2_CC_AMI_QUERY, "");
+ } else if (groovyTruth(conf.getStringKey("imageOwner"))) {
+ properties.setProperty(PROPERTY_EC2_AMI_QUERY, "owner-id=" + conf.getStringKey("imageOwner") + ";state=available;image-type=machine");
+ } else if (groovyTruth(conf.getStringKey("anyOwner"))) {
+ // set `anyOwner: true` to override the default query (which is restricted to certain owners as per below),
+ // allowing the AMI query to bind to any machine
+ // (note however, we sometimes pick defaults in JcloudsLocationFactory);
+ // (and be careful, this can give a LOT of data back, taking several minutes,
+ // and requiring extra memory allocated on the command-line)
+ properties.setProperty(PROPERTY_EC2_AMI_QUERY, "state=available;image-type=machine");
+ /*
+ * by default the following filters are applied:
+ * Filter.1.Name=owner-id&Filter.1.Value.1=137112412989&
+ * Filter.1.Value.2=063491364108&
+ * Filter.1.Value.3=099720109477&
+ * Filter.1.Value.4=411009282317&
+ * Filter.2.Name=state&Filter.2.Value.1=available&
+ * Filter.3.Name=image-type&Filter.3.Value.1=machine&
+ */
+ }
+
+ // See https://issues.apache.org/jira/browse/BROOKLYN-399
+ String region = conf.get(CLOUD_REGION_ID);
+ if (Strings.isNonBlank(region)) {
+ /*
+ * Drop availability zone suffixes. Without this deployments to regions like us-east-1b fail
+ * because jclouds throws an IllegalStateException complaining that: location id us-east-1b
+ * not found in: [{scope=PROVIDER, id=aws-ec2, description=https://ec2.us-east-1.amazonaws.com,
+ * iso3166Codes=[US-VA, US-CA, US-OR, BR-SP, IE, DE-HE, SG, AU-NSW, JP-13]}]. The exception is
+ * thrown by org.jclouds.compute.domain.internal.TemplateBuilderImpl#locationId(String).
+ */
+ if (Character.isLetter(region.charAt(region.length() - 1))) {
+ region = region.substring(0, region.length() - 1);
+ }
+ properties.setProperty(LocationConstants.PROPERTY_REGIONS, region);
+ }
+
+ // occasionally can get com.google.common.util.concurrent.UncheckedExecutionException: java.lang.RuntimeException:
+ // security group eu-central-1/jclouds#brooklyn-bxza-alex-eu-central-shoul-u2jy-nginx-ielm is not available after creating
+ // the default timeout was 500ms so let's raise it in case that helps
+ properties.setProperty(EC2Constants.PROPERTY_EC2_TIMEOUT_SECURITYGROUP_PRESENT, "" + Duration.seconds(30).toMilliseconds());
+ }
+
+ protected abstract Supplier<Credentials> makeCredentials(ConfigBag conf);
+
+ /**
+ * returns the jclouds modules we typically install
+ */
+ protected ImmutableSet<Module> getCommonModules() {
+ return ImmutableSet.<Module>of(
+ new SshjSshClientModule(),
+ new SLF4JLoggingModule(),
+ new BouncyCastleCryptoModule());
+ }
+
+ protected String getDeprecatedProperty(ConfigBag conf, String key) {
+ if (conf.containsKey(key)) {
+ LOG.warn("Jclouds using deprecated brooklyn-jclouds property " + key + ": " + Sanitizer.sanitize(conf.getAllConfig()));
+ return (String) conf.getStringKey(key);
+ } else {
+ return null;
+ }
+ }
+
+ @Override
+ public String toString() {
+ return getClass().getName();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/e6ca01c9/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/AwsEc2SessionAwareComputeServiceRegistry.java
----------------------------------------------------------------------
diff --git a/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/AwsEc2SessionAwareComputeServiceRegistry.java b/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/AwsEc2SessionAwareComputeServiceRegistry.java
new file mode 100644
index 0000000..1b3c8fe
--- /dev/null
+++ b/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/AwsEc2SessionAwareComputeServiceRegistry.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.brooklyn.location.jclouds;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import java.io.IOException;
+import java.net.URL;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+
+import org.apache.brooklyn.util.core.config.ConfigBag;
+import org.apache.brooklyn.util.exceptions.Exceptions;
+import org.jclouds.aws.domain.SessionCredentials;
+import org.jclouds.compute.ComputeService;
+import org.jclouds.domain.Credentials;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.base.Supplier;
+
+public class AwsEc2SessionAwareComputeServiceRegistry extends AbstractComputeServiceRegistry implements ComputeServiceRegistry, AwsEc2SessionAwareLocationConfig {
+
+ public static final String ACCESS_KEY_ID = "AccessKeyId";
+ public static final String SECRET_ACCESS_KEY = "SecretAccessKey";
+ public static final String TOKEN = "Token";
+ public static final String EXPIRATION = "Expiration";
+ public static final String AWS_SECURITY_CREDENTIAL_URL = "http://169.254.169.254/latest/meta-data/iam/security-credentials";
+ public static final String AWS_EXPIRATION_DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ss'Z'";
+
+ public AwsEc2SessionAwareComputeServiceRegistry(){}
+
+ @Override
+ public ComputeService findComputeService(ConfigBag conf, boolean allowReuse) {
+ return super.findComputeService(conf, false); // do not allow caching
+ }
+
+ @Override
+ protected Supplier<Credentials> makeCredentials(ConfigBag conf) {
+ Credentials credentials;
+ String identity = null, credential = null, token = null;
+ Date expiration = null;
+ String provider = getProviderFromConfig(conf);
+ String iamRoleName = getIamRoleNameFromConfig(conf);
+ if ("aws-ec2".equals(provider)) {
+ try {
+ String instanceProfileUrl = AWS_SECURITY_CREDENTIAL_URL;
+ JsonNode node = new ObjectMapper().readTree(new URL(instanceProfileUrl + "/" + iamRoleName));
+ identity = node.path(ACCESS_KEY_ID).asText();
+ credential = node.path(SECRET_ACCESS_KEY).asText();
+ token = node.path(TOKEN).asText();
+ expiration = new SimpleDateFormat(AWS_EXPIRATION_DATE_FORMAT).parse(node.path(EXPIRATION).asText());
+ } catch (IOException | ParseException e) {
+ Exceptions.propagate(e);
+ }
+ } else {
+ throw new IllegalArgumentException("Provider " + provider + " does not support session credentials");
+ }
+
+ identity = checkNotNull(identity, "identity must not be null");
+ credential = checkNotNull(credential, "credential must not be null");
+ token = checkNotNull(token, "token must not be null");
+
+ credentials = SessionCredentials.builder()
+ .accessKeyId(identity)
+ .credential(credential)
+ .sessionToken(token)
+ .expiration(expiration)
+ .build();
+ return () -> credentials;
+ }
+
+ private String getIamRoleNameFromConfig(ConfigBag conf) {
+ return checkNotNull(conf.get(IAM_ROLE_NAME), "IAM role must not be null");
+ }
+
+ @Override
+ public String toString(){
+ return getClass().getName();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/e6ca01c9/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/AwsEc2SessionAwareLocationConfig.java
----------------------------------------------------------------------
diff --git a/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/AwsEc2SessionAwareLocationConfig.java b/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/AwsEc2SessionAwareLocationConfig.java
new file mode 100644
index 0000000..d20f846
--- /dev/null
+++ b/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/AwsEc2SessionAwareLocationConfig.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.brooklyn.location.jclouds;
+
+import org.apache.brooklyn.config.ConfigKey;
+import org.apache.brooklyn.core.config.ConfigKeys;
+
+public interface AwsEc2SessionAwareLocationConfig extends JcloudsLocationConfig{
+
+ ConfigKey<String> IAM_ROLE_NAME = ConfigKeys.newStringConfigKey("iamRoleName",
+ "Use IAM role to get session credentials when connecting to AWS EC2", "brooklyn");
+}
http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/e6ca01c9/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/ComputeServiceRegistryImpl.java
----------------------------------------------------------------------
diff --git a/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/ComputeServiceRegistryImpl.java b/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/ComputeServiceRegistryImpl.java
index 02d7ea2..35087fe 100644
--- a/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/ComputeServiceRegistryImpl.java
+++ b/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/ComputeServiceRegistryImpl.java
@@ -19,214 +19,28 @@
package org.apache.brooklyn.location.jclouds;
import static com.google.common.base.Preconditions.checkNotNull;
-import static org.apache.brooklyn.util.JavaGroovyEquivalents.groovyTruth;
-import static org.jclouds.aws.ec2.reference.AWSEC2Constants.PROPERTY_EC2_AMI_QUERY;
-import static org.jclouds.aws.ec2.reference.AWSEC2Constants.PROPERTY_EC2_CC_AMI_QUERY;
-import java.util.Map;
-import java.util.Properties;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.apache.brooklyn.core.config.Sanitizer;
import org.apache.brooklyn.core.location.cloud.CloudLocationConfig;
-import org.apache.brooklyn.core.mgmt.persist.DeserializingJcloudsRenamesProvider;
-import org.apache.brooklyn.util.collections.MutableMap;
import org.apache.brooklyn.util.core.config.ConfigBag;
-import org.apache.brooklyn.util.text.Strings;
-import org.apache.brooklyn.util.time.Duration;
-import org.jclouds.Constants;
-import org.jclouds.ContextBuilder;
-import org.jclouds.azurecompute.arm.config.AzureComputeRateLimitModule;
-import org.jclouds.compute.ComputeService;
-import org.jclouds.compute.ComputeServiceContext;
-import org.jclouds.ec2.reference.EC2Constants;
-import org.jclouds.encryption.bouncycastle.config.BouncyCastleCryptoModule;
-import org.jclouds.location.reference.LocationConstants;
-import org.jclouds.logging.slf4j.config.SLF4JLoggingModule;
-import org.jclouds.sshj.config.SshjSshClientModule;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.jclouds.domain.Credentials;
-import com.google.common.base.Predicates;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Maps;
-import com.google.inject.Module;
+import com.google.common.base.Supplier;
-public class ComputeServiceRegistryImpl implements ComputeServiceRegistry, JcloudsLocationConfig {
+public class ComputeServiceRegistryImpl extends AbstractComputeServiceRegistry implements ComputeServiceRegistry, JcloudsLocationConfig {
- private static final Logger LOG = LoggerFactory.getLogger(ComputeServiceRegistryImpl.class);
-
public static final ComputeServiceRegistryImpl INSTANCE = new ComputeServiceRegistryImpl();
protected ComputeServiceRegistryImpl() {
}
-
- protected final Map<Map<?,?>,ComputeService> cachedComputeServices = new ConcurrentHashMap<Map<?,?>,ComputeService>();
-
- protected final Object createComputeServicesMutex = new Object();
@Override
- public ComputeService findComputeService(ConfigBag conf, boolean allowReuse) {
- String rawProvider = checkNotNull(conf.get(CLOUD_PROVIDER), "provider must not be null");
- String provider = DeserializingJcloudsRenamesProvider.INSTANCE.applyJcloudsRenames(rawProvider);
-
+ protected Supplier<Credentials> makeCredentials(ConfigBag conf) {
String identity = checkNotNull(conf.get(CloudLocationConfig.ACCESS_IDENTITY), "identity must not be null");
String credential = checkNotNull(conf.get(CloudLocationConfig.ACCESS_CREDENTIAL), "credential must not be null");
-
- Properties properties = new Properties();
- properties.setProperty(Constants.PROPERTY_TRUST_ALL_CERTS, Boolean.toString(true));
- properties.setProperty(Constants.PROPERTY_RELAX_HOSTNAME, Boolean.toString(true));
- properties.setProperty("jclouds.ssh.max-retries", conf.getStringKey("jclouds.ssh.max-retries") != null ?
- conf.getStringKey("jclouds.ssh.max-retries").toString() : "50");
-
- if (conf.get(OAUTH_ENDPOINT) != null) properties.setProperty(OAUTH_ENDPOINT.getName(),conf.get(OAUTH_ENDPOINT));
-
- // See https://issues.apache.org/jira/browse/BROOKLYN-394
- // For retries, the backoff times are:
- // Math.min(2^failureCount * retryDelayStart, retryDelayStart * 10) + random(10%)
- // Therefore the backoff times will be: 500ms, 1s, 2s, 4s, 5s, 5s.
- // The defaults (if not overridden here) are 50ms and 5 retires. This gives backoff
- // times of 50ms, 100ms, 200ms, 400ms, 500ms (so a total backoff time of 1.25s),
- // which is not long when you're being rate-limited and there are multiple thread all
- // retrying their API calls.
- properties.setProperty(Constants.PROPERTY_RETRY_DELAY_START, "500");
- properties.setProperty(Constants.PROPERTY_MAX_RETRIES, "6");
-
- Iterable<Module> modules = getCommonModules();
-
- // Enable aws-ec2 lazy image fetching, if given a specific imageId; otherwise customize for specific owners; or all as a last resort
- // See https://issues.apache.org/jira/browse/WHIRR-416
- if ("aws-ec2".equals(provider)) {
- // TODO convert AWS-only flags to config keys
- if (groovyTruth(conf.get(IMAGE_ID))) {
- properties.setProperty(PROPERTY_EC2_AMI_QUERY, "");
- properties.setProperty(PROPERTY_EC2_CC_AMI_QUERY, "");
- } else if (groovyTruth(conf.getStringKey("imageOwner"))) {
- properties.setProperty(PROPERTY_EC2_AMI_QUERY, "owner-id="+conf.getStringKey("imageOwner")+";state=available;image-type=machine");
- } else if (groovyTruth(conf.getStringKey("anyOwner"))) {
- // set `anyOwner: true` to override the default query (which is restricted to certain owners as per below),
- // allowing the AMI query to bind to any machine
- // (note however, we sometimes pick defaults in JcloudsLocationFactory);
- // (and be careful, this can give a LOT of data back, taking several minutes,
- // and requiring extra memory allocated on the command-line)
- properties.setProperty(PROPERTY_EC2_AMI_QUERY, "state=available;image-type=machine");
- /*
- * by default the following filters are applied:
- * Filter.1.Name=owner-id&Filter.1.Value.1=137112412989&
- * Filter.1.Value.2=063491364108&
- * Filter.1.Value.3=099720109477&
- * Filter.1.Value.4=411009282317&
- * Filter.2.Name=state&Filter.2.Value.1=available&
- * Filter.3.Name=image-type&Filter.3.Value.1=machine&
- */
- }
-
- // See https://issues.apache.org/jira/browse/BROOKLYN-399
- String region = conf.get(CLOUD_REGION_ID);
- if (Strings.isNonBlank(region)) {
- /*
- * Drop availability zone suffixes. Without this deployments to regions like us-east-1b fail
- * because jclouds throws an IllegalStateException complaining that: location id us-east-1b
- * not found in: [{scope=PROVIDER, id=aws-ec2, description=https://ec2.us-east-1.amazonaws.com,
- * iso3166Codes=[US-VA, US-CA, US-OR, BR-SP, IE, DE-HE, SG, AU-NSW, JP-13]}]. The exception is
- * thrown by org.jclouds.compute.domain.internal.TemplateBuilderImpl#locationId(String).
- */
- if (Character.isLetter(region.charAt(region.length() - 1))) {
- region = region.substring(0, region.length() - 1);
- }
- properties.setProperty(LocationConstants.PROPERTY_REGIONS, region);
- }
-
- // occasionally can get com.google.common.util.concurrent.UncheckedExecutionException: java.lang.RuntimeException:
- // security group eu-central-1/jclouds#brooklyn-bxza-alex-eu-central-shoul-u2jy-nginx-ielm is not available after creating
- // the default timeout was 500ms so let's raise it in case that helps
- properties.setProperty(EC2Constants.PROPERTY_EC2_TIMEOUT_SECURITYGROUP_PRESENT, ""+Duration.seconds(30).toMilliseconds());
- }
- else if ("azurecompute-arm".equals(provider)) {
- String region = conf.get(CLOUD_REGION_ID);
- if (Strings.isNonBlank(region)) {
- properties.setProperty(LocationConstants.PROPERTY_REGIONS, region);
- }
- // jclouds 2.0.0 does not include the rate limit module for Azure ARM. This quick fix enables this which will
- // avoid provisioning to fail due to rate limit exceeded
- // See https://issues.apache.org/jira/browse/JCLOUDS-1229
- modules = ImmutableSet.<Module>builder()
- .addAll(modules)
- .add(new AzureComputeRateLimitModule())
- .build();
- }
-
- // Add extra jclouds-specific configuration
- Map<String, Object> extra = Maps.filterKeys(conf.getAllConfig(), Predicates.containsPattern("^jclouds\\."));
- if (extra.size() > 0) {
- LOG.debug("Configuring custom jclouds property overrides for {}: {}", provider, Sanitizer.sanitize(extra));
- }
- properties.putAll(Maps.filterValues(extra, Predicates.notNull()));
-
- String endpoint = conf.get(CloudLocationConfig.CLOUD_ENDPOINT);
- if (!groovyTruth(endpoint)) endpoint = getDeprecatedProperty(conf, Constants.PROPERTY_ENDPOINT);
- if (groovyTruth(endpoint)) properties.setProperty(Constants.PROPERTY_ENDPOINT, endpoint);
-
- Map<?,?> cacheKey = MutableMap.builder()
- .putAll(properties)
- .put("provider", provider)
- .put("identity", identity)
- .put("credential", credential)
- .putIfNotNull("endpoint", endpoint)
- .build()
- .asUnmodifiable();
-
- if (allowReuse) {
- ComputeService result = cachedComputeServices.get(cacheKey);
- if (result!=null) {
- LOG.trace("jclouds ComputeService cache hit for compute service, for "+Sanitizer.sanitize(properties));
- return result;
- }
- LOG.debug("jclouds ComputeService cache miss for compute service, creating, for "+Sanitizer.sanitize(properties));
- }
-
- // Synchronizing to avoid deadlock from sun.reflect.annotation.AnnotationType.
- // See https://github.com/brooklyncentral/brooklyn/issues/974
- ComputeServiceContext computeServiceContext;
- synchronized (createComputeServicesMutex) {
- computeServiceContext = ContextBuilder.newBuilder(provider)
- .modules(modules)
- .credentials(identity, credential)
- .overrides(properties)
- .build(ComputeServiceContext.class);
- }
- final ComputeService computeService = computeServiceContext.getComputeService();
- if (allowReuse) {
- synchronized (cachedComputeServices) {
- ComputeService result = cachedComputeServices.get(cacheKey);
- if (result != null) {
- LOG.debug("jclouds ComputeService cache recovery for compute service, for "+Sanitizer.sanitize(cacheKey));
- //keep the old one, discard the new one
- computeService.getContext().close();
- return result;
- }
- LOG.debug("jclouds ComputeService created "+computeService+", adding to cache, for "+Sanitizer.sanitize(properties));
- cachedComputeServices.put(cacheKey, computeService);
- }
- }
- return computeService;
- }
-
- /** returns the jclouds modules we typically install */
- protected ImmutableSet<Module> getCommonModules() {
- return ImmutableSet.<Module> of(
- new SshjSshClientModule(),
- new SLF4JLoggingModule(),
- new BouncyCastleCryptoModule());
- }
-
- protected String getDeprecatedProperty(ConfigBag conf, String key) {
- if (conf.containsKey(key)) {
- LOG.warn("Jclouds using deprecated brooklyn-jclouds property "+key+": "+Sanitizer.sanitize(conf.getAllConfig()));
- return (String) conf.getStringKey(key);
- } else {
- return null;
- }
+ return () -> new Credentials.Builder<>()
+ .identity(identity)
+ .credential(credential)
+ .build();
}
@Override
[2/4] brooklyn-server git commit: replace custom caching with
computeIfAbsent
Posted by an...@apache.org.
replace custom caching with computeIfAbsent
Project: http://git-wip-us.apache.org/repos/asf/brooklyn-server/repo
Commit: http://git-wip-us.apache.org/repos/asf/brooklyn-server/commit/a4d8a95f
Tree: http://git-wip-us.apache.org/repos/asf/brooklyn-server/tree/a4d8a95f
Diff: http://git-wip-us.apache.org/repos/asf/brooklyn-server/diff/a4d8a95f
Branch: refs/heads/master
Commit: a4d8a95f0a2062fd278228d58cb31f9423ae90d3
Parents: e6ca01c
Author: Robert Moss <ro...@cloudsoftcorp.com>
Authored: Thu Jun 8 16:15:15 2017 +0100
Committer: Robert Moss <ro...@cloudsoftcorp.com>
Committed: Thu Jun 8 16:15:15 2017 +0100
----------------------------------------------------------------------
.../jclouds/AbstractComputeServiceRegistry.java | 303 +++++++++----------
1 file changed, 137 insertions(+), 166 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/a4d8a95f/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/AbstractComputeServiceRegistry.java
----------------------------------------------------------------------
diff --git a/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/AbstractComputeServiceRegistry.java b/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/AbstractComputeServiceRegistry.java
index 076d442..8bc3388 100644
--- a/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/AbstractComputeServiceRegistry.java
+++ b/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/AbstractComputeServiceRegistry.java
@@ -62,8 +62,8 @@ public abstract class AbstractComputeServiceRegistry implements ComputeServiceRe
@Override
public ComputeService findComputeService(ConfigBag conf, boolean allowReuse) {
- Properties properties = new Properties();
- setCommonProperties(conf, properties);
+ PropertiesBuilder propertiesBuilder = new PropertiesBuilder(conf)
+ .setCommonProperties();
Iterable<Module> modules = getCommonModules();
@@ -71,9 +71,9 @@ public abstract class AbstractComputeServiceRegistry implements ComputeServiceRe
// See https://issues.apache.org/jira/browse/WHIRR-416
String provider = getProviderFromConfig(conf);
if ("aws-ec2".equals(provider)) {
- setAWSEC2Properties(conf, properties);
+ propertiesBuilder.setAWSEC2Properties();
} else if ("azurecompute-arm".equals(provider)) {
- setAzureComputeArmProperties(conf, properties);
+ propertiesBuilder.setAzureComputeArmProperties();
// jclouds 2.0.0 does not include the rate limit module for Azure ARM. This quick fix enables this which will
// avoid provisioning to fail due to rate limit exceeded
// See https://issues.apache.org/jira/browse/JCLOUDS-1229
@@ -83,17 +83,143 @@ public abstract class AbstractComputeServiceRegistry implements ComputeServiceRe
.build();
}
- addJCloudsProperties(conf, properties);
- addEndpointProperty(conf, properties);
-
- Supplier<ComputeService> computeServiceSupplier = allowReuse
- ? new ReusableComputeServiceSupplier(conf, modules, properties)
- : new ComputeServiceSupplierImpl(conf, modules, properties);
+ Properties properties = propertiesBuilder
+ .setJCloudsProperties()
+ .setEndpointProperty()
+ .build();
+ Supplier<ComputeService> computeServiceSupplier = new ComputeServiceSupplier(conf, modules, properties);
+ if (allowReuse) {
+ return cachedComputeServices.computeIfAbsent(makeCacheKey(conf, properties), key -> computeServiceSupplier.get());
+ }
return computeServiceSupplier.get();
}
- public abstract class ComputeServiceSupplier implements Supplier<ComputeService> {
+ private Map<?, ?> makeCacheKey(ConfigBag conf, Properties properties) {
+ String provider = getProviderFromConfig(conf);
+ String identity = checkNotNull(conf.get(CloudLocationConfig.ACCESS_IDENTITY), "identity must not be null");
+ String credential = checkNotNull(conf.get(CloudLocationConfig.ACCESS_CREDENTIAL), "credential must not be null");
+ String endpoint = properties.getProperty(Constants.PROPERTY_ENDPOINT);
+ return MutableMap.builder()
+ .putAll(properties)
+ .put("provider", provider)
+ .put("identity", identity)
+ .put("credential", credential)
+ .putIfNotNull("endpoint", endpoint)
+ .build()
+ .asUnmodifiable();
+ }
+
+ public class PropertiesBuilder {
+ private ConfigBag conf;
+ private Properties properties = new Properties();
+
+ public PropertiesBuilder(ConfigBag conf) {
+ this.conf = conf;
+ }
+
+ public PropertiesBuilder setCommonProperties() {
+ properties.setProperty(Constants.PROPERTY_TRUST_ALL_CERTS, Boolean.toString(true));
+ properties.setProperty(Constants.PROPERTY_RELAX_HOSTNAME, Boolean.toString(true));
+ properties.setProperty("jclouds.ssh.max-retries", conf.getStringKey("jclouds.ssh.max-retries") != null ?
+ conf.getStringKey("jclouds.ssh.max-retries").toString() : "50");
+
+ if (conf.get(OAUTH_ENDPOINT) != null)
+ properties.setProperty(OAUTH_ENDPOINT.getName(), conf.get(OAUTH_ENDPOINT));
+
+ // See https://issues.apache.org/jira/browse/BROOKLYN-394
+ // For retries, the backoff times are:
+ // Math.min(2^failureCount * retryDelayStart, retryDelayStart * 10) + random(10%)
+ // Therefore the backoff times will be: 500ms, 1s, 2s, 4s, 5s, 5s.
+ // The defaults (if not overridden here) are 50ms and 5 retires. This gives backoff
+ // times of 50ms, 100ms, 200ms, 400ms, 500ms (so a total backoff time of 1.25s),
+ // which is not long when you're being rate-limited and there are multiple thread all
+ // retrying their API calls.
+ properties.setProperty(Constants.PROPERTY_RETRY_DELAY_START, "500");
+ properties.setProperty(Constants.PROPERTY_MAX_RETRIES, "6");
+ return this;
+ }
+
+ public PropertiesBuilder setAWSEC2Properties() {
+ // TODO convert AWS-only flags to config keys
+ if (groovyTruth(conf.get(IMAGE_ID))) {
+ properties.setProperty(PROPERTY_EC2_AMI_QUERY, "");
+ properties.setProperty(PROPERTY_EC2_CC_AMI_QUERY, "");
+ } else if (groovyTruth(conf.getStringKey("imageOwner"))) {
+ properties.setProperty(PROPERTY_EC2_AMI_QUERY, "owner-id=" + conf.getStringKey("imageOwner") + ";state=available;image-type=machine");
+ } else if (groovyTruth(conf.getStringKey("anyOwner"))) {
+ // set `anyOwner: true` to override the default query (which is restricted to certain owners as per below),
+ // allowing the AMI query to bind to any machine
+ // (note however, we sometimes pick defaults in JcloudsLocationFactory);
+ // (and be careful, this can give a LOT of data back, taking several minutes,
+ // and requiring extra memory allocated on the command-line)
+ properties.setProperty(PROPERTY_EC2_AMI_QUERY, "state=available;image-type=machine");
+ /*
+ * by default the following filters are applied:
+ * Filter.1.Name=owner-id&Filter.1.Value.1=137112412989&
+ * Filter.1.Value.2=063491364108&
+ * Filter.1.Value.3=099720109477&
+ * Filter.1.Value.4=411009282317&
+ * Filter.2.Name=state&Filter.2.Value.1=available&
+ * Filter.3.Name=image-type&Filter.3.Value.1=machine&
+ */
+ }
+
+ // See https://issues.apache.org/jira/browse/BROOKLYN-399
+ String region = conf.get(CLOUD_REGION_ID);
+ if (Strings.isNonBlank(region)) {
+ /*
+ * Drop availability zone suffixes. Without this deployments to regions like us-east-1b fail
+ * because jclouds throws an IllegalStateException complaining that: location id us-east-1b
+ * not found in: [{scope=PROVIDER, id=aws-ec2, description=https://ec2.us-east-1.amazonaws.com,
+ * iso3166Codes=[US-VA, US-CA, US-OR, BR-SP, IE, DE-HE, SG, AU-NSW, JP-13]}]. The exception is
+ * thrown by org.jclouds.compute.domain.internal.TemplateBuilderImpl#locationId(String).
+ */
+ if (Character.isLetter(region.charAt(region.length() - 1))) {
+ region = region.substring(0, region.length() - 1);
+ }
+ properties.setProperty(LocationConstants.PROPERTY_REGIONS, region);
+ }
+
+ // occasionally can get com.google.common.util.concurrent.UncheckedExecutionException: java.lang.RuntimeException:
+ // security group eu-central-1/jclouds#brooklyn-bxza-alex-eu-central-shoul-u2jy-nginx-ielm is not available after creating
+ // the default timeout was 500ms so let's raise it in case that helps
+ properties.setProperty(EC2Constants.PROPERTY_EC2_TIMEOUT_SECURITYGROUP_PRESENT, "" + Duration.seconds(30).toMilliseconds());
+ return this;
+ }
+
+ private PropertiesBuilder setAzureComputeArmProperties() {
+ String region = conf.get(CLOUD_REGION_ID);
+ if (Strings.isNonBlank(region)) {
+ properties.setProperty(LocationConstants.PROPERTY_REGIONS, region);
+ }
+ return this;
+ }
+
+ private PropertiesBuilder setJCloudsProperties() {
+ // Add extra jclouds-specific configuration
+ Map<String, Object> extra = Maps.filterKeys(conf.getAllConfig(), Predicates.containsPattern("^jclouds\\."));
+ if (extra.size() > 0) {
+ String provider = getProviderFromConfig(conf);
+ LOG.debug("Configuring custom jclouds property overrides for {}: {}", provider, Sanitizer.sanitize(extra));
+ }
+ properties.putAll(Maps.filterValues(extra, Predicates.notNull()));
+ return this;
+ }
+
+ private PropertiesBuilder setEndpointProperty() {
+ String endpoint = conf.get(CloudLocationConfig.CLOUD_ENDPOINT);
+ if (!groovyTruth(endpoint)) endpoint = getDeprecatedProperty(conf, Constants.PROPERTY_ENDPOINT);
+ if (groovyTruth(endpoint)) properties.setProperty(Constants.PROPERTY_ENDPOINT, endpoint);
+ return this;
+ }
+
+ public Properties build() {
+ return properties;
+ }
+ }
+
+ public class ComputeServiceSupplier implements Supplier<ComputeService> {
private final String provider;
private final ConfigBag conf;
@@ -121,69 +247,6 @@ public abstract class AbstractComputeServiceRegistry implements ComputeServiceRe
return computeServiceContext.getComputeService();
}
}
-
- protected ConfigBag getConf() {
- return conf;
- }
-
- protected Properties getProperties() {
- return properties;
- }
- }
-
- public class ComputeServiceSupplierImpl extends ComputeServiceSupplier {
-
- public ComputeServiceSupplierImpl(ConfigBag conf, Iterable<? extends Module> modules, Properties properties) {
- super(conf, modules, properties);
- }
- }
-
- public class ReusableComputeServiceSupplier extends ComputeServiceSupplier {
-
- private Map<?, ?> cacheKey;
-
- public ReusableComputeServiceSupplier(ConfigBag conf, Iterable<? extends Module> modules, Properties properties) {
- super(conf, modules, properties);
- this.cacheKey = makeCacheKey();
- }
-
- @Override
- public ComputeService get() {
- ComputeService result = cachedComputeServices.get(cacheKey);
- if (result != null) {
- LOG.trace("jclouds ComputeService cache hit for compute service, for " + Sanitizer.sanitize(getProperties()));
- return result;
- }
- LOG.debug("jclouds ComputeService cache miss for compute service, creating, for " + Sanitizer.sanitize(getProperties()));
- final ComputeService computeService = super.get();
- synchronized (cachedComputeServices) {
- result = cachedComputeServices.get(cacheKey);
- if (result != null) {
- LOG.debug("jclouds ComputeService cache recovery for compute service, for " + Sanitizer.sanitize(cacheKey));
- //keep the old one, discard the new one
- computeService.getContext().close();
- return result;
- }
- LOG.debug("jclouds ComputeService created " + computeService + ", adding to cache, for " + Sanitizer.sanitize(getProperties()));
- cachedComputeServices.put(cacheKey, computeService);
- }
- return result;
- }
-
- private Map<?, ?> makeCacheKey() {
- String provider = getProviderFromConfig(getConf());
- String identity = checkNotNull(getConf().get(CloudLocationConfig.ACCESS_IDENTITY), "identity must not be null");
- String credential = checkNotNull(getConf().get(CloudLocationConfig.ACCESS_CREDENTIAL), "credential must not be null");
- String endpoint = getProperties().getProperty(Constants.PROPERTY_ENDPOINT);
- return MutableMap.builder()
- .putAll(getProperties())
- .put("provider", provider)
- .put("identity", identity)
- .put("credential", credential)
- .putIfNotNull("endpoint", endpoint)
- .build()
- .asUnmodifiable();
- }
}
protected String getProviderFromConfig(ConfigBag conf) {
@@ -191,98 +254,6 @@ public abstract class AbstractComputeServiceRegistry implements ComputeServiceRe
return DeserializingJcloudsRenamesProvider.INSTANCE.applyJcloudsRenames(rawProvider);
}
- private String addEndpointProperty(ConfigBag conf, Properties properties) {
- String endpoint = conf.get(CloudLocationConfig.CLOUD_ENDPOINT);
- if (!groovyTruth(endpoint)) endpoint = getDeprecatedProperty(conf, Constants.PROPERTY_ENDPOINT);
- if (groovyTruth(endpoint)) properties.setProperty(Constants.PROPERTY_ENDPOINT, endpoint);
- return endpoint;
- }
-
- private void setCommonProperties(ConfigBag conf, Properties properties) {
- properties.setProperty(Constants.PROPERTY_TRUST_ALL_CERTS, Boolean.toString(true));
- properties.setProperty(Constants.PROPERTY_RELAX_HOSTNAME, Boolean.toString(true));
- properties.setProperty("jclouds.ssh.max-retries", conf.getStringKey("jclouds.ssh.max-retries") != null ?
- conf.getStringKey("jclouds.ssh.max-retries").toString() : "50");
-
- if (conf.get(OAUTH_ENDPOINT) != null)
- properties.setProperty(OAUTH_ENDPOINT.getName(), conf.get(OAUTH_ENDPOINT));
-
- // See https://issues.apache.org/jira/browse/BROOKLYN-394
- // For retries, the backoff times are:
- // Math.min(2^failureCount * retryDelayStart, retryDelayStart * 10) + random(10%)
- // Therefore the backoff times will be: 500ms, 1s, 2s, 4s, 5s, 5s.
- // The defaults (if not overridden here) are 50ms and 5 retires. This gives backoff
- // times of 50ms, 100ms, 200ms, 400ms, 500ms (so a total backoff time of 1.25s),
- // which is not long when you're being rate-limited and there are multiple thread all
- // retrying their API calls.
- properties.setProperty(Constants.PROPERTY_RETRY_DELAY_START, "500");
- properties.setProperty(Constants.PROPERTY_MAX_RETRIES, "6");
- }
-
- private void addJCloudsProperties(ConfigBag conf, Properties properties) {
- // Add extra jclouds-specific configuration
- Map<String, Object> extra = Maps.filterKeys(conf.getAllConfig(), Predicates.containsPattern("^jclouds\\."));
- if (extra.size() > 0) {
- String provider = getProviderFromConfig(conf);
- LOG.debug("Configuring custom jclouds property overrides for {}: {}", provider, Sanitizer.sanitize(extra));
- }
- properties.putAll(Maps.filterValues(extra, Predicates.notNull()));
- }
-
- private void setAzureComputeArmProperties(ConfigBag conf, Properties properties) {
- String region = conf.get(CLOUD_REGION_ID);
- if (Strings.isNonBlank(region)) {
- properties.setProperty(LocationConstants.PROPERTY_REGIONS, region);
- }
- }
-
- private void setAWSEC2Properties(ConfigBag conf, Properties properties) {
- // TODO convert AWS-only flags to config keys
- if (groovyTruth(conf.get(IMAGE_ID))) {
- properties.setProperty(PROPERTY_EC2_AMI_QUERY, "");
- properties.setProperty(PROPERTY_EC2_CC_AMI_QUERY, "");
- } else if (groovyTruth(conf.getStringKey("imageOwner"))) {
- properties.setProperty(PROPERTY_EC2_AMI_QUERY, "owner-id=" + conf.getStringKey("imageOwner") + ";state=available;image-type=machine");
- } else if (groovyTruth(conf.getStringKey("anyOwner"))) {
- // set `anyOwner: true` to override the default query (which is restricted to certain owners as per below),
- // allowing the AMI query to bind to any machine
- // (note however, we sometimes pick defaults in JcloudsLocationFactory);
- // (and be careful, this can give a LOT of data back, taking several minutes,
- // and requiring extra memory allocated on the command-line)
- properties.setProperty(PROPERTY_EC2_AMI_QUERY, "state=available;image-type=machine");
- /*
- * by default the following filters are applied:
- * Filter.1.Name=owner-id&Filter.1.Value.1=137112412989&
- * Filter.1.Value.2=063491364108&
- * Filter.1.Value.3=099720109477&
- * Filter.1.Value.4=411009282317&
- * Filter.2.Name=state&Filter.2.Value.1=available&
- * Filter.3.Name=image-type&Filter.3.Value.1=machine&
- */
- }
-
- // See https://issues.apache.org/jira/browse/BROOKLYN-399
- String region = conf.get(CLOUD_REGION_ID);
- if (Strings.isNonBlank(region)) {
- /*
- * Drop availability zone suffixes. Without this deployments to regions like us-east-1b fail
- * because jclouds throws an IllegalStateException complaining that: location id us-east-1b
- * not found in: [{scope=PROVIDER, id=aws-ec2, description=https://ec2.us-east-1.amazonaws.com,
- * iso3166Codes=[US-VA, US-CA, US-OR, BR-SP, IE, DE-HE, SG, AU-NSW, JP-13]}]. The exception is
- * thrown by org.jclouds.compute.domain.internal.TemplateBuilderImpl#locationId(String).
- */
- if (Character.isLetter(region.charAt(region.length() - 1))) {
- region = region.substring(0, region.length() - 1);
- }
- properties.setProperty(LocationConstants.PROPERTY_REGIONS, region);
- }
-
- // occasionally can get com.google.common.util.concurrent.UncheckedExecutionException: java.lang.RuntimeException:
- // security group eu-central-1/jclouds#brooklyn-bxza-alex-eu-central-shoul-u2jy-nginx-ielm is not available after creating
- // the default timeout was 500ms so let's raise it in case that helps
- properties.setProperty(EC2Constants.PROPERTY_EC2_TIMEOUT_SECURITYGROUP_PRESENT, "" + Duration.seconds(30).toMilliseconds());
- }
-
protected abstract Supplier<Credentials> makeCredentials(ConfigBag conf);
/**
[4/4] brooklyn-server git commit: This closes #723
Posted by an...@apache.org.
This closes #723
Project: http://git-wip-us.apache.org/repos/asf/brooklyn-server/repo
Commit: http://git-wip-us.apache.org/repos/asf/brooklyn-server/commit/78f41f8d
Tree: http://git-wip-us.apache.org/repos/asf/brooklyn-server/tree/78f41f8d
Diff: http://git-wip-us.apache.org/repos/asf/brooklyn-server/diff/78f41f8d
Branch: refs/heads/master
Commit: 78f41f8d6f5ef7e08a5de5ddd785498523f128f7
Parents: 2a94137 14e844d
Author: Andrea Turli <an...@gmail.com>
Authored: Fri Jun 9 12:49:51 2017 +0200
Committer: Andrea Turli <an...@gmail.com>
Committed: Fri Jun 9 12:49:51 2017 +0200
----------------------------------------------------------------------
.../jclouds/AbstractComputeServiceRegistry.java | 151 ++++++++++++++
...wsEc2SessionAwareComputeServiceRegistry.java | 115 +++++++++++
.../AwsEc2SessionAwareLocationConfig.java | 28 +++
.../jclouds/ComputeServiceRegistryImpl.java | 202 +------------------
.../jclouds/JCloudsPropertiesBuilder.java | 166 +++++++++++++++
5 files changed, 468 insertions(+), 194 deletions(-)
----------------------------------------------------------------------
[3/4] brooklyn-server git commit: extract JCloudsPropertiesBuilder
into own class
Posted by an...@apache.org.
extract JCloudsPropertiesBuilder into own class
Project: http://git-wip-us.apache.org/repos/asf/brooklyn-server/repo
Commit: http://git-wip-us.apache.org/repos/asf/brooklyn-server/commit/14e844d8
Tree: http://git-wip-us.apache.org/repos/asf/brooklyn-server/tree/14e844d8
Diff: http://git-wip-us.apache.org/repos/asf/brooklyn-server/diff/14e844d8
Branch: refs/heads/master
Commit: 14e844d8e24b1092cd0ddf940711e50cd4ff87ca
Parents: a4d8a95
Author: Robert Moss <ro...@cloudsoftcorp.com>
Authored: Thu Jun 8 17:59:08 2017 +0100
Committer: Robert Moss <ro...@cloudsoftcorp.com>
Committed: Thu Jun 8 17:59:08 2017 +0100
----------------------------------------------------------------------
.../jclouds/AbstractComputeServiceRegistry.java | 138 +--------------
...wsEc2SessionAwareComputeServiceRegistry.java | 18 +-
.../jclouds/JCloudsPropertiesBuilder.java | 166 +++++++++++++++++++
3 files changed, 186 insertions(+), 136 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/14e844d8/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/AbstractComputeServiceRegistry.java
----------------------------------------------------------------------
diff --git a/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/AbstractComputeServiceRegistry.java b/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/AbstractComputeServiceRegistry.java
index 8bc3388..04951e0 100644
--- a/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/AbstractComputeServiceRegistry.java
+++ b/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/AbstractComputeServiceRegistry.java
@@ -19,51 +19,37 @@
package org.apache.brooklyn.location.jclouds;
import static com.google.common.base.Preconditions.checkNotNull;
-import static org.apache.brooklyn.util.JavaGroovyEquivalents.groovyTruth;
-import static org.jclouds.aws.ec2.reference.AWSEC2Constants.PROPERTY_EC2_AMI_QUERY;
-import static org.jclouds.aws.ec2.reference.AWSEC2Constants.PROPERTY_EC2_CC_AMI_QUERY;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.ConcurrentHashMap;
-import org.apache.brooklyn.core.config.Sanitizer;
import org.apache.brooklyn.core.location.cloud.CloudLocationConfig;
import org.apache.brooklyn.core.mgmt.persist.DeserializingJcloudsRenamesProvider;
import org.apache.brooklyn.util.collections.MutableMap;
import org.apache.brooklyn.util.core.config.ConfigBag;
-import org.apache.brooklyn.util.text.Strings;
-import org.apache.brooklyn.util.time.Duration;
import org.jclouds.Constants;
import org.jclouds.ContextBuilder;
import org.jclouds.azurecompute.arm.config.AzureComputeRateLimitModule;
import org.jclouds.compute.ComputeService;
import org.jclouds.compute.ComputeServiceContext;
import org.jclouds.domain.Credentials;
-import org.jclouds.ec2.reference.EC2Constants;
import org.jclouds.encryption.bouncycastle.config.BouncyCastleCryptoModule;
-import org.jclouds.location.reference.LocationConstants;
import org.jclouds.logging.slf4j.config.SLF4JLoggingModule;
import org.jclouds.sshj.config.SshjSshClientModule;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import com.google.common.base.Predicates;
import com.google.common.base.Supplier;
import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Maps;
import com.google.inject.Module;
public abstract class AbstractComputeServiceRegistry implements ComputeServiceRegistry, JcloudsLocationConfig {
- private static final Logger LOG = LoggerFactory.getLogger(AbstractComputeServiceRegistry.class);
-
private final Map<Map<?, ?>, ComputeService> cachedComputeServices = new ConcurrentHashMap<>();
@Override
public ComputeService findComputeService(ConfigBag conf, boolean allowReuse) {
- PropertiesBuilder propertiesBuilder = new PropertiesBuilder(conf)
- .setCommonProperties();
+ JCloudsPropertiesBuilder propertiesBuilder = new JCloudsPropertiesBuilder(conf)
+ .setCommonJcloudsProperties();
Iterable<Module> modules = getCommonModules();
@@ -84,7 +70,7 @@ public abstract class AbstractComputeServiceRegistry implements ComputeServiceRe
}
Properties properties = propertiesBuilder
- .setJCloudsProperties()
+ .setCustomJcloudsProperties()
.setEndpointProperty()
.build();
@@ -110,115 +96,6 @@ public abstract class AbstractComputeServiceRegistry implements ComputeServiceRe
.asUnmodifiable();
}
- public class PropertiesBuilder {
- private ConfigBag conf;
- private Properties properties = new Properties();
-
- public PropertiesBuilder(ConfigBag conf) {
- this.conf = conf;
- }
-
- public PropertiesBuilder setCommonProperties() {
- properties.setProperty(Constants.PROPERTY_TRUST_ALL_CERTS, Boolean.toString(true));
- properties.setProperty(Constants.PROPERTY_RELAX_HOSTNAME, Boolean.toString(true));
- properties.setProperty("jclouds.ssh.max-retries", conf.getStringKey("jclouds.ssh.max-retries") != null ?
- conf.getStringKey("jclouds.ssh.max-retries").toString() : "50");
-
- if (conf.get(OAUTH_ENDPOINT) != null)
- properties.setProperty(OAUTH_ENDPOINT.getName(), conf.get(OAUTH_ENDPOINT));
-
- // See https://issues.apache.org/jira/browse/BROOKLYN-394
- // For retries, the backoff times are:
- // Math.min(2^failureCount * retryDelayStart, retryDelayStart * 10) + random(10%)
- // Therefore the backoff times will be: 500ms, 1s, 2s, 4s, 5s, 5s.
- // The defaults (if not overridden here) are 50ms and 5 retires. This gives backoff
- // times of 50ms, 100ms, 200ms, 400ms, 500ms (so a total backoff time of 1.25s),
- // which is not long when you're being rate-limited and there are multiple thread all
- // retrying their API calls.
- properties.setProperty(Constants.PROPERTY_RETRY_DELAY_START, "500");
- properties.setProperty(Constants.PROPERTY_MAX_RETRIES, "6");
- return this;
- }
-
- public PropertiesBuilder setAWSEC2Properties() {
- // TODO convert AWS-only flags to config keys
- if (groovyTruth(conf.get(IMAGE_ID))) {
- properties.setProperty(PROPERTY_EC2_AMI_QUERY, "");
- properties.setProperty(PROPERTY_EC2_CC_AMI_QUERY, "");
- } else if (groovyTruth(conf.getStringKey("imageOwner"))) {
- properties.setProperty(PROPERTY_EC2_AMI_QUERY, "owner-id=" + conf.getStringKey("imageOwner") + ";state=available;image-type=machine");
- } else if (groovyTruth(conf.getStringKey("anyOwner"))) {
- // set `anyOwner: true` to override the default query (which is restricted to certain owners as per below),
- // allowing the AMI query to bind to any machine
- // (note however, we sometimes pick defaults in JcloudsLocationFactory);
- // (and be careful, this can give a LOT of data back, taking several minutes,
- // and requiring extra memory allocated on the command-line)
- properties.setProperty(PROPERTY_EC2_AMI_QUERY, "state=available;image-type=machine");
- /*
- * by default the following filters are applied:
- * Filter.1.Name=owner-id&Filter.1.Value.1=137112412989&
- * Filter.1.Value.2=063491364108&
- * Filter.1.Value.3=099720109477&
- * Filter.1.Value.4=411009282317&
- * Filter.2.Name=state&Filter.2.Value.1=available&
- * Filter.3.Name=image-type&Filter.3.Value.1=machine&
- */
- }
-
- // See https://issues.apache.org/jira/browse/BROOKLYN-399
- String region = conf.get(CLOUD_REGION_ID);
- if (Strings.isNonBlank(region)) {
- /*
- * Drop availability zone suffixes. Without this deployments to regions like us-east-1b fail
- * because jclouds throws an IllegalStateException complaining that: location id us-east-1b
- * not found in: [{scope=PROVIDER, id=aws-ec2, description=https://ec2.us-east-1.amazonaws.com,
- * iso3166Codes=[US-VA, US-CA, US-OR, BR-SP, IE, DE-HE, SG, AU-NSW, JP-13]}]. The exception is
- * thrown by org.jclouds.compute.domain.internal.TemplateBuilderImpl#locationId(String).
- */
- if (Character.isLetter(region.charAt(region.length() - 1))) {
- region = region.substring(0, region.length() - 1);
- }
- properties.setProperty(LocationConstants.PROPERTY_REGIONS, region);
- }
-
- // occasionally can get com.google.common.util.concurrent.UncheckedExecutionException: java.lang.RuntimeException:
- // security group eu-central-1/jclouds#brooklyn-bxza-alex-eu-central-shoul-u2jy-nginx-ielm is not available after creating
- // the default timeout was 500ms so let's raise it in case that helps
- properties.setProperty(EC2Constants.PROPERTY_EC2_TIMEOUT_SECURITYGROUP_PRESENT, "" + Duration.seconds(30).toMilliseconds());
- return this;
- }
-
- private PropertiesBuilder setAzureComputeArmProperties() {
- String region = conf.get(CLOUD_REGION_ID);
- if (Strings.isNonBlank(region)) {
- properties.setProperty(LocationConstants.PROPERTY_REGIONS, region);
- }
- return this;
- }
-
- private PropertiesBuilder setJCloudsProperties() {
- // Add extra jclouds-specific configuration
- Map<String, Object> extra = Maps.filterKeys(conf.getAllConfig(), Predicates.containsPattern("^jclouds\\."));
- if (extra.size() > 0) {
- String provider = getProviderFromConfig(conf);
- LOG.debug("Configuring custom jclouds property overrides for {}: {}", provider, Sanitizer.sanitize(extra));
- }
- properties.putAll(Maps.filterValues(extra, Predicates.notNull()));
- return this;
- }
-
- private PropertiesBuilder setEndpointProperty() {
- String endpoint = conf.get(CloudLocationConfig.CLOUD_ENDPOINT);
- if (!groovyTruth(endpoint)) endpoint = getDeprecatedProperty(conf, Constants.PROPERTY_ENDPOINT);
- if (groovyTruth(endpoint)) properties.setProperty(Constants.PROPERTY_ENDPOINT, endpoint);
- return this;
- }
-
- public Properties build() {
- return properties;
- }
- }
-
public class ComputeServiceSupplier implements Supplier<ComputeService> {
private final String provider;
@@ -266,15 +143,6 @@ public abstract class AbstractComputeServiceRegistry implements ComputeServiceRe
new BouncyCastleCryptoModule());
}
- protected String getDeprecatedProperty(ConfigBag conf, String key) {
- if (conf.containsKey(key)) {
- LOG.warn("Jclouds using deprecated brooklyn-jclouds property " + key + ": " + Sanitizer.sanitize(conf.getAllConfig()));
- return (String) conf.getStringKey(key);
- } else {
- return null;
- }
- }
-
@Override
public String toString() {
return getClass().getName();
http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/14e844d8/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/AwsEc2SessionAwareComputeServiceRegistry.java
----------------------------------------------------------------------
diff --git a/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/AwsEc2SessionAwareComputeServiceRegistry.java b/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/AwsEc2SessionAwareComputeServiceRegistry.java
index 1b3c8fe..436b823 100644
--- a/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/AwsEc2SessionAwareComputeServiceRegistry.java
+++ b/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/AwsEc2SessionAwareComputeServiceRegistry.java
@@ -36,6 +36,20 @@ import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Supplier;
+/**
+ * A ComputeServiceRegistry that can infer credentials from the AWS EC2 instance profile.
+ *
+ * This only works if Brooklyn is running on an EC2 instance that has an IAM Role attached with e.g the AmazonEC2FullAccess
+ * policy set.
+ *
+ * usage:
+ *
+ * {@code
+ * jclouds.computeServiceRegistry:
+ * $brooklyn:object:
+ * type: org.apache.brooklyn.location.jclouds.AwsEc2SessionAwareComputeServiceRegistry
+ * }
+ */
public class AwsEc2SessionAwareComputeServiceRegistry extends AbstractComputeServiceRegistry implements ComputeServiceRegistry, AwsEc2SessionAwareLocationConfig {
public static final String ACCESS_KEY_ID = "AccessKeyId";
@@ -45,7 +59,9 @@ public class AwsEc2SessionAwareComputeServiceRegistry extends AbstractComputeSer
public static final String AWS_SECURITY_CREDENTIAL_URL = "http://169.254.169.254/latest/meta-data/iam/security-credentials";
public static final String AWS_EXPIRATION_DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ss'Z'";
- public AwsEc2SessionAwareComputeServiceRegistry(){}
+ public AwsEc2SessionAwareComputeServiceRegistry(){
+ // empty constructor required
+ }
@Override
public ComputeService findComputeService(ConfigBag conf, boolean allowReuse) {
http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/14e844d8/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/JCloudsPropertiesBuilder.java
----------------------------------------------------------------------
diff --git a/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/JCloudsPropertiesBuilder.java b/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/JCloudsPropertiesBuilder.java
new file mode 100644
index 0000000..8178134
--- /dev/null
+++ b/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/JCloudsPropertiesBuilder.java
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.brooklyn.location.jclouds;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static org.apache.brooklyn.util.JavaGroovyEquivalents.groovyTruth;
+import static org.jclouds.aws.ec2.reference.AWSEC2Constants.PROPERTY_EC2_AMI_QUERY;
+import static org.jclouds.aws.ec2.reference.AWSEC2Constants.PROPERTY_EC2_CC_AMI_QUERY;
+
+import java.util.Map;
+import java.util.Properties;
+
+import org.apache.brooklyn.core.config.Sanitizer;
+import org.apache.brooklyn.core.location.cloud.CloudLocationConfig;
+import org.apache.brooklyn.core.mgmt.persist.DeserializingJcloudsRenamesProvider;
+import org.apache.brooklyn.util.core.config.ConfigBag;
+import org.apache.brooklyn.util.text.Strings;
+import org.apache.brooklyn.util.time.Duration;
+import org.jclouds.Constants;
+import org.jclouds.ec2.reference.EC2Constants;
+import org.jclouds.location.reference.LocationConstants;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Predicates;
+import com.google.common.collect.Maps;
+
+public class JCloudsPropertiesBuilder implements JcloudsLocationConfig{
+ private ConfigBag conf;
+ private Properties properties = new Properties();
+
+ private static final Logger LOG = LoggerFactory.getLogger(JCloudsPropertiesBuilder.class);
+
+ public JCloudsPropertiesBuilder(ConfigBag conf) {
+ this.conf = conf;
+ }
+
+ public JCloudsPropertiesBuilder setCommonJcloudsProperties() {
+ properties.setProperty(Constants.PROPERTY_TRUST_ALL_CERTS, Boolean.toString(true));
+ properties.setProperty(Constants.PROPERTY_RELAX_HOSTNAME, Boolean.toString(true));
+ properties.setProperty("jclouds.ssh.max-retries", conf.getStringKey("jclouds.ssh.max-retries") != null ?
+ conf.getStringKey("jclouds.ssh.max-retries").toString() : "50");
+
+ if (conf.get(OAUTH_ENDPOINT) != null)
+ properties.setProperty(OAUTH_ENDPOINT.getName(), conf.get(OAUTH_ENDPOINT));
+
+ // See https://issues.apache.org/jira/browse/BROOKLYN-394
+ // For retries, the backoff times are:
+ // Math.min(2^failureCount * retryDelayStart, retryDelayStart * 10) + random(10%)
+ // Therefore the backoff times will be: 500ms, 1s, 2s, 4s, 5s, 5s.
+ // The defaults (if not overridden here) are 50ms and 5 retires. This gives backoff
+ // times of 50ms, 100ms, 200ms, 400ms, 500ms (so a total backoff time of 1.25s),
+ // which is not long when you're being rate-limited and there are multiple thread all
+ // retrying their API calls.
+ properties.setProperty(Constants.PROPERTY_RETRY_DELAY_START, "500");
+ properties.setProperty(Constants.PROPERTY_MAX_RETRIES, "6");
+ return this;
+ }
+
+ public JCloudsPropertiesBuilder setAWSEC2Properties() {
+ // TODO convert AWS-only flags to config keys
+ if (groovyTruth(conf.get(IMAGE_ID))) {
+ properties.setProperty(PROPERTY_EC2_AMI_QUERY, "");
+ properties.setProperty(PROPERTY_EC2_CC_AMI_QUERY, "");
+ } else if (groovyTruth(conf.getStringKey("imageOwner"))) {
+ properties.setProperty(PROPERTY_EC2_AMI_QUERY, "owner-id=" + conf.getStringKey("imageOwner") + ";state=available;image-type=machine");
+ } else if (groovyTruth(conf.getStringKey("anyOwner"))) {
+ // set `anyOwner: true` to override the default query (which is restricted to certain owners as per below),
+ // allowing the AMI query to bind to any machine
+ // (note however, we sometimes pick defaults in JcloudsLocationFactory);
+ // (and be careful, this can give a LOT of data back, taking several minutes,
+ // and requiring extra memory allocated on the command-line)
+ properties.setProperty(PROPERTY_EC2_AMI_QUERY, "state=available;image-type=machine");
+ /*
+ * by default the following filters are applied:
+ * Filter.1.Name=owner-id&Filter.1.Value.1=137112412989&
+ * Filter.1.Value.2=063491364108&
+ * Filter.1.Value.3=099720109477&
+ * Filter.1.Value.4=411009282317&
+ * Filter.2.Name=state&Filter.2.Value.1=available&
+ * Filter.3.Name=image-type&Filter.3.Value.1=machine&
+ */
+ }
+
+ // See https://issues.apache.org/jira/browse/BROOKLYN-399
+ String region = conf.get(CLOUD_REGION_ID);
+ if (Strings.isNonBlank(region)) {
+ /*
+ * Drop availability zone suffixes. Without this deployments to regions like us-east-1b fail
+ * because jclouds throws an IllegalStateException complaining that: location id us-east-1b
+ * not found in: [{scope=PROVIDER, id=aws-ec2, description=https://ec2.us-east-1.amazonaws.com,
+ * iso3166Codes=[US-VA, US-CA, US-OR, BR-SP, IE, DE-HE, SG, AU-NSW, JP-13]}]. The exception is
+ * thrown by org.jclouds.compute.domain.internal.TemplateBuilderImpl#locationId(String).
+ */
+ if (Character.isLetter(region.charAt(region.length() - 1))) {
+ region = region.substring(0, region.length() - 1);
+ }
+ properties.setProperty(LocationConstants.PROPERTY_REGIONS, region);
+ }
+
+ // occasionally can get com.google.common.util.concurrent.UncheckedExecutionException: java.lang.RuntimeException:
+ // security group eu-central-1/jclouds#brooklyn-bxza-alex-eu-central-shoul-u2jy-nginx-ielm is not available after creating
+ // the default timeout was 500ms so let's raise it in case that helps
+ properties.setProperty(EC2Constants.PROPERTY_EC2_TIMEOUT_SECURITYGROUP_PRESENT, "" + Duration.seconds(30).toMilliseconds());
+ return this;
+ }
+
+ public JCloudsPropertiesBuilder setAzureComputeArmProperties() {
+ String region = conf.get(CloudLocationConfig.CLOUD_REGION_ID);
+ if (Strings.isNonBlank(region)) {
+ properties.setProperty(LocationConstants.PROPERTY_REGIONS, region);
+ }
+ return this;
+ }
+
+ public JCloudsPropertiesBuilder setCustomJcloudsProperties() {
+ Map<String, Object> extra = Maps.filterKeys(conf.getAllConfig(), Predicates.containsPattern("^jclouds\\."));
+ if (extra.size() > 0) {
+ String provider = getProviderFromConfig(conf);
+ LOG.debug("Configuring custom jclouds property overrides for {}: {}", provider, Sanitizer.sanitize(extra));
+ }
+ properties.putAll(Maps.filterValues(extra, Predicates.notNull()));
+ return this;
+ }
+
+ public JCloudsPropertiesBuilder setEndpointProperty() {
+ String endpoint = conf.get(CloudLocationConfig.CLOUD_ENDPOINT);
+ if (!groovyTruth(endpoint)) endpoint = getDeprecatedProperty(conf, Constants.PROPERTY_ENDPOINT);
+ if (groovyTruth(endpoint)) properties.setProperty(Constants.PROPERTY_ENDPOINT, endpoint);
+ return this;
+ }
+
+ public Properties build() {
+ return properties;
+ }
+
+ private String getDeprecatedProperty(ConfigBag conf, String key) {
+ if (conf.containsKey(key)) {
+ LOG.warn("Jclouds using deprecated brooklyn-jclouds property " + key + ": " + Sanitizer.sanitize(conf.getAllConfig()));
+ return (String) conf.getStringKey(key);
+ } else {
+ return null;
+ }
+ }
+
+ private String getProviderFromConfig(ConfigBag conf) {
+ String rawProvider = checkNotNull(conf.get(CLOUD_PROVIDER), "provider must not be null");
+ return DeserializingJcloudsRenamesProvider.INSTANCE.applyJcloudsRenames(rawProvider);
+ }
+}