You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@brooklyn.apache.org by ge...@apache.org on 2017/02/24 15:18:15 UTC

[4/5] brooklyn-server git commit: LocationNetworkInfoCustomizer

LocationNetworkInfoCustomizer

LocationNetworkInfoCustomizer is a customiser that gives users much more
control over which addresses and credentials are used when connecting to
instances provisioned by JcloudsLocation.


Project: http://git-wip-us.apache.org/repos/asf/brooklyn-server/repo
Commit: http://git-wip-us.apache.org/repos/asf/brooklyn-server/commit/3fd64c89
Tree: http://git-wip-us.apache.org/repos/asf/brooklyn-server/tree/3fd64c89
Diff: http://git-wip-us.apache.org/repos/asf/brooklyn-server/diff/3fd64c89

Branch: refs/heads/master
Commit: 3fd64c8992c5f561b591ea0239ceabbc5ae2719d
Parents: 1a6fd6a
Author: Sam Corbett <sa...@cloudsoftcorp.com>
Authored: Tue Jan 24 18:00:54 2017 +0000
Committer: Sam Corbett <sa...@cloudsoftcorp.com>
Committed: Fri Feb 24 14:32:32 2017 +0000

----------------------------------------------------------------------
 .../location/cloud/CloudLocationConfig.java     |  23 +-
 .../brooklyn/util/core/task/BasicTask.java      |   4 +-
 .../location/jclouds/ConnectivityResolver.java  |  55 +++
 .../jclouds/ConnectivityResolverOptions.java    | 245 ++++++++++
 .../jclouds/DefaultConnectivityResolver.java    | 490 +++++++++++++++++++
 .../location/jclouds/JcloudsLocation.java       | 381 ++++++--------
 .../location/jclouds/JcloudsLocationConfig.java |   4 +
 .../brooklyn/location/jclouds/JcloudsUtil.java  |   1 -
 .../jclouds/ManagementAddressResolveResult.java |  54 ++
 .../jclouds/AbstractJcloudsStubbedLiveTest.java |   3 +-
 ...BasicLocationNetworkInfoInitializerTest.java |  44 ++
 .../DefaultConnectivityResolverTest.java        | 262 ++++++++++
 ...dsByonLocationResolverStubbedRebindTest.java |   2 +
 .../JcloudsReachableAddressStubbedTest.java     |  69 ++-
 .../MachineLifecycleEffectorTasks.java          |  16 +-
 .../brooklyn/entity/AbstractEc2LiveTest.java    |  17 -
 16 files changed, 1407 insertions(+), 263 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/3fd64c89/core/src/main/java/org/apache/brooklyn/core/location/cloud/CloudLocationConfig.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/brooklyn/core/location/cloud/CloudLocationConfig.java b/core/src/main/java/org/apache/brooklyn/core/location/cloud/CloudLocationConfig.java
index baa6565..d5051ea 100644
--- a/core/src/main/java/org/apache/brooklyn/core/location/cloud/CloudLocationConfig.java
+++ b/core/src/main/java/org/apache/brooklyn/core/location/cloud/CloudLocationConfig.java
@@ -82,16 +82,17 @@ public interface CloudLocationConfig {
         "vmNameSaltLength", "Number of characters to use for a random identifier inserted in hostname "
             + "to uniquely identify machines", 4);
 
-    public static final ConfigKey<String> POLL_FOR_FIRST_REACHABLE_ADDRESS = ConfigKeys.newStringConfigKey("pollForFirstReachableAddress", 
-            "Whether and how long to wait for reaching the VM's ip:port; "
-            + "if 'false', will default to the node's first public IP (or private if no public IPs); "
-            + "if 'true' uses default duration; otherwise accepts a time string e.g. '5m' (the default) or a number of milliseconds", "5m");
+    ConfigKey<String> POLL_FOR_FIRST_REACHABLE_ADDRESS = ConfigKeys.newStringConfigKey("pollForFirstReachableAddress",
+            "Whether and how long to wait for reaching the VM's ip:port to be accessible over SSH or WinRM; "
+            + "if 'false', the location will will choose a public or private IP as appropriate; "
+            + "if 'true' uses default duration; otherwise accepts a time string e.g. '5m' (the default) or a number of milliseconds",
+            "5m");
 
     @SuppressWarnings("serial")
     ConfigKey<Predicate<? super HostAndPort>> POLL_FOR_FIRST_REACHABLE_ADDRESS_PREDICATE = ConfigKeys.newConfigKey(
             new TypeToken<Predicate<? super HostAndPort>>(){}, 
             "pollForFirstReachableAddress.predicate",
-            "Predicate<HostAndPort> implementation which checks whether machine is up or not.");
+            "Predicate<HostAndPort> implementation which checks whether an ip:port is reachable.");
 
     @SuppressWarnings("serial")
     ConfigKey<Class<? extends Predicate<? super HostAndPort>>> POLL_FOR_FIRST_REACHABLE_ADDRESS_PREDICATE_TYPE = ConfigKeys.newConfigKey(
@@ -101,15 +102,19 @@ public interface CloudLocationConfig {
                     "Other keys prefixed with pollForFirstReachableAddress.predicate.<property> will be passed to the Map constructor of the Predicate<HostAndPort> implementation.",
             Networking.IsReachablePredicate.class);
 
-    public static final ConfigKey<String> WAIT_FOR_SSHABLE = ConfigKeys.newStringConfigKey("waitForSshable",
+    /** @deprecated since 0.11.0 use {@link #POLL_FOR_FIRST_REACHABLE_ADDRESS} instead. */
+    @Deprecated
+    ConfigKey<String> WAIT_FOR_SSHABLE = ConfigKeys.newStringConfigKey("waitForSshable",
             "Whether and how long to wait for a newly provisioned VM to be accessible via ssh; " +
             "if 'false', won't check; if 'true' uses default duration; otherwise accepts a time string e.g. '5m' (the default) or a number of milliseconds", "5m");
 
-    public static final ConfigKey<String> WAIT_FOR_WINRM_AVAILABLE = ConfigKeys.newStringConfigKey("waitForWinRmAvailable",
+    /** @deprecated since 0.11.0 use {@link #POLL_FOR_FIRST_REACHABLE_ADDRESS} instead. */
+    @Deprecated
+    ConfigKey<String> WAIT_FOR_WINRM_AVAILABLE = ConfigKeys.newStringConfigKey("waitForWinRmAvailable",
             "Whether and how long to wait for a newly provisioned VM to be accessible via WinRm; " +
-                    "if 'false', won't check; if 'true' uses default duration; otherwise accepts a time string e.g. '30m' (the default) or a number of milliseconds", "30m");
+            "if 'false', won't check; if 'true' uses default duration; otherwise accepts a time string e.g. '30m' (the default) or a number of milliseconds", "30m");
 
-    public static final ConfigKey<Boolean> LOG_CREDENTIALS = ConfigKeys.newBooleanConfigKey(
+    ConfigKey<Boolean> LOG_CREDENTIALS = ConfigKeys.newBooleanConfigKey(
             "logCredentials", 
             "Whether to log credentials of a new VM - strongly recommended never be used in production, as it is a big security hole!",
             false);

http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/3fd64c89/core/src/main/java/org/apache/brooklyn/util/core/task/BasicTask.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/brooklyn/util/core/task/BasicTask.java b/core/src/main/java/org/apache/brooklyn/util/core/task/BasicTask.java
index 14d15c0..583f404 100644
--- a/core/src/main/java/org/apache/brooklyn/util/core/task/BasicTask.java
+++ b/core/src/main/java/org/apache/brooklyn/util/core/task/BasicTask.java
@@ -377,7 +377,9 @@ public class BasicTask<T> implements TaskInternal<T> {
     public synchronized void blockUntilStarted() {
         blockUntilStarted(null);
     }
-    
+
+    // TODO: This should log a message if timeout is null and the method blocks for an unreasonably long time -
+    // it probably means someone called .get() and forgot to submit the task.
     @Override
     public synchronized boolean blockUntilStarted(Duration timeout) {
         Long endTime = timeout==null ? null : System.currentTimeMillis() + timeout.toMillisecondsRoundingUp();

http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/3fd64c89/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/ConnectivityResolver.java
----------------------------------------------------------------------
diff --git a/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/ConnectivityResolver.java b/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/ConnectivityResolver.java
new file mode 100644
index 0000000..30ae603
--- /dev/null
+++ b/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/ConnectivityResolver.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.brooklyn.location.jclouds;
+
+import org.apache.brooklyn.api.sensor.AttributeSensor;
+import org.apache.brooklyn.core.sensor.Sensors;
+import org.apache.brooklyn.util.core.config.ConfigBag;
+import org.jclouds.compute.domain.NodeMetadata;
+
+import com.google.common.annotations.Beta;
+import com.google.common.reflect.TypeToken;
+
+/**
+ * LocationNetworkInfoCustomizers are used by {@link JcloudsLocation} to determine the host and port
+ * on which connections to locations should be made and the credentials that should be used.
+ *
+ * @see JcloudsLocationConfig#CONNECTIVITY_RESOLVER
+ */
+@Beta
+public interface ConnectivityResolver {
+
+    AttributeSensor<Iterable<String>> PUBLIC_ADDRESSES = Sensors.newSensor(new TypeToken<Iterable<String>>() {},
+            "host.addresses.public", "Public addresses on an instance");
+
+    AttributeSensor<Iterable<String>> PRIVATE_ADDRESSES = Sensors.newSensor(new TypeToken<Iterable<String>>() {},
+            "host.addresses.private", "Private addresses on an instance");
+
+    /**
+     * @param location       The caller
+     * @param node           The node the caller has created
+     * @param config         The configuration the caller used to create the node
+     * @param resolveOptions Additional options the caller has chosen when creating the node
+     * @return The HostAndPort and LoginCredentials to use when connecting to the node.
+     */
+    ManagementAddressResolveResult resolve(
+            JcloudsLocation location, NodeMetadata node, ConfigBag config, ConnectivityResolverOptions resolveOptions);
+
+}

http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/3fd64c89/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/ConnectivityResolverOptions.java
----------------------------------------------------------------------
diff --git a/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/ConnectivityResolverOptions.java b/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/ConnectivityResolverOptions.java
new file mode 100644
index 0000000..9af1e60
--- /dev/null
+++ b/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/ConnectivityResolverOptions.java
@@ -0,0 +1,245 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.brooklyn.location.jclouds;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+import org.apache.brooklyn.util.time.Duration;
+import org.jclouds.domain.LoginCredentials;
+
+import com.google.common.annotations.Beta;
+import com.google.common.base.Optional;
+import com.google.common.base.Predicate;
+import com.google.common.net.HostAndPort;
+
+/**
+ * Holds parameters to be used by a {@link ConnectivityResolver}.
+ */
+@Beta
+public class ConnectivityResolverOptions {
+
+    public static Builder builder() {
+        return new Builder();
+    }
+
+    public static class Builder {
+        private boolean isWindows = false;
+
+        private boolean waitForConnectable;
+        private boolean pollForReachableAddresses;
+        private Predicate<? super HostAndPort> reachableAddressPredicate;
+        private Duration reachableAddressTimeout;
+        private boolean propagatePollForReachableFailure;
+
+        private LoginCredentials initialCredentials;
+        private LoginCredentials userCredentials;
+
+        private int defaultLoginPort;
+        private boolean usePortForwarding;
+        private HostAndPort portForwardSshOverride;
+        private boolean isRebinding;
+        private boolean skipJcloudsSshing;
+
+        public Builder pollForReachableAddresses(
+                @Nonnull Predicate<? super HostAndPort> reachable,
+                @Nonnull Duration timeout,
+                boolean propagatePollFailure) {
+            this.pollForReachableAddresses = true;
+            this.reachableAddressPredicate = reachable;
+            this.reachableAddressTimeout = timeout;
+            this.propagatePollForReachableFailure = propagatePollFailure;
+            return this;
+        }
+
+        public Builder noPollForReachableAddresses() {
+            this.pollForReachableAddresses = false;
+            this.reachableAddressPredicate = null;
+            this.reachableAddressTimeout = null;
+            this.propagatePollForReachableFailure = false;
+            return this;
+        }
+
+        public Builder initialCredentials(@Nullable LoginCredentials initialCredentials) {
+            this.initialCredentials = initialCredentials;
+            return this;
+        }
+
+        public Builder userCredentials(@Nullable LoginCredentials userCredentials) {
+            this.userCredentials = userCredentials;
+            return this;
+        }
+
+        public Builder isWindows(boolean windows) {
+            isWindows = windows;
+            return this;
+        }
+
+        /** Indicate the host and port that should be used over all others. Normally used in tandem with a port forwarder. */
+        public Builder portForwardSshOverride(@Nullable HostAndPort hostAndPortOverride) {
+            this.portForwardSshOverride = hostAndPortOverride;
+            return this;
+        }
+
+        public Builder isRebinding(boolean isRebinding) {
+            this.isRebinding = isRebinding;
+            return this;
+        }
+
+        public Builder skipJcloudsSshing(boolean skipJcloudsSshing) {
+            this.skipJcloudsSshing = skipJcloudsSshing;
+            return this;
+        }
+
+        public Builder defaultLoginPort(int defaultLoginPort) {
+            this.defaultLoginPort = defaultLoginPort;
+            return this;
+        }
+
+        public Builder usePortForwarding(boolean usePortForwarding) {
+            this.usePortForwarding = usePortForwarding;
+            return this;
+        }
+
+        public Builder waitForConnectable(boolean waitForConnectable) {
+            this.waitForConnectable = waitForConnectable;
+            return this;
+        }
+
+        public ConnectivityResolverOptions build() {
+            return new ConnectivityResolverOptions(
+                    isWindows, waitForConnectable, pollForReachableAddresses, reachableAddressPredicate,
+                    propagatePollForReachableFailure, reachableAddressTimeout, initialCredentials, userCredentials,
+                    defaultLoginPort, usePortForwarding, portForwardSshOverride, isRebinding, skipJcloudsSshing);
+        }
+    }
+
+    private final boolean isWindows;
+
+    /** Wait for Windows machines to be available over WinRM and other machines over SSH */
+    // TODO: Merge this with pollForReachable when waitForSshable and waitForWinRmable deleted.
+    private final boolean waitForConnectable;
+
+    /** Wait for a machine's ip:port to be available. */
+    private final boolean pollForReachableAddresses;
+    private final Predicate<? super HostAndPort> reachableAddressPredicate;
+    private final boolean propagatePollForReachableFailure;
+    private final Duration reachableAddressTimeout;
+
+    private final LoginCredentials initialCredentials;
+    private final LoginCredentials userCredentials;
+    private final int defaultLoginPort;
+
+    // TODO: Can usePortForwarding and portForwardSshOverride be merged?
+    private final boolean usePortForwarding;
+    private final HostAndPort portForwardSshOverride;
+    private final boolean isRebinding;
+    private final boolean skipJcloudsSshing;
+
+
+    protected ConnectivityResolverOptions(boolean isWindows, boolean waitForConnectable, boolean pollForReachableAddresses, Predicate<? super HostAndPort> reachableAddressPredicate, boolean propagatePollForReachableFailure, Duration reachableAddressTimeout, LoginCredentials initialCredentials, LoginCredentials userCredentials, int defaultLoginPort, boolean usePortForwarding, HostAndPort portForwardSshOverride, boolean isRebinding, boolean skipJcloudsSshing) {
+        this.isWindows = isWindows;
+        this.waitForConnectable = waitForConnectable;
+        this.pollForReachableAddresses = pollForReachableAddresses;
+        this.reachableAddressPredicate = reachableAddressPredicate;
+        this.propagatePollForReachableFailure = propagatePollForReachableFailure;
+        this.reachableAddressTimeout = reachableAddressTimeout;
+        this.initialCredentials = initialCredentials;
+        this.userCredentials = userCredentials;
+        this.defaultLoginPort = defaultLoginPort;
+        this.usePortForwarding = usePortForwarding;
+        this.portForwardSshOverride = portForwardSshOverride;
+        this.isRebinding = isRebinding;
+        this.skipJcloudsSshing = skipJcloudsSshing;
+    }
+
+    public boolean isWindows() {
+        return isWindows;
+    }
+
+    public boolean waitForConnectable() {
+        return waitForConnectable;
+    }
+
+    public boolean pollForReachableAddresses() {
+        return pollForReachableAddresses;
+    }
+
+    public Predicate<? super HostAndPort> reachableAddressPredicate() {
+        return reachableAddressPredicate;
+    }
+
+    public Duration reachableAddressTimeout() {
+        return reachableAddressTimeout;
+    }
+
+    public boolean propagatePollForReachableFailure() {
+        return propagatePollForReachableFailure;
+    }
+
+    public Optional<LoginCredentials> initialCredentials() {
+        return Optional.fromNullable(initialCredentials);
+    }
+
+    public Optional<LoginCredentials> userCredentials() {
+        return Optional.fromNullable(userCredentials);
+    }
+
+    public boolean usePortForwarding() {
+        return usePortForwarding;
+    }
+
+    public Optional<HostAndPort> portForwardSshOverride() {
+        return Optional.fromNullable(portForwardSshOverride);
+    }
+
+    public int defaultLoginPort() {
+        return defaultLoginPort;
+    }
+
+    public boolean skipJcloudsSshing() {
+        return skipJcloudsSshing;
+    }
+
+    public boolean isRebinding() {
+        return isRebinding;
+    }
+
+    public Builder toBuilder() {
+        Builder builder = builder()
+                .isWindows(isWindows)
+                .waitForConnectable(waitForConnectable)
+                .usePortForwarding(usePortForwarding)
+                .portForwardSshOverride(portForwardSshOverride)
+                .skipJcloudsSshing(skipJcloudsSshing)
+                .initialCredentials(initialCredentials)
+                .userCredentials(userCredentials)
+                .isRebinding(isRebinding)
+                .defaultLoginPort(defaultLoginPort)
+                ;
+        if (pollForReachableAddresses) {
+            builder.pollForReachableAddresses(reachableAddressPredicate, reachableAddressTimeout, propagatePollForReachableFailure);
+        } else {
+            builder.noPollForReachableAddresses();
+        }
+        return builder;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/3fd64c89/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/DefaultConnectivityResolver.java
----------------------------------------------------------------------
diff --git a/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/DefaultConnectivityResolver.java b/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/DefaultConnectivityResolver.java
new file mode 100644
index 0000000..b7c90eb
--- /dev/null
+++ b/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/DefaultConnectivityResolver.java
@@ -0,0 +1,490 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.brooklyn.location.jclouds;
+
+import java.util.Iterator;
+import java.util.Map;
+import javax.annotation.Nullable;
+
+import org.apache.brooklyn.api.entity.Entity;
+import org.apache.brooklyn.api.entity.EntityInitializer;
+import org.apache.brooklyn.api.entity.EntityLocal;
+import org.apache.brooklyn.config.ConfigKey;
+import org.apache.brooklyn.core.config.ConfigKeys;
+import org.apache.brooklyn.core.entity.Attributes;
+import org.apache.brooklyn.core.entity.BrooklynConfigKeys;
+import org.apache.brooklyn.core.location.LocationConfigKeys;
+import org.apache.brooklyn.core.mgmt.BrooklynTaskTags;
+import org.apache.brooklyn.core.objs.BasicConfigurableObject;
+import org.apache.brooklyn.util.core.config.ConfigBag;
+import org.apache.brooklyn.util.core.task.Tasks;
+import org.apache.brooklyn.util.exceptions.Exceptions;
+import org.apache.brooklyn.util.net.Networking;
+import org.apache.brooklyn.util.time.Duration;
+import org.jclouds.compute.domain.NodeMetadata;
+import org.jclouds.domain.LoginCredentials;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.annotations.Beta;
+import com.google.common.base.MoreObjects;
+import com.google.common.base.Optional;
+import com.google.common.base.Predicate;
+import com.google.common.base.Stopwatch;
+import com.google.common.base.Supplier;
+import com.google.common.collect.FluentIterable;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.net.HostAndPort;
+
+/**
+ * DefaultConnectivityResolver provides the default implementation of
+ * {@link ConnectivityResolver}. It exposes options to have JcloudsLocation
+ * prefer to contact VMs on private addresses and can be injected on a
+ * per-entity basis. For example:
+ * <pre>
+ * services:
+ * - type: server
+ *   location: the-same-private-network-as-brooklyn
+ *   brooklyn.initializers:
+ *   - type: org.apache.brooklyn.location.jclouds.DefaultConnectivityResolver
+ *     brooklyn.config:
+ *       mode: ONLY_PRIVATE
+ * - type: server
+ *   location: another-cloud
+ *   # implicit use of PREFER_PUBLIC.
+ * </pre>
+ * Would result in the first entity being managed on the instance's private address (and deployment
+ * failing if this was not possible) and the second being managed on its public address. Graceful
+ * fallback is possible by replacing ONLY_PRIVATE with PREFER_PRIVATE. There are PUBLIC variants of
+ * each of these.
+ * <p>
+ * DefaultConnectivityResolver is the default location network info customizer used by
+ * {@link JcloudsLocation} when {@link JcloudsLocationConfig#CONNECTIVITY_RESOLVER}
+ * is unset.
+ * <p>
+ * When used as an {@link EntityInitializer} the instance inserts itself into the entity's
+ * provisioning properties under the {@link JcloudsLocationConfig#CONNECTIVITY_RESOLVER}
+ * subkey.
+ * <p>
+ * This class is annotated @Beta and is likely to change in the future.
+ */
+@Beta
+public class DefaultConnectivityResolver extends BasicConfigurableObject implements ConnectivityResolver, EntityInitializer {
+
+    private static final Logger LOG = LoggerFactory.getLogger(DefaultConnectivityResolver.class);
+
+    public enum NetworkMode {
+        /**
+         * Check each node's {@link NodeMetadata#getPublicAddresses() public addresses}
+         * for reachability before its {@link NodeMetadata#getPrivateAddresses() private addresses}.
+         */
+        PREFER_PUBLIC,
+        /**
+         * Check each node's {@link NodeMetadata#getPrivateAddresses() private addresses}
+         * for reachability before its {@link NodeMetadata#getPublicAddresses() public addresses}.
+         */
+        PREFER_PRIVATE,
+        /**
+         * Check only a node's {@link NodeMetadata#getPublicAddresses() public addresses} for reachability.
+         */
+        ONLY_PUBLIC,
+        /**
+         * Check only a node's {@link NodeMetadata#getPrivateAddresses()}  private addresses} for reachability.
+         */
+        ONLY_PRIVATE
+    }
+
+    public static final ConfigKey<NetworkMode> NETWORK_MODE = ConfigKeys.newConfigKey(NetworkMode.class,
+            "mode", "Operation mode: PREFER_PUBLIC, PREFER_PRIVATE, ONLY_PUBLIC or ONLY_PRIVATE");
+
+    @Beta
+    public static final ConfigKey<Boolean> CHECK_CREDENTIALS = ConfigKeys.newBooleanConfigKey(
+            "checkCredentials",
+            "Indicates that credentials should be tested when determining endpoint reachability.",
+            Boolean.TRUE);
+
+    public static final ConfigKey<Boolean> PUBLISH_NETWORKS = ConfigKeys.newBooleanConfigKey(
+            "publishNetworks",
+            "Indicates that the customizer should publish addresses as sensors on each entity",
+            Boolean.TRUE);
+
+    // --------------------------------------------------------------------------------------
+
+    public DefaultConnectivityResolver() {
+        this(ImmutableMap.of());
+    }
+
+    public DefaultConnectivityResolver(Map<?, ?> params) {
+        this(ConfigBag.newInstance(params));
+    }
+
+    public DefaultConnectivityResolver(final ConfigBag params) {
+        for (Map.Entry<String, Object> entry : params.getAllConfig().entrySet()) {
+            config().set(ConfigKeys.newConfigKey(Object.class, entry.getKey()), entry.getValue());
+        }
+    }
+
+    // --------------------------------------------------------------------------------------
+
+    /**
+     * Sets the instance as the value of {@link JcloudsLocationConfig#CONNECTIVITY_RESOLVER}
+     * in entity's provisioning properties.
+     */
+    @Override
+    public void apply(EntityLocal entity) {
+        final String sensorName = JcloudsLocationConfig.CONNECTIVITY_RESOLVER.getName();
+        ConfigKey<Object> subkey = BrooklynConfigKeys.PROVISIONING_PROPERTIES.subKey(sensorName);
+        entity.config().set(subkey, this);
+        LOG.debug("{} set itself as the {} on {}", new Object[]{this, sensorName, entity});
+    }
+
+    // --------------------------------------------------------------------------------------
+
+    /**
+     * Combines the given resolve options with the customiser's configuration to determine the
+     * best address and credential pair for management. In particular, if the resolve options
+     * allow it will check that the credential is actually valid for the address.
+     */
+    @Override
+    public ManagementAddressResolveResult resolve(
+            JcloudsLocation location, NodeMetadata node, ConfigBag config, ConnectivityResolverOptions options) {
+        LOG.debug("{} resolving management parameters for {}, node={}, config={}, options={}",
+                new Object[]{this, location, node, config, options});
+        final Stopwatch timer = Stopwatch.createStarted();
+        // Should only be null in tests.
+        final Entity contextEntity = getContextEntity(config);
+        if (shouldPublishNetworks() && !options.isRebinding() && contextEntity != null) {
+            publishNetworks(node, contextEntity);
+        }
+        HostAndPort hapChoice = null;
+        LoginCredentials credChoice = null;
+
+        final Iterable<HostAndPort> managementCandidates = getManagementCandidates(location, node, config, options);
+        final Iterable<LoginCredentials> credentialCandidates = getCredentialCandidates(location, node, options, config);
+
+        // Try each pair of address and credential until one succeeds.
+        if (shouldCheckCredentials() && options.pollForReachableAddresses()) {
+            for (HostAndPort hap : managementCandidates) {
+                for (LoginCredentials cred : credentialCandidates) {
+                    LOG.trace("Testing host={} with credential={}", hap, cred);
+                    if (checkCredential(location, hap, cred, config, options.isWindows())) {
+                        hapChoice = hap;
+                        credChoice = cred;
+                        break;
+                    }
+                }
+                if (hapChoice != null) break;
+            }
+        } else if (shouldCheckCredentials()) {
+            LOG.debug("{} set on {} but pollForFirstReachableAddress={}",
+                    new Object[]{CHECK_CREDENTIALS.getName(), this, options.pollForReachableAddresses()});
+        }
+
+        if (hapChoice == null) {
+            LOG.trace("Choosing first management candidate given node={} and mode={}", node, getNetworkMode());
+            hapChoice = Iterables.getFirst(managementCandidates, null);
+        }
+        if (hapChoice == null) {
+            LOG.trace("Choosing first address of node={} in mode={}", node, getNetworkMode());
+            final Iterator<String> hit = getResolvableAddressesWithMode(node).iterator();
+            if (hit.hasNext()) HostAndPort.fromHost(hit.next());
+        }
+
+        if (hapChoice == null) {
+            throw new IllegalStateException("jclouds did not return any IP addresses matching " + getNetworkMode() + " in " + location);
+        }
+        if (credChoice == null) {
+            credChoice = Iterables.getFirst(credentialCandidates, null);
+            if (credChoice == null) {
+                throw new IllegalStateException("No credentials configured for " + location);
+            }
+        }
+
+        // Treat AWS as a special case because the DNS fully qualified hostname in AWS is
+        // (normally?!) a good way to refer to the VM from both inside and outside of the region.
+        if (!isNetworkModeSet() && !options.isWindows()) {
+            final boolean lookupAwsHostname = Boolean.TRUE.equals(config.get(JcloudsLocationConfig.LOOKUP_AWS_HOSTNAME));
+            String provider = config.get(JcloudsLocationConfig.CLOUD_PROVIDER);
+            if (provider == null) {
+                provider = location.getProvider();
+            }
+            if (options.waitForConnectable() && "aws-ec2".equals(provider) && lookupAwsHostname) {
+                // getHostnameAws sshes to the machine and curls 169.254.169.254/latest/meta-data/public-hostname.
+                try {
+                    LOG.debug("Resolving AWS hostname of {}", location);
+                    String result = location.getHostnameAws(hapChoice, credChoice, config);
+                    hapChoice = HostAndPort.fromParts(result, hapChoice.getPort());
+                    LOG.debug("Resolved AWS hostname of {}: {}", location, result);
+                } catch (Exception e) {
+                    LOG.debug("Failed to resolve AWS hostname of " + location, e);
+                }
+            }
+        }
+
+        if (contextEntity != null) {
+            contextEntity.sensors().set(Attributes.ADDRESS, hapChoice.getHostText());
+        }
+        ManagementAddressResolveResult result = new ManagementAddressResolveResult(hapChoice, credChoice);
+        LOG.debug("{} resolved management parameters for {} in {}: {}",
+                new Object[]{this, location, Duration.of(timer), result});
+        return result;
+    }
+
+    private boolean shouldPublishNetworks() {
+        return Boolean.TRUE.equals(config().get(PUBLISH_NETWORKS));
+    }
+
+    void publishNetworks(NodeMetadata node, Entity entity) {
+        if (entity.sensors().get(PRIVATE_ADDRESSES) == null) {
+            entity.sensors().set(PRIVATE_ADDRESSES, ImmutableSet.copyOf(node.getPrivateAddresses()));
+        }
+        if (entity.sensors().get(PUBLIC_ADDRESSES) == null) {
+            entity.sensors().set(PUBLIC_ADDRESSES, ImmutableSet.copyOf(node.getPublicAddresses()));
+        }
+    }
+
+    // --------------------------------------------------------------------------------------
+
+    /**
+     * Returns the hosts and ports that should be considered when determining the address
+     * to use when connecting to the location by assessing the following criteria:
+     * <ol>
+     *     <li>Use the hostAndPortOverride set in options.</li>
+     *     <li>If the machine is connectable, user credentials are given and the machine is provisioned
+     *     in AWS then use {@link JcloudsLocation#getHostnameAws(NodeMetadata, Optional, Supplier, ConfigBag)}.</li>
+     *     <li>If the machine is connectable and pollForFirstReachableAddress is set in options then use all
+     *     {@link #getReachableAddresses reachable} addresses.</li>
+     *     <li>Use the first address that is resolvable with {@link #isAddressResolvable}.</li>
+     *     <li>Use the first address in the node's public then private addresses.</li>
+     * </ol>
+     */
+    protected Iterable<HostAndPort> getManagementCandidates(
+            JcloudsLocation location, NodeMetadata node, ConfigBag config, ConnectivityResolverOptions options) {
+        final Optional<HostAndPort> portForwardSshOverride = options.portForwardSshOverride();
+
+        if (portForwardSshOverride.isPresent()) {
+            // Don't try to resolve it; just use it
+            int port = portForwardSshOverride.get().hasPort()
+                       ? portForwardSshOverride.get().getPort()
+                       : options.defaultLoginPort();
+            final HostAndPort override = HostAndPort.fromParts(portForwardSshOverride.get().getHostText(), port);
+            switch (getNetworkMode()) {
+            case ONLY_PRIVATE:
+                LOG.info("Ignoring mode {} in favour of port forwarding override for management candidates of {}: {}",
+                        new Object[]{NetworkMode.ONLY_PRIVATE.name(), location, override});
+                break;
+            default:
+                LOG.debug("Using host and port override for management candidates of {}: {}", location, override);
+            }
+            return ImmutableList.of(override);
+        }
+
+        if (options.pollForReachableAddresses() && options.reachableAddressPredicate() != null) {
+            LOG.debug("Using reachable addresses for management candidates of {}", location);
+            try {
+                final Predicate<? super HostAndPort> predicate = options.reachableAddressPredicate();
+                return getReachableAddresses(node, predicate, options.reachableAddressTimeout());
+            } catch (RuntimeException e) {
+                if (options.propagatePollForReachableFailure()) {
+                    throw Exceptions.propagate(e);
+                } else {
+                    LOG.warn("No reachable address ({}/{}); falling back to any advertised address; may cause future failures",
+                            location.getCreationString(config), node);
+                }
+            }
+        } else if (options.pollForReachableAddresses()) {
+            throw new IllegalStateException(this + " was configured to expect " + node + " to be reachable " +
+                    "and to poll for its reachable addresses but the predicate to determine reachability was null");
+        }
+
+        Iterable<String> addresses = getResolvableAddressesWithMode(node);
+        LOG.debug("Using first resolvable address in {} for management candidates of {}", Iterables.toString(addresses), location);
+        for (String address : addresses) {
+            if (isAddressResolvable(address)) {
+                return ImmutableList.of(HostAndPort.fromParts(address, options.defaultLoginPort()));
+            } else {
+                LOG.debug("Unresolvable address: " + address);
+            }
+        }
+
+        LOG.warn("No resolvable address in {} ({}/{}); using first; may cause future failures",
+                new Object[]{addresses, location.getCreationString(config), node});
+        String host = Iterables.getFirst(addresses, null);
+        if (host != null) {
+            return ImmutableList.of(HostAndPort.fromParts(host, options.defaultLoginPort()));
+        } else {
+            return ImmutableList.of();
+        }
+    }
+
+    /**
+     * Returns all reachable addresses according to reachablePredicate.
+     * Iterators are ordered according to the configured {@link #getNetworkMode() mode}.
+     */
+    protected Iterable<HostAndPort> getReachableAddresses(NodeMetadata node, Predicate<? super HostAndPort> reachablePredicate, Duration timeout) {
+        if (timeout == null) timeout = Duration.FIVE_MINUTES;
+        Iterable<String> candidates = getResolvableAddressesWithMode(node);
+        return JcloudsUtil.getReachableAddresses(candidates, node.getLoginPort(), timeout, reachablePredicate);
+    }
+
+    protected Iterable<String> getResolvableAddressesWithMode(NodeMetadata node) {
+        Iterable<String> base;
+        switch (getNetworkMode()) {
+        case ONLY_PRIVATE:
+            base = node.getPrivateAddresses();
+            break;
+        case ONLY_PUBLIC:
+            base = node.getPublicAddresses();
+            break;
+        case PREFER_PRIVATE:
+            base = Iterables.concat(node.getPrivateAddresses(), node.getPublicAddresses());
+            break;
+        case PREFER_PUBLIC:
+        default:
+            base = Iterables.concat(node.getPublicAddresses(), node.getPrivateAddresses());
+        }
+        return FluentIterable.from(base)
+                .filter(new AddressResolvable());
+    }
+
+    protected static boolean isAddressResolvable(String addr) {
+        try {
+            Networking.getInetAddressWithFixedName(addr);
+            return true; // fine, it resolves
+        } catch (RuntimeException e) {
+            Exceptions.propagateIfFatal(e);
+            return false;
+        }
+    }
+
+    private static class AddressResolvable implements Predicate<String> {
+        @Override
+        public boolean apply(@Nullable String input) {
+            return isAddressResolvable(input);
+        }
+    }
+
+    // --------------------------------------------------------------------------------------
+
+    protected boolean shouldCheckCredentials() {
+        return Boolean.TRUE.equals(config().get(CHECK_CREDENTIALS));
+    }
+
+    protected boolean checkCredential(
+            JcloudsLocation location, HostAndPort hostAndPort, LoginCredentials credentials,
+            ConfigBag config, boolean isWindows) {
+        try {
+            if (isWindows) {
+                location.waitForWinRmAvailable(credentials, hostAndPort, config);
+            } else {
+                location.waitForSshable(hostAndPort, ImmutableList.of(credentials), config);
+            }
+            return true;
+        } catch (IllegalStateException e) {
+            return false;
+        }
+    }
+
+    protected Iterable<LoginCredentials> getCredentialCandidates(
+            JcloudsLocation location, NodeMetadata node, ConnectivityResolverOptions options, ConfigBag setup) {
+        LoginCredentials userCredentials = null;
+        // Figure out which login credentials to use. We only make a connection with
+        // initialCredentials when jclouds didn't do any sshing and wait for connectable is true.
+        // 0. if jclouds didn't do anything and we should wait for the machine then initial credentials is
+        //    whatever waitForSshable determines and then create the user ourselves.
+        if (options.skipJcloudsSshing() && options.waitForConnectable()) {
+            if (options.isWindows() && options.initialCredentials().isPresent()) {
+                return ImmutableList.of(options.initialCredentials().get());
+            } else {
+                return location.generateCredentials(node.getCredentials(), setup.get(JcloudsLocationConfig.LOGIN_USER));
+            }
+        }
+
+        // 1. Were they configured by the user?
+        LoginCredentials customCredentials = setup.get(JcloudsLocationConfig.CUSTOM_CREDENTIALS);
+        if (customCredentials != null) {
+            userCredentials = customCredentials;
+            //set userName and other data, from these credentials
+            Object oldUsername = setup.put(JcloudsLocationConfig.USER, customCredentials.getUser());
+            LOG.debug("Using username {}, from custom credentials, on node {}. User was previously {}",
+                    new Object[]{customCredentials.getUser(), node, oldUsername});
+            if (customCredentials.getOptionalPassword().isPresent()) {
+                setup.put(JcloudsLocationConfig.PASSWORD, customCredentials.getOptionalPassword().get());
+            }
+            if (customCredentials.getOptionalPrivateKey().isPresent()) {
+                setup.put(JcloudsLocationConfig.PRIVATE_KEY_DATA, customCredentials.getOptionalPrivateKey().get());
+            }
+        }
+        // 2. Can they be extracted from the setup+node?
+        if ((userCredentials == null ||
+                (!userCredentials.getOptionalPassword().isPresent() && !userCredentials.getOptionalPrivateKey().isPresent())) &&
+                options.initialCredentials().isPresent()) {
+            // We either don't have any userCredentials, or it is missing both a password/key.
+            if (userCredentials != null) {
+                LOG.debug("Custom credential from {} is missing both password and private key; " +
+                        "extracting them from the VM: {}", JcloudsLocationConfig.CUSTOM_CREDENTIALS.getName(), userCredentials);
+            }
+            // TODO See waitForSshable, which now handles if the node.getLoginCredentials has both a password+key
+            userCredentials = location.extractVmCredentials(setup, node, options.initialCredentials().get());
+        }
+        if (userCredentials == null) {
+            // only happens if something broke above...
+            userCredentials = node.getCredentials();
+        }
+
+        return userCredentials != null? ImmutableList.of(userCredentials) : ImmutableList.<LoginCredentials>of();
+    }
+
+    // --------------------------------------------------------------------------------------
+
+    protected Entity getContextEntity(ConfigBag configBag) {
+        Object context = configBag.get(LocationConfigKeys.CALLER_CONTEXT);
+        if (context instanceof Entity) {
+            return (Entity) context;
+        } else {
+            Entity taskContext = BrooklynTaskTags.getContextEntity(Tasks.current());
+            if (taskContext != null) {
+                return taskContext;
+            }
+        }
+        LOG.warn("No context entity found in config or current task");
+        return null;
+    }
+
+    protected NetworkMode getNetworkMode() {
+        NetworkMode networkMode = config().get(NETWORK_MODE);
+        return networkMode != null ? networkMode : NetworkMode.PREFER_PUBLIC;
+    }
+
+    private boolean isNetworkModeSet() {
+        return config().get(NETWORK_MODE) != null;
+    }
+
+    @Override
+    public String toString() {
+        return MoreObjects.toStringHelper(this)
+                .add("mode", getNetworkMode())
+                .toString();
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/3fd64c89/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/JcloudsLocation.java
----------------------------------------------------------------------
diff --git a/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/JcloudsLocation.java b/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/JcloudsLocation.java
index b221641..8e25d64 100644
--- a/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/JcloudsLocation.java
+++ b/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/JcloudsLocation.java
@@ -430,6 +430,11 @@ public class JcloudsLocation extends AbstractCloudMachineProvisioningLocation im
         return result;
     }
 
+    public ConnectivityResolver getLocationNetworkInfoCustomizer(ConfigBag setup) {
+        ConnectivityResolver configured = setup.get(CONNECTIVITY_RESOLVER);
+        return (configured != null) ? configured : new DefaultConnectivityResolver();
+    }
+
     protected Collection<MachineLocationCustomizer> getMachineCustomizers(ConfigBag setup) {
         Collection<MachineLocationCustomizer> customizers = setup.get(MACHINE_LOCATION_CUSTOMIZERS);
         return (customizers == null ? ImmutableList.<MachineLocationCustomizer>of() : customizers);
@@ -626,18 +631,46 @@ public class JcloudsLocation extends AbstractCloudMachineProvisioningLocation im
         }
     }
 
+    protected ConnectivityResolverOptions.Builder getConnectivityOptionsBuilder(ConfigBag setup, boolean isWindows) {
+        boolean waitForSshable = !"false".equalsIgnoreCase(setup.get(JcloudsLocationConfig.WAIT_FOR_SSHABLE));
+        boolean waitForWinRmable = !"false".equalsIgnoreCase(setup.get(JcloudsLocationConfig.WAIT_FOR_WINRM_AVAILABLE));
+        boolean waitForConnectable = isWindows ? waitForWinRmable : waitForSshable;
+
+        boolean usePortForwarding = setup.get(JcloudsLocationConfig.USE_PORT_FORWARDING);
+        boolean skipJcloudsSshing = usePortForwarding ||
+                Boolean.FALSE.equals(setup.get(JcloudsLocationConfig.USE_JCLOUDS_SSH_INIT));
+
+        ConnectivityResolverOptions.Builder builder = ConnectivityResolverOptions.builder()
+                .waitForConnectable(waitForConnectable)
+                .usePortForwarding(usePortForwarding)
+                .skipJcloudsSshing(skipJcloudsSshing);
+
+        String pollForFirstReachable = setup.get(JcloudsLocationConfig.POLL_FOR_FIRST_REACHABLE_ADDRESS);
+        boolean pollEnabled = !"false".equalsIgnoreCase(pollForFirstReachable);
+
+        if (pollEnabled) {
+            Predicate<? super HostAndPort> reachableAddressesPredicate = getReachableAddressesPredicate(setup);
+            Duration pollTimeout = "true".equals(pollForFirstReachable)
+                                   ? Duration.FIVE_MINUTES
+                                   : Duration.of(pollForFirstReachable);
+            builder.pollForReachableAddresses(reachableAddressesPredicate, pollTimeout, true);
+        }
+        return builder;
+    }
+
     protected MachineLocation obtainOnce(ConfigBag setup) throws NoMachinesAvailableException {
         AccessController.Response access = getManagementContext().getAccessController().canProvisionLocation(this);
         if (!access.isAllowed()) {
             throw new IllegalStateException("Access controller forbids provisioning in "+this+": "+access.getMsg());
         }
 
-        boolean waitForSshable = !"false".equalsIgnoreCase(setup.get(WAIT_FOR_SSHABLE));
-        boolean waitForWinRmable = !"false".equalsIgnoreCase(setup.get(WAIT_FOR_WINRM_AVAILABLE));
-        boolean usePortForwarding = setup.get(USE_PORT_FORWARDING);
-        boolean skipJcloudsSshing = Boolean.FALSE.equals(setup.get(USE_JCLOUDS_SSH_INIT)) || usePortForwarding;
+        Predicate<? super HostAndPort> reachablePredicate = getReachableAddressesPredicate(setup);
+        ConnectivityResolverOptions options = getConnectivityOptionsBuilder(setup, false).build();
+
+        // FIXME How do we influence the node.getLoginPort, so it is set correctly for Windows?
+        // Setup port-forwarding, if required
         JcloudsPortForwarderExtension portForwarder = setup.get(PORT_FORWARDER);
-        if (usePortForwarding) checkNotNull(portForwarder, "portForwarder, when use-port-forwarding enabled");
+        if (options.usePortForwarding()) checkNotNull(portForwarder, "portForwarder, when use-port-forwarding enabled");
 
         final ComputeService computeService = getComputeService(setup);
         CloudMachineNamer cloudMachineNamer = getCloudMachineNamer(setup);
@@ -676,12 +709,14 @@ public class JcloudsLocation extends AbstractCloudMachineProvisioningLocation im
                 // Setup the template
                 template = buildTemplate(computeService, setup, customizers);
                 boolean expectWindows = isWindows(template, setup);
-                if (!skipJcloudsSshing) {
+                if (!options.skipJcloudsSshing()) {
                     if (expectWindows) {
                         // TODO Was this too early to look at template.getImage? e.g. customizeTemplate could subsequently modify it.
                         LOG.warn("Ignoring invalid configuration for Windows provisioning of "+template.getImage()+": "+USE_JCLOUDS_SSH_INIT.getName()+" should be false");
-                        skipJcloudsSshing = true;
-                    } else if (waitForSshable) {
+                        options = options.toBuilder()
+                                .skipJcloudsSshing(true)
+                                .build();
+                    } else if (options.waitForConnectable()) {
                         userCredentials = initTemplateForCreateUser(template, setup);
                     }
                 }
@@ -723,7 +758,6 @@ public class JcloudsLocation extends AbstractCloudMachineProvisioningLocation im
                 throw new IllegalStateException("No nodes returned by jclouds create-nodes in " + getCreationString(setup));
 
             boolean windows = isWindows(node, setup);
-            boolean waitForConnectable = (windows) ? waitForWinRmable : waitForSshable;
 
             if (windows) {
                 int newLoginPort = node.getLoginPort() == 22
@@ -739,77 +773,54 @@ public class JcloudsLocation extends AbstractCloudMachineProvisioningLocation im
                         .credentials(LoginCredentials.builder(node.getCredentials()).user(newLoginUser).build())
                         .build();
             }
-            // FIXME How do we influence the node.getLoginPort, so it is set correctly for Windows?
-            // Setup port-forwarding, if required
-            Optional<HostAndPort> sshHostAndPortOverride;
-            if (usePortForwarding) {
-                sshHostAndPortOverride = Optional.of(portForwarder.openPortForwarding(
+            Optional<HostAndPort> portForwardSshOverride;
+            if (options.usePortForwarding()) {
+                portForwardSshOverride = Optional.of(portForwarder.openPortForwarding(
                         node,
                         node.getLoginPort(),
                         Optional.<Integer>absent(),
                         Protocol.TCP,
                         Cidr.UNIVERSAL));
             } else {
-                sshHostAndPortOverride = Optional.absent();
+                portForwardSshOverride = Optional.absent();
             }
 
-            LoginCredentials initialCredentials = node.getCredentials();
+            options = options.toBuilder()
+                    .isWindows(windows)
+                    .defaultLoginPort(node.getLoginPort())
+                    .portForwardSshOverride(portForwardSshOverride.orNull())
+                    .initialCredentials(node.getCredentials())
+                    .userCredentials(userCredentials)
+                    .build();
 
-            final HostAndPort managementHostAndPort = resolveManagementHostAndPort(
-                    node, Optional.fromNullable(userCredentials), sshHostAndPortOverride, setup,
-                    new ResolveOptions()
-                            .windows(windows)
-                            .expectConnectable(waitForConnectable)
-                            .pollForFirstReachableAddress(!"false".equalsIgnoreCase(setup.get(POLL_FOR_FIRST_REACHABLE_ADDRESS)))
-                            .propagatePollForReachableFailure(true));
+            ConnectivityResolver networkInfoCustomizer = getLocationNetworkInfoCustomizer(setup);
 
-            if (skipJcloudsSshing) {
-                if (waitForConnectable) {
-                    if (windows) {
-                        // TODO Does jclouds support any windows user setup?
-                        initialCredentials = waitForWinRmAvailable(initialCredentials, managementHostAndPort, setup);
-                    } else {
-                        initialCredentials = waitForSshable(computeService, node, managementHostAndPort, setup);
-                    }
-                    userCredentials = createUser(computeService, node, managementHostAndPort, initialCredentials, setup);
-                }
-            }
+            ManagementAddressResolveResult hostPortCred = networkInfoCustomizer.resolve(this, node, setup, options);
+            final HostAndPort managementHostAndPort = hostPortCred.hostAndPort();
+            LoginCredentials creds = hostPortCred.credentials();
+            LOG.info("Using host-and-port={} and user={} when connecting to {}",
+                    new Object[]{managementHostAndPort, creds.getUser(), node});
 
-            // Figure out which login-credentials to use
-            LoginCredentials customCredentials = setup.get(CUSTOM_CREDENTIALS);
-            if (customCredentials != null) {
-                userCredentials = customCredentials;
-                //set userName and other data, from these credentials
-                Object oldUsername = setup.put(USER, customCredentials.getUser());
-                LOG.debug("node {} username {} / {} (customCredentials)", new Object[] { node, customCredentials.getUser(), oldUsername });
-                if (customCredentials.getOptionalPassword().isPresent()) setup.put(PASSWORD, customCredentials.getOptionalPassword().get());
-                if (customCredentials.getOptionalPrivateKey().isPresent()) setup.put(PRIVATE_KEY_DATA, customCredentials.getOptionalPrivateKey().get());
-            }
-            if (userCredentials == null || (!userCredentials.getOptionalPassword().isPresent() && !userCredentials.getOptionalPrivateKey().isPresent())) {
-                // We either don't have any userCredentials, or it is missing both a password/key.
-                // TODO See waitForSshable, which now handles if the node.getLoginCredentials has both a password+key
-                userCredentials = extractVmCredentials(setup, node, initialCredentials);
+            if (options.skipJcloudsSshing() && options.waitForConnectable()) {
+                LoginCredentials createdCredentials = createUser(computeService, node, managementHostAndPort, creds, setup);
+                if (createdCredentials != null) {
+                    userCredentials = createdCredentials;
+                }
             }
             if (userCredentials == null) {
-                // TODO See waitForSshable, which now handles if the node.getLoginCredentials has both a password+key
-                userCredentials = extractVmCredentials(setup, node, initialCredentials);
-            }
-            if (userCredentials != null) {
-                node = NodeMetadataBuilder.fromNodeMetadata(node).credentials(userCredentials).build();
-            } else {
-                // only happens if something broke above...
-                userCredentials = LoginCredentials.fromCredentials(node.getCredentials());
+                userCredentials = creds;
             }
+
             // store the credentials, in case they have changed
             putIfPresentButDifferent(setup, JcloudsLocationConfig.PASSWORD, userCredentials.getOptionalPassword().orNull());
             putIfPresentButDifferent(setup, JcloudsLocationConfig.PRIVATE_KEY_DATA, userCredentials.getOptionalPrivateKey().orNull());
 
             // Wait for the VM to be reachable over SSH
-            if (waitForSshable && !windows) {
+            if (options.waitForConnectable() && !options.isWindows()) {
                 waitForSshable(computeService, node, managementHostAndPort, ImmutableList.of(userCredentials), setup);
             } else {
-                LOG.debug("Skipping ssh check for {} ({}) due to config waitForSshable={}, windows={}",
-                        new Object[]{node, getCreationString(setup), waitForSshable, windows});
+                LOG.debug("Skipping ssh check for {} ({}) due to config waitForConnectable={}, windows={}",
+                        new Object[]{node, getCreationString(setup), options.waitForConnectable(), windows});
             }
 
             // Do not store the credentials on the node as this may leak the credentials if they
@@ -831,9 +842,9 @@ public class JcloudsLocation extends AbstractCloudMachineProvisioningLocation im
                 portForwardManager = (PortForwardManager) getManagementContext().getLocationRegistry().getLocationManaged(PortForwardManagerLocationResolver.PFM_GLOBAL_SPEC);
             }
 
-            if (usePortForwarding && sshHostAndPortOverride.isPresent()) {
+            if (options.usePortForwarding() && portForwardSshOverride.isPresent()) {
                 // Now that we have the sshMachineLocation, we can associate the port-forwarding address with it.
-                portForwardManager.associate(node.getId(), sshHostAndPortOverride.get(), machineLocation, node.getLoginPort());
+                portForwardManager.associate(node.getId(), portForwardSshOverride.get(), machineLocation, node.getLoginPort());
             }
 
             if ("docker".equals(this.getProvider())) {
@@ -850,7 +861,7 @@ public class JcloudsLocation extends AbstractCloudMachineProvisioningLocation im
 
             List<String> customisationForLogging = new ArrayList<String>();
             // Apply same securityGroups rules to iptables, if iptables is running on the node
-            if (waitForSshable) {
+            if (options.waitForConnectable()) {
 
                 String setupScript = setup.get(JcloudsLocationConfig.CUSTOM_MACHINE_SETUP_SCRIPT_URL);
                 List<String> setupScripts = setup.get(JcloudsLocationConfig.CUSTOM_MACHINE_SETUP_SCRIPT_URL_LIST);
@@ -907,7 +918,6 @@ public class JcloudsLocation extends AbstractCloudMachineProvisioningLocation im
                     }
                 }
 
-
                 if (setup.get(GENERATE_HOSTNAME)) {
                     if (windows) {
                         // TODO: Generate Windows Hostname
@@ -1739,107 +1749,6 @@ public class JcloudsLocation extends AbstractCloudMachineProvisioningLocation im
         return userCreation.credentials();
     }
 
-    private static class ResolveOptions {
-        boolean pollForFirstReachableAddress;
-        boolean expectConnectable;
-        boolean isWindows;
-        boolean propagatePollForReachableFailure;
-
-        ResolveOptions pollForFirstReachableAddress(boolean val) {
-            pollForFirstReachableAddress = val;
-            return this;
-        }
-        ResolveOptions expectConnectable(boolean val) {
-            expectConnectable = val;
-            return this;
-        }
-        ResolveOptions windows(boolean val) {
-            isWindows = val;
-            return this;
-        }
-        public ResolveOptions propagatePollForReachableFailure(boolean val) {
-            this.propagatePollForReachableFailure = val;
-            return this;
-        }
-    }
-
-    /**
-     * Infers the hostAndPort to use for subsequent creation of the
-     * {@link JcloudsSshMachineLocation} or {@link JcloudsWinRmMachineLocation}.
-     * This is expected to be the login host:port, for connecting to the VM
-     * via ssh or WinRM.
-     *
-     * However, some VMs provisioned will not be sshable or reachable at all.
-     * In such cases, this method will likely return the first address returned by
-     * jclouds.
-     *
-     * For AWS, if we are allowed to SSH to the VM to find out its DNS name, then we'll
-     * return that fully qualified name (which we expect to be reachable from inside
-     * and outside the AWS region).
-     */
-    private HostAndPort resolveManagementHostAndPort(
-            NodeMetadata node, Optional<LoginCredentials> userCredentials,
-            Optional<HostAndPort> hostAndPortOverride, ConfigBag config, ResolveOptions options) {
-        boolean lookupAwsHostname = Boolean.TRUE.equals(config.get(LOOKUP_AWS_HOSTNAME));
-        String provider = config.get(CLOUD_PROVIDER);
-        if (provider == null) provider= getProvider();
-        int defaultPort;
-        if (options.isWindows) {
-            defaultPort = config.get(WinRmMachineLocation.USE_HTTPS_WINRM) ? 5986 : 5985;
-        } else {
-            defaultPort = node.getLoginPort();
-        }
-
-        if (hostAndPortOverride.isPresent()) {
-            // Don't try to resolve it; just use it
-            int port = hostAndPortOverride.get().hasPort() ? hostAndPortOverride.get().getPort() : defaultPort;
-            return HostAndPort.fromParts(hostAndPortOverride.get().getHostText(), port);
-        }
-        if (options.expectConnectable && userCredentials.isPresent() && "aws-ec2".equals(provider) && lookupAwsHostname) {
-            // Treat AWS as a special case because the DNS fully qualified hostname in AWS is
-            // (normally?!) a good way to refer to the VM from both inside and outside of the
-            // region.
-            Maybe<String> result = getHostnameAws(node, hostAndPortOverride, Suppliers.ofInstance(userCredentials.get()), config);
-            if (result.isPresent()) {
-                return HostAndPort.fromParts(result.get(), defaultPort);
-            }
-        }
-        if (options.expectConnectable && options.pollForFirstReachableAddress) {
-            try {
-                String firstReachableAddress = getFirstReachableAddress(node, config);
-                return HostAndPort.fromParts(firstReachableAddress, defaultPort);
-            } catch (RuntimeException e) {
-                if (options.propagatePollForReachableFailure) {
-                    throw Exceptions.propagate(e);
-                } else {
-                    LOG.warn("No reachable address ({}/{}); falling back to any advertised address; may cause future failures",
-                            getCreationString(config), node);
-                }
-            }
-        }
-
-        Iterable<String> addresses = Iterables.concat(node.getPublicAddresses(), node.getPrivateAddresses());
-        for (String address : addresses) {
-            if (isAddressResolvable(address)) {
-                return HostAndPort.fromParts(address, defaultPort);
-            }
-        }
-        LOG.warn("No resolvable address in {} ({}/{}); using first; may cause future failures",
-                new Object[]{addresses, getCreationString(config), node});
-        String host = Iterables.get(addresses, 0);
-        return HostAndPort.fromParts(host, defaultPort);
-    }
-
-    private boolean isAddressResolvable(String addr) {
-        try {
-            Networking.getInetAddressWithFixedName(addr);
-            return true; // fine, it resolves
-        } catch (RuntimeException e) {
-            Exceptions.propagateIfFatal(e);
-            return false;
-        }
-    }
-
     /** @deprecated since 0.11.0 use {@link CreateUserStatements} instead. */
     @Deprecated
     protected static class UserCreation extends CreateUserStatements  {
@@ -1918,15 +1827,21 @@ public class JcloudsLocation extends AbstractCloudMachineProvisioningLocation im
     protected JcloudsMachineLocation registerMachineLocation(ConfigBag setup, NodeMetadata node) {
         ComputeService computeService = getComputeService(setup);
         boolean windows = isWindows(node, setup);
-        
-        HostAndPort managementHostAndPort = resolveManagementHostAndPort(
-                node, Optional.<LoginCredentials>absent(), Optional.<HostAndPort>absent(), setup,
-                new ResolveOptions()
-                        .windows(windows)
-                        .expectConnectable(true)
-                        .propagatePollForReachableFailure(false)
-                        .pollForFirstReachableAddress(!"false".equalsIgnoreCase(setup.get(POLL_FOR_FIRST_REACHABLE_ADDRESS))));
-        
+
+        // Not publishing networks since they should have previously been published.
+        ConnectivityResolverOptions options = getConnectivityOptionsBuilder(setup, windows)
+                .initialCredentials(node.getCredentials())
+                .userCredentials(node.getCredentials())
+                .defaultLoginPort(node.getLoginPort())
+                .isRebinding(true)
+                .build();
+        HostAndPort managementHostAndPort = getLocationNetworkInfoCustomizer(setup)
+                .resolve(this, node, setup, options)
+                .hostAndPort();
+
+        if (managementHostAndPort == null) {
+            throw new IllegalStateException("Could not resolve management host and port for " + node + " given options: " + options);
+        }
 
         if (windows) {
             return registerWinRmMachineLocation(computeService, node, Optional.<Template>absent(), node.getCredentials(), managementHostAndPort, setup);
@@ -2451,8 +2366,8 @@ public class JcloudsLocation extends AbstractCloudMachineProvisioningLocation im
         String user = getUser(setup);
         OsCredential localCredentials = LocationConfigUtils.getOsCredential(setup).checkNoErrors();
         
-        LOG.debug("Credentials extracted for {}: {}/{} with {}/{}", new Object[] { node,
-            user, nodeCredentials.getUser(), localCredentials, nodeCredentials });
+        LOG.debug("Credentials extracted for {}: {}/{} with {}/{}", new Object[] {
+                node, user, nodeCredentials.getUser(), localCredentials, nodeCredentials });
 
         if (Strings.isNonBlank(nodeCredentials.getUser())) {
             if (Strings.isBlank(user)) {
@@ -2493,37 +2408,10 @@ public class JcloudsLocation extends AbstractCloudMachineProvisioningLocation im
         String result;
         if (enabled) {
             Duration timeout = "true".equals(pollForFirstReachable) ? Duration.FIVE_MINUTES : Duration.of(pollForFirstReachable);
-
-            Predicate<? super HostAndPort> pollForFirstReachableHostAndPortPredicate;
-            if (setup.get(POLL_FOR_FIRST_REACHABLE_ADDRESS_PREDICATE) != null) {
-                LOG.debug("{} polling for first reachable address with {}",
-                        this, setup.get(POLL_FOR_FIRST_REACHABLE_ADDRESS_PREDICATE));
-                pollForFirstReachableHostAndPortPredicate = setup.get(POLL_FOR_FIRST_REACHABLE_ADDRESS_PREDICATE);
-            } else {
-                LOG.debug("{} polling for first reachable address with instance of {}",
-                        this, POLL_FOR_FIRST_REACHABLE_ADDRESS_PREDICATE_TYPE.getName());
-
-                Class<? extends Predicate<? super HostAndPort>> predicateType =
-                        setup.get(POLL_FOR_FIRST_REACHABLE_ADDRESS_PREDICATE_TYPE);
-
-                Map<String, Object> args = MutableMap.of();
-                ConfigUtils.addUnprefixedConfigKeyInConfigBack(
-                        POLL_FOR_FIRST_REACHABLE_ADDRESS_PREDICATE.getName() + ".", setup, args);
-                try {
-                    pollForFirstReachableHostAndPortPredicate = predicateType.getConstructor(Map.class).newInstance(args);
-                } catch (NoSuchMethodException|IllegalAccessException e) {
-                    try {
-                        pollForFirstReachableHostAndPortPredicate = predicateType.newInstance();
-                    } catch (IllegalAccessException|InstantiationException newInstanceException) {
-                        throw Exceptions.propagate("Instantiating " + predicateType + " failed.", newInstanceException);
-                    }
-                } catch (InvocationTargetException|InstantiationException e) {
-                    throw Exceptions.propagate("Problem trying to instantiate " + predicateType + " with Map constructor.", e);
-                }
-            }
-
+            Predicate<? super HostAndPort> predicate = getReachableAddressesPredicate(setup);
+            LOG.debug("{} polling for first reachable address with {}", this, predicate);
             // Throws if no suitable address is found.
-            result = JcloudsUtil.getFirstReachableAddress(node, timeout, pollForFirstReachableHostAndPortPredicate);
+            result = JcloudsUtil.getFirstReachableAddress(node, timeout, predicate);
             LOG.debug("Using first-reachable address "+result+" for node "+node+" in "+this);
         } else {
             result = Iterables.getFirst(Iterables.concat(node.getPublicAddresses(), node.getPrivateAddresses()), null);
@@ -2535,6 +2423,31 @@ public class JcloudsLocation extends AbstractCloudMachineProvisioningLocation im
         return result;
     }
 
+    private Predicate<? super HostAndPort> getReachableAddressesPredicate(ConfigBag config) {
+        Predicate<? super HostAndPort> pollForFirstReachableHostAndPortPredicate;
+        if (config.get(POLL_FOR_FIRST_REACHABLE_ADDRESS_PREDICATE) != null) {
+            return config.get(POLL_FOR_FIRST_REACHABLE_ADDRESS_PREDICATE);
+        } else {
+            Class<? extends Predicate<? super HostAndPort>> predicateType =
+                    config.get(POLL_FOR_FIRST_REACHABLE_ADDRESS_PREDICATE_TYPE);
+
+            Map<String, Object> args = MutableMap.of();
+            ConfigUtils.addUnprefixedConfigKeyInConfigBack(
+                    POLL_FOR_FIRST_REACHABLE_ADDRESS_PREDICATE.getName() + ".", config, args);
+            try {
+                return predicateType.getConstructor(Map.class).newInstance(args);
+            } catch (NoSuchMethodException | IllegalAccessException e) {
+                try {
+                    return pollForFirstReachableHostAndPortPredicate = predicateType.newInstance();
+                } catch (IllegalAccessException | InstantiationException newInstanceException) {
+                    throw Exceptions.propagate("Failed to instantiate " + predicateType, newInstanceException);
+                }
+            } catch (InvocationTargetException | InstantiationException e) {
+                throw Exceptions.propagate("Failed to instantiate " + predicateType + " with Map constructor", e);
+            }
+        }
+    }
+
     protected LoginCredentials waitForWinRmAvailable(LoginCredentials credentialsToTry, final HostAndPort managementHostAndPort, ConfigBag setup) {
         String waitForWinrmAvailable = setup.get(WAIT_FOR_WINRM_AVAILABLE);
         checkArgument(!"false".equalsIgnoreCase(waitForWinrmAvailable), "waitForWinRmAvailable called despite waitForWinRmAvailable=%s", waitForWinrmAvailable);
@@ -2627,24 +2540,31 @@ public class JcloudsLocation extends AbstractCloudMachineProvisioningLocation im
         return credsSuccessful.get();
     }
 
-    protected LoginCredentials waitForSshable(final ComputeService computeService, final NodeMetadata node, HostAndPort managementHostAndPort, ConfigBag setup) {
-        LoginCredentials nodeCreds = node.getCredentials();
+    protected LoginCredentials waitForSshableGuessCredentials(final ComputeService computeService, final NodeMetadata node, HostAndPort managementHostAndPort, ConfigBag setup) {
+        // See https://issues.apache.org/jira/browse/BROOKLYN-186
+        // Handle where jclouds gives us the wrong login user (!) and both a password + ssh key.
+        // Try all the permutations to find the one that works.
+        Iterable<LoginCredentials> credentialsToTry = generateCredentials(node.getCredentials(), setup.get(LOGIN_USER));
+        return waitForSshable(computeService, node, managementHostAndPort, credentialsToTry, setup);
+    }
+
+    /** @deprecated Since 0.11.0. Use {@link #waitForSshableGuessCredentials} instead. */
+    @Deprecated
+    protected LoginCredentials waitForSshable(ComputeService computeService, NodeMetadata node, HostAndPort managementHostAndPort, ConfigBag setup) {
+        return waitForSshableGuessCredentials(computeService, node, managementHostAndPort, setup);
+    }
+
+    /** @return An Iterable of credentials based on nodeCreds containing different parameters. */
+    Iterable<LoginCredentials> generateCredentials(LoginCredentials nodeCreds, @Nullable String loginUserOverride) {
         String nodeUser = nodeCreds.getUser();
-        String loginUserOverride = setup.get(LOGIN_USER);
         Set<String> users = MutableSet.of();
-
         if (Strings.isNonBlank(nodeUser)) {
             users.add(nodeUser);
         }
-
         if (Strings.isNonBlank(loginUserOverride)) {
             users.add(loginUserOverride);
         }
-
-        // See https://issues.apache.org/jira/browse/BROOKLYN-186
-        // Handle where jclouds gives us the wrong login user (!) and both a password + ssh key.
-        // Try all the permutations to find the one that works.
-        List<LoginCredentials> credentialsToTry = Lists.newArrayList();
+        List<LoginCredentials> credentialsToTry = new ArrayList<>();
         for (String user : users) {
             if (nodeCreds.getOptionalPassword().isPresent() && nodeCreds.getOptionalPrivateKey().isPresent()) {
                 credentialsToTry.add(LoginCredentials.builder(nodeCreds).noPassword().user(user).build());
@@ -2653,14 +2573,22 @@ public class JcloudsLocation extends AbstractCloudMachineProvisioningLocation im
                 credentialsToTry.add(LoginCredentials.builder(nodeCreds).user(user).build());
             }
         }
+        return credentialsToTry;
+    }
 
-        return waitForSshable(computeService, node, managementHostAndPort, credentialsToTry, setup);
+    /** @deprecated since 0.11.0 use {@link #waitForSshable(HostAndPort, Iterable, ConfigBag)} instead */
+    @Deprecated
+    protected LoginCredentials waitForSshable(
+            final ComputeService computeService, final NodeMetadata node, HostAndPort hostAndPort,
+            Iterable<LoginCredentials> credentialsToTry, ConfigBag setup) {
+        return waitForSshable(hostAndPort, credentialsToTry, setup);
     }
 
-    protected LoginCredentials waitForSshable(final ComputeService computeService, final NodeMetadata node, HostAndPort hostAndPort, List<LoginCredentials> credentialsToTry, ConfigBag setup) {
+    protected LoginCredentials waitForSshable(
+            HostAndPort hostAndPort, Iterable<LoginCredentials> credentialsToTry, ConfigBag setup) {
         String waitForSshable = setup.get(WAIT_FOR_SSHABLE);
-        checkArgument(!"false".equalsIgnoreCase(waitForSshable), "waitForSshable called despite waitForSshable=%s for %s", waitForSshable, node);
-        checkArgument(credentialsToTry.size() > 0, "waitForSshable called without credentials for %s", node);
+        checkArgument(!"false".equalsIgnoreCase(waitForSshable), "waitForSshable called despite waitForSshable=%s for %s", waitForSshable, hostAndPort);
+        checkArgument(!Iterables.isEmpty(credentialsToTry), "waitForSshable called without credentials for %s", hostAndPort);
 
         Duration timeout = null;
         try {
@@ -2743,7 +2671,7 @@ public class JcloudsLocation extends AbstractCloudMachineProvisioningLocation im
         return defaultPort;
     }
 
-    protected void waitForReachable(Callable<Boolean> checker, String hostAndPort, List<LoginCredentials> credentialsToLog, ConfigBag setup, Duration timeout) {
+    protected void waitForReachable(Callable<Boolean> checker, String hostAndPort, Iterable<LoginCredentials> credentialsToLog, ConfigBag setup, Duration timeout) {
         if (LOG.isDebugEnabled()) {
             List<String> credsToString = Lists.newArrayList();
             for (LoginCredentials creds : credentialsToLog) {
@@ -2764,8 +2692,8 @@ public class JcloudsLocation extends AbstractCloudMachineProvisioningLocation im
                     new Object[] {
                             getCreationString(setup), timeout,
                             hostAndPort,
-                            credentialsToLog.size(),
-                            Strings.s(credentialsToLog.size()),
+                            Iterables.size(credentialsToLog),
+                            Strings.s(Iterables.size(credentialsToLog)),
                             (credsToString.size() == 1) ? credsToString.get(0) : "(multiple!):" + Joiner.on("\n\t").join(credsToString)
                     });
         }
@@ -2975,7 +2903,7 @@ public class JcloudsLocation extends AbstractCloudMachineProvisioningLocation im
         }
     }
 
-    private Maybe<String> getHostnameAws(NodeMetadata node, Optional<HostAndPort> sshHostAndPort, Supplier<? extends LoginCredentials> userCredentials, ConfigBag setup) {
+    Maybe<String> getHostnameAws(NodeMetadata node, Optional<HostAndPort> sshHostAndPort, Supplier<? extends LoginCredentials> userCredentials, ConfigBag setup) {
         HostAndPort inferredHostAndPort = null;
         boolean waitForSshable = !"false".equalsIgnoreCase(setup.get(WAIT_FOR_SSHABLE));
         if (!waitForSshable) {
@@ -2993,7 +2921,7 @@ public class JcloudsLocation extends AbstractCloudMachineProvisioningLocation im
         }
         if (sshHostAndPort.isPresent() || inferredHostAndPort != null) {
             if (isWindows(node, setup)) {
-                LOG.warn("Error querying aws-ec2 Windows instance "+node.getId()+"@"+node.getLocation()+" over ssh for its hostname; falling back to jclouds metadata for address");
+                LOG.warn("Cannot query aws-ec2 Windows instance "+node.getId()+"@"+node.getLocation()+" over ssh for its hostname; falling back to jclouds metadata for address");
             } else {
                 HostAndPort hostAndPortToUse = sshHostAndPort.isPresent() ? sshHostAndPort.get() : inferredHostAndPort;
                 try {
@@ -3006,7 +2934,7 @@ public class JcloudsLocation extends AbstractCloudMachineProvisioningLocation im
         return Maybe.absent();
     }
 
-    private String getHostnameAws(HostAndPort hostAndPort, LoginCredentials userCredentials, ConfigBag setup) {
+    String getHostnameAws(HostAndPort hostAndPort, LoginCredentials userCredentials, ConfigBag setup) {
         SshMachineLocation sshLocByIp = null;
         try {
             // TODO messy way to get an SSH session
@@ -3036,6 +2964,9 @@ public class JcloudsLocation extends AbstractCloudMachineProvisioningLocation im
         return new JcloudsBlobStoreBasedObjectStore(this, container);
     }
 
+
+
+
     // ------------ static converters (could go to a new file) ------------------
 
     /** @deprecated since 0.11.0 without replacement */

http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/3fd64c89/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/JcloudsLocationConfig.java
----------------------------------------------------------------------
diff --git a/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/JcloudsLocationConfig.java b/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/JcloudsLocationConfig.java
index eb47a8f..d06f7cc 100644
--- a/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/JcloudsLocationConfig.java
+++ b/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/JcloudsLocationConfig.java
@@ -211,6 +211,10 @@ public interface JcloudsLocationConfig extends CloudLocationConfig {
             "customizersSupplierType", "Optional type of a Supplier<Collection<JcloudsLocationCustomizer>> " +
             "(to be class-loaded and constructed with either a ConfigBag or no-arg constructor)");
 
+    ConfigKey<ConnectivityResolver> CONNECTIVITY_RESOLVER = ConfigKeys.newConfigKey(ConnectivityResolver.class,
+            "connectivityResolver",
+            "Optional instance of a ConnectivityResolver that the location will use in favour of " + DefaultConnectivityResolver.class.getSimpleName());
+
     public static final ConfigKey<String> LOCAL_TEMP_DIR = SshTool.PROP_LOCAL_TEMP_DIR;
     
     public static final ConfigKey<Integer> OVERRIDE_RAM = ConfigKeys.newIntegerConfigKey("overrideRam", "Custom ram value");    

http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/3fd64c89/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/JcloudsUtil.java
----------------------------------------------------------------------
diff --git a/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/JcloudsUtil.java b/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/JcloudsUtil.java
index 88cdf1b..480d0d6 100644
--- a/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/JcloudsUtil.java
+++ b/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/JcloudsUtil.java
@@ -40,7 +40,6 @@ import org.apache.brooklyn.core.config.Sanitizer;
 import org.apache.brooklyn.core.location.LocationConfigKeys;
 import org.apache.brooklyn.core.location.cloud.CloudLocationConfig;
 import org.apache.brooklyn.util.core.config.ConfigBag;
-import org.apache.brooklyn.util.exceptions.Exceptions;
 import org.apache.brooklyn.util.net.Networking;
 import org.apache.brooklyn.util.net.Protocol;
 import org.apache.brooklyn.util.net.ReachableSocketFinder;

http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/3fd64c89/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/ManagementAddressResolveResult.java
----------------------------------------------------------------------
diff --git a/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/ManagementAddressResolveResult.java b/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/ManagementAddressResolveResult.java
new file mode 100644
index 0000000..c285738
--- /dev/null
+++ b/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/ManagementAddressResolveResult.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.brooklyn.location.jclouds;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import org.apache.brooklyn.core.config.Sanitizer;
+import org.jclouds.domain.LoginCredentials;
+
+import com.google.common.base.MoreObjects;
+import com.google.common.net.HostAndPort;
+
+public class ManagementAddressResolveResult {
+    private final HostAndPort hostAndPort;
+    private final LoginCredentials credentials;
+
+    ManagementAddressResolveResult(HostAndPort hostAndPort, LoginCredentials credentials) {
+        this.hostAndPort = checkNotNull(hostAndPort, "hostAndPort");
+        this.credentials = checkNotNull(credentials, "credentials");
+    }
+
+    public HostAndPort hostAndPort() {
+        return hostAndPort;
+    }
+
+    public LoginCredentials credentials() {
+        return credentials;
+    }
+
+    @Override
+    public String toString() {
+        return MoreObjects.toStringHelper(this)
+                .add("hostAndPort", hostAndPort)
+                .add("credentials", credentials)
+                .toString();
+    }
+}

http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/3fd64c89/locations/jclouds/src/test/java/org/apache/brooklyn/location/jclouds/AbstractJcloudsStubbedLiveTest.java
----------------------------------------------------------------------
diff --git a/locations/jclouds/src/test/java/org/apache/brooklyn/location/jclouds/AbstractJcloudsStubbedLiveTest.java b/locations/jclouds/src/test/java/org/apache/brooklyn/location/jclouds/AbstractJcloudsStubbedLiveTest.java
index 3190d33..9284459 100644
--- a/locations/jclouds/src/test/java/org/apache/brooklyn/location/jclouds/AbstractJcloudsStubbedLiveTest.java
+++ b/locations/jclouds/src/test/java/org/apache/brooklyn/location/jclouds/AbstractJcloudsStubbedLiveTest.java
@@ -80,7 +80,8 @@ public abstract class AbstractJcloudsStubbedLiveTest extends AbstractJcloudsLive
                 locationSpec,
                 jcloudsLocationConfig(ImmutableMap.<Object, Object>of(
                         JcloudsLocationConfig.COMPUTE_SERVICE_REGISTRY, computeServiceRegistry,
-                        JcloudsLocationConfig.WAIT_FOR_SSHABLE, "false")));
+                        JcloudsLocationConfig.WAIT_FOR_SSHABLE, "false",
+                        JcloudsLocationConfig.POLL_FOR_FIRST_REACHABLE_ADDRESS, "false")));
         return jcloudsLocation;
     }
     

http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/3fd64c89/locations/jclouds/src/test/java/org/apache/brooklyn/location/jclouds/BasicLocationNetworkInfoInitializerTest.java
----------------------------------------------------------------------
diff --git a/locations/jclouds/src/test/java/org/apache/brooklyn/location/jclouds/BasicLocationNetworkInfoInitializerTest.java b/locations/jclouds/src/test/java/org/apache/brooklyn/location/jclouds/BasicLocationNetworkInfoInitializerTest.java
new file mode 100644
index 0000000..51fcad2
--- /dev/null
+++ b/locations/jclouds/src/test/java/org/apache/brooklyn/location/jclouds/BasicLocationNetworkInfoInitializerTest.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.brooklyn.location.jclouds;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
+
+import org.apache.brooklyn.api.entity.EntitySpec;
+import org.apache.brooklyn.config.ConfigKey;
+import org.apache.brooklyn.core.entity.BrooklynConfigKeys;
+import org.apache.brooklyn.core.test.BrooklynAppUnitTestSupport;
+import org.apache.brooklyn.core.test.entity.TestEntity;
+import org.testng.annotations.Test;
+
+public class BasicLocationNetworkInfoInitializerTest extends BrooklynAppUnitTestSupport {
+
+    @Test
+    public void testInitializerSetsConfigKeyOnEntity() {
+        TestEntity entity = app.createAndManageChild(EntitySpec.create(TestEntity.class)
+                .addInitializer(DefaultConnectivityResolver.class));
+        final ConfigKey<Object> key = BrooklynConfigKeys.PROVISIONING_PROPERTIES.subKey(JcloudsLocationConfig.CONNECTIVITY_RESOLVER.getName());
+        final Object value = entity.config().get(key);
+        assertNotNull(value, "no value on " + entity + " for " + key);
+        assertEquals(value.getClass(), DefaultConnectivityResolver.class);
+    }
+
+}