You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by mm...@apache.org on 2022/06/14 13:46:16 UTC
[ignite-extensions] branch master updated: IGNITE-17137 Move ignite-cloud IP finder to the Ignite Extensions project (#157)
This is an automated email from the ASF dual-hosted git repository.
mmuzaf pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite-extensions.git
The following commit(s) were added to refs/heads/master by this push:
new 05b7236 IGNITE-17137 Move ignite-cloud IP finder to the Ignite Extensions project (#157)
05b7236 is described below
commit 05b7236db09db1f5e6e6ef3a9c06f9dec33e7213
Author: Maxim Muzafarov <ma...@gmail.com>
AuthorDate: Tue Jun 14 16:46:12 2022 +0300
IGNITE-17137 Move ignite-cloud IP finder to the Ignite Extensions project (#157)
---
modules/cloud-ext/README.txt | 32 ++
modules/cloud-ext/assembly/cloud-ext.xml | 34 ++
modules/cloud-ext/licenses/apache-2.0.txt | 202 +++++++++
modules/cloud-ext/pom.xml | 117 +++++
.../ipfinder/cloud/TcpDiscoveryCloudIpFinder.java | 481 +++++++++++++++++++++
.../discovery/tcp/ipfinder/cloud/package-info.java | 22 +
.../cloud/TcpDiscoveryCloudIpFinderSelfTest.java | 119 +++++
.../discovery/tcp/ipfinder/cloud/package-info.java | 23 +
.../ignite/testsuites/IgniteCloudTestSuite.java | 103 +++++
pom.xml | 1 +
10 files changed, 1134 insertions(+)
diff --git a/modules/cloud-ext/README.txt b/modules/cloud-ext/README.txt
new file mode 100644
index 0000000..015aad9
--- /dev/null
+++ b/modules/cloud-ext/README.txt
@@ -0,0 +1,32 @@
+Apache Ignite Cloud Module
+------------------------
+
+Apache Ignite Cloud module provides Apache jclouds implementations of IP finder for TCP discovery.
+
+To enable Cloud module when starting a standalone node, move 'ignite-cloud' folder to
+'libs' folder before running 'ignite.{sh|bat}' script. The content of the module folder will
+be added to classpath in this case.
+
+Importing Cloud Module In Maven Project
+-------------------------------------
+
+If you are using Maven to manage dependencies of your project, you can add Cloud module
+dependency like this (replace '${ignite.version}' with actual Ignite version you are
+interested in):
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
+ http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ ...
+ <dependencies>
+ ...
+ <dependency>
+ <groupId>org.apache.ignite</groupId>
+ <artifactId>ignite-cloud-ext</artifactId>
+ <version>1.0.0</version>
+ </dependency>
+ ...
+ </dependencies>
+ ...
+</project>
diff --git a/modules/cloud-ext/assembly/cloud-ext.xml b/modules/cloud-ext/assembly/cloud-ext.xml
new file mode 100644
index 0000000..0c28f4c
--- /dev/null
+++ b/modules/cloud-ext/assembly/cloud-ext.xml
@@ -0,0 +1,34 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2
+ http://maven.apache.org/xsd/assembly-1.1.2.xsd">
+ <id>bin</id>
+ <includeBaseDirectory>false</includeBaseDirectory>
+
+ <formats>
+ <format>zip</format>
+ </formats>
+
+ <componentDescriptors>
+ <componentDescriptor>../../assembly/bin-component-shared.xml</componentDescriptor>
+ </componentDescriptors>
+</assembly>
diff --git a/modules/cloud-ext/licenses/apache-2.0.txt b/modules/cloud-ext/licenses/apache-2.0.txt
new file mode 100644
index 0000000..d645695
--- /dev/null
+++ b/modules/cloud-ext/licenses/apache-2.0.txt
@@ -0,0 +1,202 @@
+
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
diff --git a/modules/cloud-ext/pom.xml b/modules/cloud-ext/pom.xml
new file mode 100644
index 0000000..8006223
--- /dev/null
+++ b/modules/cloud-ext/pom.xml
@@ -0,0 +1,117 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.ignite</groupId>
+ <artifactId>ignite-parent-ext-internal</artifactId>
+ <version>1</version>
+ <relativePath>../../parent-internal/pom.xml</relativePath>
+ </parent>
+
+ <artifactId>ignite-cloud-ext</artifactId>
+ <version>1.0.0-SNAPSHOT</version>
+
+ <url>https://ignite.apache.org</url>
+
+ <properties>
+ <jcloud.version>2.0.0</jcloud.version>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>ignite-core</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.jclouds</groupId>
+ <artifactId>jclouds-allcompute</artifactId>
+ <version>${jcloud.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.jclouds.labs</groupId>
+ <artifactId>google-compute-engine</artifactId>
+ <version>1.9.3</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.jclouds.labs</groupId>
+ <artifactId>docker</artifactId>
+ <version>1.9.3</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.jclouds.provider</groupId>
+ <artifactId>cloudsigma-zrh</artifactId>
+ <version>1.8.0</version>
+ </dependency>
+
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>ignite-core</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-beans</artifactId>
+ <version>${spring.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-context</artifactId>
+ <version>${spring.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-core</artifactId>
+ <version>${spring.version}</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-deploy-plugin</artifactId>
+ <version>2.8.2</version>
+ <configuration>
+ <skip>false</skip>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/modules/cloud-ext/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/cloud/TcpDiscoveryCloudIpFinder.java b/modules/cloud-ext/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/cloud/TcpDiscoveryCloudIpFinder.java
new file mode 100644
index 0000000..57e8c50
--- /dev/null
+++ b/modules/cloud-ext/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/cloud/TcpDiscoveryCloudIpFinder.java
@@ -0,0 +1,481 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.tcp.ipfinder.cloud;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.Properties;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicBoolean;
+import com.google.common.base.Charsets;
+import com.google.common.base.Predicate;
+import com.google.common.base.Supplier;
+import com.google.common.io.Files;
+import org.apache.ignite.internal.IgniteInterruptedCheckedException;
+import org.apache.ignite.internal.util.tostring.GridToStringExclude;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.spi.IgniteSpiConfiguration;
+import org.apache.ignite.spi.IgniteSpiException;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinderAdapter;
+import org.jclouds.Constants;
+import org.jclouds.ContextBuilder;
+import org.jclouds.compute.ComputeService;
+import org.jclouds.compute.ComputeServiceContext;
+import org.jclouds.compute.domain.ComputeMetadata;
+import org.jclouds.compute.domain.NodeMetadata;
+import org.jclouds.domain.Credentials;
+import org.jclouds.domain.Location;
+import org.jclouds.googlecloud.GoogleCredentialsFromJson;
+import org.jclouds.location.reference.LocationConstants;
+
+/**
+ * IP finder for automatic lookup of nodes running in a cloud.
+ * <p>
+ * Implementation is based on Apache jclouds multi-cloud toolkit.
+ * For information about jclouds visit <a href="https://jclouds.apache.org/">jclouds.apache.org</a>.
+ * <h1 class="header">Configuration</h1>
+ * <h2 class="header">Mandatory</h2>
+ * <ul>
+ * <li>Cloud provider (see {@link #setProvider(String)})</li>
+ * <li>Identity (see {@link #setIdentity(String)})</li>
+ * </ul>
+ * <h2 class="header">Optional</h2>
+ * <ul>
+ * <li>Credential (see {@link #setCredential(String)})</li>
+ * <li>Credential path (see {@link #setCredentialPath(String)}</li>
+ * <li>Regions (see {@link #setRegions(Collection)})</li>
+ * <li>Zones (see {@link #setZones(Collection)}</li>
+ * </ul>
+ * </p>
+ * <p>
+ * The finder forms nodes addresses, that possibly running Ignite, by getting private and public IPs of all
+ * VMs in a cloud and adding a port number to them.
+ * The port is either the one that is set with {@link TcpDiscoverySpi#setLocalPort(int)} or
+ * {@link TcpDiscoverySpi#DFLT_PORT}.
+ * Make sure that all VMs start Ignite instances on the same port, otherwise they will not be able to discover each
+ * other using this IP finder.
+ * </p>
+ * <p>
+ * Both {@link #registerAddresses(Collection)} and {@link #unregisterAddresses(Collection)} has no effect.
+ * </p>
+ * <p>
+ * Note, this finder is only workable when it used directly by cloud VM.
+ * Choose another implementation of {@link org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder} for local
+ * or home network tests.
+ * </p>
+ * <h2 class="header">Java Example</h2>
+ * <pre name="code" class="java">
+ * String accountId = "your_account_id";
+ * String accountKey = "your_account_key";
+ *
+ * TcpDiscoveryCloudIpFinder ipFinder = new TcpDiscoveryCloudIpFinder();
+ *
+ * ipFinder.setProvider("aws-ec2");
+ * ipFinder.setIdentity(accountId);
+ * ipFinder.setCredential(accountKey);
+ * ipFinder.setRegions(Collections.<String>emptyList().add("us-east-1"));
+ * ipFinder.setZones(Arrays.asList("us-east-1b", "us-east-1e"));
+ * </pre>
+ * <h2 class="header">Spring Example</h2>
+ * TcpDiscoveryCloudIpFinder can be configured from Spring XML configuration file:
+ * <pre name="code" class="xml">
+ * <bean id="grid.custom.cfg" class="org.apache.ignite.configuration.IgniteConfiguration" singleton="true">
+ * ...
+ * <property name="discoverySpi">
+ * <bean class="org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi">
+ * <property name="ipFinder">
+ * <bean class="org.apache.ignite.spi.discovery.tcp.ipfinder.cloud.TcpDiscoveryCloudIpFinder"/>
+ * <property name="provider" value="google-compute-engine"/>
+ * <property name="identity" value="your_service_account_email"/>
+ * <property name="credentialPath" value="path_to_json_key"/>
+ * <property name="zones">
+ * <list>
+ * <value>us-central1-a</value>
+ * <value>asia-east1-a</value>
+ * </list>
+ * </property>
+ * </bean>
+ * </property>
+ *
+ * <property name="socketTimeout" value="400"/>
+ * </bean>
+ * </property>
+ * ...
+ * </bean>
+ * </pre>
+ * <p>
+ * <img src="http://ignite.apache.org/images/spring-small.png">
+ * <br>
+ * For information about Spring framework visit <a href="http://www.springframework.org/">www.springframework.org</a>
+ */
+public class TcpDiscoveryCloudIpFinder extends TcpDiscoveryIpFinderAdapter {
+ /** JCloud default connection timeout. */
+ private static final String JCLOUD_CONNECTION_TIMEOUT = "10000"; //10 secs
+
+ /** Cloud provider. */
+ private String provider;
+
+ /** Cloud specific identity (user name, email address, etc.). */
+ private String identity;
+
+ /** Cloud specific credential (password, access key, etc.). */
+ @GridToStringExclude
+ private String credential;
+
+ /** Path to a cloud specific credential. */
+ @GridToStringExclude
+ private String credentialPath;
+
+ /** Regions where VMs are located. */
+ private TreeSet<String> regions;
+
+ /** Zones where VMs are located. */
+ private TreeSet<String> zones;
+
+ /** Nodes filter by regions and zones. */
+ private Predicate<ComputeMetadata> nodesFilter;
+
+ /** Init guard. */
+ @GridToStringExclude
+ private final AtomicBoolean initGuard = new AtomicBoolean();
+
+ /** Init latch. */
+ @GridToStringExclude
+ private final CountDownLatch initLatch = new CountDownLatch(1);
+
+ /** JCloud compute service. */
+ private ComputeService computeService;
+
+ /**
+ * Constructor.
+ */
+ public TcpDiscoveryCloudIpFinder() {
+ setShared(true);
+ }
+
+ /** {@inheritDoc} */
+ @Override public Collection<InetSocketAddress> getRegisteredAddresses() throws IgniteSpiException {
+ initComputeService();
+
+ Collection<InetSocketAddress> addresses = new LinkedList<>();
+
+ try {
+ Set<NodeMetadata> nodes;
+
+ if (nodesFilter != null)
+ nodes = (Set<NodeMetadata>)computeService.listNodesDetailsMatching(nodesFilter);
+ else {
+ nodes = new HashSet<>();
+
+ for (ComputeMetadata metadata : computeService.listNodes())
+ nodes.add(computeService.getNodeMetadata(metadata.getId()));
+ }
+
+ for (NodeMetadata metadata : nodes) {
+ if (metadata.getStatus() != NodeMetadata.Status.RUNNING)
+ continue;
+
+ for (String addr : metadata.getPrivateAddresses())
+ addresses.add(new InetSocketAddress(addr, 0));
+
+ for (String addr : metadata.getPublicAddresses())
+ addresses.add(new InetSocketAddress(addr, 0));
+ }
+ }
+ catch (Exception e) {
+ throw new IgniteSpiException("Failed to get registered addresses for the provider: " + provider, e);
+ }
+
+ return addresses;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void registerAddresses(Collection<InetSocketAddress> addrs) throws IgniteSpiException {
+ // No-op
+ }
+
+ /** {@inheritDoc} */
+ @Override public void unregisterAddresses(Collection<InetSocketAddress> addrs) throws IgniteSpiException {
+ // No-op
+ }
+
+ /**
+ * Sets the cloud provider to use.
+ *
+ * <a href="https://jclouds.apache.org/reference/providers/#compute">Apache jclouds providers list</a> from
+ * ComputeService section contains names of all supported providers.
+ *
+ * @param provider Provider name.
+ * @return {@code this} for chaining.
+ */
+ @IgniteSpiConfiguration(optional = false)
+ public TcpDiscoveryCloudIpFinder setProvider(String provider) {
+ this.provider = provider;
+
+ return this;
+ }
+
+ /**
+ * Sets the identity that is used as a user name during a connection to the cloud.
+ * Depending on a cloud platform it can be an email address, user name, etc.
+ *
+ * Refer to <a href="http://jclouds.apache.org/guides/">Apache jclouds guide</a> to get concrete information on
+ * what is used as an identity for a particular cloud platform.
+ *
+ * @param identity Identity to use during authentication on the cloud.
+ * @return {@code this} for chaining.
+ */
+ @IgniteSpiConfiguration(optional = false)
+ public TcpDiscoveryCloudIpFinder setIdentity(String identity) {
+ this.identity = identity;
+
+ return this;
+ }
+
+ /**
+ * Sets credential that is used during authentication on the cloud.
+ * Depending on a cloud platform it can be a password or access key.
+ *
+ * Refer to <a href="http://jclouds.apache.org/guides/">Apache jclouds guide</a> to get concrete information on
+ * what is used as an credential for a particular cloud platform.
+ *
+ * @param credential Credential to use during authentication on the cloud.
+ * @return {@code this} for chaining.
+ */
+ @IgniteSpiConfiguration(optional = true)
+ public TcpDiscoveryCloudIpFinder setCredential(String credential) {
+ this.credential = credential;
+
+ return this;
+ }
+
+ /**
+ * Sets the path to a credential that is used during authentication on the cloud.
+ *
+ * This method should be used when an access key or private key is stored in a file.
+ * Content of the file, referred by @{code credentialPath}, is fully read and used as a access key or private key
+ * during authentication.
+ *
+ * Refer to <a href="http://jclouds.apache.org/guides/">Apache jclouds guide</a> to get concrete information on
+ * what is used as an credential for a particular cloud platform.
+ *
+ * @param credentialPath Path to the credential to use during authentication on the cloud.
+ * @return {@code this} for chaining.
+ */
+ @IgniteSpiConfiguration(optional = true)
+ public TcpDiscoveryCloudIpFinder setCredentialPath(String credentialPath) {
+ this.credentialPath = credentialPath;
+
+ return this;
+ }
+
+ /**
+ * Sets list of zones where VMs are located.
+ *
+ * If the zones are not set then every zone from regions, set by {@link #setRegions(Collection)}}, will be
+ * taken into account.
+ *
+ * Note, that some cloud providers, like Rackspace, doesn't have a notion of a zone. For such
+ * providers a call to this method is redundant.
+ *
+ * @param zones Zones where VMs are located or null if to take every zone into account.
+ * @return {@code this} for chaining.
+ */
+ @IgniteSpiConfiguration(optional = true)
+ public TcpDiscoveryCloudIpFinder setZones(Collection<String> zones) {
+ if (!F.isEmpty(zones))
+ this.zones = new TreeSet<>(zones);
+
+ return this;
+ }
+
+ /**
+ * Sets list of regions where VMs are located.
+ *
+ * If the regions are not set then every region, that a cloud provider has, will be investigated. This could lead
+ * to significant performance degradation.
+ *
+ * Note, that some cloud providers, like Google Compute Engine, doesn't have a notion of a region. For such
+ * providers a call to this method is redundant.
+ *
+ * @param regions Regions where VMs are located or null if to check every region a provider has.
+ * @return {@code this} for chaining.
+ */
+ @IgniteSpiConfiguration(optional = true)
+ public TcpDiscoveryCloudIpFinder setRegions(Collection<String> regions) {
+ if (!F.isEmpty(regions))
+ this.regions = new TreeSet<>(regions);
+
+ return this;
+ }
+
+ /**
+ * Initializes Apache jclouds compute service.
+ */
+ private void initComputeService() {
+ if (initGuard.compareAndSet(false, true))
+ try {
+ if (provider == null)
+ throw new IgniteSpiException("Cloud provider is not set.");
+
+ if (identity == null)
+ throw new IgniteSpiException("Cloud identity is not set.");
+
+ if (credential != null && credentialPath != null)
+ throw new IgniteSpiException("Both credential and credentialPath are set. Use only one method.");
+
+ if (credentialPath != null)
+ credential = getCredentialFromFile();
+
+ try {
+ ContextBuilder ctxBuilder = ContextBuilder.newBuilder(provider);
+
+ ctxBuilder.credentials(identity, credential);
+
+ Properties properties = new Properties();
+ properties.setProperty(Constants.PROPERTY_SO_TIMEOUT, JCLOUD_CONNECTION_TIMEOUT);
+ properties.setProperty(Constants.PROPERTY_CONNECTION_TIMEOUT, JCLOUD_CONNECTION_TIMEOUT);
+
+ if (!F.isEmpty(regions))
+ properties.setProperty(LocationConstants.PROPERTY_REGIONS, keysSetToStr(regions));
+
+ if (!F.isEmpty(zones))
+ properties.setProperty(LocationConstants.PROPERTY_ZONES, keysSetToStr(zones));
+
+ ctxBuilder.overrides(properties);
+
+ computeService = ctxBuilder.buildView(ComputeServiceContext.class).getComputeService();
+
+ if (!F.isEmpty(zones) || !F.isEmpty(regions)) {
+ nodesFilter = new Predicate<ComputeMetadata>() {
+ @Override public boolean apply(ComputeMetadata computeMetadata) {
+ String region = null;
+ String zone = null;
+
+ Location location = computeMetadata.getLocation();
+
+ while (location != null) {
+ switch (location.getScope()) {
+ case ZONE:
+ zone = location.getId();
+ break;
+
+ case REGION:
+ region = location.getId();
+ break;
+ }
+
+ location = location.getParent();
+ }
+
+ if (regions != null && region != null && !regions.contains(region))
+ return false;
+
+ if (zones != null && zone != null && !zones.contains(zone))
+ return false;
+
+ return true;
+ }
+ };
+ }
+ }
+ catch (Exception e) {
+ throw new IgniteSpiException("Failed to connect to the provider: " + provider, e);
+ }
+ }
+ finally {
+ initLatch.countDown();
+ }
+ else {
+ try {
+ U.await(initLatch);
+ }
+ catch (IgniteInterruptedCheckedException e) {
+ throw new IgniteSpiException("Thread has been interrupted.", e);
+ }
+
+ if (computeService == null)
+ throw new IgniteSpiException("Ip finder has not been initialized properly.");
+ }
+ }
+
+ /**
+ * Reads credential info from {@link #credentialPath} and returns in a string format.
+ *
+ * @return Credential in {@code String} representation.
+ * @throws IgniteSpiException In case of error.
+ */
+ private String getCredentialFromFile() throws IgniteSpiException {
+ try {
+ String fileContents = Files.toString(new File(credentialPath), Charsets.UTF_8);
+
+ if (provider.equals("google-compute-engine")) {
+ Supplier<Credentials> credentialSupplier = new GoogleCredentialsFromJson(fileContents);
+
+ return credentialSupplier.get().credential;
+ }
+
+ return fileContents;
+ }
+ catch (IOException e) {
+ throw new IgniteSpiException("Failed to retrieve the private key from the file: " + credentialPath, e);
+ }
+ }
+
+ /**
+ * Converts set keys to string.
+ *
+ * @param set Set.
+ * @return String where keys delimited by ','.
+ */
+ private String keysSetToStr(Set<String> set) {
+ Iterator<String> iter = set.iterator();
+ StringBuilder builder = new StringBuilder();
+
+ while (iter.hasNext()) {
+ builder.append(iter.next());
+
+ if (iter.hasNext())
+ builder.append(',');
+ }
+
+ return builder.toString();
+ }
+
+ /** {@inheritDoc} */
+ @Override public TcpDiscoveryCloudIpFinder setShared(boolean shared) {
+ super.setShared(shared);
+
+ return this;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(TcpDiscoveryCloudIpFinder.class, this);
+ }
+}
diff --git a/modules/cloud-ext/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/cloud/package-info.java b/modules/cloud-ext/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/cloud/package-info.java
new file mode 100644
index 0000000..039ca9d
--- /dev/null
+++ b/modules/cloud-ext/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/cloud/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * <!-- Package description. -->
+ * Contains Apache jclouds based IP finder.
+ */
+
+package org.apache.ignite.spi.discovery.tcp.ipfinder.cloud;
diff --git a/modules/cloud-ext/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/cloud/TcpDiscoveryCloudIpFinderSelfTest.java b/modules/cloud-ext/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/cloud/TcpDiscoveryCloudIpFinderSelfTest.java
new file mode 100644
index 0000000..378769c
--- /dev/null
+++ b/modules/cloud-ext/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/cloud/TcpDiscoveryCloudIpFinderSelfTest.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.tcp.ipfinder.cloud;
+
+import java.net.InetSocketAddress;
+import java.util.Collection;
+import com.google.common.collect.ImmutableList;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinderAbstractSelfTest;
+import org.apache.ignite.testsuites.IgniteCloudTestSuite;
+import org.junit.Ignore;
+import org.junit.Test;
+
+/**
+ * TcpDiscoveryCloudIpFinder test.
+ */
+public class TcpDiscoveryCloudIpFinderSelfTest extends
+ TcpDiscoveryIpFinderAbstractSelfTest<TcpDiscoveryCloudIpFinder> {
+ /**
+ * Constructor.
+ *
+ * @throws Exception If any error occurs.
+ */
+ public TcpDiscoveryCloudIpFinderSelfTest() throws Exception {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
+ @Override protected TcpDiscoveryCloudIpFinder ipFinder() throws Exception {
+ // No-op.
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Test
+ @Override public void testIpFinder() throws Exception {
+ // No-op
+ }
+
+ /**
+ * Tests GCE.
+ *
+ * @throws Exception If any error occurs.
+ */
+ @Ignore("https://issues.apache.org/jira/browse/IGNITE-1585")
+ @Test
+ public void testGoogleComputeEngine() throws Exception {
+ testCloudProvider("google-compute-engine");
+ }
+
+ /**
+ * Tests Rackspace.
+ *
+ * @throws Exception If any error occurs.
+ */
+ @Ignore("https://issues.apache.org/jira/browse/IGNITE-9444")
+ @Test
+ public void testRackspace() throws Exception {
+ testCloudProvider("rackspace-cloudservers-us");
+ }
+
+ /**
+ * Tests a given provider.
+ *
+ * @param provider Provider name.
+ * @throws Exception If any error occurs.
+ */
+ private void testCloudProvider(String provider) throws Exception {
+ info("Testing provider: " + provider);
+
+ TcpDiscoveryCloudIpFinder ipFinder = new TcpDiscoveryCloudIpFinder();
+
+ injectLogger(ipFinder);
+
+ ipFinder.setProvider(provider);
+ ipFinder.setIdentity(IgniteCloudTestSuite.getAccessKey(provider));
+ ipFinder.setRegions(IgniteCloudTestSuite.getRegions(provider));
+ ipFinder.setZones(IgniteCloudTestSuite.getZones(provider));
+
+ if (provider.equals("google-compute-engine"))
+ ipFinder.setCredentialPath(IgniteCloudTestSuite.getSecretKey(provider));
+ else
+ ipFinder.setCredential(IgniteCloudTestSuite.getSecretKey(provider));
+
+ Collection<InetSocketAddress> addresses = ipFinder.getRegisteredAddresses();
+
+ for (InetSocketAddress addr : addresses)
+ info("Registered instance: " + addr.getAddress().getHostAddress() + ":" + addr.getPort());
+
+ ipFinder.unregisterAddresses(addresses);
+
+ assert addresses.size() == ipFinder.getRegisteredAddresses().size();
+
+ ipFinder.registerAddresses(ImmutableList.of(
+ new InetSocketAddress("192.168.0.1", TcpDiscoverySpi.DFLT_PORT)));
+
+ assert addresses.size() == ipFinder.getRegisteredAddresses().size();
+ }
+}
diff --git a/modules/cloud-ext/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/cloud/package-info.java b/modules/cloud-ext/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/cloud/package-info.java
new file mode 100644
index 0000000..ec314a5
--- /dev/null
+++ b/modules/cloud-ext/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/cloud/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <!-- Package description. -->
+ * Contains internal tests or test related classes and interfaces.
+ */
+
+package org.apache.ignite.spi.discovery.tcp.ipfinder.cloud;
diff --git a/modules/cloud-ext/src/test/java/org/apache/ignite/testsuites/IgniteCloudTestSuite.java b/modules/cloud-ext/src/test/java/org/apache/ignite/testsuites/IgniteCloudTestSuite.java
new file mode 100644
index 0000000..f8780f4
--- /dev/null
+++ b/modules/cloud-ext/src/test/java/org/apache/ignite/testsuites/IgniteCloudTestSuite.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.testsuites;
+
+import java.util.Collection;
+import java.util.LinkedList;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.cloud.TcpDiscoveryCloudIpFinderSelfTest;
+import org.junit.runner.RunWith;
+import org.junit.runners.Suite;
+
+/**
+ * Ignite Cloud integration test.
+ */
+@RunWith(Suite.class)
+@Suite.SuiteClasses({TcpDiscoveryCloudIpFinderSelfTest.class})
+public class IgniteCloudTestSuite {
+ /**
+ * <a href="http://jclouds.apache.org/guides/google/">Service Account email<a/> for GCE.
+ * <a href="http://jclouds.apache.org/guides/aws/">Access Key ID</a> for Amazon.
+ * <a href="http://jclouds.apache.org/guides/rackspace/">Username</a> for Rackspace.
+ *
+ * @return Access key.
+ */
+ public static String getAccessKey(String provider) {
+ String key = System.getenv("test." + provider + ".access.key");
+
+ assert key != null : "Environment variable 'test." + provider + ".access.key' is not set";
+
+ return key;
+ }
+
+ /**
+ * <a href="http://jclouds.apache.org/guides/google/">Path to pkcs12 file<a/> for GCE.
+ * <a href="http://jclouds.apache.org/guides/aws/">Access Key</a> for Amazon.
+ * <a href="http://jclouds.apache.org/guides/rackspace/">API key</a> for Rackspace.
+ *
+ * @return Secret key.
+ */
+ public static String getSecretKey(String provider) {
+ String key = System.getenv("test." + provider + ".secret.key");
+
+ assert key != null : "Environment variable 'test." + provider + ".secret.key' is not set";
+
+ return key;
+ }
+
+ /**
+ * Zones where VMs are located.
+ *
+ * @return Zones list or null.
+ */
+ public static Collection<String> getZones(String provider) {
+ String zonesStr = System.getenv("test." + provider + ".zones.list");
+
+ if (zonesStr == null)
+ return null;
+
+ String[] zonesArr = zonesStr.split(",");
+
+ LinkedList<String> list = new LinkedList<>();
+
+ for (String zone : zonesArr)
+ list.add(zone.trim());
+
+ return list;
+ }
+
+ /**
+ * Regions where VMs are located.
+ *
+ * @return Zones list or null.
+ */
+ public static Collection<String> getRegions(String provider) {
+ String regionStr = System.getenv("test." + provider + ".regions.list");
+
+ if (regionStr == null)
+ return null;
+
+ String[] zonesArr = regionStr.split(",");
+
+ LinkedList<String> list = new LinkedList<>();
+
+ for (String zone : zonesArr)
+ list.add(zone.trim());
+
+ return list;
+ }
+}
diff --git a/pom.xml b/pom.xml
index 93863ed..4c3ad1a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -60,6 +60,7 @@
<module>modules/spark-ext</module>
<module>modules/mesos-ext</module>
<module>modules/yarn-ext</module>
+ <module>modules/cloud-ext</module>
</modules>
<profiles>