You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by il...@apache.org on 2021/04/16 16:53:38 UTC
[ignite] branch master updated: IGNITE-14346 Implement Azure Blob
Storage based IP Finder - Fixes #8897.
This is an automated email from the ASF dual-hosted git repository.
ilyak pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push:
new e5d266e IGNITE-14346 Implement Azure Blob Storage based IP Finder - Fixes #8897.
e5d266e is described below
commit e5d266ede810c39b4ad8a3376b50ae689d5c8b86
Author: Atri Sharma <at...@gmail.com>
AuthorDate: Fri Apr 16 18:37:32 2021 +0300
IGNITE-14346 Implement Azure Blob Storage based IP Finder - Fixes #8897.
Signed-off-by: Ilya Kasnacheev <il...@gmail.com>
---
assembly/dependencies-apache-ignite-slim.xml | 1 +
assembly/libs/README.txt | 1 +
docs/_docs/clustering/discovery-in-the-cloud.adoc | 42 +++
docs/_docs/code-snippets/java/pom.xml | 5 +
.../ignite/snippets/DiscoveryInTheCloud.java | 24 ++
docs/_docs/setup.adoc | 1 +
modules/azure/README.txt | 32 ++
modules/azure/pom.xml | 342 ++++++++++++++++++
.../azure/TcpDiscoveryAzureBlobStoreIpFinder.java | 382 +++++++++++++++++++++
.../spi/discovery/tcp/ipfinder/package-info.java | 23 ++
...TcpDiscoveryAzureBlobStoreIpFinderSelfTest.java | 91 +++++
.../discovery/tcp/ipfinder/azure/package-info.java | 21 ++
.../ignite/testsuites/IgniteAzureTestSuite.java | 61 ++++
parent/pom.xml | 2 +
pom.xml | 1 +
15 files changed, 1029 insertions(+)
diff --git a/assembly/dependencies-apache-ignite-slim.xml b/assembly/dependencies-apache-ignite-slim.xml
index 6d222c7..43f0fdf 100644
--- a/assembly/dependencies-apache-ignite-slim.xml
+++ b/assembly/dependencies-apache-ignite-slim.xml
@@ -145,6 +145,7 @@
<!-- Removed from slim packaging are: -->
<exclude>org.apache.ignite:ignite-aop</exclude>
<exclude>org.apache.ignite:ignite-aws</exclude>
+ <exclude>org.apache.ignite:ignite-azure</exclude>
<exclude>org.apache.ignite:ignite-cassandra-serializers</exclude>
<exclude>org.apache.ignite:ignite-cassandra-store</exclude>
<exclude>org.apache.ignite:ignite-cloud</exclude>
diff --git a/assembly/libs/README.txt b/assembly/libs/README.txt
index 39f342b..12d1256 100644
--- a/assembly/libs/README.txt
+++ b/assembly/libs/README.txt
@@ -73,6 +73,7 @@ All optional modules can be imported just like the core module, but with differe
The following modules are available:
- ignite-aop (for AOP-based grid-enabling)
- ignite-aws (for seemless cluster discovery on AWS S3)
+- ignite-azure (for automatic cluster discovery on Azure Blob Storage)
- ignite-cassandra (for Apache Cassandra integration)
- ignite-cloud (for Apache JClouds integration)
- ignite-gce (for automatic cluster discovery on Google Compute Engine)
diff --git a/docs/_docs/clustering/discovery-in-the-cloud.adoc b/docs/_docs/clustering/discovery-in-the-cloud.adoc
index 0d74b81..4c73f7825 100644
--- a/docs/_docs/clustering/discovery-in-the-cloud.adoc
+++ b/docs/_docs/clustering/discovery-in-the-cloud.adoc
@@ -34,6 +34,7 @@ To mitigate the constantly changing IP addresses problem, Ignite supports a numb
* Amazon S3 IP Finder
* Amazon ELB IP Finder
* Google Cloud Storage IP Finder
+* Azure Blob Storage IP Finder
TIP: Cloud-based IP Finders allow you to create your configuration once and reuse it for all instances.
@@ -268,3 +269,44 @@ include::{javaFile}[tag=google,indent=0]
tab:C#/.NET[unsupported]
tab:C++[unsupported]
--
+
+== Azure Blob Storage
+
+Ignite supports automatic node discovery by utilizing Azure Blob Storage.
+This mechanism is implemented in `TcpDiscoveryAzureBlobStorageIpFinder`.
+On start-up, each node registers its IP address in the storage and discovers other nodes by reading the storage.
+
+IMPORTANT: To use `TcpDiscoveryAzureBlobStorageIpFinder`, enable the `ignite-azure` link:setup#enabling-modules[module] in your application.
+
+Here is an example of how to configure Azure Blob Storage based IP finder:
+
+[tabs]
+--
+tab:XML[]
+[source,xml]
+----
+<bean class="org.apache.ignite.configuration.IgniteConfiguration">
+
+ <property name="discoverySpi">
+ <bean class="org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi">
+ <property name="ipFinder">
+ <bean class="org.apache.ignite.spi.discovery.tcp.ipfinder.azure.TcpDiscoveryAzureBlobStoreIpFinder">
+ <property name="accountName" value="YOUR_AZURE_BLOB_STORAGE_ACCOUNT_NAME"/>
+ <property name="accountKey" value="YOUR_AZURE_BLOB_STORAGE_ACCOUNT_KEY"/>
+ <property name="accountEndpoint" value="YOUR_END_POINT"/>
+ <property name="containerName" value="YOUR_CONTAINER_NAME"/>
+ </bean>
+ </property>
+ </bean>
+ </property>
+</bean>
+----
+
+tab:Java[]
+[source,java]
+----
+include::{javaFile}[tag=google,indent=0]
+----
+tab:C#/.NET[unsupported]
+tab:C++[unsupported]
+--
diff --git a/docs/_docs/code-snippets/java/pom.xml b/docs/_docs/code-snippets/java/pom.xml
index de5623d..4589866 100644
--- a/docs/_docs/code-snippets/java/pom.xml
+++ b/docs/_docs/code-snippets/java/pom.xml
@@ -91,6 +91,11 @@
</dependency>
<dependency>
<groupId>org.apache.ignite</groupId>
+ <artifactId>ignite-azure</artifactId>
+ <version>${ignite.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.ignite</groupId>
<artifactId>ignite-compress</artifactId>
<version>${ignite.version}</version>
</dependency>
diff --git a/docs/_docs/code-snippets/java/src/main/java/org/apache/ignite/snippets/DiscoveryInTheCloud.java b/docs/_docs/code-snippets/java/src/main/java/org/apache/ignite/snippets/DiscoveryInTheCloud.java
index 576b36d..607eb5a 100644
--- a/docs/_docs/code-snippets/java/src/main/java/org/apache/ignite/snippets/DiscoveryInTheCloud.java
+++ b/docs/_docs/code-snippets/java/src/main/java/org/apache/ignite/snippets/DiscoveryInTheCloud.java
@@ -148,4 +148,28 @@ public class DiscoveryInTheCloud {
Ignition.start(cfg);
//end::google[]
}
+
+ public static void azureBlobStorageExample() {
+ //tag::azureBlobStorage[]
+ TcpDiscoverySpi spi = new TcpDiscoverySpi();
+
+ TcpDiscoveryAzureBlobStorageIpFinder ipFinder = new TcpDiscoveryGoogleStorageIpFinder();
+
+ finder.setAccountName("yourAccountName");
+ finder.setAccountKey("yourAccountKey");
+ finder.setAccountEndpoint("yourEndpoint");
+
+ finder.setContainerName("yourContainerName");
+
+ spi.setIpFinder(ipFinder);
+
+ IgniteConfiguration cfg = new IgniteConfiguration();
+
+ // Override default discovery SPI.
+ cfg.setDiscoverySpi(spi);
+
+ // Start the node.
+ Ignition.start(cfg);
+ //end::azureBlobStorage[]
+ }
}
diff --git a/docs/_docs/setup.adoc b/docs/_docs/setup.adoc
index 603934b..754f4e7 100644
--- a/docs/_docs/setup.adoc
+++ b/docs/_docs/setup.adoc
@@ -203,6 +203,7 @@ adding @Gridify annotation to it.
|ignite-aws |Cluster discovery on AWS S3. Refer to link:clustering/discovery-in-the-cloud#amazon-s3-ip-finder[Amazon S3 IP Finder] for details.
+|ignite-azure| Ignite Azure provides Azure Blob Storage-based implementation of IP finder for TCP discovery.
|ignite-cassandra-serializers | The Ignite Cassandra Serializers module provides additional serializers to store objects as BLOBs in Cassandra. The module could be used as in conjunction with the Ignite Cassandra Store module.
diff --git a/modules/azure/README.txt b/modules/azure/README.txt
new file mode 100644
index 0000000..33501f0
--- /dev/null
+++ b/modules/azure/README.txt
@@ -0,0 +1,32 @@
+Apache Ignite Azure Module
+------------------------
+
+Apache Ignite Azure module provides Azure Blob Storage based implementation of IP finder for TCP discovery.
+
+To enable Azure module when starting a standalone node, move 'optional/ignite-azure' folder to
+'libs' folder before running 'ignite.{sh|bat}' script. The content of the module folder will
+be added to classpath in this case.
+
+Importing Azure Module In Maven Project
+-------------------------------------
+
+If you are using Maven to manage dependencies of your project, you can add Azure module
+dependency like this (replace '${ignite.version}' with actual Ignite version you are
+interested in):
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
+ http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ ...
+ <dependencies>
+ ...
+ <dependency>
+ <groupId>org.apache.ignite</groupId>
+ <artifactId>ignite-azure</artifactId>
+ <version>${ignite.version}</version>
+ </dependency>
+ ...
+ </dependencies>
+ ...
+</project>
diff --git a/modules/azure/pom.xml b/modules/azure/pom.xml
new file mode 100644
index 0000000..c3951b3
--- /dev/null
+++ b/modules/azure/pom.xml
@@ -0,0 +1,342 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>ignite-parent</artifactId>
+ <groupId>org.apache.ignite</groupId>
+ <version>1</version>
+ <relativePath>../../parent</relativePath>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>ignite-azure</artifactId>
+ <version>2.11.0-SNAPSHOT</version>
+ <url>http://ignite.apache.org</url>
+
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>com.azure</groupId>
+ <artifactId>azure-sdk-bom</artifactId>
+ <version>1.0.2</version>
+ <type>pom</type>
+ <scope>import</scope>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.ignite</groupId>
+ <artifactId>ignite-core</artifactId>
+ <version>${project.version}</version>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.ignite</groupId>
+ <artifactId>ignite-core</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.ignite</groupId>
+ <artifactId>ignite-tools</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <!-- https://mvnrepository.com/artifact/org.jetbrains/annotations -->
+ <dependency>
+ <groupId>org.jetbrains</groupId>
+ <artifactId>annotations</artifactId>
+ <version>${jetbrains.annotations.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.azure</groupId>
+ <artifactId>azure-core-http-netty</artifactId>
+ <version>1.0.0</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.azure</groupId>
+ <artifactId>azure-core</artifactId>
+ <version>1.0.0</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.azure</groupId>
+ <artifactId>azure-storage-blob</artifactId>
+ <version>${azure.sdk.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.azure</groupId>
+ <artifactId>azure-storage-common</artifactId>
+ <version>${azure.sdk.version}</version>
+ </dependency>
+
+ <!-- https://mvnrepository.com/artifact/com.fasterxml.jackson.core/jackson-annotations -->
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ <version>${jackson.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.azure</groupId>
+ <artifactId>azure-security-keyvault-secrets</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.azure</groupId>
+ <artifactId>azure-identity</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>${jackson.version}</version>
+ </dependency>
+
+ <!-- https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml -->
+ <dependency>
+ <groupId>com.fasterxml.jackson.dataformat</groupId>
+ <artifactId>jackson-dataformat-xml</artifactId>
+ <version>${jackson.version}</version>
+ </dependency>
+
+ <!-- https://mvnrepository.com/artifact/com.fasterxml.jackson.datatype/jackson-datatype-jsr310 -->
+ <dependency>
+ <groupId>com.fasterxml.jackson.datatype</groupId>
+ <artifactId>jackson-datatype-jsr310</artifactId>
+ <version>${jackson.version}</version>
+ </dependency>
+
+ <!-- https://mvnrepository.com/artifact/com.fasterxml.jackson.module/jackson-module-jaxb-annotations -->
+ <dependency>
+ <groupId>com.fasterxml.jackson.module</groupId>
+ <artifactId>jackson-module-jaxb-annotations</artifactId>
+ <version>${jackson.version}</version>
+ </dependency>
+
+ <!-- https://mvnrepository.com/artifact/org.apache.logging.log4j/log4j-api -->
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-api</artifactId>
+ <version>${log4j.version}</version>
+ </dependency>
+
+ <!-- https://mvnrepository.com/artifact/org.apache.logging.log4j/log4j-core -->
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-core</artifactId>
+ <version>${log4j.version}</version>
+ </dependency>
+
+ <!-- https://mvnrepository.com/artifact/org.apache.logging.log4j/log4j-slf4j-impl -->
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-slf4j-impl</artifactId>
+ <version>${log4j.version}</version>
+ </dependency>
+
+ <!-- https://mvnrepository.com/artifact/io.netty/netty-buffer -->
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-buffer</artifactId>
+ <version>${azure.netty.version}</version>
+ </dependency>
+
+ <!-- https://mvnrepository.com/artifact/io.netty/netty-codec -->
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-codec</artifactId>
+ <version>${azure.netty.version}</version>
+ </dependency>
+
+ <!-- https://mvnrepository.com/artifact/io.netty/netty-codec-http2 -->
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-codec-http2</artifactId>
+ <version>${azure.netty.version}</version>
+ </dependency>
+
+ <!-- https://mvnrepository.com/artifact/io.netty/netty-codec-http -->
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-codec-http</artifactId>
+ <version>${azure.netty.version}</version>
+ </dependency>
+
+ <!-- https://mvnrepository.com/artifact/io.netty/netty-codec-socks -->
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-codec-socks</artifactId>
+ <version>${azure.netty.version}</version>
+ </dependency>
+
+ <!-- https://mvnrepository.com/artifact/io.netty/netty-common -->
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-common</artifactId>
+ <version>${azure.netty.version}</version>
+ </dependency>
+
+ <!-- https://mvnrepository.com/artifact/io.netty/netty-handler -->
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-handler</artifactId>
+ <version>${azure.netty.version}</version>
+ </dependency>
+
+ <!-- https://mvnrepository.com/artifact/io.netty/netty-handler-proxy -->
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-handler-proxy</artifactId>
+ <version>${azure.netty.version}</version>
+ </dependency>
+
+ <!-- https://mvnrepository.com/artifact/io.netty/netty-resolver -->
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-resolver</artifactId>
+ <version>${azure.netty.version}</version>
+ </dependency>
+
+ <!-- https://mvnrepository.com/artifact/io.netty/netty-transport -->
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-transport</artifactId>
+ <version>${azure.netty.version}</version>
+ </dependency>
+
+ <!-- https://mvnrepository.com/artifact/io.netty/netty-transport-native-epoll -->
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-transport-native-epoll</artifactId>
+ <version>${azure.netty.version}</version>
+ </dependency>
+
+ <!-- https://mvnrepository.com/artifact/io.netty/netty-transport-native-kqueue -->
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-transport-native-kqueue</artifactId>
+ <version>${azure.netty.version}</version>
+ </dependency>
+
+ <!-- https://mvnrepository.com/artifact/io.netty/netty-transport-native-kqueue -->
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-transport-native-kqueue</artifactId>
+ <version>${azure.netty.version}</version>
+ </dependency>
+
+ <!-- https://mvnrepository.com/artifact/io.netty/netty-transport-native-unix-common -->
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-transport-native-unix-common</artifactId>
+ <version>${azure.netty.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>${slf4j.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.reactivestreams</groupId>
+ <artifactId>reactive-streams</artifactId>
+ <version>1.0.3</version>
+ </dependency>
+
+ <!-- https://mvnrepository.com/artifact/io.projectreactor/reactor-core -->
+ <dependency>
+ <groupId>io.projectreactor</groupId>
+ <artifactId>reactor-core</artifactId>
+ <version>3.3.0.RELEASE</version>
+ </dependency>
+
+ <!-- https://mvnrepository.com/artifact/io.projectreactor.netty/reactor-netty -->
+ <dependency>
+ <groupId>io.projectreactor.netty</groupId>
+ <artifactId>reactor-netty</artifactId>
+ <version>0.9.0.RELEASE</version>
+ </dependency>
+
+ <!-- https://mvnrepository.com/artifact/org.codehaus.woodstox/stax2-api -->
+ <dependency>
+ <groupId>org.codehaus.woodstox</groupId>
+ <artifactId>stax2-api</artifactId>
+ <version>4.2.1</version>
+ </dependency>
+
+ <!-- https://mvnrepository.com/artifact/com.fasterxml.jackson.core/jackson-core -->
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ <version>${jackson.version}</version>
+ </dependency>
+
+ <!-- https://mvnrepository.com/artifact/org.reflections/reflections -->
+ <dependency>
+ <groupId>org.reflections</groupId>
+ <artifactId>reflections</artifactId>
+ <version>0.9.12</version>
+ </dependency>
+
+ <!-- https://mvnrepository.com/artifact/org.slf4j/slf4j-simple -->
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-simple</artifactId>
+ <version>${slf4j.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-beans</artifactId>
+ <version>${spring.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-context</artifactId>
+ <version>${spring.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-core</artifactId>
+ <version>${spring.version}</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+</project>
diff --git a/modules/azure/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/azure/TcpDiscoveryAzureBlobStoreIpFinder.java b/modules/azure/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/azure/TcpDiscoveryAzureBlobStoreIpFinder.java
new file mode 100644
index 0000000..031554b
--- /dev/null
+++ b/modules/azure/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/azure/TcpDiscoveryAzureBlobStoreIpFinder.java
@@ -0,0 +1,382 @@
+package org.apache.ignite.spi.discovery.tcp.ipfinder.azure;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.UnsupportedEncodingException;
+import java.net.InetSocketAddress;
+import java.net.URLEncoder;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import com.azure.storage.blob.BlobContainerClient;
+import com.azure.storage.blob.BlobServiceClient;
+import com.azure.storage.blob.BlobServiceClientBuilder;
+import com.azure.storage.blob.models.BlobItem;
+import com.azure.storage.blob.models.BlobStorageException;
+import com.azure.storage.blob.specialized.BlockBlobClient;
+import com.azure.storage.common.StorageSharedKeyCredential;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.internal.IgniteInterruptedCheckedException;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.resources.LoggerResource;
+import org.apache.ignite.spi.IgniteSpiConfiguration;
+import org.apache.ignite.spi.IgniteSpiException;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinderAdapter;
+
+import static com.nimbusds.oauth2.sdk.util.URLUtils.CHARSET;
+
+/**
+ * Azure Blob Storage based IP Finder
+ * <p>
+ * For information about Blob Storage visit <a href="https://azure.microsoft.com/en-in/services/storage/blobs/">azure.microsoft.com</a>.
+ * <h1 class="header">Configuration</h1>
+ * <h2 class="header">Mandatory</h2>
+ * <ul>
+ * <li>AccountName (see {@link #setAccountName(String)})</li>
+ * <li>AccountKey (see {@link #setAccountKey(String)})</li>
+ * <li>Account Endpoint (see {@link #setAccountEndpoint(String)})</li>
+ * <li>Container Name (see {@link #setContainerName(String)})</li>
+ * </ul>
+ * <h2 class="header">Optional</h2>
+ * <ul>
+ * <li>Shared flag (see {@link #setShared(boolean)})</li>
+ * </ul>
+ * <p>
+ * The finder will create a container with the provided name. The container will contain entries named
+ * like the following: {@code 192.168.1.136#1001}.
+ * <p>
+ * Note that storing data in Azure Blob Storage service will result in charges to your Azure account.
+ * Choose another implementation of {@link org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder} for local
+ * or home network tests.
+ * <p>
+ * Note that this finder is shared by default (see {@link org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder#isShared()}.
+ */
+public class TcpDiscoveryAzureBlobStoreIpFinder extends TcpDiscoveryIpFinderAdapter {
+ /** Default object's content. */
+ private static final byte[] OBJECT_CONTENT = new byte[0];
+
+ /** Grid logger. */
+ @LoggerResource
+ private IgniteLogger log;
+
+ /** Azure Blob Storage's account name*/
+ private String accountName;
+
+ /** Azure Blob Storage's account key */
+ private String accountKey;
+
+ /** End point URL */
+ private String endPoint;
+
+ /** Container name */
+ private String containerName;
+
+ /** Storage credential */
+ StorageSharedKeyCredential credential;
+
+ /** Blob service client */
+ private BlobServiceClient blobServiceClient;
+
+ /** Blob container client */
+ private BlobContainerClient blobContainerClient;
+
+ /** Init routine guard. */
+ private final AtomicBoolean initGuard = new AtomicBoolean();
+
+ /** Init routine latch. */
+ private final CountDownLatch initLatch = new CountDownLatch(1);
+
+ /**
+ * Default constructor
+ */
+ public TcpDiscoveryAzureBlobStoreIpFinder() {
+ setShared(true);
+ }
+
+ /** {@inheritDoc} */
+ @Override public Collection<InetSocketAddress> getRegisteredAddresses() throws IgniteSpiException {
+ init();
+
+ Collection<InetSocketAddress> addrs = new ArrayList<>();
+ Set<String> seenBlobNames = new HashSet<>();
+
+ Iterator<BlobItem> blobItemIterator = blobContainerClient.listBlobs().iterator();
+
+ while (blobItemIterator.hasNext()) {
+ BlobItem blobItem = blobItemIterator.next();
+
+ // https://github.com/Azure/azure-sdk-for-java/issues/20515
+ if (seenBlobNames.contains(blobItem.getName())) {
+ break;
+ }
+
+ try {
+ if (!blobItem.isDeleted()) {
+ addrs.add(addrFromString(blobItem.getName()));
+ seenBlobNames.add(blobItem.getName());
+ }
+ }
+ catch (Exception e) {
+ throw new IgniteSpiException("Failed to get content from the container: " + containerName, e);
+ }
+ }
+
+ return addrs;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void registerAddresses(Collection<InetSocketAddress> addrs) throws IgniteSpiException {
+ assert !F.isEmpty(addrs);
+
+ init();
+
+ for (InetSocketAddress addr : addrs) {
+ String key = keyFromAddr(addr);
+ try {
+ key = URLEncoder.encode(key, CHARSET);
+ } catch (UnsupportedEncodingException e) {
+ throw new IgniteSpiException("Unable to encode URL due to error "
+ + e.getMessage());
+ }
+
+ BlockBlobClient blobClient = blobContainerClient.getBlobClient(key).getBlockBlobClient();
+ InputStream dataStream = new ByteArrayInputStream(OBJECT_CONTENT);
+
+ try {
+ blobClient.upload(dataStream, OBJECT_CONTENT.length);
+ }
+ catch (BlobStorageException e) {
+ // If the blob already exists, ignore
+ if (!(e.getStatusCode() == 409)) {
+ throw new IgniteSpiException("Failed to upload blob with exception " +
+ e.getMessage());
+ }
+ }
+
+ try {
+ dataStream.close();
+ }
+ catch (IOException e) {
+ throw new IgniteSpiException(e.getMessage());
+ }
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void unregisterAddresses(Collection<InetSocketAddress> addrs) throws IgniteSpiException {
+ assert !F.isEmpty(addrs);
+
+ init();
+
+ for (InetSocketAddress addr : addrs) {
+ String key = keyFromAddr(addr);
+
+ try {
+ blobContainerClient.getBlobClient(key).delete();
+ } catch (Exception e) {
+ // https://github.com/Azure/azure-sdk-for-java/issues/20551
+ if ((!(e.getMessage().contains("InterruptedException"))) || (e instanceof BlobStorageException
+ && (!((BlobStorageException) e).getErrorCode().equals(404)))) {
+ throw new IgniteSpiException("Failed to delete entry [containerName=" + containerName +
+ ", entry=" + key + ']', e);
+ }
+ }
+ }
+ }
+
+ /**
+ * Sets Azure Blob Storage Account Name.
+ * <p>
+ * For details refer to Azure Blob Storage API reference.
+ *
+ * @param accountName Account Name
+ * @return {@code this} for chaining.
+ */
+ @IgniteSpiConfiguration(optional = false)
+ public TcpDiscoveryAzureBlobStoreIpFinder setAccountName(String accountName) {
+ this.accountName = accountName;
+
+ return this;
+ }
+
+ /**
+ * Sets Azure Blob Storage Account Key
+ * <p>
+ * For details refer to Azure Blob Storage API reference.
+ *
+ * @param accountKey Account Key
+ * @return {@code this} for chaining.
+ */
+ @IgniteSpiConfiguration(optional = false)
+ public TcpDiscoveryAzureBlobStoreIpFinder setAccountKey(String accountKey) {
+ this.accountKey = accountKey;
+
+ return this;
+ }
+
+ /**
+ * Sets Azure Blob Storage endpoint
+ * <p>
+ * For details refer to Azure Blob Storage API reference.
+ *
+ * @param endPoint Endpoint for storage
+ * @return {@code this} for chaining.
+ */
+ @IgniteSpiConfiguration(optional = false)
+ public TcpDiscoveryAzureBlobStoreIpFinder setAccountEndpoint(String endPoint) {
+ this.endPoint = endPoint;
+
+ return this;
+ }
+
+ /**
+ * Sets container name for using in the context
+ * If the container name doesn't exist Ignite will automatically create itß.
+ *
+ * @param containerName Container Name.
+ * @return {@code this} for chaining.
+ */
+ @IgniteSpiConfiguration(optional = false)
+ public TcpDiscoveryAzureBlobStoreIpFinder setContainerName(String containerName) {
+ this.containerName = containerName;
+
+ return this;
+ }
+
+ /**
+ * Initialize the IP finder
+ * @throws IgniteSpiException
+ */
+ private void init() throws IgniteSpiException {
+ if (initGuard.compareAndSet(false, true)) {
+ if (accountKey == null || accountName == null || containerName == null || endPoint == null) {
+ throw new IgniteSpiException(
+ "One or more of the required parameters is not set [accountName=" +
+ accountName + ", accountKey=" + accountKey + ", containerName=" +
+ containerName + ", endPoint=" + endPoint + "]");
+ }
+
+ try {
+ credential = new StorageSharedKeyCredential(accountName, accountKey);
+ blobServiceClient = new BlobServiceClientBuilder().endpoint(endPoint).credential(credential).buildClient();
+ blobContainerClient = blobServiceClient.getBlobContainerClient(containerName);
+
+ if (!blobContainerClient.exists()) {
+ U.warn(log, "Container doesn't exist, will create it [containerName=" + containerName + "]");
+
+ blobContainerClient.create();
+ }
+ }
+ finally {
+ initLatch.countDown();
+ }
+ }
+ else {
+ try {
+ U.await(initLatch);
+ }
+ catch (IgniteInterruptedCheckedException e) {
+ throw new IgniteSpiException("Thread has been interrupted.", e);
+ }
+
+ try {
+ if (!blobContainerClient.exists())
+ throw new IgniteSpiException("IpFinder has not been initialized properly");
+ } catch (Exception e) {
+ // Check if this is a nested exception wrapping an InterruptedException
+ // https://github.com/Azure/azure-sdk-for-java/issues/20551
+ if (!(e.getCause() instanceof InterruptedException)) {
+ throw e;
+ }
+ }
+ }
+ }
+
+ /**
+ * Constructs a node address from bucket's key.
+ *
+ * @param key Bucket key.
+ * @return Node address.
+ * @throws IgniteSpiException In case of error.
+ */
+ private InetSocketAddress addrFromString(String key) throws IgniteSpiException {
+ //TODO: This needs to move out to a generic helper class
+ String[] res = key.split("#");
+
+ if (res.length != 2)
+ throw new IgniteSpiException("Invalid address string: " + key);
+
+ int port;
+
+ try {
+ port = Integer.parseInt(res[1]);
+ }
+ catch (NumberFormatException ignored) {
+ throw new IgniteSpiException("Invalid port number: " + res[1]);
+ }
+
+ return new InetSocketAddress(res[0], port);
+ }
+
+ /**
+ * Constructs bucket's key from an address.
+ *
+ * @param addr Node address.
+ * @return Bucket key.
+ */
+ private String keyFromAddr(InetSocketAddress addr) {
+ // TODO: This needs to move out to a generic helper class
+ return addr.getAddress().getHostAddress() + "#" + addr.getPort();
+ }
+
+ /**
+ * Used by TEST SUITES only. Called through reflection.
+ *
+ * @param containerName Container to delete
+ */
+ private void removeContainer(String containerName) {
+ init();
+
+ try {
+ blobServiceClient.getBlobContainerClient(containerName).delete();
+ }
+ catch (Exception e) {
+ throw new IgniteSpiException("Failed to remove the container: " + containerName, e);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public TcpDiscoveryAzureBlobStoreIpFinder setShared(boolean shared) {
+ super.setShared(shared);
+
+ return this;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(TcpDiscoveryAzureBlobStoreIpFinder.class, this);
+ }
+}
diff --git a/modules/azure/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/package-info.java b/modules/azure/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/package-info.java
new file mode 100644
index 0000000..b8410b3
--- /dev/null
+++ b/modules/azure/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <!-- Package description. -->
+ * Contains Azure Blob Storage IP finder.
+ */
+
+package org.apache.ignite.spi.discovery.tcp.ipfinder;
diff --git a/modules/azure/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/azure/TcpDiscoveryAzureBlobStoreIpFinderSelfTest.java b/modules/azure/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/azure/TcpDiscoveryAzureBlobStoreIpFinderSelfTest.java
new file mode 100644
index 0000000..3c172c2
--- /dev/null
+++ b/modules/azure/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/azure/TcpDiscoveryAzureBlobStoreIpFinderSelfTest.java
@@ -0,0 +1,91 @@
+package org.apache.ignite.spi.discovery.tcp.ipfinder.azure;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.lang.reflect.Method;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.util.Collection;
+
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinderAbstractSelfTest;
+import org.apache.ignite.testsuites.IgniteAzureTestSuite;
+
+public class TcpDiscoveryAzureBlobStoreIpFinderSelfTest
+ extends TcpDiscoveryIpFinderAbstractSelfTest<TcpDiscoveryAzureBlobStoreIpFinder> {
+ private static String containerName;
+
+ /**
+ * Constructor.
+ *
+ * @throws Exception If any error occurs.
+ */
+ public TcpDiscoveryAzureBlobStoreIpFinderSelfTest() throws Exception {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ containerName = "ip-finder-test-container-" + InetAddress.getLocalHost().getAddress()[3];
+
+ super.beforeTestsStarted();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTestsStopped() {
+ try {
+ Method method = TcpDiscoveryAzureBlobStoreIpFinder.class.getDeclaredMethod("removeContainer", String.class);
+
+ method.setAccessible(true);
+
+ method.invoke(finder, containerName);
+ }
+ catch (Exception e) {
+ log.warning("Failed to remove bucket on Azure [containerName=" + containerName + ", mes=" + e.getMessage() + ']');
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override protected TcpDiscoveryAzureBlobStoreIpFinder ipFinder() throws Exception {
+ TcpDiscoveryAzureBlobStoreIpFinder finder = new TcpDiscoveryAzureBlobStoreIpFinder();
+
+ injectLogger(finder);
+
+ assert finder.isShared() : "Ip finder must be shared by default.";
+
+ finder.setAccountName(IgniteAzureTestSuite.getAccountName());
+ finder.setAccountKey(IgniteAzureTestSuite.getAccountKey());
+ finder.setAccountEndpoint(IgniteAzureTestSuite.getEndpoint());
+
+ finder.setContainerName(containerName);
+
+ for (int i = 0; i < 5; i++) {
+ Collection<InetSocketAddress> addrs = finder.getRegisteredAddresses();
+
+ if (!addrs.isEmpty())
+ finder.unregisterAddresses(addrs);
+ else
+ return finder;
+
+ U.sleep(1000);
+ }
+
+ if (!finder.getRegisteredAddresses().isEmpty())
+ throw new Exception("Failed to initialize IP finder.");
+
+ return finder;
+ }
+}
diff --git a/modules/azure/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/azure/package-info.java b/modules/azure/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/azure/package-info.java
new file mode 100644
index 0000000..c3f6711
--- /dev/null
+++ b/modules/azure/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/azure/package-info.java
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * <!-- Package description. -->
+ * Contains Azure Blob Storage IP finder internal tests.
+ */
+package org.apache.ignite.spi.discovery.tcp.ipfinder.azure;
diff --git a/modules/azure/src/test/java/org/apache/ignite/testsuites/IgniteAzureTestSuite.java b/modules/azure/src/test/java/org/apache/ignite/testsuites/IgniteAzureTestSuite.java
new file mode 100644
index 0000000..0549568
--- /dev/null
+++ b/modules/azure/src/test/java/org/apache/ignite/testsuites/IgniteAzureTestSuite.java
@@ -0,0 +1,61 @@
+package org.apache.ignite.testsuites;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.ignite.spi.discovery.tcp.ipfinder.azure.TcpDiscoveryAzureBlobStoreIpFinderSelfTest;
+import org.junit.runner.RunWith;
+import org.junit.runners.Suite;
+
+/**
+ * Azure integration tests
+ */
+@RunWith(Suite.class)
+@Suite.SuiteClasses({TcpDiscoveryAzureBlobStoreIpFinderSelfTest.class})
+public class IgniteAzureTestSuite {
+ /**
+ * @return Account Name
+ */
+ public static String getAccountName() {
+ String id = System.getenv("test.azure.account.name");
+
+ assert id != null : "Environment variable 'test.azure.account.name' is not set";
+
+ return id;
+ }
+
+ /**
+ * @return Account Key
+ */
+ public static String getAccountKey() {
+ String path = System.getenv("test.azure.account.key");
+
+ assert path != null : "Environment variable 'test.azure.account.key' is not set";
+
+ return path;
+ }
+
+ /**
+ * @return Endpoint
+ */
+ public static String getEndpoint() {
+ String name = System.getenv("test.azure.endpoint");
+
+ assert name != null : "Environment variable 'test.azure.endpoint' is not set";
+
+ return name;
+ }
+}
diff --git a/parent/pom.xml b/parent/pom.xml
index b227c30..05203a1 100644
--- a/parent/pom.xml
+++ b/parent/pom.xml
@@ -54,6 +54,8 @@
<aspectj.version>1.8.13</aspectj.version>
<aws.sdk.bundle.version>1.10.12_1</aws.sdk.bundle.version>
<aws.sdk.version>1.11.75</aws.sdk.version>
+ <azure.sdk.version>12.0.0</azure.sdk.version>
+ <azure.netty.version>4.1.54.Final</azure.netty.version>
<camel.version>2.22.0</camel.version>
<aws.encryption.sdk.version>1.3.2</aws.encryption.sdk.version>
<bouncycastle.version>1.60</bouncycastle.version>
diff --git a/pom.xml b/pom.xml
index a054b86..ed2c607 100644
--- a/pom.xml
+++ b/pom.xml
@@ -67,6 +67,7 @@
<module>modules/jcl</module>
<module>modules/codegen</module>
<module>modules/gce</module>
+ <module>modules/azure</module>
<module>modules/cloud</module>
<module>modules/mesos</module>
<module>modules/yarn</module>