You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by jf...@apache.org on 2020/12/17 00:26:06 UTC
[nifi] branch main updated: NIFI-7386 Added Azure Azurite emulator
credentials service
This is an automated email from the ASF dual-hosted git repository.
jfrazee pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new 9420305 NIFI-7386 Added Azure Azurite emulator credentials service
9420305 is described below
commit 942030579a055b568eef9de2849e804ad034f174
Author: sjyang18 <il...@hotmail.com>
AuthorDate: Wed May 20 02:47:14 2020 +0000
NIFI-7386 Added Azure Azurite emulator credentials service
This closes #4286
Signed-off-by: Joey Frazee <jf...@apache.org>
---
.../storage/queue/AbstractAzureQueueStorage.java | 9 +--
.../azure/storage/utils/AzureStorageUtils.java | 29 ++++++--
...torageEmulatorCredentialsControllerService.java | 79 ++++++++++++++++++++++
.../AzureStorageEmulatorCredentialsDetails.java | 28 ++++++++
.../org.apache.nifi.controller.ControllerService | 1 +
...torageEmulatorCredentialsControllerService.java | 55 +++++++++++++++
.../storage/AzureStorageCredentialsDetails.java | 4 ++
7 files changed, 191 insertions(+), 14 deletions(-)
diff --git a/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/storage/queue/AbstractAzureQueueStorage.java b/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/storage/queue/AbstractAzureQueueStorage.java
index c077df6..968cdc2 100644
--- a/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/storage/queue/AbstractAzureQueueStorage.java
+++ b/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/storage/queue/AbstractAzureQueueStorage.java
@@ -68,15 +68,8 @@ public abstract class AbstractAzureQueueStorage extends AbstractProcessor {
protected final CloudQueueClient createCloudQueueClient(final ProcessContext context, final FlowFile flowFile) throws URISyntaxException {
final AzureStorageCredentialsDetails storageCredentialsDetails = AzureStorageUtils.getStorageCredentialsDetails(context, flowFile);
- final CloudStorageAccount cloudStorageAccount =
- new CloudStorageAccount(
- storageCredentialsDetails.getStorageCredentials(),
- true,
- storageCredentialsDetails.getStorageSuffix(),
- storageCredentialsDetails.getStorageAccountName()
- );
+ final CloudStorageAccount cloudStorageAccount = AzureStorageUtils.getCloudStorageAccount(storageCredentialsDetails);
final CloudQueueClient cloudQueueClient = cloudStorageAccount.createCloudQueueClient();
-
return cloudQueueClient;
}
diff --git a/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/storage/utils/AzureStorageUtils.java b/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/storage/utils/AzureStorageUtils.java
index 8eebb2c..cf3c8a0 100644
--- a/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/storage/utils/AzureStorageUtils.java
+++ b/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/storage/utils/AzureStorageUtils.java
@@ -16,6 +16,7 @@
*/
package org.apache.nifi.processors.azure.storage.utils;
+import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Collection;
@@ -44,6 +45,7 @@ import org.apache.nifi.proxy.ProxyConfiguration;
import org.apache.nifi.proxy.ProxySpec;
import org.apache.nifi.services.azure.storage.AzureStorageCredentialsDetails;
import org.apache.nifi.services.azure.storage.AzureStorageCredentialsService;
+import org.apache.nifi.services.azure.storage.AzureStorageEmulatorCredentialsDetails;
public final class AzureStorageUtils {
public static final String BLOCK = "Block";
@@ -152,16 +154,31 @@ public final class AzureStorageUtils {
*/
public static CloudBlobClient createCloudBlobClient(ProcessContext context, ComponentLog logger, FlowFile flowFile) throws URISyntaxException {
final AzureStorageCredentialsDetails storageCredentialsDetails = getStorageCredentialsDetails(context, flowFile);
- final CloudStorageAccount cloudStorageAccount = new CloudStorageAccount(
- storageCredentialsDetails.getStorageCredentials(),
- true,
- storageCredentialsDetails.getStorageSuffix(),
- storageCredentialsDetails.getStorageAccountName());
+ final CloudStorageAccount cloudStorageAccount = getCloudStorageAccount(storageCredentialsDetails);
final CloudBlobClient cloudBlobClient = cloudStorageAccount.createCloudBlobClient();
-
return cloudBlobClient;
}
+ public static CloudStorageAccount getCloudStorageAccount(final AzureStorageCredentialsDetails storageCredentialsDetails) throws URISyntaxException {
+ final CloudStorageAccount cloudStorageAccount;
+ if (storageCredentialsDetails instanceof AzureStorageEmulatorCredentialsDetails) {
+ AzureStorageEmulatorCredentialsDetails emulatorCredentials = (AzureStorageEmulatorCredentialsDetails) storageCredentialsDetails;
+ final String proxyUri = emulatorCredentials.getDevelopmentStorageProxyUri();
+ if (proxyUri != null) {
+ cloudStorageAccount = CloudStorageAccount.getDevelopmentStorageAccount(new URI(proxyUri));
+ } else {
+ cloudStorageAccount = CloudStorageAccount.getDevelopmentStorageAccount();
+ }
+ } else {
+ cloudStorageAccount = new CloudStorageAccount(
+ storageCredentialsDetails.getStorageCredentials(),
+ true,
+ storageCredentialsDetails.getStorageSuffix(),
+ storageCredentialsDetails.getStorageAccountName());
+ }
+ return cloudStorageAccount;
+ }
+
public static AzureStorageCredentialsDetails getStorageCredentialsDetails(PropertyContext context, FlowFile flowFile) {
final Map<String, String> attributes = flowFile != null ? flowFile.getAttributes() : Collections.emptyMap();
diff --git a/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/services/azure/storage/AzureStorageEmulatorCredentialsControllerService.java b/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/services/azure/storage/AzureStorageEmulatorCredentialsControllerService.java
new file mode 100644
index 0000000..42aac0e
--- /dev/null
+++ b/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/services/azure/storage/AzureStorageEmulatorCredentialsControllerService.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.services.azure.storage;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnEnabled;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.controller.AbstractControllerService;
+import org.apache.nifi.controller.ConfigurationContext;
+import org.apache.nifi.processor.util.StandardValidators;
+
+@Tags({ "azure", "microsoft", "emulator", "storage", "blob", "queue", "credentials" })
+@CapabilityDescription("Defines credentials for Azure Storage processors that connects to Azurite emulator.")
+public class AzureStorageEmulatorCredentialsControllerService extends AbstractControllerService implements AzureStorageCredentialsService {
+
+ public static final PropertyDescriptor DEVELOPMENT_STORAGE_PROXY_URI = new PropertyDescriptor.Builder()
+ .name("azurite-uri")
+ .displayName("Storage Emulator URI")
+ .description("URI to connect to Azure Storage Emulator (Azurite)")
+ .required(false)
+ .sensitive(false)
+ .addValidator(StandardValidators.URI_VALIDATOR)
+ .build();
+
+ private static final List<PropertyDescriptor> PROPERTIES =
+ Collections.unmodifiableList(Arrays.asList(DEVELOPMENT_STORAGE_PROXY_URI));
+
+ private String azuriteProxyUri;
+
+ @Override
+ protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
+ return PROPERTIES;
+ }
+
+ @Override
+ protected Collection<ValidationResult> customValidate(final ValidationContext validationContext) {
+ final List<ValidationResult> results = new ArrayList<>();
+ return results;
+ }
+
+ @OnEnabled
+ public void onEnabled(final ConfigurationContext context) {
+ this.azuriteProxyUri = context.getProperty(DEVELOPMENT_STORAGE_PROXY_URI).getValue();
+ }
+
+ public String getProxyUri() {
+ return azuriteProxyUri;
+ }
+
+ @Override
+ public AzureStorageCredentialsDetails getStorageCredentialsDetails(final Map<String, String> attributes) {
+ return new AzureStorageEmulatorCredentialsDetails(azuriteProxyUri);
+
+ }
+}
diff --git a/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/services/azure/storage/AzureStorageEmulatorCredentialsDetails.java b/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/services/azure/storage/AzureStorageEmulatorCredentialsDetails.java
new file mode 100644
index 0000000..1f0f798
--- /dev/null
+++ b/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/services/azure/storage/AzureStorageEmulatorCredentialsDetails.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.services.azure.storage;
+
+public class AzureStorageEmulatorCredentialsDetails extends AzureStorageCredentialsDetails {
+ private String developmentStorageProxyUri;
+
+ public AzureStorageEmulatorCredentialsDetails(String developmentStorageProxyUri) {
+ this.developmentStorageProxyUri = developmentStorageProxyUri;
+ }
+ public String getDevelopmentStorageProxyUri() {
+ return developmentStorageProxyUri;
+ }
+}
diff --git a/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/resources/META-INF/services/org.apache.nifi.controller.ControllerService b/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/resources/META-INF/services/org.apache.nifi.controller.ControllerService
index 5e63d4b..3d5ffff 100644
--- a/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/resources/META-INF/services/org.apache.nifi.controller.ControllerService
+++ b/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/resources/META-INF/services/org.apache.nifi.controller.ControllerService
@@ -16,3 +16,4 @@ org.apache.nifi.services.azure.storage.AzureStorageCredentialsControllerService
org.apache.nifi.services.azure.storage.AzureStorageCredentialsControllerServiceLookup
org.apache.nifi.services.azure.storage.ADLSCredentialsControllerService
org.apache.nifi.services.azure.cosmos.document.AzureCosmosDBClientService
+org.apache.nifi.services.azure.storage.AzureStorageEmulatorCredentialsControllerService
diff --git a/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/test/java/org/apache/nifi/services/azure/storage/TestAzureStorageEmulatorCredentialsControllerService.java b/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/test/java/org/apache/nifi/services/azure/storage/TestAzureStorageEmulatorCredentialsControllerService.java
new file mode 100644
index 0000000..40f5545
--- /dev/null
+++ b/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/test/java/org/apache/nifi/services/azure/storage/TestAzureStorageEmulatorCredentialsControllerService.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.services.azure.storage;
+
+import org.apache.nifi.reporting.InitializationException;
+import org.apache.nifi.util.NoOpProcessor;
+import org.apache.nifi.util.TestRunner;
+import org.apache.nifi.util.TestRunners;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestAzureStorageEmulatorCredentialsControllerService {
+
+ private static final String TEST_ENVIRONMENT_URI = "http://127.0.0.1";
+
+ private TestRunner runner;
+ private AzureStorageCredentialsService credentialsService;
+
+ @Before
+ public void setUp() throws InitializationException {
+ runner = TestRunners.newTestRunner(NoOpProcessor.class);
+ credentialsService = new AzureStorageEmulatorCredentialsControllerService();
+ runner.addControllerService("credentials-service", credentialsService);
+ }
+
+ @Test
+ public void testValidWithProxyURI() {
+ configureProxyURI();
+
+ runner.assertValid(credentialsService);
+ }
+
+ @Test
+ public void testValidWithoutProxyURI() {
+ runner.assertValid(credentialsService);
+ }
+
+ private void configureProxyURI() {
+ runner.setProperty(credentialsService, AzureStorageEmulatorCredentialsControllerService.DEVELOPMENT_STORAGE_PROXY_URI, TEST_ENVIRONMENT_URI);
+ }
+}
diff --git a/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-services-api/src/main/java/org/apache/nifi/services/azure/storage/AzureStorageCredentialsDetails.java b/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-services-api/src/main/java/org/apache/nifi/services/azure/storage/AzureStorageCredentialsDetails.java
index fc1ccaf..f009ab3 100644
--- a/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-services-api/src/main/java/org/apache/nifi/services/azure/storage/AzureStorageCredentialsDetails.java
+++ b/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-services-api/src/main/java/org/apache/nifi/services/azure/storage/AzureStorageCredentialsDetails.java
@@ -26,6 +26,10 @@ public class AzureStorageCredentialsDetails {
private final StorageCredentials storageCredentials;
+ public AzureStorageCredentialsDetails() {
+ this(null, null, null);
+ }
+
@Deprecated
public AzureStorageCredentialsDetails(String storageAccountName, StorageCredentials storageCredentials) {
this(storageAccountName, null, storageCredentials);