You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by mi...@apache.org on 2024/03/04 10:18:06 UTC
(jackrabbit-oak) branch trunk updated: OAK-10670: add support for service principal in oak-upgrade (#1329)
This is an automated email from the ASF dual-hosted git repository.
miroslav pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
The following commit(s) were added to refs/heads/trunk by this push:
new 8f97e2a0e9 OAK-10670: add support for service principal in oak-upgrade (#1329)
8f97e2a0e9 is described below
commit 8f97e2a0e9284553301a8885bdd749e0e44c12ec
Author: t-rana <14...@users.noreply.github.com>
AuthorDate: Mon Mar 4 15:48:00 2024 +0530
OAK-10670: add support for service principal in oak-upgrade (#1329)
* OAK-10670: add service principal support in oak upgrade
* OAK-10670: add service principal support in oak upgrade
* OAK-10670: add service principal support in oak upgrade
* OAK-10670: add service principal support in oak upgrade
* OAK-10670: minor change
* OAK-10670: delete tempDir after closing filestore
* OAK-10670: add license to new files
---
.../oak/segment/azure/AzureUtilities.java | 46 +++--
.../jackrabbit/oak/segment/azure/package-info.java | 2 +-
.../oak/segment/azure/tool/ToolUtils.java | 9 +-
.../oak/upgrade/cli/node/SegmentAzureFactory.java | 52 +++---
.../oak/upgrade/cli/parser/StoreType.java | 2 +
...gmentTarToSegmentAzureServicePrincipalTest.java | 69 ++++++++
...entAzureServicePrincipalNodeStoreContainer.java | 116 +++++++++++++
.../upgrade/cli/node/SegmentAzureFactoryTest.java | 190 +++++++++++++++++++++
8 files changed, 449 insertions(+), 37 deletions(-)
diff --git a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureUtilities.java b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureUtilities.java
index 46e410e398..5817d78a44 100644
--- a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureUtilities.java
+++ b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureUtilities.java
@@ -16,17 +16,6 @@
*/
package org.apache.jackrabbit.oak.segment.azure;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.nio.file.Paths;
-import java.security.InvalidKeyException;
-import java.util.ArrayList;
-import java.util.EnumSet;
-import java.util.List;
-
import com.azure.core.credential.TokenRequestContext;
import com.azure.identity.ClientSecretCredential;
import com.azure.identity.ClientSecretCredentialBuilder;
@@ -41,6 +30,7 @@ import com.microsoft.azure.storage.blob.BlobListingDetails;
import com.microsoft.azure.storage.blob.CloudBlob;
import com.microsoft.azure.storage.blob.CloudBlobContainer;
import com.microsoft.azure.storage.blob.CloudBlobDirectory;
+import com.microsoft.azure.storage.blob.LeaseStatus;
import com.microsoft.azure.storage.blob.ListBlobItem;
import org.apache.jackrabbit.oak.commons.Buffer;
import org.apache.jackrabbit.oak.segment.spi.RepositoryNotReachableException;
@@ -48,6 +38,17 @@ import org.jetbrains.annotations.NotNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.nio.file.Paths;
+import java.security.InvalidKeyException;
+import java.util.ArrayList;
+import java.util.EnumSet;
+import java.util.List;
+
public final class AzureUtilities {
public static final String AZURE_ACCOUNT_NAME = "AZURE_ACCOUNT_NAME";
public static final String AZURE_SECRET_KEY = "AZURE_SECRET_KEY";
@@ -110,7 +111,7 @@ public final class AzureUtilities {
}
public static CloudBlobDirectory cloudBlobDirectoryFrom(StorageCredentials credentials,
- String uri, String dir) throws URISyntaxException, StorageException {
+ String uri, String dir) throws URISyntaxException, StorageException {
StorageUri storageUri = new StorageUri(new URI(uri));
CloudBlobContainer container = new CloudBlobContainer(storageUri, credentials);
@@ -120,7 +121,7 @@ public final class AzureUtilities {
}
public static CloudBlobDirectory cloudBlobDirectoryFrom(String connection, String containerName,
- String dir) throws InvalidKeyException, URISyntaxException, StorageException {
+ String dir) throws InvalidKeyException, URISyntaxException, StorageException {
CloudStorageAccount cloud = CloudStorageAccount.parse(connection);
CloudBlobContainer container = cloud.createCloudBlobClient().getContainerReference(containerName);
container.createIfNotExists();
@@ -140,7 +141,7 @@ public final class AzureUtilities {
}
private static ResultSegment<ListBlobItem> listBlobsInSegments(CloudBlobDirectory directory,
- ResultContinuation token) throws IOException {
+ ResultContinuation token) throws IOException {
ResultSegment<ListBlobItem> result = null;
IOException lastException = null;
for (int sleep = 10; sleep <= 10000; sleep *= 10) { //increment the sleep time in steps.
@@ -171,6 +172,21 @@ public final class AzureUtilities {
}
}
+ public static void deleteAllBlobs(@NotNull CloudBlobDirectory directory) throws URISyntaxException, StorageException, InterruptedException {
+ for (ListBlobItem blobItem : directory.listBlobs()) {
+ if (blobItem instanceof CloudBlob) {
+ CloudBlob cloudBlob = (CloudBlob) blobItem;
+ if (cloudBlob.getProperties().getLeaseStatus() == LeaseStatus.LOCKED) {
+ cloudBlob.breakLease(0);
+ }
+ cloudBlob.deleteIfExists();
+ } else if (blobItem instanceof CloudBlobDirectory) {
+ CloudBlobDirectory cloudBlobDirectory = (CloudBlobDirectory) blobItem;
+ deleteAllBlobs(cloudBlobDirectory);
+ }
+ }
+ }
+
private static class ByteBufferOutputStream extends OutputStream {
@NotNull
@@ -182,7 +198,7 @@ public final class AzureUtilities {
@Override
public void write(int b) {
- buffer.put((byte)b);
+ buffer.put((byte) b);
}
@Override
diff --git a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/package-info.java b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/package-info.java
index 46bf646176..e054751151 100644
--- a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/package-info.java
+++ b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/package-info.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
@Internal(since = "1.0.0")
-@Version("2.2.0")
+@Version("2.3.0")
package org.apache.jackrabbit.oak.segment.azure;
import org.apache.jackrabbit.oak.commons.annotations.Internal;
diff --git a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtils.java b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtils.java
index 51984f440c..f578734bcd 100644
--- a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtils.java
+++ b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtils.java
@@ -172,12 +172,12 @@ public class ToolUtils {
public static CloudBlobDirectory createCloudBlobDirectory(String path) {
return createCloudBlobDirectory(path, ENVIRONMENT);
}
-
+
public static CloudBlobDirectory createCloudBlobDirectory(String path, Environment environment) {
Map<String, String> config = parseAzureConfigurationFromUri(path);
String accountName = config.get(KEY_ACCOUNT_NAME);
-
+
StorageCredentials credentials;
if (config.containsKey(KEY_SHARED_ACCESS_SIGNATURE)) {
credentials = new StorageCredentialsSharedAccessSignature(config.get(KEY_SHARED_ACCESS_SIGNATURE));
@@ -196,6 +196,11 @@ public class ToolUtils {
}
}
+ @NotNull
+ public static StorageCredentials getStorageCredentialsFromAccountAndEnv(@NotNull String accountName) {
+ return getStorageCredentialsFromAccountAndEnv(accountName, ENVIRONMENT);
+ }
+
@NotNull
private static StorageCredentials getStorageCredentialsFromAccountAndEnv(String accountName, Environment environment) {
String clientId = environment.getVariable(AZURE_CLIENT_ID);
diff --git a/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentAzureFactory.java b/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentAzureFactory.java
index 7ee319f93b..a48ecd6de6 100644
--- a/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentAzureFactory.java
+++ b/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentAzureFactory.java
@@ -16,17 +16,17 @@
*/
package org.apache.jackrabbit.oak.upgrade.cli.node;
-import static org.apache.jackrabbit.oak.segment.SegmentCache.DEFAULT_SEGMENT_CACHE_MB;
-import static org.apache.jackrabbit.oak.upgrade.cli.node.FileStoreUtils.asCloseable;
-
-import java.io.File;
-import java.io.IOException;
-import java.net.URISyntaxException;
-import java.security.InvalidKeyException;
-
+import com.microsoft.azure.storage.StorageCredentials;
+import com.microsoft.azure.storage.StorageCredentialsSharedAccessSignature;
+import com.microsoft.azure.storage.StorageException;
+import com.microsoft.azure.storage.blob.CloudBlobDirectory;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.jackrabbit.guava.common.io.Closer;
+import org.apache.jackrabbit.guava.common.io.Files;
import org.apache.jackrabbit.oak.segment.SegmentNodeStoreBuilders;
import org.apache.jackrabbit.oak.segment.azure.AzurePersistence;
import org.apache.jackrabbit.oak.segment.azure.AzureUtilities;
+import org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils;
import org.apache.jackrabbit.oak.segment.file.FileStore;
import org.apache.jackrabbit.oak.segment.file.FileStoreBuilder;
import org.apache.jackrabbit.oak.segment.file.InvalidFileStoreVersionException;
@@ -35,15 +35,17 @@ import org.apache.jackrabbit.oak.spi.blob.BlobStore;
import org.apache.jackrabbit.oak.spi.state.NodeStore;
import org.apache.jackrabbit.oak.upgrade.cli.node.FileStoreUtils.NodeStoreWithFileStore;
-import org.apache.jackrabbit.guava.common.io.Closer;
-import org.apache.jackrabbit.guava.common.io.Files;
-import com.microsoft.azure.storage.StorageCredentials;
-import com.microsoft.azure.storage.StorageCredentialsAccountAndKey;
-import com.microsoft.azure.storage.StorageException;
-import com.microsoft.azure.storage.blob.CloudBlobDirectory;
+import java.io.File;
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.security.InvalidKeyException;
+
+import static org.apache.jackrabbit.oak.segment.SegmentCache.DEFAULT_SEGMENT_CACHE_MB;
+import static org.apache.jackrabbit.oak.upgrade.cli.node.FileStoreUtils.asCloseable;
public class SegmentAzureFactory implements NodeStoreFactory {
private final String accountName;
+ private final String sasToken;
private final String uri;
private final String connectionString;
private final String containerName;
@@ -58,6 +60,7 @@ public class SegmentAzureFactory implements NodeStoreFactory {
private final boolean readOnly;
private String accountName;
+ private String sasToken;
private String uri;
private String connectionString;
private String containerName;
@@ -73,6 +76,11 @@ public class SegmentAzureFactory implements NodeStoreFactory {
return this;
}
+ public Builder sasToken(String sasToken) {
+ this.sasToken = sasToken;
+ return this;
+ }
+
public Builder uri(String uri) {
this.uri = uri;
return this;
@@ -95,6 +103,7 @@ public class SegmentAzureFactory implements NodeStoreFactory {
public SegmentAzureFactory(Builder builder) {
this.accountName = builder.accountName;
+ this.sasToken = builder.sasToken;
this.uri = builder.uri;
this.connectionString = builder.connectionString;
this.containerName = builder.containerName;
@@ -142,12 +151,17 @@ public class SegmentAzureFactory implements NodeStoreFactory {
private AzurePersistence createAzurePersistence() throws StorageException, URISyntaxException, InvalidKeyException {
CloudBlobDirectory cloudBlobDirectory = null;
- if (accountName != null && uri != null) {
- String key = System.getenv("AZURE_SECRET_KEY");
- StorageCredentials credentials = new StorageCredentialsAccountAndKey(accountName, key);
- cloudBlobDirectory = AzureUtilities.cloudBlobDirectoryFrom(credentials, uri, dir);
- } else if (connectionString != null && containerName != null) {
+ // connection string will take precedence over accountkey / sas / service principal
+ if (StringUtils.isNoneBlank(connectionString, containerName)) {
cloudBlobDirectory = AzureUtilities.cloudBlobDirectoryFrom(connectionString, containerName, dir);
+ } else if (StringUtils.isNoneBlank(accountName, uri)) {
+ StorageCredentials credentials = null;
+ if (StringUtils.isNotBlank(sasToken)) {
+ credentials = new StorageCredentialsSharedAccessSignature(sasToken);
+ } else {
+ credentials = ToolUtils.getStorageCredentialsFromAccountAndEnv(accountName);
+ }
+ cloudBlobDirectory = AzureUtilities.cloudBlobDirectoryFrom(credentials, uri, dir);
}
if (cloudBlobDirectory == null) {
diff --git a/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/StoreType.java b/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/StoreType.java
index 5d46078c86..b9b961b5eb 100644
--- a/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/StoreType.java
+++ b/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/StoreType.java
@@ -21,6 +21,7 @@ import static org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationPar
import static org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.KEY_CONNECTION_STRING;
import static org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.KEY_CONTAINER_NAME;
import static org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.KEY_DIR;
+import static org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.KEY_SHARED_ACCESS_SIGNATURE;
import static org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.KEY_STORAGE_URI;
import static org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.isCustomAzureConnectionString;
import static org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.parseAzureConfigurationFromCustomConnection;
@@ -175,6 +176,7 @@ public enum StoreType {
return new StoreFactory(new SegmentAzureFactory.Builder(config.get(KEY_DIR),
migrationOptions.getCacheSizeInMB(), direction == MigrationDirection.SRC)
.accountName(config.get(KEY_ACCOUNT_NAME))
+ .sasToken(config.get(KEY_SHARED_ACCESS_SIGNATURE))
.uri(config.get(KEY_STORAGE_URI))
.build()
);
diff --git a/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/SegmentTarToSegmentAzureServicePrincipalTest.java b/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/SegmentTarToSegmentAzureServicePrincipalTest.java
new file mode 100644
index 0000000000..1d42f0ee29
--- /dev/null
+++ b/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/SegmentTarToSegmentAzureServicePrincipalTest.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.upgrade.cli;
+
+import org.apache.jackrabbit.oak.segment.azure.AzureUtilities;
+import org.apache.jackrabbit.oak.segment.azure.util.Environment;
+import org.apache.jackrabbit.oak.upgrade.cli.container.NodeStoreContainer;
+import org.apache.jackrabbit.oak.upgrade.cli.container.SegmentAzureServicePrincipalNodeStoreContainer;
+import org.apache.jackrabbit.oak.upgrade.cli.container.SegmentTarNodeStoreContainer;
+
+import java.io.IOException;
+
+import static org.junit.Assume.assumeNotNull;
+
+public class SegmentTarToSegmentAzureServicePrincipalTest extends AbstractOak2OakTest {
+ private static boolean skipTest = true;
+ private static final Environment ENVIRONMENT = new Environment();
+ private final NodeStoreContainer source;
+ private final NodeStoreContainer destination;
+
+ @Override
+ public void prepare() throws Exception {
+ assumeNotNull(ENVIRONMENT.getVariable(AzureUtilities.AZURE_ACCOUNT_NAME), ENVIRONMENT.getVariable(AzureUtilities.AZURE_TENANT_ID),
+ ENVIRONMENT.getVariable(AzureUtilities.AZURE_CLIENT_ID), ENVIRONMENT.getVariable(AzureUtilities.AZURE_CLIENT_SECRET));
+ skipTest = false;
+ super.prepare();
+ }
+
+ @Override
+ public void clean() throws IOException {
+ if (!skipTest) {
+ super.clean();
+ }
+ }
+
+ public SegmentTarToSegmentAzureServicePrincipalTest() throws IOException {
+ source = new SegmentTarNodeStoreContainer();
+ destination = new SegmentAzureServicePrincipalNodeStoreContainer();
+ }
+
+ @Override
+ protected NodeStoreContainer getSourceContainer() {
+ return source;
+ }
+
+ @Override
+ protected NodeStoreContainer getDestinationContainer() {
+ return destination;
+ }
+
+ @Override
+ protected String[] getArgs() {
+ return new String[]{source.getDescription(), destination.getDescription()};
+ }
+}
diff --git a/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/container/SegmentAzureServicePrincipalNodeStoreContainer.java b/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/container/SegmentAzureServicePrincipalNodeStoreContainer.java
new file mode 100644
index 0000000000..2347d3e210
--- /dev/null
+++ b/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/container/SegmentAzureServicePrincipalNodeStoreContainer.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.upgrade.cli.container;
+
+import com.microsoft.azure.storage.blob.CloudBlobDirectory;
+import org.apache.jackrabbit.guava.common.io.Files;
+import org.apache.jackrabbit.oak.segment.SegmentNodeStoreBuilders;
+import org.apache.jackrabbit.oak.segment.azure.AzurePersistence;
+import org.apache.jackrabbit.oak.segment.azure.AzureUtilities;
+import org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils;
+import org.apache.jackrabbit.oak.segment.azure.util.Environment;
+import org.apache.jackrabbit.oak.segment.file.FileStore;
+import org.apache.jackrabbit.oak.segment.file.FileStoreBuilder;
+import org.apache.jackrabbit.oak.segment.file.InvalidFileStoreVersionException;
+import org.apache.jackrabbit.oak.spi.blob.BlobStore;
+import org.apache.jackrabbit.oak.spi.state.NodeStore;
+import org.apache.jackrabbit.oak.upgrade.cli.node.FileStoreUtils;
+
+import java.io.File;
+import java.io.IOException;
+
+public class SegmentAzureServicePrincipalNodeStoreContainer implements NodeStoreContainer {
+ private static final Environment ENVIRONMENT = new Environment();
+ private static final String CONTAINER_NAME = "oak-migration-test";
+ private static final String DIR = "repository";
+ private static final String AZURE_SEGMENT_STORE_PATH = "https://%s.blob.core.windows.net/%s/%s";
+
+ private final BlobStore blobStore;
+ private FileStore fs;
+ private File tmpDir;
+ private AzurePersistence azurePersistence;
+
+ public SegmentAzureServicePrincipalNodeStoreContainer() {
+ this(null);
+ }
+
+ public SegmentAzureServicePrincipalNodeStoreContainer(BlobStore blobStore) {
+ this.blobStore = blobStore;
+ }
+
+
+ @Override
+ public NodeStore open() throws IOException {
+ try {
+ azurePersistence = createAzurePersistence();
+ } catch (Exception e) {
+ throw new IllegalStateException(e);
+ }
+
+ tmpDir = Files.createTempDir();
+ FileStoreBuilder builder = FileStoreBuilder.fileStoreBuilder(tmpDir)
+ .withCustomPersistence(azurePersistence).withMemoryMapping(false);
+ if (blobStore != null) {
+ builder.withBlobStore(blobStore);
+ }
+
+ try {
+ fs = builder.build();
+ } catch (InvalidFileStoreVersionException e) {
+ throw new IllegalStateException(e);
+ }
+
+ return new FileStoreUtils.NodeStoreWithFileStore(SegmentNodeStoreBuilders.builder(fs).build(), fs);
+ }
+
+ private AzurePersistence createAzurePersistence() {
+ if (azurePersistence != null) {
+ return azurePersistence;
+ }
+ String path = String.format(AZURE_SEGMENT_STORE_PATH, ENVIRONMENT.getVariable(AzureUtilities.AZURE_ACCOUNT_NAME),
+ CONTAINER_NAME, DIR);
+ CloudBlobDirectory cloudBlobDirectory = ToolUtils.createCloudBlobDirectory(path, ENVIRONMENT);
+ return new AzurePersistence(cloudBlobDirectory);
+ }
+
+ @Override
+ public void close() {
+ if (fs != null) {
+ fs.close();
+ fs = null;
+ }
+ if (tmpDir != null) {
+ tmpDir.delete();
+ }
+ }
+
+ @Override
+ public void clean() throws IOException {
+ AzurePersistence azurePersistence = createAzurePersistence();
+ try {
+ AzureUtilities.deleteAllBlobs(azurePersistence.getSegmentstoreDirectory());
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+ }
+
+ @Override
+ public String getDescription() {
+ return "az:" + String.format(AZURE_SEGMENT_STORE_PATH, ENVIRONMENT.getVariable(AzureUtilities.AZURE_ACCOUNT_NAME),
+ CONTAINER_NAME, DIR);
+ }
+}
\ No newline at end of file
diff --git a/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentAzureFactoryTest.java b/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentAzureFactoryTest.java
new file mode 100644
index 0000000000..a28803f597
--- /dev/null
+++ b/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentAzureFactoryTest.java
@@ -0,0 +1,190 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.upgrade.cli.node;
+
+import com.microsoft.azure.storage.CloudStorageAccount;
+import com.microsoft.azure.storage.SharedAccessAccountPermissions;
+import com.microsoft.azure.storage.SharedAccessAccountPolicy;
+import com.microsoft.azure.storage.SharedAccessAccountResourceType;
+import com.microsoft.azure.storage.SharedAccessAccountService;
+import com.microsoft.azure.storage.blob.CloudBlobDirectory;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.jackrabbit.guava.common.io.Closer;
+import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzuriteDockerRule;
+import org.apache.jackrabbit.oak.segment.azure.AzureUtilities;
+import org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils;
+import org.apache.jackrabbit.oak.segment.azure.util.Environment;
+import org.apache.jackrabbit.oak.upgrade.cli.CliUtils;
+import org.jetbrains.annotations.NotNull;
+import org.junit.ClassRule;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.Date;
+import java.util.EnumSet;
+
+import static org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_ACCOUNT_NAME;
+import static org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_CLIENT_ID;
+import static org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_CLIENT_SECRET;
+import static org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_SECRET_KEY;
+import static org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_TENANT_ID;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assume.assumeNotNull;
+import static org.junit.Assume.assumeTrue;
+
+public class SegmentAzureFactoryTest {
+
+ @ClassRule
+ public static final AzuriteDockerRule azurite = new AzuriteDockerRule();
+
+ private static final Environment ENVIRONMENT = new Environment();
+ private static final String CONTAINER_NAME = "oak-test";
+ private static final String DIR = "repository";
+ private static final String CONNECTION_URI = "https://%s.blob.core.windows.net/%s";
+
+
+ @Test
+ public void testConnectionWithConnectionString_accessKey() throws IOException {
+ String connectionStringWithPlaceholder = "DefaultEndpointsProtocol=https;AccountName=%s;AccountKey=%s;BlobEndpoint=http://127.0.0.1:%s/%s;";
+ String connectionString = String.format(connectionStringWithPlaceholder, AzuriteDockerRule.ACCOUNT_NAME, AzuriteDockerRule.ACCOUNT_KEY, azurite.getMappedPort(), AzuriteDockerRule.ACCOUNT_NAME);
+ SegmentAzureFactory segmentAzureFactory = new SegmentAzureFactory.Builder("respository", 256,
+ false)
+ .connectionString(connectionString)
+ .containerName(CONTAINER_NAME)
+ .build();
+ Closer closer = Closer.create();
+ CliUtils.handleSigInt(closer);
+ FileStoreUtils.NodeStoreWithFileStore nodeStore = (FileStoreUtils.NodeStoreWithFileStore) segmentAzureFactory.create(null, closer);
+ assertEquals(1, nodeStore.getFileStore().getSegmentCount());
+ closer.close();
+ }
+
+ @Test
+ public void testConnectionWithConnectionString_sas() throws IOException {
+ String sasToken = getAccountSasToken();
+ String connectionStringWithPlaceholder = "DefaultEndpointsProtocol=http;AccountName=%s;SharedAccessSignature=%s;BlobEndpoint=http://127.0.0.1:%s/%s;";
+ String connectionString = String.format(connectionStringWithPlaceholder, AzuriteDockerRule.ACCOUNT_NAME, sasToken, azurite.getMappedPort(), AzuriteDockerRule.ACCOUNT_NAME);
+ SegmentAzureFactory segmentAzureFactory = new SegmentAzureFactory.Builder(DIR, 256,
+ false)
+ .connectionString(connectionString)
+ .containerName(CONTAINER_NAME)
+ .build();
+ Closer closer = Closer.create();
+ CliUtils.handleSigInt(closer);
+ FileStoreUtils.NodeStoreWithFileStore nodeStore = (FileStoreUtils.NodeStoreWithFileStore) segmentAzureFactory.create(null, closer);
+ assertEquals(1, nodeStore.getFileStore().getSegmentCount());
+ closer.close();
+ }
+
+ /* if AZURE_TENANT_ID, AZURE_CLIENT_ID, AZURE_CLIENT_SECRET are already set in environment variables
+ * then they will be given preference and authentication will be done via service principals and this
+ * test will be skipped
+ * */
+ @Test
+ public void testConnectionWithUri_accessKey() throws IOException {
+ assumeTrue(StringUtils.isBlank(ENVIRONMENT.getVariable(AZURE_TENANT_ID)));
+ assumeTrue(StringUtils.isBlank(ENVIRONMENT.getVariable(AZURE_CLIENT_ID)));
+ assumeTrue(StringUtils.isBlank(ENVIRONMENT.getVariable(AZURE_CLIENT_SECRET)));
+
+ assumeNotNull(ENVIRONMENT.getVariable(AZURE_ACCOUNT_NAME), ENVIRONMENT.getVariable(AZURE_SECRET_KEY));
+
+ final String CONTAINER_NAME = "oak-migration-test";
+
+ String uri = String.format(CONNECTION_URI, ENVIRONMENT.getVariable(AZURE_ACCOUNT_NAME), CONTAINER_NAME);
+ Closer closer = Closer.create();
+ try {
+ SegmentAzureFactory segmentAzureFactory = new SegmentAzureFactory.Builder(DIR, 256,
+ false)
+ .accountName(ENVIRONMENT.getVariable(AZURE_ACCOUNT_NAME))
+ .uri(uri)
+ .build();
+ closer = Closer.create();
+ CliUtils.handleSigInt(closer);
+ FileStoreUtils.NodeStoreWithFileStore nodeStore = (FileStoreUtils.NodeStoreWithFileStore) segmentAzureFactory.create(null, closer);
+ assertEquals(1, nodeStore.getFileStore().getSegmentCount());
+ } finally {
+ closer.close();
+ cleanup(uri);
+ }
+ }
+
+ @Test
+ public void testConnectionWithUri_servicePrincipal() throws IOException, InterruptedException {
+ assumeNotNull(ENVIRONMENT.getVariable(AZURE_ACCOUNT_NAME), ENVIRONMENT.getVariable(AZURE_TENANT_ID),
+ ENVIRONMENT.getVariable(AZURE_CLIENT_ID), ENVIRONMENT.getVariable(AZURE_CLIENT_SECRET));
+
+ final String CONTAINER_NAME = "oak-migration-test";
+
+ String uri = String.format(CONNECTION_URI, ENVIRONMENT.getVariable(AZURE_ACCOUNT_NAME), CONTAINER_NAME);
+ Closer closer = Closer.create();
+ try {
+ SegmentAzureFactory segmentAzureFactory = new SegmentAzureFactory.Builder(DIR, 256,
+ false)
+ .accountName(ENVIRONMENT.getVariable(AZURE_ACCOUNT_NAME))
+ .uri(uri)
+ .build();
+
+ CliUtils.handleSigInt(closer);
+ FileStoreUtils.NodeStoreWithFileStore nodeStore = (FileStoreUtils.NodeStoreWithFileStore) segmentAzureFactory.create(null, closer);
+ assertEquals(1, nodeStore.getFileStore().getSegmentCount());
+ } finally {
+ closer.close();
+ cleanup(uri);
+ }
+ }
+
+ private void cleanup(String uri) {
+ uri = uri + "/" + DIR;
+ try {
+ CloudBlobDirectory cloudBlobDirectory = ToolUtils.createCloudBlobDirectory(uri, ENVIRONMENT);
+ AzureUtilities.deleteAllBlobs(cloudBlobDirectory);
+ } catch (Exception e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
+
+ @NotNull
+ private String getAccountSasToken() {
+ try {
+ CloudStorageAccount cloudStorageAccount = azurite.getCloudStorageAccount();
+ return cloudStorageAccount.generateSharedAccessSignature(getPolicy());
+ } catch (Exception e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
+ @NotNull
+ private SharedAccessAccountPolicy getPolicy() {
+ SharedAccessAccountPolicy sharedAccessAccountPolicy = new SharedAccessAccountPolicy();
+ EnumSet<SharedAccessAccountPermissions> sharedAccessAccountPermissions = EnumSet.of(SharedAccessAccountPermissions.CREATE,
+ SharedAccessAccountPermissions.DELETE, SharedAccessAccountPermissions.READ, SharedAccessAccountPermissions.UPDATE,
+ SharedAccessAccountPermissions.WRITE, SharedAccessAccountPermissions.LIST);
+ EnumSet<SharedAccessAccountService> sharedAccessAccountServices = EnumSet.of(SharedAccessAccountService.BLOB);
+ EnumSet<SharedAccessAccountResourceType> sharedAccessAccountResourceTypes = EnumSet.of(
+ SharedAccessAccountResourceType.CONTAINER, SharedAccessAccountResourceType.OBJECT, SharedAccessAccountResourceType.SERVICE);
+
+ sharedAccessAccountPolicy.setPermissions(sharedAccessAccountPermissions);
+ sharedAccessAccountPolicy.setServices(sharedAccessAccountServices);
+ sharedAccessAccountPolicy.setResourceTypes(sharedAccessAccountResourceTypes);
+ sharedAccessAccountPolicy.setSharedAccessExpiryTime(Date.from(Instant.now().plus(Duration.ofDays(7))));
+ return sharedAccessAccountPolicy;
+ }
+
+}