You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by ad...@apache.org on 2018/07/25 13:25:30 UTC
svn commit: r1836633 - in /jackrabbit/oak/trunk:
oak-run/src/main/java/org/apache/jackrabbit/oak/run/
oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/
oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/...
Author: adulceanu
Date: Wed Jul 25 13:25:30 2018
New Revision: 1836633
URL: http://svn.apache.org/viewvc?rev=1836633&view=rev
Log:
OAK-7459 - oak-run compact should support Azure Segment Store
Added:
jackrabbit/oak/trunk/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/
jackrabbit/oak/trunk/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java
jackrabbit/oak/trunk/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/util/
jackrabbit/oak/trunk/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/util/AzureConfigurationParserUtils.java
- copied, changed from r1836549, jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/AzureParserUtils.java
jackrabbit/oak/trunk/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/config/
jackrabbit/oak/trunk/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/config/AzureConfigurationParserUtilsTest.java
- copied, changed from r1836549, jackrabbit/oak/trunk/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/parser/AzureParserUtilsTest.java
Removed:
jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/AzureParserUtils.java
jackrabbit/oak/trunk/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/parser/AzureParserUtilsTest.java
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java
jackrabbit/oak/trunk/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureUtilities.java
jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentAzureFactory.java
jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/StoreType.java
Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java?rev=1836633&r1=1836632&r2=1836633&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java (original)
+++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java Wed Jul 25 13:25:30 2018
@@ -24,6 +24,7 @@ import joptsimple.OptionParser;
import joptsimple.OptionSet;
import joptsimple.OptionSpec;
import org.apache.jackrabbit.oak.run.commons.Command;
+import org.apache.jackrabbit.oak.segment.azure.tool.AzureCompact;
import org.apache.jackrabbit.oak.segment.tool.Compact;
class CompactCommand implements Command {
@@ -36,13 +37,13 @@ class CompactCommand implements Command
public void execute(String... args) throws Exception {
OptionParser parser = new OptionParser();
OptionSpec<String> directoryArg = parser.nonOptions(
- "Path to segment store (required)").ofType(String.class);
+ "Path/URI to TAR/remote segment store (required)").ofType(String.class);
OptionSpec<Boolean> mmapArg = parser.accepts("mmap",
"Use memory mapped access if true, use file access if false. " +
"If not specified, memory mapped access is used on 64 bit " +
- "systems and file access is used on 32 bit systems. On " +
- "Windows, regular file access is always enforced and this " +
- "option is ignored.")
+ "systems and file access is used on 32 bit systems. For " +
+ "remote segment stores and on Windows, regular file access " +
+ "is always enforced and this option is ignored.")
.withOptionalArg()
.ofType(Boolean.class);
OptionSpec<Boolean> forceArg = parser.accepts("force",
@@ -61,15 +62,27 @@ class CompactCommand implements Command
System.exit(-1);
}
- int code = Compact.builder()
- .withPath(new File(path))
- .withForce(isTrue(forceArg.value(options)))
- .withMmap(mmapArg.value(options))
- .withOs(StandardSystemProperty.OS_NAME.value())
- .withSegmentCacheSize(Integer.getInteger("cache", 256))
- .withGCLogInterval(Long.getLong("compaction-progress-log", 150000))
- .build()
- .run();
+ int code = 0;
+
+ if (path.startsWith("az:")) {
+ code = AzureCompact.builder()
+ .withPath(path.substring(3))
+ .withForce(isTrue(forceArg.value(options)))
+ .withSegmentCacheSize(Integer.getInteger("cache", 256))
+ .withGCLogInterval(Long.getLong("compaction-progress-log", 150000))
+ .build()
+ .run();
+ } else {
+ code = Compact.builder()
+ .withPath(new File(path))
+ .withForce(isTrue(forceArg.value(options)))
+ .withMmap(mmapArg.value(options))
+ .withOs(StandardSystemProperty.OS_NAME.value())
+ .withSegmentCacheSize(Integer.getInteger("cache", 256))
+ .withGCLogInterval(Long.getLong("compaction-progress-log", 150000))
+ .build()
+ .run();
+ }
System.exit(code);
}
Modified: jackrabbit/oak/trunk/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureUtilities.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureUtilities.java?rev=1836633&r1=1836632&r2=1836633&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureUtilities.java (original)
+++ jackrabbit/oak/trunk/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureUtilities.java Wed Jul 25 13:25:30 2018
@@ -16,17 +16,23 @@
*/
package org.apache.jackrabbit.oak.segment.azure;
+import com.microsoft.azure.storage.CloudStorageAccount;
+import com.microsoft.azure.storage.StorageCredentials;
import com.microsoft.azure.storage.StorageException;
+import com.microsoft.azure.storage.StorageUri;
import com.microsoft.azure.storage.blob.BlobListingDetails;
import com.microsoft.azure.storage.blob.CloudBlob;
+import com.microsoft.azure.storage.blob.CloudBlobContainer;
import com.microsoft.azure.storage.blob.CloudBlobDirectory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
+import java.net.URI;
import java.net.URISyntaxException;
import java.nio.ByteBuffer;
import java.nio.file.Paths;
+import java.security.InvalidKeyException;
import java.util.EnumSet;
import java.util.UUID;
import java.util.stream.Stream;
@@ -89,4 +95,21 @@ public final class AzureUtilities {
}
});
}
+
+ public static CloudBlobDirectory cloudBlobDirectoryFrom(StorageCredentials credentials,
+ String uri, String dir) throws URISyntaxException, StorageException {
+ StorageUri storageUri = new StorageUri(new URI(uri));
+ CloudBlobContainer container = new CloudBlobContainer(storageUri, credentials);
+
+ return container.getDirectoryReference(dir);
+ }
+
+ public static CloudBlobDirectory cloudBlobDirectoryFrom(String connection, String containerName,
+ String dir) throws InvalidKeyException, URISyntaxException, StorageException {
+ CloudStorageAccount cloud = CloudStorageAccount.parse(connection.toString());
+ CloudBlobContainer container = cloud.createCloudBlobClient().getContainerReference(containerName);
+ container.createIfNotExists();
+
+ return container.getDirectoryReference(dir);
+ }
}
Added: jackrabbit/oak/trunk/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java?rev=1836633&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java (added)
+++ jackrabbit/oak/trunk/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java Wed Jul 25 13:25:30 2018
@@ -0,0 +1,276 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.jackrabbit.oak.segment.azure.tool;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
+import static org.apache.jackrabbit.oak.segment.SegmentCache.DEFAULT_SEGMENT_CACHE_MB;
+import static org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.KEY_ACCOUNT_NAME;
+import static org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.KEY_DIR;
+import static org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.KEY_STORAGE_URI;
+import static org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.parseAzureConfigurationFromUri;
+import static org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions.defaultGCOptions;
+
+import java.io.IOException;
+import java.io.PrintStream;
+import java.net.URISyntaxException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.jackrabbit.oak.segment.SegmentCache;
+import org.apache.jackrabbit.oak.segment.azure.AzureJournalFile;
+import org.apache.jackrabbit.oak.segment.azure.AzurePersistence;
+import org.apache.jackrabbit.oak.segment.azure.AzureUtilities;
+import org.apache.jackrabbit.oak.segment.file.FileStore;
+import org.apache.jackrabbit.oak.segment.file.FileStoreBuilder;
+import org.apache.jackrabbit.oak.segment.file.InvalidFileStoreVersionException;
+import org.apache.jackrabbit.oak.segment.file.JournalReader;
+import org.apache.jackrabbit.oak.segment.spi.monitor.FileStoreMonitorAdapter;
+import org.apache.jackrabbit.oak.segment.spi.monitor.IOMonitorAdapter;
+import org.apache.jackrabbit.oak.segment.spi.persistence.JournalFile;
+import org.apache.jackrabbit.oak.segment.spi.persistence.JournalFileWriter;
+import org.apache.jackrabbit.oak.segment.spi.persistence.SegmentArchiveManager;
+import org.apache.jackrabbit.oak.segment.spi.persistence.SegmentNodeStorePersistence;
+import org.apache.jackrabbit.oak.segment.tool.Compact;
+
+import com.google.common.base.Stopwatch;
+import com.google.common.io.Files;
+import com.microsoft.azure.storage.StorageCredentials;
+import com.microsoft.azure.storage.StorageCredentialsAccountAndKey;
+import com.microsoft.azure.storage.StorageException;
+import com.microsoft.azure.storage.blob.CloudBlobDirectory;
+
+/**
+ * Perform an offline compaction of an existing Azure Segment Store.
+ */
+public class AzureCompact {
+
+ /**
+ * Create a builder for the {@link Compact} command.
+ *
+ * @return an instance of {@link Builder}.
+ */
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ /**
+ * Collect options for the {@link Compact} command.
+ */
+ public static class Builder {
+
+ private String path;
+
+ private boolean force;
+
+ private long gcLogInterval = 150000;
+
+ private int segmentCacheSize = DEFAULT_SEGMENT_CACHE_MB;
+
+ private Builder() {
+ // Prevent external instantiation.
+ }
+
+ /**
+ * The path (URI) to an existing segment store. This parameter is required.
+ *
+ * @param path
+ * the path to an existing segment store.
+ * @return this builder.
+ */
+ public Builder withPath(String path) {
+ this.path = checkNotNull(path);
+ return this;
+ }
+
+ /**
+ * Whether to fail if run on an older version of the store of force upgrading
+ * its format.
+ *
+ * @param force
+ * upgrade iff {@code true}
+ * @return this builder.
+ */
+ public Builder withForce(boolean force) {
+ this.force = force;
+ return this;
+ }
+
+ /**
+ * The size of the segment cache in MB. The default of
+ * {@link SegmentCache#DEFAULT_SEGMENT_CACHE_MB} when this method is not
+ * invoked.
+ *
+ * @param segmentCacheSize
+ * cache size in MB
+ * @return this builder
+ * @throws IllegalArgumentException
+ * if {@code segmentCacheSize} is not a positive integer.
+ */
+ public Builder withSegmentCacheSize(int segmentCacheSize) {
+ checkArgument(segmentCacheSize > 0, "segmentCacheSize must be strictly positive");
+ this.segmentCacheSize = segmentCacheSize;
+ return this;
+ }
+
+ /**
+ * The number of nodes after which an update about the compaction process is
+ * logged. Set to a negative number to disable progress logging. If not
+ * specified, it defaults to 150,000 nodes.
+ *
+ * @param gcLogInterval
+ * The log interval.
+ * @return this builder.
+ */
+ public Builder withGCLogInterval(long gcLogInterval) {
+ this.gcLogInterval = gcLogInterval;
+ return this;
+ }
+
+ /**
+ * Create an executable version of the {@link Compact} command.
+ *
+ * @return an instance of {@link Runnable}.
+ */
+ public AzureCompact build() {
+ checkNotNull(path);
+ return new AzureCompact(this);
+ }
+
+ }
+
+ private static String printableStopwatch(Stopwatch s) {
+ return String.format("%s (%ds)", s, s.elapsed(TimeUnit.SECONDS));
+ }
+
+ private final String path;
+
+ private final int segmentCacheSize;
+
+ private final boolean strictVersionCheck;
+
+ private final long gcLogInterval;
+
+ private AzureCompact(Builder builder) {
+ this.path = builder.path;
+ this.segmentCacheSize = builder.segmentCacheSize;
+ this.strictVersionCheck = !builder.force;
+ this.gcLogInterval = builder.gcLogInterval;
+ }
+
+ public int run() {
+ Stopwatch watch = Stopwatch.createStarted();
+ CloudBlobDirectory cloudBlobDirectory = null;
+ try {
+ cloudBlobDirectory = createCloudBlobDirectory();
+ } catch (URISyntaxException | StorageException e1) {
+ throw new IllegalArgumentException(
+ "Could not connect to the Azure Storage. Please verify the path provided!");
+ }
+
+ SegmentNodeStorePersistence persistence = new AzurePersistence(cloudBlobDirectory);
+ SegmentArchiveManager archiveManager = null;
+ try {
+ archiveManager = persistence.createArchiveManager(false, new IOMonitorAdapter(),
+ new FileStoreMonitorAdapter());
+ } catch (IOException e) {
+ throw new IllegalArgumentException(
+ "Could not access the Azure Storage. Please verify the path provided!");
+ }
+
+ System.out.printf("Compacting %s\n", path);
+ System.out.printf(" before\n");
+ List<String> beforeArchives = Collections.emptyList();
+ try {
+ beforeArchives = archiveManager.listArchives();
+ } catch (IOException e) {
+ System.err.println(e);
+ }
+
+ printArchives(System.out, beforeArchives);
+ System.out.printf(" -> compacting\n");
+
+ try (FileStore store = newFileStore(persistence)) {
+ if (!store.compactFull()) {
+ System.out.printf("Compaction cancelled after %s.\n", printableStopwatch(watch));
+ return 1;
+ }
+ System.out.printf(" -> cleaning up\n");
+ store.cleanup();
+ JournalFile journal = new AzureJournalFile(cloudBlobDirectory, "journal.log");
+ String head;
+ try (JournalReader journalReader = new JournalReader(journal)) {
+ head = String.format("%s root %s\n", journalReader.next().getRevision(), System.currentTimeMillis());
+ }
+
+ try (JournalFileWriter journalWriter = journal.openJournalWriter()) {
+ System.out.printf(" -> writing new %s: %s\n", journal.getName(), head);
+ journalWriter.truncate();
+ journalWriter.writeLine(head);
+ }
+ } catch (Exception e) {
+ watch.stop();
+ e.printStackTrace(System.err);
+ System.out.printf("Compaction failed after %s.\n", printableStopwatch(watch));
+ return 1;
+ }
+
+ watch.stop();
+ System.out.printf(" after\n");
+ List<String> afterArchives = Collections.emptyList();
+ try {
+ afterArchives = archiveManager.listArchives();
+ } catch (IOException e) {
+ System.err.println(e);
+ }
+ printArchives(System.out, afterArchives);
+ System.out.printf("Compaction succeeded in %s.\n", printableStopwatch(watch));
+ return 0;
+ }
+
+ private static void printArchives(PrintStream s, List<String> archives) {
+ for (String a : archives) {
+ s.printf(" %s\n", a);
+ }
+ }
+
+ private FileStore newFileStore(SegmentNodeStorePersistence persistence)
+ throws IOException, InvalidFileStoreVersionException, URISyntaxException, StorageException {
+ FileStoreBuilder builder = FileStoreBuilder.fileStoreBuilder(Files.createTempDir())
+ .withCustomPersistence(persistence).withMemoryMapping(false).withStrictVersionCheck(strictVersionCheck)
+ .withSegmentCacheSize(segmentCacheSize)
+ .withGCOptions(defaultGCOptions().setOffline().setGCLogInterval(gcLogInterval));
+
+ return builder.build();
+ }
+
+ private CloudBlobDirectory createCloudBlobDirectory() throws URISyntaxException, StorageException {
+ Map<String, String> config = parseAzureConfigurationFromUri(path);
+
+ String accountName = config.get(KEY_ACCOUNT_NAME);
+ String key = System.getenv("AZURE_SECRET_KEY");
+ StorageCredentials credentials = new StorageCredentialsAccountAndKey(accountName, key);
+
+ String uri = config.get(KEY_STORAGE_URI);
+ String dir = config.get(KEY_DIR);
+
+ return AzureUtilities.cloudBlobDirectoryFrom(credentials, uri, dir);
+ }
+}
Copied: jackrabbit/oak/trunk/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/util/AzureConfigurationParserUtils.java (from r1836549, jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/AzureParserUtils.java)
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/util/AzureConfigurationParserUtils.java?p2=jackrabbit/oak/trunk/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/util/AzureConfigurationParserUtils.java&p1=jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/AzureParserUtils.java&r1=1836549&r2=1836633&rev=1836633&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/AzureParserUtils.java (original)
+++ jackrabbit/oak/trunk/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/util/AzureConfigurationParserUtils.java Wed Jul 25 13:25:30 2018
@@ -14,9 +14,9 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.jackrabbit.oak.upgrade.cli.parser;
+package org.apache.jackrabbit.oak.segment.azure.util;
-import static org.apache.jackrabbit.oak.upgrade.cli.parser.AzureParserUtils.AzureConnectionKey.*;
+import static org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.AzureConnectionKey.*;
import java.net.URI;
import java.net.URISyntaxException;
@@ -28,7 +28,7 @@ import java.util.Map;
* string, container name, uri, etc.) from custom encoded String or Azure
* standard URI.
*/
-public class AzureParserUtils {
+public class AzureConfigurationParserUtils {
public enum AzureConnectionKey {
DEFAULT_ENDPOINTS_PROTOCOL("DefaultEndpointsProtocol"),
ACCOUNT_NAME("AccountName"),
@@ -54,7 +54,7 @@ public class AzureParserUtils {
public static final String KEY_STORAGE_URI = "storageUri";
public static final String KEY_DIR = "directory";
- private AzureParserUtils() {
+ private AzureConfigurationParserUtils() {
// prevent instantiation
}
@@ -152,3 +152,4 @@ public class AzureParserUtils {
return config;
}
}
+
Copied: jackrabbit/oak/trunk/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/config/AzureConfigurationParserUtilsTest.java (from r1836549, jackrabbit/oak/trunk/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/parser/AzureParserUtilsTest.java)
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/config/AzureConfigurationParserUtilsTest.java?p2=jackrabbit/oak/trunk/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/config/AzureConfigurationParserUtilsTest.java&p1=jackrabbit/oak/trunk/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/parser/AzureParserUtilsTest.java&r1=1836549&r2=1836633&rev=1836633&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/parser/AzureParserUtilsTest.java (original)
+++ jackrabbit/oak/trunk/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/config/AzureConfigurationParserUtilsTest.java Wed Jul 25 13:25:30 2018
@@ -14,25 +14,26 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.jackrabbit.oak.upgrade.cli.parser;
+package org.apache.jackrabbit.oak.segment.azure.config;
-import static org.apache.jackrabbit.oak.upgrade.cli.parser.AzureParserUtils.KEY_ACCOUNT_NAME;
-import static org.apache.jackrabbit.oak.upgrade.cli.parser.AzureParserUtils.KEY_CONNECTION_STRING;
-import static org.apache.jackrabbit.oak.upgrade.cli.parser.AzureParserUtils.KEY_CONTAINER_NAME;
-import static org.apache.jackrabbit.oak.upgrade.cli.parser.AzureParserUtils.KEY_DIR;
-import static org.apache.jackrabbit.oak.upgrade.cli.parser.AzureParserUtils.KEY_STORAGE_URI;
+import static org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.KEY_ACCOUNT_NAME;
+import static org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.KEY_CONNECTION_STRING;
+import static org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.KEY_CONTAINER_NAME;
+import static org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.KEY_DIR;
+import static org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.KEY_STORAGE_URI;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import java.util.Map;
+import org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils;
import org.junit.Test;
-public class AzureParserUtilsTest {
+public class AzureConfigurationParserUtilsTest {
@Test
- public void testParseConnectionDetailsFromCustomConnection() throws CliArgumentException {
+ public void testParseConnectionDetailsFromCustomConnection() {
StringBuilder conn = new StringBuilder();
StringBuilder connStr = new StringBuilder();
connStr.append("DefaultEndpointsProtocol=https;");
@@ -44,16 +45,16 @@ public class AzureParserUtilsTest {
conn.append("ContainerName=oak-test;");
conn.append("Directory=repository");
- assertTrue(AzureParserUtils.isCustomAzureConnectionString(conn.toString()));
+ assertTrue(AzureConfigurationParserUtils.isCustomAzureConnectionString(conn.toString()));
- Map<String, String> config = AzureParserUtils.parseAzureConfigurationFromCustomConnection(conn.toString());
+ Map<String, String> config = AzureConfigurationParserUtils.parseAzureConfigurationFromCustomConnection(conn.toString());
assertEquals(connStr.toString(), config.get(KEY_CONNECTION_STRING));
assertEquals("oak-test", config.get(KEY_CONTAINER_NAME));
assertEquals("repository", config.get(KEY_DIR));
}
@Test
- public void testParseConnectionDetailsFromCustomConnectionShuffledKeys() throws CliArgumentException {
+ public void testParseConnectionDetailsFromCustomConnectionShuffledKeys() {
StringBuilder conn = new StringBuilder();
conn.append("Directory=repository;");
conn.append("DefaultEndpointsProtocol=https;");
@@ -62,21 +63,21 @@ public class AzureParserUtilsTest {
conn.append("BlobEndpoint=http://127.0.0.1:32806/myaccount;");
conn.append("AccountKey=mykey==");
- assertTrue(AzureParserUtils.isCustomAzureConnectionString(conn.toString()));
+ assertTrue(AzureConfigurationParserUtils.isCustomAzureConnectionString(conn.toString()));
String azureConn = "DefaultEndpointsProtocol=https;AccountName=myaccount;AccountKey=mykey==;BlobEndpoint=http://127.0.0.1:32806/myaccount;";
- Map<String, String> config = AzureParserUtils.parseAzureConfigurationFromCustomConnection(conn.toString());
+ Map<String, String> config = AzureConfigurationParserUtils.parseAzureConfigurationFromCustomConnection(conn.toString());
assertEquals(azureConn, config.get(KEY_CONNECTION_STRING));
assertEquals("oak-test", config.get(KEY_CONTAINER_NAME));
assertEquals("repository", config.get(KEY_DIR));
}
@Test
- public void testParseConnectionDetailsFromUri() throws CliArgumentException {
+ public void testParseConnectionDetailsFromUri() {
String uri = "https://myaccount.blob.core.windows.net/oak-test/repository";
- assertFalse(AzureParserUtils.isCustomAzureConnectionString(uri));
+ assertFalse(AzureConfigurationParserUtils.isCustomAzureConnectionString(uri));
- Map<String, String> config = AzureParserUtils.parseAzureConfigurationFromUri(uri);
+ Map<String, String> config = AzureConfigurationParserUtils.parseAzureConfigurationFromUri(uri);
assertEquals("myaccount", config.get(KEY_ACCOUNT_NAME));
assertEquals("https://myaccount.blob.core.windows.net/oak-test", config.get(KEY_STORAGE_URI));
Modified: jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentAzureFactory.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentAzureFactory.java?rev=1836633&r1=1836632&r2=1836633&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentAzureFactory.java (original)
+++ jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentAzureFactory.java Wed Jul 25 13:25:30 2018
@@ -19,12 +19,12 @@ package org.apache.jackrabbit.oak.upgrad
import static org.apache.jackrabbit.oak.upgrade.cli.node.FileStoreUtils.asCloseable;
import java.io.IOException;
-import java.net.URI;
import java.net.URISyntaxException;
import java.security.InvalidKeyException;
import org.apache.jackrabbit.oak.segment.SegmentNodeStoreBuilders;
import org.apache.jackrabbit.oak.segment.azure.AzurePersistence;
+import org.apache.jackrabbit.oak.segment.azure.AzureUtilities;
import org.apache.jackrabbit.oak.segment.file.FileStore;
import org.apache.jackrabbit.oak.segment.file.FileStoreBuilder;
import org.apache.jackrabbit.oak.segment.file.InvalidFileStoreVersionException;
@@ -35,12 +35,10 @@ import org.apache.jackrabbit.oak.upgrade
import com.google.common.io.Closer;
import com.google.common.io.Files;
-import com.microsoft.azure.storage.CloudStorageAccount;
import com.microsoft.azure.storage.StorageCredentials;
import com.microsoft.azure.storage.StorageCredentialsAccountAndKey;
import com.microsoft.azure.storage.StorageException;
-import com.microsoft.azure.storage.StorageUri;
-import com.microsoft.azure.storage.blob.CloudBlobContainer;
+import com.microsoft.azure.storage.blob.CloudBlobDirectory;
public class SegmentAzureFactory implements NodeStoreFactory {
private final String accountName;
@@ -132,28 +130,21 @@ public class SegmentAzureFactory impleme
}
private AzurePersistence createAzurePersistence() throws StorageException, URISyntaxException, InvalidKeyException {
- AzurePersistence azPersistence = null;
+ CloudBlobDirectory cloudBlobDirectory = null;
if (accountName != null && uri != null) {
String key = System.getenv("AZURE_SECRET_KEY");
StorageCredentials credentials = new StorageCredentialsAccountAndKey(accountName, key);
- StorageUri storageUri = new StorageUri(new URI(uri));
- CloudBlobContainer cloudBlobContainer = new CloudBlobContainer(storageUri, credentials);
-
- azPersistence = new AzurePersistence(cloudBlobContainer.getDirectoryReference(dir));
+ cloudBlobDirectory = AzureUtilities.cloudBlobDirectoryFrom(credentials, uri, dir);
} else if (connectionString != null && containerName != null) {
- CloudStorageAccount cloud = CloudStorageAccount.parse(connectionString.toString());
- CloudBlobContainer container = cloud.createCloudBlobClient().getContainerReference(containerName);
- container.createIfNotExists();
-
- azPersistence = new AzurePersistence(container.getDirectoryReference(dir));
+ cloudBlobDirectory = AzureUtilities.cloudBlobDirectoryFrom(connectionString, containerName, dir);
}
- if (azPersistence == null) {
+ if (cloudBlobDirectory == null) {
throw new IllegalArgumentException("Could not connect to Azure storage. Too few connection parameters specified!");
}
- return azPersistence;
+ return new AzurePersistence(cloudBlobDirectory);
}
@Override
Modified: jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/StoreType.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/StoreType.java?rev=1836633&r1=1836632&r2=1836633&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/StoreType.java (original)
+++ jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/StoreType.java Wed Jul 25 13:25:30 2018
@@ -17,19 +17,18 @@
package org.apache.jackrabbit.oak.upgrade.cli.parser;
import static org.apache.commons.lang.StringUtils.removeStart;
+import static org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.KEY_ACCOUNT_NAME;
+import static org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.KEY_CONNECTION_STRING;
+import static org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.KEY_CONTAINER_NAME;
+import static org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.KEY_DIR;
+import static org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.KEY_STORAGE_URI;
+import static org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.isCustomAzureConnectionString;
+import static org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.parseAzureConfigurationFromCustomConnection;
+import static org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.parseAzureConfigurationFromUri;
import static org.apache.jackrabbit.oak.upgrade.cli.node.Jackrabbit2Factory.isJcr2Repository;
import static org.apache.jackrabbit.oak.upgrade.cli.node.Jackrabbit2Factory.isRepositoryXml;
-import static org.apache.jackrabbit.oak.upgrade.cli.parser.AzureParserUtils.isCustomAzureConnectionString;
-import static org.apache.jackrabbit.oak.upgrade.cli.parser.AzureParserUtils.parseAzureConfigurationFromCustomConnection;
-import static org.apache.jackrabbit.oak.upgrade.cli.parser.AzureParserUtils.parseAzureConfigurationFromUri;
import static org.apache.jackrabbit.oak.upgrade.cli.parser.StoreArguments.SEGMENT_AZURE_PREFIX;
import static org.apache.jackrabbit.oak.upgrade.cli.parser.StoreArguments.SEGMENT_OLD_PREFIX;
-import static org.apache.jackrabbit.oak.upgrade.cli.parser.AzureParserUtils.KEY_ACCOUNT_NAME;
-import static org.apache.jackrabbit.oak.upgrade.cli.parser.AzureParserUtils.KEY_STORAGE_URI;
-import static org.apache.jackrabbit.oak.upgrade.cli.parser.AzureParserUtils.KEY_CONNECTION_STRING;
-import static org.apache.jackrabbit.oak.upgrade.cli.parser.AzureParserUtils.KEY_CONTAINER_NAME;
-import static org.apache.jackrabbit.oak.upgrade.cli.parser.AzureParserUtils.KEY_DIR;
-
import java.util.Map;