You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by am...@apache.org on 2017/03/24 09:31:43 UTC
svn commit: r1788389 - in /jackrabbit/oak/trunk/oak-run: ./
src/main/java/org/apache/jackrabbit/oak/fixture/
src/main/java/org/apache/jackrabbit/oak/run/
Author: amitj
Date: Fri Mar 24 09:31:43 2017
New Revision: 1788389
URL: http://svn.apache.org/viewvc?rev=1788389&view=rev
Log:
OAK-4933: Create a data store implementation that integrates with Microsoft Azure Blob Storage
- Support for Azure in oak-run
Modified:
jackrabbit/oak/trunk/oak-run/pom.xml
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/fixture/BlobStoreFixture.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/fixture/DataStoreUtils.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCheckCommand.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Utils.java
Modified: jackrabbit/oak/trunk/oak-run/pom.xml
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/pom.xml?rev=1788389&r1=1788388&r2=1788389&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/pom.xml (original)
+++ jackrabbit/oak/trunk/oak-run/pom.xml Fri Mar 24 09:31:43 2017
@@ -346,6 +346,13 @@
</dependency>
<dependency>
+ <groupId>org.apache.jackrabbit</groupId>
+ <artifactId>oak-blob-cloud-azure</artifactId>
+ <version>${project.version}</version>
+ <optional>true</optional>
+ </dependency>
+
+ <dependency>
<groupId>org.apache.sling</groupId>
<artifactId>org.apache.sling.testing.osgi-mock</artifactId>
<scope>compile</scope>
Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/fixture/BlobStoreFixture.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/fixture/BlobStoreFixture.java?rev=1788389&r1=1788388&r2=1788389&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/fixture/BlobStoreFixture.java (original)
+++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/fixture/BlobStoreFixture.java Fri Mar 24 09:31:43 2017
@@ -45,7 +45,7 @@ import org.apache.jackrabbit.oak.stats.S
import static com.google.common.base.Preconditions.checkNotNull;
import static org.apache.jackrabbit.oak.fixture.DataStoreUtils.cleanup;
-import static org.apache.jackrabbit.oak.fixture.DataStoreUtils.configureIfS3DataStore;
+import static org.apache.jackrabbit.oak.fixture.DataStoreUtils.configureIfCloudDataStore;
public abstract class BlobStoreFixture implements Closeable{
private final String name;
@@ -85,7 +85,7 @@ public abstract class BlobStoreFixture i
String className = System.getProperty("dataStore");
if (className != null) {
- return getDataStore(basedir, fdsCacheInMB);
+ return getDataStore(basedir, fdsCacheInMB, statisticsProvider);
}
String blobStore = System.getProperty("blobStoreType");
@@ -178,7 +178,8 @@ public abstract class BlobStoreFixture i
};
}
- public static BlobStoreFixture getDataStore(final File basedir, final int fdsCacheInMB) {
+ public static BlobStoreFixture getDataStore(final File basedir, final int fdsCacheInMB,
+ final StatisticsProvider statisticsProvider) {
return new BlobStoreFixture("DS") {
private DataStore dataStore;
private BlobStore blobStore;
@@ -193,7 +194,8 @@ public abstract class BlobStoreFixture i
dataStore = Class.forName(className).asSubclass(DataStore.class).newInstance();
config = getConfig();
configure(dataStore, config);
- dataStore = configureIfS3DataStore(className, dataStore, config, unique.toLowerCase());
+
+ dataStore = configureIfCloudDataStore(className, dataStore, config, unique.toLowerCase(), statisticsProvider);
storeDir = new File(basedir, unique);
dataStore.init(storeDir.getAbsolutePath());
blobStore = new DataStoreBlobStore(dataStore, true, fdsCacheInMB);
@@ -209,7 +211,7 @@ public abstract class BlobStoreFixture i
if (blobStore instanceof DataStoreBlobStore) {
try {
((DataStoreBlobStore) blobStore).close();
- cleanup(storeDir, config);
+ cleanup(storeDir, config, unique.toLowerCase());
} catch (Exception e) {
throw new RuntimeException(e);
}
Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/fixture/DataStoreUtils.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/fixture/DataStoreUtils.java?rev=1788389&r1=1788388&r2=1788389&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/fixture/DataStoreUtils.java (original)
+++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/fixture/DataStoreUtils.java Fri Mar 24 09:31:43 2017
@@ -31,61 +31,84 @@ import com.amazonaws.services.s3.model.O
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.amazonaws.services.s3.transfer.TransferManager;
import com.google.common.base.Strings;
+import com.microsoft.azure.storage.blob.CloudBlobContainer;
+
import org.apache.commons.io.FileUtils;
import org.apache.jackrabbit.core.data.DataStore;
import org.apache.jackrabbit.oak.blob.cloud.aws.s3.SharedS3DataStore;
+import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureConstants;
+import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureDataStore;
import org.apache.jackrabbit.oak.blob.cloud.s3.S3Constants;
import org.apache.jackrabbit.oak.blob.cloud.s3.S3DataStore;
import org.apache.jackrabbit.oak.blob.cloud.s3.Utils;
+import org.apache.jackrabbit.oak.stats.StatisticsProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
- * Extension to {@link DataStoreUtils} to enable S3 extensions for cleaning and initialization.
+ * Extension to {@link DataStoreUtils} to enable S3 / AzureBlob extensions for cleaning and initialization.
*/
public class DataStoreUtils {
private static final Logger log = LoggerFactory.getLogger(DataStoreUtils.class);
private static Class JR2_S3 = SharedS3DataStore.class;
private static Class S3 = S3DataStore.class;
+ private static Class AZURE = AzureDataStore.class;
public static boolean isS3DataStore(String dsName) {
return (dsName != null) && (dsName.equals(S3.getName()) || dsName.equals(JR2_S3.getName()));
}
- public static DataStore configureIfS3DataStore(String className, DataStore ds,
- Map<String, ?> config, String bucket) throws Exception {
+ public static boolean isAzureDataStore(String dsName) {
+ return (dsName != null) &&
+ (dsName.equals(AZURE.getName()));
+ }
+
+ public static DataStore configureIfCloudDataStore(String className, DataStore ds,
+ Map<String, ?> config, String bucket,
+ StatisticsProvider statisticsProvider) throws Exception {
// Add bucket info
Properties props = new Properties();
props.putAll(config);
- props.setProperty(S3Constants.S3_BUCKET, bucket);
- // Set the props object
- if (S3.getName().equals(className)) {
- ((S3DataStore) ds).setProperties(props);
- } else
- if (JR2_S3.getName().equals(className)) {
- ((org.apache.jackrabbit.oak.blob.cloud.aws.s3.SharedS3DataStore) ds).setProperties(props);
+ log.info("Using bucket [ {} ]", bucket);
+
+ if (isS3DataStore(className)) {
+ props.setProperty(S3Constants.S3_BUCKET, bucket);
+
+ // Set the props object
+ if (S3.getName().equals(className)) {
+ ((S3DataStore) ds).setProperties(props);
+ ((S3DataStore) ds).setStatisticsProvider(statisticsProvider);
+ } else if (JR2_S3.getName().equals(className)) {
+ ((org.apache.jackrabbit.oak.blob.cloud.aws.s3.SharedS3DataStore) ds).setProperties(props);
+ }
+ } else if (isAzureDataStore(className)) {
+ props.setProperty(AzureConstants.AZURE_BLOB_CONTAINER_NAME, bucket);
+ ((AzureDataStore) ds).setProperties(props);
+ ((AzureDataStore) ds).setStatisticsProvider(statisticsProvider);
}
+
return ds;
}
/**
- * Clean directory and if S3 bucket configured delete that.
+ * Clean directory and if S3 bucket/Azure container is configured delete that.
*
* @param storeDir the local directory
* @param config the datastore config
+ * @param bucket the S3 bucket name / Azure container name
* @throws Exception
*/
- public static void cleanup(File storeDir, Map<String, ?> config) throws Exception {
+ public static void cleanup(File storeDir, Map<String, ?> config, String bucket) throws Exception {
FileUtils.deleteQuietly(storeDir);
- String bucket = null;
if (config.containsKey(S3Constants.S3_BUCKET)) {
- bucket = (String) config.get(S3Constants.S3_BUCKET);
if (!Strings.isNullOrEmpty(bucket)) {
deleteBucket(bucket, config, new Date());
}
+ } else if (config.containsKey(AzureConstants.AZURE_BLOB_CONTAINER_NAME)) {
+ deleteAzureContainer(config, bucket);
}
}
@@ -124,4 +147,22 @@ public class DataStoreUtils {
tmx.shutdownNow();
s3service.shutdown();
}
+
+ public static void deleteAzureContainer(Map<String, ?> config, String containerName) throws Exception {
+ String accountName = (String)config.get(AzureConstants.AZURE_STORAGE_ACCOUNT_NAME);
+ String accountKey = (String)config.get(AzureConstants.AZURE_STORAGE_ACCOUNT_KEY);
+ if (Strings.isNullOrEmpty(containerName) ||
+ Strings.isNullOrEmpty(accountName) ||
+ Strings.isNullOrEmpty(accountKey)) {
+ return;
+ }
+ log.info("deleting container [" + containerName + "]");
+ CloudBlobContainer container = org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.Utils
+ .getBlobContainer(org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.Utils.getConnectionString(accountName, accountKey), containerName);
+ if (container.deleteIfExists()) {
+ log.info("container [ " + containerName + "] deleted");
+ } else {
+ log.info("container [" + containerName + "] doesn't exists");
+ }
+ }
}
Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCheckCommand.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCheckCommand.java?rev=1788389&r1=1788388&r2=1788389&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCheckCommand.java (original)
+++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCheckCommand.java Fri Mar 24 09:31:43 2017
@@ -78,7 +78,7 @@ public class DataStoreCheckCommand imple
String helpStr =
"datastorecheck [--id] [--ref] [--consistency] [--store <path>|<mongo_uri>] "
- + "[--s3ds <s3ds_config>|--fds <fds_config>] [--dump <path>]";
+ + "[--s3ds <s3ds_config>|--fds <fds_config>|--azureblobds <azureblobds_config>] [--dump <path>]";
Closer closer = Closer.create();
try {
Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Utils.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Utils.java?rev=1788389&r1=1788388&r2=1788389&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Utils.java (original)
+++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Utils.java Fri Mar 24 09:31:43 2017
@@ -21,6 +21,7 @@ import static java.util.Arrays.asList;
import static org.apache.jackrabbit.oak.commons.PropertiesUtil.populate;
import java.io.Closeable;
+import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.Dictionary;
@@ -31,9 +32,11 @@ import java.util.Properties;
import javax.annotation.Nullable;
import javax.jcr.RepositoryException;
+import org.apache.commons.io.FileUtils;
import org.apache.felix.cm.file.ConfigurationHandler;
import org.apache.jackrabbit.core.data.DataStore;
import org.apache.jackrabbit.core.data.DataStoreException;
+import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureDataStore;
import org.apache.jackrabbit.oak.blob.cloud.aws.s3.SharedS3DataStore;
import org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore;
import org.apache.jackrabbit.oak.plugins.blob.datastore.OakFileDataStore;
@@ -45,6 +48,7 @@ import org.apache.jackrabbit.oak.spi.sta
import com.google.common.collect.Maps;
import com.google.common.io.Closer;
+import com.google.common.io.Files;
import com.mongodb.MongoClientURI;
import com.mongodb.MongoURI;
@@ -126,10 +130,13 @@ class Utils {
parser.accepts("s3ds", "S3DataStore config").withRequiredArg().ofType(String.class);
ArgumentAcceptingOptionSpec<String> fdsConfig =
parser.accepts("fds", "FileDataStore config").withRequiredArg().ofType(String.class);
+ ArgumentAcceptingOptionSpec<String> azureBlobDSConfig =
+ parser.accepts("azureblobds", "AzureBlobStorageDataStore config").withRequiredArg().ofType(String.class);
+
OptionSet options = parser.parse(args);
- if (!options.has(s3dsConfig) && !options.has(fdsConfig)) {
+ if (!options.has(s3dsConfig) && !options.has(fdsConfig) && !options.has(azureBlobDSConfig)) {
return null;
}
@@ -141,6 +148,15 @@ class Utils {
s3ds.setProperties(props);
s3ds.init(null);
delegate = s3ds;
+ } else if (options.has(azureBlobDSConfig)) {
+ AzureDataStore azureds = new AzureDataStore();
+ String cfgPath = azureBlobDSConfig.value(options);
+ Properties props = loadAndTransformProps(cfgPath);
+ azureds.setProperties(props);
+ File homeDir = Files.createTempDir();
+ azureds.init(homeDir.getAbsolutePath());
+ closer.register(asCloseable(homeDir));
+ delegate = azureds;
} else {
delegate = new OakFileDataStore();
String cfgPath = fdsConfig.value(options);
@@ -188,6 +204,16 @@ class Utils {
};
}
+ static Closeable asCloseable(final File dir) {
+ return new Closeable() {
+
+ @Override
+ public void close() throws IOException {
+ FileUtils.deleteDirectory(dir);
+ }
+ };
+ }
+
private static Properties loadAndTransformProps(String cfgPath) throws IOException {
Dictionary dict = ConfigurationHandler.read(new FileInputStream(cfgPath));
@@ -199,4 +225,4 @@ class Utils {
}
return props;
}
-}
\ No newline at end of file
+}