You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by am...@apache.org on 2016/09/23 08:24:56 UTC
svn commit: r1762024 - in /jackrabbit/oak/trunk: oak-blob-cloud/
oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/
oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/
oak-core/src/test/java/org/apache/jackrabbit/o...
Author: amitj
Date: Fri Sep 23 08:24:55 2016
New Revision: 1762024
URL: http://svn.apache.org/viewvc?rev=1762024&view=rev
Log:
OAK-4848: Improve oak-blob-cloud tests
* Inherit tests from base class copied from JR2
* Move initialization and check to S3DataStoreUtils. Can now be used with the same mechanism as already available directly with system properties
* Add standard @Before/@After annotations to setup/tearDown test cases
Added:
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/AbstractDataStoreTest.java (with props)
Removed:
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/TestAll.java
Modified:
jackrabbit/oak/trunk/oak-blob-cloud/pom.xml
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/S3DataStoreUtils.java
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/TestS3DSAsyncTouch.java
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/TestS3DSWithSSES3.java
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/TestS3DSWithSmallCache.java
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/TestS3Ds.java
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/TestS3DsCacheOff.java
Modified: jackrabbit/oak/trunk/oak-blob-cloud/pom.xml
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-cloud/pom.xml?rev=1762024&r1=1762023&r2=1762024&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-blob-cloud/pom.xml (original)
+++ jackrabbit/oak/trunk/oak-blob-cloud/pom.xml Fri Sep 23 08:24:55 2016
@@ -46,15 +46,6 @@
</instructions>
</configuration>
</plugin>
- <plugin>
- <artifactId>maven-surefire-plugin</artifactId>
- <configuration>
- <includes>
- <include>**/cloud/**/TestAll.java</include>
- <include>**/cloud/**/TestS3DataStore.java</include>
- </includes>
- </configuration>
- </plugin>
</plugins>
</build>
Modified: jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/S3DataStoreUtils.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/S3DataStoreUtils.java?rev=1762024&r1=1762023&r2=1762024&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/S3DataStoreUtils.java (original)
+++ jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/S3DataStoreUtils.java Fri Sep 23 08:24:55 2016
@@ -18,9 +18,12 @@
*/
package org.apache.jackrabbit.oak.blob.cloud;
+import java.io.FileInputStream;
+import java.io.InputStream;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
+import java.util.Map;
import java.util.Properties;
import com.amazonaws.services.s3.AmazonS3Client;
@@ -28,12 +31,18 @@ import com.amazonaws.services.s3.model.D
import com.amazonaws.services.s3.model.ObjectListing;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.amazonaws.services.s3.transfer.TransferManager;
+import com.google.common.base.Predicate;
+import com.google.common.base.Strings;
+import com.google.common.collect.Maps;
+import org.apache.commons.io.IOUtils;
import org.apache.jackrabbit.core.data.Backend;
import org.apache.jackrabbit.core.data.DataStore;
import org.apache.jackrabbit.oak.blob.cloud.aws.s3.S3Backend;
+import org.apache.jackrabbit.oak.blob.cloud.aws.s3.S3Constants;
import org.apache.jackrabbit.oak.blob.cloud.aws.s3.S3DataStore;
import org.apache.jackrabbit.oak.blob.cloud.aws.s3.SharedS3DataStore;
import org.apache.jackrabbit.oak.blob.cloud.aws.s3.Utils;
+import org.apache.jackrabbit.oak.commons.PropertiesUtil;
import org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -44,10 +53,71 @@ import org.slf4j.LoggerFactory;
public class S3DataStoreUtils extends DataStoreUtils {
private static final Logger log = LoggerFactory.getLogger(S3DataStoreUtils.class);
+ private static final String DEFAULT_CONFIG_PATH = "./src/test/resources/aws.properties";
+
public static boolean isS3DataStore() {
String dsName = System.getProperty(DS_CLASS_NAME);
- return (dsName != null) && (dsName.equals(S3DataStore.class.getName()) || dsName
+ boolean s3Class = (dsName != null) && (dsName.equals(S3DataStore.class.getName()) || dsName
.equals(SharedS3DataStore.class.getName()));
+ if (!isS3Configured()) {
+ return false;
+ }
+ return s3Class;
+ }
+
+ /**
+ * Check for presence of mandatory properties.
+ *
+ * @return true if mandatory props configured.
+ */
+ public static boolean isS3Configured() {
+ Properties props = getS3Config();
+ if (!props.containsKey(S3Constants.ACCESS_KEY) || !props.containsKey(S3Constants.SECRET_KEY)
+ || !(props.containsKey(S3Constants.S3_REGION) || props.containsKey(S3Constants.S3_END_POINT))) {
+ return false;
+ }
+ return true;
+ }
+
+ /**
+ * Read any config property configured.
+ * Also, read any props available as system properties.
+ * System properties take precedence.
+ *
+ * @return Properties instance
+ */
+ public static Properties getS3Config() {
+ String config = System.getProperty("config");
+ if (Strings.isNullOrEmpty(config)) {
+ config = DEFAULT_CONFIG_PATH;
+ }
+ Properties props = new Properties();
+ InputStream is = null;
+ try {
+ props.load(new FileInputStream(config));
+ } catch (Exception e) {
+ e.printStackTrace();
+ } finally {
+ IOUtils.closeQuietly(is);
+ }
+ props.putAll(getConfig());
+ Map filtered = Maps.filterEntries(Maps.fromProperties(props),
+ new Predicate<Map.Entry<? extends Object, ? extends Object>>() {
+ @Override
+ public boolean apply(Map.Entry<? extends Object, ? extends Object> input) {
+ return !Strings.isNullOrEmpty((String) input.getValue());
+ }
+ });
+ props = new Properties();
+ props.putAll(filtered);
+ return props;
+ }
+
+ public static DataStore getS3DataStore(String className, String homeDir) throws Exception {
+ DataStore ds = Class.forName(className).asSubclass(DataStore.class).newInstance();
+ PropertiesUtil.populate(ds, Maps.fromProperties(getS3Config()), false);
+ ds.init(homeDir);
+ return ds;
}
/**
@@ -69,7 +139,7 @@ public class S3DataStoreUtils extends Da
private static void deleteBucket(String bucket, Date date) throws Exception {
log.info("cleaning bucket [" + bucket + "]");
- Properties props = Utils.readConfig((String) getConfig().get("config"));
+ Properties props = getS3Config();
AmazonS3Client s3service = Utils.openService(props);
TransferManager tmx = new TransferManager(s3service);
if (s3service.doesBucketExist(bucket)) {
@@ -93,7 +163,7 @@ public class S3DataStoreUtils extends Da
prevObjectListing = s3service.listNextBatchOfObjects(prevObjectListing);
}
}
- //s3service.deleteBucket(bucket);
+ s3service.deleteBucket(bucket);
log.info("bucket [ " + bucket + "] cleaned");
} else {
log.info("bucket [" + bucket + "] doesn't exists");
Modified: jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/TestS3DSAsyncTouch.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/TestS3DSAsyncTouch.java?rev=1762024&r1=1762023&r2=1762024&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/TestS3DSAsyncTouch.java (original)
+++ jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/TestS3DSAsyncTouch.java Fri Sep 23 08:24:55 2016
@@ -16,34 +16,31 @@
*/
package org.apache.jackrabbit.oak.blob.cloud.aws.s3;
-import java.io.IOException;
-
import javax.jcr.RepositoryException;
import org.apache.jackrabbit.core.data.CachingDataStore;
+import org.apache.jackrabbit.oak.blob.cloud.S3DataStoreUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Test {@link org.apache.jackrabbit.core.data.CachingDataStore} with
- * {@link org.apache.jackrabbit.core.data.CachingDataStore#setTouchAsync(boolean) set to true. It requires
- * to pass aws config file via system property. For e.g.
- * -Dconfig=/opt/cq/aws.properties. Sample aws properties located at
+ * {@link org.apache.jackrabbit.core.data.CachingDataStore#setTouchAsync(boolean) set to true.
+ * It requires to pass aws config file via system property or system properties by prefixing with 'ds.'.
+ * See details @ {@link S3DataStoreUtils}.
+ * For e.g. -Dconfig=/opt/cq/aws.properties. Sample aws properties located at
* src/test/resources/aws.properties
*/
public class TestS3DSAsyncTouch extends TestS3Ds {
protected static final Logger LOG = LoggerFactory.getLogger(TestS3DSAsyncTouch.class);
- public TestS3DSAsyncTouch() throws IOException {
-
- }
-
@Override
protected CachingDataStore createDataStore() throws RepositoryException {
S3DataStore s3ds = new S3DataStore();
s3ds.setProperties(props);
s3ds.setTouchAsync(true);
s3ds.setSecret("123456");
+ s3ds.setAsyncUploadLimit(0);
s3ds.init(dataStoreDir);
s3ds.updateModifiedDateOnAccess(System.currentTimeMillis() + 50 * 1000);
sleep(1000);
Modified: jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/TestS3DSWithSSES3.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/TestS3DSWithSSES3.java?rev=1762024&r1=1762023&r2=1762024&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/TestS3DSWithSSES3.java (original)
+++ jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/TestS3DSWithSSES3.java Fri Sep 23 08:24:55 2016
@@ -18,25 +18,31 @@
package org.apache.jackrabbit.oak.blob.cloud.aws.s3;
import java.io.ByteArrayInputStream;
-import java.io.IOException;
import javax.jcr.RepositoryException;
import org.apache.jackrabbit.core.data.CachingDataStore;
import org.apache.jackrabbit.core.data.DataRecord;
+import org.apache.jackrabbit.oak.blob.cloud.S3DataStoreUtils;
+import org.junit.Assert;
+import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static org.junit.Assert.fail;
+
/**
* Test S3DataStore operation with SSE_S3 encryption.
+ * It requires to pass aws config file via system property or system properties by prefixing with 'ds.'.
+ * See details @ {@link S3DataStoreUtils}.
+ * For e.g. -Dconfig=/opt/cq/aws.properties. Sample aws properties located at
+ * src/test/resources/aws.properties
+
*/
public class TestS3DSWithSSES3 extends TestS3Ds {
protected static final Logger LOG = LoggerFactory.getLogger(TestS3DSWithSSES3.class);
- public TestS3DSWithSSES3() throws IOException {
- }
-
@Override
protected CachingDataStore createDataStore() throws RepositoryException {
props.setProperty(S3Constants.S3_ENCRYPTION,
@@ -52,6 +58,7 @@ public class TestS3DSWithSSES3 extends T
/**
* Test data migration enabling SSE_S3 encryption.
*/
+ @Test
public void testDataMigration() {
try {
String bucket = props.getProperty(S3Constants.S3_BUCKET);
@@ -62,7 +69,7 @@ public class TestS3DSWithSSES3 extends T
byte[] data = new byte[dataLength];
randomGen.nextBytes(data);
DataRecord rec = s3ds.addRecord(new ByteArrayInputStream(data));
- assertEquals(data.length, rec.getLength());
+ Assert.assertEquals(data.length, rec.getLength());
assertRecord(data, rec);
s3ds.close();
@@ -77,7 +84,7 @@ public class TestS3DSWithSSES3 extends T
s3ds.init(dataStoreDir);
rec = s3ds.getRecord(rec.getIdentifier());
- assertEquals(data.length, rec.getLength());
+ Assert.assertEquals(data.length, rec.getLength());
assertRecord(data, rec);
randomGen.nextBytes(data);
Modified: jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/TestS3DSWithSmallCache.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/TestS3DSWithSmallCache.java?rev=1762024&r1=1762023&r2=1762024&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/TestS3DSWithSmallCache.java (original)
+++ jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/TestS3DSWithSmallCache.java Fri Sep 23 08:24:55 2016
@@ -16,28 +16,27 @@
*/
package org.apache.jackrabbit.oak.blob.cloud.aws.s3;
-import java.io.IOException;
-
import javax.jcr.RepositoryException;
import org.apache.jackrabbit.core.data.CachingDataStore;
import org.apache.jackrabbit.core.data.LocalCache;
+import org.apache.jackrabbit.oak.blob.cloud.S3DataStoreUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Test {@link CachingDataStore} with S3Backend and with very small size (@link
- * {@link LocalCache}. It requires to pass aws config file via system property.
+ * {@link LocalCache}.
+ * It requires to pass aws config file via system property or system properties by prefixing with 'ds.'.
+ * See details @ {@link S3DataStoreUtils}.
* For e.g. -Dconfig=/opt/cq/aws.properties. Sample aws properties located at
* src/test/resources/aws.properties
+
*/
public class TestS3DSWithSmallCache extends TestS3Ds {
protected static final Logger LOG = LoggerFactory.getLogger(TestS3DSWithSmallCache.class);
- public TestS3DSWithSmallCache() throws IOException {
- }
-
@Override
protected CachingDataStore createDataStore() throws RepositoryException {
S3DataStore s3ds = new S3DataStore();
Modified: jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/TestS3Ds.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/TestS3Ds.java?rev=1762024&r1=1762023&r2=1762024&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/TestS3Ds.java (original)
+++ jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/TestS3Ds.java Fri Sep 23 08:24:55 2016
@@ -16,70 +16,62 @@
*/
package org.apache.jackrabbit.oak.blob.cloud.aws.s3;
-import java.io.IOException;
-import java.util.ArrayList;
import java.util.Date;
-import java.util.List;
import java.util.Properties;
import javax.jcr.RepositoryException;
-import com.google.common.base.Strings;
-import org.apache.jackrabbit.core.data.Backend;
import org.apache.jackrabbit.core.data.CachingDataStore;
-import org.apache.jackrabbit.core.data.TestCaseBase;
+import org.apache.jackrabbit.oak.blob.cloud.S3DataStoreUtils;
+import org.apache.jackrabbit.oak.plugins.blob.datastore.AbstractDataStoreTest;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.BeforeClass;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.amazonaws.services.s3.AmazonS3Client;
-import com.amazonaws.services.s3.model.DeleteObjectsRequest;
-import com.amazonaws.services.s3.model.ObjectListing;
-import com.amazonaws.services.s3.model.S3ObjectSummary;
-import com.amazonaws.services.s3.transfer.TransferManager;
+import static org.apache.jackrabbit.oak.blob.cloud.S3DataStoreUtils.getS3Config;
+import static org.apache.jackrabbit.oak.blob.cloud.S3DataStoreUtils.isS3Configured;
+import static org.junit.Assume.assumeTrue;
/**
- * Test {@link org.apache.jackrabbit.core.data.CachingDataStore} with S3Backend and local cache on. It requires
- * to pass aws config file via system property. For e.g.
- * -Dconfig=/opt/cq/aws.properties. Sample aws properties located at
+ * Test {@link org.apache.jackrabbit.core.data.CachingDataStore} with S3Backend and local cache on.
+ * It requires to pass aws config file via system property or system properties by prefixing with 'ds.'.
+ * See details @ {@link S3DataStoreUtils}.
+ * For e.g. -Dconfig=/opt/cq/aws.properties. Sample aws properties located at
* src/test/resources/aws.properties
*/
-public class TestS3Ds extends TestCaseBase {
+public class TestS3Ds extends AbstractDataStoreTest {
protected static final Logger LOG = LoggerFactory.getLogger(TestS3Ds.class);
- private static final String DEFAULT_CONFIG_PATH = "./src/test/resources/aws.properties";
-
private Date startTime = null;
protected Properties props;
- protected String config;
-
- public TestS3Ds() throws IOException {
- config = System.getProperty(CONFIG);
- if (Strings.isNullOrEmpty(config)) {
- config = DEFAULT_CONFIG_PATH;
- }
- props = Utils.readConfig(config);
+ @BeforeClass
+ public static void assumptions() {
+ assumeTrue(isS3Configured());
}
-
@Override
- protected void setUp() throws Exception {
+ @Before
+ public void setUp() throws Exception {
+ props = getS3Config();
startTime = new Date();
- super.setUp();
- String bucket = String.valueOf(randomGen.nextInt(9999)) + "-"
- + String.valueOf(randomGen.nextInt(9999)) + "-test";
+ String bucket =
+ String.valueOf(randomGen.nextInt(9999)) + "-" + String.valueOf(randomGen.nextInt(9999))
+ + "-test";
props.setProperty(S3Constants.S3_BUCKET, bucket);
- // delete bucket if exists
- deleteBucket(bucket);
+ super.setUp();
}
@Override
- protected void tearDown() {
+ @After
+ public void tearDown() {
try {
- deleteBucket();
super.tearDown();
+ S3DataStoreUtils.cleanup(ds, startTime);
} catch (Exception ignore) {
}
@@ -94,52 +86,4 @@ public class TestS3Ds extends TestCaseBa
sleep(1000);
return s3ds;
}
-
- /**
- * Cleaning of bucket after test run.
- */
- /**
- * Cleaning of bucket after test run.
- */
- public void deleteBucket() throws Exception {
- Backend backend = ((S3DataStore) ds).getBackend();
- String bucket = ((S3Backend) backend).getBucket();
- deleteBucket(bucket);
- }
-
- public void deleteBucket(String bucket) throws Exception {
- LOG.info("deleting bucket [" + bucket + "]");
- Properties props = Utils.readConfig(config);
- AmazonS3Client s3service = Utils.openService(props);
- TransferManager tmx = new TransferManager(s3service);
-
- if (s3service.doesBucketExist(bucket)) {
- for (int i = 0; i < 4; i++) {
- tmx.abortMultipartUploads(bucket, startTime);
- ObjectListing prevObjectListing = s3service.listObjects(bucket);
- while (prevObjectListing != null) {
- List<DeleteObjectsRequest.KeyVersion> deleteList = new ArrayList<DeleteObjectsRequest.KeyVersion>();
- for (S3ObjectSummary s3ObjSumm : prevObjectListing.getObjectSummaries()) {
- deleteList.add(new DeleteObjectsRequest.KeyVersion(
- s3ObjSumm.getKey()));
- }
- if (deleteList.size() > 0) {
- DeleteObjectsRequest delObjsReq = new DeleteObjectsRequest(
- bucket);
- delObjsReq.setKeys(deleteList);
- s3service.deleteObjects(delObjsReq);
- }
- if (!prevObjectListing.isTruncated()) break;
- prevObjectListing = s3service.listNextBatchOfObjects(prevObjectListing);
- }
- }
- s3service.deleteBucket(bucket);
- LOG.info("bucket [ " + bucket + "] deleted");
-
- } else {
- LOG.info("bucket [" + bucket + "] doesn't exists");
- }
- tmx.shutdownNow();
- s3service.shutdown();
- }
}
Modified: jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/TestS3DsCacheOff.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/TestS3DsCacheOff.java?rev=1762024&r1=1762023&r2=1762024&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/TestS3DsCacheOff.java (original)
+++ jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/TestS3DsCacheOff.java Fri Sep 23 08:24:55 2016
@@ -16,27 +16,26 @@
*/
package org.apache.jackrabbit.oak.blob.cloud.aws.s3;
-import java.io.IOException;
-
import javax.jcr.RepositoryException;
import org.apache.jackrabbit.core.data.CachingDataStore;
+import org.apache.jackrabbit.oak.blob.cloud.S3DataStoreUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Test {@link org.apache.jackrabbit.core.data.CachingDataStore} with S3Backend
- * and local cache Off. It requires to pass aws config file via system property.
+ * and local cache Off.
+ * It requires to pass aws config file via system property or system properties by prefixing with 'ds.'.
+ * See details @ {@link S3DataStoreUtils}.
* For e.g. -Dconfig=/opt/cq/aws.properties. Sample aws properties located at
* src/test/resources/aws.properties
+
*/
public class TestS3DsCacheOff extends TestS3Ds {
protected static final Logger LOG = LoggerFactory.getLogger(TestS3DsCacheOff.class);
- public TestS3DsCacheOff() throws IOException {
- }
-
@Override
protected CachingDataStore createDataStore() throws RepositoryException {
S3DataStore s3ds = new S3DataStore();
@@ -47,4 +46,4 @@ public class TestS3DsCacheOff extends Te
sleep(1000);
return s3ds;
}
-}
\ No newline at end of file
+}
Added: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/AbstractDataStoreTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/AbstractDataStoreTest.java?rev=1762024&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/AbstractDataStoreTest.java (added)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/AbstractDataStoreTest.java Fri Sep 23 08:24:55 2016
@@ -0,0 +1,649 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.blob.datastore;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.SequenceInputStream;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Properties;
+import java.util.Random;
+
+import javax.jcr.RepositoryException;
+
+import org.apache.jackrabbit.core.data.DataIdentifier;
+import org.apache.jackrabbit.core.data.DataRecord;
+import org.apache.jackrabbit.core.data.DataStore;
+import org.apache.jackrabbit.core.data.DataStoreException;
+import org.apache.jackrabbit.core.data.MultiDataStoreAware;
+import org.apache.jackrabbit.core.data.RandomInputStream;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/**
+ * Test base class for {@link DataStore} which covers all scenarios.
+ * Copied from {@link org.apache.jackrabbit.core.data.TestCaseBase}.
+ */
+public abstract class AbstractDataStoreTest {
+
+ /**
+ * Logger
+ */
+ protected static final Logger LOG = LoggerFactory.getLogger(AbstractDataStoreTest.class);
+
+ @Rule
+ public TemporaryFolder folder = new TemporaryFolder(new File("target"));
+
+ /**
+ * length of record to be added
+ */
+ protected int dataLength = 123456;
+
+ /**
+ * datastore directory path
+ */
+ protected String dataStoreDir;
+
+ protected DataStore ds;
+
+ /**
+ * Random number generator to populate data
+ */
+ protected Random randomGen = new Random();
+
+ /**
+ * Delete temporary directory.
+ */
+ @Before
+ protected void setUp() throws Exception {
+ dataStoreDir = folder.newFolder().getAbsolutePath();
+ ds = createDataStore();
+ }
+
+ @After
+ protected void tearDown() {
+ try {
+ ds.close();
+ } catch (DataStoreException e) {
+ LOG.info("Error in close ds", e);
+ }
+ }
+
+ /**
+ * Testcase to validate {@link DataStore#addRecord(InputStream)} API.
+ */
+ @Test
+ public void testAddRecord() {
+ try {
+ long start = System.currentTimeMillis();
+ LOG.info("Testcase: " + this.getClass().getName()
+ + "#addRecord, testDir=" + dataStoreDir);
+ doAddRecordTest();
+ LOG.info("Testcase: " + this.getClass().getName()
+ + "#addRecord finished, time taken = ["
+ + (System.currentTimeMillis() - start) + "]ms");
+ } catch (Exception e) {
+ LOG.error("error:", e);
+ fail(e.getMessage());
+ }
+ }
+
+ /**
+ * Testcase to validate {@link DataStore#getRecord(DataIdentifier)} API.
+ */
+ @Test
+ public void testGetRecord() {
+ try {
+ long start = System.currentTimeMillis();
+ LOG.info("Testcase: " + this.getClass().getName()
+ + "#testGetRecord, testDir=" + dataStoreDir);
+ doGetRecordTest();
+ LOG.info("Testcase: " + this.getClass().getName()
+ + "#testGetRecord finished, time taken = ["
+ + (System.currentTimeMillis() - start) + "]ms");
+ } catch (Exception e) {
+ LOG.error("error:", e);
+ }
+ }
+
+ /**
+ * Testcase to validate {@link DataStore#getAllIdentifiers()} API.
+ */
+ @Test
+ public void testGetAllIdentifiers() {
+ try {
+ long start = System.currentTimeMillis();
+ LOG.info("Testcase: " + this.getClass().getName()
+ + "#testGetAllIdentifiers, testDir=" + dataStoreDir);
+ doGetAllIdentifiersTest();
+ LOG.info("Testcase: " + this.getClass().getName()
+ + "#testGetAllIdentifiers finished, time taken = ["
+ + (System.currentTimeMillis() - start) + "]ms");
+ } catch (Exception e) {
+ LOG.error("error:", e);
+ fail(e.getMessage());
+ }
+ }
+
+ /**
+ * Testcase to validate {@link DataStore#updateModifiedDateOnAccess(long)}
+ * API.
+ */
+ @Test
+ public void testUpdateLastModifiedOnAccess() {
+ try {
+ long start = System.currentTimeMillis();
+ LOG.info("Testcase: " + this.getClass().getName()
+ + "#testUpdateLastModifiedOnAccess, testDir=" + dataStoreDir);
+ doUpdateLastModifiedOnAccessTest();
+ LOG.info("Testcase: " + this.getClass().getName()
+ + "#testUpdateLastModifiedOnAccess finished, time taken = ["
+ + (System.currentTimeMillis() - start) + "]ms");
+ } catch (Exception e) {
+ LOG.error("error:", e);
+ }
+ }
+
+ /**
+ * Testcase to validate
+ * {@link MultiDataStoreAware#deleteRecord(DataIdentifier)}.API.
+ */
+ @Test
+ public void testDeleteRecord() {
+ try {
+ long start = System.currentTimeMillis();
+ LOG.info("Testcase: " + this.getClass().getName()
+ + "#testDeleteRecord, testDir=" + dataStoreDir);
+ doDeleteRecordTest();
+ LOG.info("Testcase: " + this.getClass().getName()
+ + "#testDeleteRecord finished, time taken = ["
+ + (System.currentTimeMillis() - start) + "]ms");
+ } catch (Exception e) {
+ LOG.error("error:", e);
+ fail(e.getMessage());
+ }
+ }
+
+ /**
+ * Testcase to validate {@link DataStore#deleteAllOlderThan(long)} API.
+ */
+ @Test
+ public void testDeleteAllOlderThan() {
+ try {
+ long start = System.currentTimeMillis();
+ LOG.info("Testcase: " + this.getClass().getName()
+ + "#testDeleteAllOlderThan, testDir=" + dataStoreDir);
+ doDeleteAllOlderThan();
+ LOG.info("Testcase: " + this.getClass().getName()
+ + "#testDeleteAllOlderThan finished, time taken = ["
+ + (System.currentTimeMillis() - start) + "]ms");
+ } catch (Exception e) {
+ LOG.error("error:", e);
+ fail(e.getMessage());
+ }
+ }
+
+ /**
+ * Testcase to validate {@link DataStore#getRecordFromReference(String)}
+ */
+ @Test
+ public void testReference() {
+ try {
+ long start = System.currentTimeMillis();
+ LOG.info("Testcase: " + this.getClass().getName()
+ + "#testReference, testDir=" + dataStoreDir);
+ doReferenceTest();
+ LOG.info("Testcase: " + this.getClass().getName()
+ + "#testReference finished, time taken = ["
+ + (System.currentTimeMillis() - start) + "]ms");
+ } catch (Exception e) {
+ LOG.error("error:", e);
+ fail(e.getMessage());
+ }
+ }
+
+ /**
+ * Testcase to validate mixed scenario use of {@link DataStore}.
+ */
+ @Test
+ public void testSingleThread() {
+ try {
+ long start = System.currentTimeMillis();
+ LOG.info("Testcase: " + this.getClass().getName()
+ + "#testSingleThread, testDir=" + dataStoreDir);
+ doTestSingleThread();
+ LOG.info("Testcase: " + this.getClass().getName()
+ + "#testSingleThread finished, time taken = ["
+ + (System.currentTimeMillis() - start) + "]ms");
+ } catch (Exception e) {
+ LOG.error("error:", e);
+ fail(e.getMessage());
+ }
+ }
+
+ /**
+ * Testcase to validate mixed scenario use of {@link DataStore} in
+ * multi-threaded concurrent environment.
+ */
+ @Test
+ public void testMultiThreaded() {
+ try {
+ long start = System.currentTimeMillis();
+ LOG.info("Testcase: " + this.getClass().getName()
+ + "#testMultiThreaded, testDir=" + dataStoreDir);
+ doTestMultiThreaded();
+ LOG.info("Testcase: " + this.getClass().getName()
+ + "#testMultiThreaded finished, time taken = ["
+ + (System.currentTimeMillis() - start) + "]ms");
+ } catch (Exception e) {
+ LOG.error("error:", e);
+ fail(e.getMessage());
+ }
+
+ }
+
+ protected abstract DataStore createDataStore() throws RepositoryException ;
+
+ /**
+ * Test {@link DataStore#addRecord(InputStream)} and assert length of added
+ * record.
+ */
+ protected void doAddRecordTest() throws Exception {
+ byte[] data = new byte[dataLength];
+ randomGen.nextBytes(data);
+ DataRecord rec = ds.addRecord(new ByteArrayInputStream(data));
+ Assert.assertEquals(data.length, rec.getLength());
+ assertRecord(data, rec);
+ }
+
+ /**
+ * Test {@link DataStore#getRecord(DataIdentifier)} and assert length and
+ * inputstream.
+ */
+ protected void doGetRecordTest() throws Exception {
+ byte[] data = new byte[dataLength];
+ randomGen.nextBytes(data);
+ DataRecord rec = ds.addRecord(new ByteArrayInputStream(data));
+ rec = ds.getRecord(rec.getIdentifier());
+ Assert.assertEquals(data.length, rec.getLength());
+ assertRecord(data, rec);
+ }
+
+ /**
+ * Test {@link MultiDataStoreAware#deleteRecord(DataIdentifier)}.
+ */
+ protected void doDeleteRecordTest() throws Exception {
+ Random random = randomGen;
+ byte[] data1 = new byte[dataLength];
+ random.nextBytes(data1);
+ DataRecord rec1 = ds.addRecord(new ByteArrayInputStream(data1));
+
+ byte[] data2 = new byte[dataLength];
+ random.nextBytes(data2);
+ DataRecord rec2 = ds.addRecord(new ByteArrayInputStream(data2));
+
+ byte[] data3 = new byte[dataLength];
+ random.nextBytes(data3);
+ DataRecord rec3 = ds.addRecord(new ByteArrayInputStream(data3));
+
+ ((MultiDataStoreAware)ds).deleteRecord(rec2.getIdentifier());
+
+ assertNull("rec2 should be null",
+ ds.getRecordIfStored(rec2.getIdentifier()));
+ assertEquals(new ByteArrayInputStream(data1),
+ ds.getRecord(rec1.getIdentifier()).getStream());
+ assertEquals(new ByteArrayInputStream(data3),
+ ds.getRecord(rec3.getIdentifier()).getStream());
+ }
+
+ /**
+ * Test {@link DataStore#getAllIdentifiers()} and asserts all identifiers
+ * are returned.
+ */
+ protected void doGetAllIdentifiersTest() throws Exception {
+ List<DataIdentifier> list = new ArrayList<DataIdentifier>();
+ Random random = randomGen;
+ byte[] data = new byte[dataLength];
+ random.nextBytes(data);
+ DataRecord rec = ds.addRecord(new ByteArrayInputStream(data));
+ list.add(rec.getIdentifier());
+
+ data = new byte[dataLength];
+ random.nextBytes(data);
+ rec = ds.addRecord(new ByteArrayInputStream(data));
+ list.add(rec.getIdentifier());
+
+ data = new byte[dataLength];
+ random.nextBytes(data);
+ rec = ds.addRecord(new ByteArrayInputStream(data));
+ list.add(rec.getIdentifier());
+
+ Iterator<DataIdentifier> itr = ds.getAllIdentifiers();
+ while (itr.hasNext()) {
+ assertTrue("record found on list", list.remove(itr.next()));
+ }
+ Assert.assertEquals(0, list.size());
+ }
+
+ /**
+ * Asserts that timestamp of all records accessed after
+ * {@link DataStore#updateModifiedDateOnAccess(long)} invocation.
+ */
+ protected void doUpdateLastModifiedOnAccessTest() throws Exception {
+ Random random = randomGen;
+ byte[] data = new byte[dataLength];
+ random.nextBytes(data);
+ DataRecord rec1 = ds.addRecord(new ByteArrayInputStream(data));
+
+ data = new byte[dataLength];
+ random.nextBytes(data);
+ DataRecord rec2 = ds.addRecord(new ByteArrayInputStream(data));
+ LOG.debug("rec2 timestamp=" + rec2.getLastModified());
+
+ // sleep for some time to ensure that async upload completes in backend.
+ sleep(6000);
+ long updateTime = System.currentTimeMillis();
+ LOG.debug("updateTime=" + updateTime);
+ ds.updateModifiedDateOnAccess(updateTime);
+
+ // sleep to workaround System.currentTimeMillis granularity.
+ sleep(3000);
+ data = new byte[dataLength];
+ random.nextBytes(data);
+ DataRecord rec3 = ds.addRecord(new ByteArrayInputStream(data));
+
+ data = new byte[dataLength];
+ random.nextBytes(data);
+ DataRecord rec4 = ds.addRecord(new ByteArrayInputStream(data));
+
+ rec1 = ds.getRecord(rec1.getIdentifier());
+
+ Assert.assertEquals("rec1 touched", true, rec1.getLastModified() > updateTime);
+ LOG.debug("rec2 timestamp=" + rec2.getLastModified());
+ Assert.assertEquals("rec2 not touched", true,
+ rec2.getLastModified() < updateTime);
+ Assert.assertEquals("rec3 touched", true, rec3.getLastModified() > updateTime);
+ Assert.assertEquals("rec4 touched", true, rec4.getLastModified() > updateTime);
+ }
+
+ /**
+ * Asserts that {@link DataStore#deleteAllOlderThan(long)} only deleted
+ * records older than argument passed.
+ */
+ protected void doDeleteAllOlderThan() throws Exception {
+ Random random = randomGen;
+ byte[] data = new byte[dataLength];
+ random.nextBytes(data);
+ DataRecord rec1 = ds.addRecord(new ByteArrayInputStream(data));
+
+ data = new byte[dataLength];
+ random.nextBytes(data);
+ DataRecord rec2 = ds.addRecord(new ByteArrayInputStream(data));
+
+ // sleep for some time to ensure that async upload completes in backend.
+ sleep(10000);
+ long updateTime = System.currentTimeMillis();
+ ds.updateModifiedDateOnAccess(updateTime);
+
+ // sleep to workaround System.currentTimeMillis granularity.
+ sleep(3000);
+ data = new byte[dataLength];
+ random.nextBytes(data);
+ DataRecord rec3 = ds.addRecord(new ByteArrayInputStream(data));
+
+ data = new byte[dataLength];
+ random.nextBytes(data);
+ DataRecord rec4 = ds.addRecord(new ByteArrayInputStream(data));
+
+ rec1 = ds.getRecord(rec1.getIdentifier());
+ ds.clearInUse();
+ Assert.assertEquals("only rec2 should be deleted", 1,
+ ds.deleteAllOlderThan(updateTime));
+ assertNull("rec2 should be null",
+ ds.getRecordIfStored(rec2.getIdentifier()));
+
+ Iterator<DataIdentifier> itr = ds.getAllIdentifiers();
+ List<DataIdentifier> list = new ArrayList<DataIdentifier>();
+ list.add(rec1.getIdentifier());
+ list.add(rec3.getIdentifier());
+ list.add(rec4.getIdentifier());
+ while (itr.hasNext()) {
+ assertTrue("record found on list", list.remove(itr.next()));
+ }
+
+ Assert.assertEquals("touched records found", 0, list.size());
+ Assert.assertEquals("rec1 touched", true, rec1.getLastModified() > updateTime);
+ Assert.assertEquals("rec3 touched", true, rec3.getLastModified() > updateTime);
+ Assert.assertEquals("rec4 touched", true, rec4.getLastModified() > updateTime);
+ }
+
+ /**
+ * Test if record can be accessed via
+ * {@link DataStore#getRecordFromReference(String)}
+ */
+ protected void doReferenceTest() throws Exception {
+ byte[] data = new byte[dataLength];
+ randomGen.nextBytes(data);
+ String reference;
+ DataRecord record = ds.addRecord(new ByteArrayInputStream(data));
+ reference = record.getReference();
+ assertReference(data, reference, ds);
+ }
+
+ /**
+ * Method to validate mixed scenario use of {@link DataStore}.
+ */
+ protected void doTestSingleThread() throws Exception {
+ doTestMultiThreaded(ds, 1);
+ }
+
+ /**
+ * Method to validate mixed scenario use of {@link DataStore} in
+ * multi-threaded concurrent environment.
+ */
+ protected void doTestMultiThreaded() throws Exception {
+ doTestMultiThreaded(ds, 4);
+ }
+
+ /**
+ * Method to assert record with byte array.
+ */
+ protected void assertRecord(byte[] expected, DataRecord record)
+ throws DataStoreException, IOException {
+ InputStream stream = record.getStream();
+ try {
+ for (int i = 0; i < expected.length; i++) {
+ Assert.assertEquals(expected[i] & 0xff, stream.read());
+ }
+ Assert.assertEquals(-1, stream.read());
+ } finally {
+ stream.close();
+ }
+ }
+
+ /**
+ * Method to run {@link AbstractDataStoreTest#doTest(DataStore, int)} in multiple
+ * concurrent threads.
+ */
+ protected void doTestMultiThreaded(final DataStore ds, int threadCount)
+ throws Exception {
+ final Exception[] exception = new Exception[1];
+ Thread[] threads = new Thread[threadCount];
+ for (int i = 0; i < threadCount; i++) {
+ final int x = i;
+ Thread t = new Thread() {
+ public void run() {
+ try {
+ doTest(ds, x);
+ } catch (Exception e) {
+ exception[0] = e;
+ }
+ }
+ };
+ threads[i] = t;
+ t.start();
+ }
+ for (int i = 0; i < threadCount; i++) {
+ threads[i].join();
+ }
+ if (exception[0] != null) {
+ throw exception[0];
+ }
+ }
+
+ /**
+ * Assert randomly read stream from record.
+ */
+ void doTest(DataStore ds, int offset) throws Exception {
+ ArrayList<DataRecord> list = new ArrayList<DataRecord>();
+ HashMap<DataRecord, Integer> map = new HashMap<DataRecord, Integer>();
+ for (int i = 0; i < 10; i++) {
+ int size = 100000 - (i * 100);
+ RandomInputStream in = new RandomInputStream(size + offset, size);
+ DataRecord rec = ds.addRecord(in);
+ list.add(rec);
+ map.put(rec, new Integer(size));
+ }
+ Random random = new Random(1);
+ for (int i = 0; i < list.size(); i++) {
+ int pos = random.nextInt(list.size());
+ DataRecord rec = list.get(pos);
+ int size = map.get(rec);
+ rec = ds.getRecord(rec.getIdentifier());
+ Assert.assertEquals(size, rec.getLength());
+ RandomInputStream expected = new RandomInputStream(size + offset,
+ size);
+ InputStream in = rec.getStream();
+ // Workaround for race condition that can happen with low cache size relative to the test
+ // read immediately
+ byte[] buffer = new byte[1];
+ in.read(buffer);
+ in = new SequenceInputStream(new ByteArrayInputStream(buffer), in);
+
+ if (random.nextBoolean()) {
+ in = readInputStreamRandomly(in, random);
+ }
+ assertEquals(expected, in);
+ }
+ }
+
+ InputStream readInputStreamRandomly(InputStream in, Random random)
+ throws IOException {
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ byte[] buffer = new byte[8000];
+ while (true) {
+ if (random.nextBoolean()) {
+ int x = in.read();
+ if (x < 0) {
+ break;
+ }
+ out.write(x);
+ } else {
+ if (random.nextBoolean()) {
+ int l = in.read(buffer);
+ if (l < 0) {
+ break;
+ }
+ out.write(buffer, 0, l);
+ } else {
+ int offset = random.nextInt(buffer.length / 2);
+ int len = random.nextInt(buffer.length / 2);
+ int l = in.read(buffer, offset, len);
+ if (l < 0) {
+ break;
+ }
+ out.write(buffer, offset, l);
+ }
+ }
+ }
+ in.close();
+ return new ByteArrayInputStream(out.toByteArray());
+ }
+
+ /**
+ * Assert two inputstream
+ */
+ protected void assertEquals(InputStream a, InputStream b)
+ throws IOException {
+ try {
+ assertTrue("binary not equal",
+ org.apache.commons.io.IOUtils.contentEquals(a, b));
+ } finally {
+ try {
+ a.close();
+ } catch (Exception ignore) {
+ }
+ try {
+ b.close();
+ } catch (Exception ignore) {
+ }
+ }
+ }
+
+ /**
+ * Assert inputstream read from reference.
+ */
+ protected void assertReference(byte[] expected, String reference,
+ DataStore store) throws Exception {
+ DataRecord record = store.getRecordFromReference(reference);
+ assertNotNull(record);
+ Assert.assertEquals(expected.length, record.getLength());
+
+ InputStream stream = record.getStream();
+ try {
+ assertTrue("binary not equal",
+ org.apache.commons.io.IOUtils.contentEquals(
+ new ByteArrayInputStream(expected), stream));
+ } finally {
+ stream.close();
+ }
+ }
+
+ /**
+ * Utility method to stop execution for duration time.
+ *
+ * @param duration
+ * time in milli seconds
+ */
+ protected void sleep(long duration) {
+ long expected = System.currentTimeMillis() + duration;
+ while (System.currentTimeMillis() < expected) {
+ try {
+ Thread.sleep(1);
+ } catch (InterruptedException ie) {
+
+ }
+ }
+ }
+}
Propchange: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/AbstractDataStoreTest.java
------------------------------------------------------------------------------
svn:eol-style = native