You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@jackrabbit.apache.org by am...@apache.org on 2014/10/17 09:30:57 UTC
svn commit: r1632482 - in /jackrabbit/trunk:
jackrabbit-aws-ext/src/main/java/org/apache/jackrabbit/aws/ext/ds/
jackrabbit-aws-ext/src/test/java/org/apache/jackrabbit/aws/ext/
jackrabbit-aws-ext/src/test/java/org/apache/jackrabbit/aws/ext/ds/
jackrabbi...
Author: amitj
Date: Fri Oct 17 07:30:57 2014
New Revision: 1632482
URL: http://svn.apache.org/r1632482
Log:
JCR-3816: [aws-ext]S3DS not able update lastModified of record > 5GB
JCR-3817: [jackrabbit-aws-ext] Performance of operation degrades while running DS GC
Applying patch from Shashank Gupta
Added:
jackrabbit/trunk/jackrabbit-aws-ext/src/test/java/org/apache/jackrabbit/aws/ext/ds/TestS3DSAsyncTouch.java (with props)
jackrabbit/trunk/jackrabbit-aws-ext/src/test/java/org/apache/jackrabbit/aws/ext/ds/TestS3DSWithSmallCache.java (with props)
jackrabbit/trunk/jackrabbit-data/src/main/java/org/apache/jackrabbit/core/data/AsyncTouchCallback.java (with props)
jackrabbit/trunk/jackrabbit-data/src/main/java/org/apache/jackrabbit/core/data/AsyncTouchResult.java (with props)
Modified:
jackrabbit/trunk/jackrabbit-aws-ext/src/main/java/org/apache/jackrabbit/aws/ext/ds/S3Backend.java
jackrabbit/trunk/jackrabbit-aws-ext/src/test/java/org/apache/jackrabbit/aws/ext/TestAll.java
jackrabbit/trunk/jackrabbit-data/src/main/java/org/apache/jackrabbit/core/data/Backend.java
jackrabbit/trunk/jackrabbit-data/src/main/java/org/apache/jackrabbit/core/data/CachingDataStore.java
jackrabbit/trunk/jackrabbit-data/src/test/java/org/apache/jackrabbit/core/data/InMemoryBackend.java
jackrabbit/trunk/jackrabbit-data/src/test/java/org/apache/jackrabbit/core/data/TestCaseBase.java
Modified: jackrabbit/trunk/jackrabbit-aws-ext/src/main/java/org/apache/jackrabbit/aws/ext/ds/S3Backend.java
URL: http://svn.apache.org/viewvc/jackrabbit/trunk/jackrabbit-aws-ext/src/main/java/org/apache/jackrabbit/aws/ext/ds/S3Backend.java?rev=1632482&r1=1632481&r2=1632482&view=diff
==============================================================================
--- jackrabbit/trunk/jackrabbit-aws-ext/src/main/java/org/apache/jackrabbit/aws/ext/ds/S3Backend.java (original)
+++ jackrabbit/trunk/jackrabbit-aws-ext/src/main/java/org/apache/jackrabbit/aws/ext/ds/S3Backend.java Fri Oct 17 07:30:57 2014
@@ -35,6 +35,8 @@ import java.util.concurrent.TimeUnit;
import org.apache.jackrabbit.aws.ext.S3Constants;
import org.apache.jackrabbit.aws.ext.Utils;
+import org.apache.jackrabbit.core.data.AsyncTouchCallback;
+import org.apache.jackrabbit.core.data.AsyncTouchResult;
import org.apache.jackrabbit.core.data.AsyncUploadCallback;
import org.apache.jackrabbit.core.data.AsyncUploadResult;
import org.apache.jackrabbit.core.data.Backend;
@@ -59,6 +61,7 @@ import com.amazonaws.services.s3.model.P
import com.amazonaws.services.s3.model.Region;
import com.amazonaws.services.s3.model.S3Object;
import com.amazonaws.services.s3.model.S3ObjectSummary;
+import com.amazonaws.services.s3.transfer.Copy;
import com.amazonaws.services.s3.transfer.TransferManager;
import com.amazonaws.services.s3.transfer.Upload;
@@ -160,7 +163,7 @@ public class S3Backend implements Backen
}
String propEndPoint = prop.getProperty(S3Constants.S3_END_POINT);
- if (propEndPoint != null & !"".equals(propEndPoint)) {
+ if ((propEndPoint != null) & !"".equals(propEndPoint)) {
endpoint = propEndPoint;
}
/*
@@ -290,7 +293,8 @@ public class S3Backend implements Backen
CopyObjectRequest copReq = new CopyObjectRequest(bucket,
key, bucket, key);
copReq.setNewObjectMetadata(objectMetaData);
- s3service.copyObject(copReq);
+ Copy copy = tmx.copy(copReq);
+ copy.waitForCopyResult();
LOG.debug("[{}] touched took [{}] ms. ", identifier,
(System.currentTimeMillis() - start));
}
@@ -319,6 +323,77 @@ public class S3Backend implements Backen
retVal, (System.currentTimeMillis() - start) });
return retVal;
}
+
+ @Override
+ public void touchAsync(final DataIdentifier identifier,
+ final long minModifiedDate, final AsyncTouchCallback callback)
+ throws DataStoreException {
+ ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
+ try {
+ if (callback == null) {
+ throw new IllegalArgumentException(
+ "callback parameter cannot be null in touchAsync");
+ }
+ Thread.currentThread().setContextClassLoader(
+ getClass().getClassLoader());
+
+ asyncWriteExecuter.execute(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ touch(identifier, minModifiedDate);
+ callback.onSuccess(new AsyncTouchResult(identifier));
+ } catch (DataStoreException e) {
+ AsyncTouchResult result = new AsyncTouchResult(
+ identifier);
+ result.setException(e);
+ callback.onFailure(result);
+ }
+ }
+ });
+ } catch (Exception e) {
+ callback.onAbort(new AsyncTouchResult(identifier));
+ throw new DataStoreException("Cannot touch the record "
+ + identifier.toString(), e);
+ } finally {
+ if (contextClassLoader != null) {
+ Thread.currentThread().setContextClassLoader(contextClassLoader);
+ }
+ }
+
+ }
+
+ @Override
+ public void touch(DataIdentifier identifier, long minModifiedDate)
+ throws DataStoreException {
+ ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
+ try {
+ final long start = System.currentTimeMillis();
+ final String key = getKeyName(identifier);
+ if (minModifiedDate > 0
+ && minModifiedDate > getLastModified(identifier)) {
+ CopyObjectRequest copReq = new CopyObjectRequest(bucket, key,
+ bucket, key);
+ copReq.setNewObjectMetadata(new ObjectMetadata());
+ Copy copy = tmx.copy(copReq);
+ copy.waitForCompletion();
+ LOG.debug("[{}] touched. time taken [{}] ms ", new Object[] {
+ identifier, (System.currentTimeMillis() - start) });
+ } else {
+ LOG.debug("[{}] touch not required. time taken [{}] ms ",
+ new Object[] { identifier,
+ (System.currentTimeMillis() - start) });
+ }
+
+ } catch (Exception e) {
+ throw new DataStoreException("Error occured in touching key ["
+ + identifier.toString() + "]", e);
+ } finally {
+ if (contextClassLoader != null) {
+ Thread.currentThread().setContextClassLoader(contextClassLoader);
+ }
+ }
+ }
@Override
public InputStream read(DataIdentifier identifier)
@@ -471,8 +546,16 @@ public class S3Backend implements Backen
getIdentifierName(s3ObjSumm.getKey()));
long lastModified = s3ObjSumm.getLastModified().getTime();
LOG.debug("Identifier [{}]'s lastModified = [{}]", identifier, lastModified);
- if (!store.isInUse(identifier) && lastModified < min) {
- LOG.debug("add id [{}] to delete lists", s3ObjSumm.getKey());
+ if (lastModified < min
+ && store.confirmDelete(identifier)
+ // confirm once more that record's lastModified < min
+ // order is important here
+ && s3service.getObjectMetadata(bucket,
+ s3ObjSumm.getKey()).getLastModified().getTime() < min) {
+
+
+ LOG.debug("add id [{}] to delete lists",
+ s3ObjSumm.getKey());
deleteList.add(new DeleteObjectsRequest.KeyVersion(
s3ObjSumm.getKey()));
deleteIdSet.add(identifier);
@@ -513,10 +596,10 @@ public class S3Backend implements Backen
@Override
public void close() {
// backend is closing. abort all mulitpart uploads from start.
+ asyncWriteExecuter.shutdownNow();
tmx.abortMultipartUploads(bucket, startTime);
tmx.shutdownNow();
s3service.shutdown();
- asyncWriteExecuter.shutdownNow();
LOG.info("S3Backend closed.");
}
@@ -567,10 +650,20 @@ public class S3Backend implements Backen
CopyObjectRequest copReq = new CopyObjectRequest(bucket, key,
bucket, key);
copReq.setNewObjectMetadata(objectMetaData);
- s3service.copyObject(copReq);
- LOG.debug("lastModified of [{}] updated successfully.", identifier);
- if (callback != null) {
- callback.onSuccess(new AsyncUploadResult(identifier, file));
+ Copy copy = tmx.copy(copReq);
+ try {
+ copy.waitForCopyResult();
+ LOG.debug("lastModified of [{}] updated successfully.", identifier);
+ if (callback != null) {
+ callback.onSuccess(new AsyncUploadResult(identifier, file));
+ }
+ }catch (Exception e2) {
+ AsyncUploadResult asyncUpRes= new AsyncUploadResult(identifier, file);
+ asyncUpRes.setException(e2);
+ if (callback != null) {
+ callback.onAbort(asyncUpRes);
+ }
+ throw new DataStoreException("Could not upload " + key, e2);
}
}
@@ -594,10 +687,12 @@ public class S3Backend implements Backen
identifier, file));
}
}
- } catch (Exception e2) {
- if (!asyncUpload) {
- callback.onAbort(new AsyncUploadResult(identifier, file));
- }
+ } catch (Exception e2 ) {
+ AsyncUploadResult asyncUpRes= new AsyncUploadResult(identifier, file);
+ asyncUpRes.setException(e2);
+ if (callback != null) {
+ callback.onAbort(asyncUpRes);
+ }
throw new DataStoreException("Could not upload " + key, e2);
}
}
@@ -721,6 +816,7 @@ public class S3Backend implements Backen
}
return key.substring(0, 4) + key.substring(5);
}
+
/**
* The class renames object key in S3 in a thread.
@@ -737,8 +833,15 @@ public class S3Backend implements Backen
String newS3Key = convertKey(oldKey);
CopyObjectRequest copReq = new CopyObjectRequest(bucket,
oldKey, bucket, newS3Key);
- s3service.copyObject(copReq);
- LOG.debug("[{}] renamed to [{}] ", oldKey, newS3Key);
+ Copy copy = tmx.copy(copReq);
+ try {
+ copy.waitForCopyResult();
+ LOG.debug("[{}] renamed to [{}] ", oldKey, newS3Key);
+ } catch (InterruptedException ie) {
+ LOG.error(" Exception in renaming [{}] to [{}] ",
+ new Object[] { ie, oldKey, newS3Key });
+ }
+
} finally {
if (contextClassLoader != null) {
Thread.currentThread().setContextClassLoader(
@@ -797,7 +900,7 @@ public class S3Backend implements Backen
}
}
}
-
+
/**
* This class implements {@link Runnable} interface to upload {@link File}
* to S3 asynchronously.
@@ -828,4 +931,6 @@ public class S3Backend implements Backen
}
}
+
+
}
Modified: jackrabbit/trunk/jackrabbit-aws-ext/src/test/java/org/apache/jackrabbit/aws/ext/TestAll.java
URL: http://svn.apache.org/viewvc/jackrabbit/trunk/jackrabbit-aws-ext/src/test/java/org/apache/jackrabbit/aws/ext/TestAll.java?rev=1632482&r1=1632481&r2=1632482&view=diff
==============================================================================
--- jackrabbit/trunk/jackrabbit-aws-ext/src/test/java/org/apache/jackrabbit/aws/ext/TestAll.java (original)
+++ jackrabbit/trunk/jackrabbit-aws-ext/src/test/java/org/apache/jackrabbit/aws/ext/TestAll.java Fri Oct 17 07:30:57 2014
@@ -22,7 +22,9 @@ import junit.framework.TestCase;
import junit.framework.TestSuite;
import org.apache.jackrabbit.aws.ext.ds.TestS3Ds;
+import org.apache.jackrabbit.aws.ext.ds.TestS3DSAsyncTouch;
import org.apache.jackrabbit.aws.ext.ds.TestS3DsCacheOff;
+import org.apache.jackrabbit.aws.ext.ds.TestS3DSWithSmallCache;
import org.apache.jackrabbit.core.data.TestCaseBase;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -36,7 +38,7 @@ public class TestAll extends TestCase {
/**
* <code>TestAll</code> suite that executes all tests inside this module. To
- * run test cases agains Amazon S3 pass AWS configuration properties file as
+ * run test cases against Amazon S3 pass AWS configuration properties file as
* system property -Dconfig=/opt/cq/aws.properties. Sample aws properties
* located at src/test/resources/aws.properties.
*/
@@ -46,6 +48,8 @@ public class TestAll extends TestCase {
LOG.info("config= " + config);
if (config != null && !"".equals(config.trim())) {
suite.addTestSuite(TestS3Ds.class);
+ suite.addTestSuite(TestS3DSAsyncTouch.class);
+ suite.addTestSuite(TestS3DSWithSmallCache.class);
suite.addTestSuite(TestS3DsCacheOff.class);
}
return suite;
Added: jackrabbit/trunk/jackrabbit-aws-ext/src/test/java/org/apache/jackrabbit/aws/ext/ds/TestS3DSAsyncTouch.java
URL: http://svn.apache.org/viewvc/jackrabbit/trunk/jackrabbit-aws-ext/src/test/java/org/apache/jackrabbit/aws/ext/ds/TestS3DSAsyncTouch.java?rev=1632482&view=auto
==============================================================================
--- jackrabbit/trunk/jackrabbit-aws-ext/src/test/java/org/apache/jackrabbit/aws/ext/ds/TestS3DSAsyncTouch.java (added)
+++ jackrabbit/trunk/jackrabbit-aws-ext/src/test/java/org/apache/jackrabbit/aws/ext/ds/TestS3DSAsyncTouch.java Fri Oct 17 07:30:57 2014
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.aws.ext.ds;
+
+import javax.jcr.RepositoryException;
+
+import org.apache.jackrabbit.core.data.CachingDataStore;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Test {@link CachingDataStore} with
+ * {@link CachingDataStore#setTouchAsync(boolean) set to true. It requires
+ * to pass aws config file via system property. For e.g.
+ * -Dconfig=/opt/cq/aws.properties. Sample aws properties located at
+ * src/test/resources/aws.properties
+ */
+public class TestS3DSAsyncTouch extends TestS3Ds {
+
+ public TestS3DSAsyncTouch() {
+ config = System.getProperty(CONFIG);
+ memoryBackend = false;
+ noCache = false;
+ }
+
+ protected CachingDataStore createDataStore() throws RepositoryException {
+ ds = new S3TestDataStore(String.valueOf(randomGen.nextInt(9999)) + "-test");
+ ds.setConfig(config);
+ ds.init(dataStoreDir);
+ ds.setTouchAsync(true);
+ ds.updateModifiedDateOnAccess(System.currentTimeMillis()+ 50* 1000);
+ return ds;
+ }
+}
Propchange: jackrabbit/trunk/jackrabbit-aws-ext/src/test/java/org/apache/jackrabbit/aws/ext/ds/TestS3DSAsyncTouch.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: jackrabbit/trunk/jackrabbit-aws-ext/src/test/java/org/apache/jackrabbit/aws/ext/ds/TestS3DSWithSmallCache.java
URL: http://svn.apache.org/viewvc/jackrabbit/trunk/jackrabbit-aws-ext/src/test/java/org/apache/jackrabbit/aws/ext/ds/TestS3DSWithSmallCache.java?rev=1632482&view=auto
==============================================================================
--- jackrabbit/trunk/jackrabbit-aws-ext/src/test/java/org/apache/jackrabbit/aws/ext/ds/TestS3DSWithSmallCache.java (added)
+++ jackrabbit/trunk/jackrabbit-aws-ext/src/test/java/org/apache/jackrabbit/aws/ext/ds/TestS3DSWithSmallCache.java Fri Oct 17 07:30:57 2014
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.aws.ext.ds;
+
+import javax.jcr.RepositoryException;
+
+import org.apache.jackrabbit.core.data.CachingDataStore;
+import org.apache.jackrabbit.core.data.LocalCache;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Test {@link CachingDataStore} with S3Backend and with very small size (@link
+ * {@link LocalCache}. It requires to pass aws config file via system property.
+ * For e.g. -Dconfig=/opt/cq/aws.properties. Sample aws properties located at
+ * src/test/resources/aws.properties
+ */
+public class TestS3DSWithSmallCache extends TestS3Ds {
+
+ public TestS3DSWithSmallCache() {
+ config = System.getProperty(CONFIG);
+ memoryBackend = false;
+ noCache = false;
+ }
+
+ protected CachingDataStore createDataStore() throws RepositoryException {
+ ds = new S3TestDataStore(String.valueOf(randomGen.nextInt(9999)) + "-test");
+ ds.setConfig(config);
+ ds.setCacheSize(dataLength * 10);
+ ds.setCachePurgeTrigFactor(0.5d);
+ ds.setCachePurgeResizeFactor(0.4d);
+ ds.init(dataStoreDir);
+ return ds;
+ }
+}
Propchange: jackrabbit/trunk/jackrabbit-aws-ext/src/test/java/org/apache/jackrabbit/aws/ext/ds/TestS3DSWithSmallCache.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: jackrabbit/trunk/jackrabbit-data/src/main/java/org/apache/jackrabbit/core/data/AsyncTouchCallback.java
URL: http://svn.apache.org/viewvc/jackrabbit/trunk/jackrabbit-data/src/main/java/org/apache/jackrabbit/core/data/AsyncTouchCallback.java?rev=1632482&view=auto
==============================================================================
--- jackrabbit/trunk/jackrabbit-data/src/main/java/org/apache/jackrabbit/core/data/AsyncTouchCallback.java (added)
+++ jackrabbit/trunk/jackrabbit-data/src/main/java/org/apache/jackrabbit/core/data/AsyncTouchCallback.java Fri Oct 17 07:30:57 2014
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.jackrabbit.core.data;
+/**
+ * This interface defines callback methods to reflect the status of asynchronous
+ * touch.
+ */
+public interface AsyncTouchCallback {
+
+
+ /**
+ * Callback method for successful asynchronous touch.
+ */
+ public void onSuccess(AsyncTouchResult result);
+
+ /**
+ * Callback method for failed asynchronous touch.
+ */
+ public void onFailure(AsyncTouchResult result);
+
+ /**
+ * Callback method for aborted asynchronous touch.
+ */
+ public void onAbort(AsyncTouchResult result);
+
+}
Propchange: jackrabbit/trunk/jackrabbit-data/src/main/java/org/apache/jackrabbit/core/data/AsyncTouchCallback.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: jackrabbit/trunk/jackrabbit-data/src/main/java/org/apache/jackrabbit/core/data/AsyncTouchResult.java
URL: http://svn.apache.org/viewvc/jackrabbit/trunk/jackrabbit-data/src/main/java/org/apache/jackrabbit/core/data/AsyncTouchResult.java?rev=1632482&view=auto
==============================================================================
--- jackrabbit/trunk/jackrabbit-data/src/main/java/org/apache/jackrabbit/core/data/AsyncTouchResult.java (added)
+++ jackrabbit/trunk/jackrabbit-data/src/main/java/org/apache/jackrabbit/core/data/AsyncTouchResult.java Fri Oct 17 07:30:57 2014
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.core.data;
+
+/**
+ *
+ * The class holds the result of asynchronous touch to {@link Backend}
+ */
+public class AsyncTouchResult {
+ /**
+ * {@link DataIdentifier} on which asynchronous touch is initiated.
+ */
+ private final DataIdentifier identifier;
+ /**
+ * Any {@link Exception} which is raised in asynchronously touch.
+ */
+ private Exception exception;
+
+ public AsyncTouchResult(DataIdentifier identifier) {
+ super();
+ this.identifier = identifier;
+ }
+
+ public DataIdentifier getIdentifier() {
+ return identifier;
+ }
+
+ public Exception getException() {
+ return exception;
+ }
+
+ public void setException(Exception exception) {
+ this.exception = exception;
+ }
+
+}
Propchange: jackrabbit/trunk/jackrabbit-data/src/main/java/org/apache/jackrabbit/core/data/AsyncTouchResult.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: jackrabbit/trunk/jackrabbit-data/src/main/java/org/apache/jackrabbit/core/data/Backend.java
URL: http://svn.apache.org/viewvc/jackrabbit/trunk/jackrabbit-data/src/main/java/org/apache/jackrabbit/core/data/Backend.java?rev=1632482&r1=1632481&r2=1632482&view=diff
==============================================================================
--- jackrabbit/trunk/jackrabbit-data/src/main/java/org/apache/jackrabbit/core/data/Backend.java (original)
+++ jackrabbit/trunk/jackrabbit-data/src/main/java/org/apache/jackrabbit/core/data/Backend.java Fri Oct 17 07:30:57 2014
@@ -89,10 +89,7 @@ public interface Backend {
void write(DataIdentifier identifier, File file) throws DataStoreException;
/**
- * Write file to backend in asynchronous mode. Backend implmentation may
- * choose not to write asynchronously but it requires to call
- * {@link AsyncUploadCallback#call(DataIdentifier, File, com.day.crx.cloud.s3.ds.AsyncUploadCallback.RESULT)}
- * after upload succeed or failed.
+ * Write file to backend in asynchronous mode.
*
* @param identifier
* @param file
@@ -131,6 +128,35 @@ public interface Backend {
* @throws DataStoreException
*/
boolean exists(DataIdentifier identifier) throws DataStoreException;
+
+ /**
+ * Update the lastModified of record if it's lastModified < minModifiedDate.
+ *
+ * @param identifier
+ * @param minModifiedDate
+ * @throws DataStoreException
+ */
+ void touch(final DataIdentifier identifier, long minModifiedDate)
+ throws DataStoreException;
+
+ /**
+ * Update the lastModified of record if it's lastModified < minModifiedDate
+ * asynchronously. Result of update is passed using appropriate
+ * {@link AsyncTouchCallback} methods. If identifier's lastModified >
+ * minModified {@link AsyncTouchCallback#onAbort(AsyncTouchResult)} is
+ * called. Any exception is communicated through
+ * {@link AsyncTouchCallback#onFailure(AsyncTouchResult)} . On successful
+ * update of lastModified,
+ * {@link AsyncTouchCallback#onSuccess(AsyncTouchResult)(AsyncTouchResult)}
+ * is invoked.
+ *
+ * @param identifier
+ * @param minModifiedDate
+ * @param callback
+ * @throws DataStoreException
+ */
+ void touchAsync(final DataIdentifier identifier, long minModifiedDate,
+ final AsyncTouchCallback callback) throws DataStoreException;
/**
* Close backend and release resources like database connection if any.
Modified: jackrabbit/trunk/jackrabbit-data/src/main/java/org/apache/jackrabbit/core/data/CachingDataStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/trunk/jackrabbit-data/src/main/java/org/apache/jackrabbit/core/data/CachingDataStore.java?rev=1632482&r1=1632481&r2=1632482&view=diff
==============================================================================
--- jackrabbit/trunk/jackrabbit-data/src/main/java/org/apache/jackrabbit/core/data/CachingDataStore.java (original)
+++ jackrabbit/trunk/jackrabbit-data/src/main/java/org/apache/jackrabbit/core/data/CachingDataStore.java Fri Oct 17 07:30:57 2014
@@ -75,10 +75,11 @@ import org.slf4j.LoggerFactory;
* <param name="{@link #setConcurrentUploadsThreads(int) concurrentUploadsThreads}" value="10"/>
* <param name="{@link #setAsyncUploadLimit(int) asyncUploadLimit}" value="100"/>
* <param name="{@link #setUploadRetries(int) uploadRetries}" value="3"/>
+ * <param name="{@link #setTouchAsync(boolean) touchAsync}" value="false"/>
* </DataStore>
*/
public abstract class CachingDataStore extends AbstractDataStore implements
- MultiDataStoreAware, AsyncUploadCallback {
+ MultiDataStoreAware, AsyncUploadCallback, AsyncTouchCallback {
/**
* Logger instance.
@@ -112,6 +113,12 @@ public abstract class CachingDataStore e
* is not required to be persisted.
*/
protected final Map<DataIdentifier, Integer> uploadRetryMap = new ConcurrentHashMap<DataIdentifier, Integer>(5);
+
+ /**
+ * In memory map to hold in-progress asynchronous touch. Once touch is
+ * successful corresponding entry is flushed from the map.
+ */
+ protected final Map<DataIdentifier, Long> asyncTouchCache = new ConcurrentHashMap<DataIdentifier, Long>(5);
protected Backend backend;
@@ -127,6 +134,11 @@ public abstract class CachingDataStore e
private File tmpDir;
private String secret;
+
+ /**
+ * Flag to indicate if lastModified is updated asynchronously.
+ */
+ private boolean touchAsync = false;
/**
* The optional backend configuration.
@@ -385,54 +397,53 @@ public abstract class CachingDataStore e
public DataRecord getRecord(DataIdentifier identifier)
throws DataStoreException {
String fileName = getFileName(identifier);
- boolean touch = minModifiedDate > 0 ? true : false;
- synchronized (this) {
- try {
- if (asyncWriteCache.hasEntry(fileName, touch)) {
- usesIdentifier(identifier);
- return new CachingDataRecord(this, identifier);
- } else if (cache.getFileIfStored(fileName) != null) {
- if (touch) {
- backend.exists(identifier, touch);
- }
- usesIdentifier(identifier);
- return new CachingDataRecord(this, identifier);
- } else if (backend.exists(identifier, touch)) {
- usesIdentifier(identifier);
- return new CachingDataRecord(this, identifier);
- }
-
- } catch (IOException ioe) {
- throw new DataStoreException("error in getting record ["
- + identifier + "]", ioe);
+ try {
+ if (asyncWriteCache.hasEntry(fileName, minModifiedDate > 0)) {
+ LOG.debug("[{}] record retrieved from asyncUploadmap",
+ identifier);
+ usesIdentifier(identifier);
+ return new CachingDataRecord(this, identifier);
+ } else if (cache.getFileIfStored(fileName) != null
+ || backend.exists(identifier)) {
+ LOG.debug("[{}] record retrieved from local cache or backend",
+ identifier);
+ touchInternal(identifier);
+ usesIdentifier(identifier);
+ return new CachingDataRecord(this, identifier);
}
+
+ } catch (IOException ioe) {
+ throw new DataStoreException("error in getting record ["
+ + identifier + "]", ioe);
}
throw new DataStoreException("Record not found: " + identifier);
}
-
+
/**
* Get a data record for the given identifier or null it data record doesn't
* exist in {@link Backend}
*
- * @param identifier
- * identifier of record.
+ * @param identifier identifier of record.
* @return the {@link CachingDataRecord} or null.
*/
@Override
public DataRecord getRecordIfStored(DataIdentifier identifier)
throws DataStoreException {
String fileName = getFileName(identifier);
- boolean touch = minModifiedDate > 0 ? true : false;
- synchronized (this) {
- try {
- if (asyncWriteCache.hasEntry(fileName, touch)
- || backend.exists(identifier, touch)) {
- usesIdentifier(identifier);
- return new CachingDataRecord(this, identifier);
- }
- } catch (IOException ioe) {
- throw new DataStoreException(ioe);
+ try {
+ if (asyncWriteCache.hasEntry(fileName, minModifiedDate > 0)) {
+ LOG.debug("[{}] record retrieved from asyncuploadmap",
+ identifier);
+ usesIdentifier(identifier);
+ return new CachingDataRecord(this, identifier);
+ } else if (backend.exists(identifier)) {
+ LOG.debug("[{}] record retrieved from backend", identifier);
+ touchInternal(identifier);
+ usesIdentifier(identifier);
+ return new CachingDataRecord(this, identifier);
}
+ } catch (IOException ioe) {
+ throw new DataStoreException(ioe);
}
return null;
}
@@ -534,14 +545,20 @@ public abstract class CachingDataStore e
long lastModified = asyncWriteCache.getLastModified(fileName);
if (lastModified != 0) {
LOG.debug(
- "identifier [{}]'s lastModified retrireved from AsyncUploadCache ",
- identifier);
+ "identifier [{}], lastModified=[{}] retrireved from AsyncUploadCache ",
+ identifier, lastModified);
+ } else if (asyncTouchCache.get(identifier) != null) {
+ lastModified = asyncTouchCache.get(identifier);
+ LOG.debug(
+ "identifier [{}], lastModified=[{}] retrireved from asyncTouchCache ",
+ identifier, lastModified);
} else {
- lastModified = backend.getLastModified(identifier);
+ lastModified = backend.getLastModified(identifier);
+ LOG.debug(
+ "identifier [{}], lastModified=[{}] retrireved from backend ",
+ identifier, lastModified);
}
- LOG.debug("identifier= [{}], lastModified=[{}]", identifier,
- lastModified);
return lastModified;
}
@@ -555,23 +572,8 @@ public abstract class CachingDataStore e
if (length != null) {
return length.longValue();
} else {
- InputStream in = null;
- InputStream cachedStream = null;
- try {
- in = backend.read(identifier);
- cachedStream = cache.store(fileName, in);
- } catch (IOException e) {
- throw new DataStoreException("IO Exception: " + identifier, e);
- } finally {
- IOUtils.closeQuietly(in);
- IOUtils.closeQuietly(cachedStream);
- }
- length = cache.getFileLength(fileName);
- if (length != null) {
- return length.longValue();
- }
+ return backend.getLength(identifier);
}
- return backend.getLength(identifier);
}
@Override
@@ -600,6 +602,10 @@ public abstract class CachingDataStore e
if (cachedResult.doRequiresDelete()) {
// added record already marked for delete
deleteRecord(identifier);
+ } else {
+ // async upload took lot of time.
+ // getRecord to touch if required.
+ getRecord(identifier);
}
} catch (IOException ie) {
LOG.warn("Cannot remove pending file upload. Dataidentifer [ "
@@ -662,6 +668,8 @@ public abstract class CachingDataStore e
File file = result.getFile();
String fileName = getFileName(identifier);
try {
+ // remove from failed upload map if any.
+ uploadRetryMap.remove(identifier);
asyncWriteCache.remove(fileName);
LOG.info(
"Async Upload Aborted. Dataidentifer [{}], file [{}] removed from AsyncCache.",
@@ -672,6 +680,89 @@ public abstract class CachingDataStore e
}
}
+
+ @Override
+ public void onSuccess(AsyncTouchResult result) {
+ asyncTouchCache.remove(result.getIdentifier());
+ LOG.debug(" Async Touch succeed. Removed [{}] from asyncTouchCache",
+ result.getIdentifier());
+
+ }
+
+ @Override
+ public void onFailure(AsyncTouchResult result) {
+ LOG.warn(" Async Touch failed. Not removing [{}] from asyncTouchCache",
+ result.getIdentifier());
+ if (result.getException() != null) {
+ LOG.debug(" Async Touch failed. exception", result.getException());
+ }
+ }
+
+ @Override
+ public void onAbort(AsyncTouchResult result) {
+ asyncTouchCache.remove(result.getIdentifier());
+ LOG.debug(" Async Touch aborted. Removed [{}] from asyncTouchCache",
+ result.getIdentifier());
+ }
+
+ /**
+ * Method to confirm that identifier can be deleted from {@link Backend}
+ *
+ * @param identifier
+ * @return
+ */
+ public boolean confirmDelete(DataIdentifier identifier) {
+ if (isInUse(identifier)) {
+ LOG.debug("identifier [{}] is inUse confirmDelete= false ",
+ identifier);
+ return false;
+ }
+
+ String fileName = getFileName(identifier);
+ long lastModified = asyncWriteCache.getLastModified(fileName);
+ if (lastModified != 0) {
+ LOG.debug(
+ "identifier [{}] is asyncWriteCache map confirmDelete= false ",
+ identifier);
+ return false;
+
+ }
+ if (asyncTouchCache.get(identifier) != null) {
+ LOG.debug(
+ "identifier [{}] is asyncTouchCache confirmDelete = false ",
+ identifier);
+ return false;
+ }
+
+ return true;
+ }
+
+ /**
+ * Internal method to touch identifier in @link {@link Backend}. if
+ * {@link #touchAsync}, the record is updated asynchronously.
+ *
+ * @param identifier
+ * @throws DataStoreException
+ */
+ private void touchInternal(DataIdentifier identifier)
+ throws DataStoreException {
+
+ if (touchAsync) {
+ Long lastModified = asyncTouchCache.put(identifier,
+ System.currentTimeMillis());
+
+ if (lastModified == null) {
+ LOG.debug("Async touching [{}] ", identifier);
+ backend.touchAsync(identifier, minModifiedDate, this);
+ } else {
+ LOG.debug( "Touched in asyncTouchMap [{}]", identifier);
+ }
+
+ } else {
+ backend.touch(identifier, minModifiedDate);
+ }
+ }
+
/**
* Returns a unique temporary file to be used for creating a new data
@@ -972,6 +1063,12 @@ public abstract class CachingDataStore e
public void setUploadRetries(int uploadRetries) {
this.uploadRetries = uploadRetries;
}
+
+
+
+ public void setTouchAsync(boolean touchAsync) {
+ this.touchAsync = touchAsync;
+ }
public Backend getBackend() {
return backend;
Modified: jackrabbit/trunk/jackrabbit-data/src/test/java/org/apache/jackrabbit/core/data/InMemoryBackend.java
URL: http://svn.apache.org/viewvc/jackrabbit/trunk/jackrabbit-data/src/test/java/org/apache/jackrabbit/core/data/InMemoryBackend.java?rev=1632482&r1=1632481&r2=1632482&view=diff
==============================================================================
--- jackrabbit/trunk/jackrabbit-data/src/test/java/org/apache/jackrabbit/core/data/InMemoryBackend.java (original)
+++ jackrabbit/trunk/jackrabbit-data/src/test/java/org/apache/jackrabbit/core/data/InMemoryBackend.java Fri Oct 17 07:30:57 2014
@@ -29,12 +29,6 @@ import java.util.Iterator;
import java.util.Map;
import java.util.Set;
-import org.apache.jackrabbit.core.data.AsyncUploadCallback;
-import org.apache.jackrabbit.core.data.Backend;
-import org.apache.jackrabbit.core.data.CachingDataStore;
-import org.apache.jackrabbit.core.data.DataIdentifier;
-import org.apache.jackrabbit.core.data.DataStoreException;
-
/**
* An in-memory backend implementation used to speed up testing.
*/
@@ -43,12 +37,15 @@ public class InMemoryBackend implements
private HashMap<DataIdentifier, byte[]> data = new HashMap<DataIdentifier, byte[]>();
private HashMap<DataIdentifier, Long> timeMap = new HashMap<DataIdentifier, Long>();
+
+ private CachingDataStore store;
@Override
public void init(CachingDataStore store, String homeDir, String config)
throws DataStoreException {
// ignore
log("init");
+ this.store = store;
}
@Override
@@ -110,7 +107,8 @@ public class InMemoryBackend implements
for (Map.Entry<DataIdentifier, Long> entry : timeMap.entrySet()) {
DataIdentifier identifier = entry.getKey();
long timestamp = entry.getValue();
- if (timestamp < min) {
+ if (timestamp < min && !store.isInUse(identifier)
+ && store.confirmDelete(identifier)) {
tobeDeleted.add(identifier);
}
}
@@ -140,6 +138,18 @@ public class InMemoryBackend implements
}
return retVal;
}
+
+ @Override
+ public void touch(DataIdentifier identifier, long minModifiedDate) {
+ timeMap.put(identifier, System.currentTimeMillis());
+ }
+
+ @Override
+ public void touchAsync(DataIdentifier identifier, long minModifiedDate,
+ AsyncTouchCallback callback) {
+ timeMap.put(identifier, System.currentTimeMillis());
+ callback.onSuccess(new AsyncTouchResult(identifier));
+ }
private void write(final DataIdentifier identifier, final File file,
final boolean async, final AsyncUploadCallback callback)
Modified: jackrabbit/trunk/jackrabbit-data/src/test/java/org/apache/jackrabbit/core/data/TestCaseBase.java
URL: http://svn.apache.org/viewvc/jackrabbit/trunk/jackrabbit-data/src/test/java/org/apache/jackrabbit/core/data/TestCaseBase.java?rev=1632482&r1=1632481&r2=1632482&view=diff
==============================================================================
--- jackrabbit/trunk/jackrabbit-data/src/test/java/org/apache/jackrabbit/core/data/TestCaseBase.java (original)
+++ jackrabbit/trunk/jackrabbit-data/src/test/java/org/apache/jackrabbit/core/data/TestCaseBase.java Fri Oct 17 07:30:57 2014
@@ -32,13 +32,6 @@ import javax.jcr.RepositoryException;
import junit.framework.TestCase;
import org.apache.commons.io.FileUtils;
-import org.apache.jackrabbit.core.data.CachingDataStore;
-import org.apache.jackrabbit.core.data.DataIdentifier;
-import org.apache.jackrabbit.core.data.DataRecord;
-import org.apache.jackrabbit.core.data.DataStore;
-import org.apache.jackrabbit.core.data.DataStoreException;
-import org.apache.jackrabbit.core.data.LocalCache;
-import org.apache.jackrabbit.core.data.MultiDataStoreAware;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -68,7 +61,7 @@ public abstract class TestCaseBase exten
protected String config;
/**
- * Parameter to use in-memory backend. If false {@link S3Backend}
+ * Parameter to use in-memory backend.
*/
protected boolean memoryBackend = true;
@@ -81,7 +74,7 @@ public abstract class TestCaseBase exten
/**
* length of record to be added
*/
- private int dataLength = 123456;
+ protected int dataLength = 123456;
/**
* datastore directory path
@@ -154,7 +147,7 @@ public abstract class TestCaseBase exten
LOG.error("error:", e);
}
}
-
+
/**
* Testcase to validate {@link DataStore#getAllIdentifiers()} API.
*/
@@ -253,10 +246,10 @@ public abstract class TestCaseBase exten
try {
long start = System.currentTimeMillis();
LOG.info("Testcase: " + this.getClass().getName()
- + "#test, testDir=" + dataStoreDir);
+ + "#testSingleThread, testDir=" + dataStoreDir);
doTestSingleThread();
LOG.info("Testcase: " + this.getClass().getName()
- + "#test finished, time taken = ["
+ + "#testSingleThread finished, time taken = ["
+ (System.currentTimeMillis() - start) + "]ms");
} catch (Exception e) {
LOG.error("error:", e);
@@ -446,7 +439,7 @@ public abstract class TestCaseBase exten
DataRecord rec2 = ds.addRecord(new ByteArrayInputStream(data));
// sleep for some time to ensure that async upload completes in backend.
- sleep(6000);
+ sleep(10000);
long updateTime = System.currentTimeMillis();
ds.updateModifiedDateOnAccess(updateTime);
@@ -574,7 +567,7 @@ public abstract class TestCaseBase exten
ArrayList<DataRecord> list = new ArrayList<DataRecord>();
HashMap<DataRecord, Integer> map = new HashMap<DataRecord, Integer>();
for (int i = 0; i < 10; i++) {
- int size = 1000000 - (i * 100);
+ int size = 100000 - (i * 100);
RandomInputStream in = new RandomInputStream(size + offset, size);
DataRecord rec = ds.addRecord(in);
list.add(rec);