You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by am...@apache.org on 2016/07/19 03:48:23 UTC
svn commit: r1753331 - in /jackrabbit/oak/trunk:
oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/
oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/
oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/data...
Author: amitj
Date: Tue Jul 19 03:48:23 2016
New Revision: 1753331
URL: http://svn.apache.org/viewvc?rev=1753331&view=rev
Log:
OAK-4565: S3Backend fails to upload large metadata records
* Added an overloaded method to add metadata records
* MarkSweepGarbageCollector now uses this method to add records
Modified:
jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/S3Backend.java
jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/SharedS3Backend.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStore.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/OakFileDataStore.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/SharedS3DataStore.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStoreUtilsTest.java
Modified: jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/S3Backend.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/S3Backend.java?rev=1753331&r1=1753330&r2=1753331&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/S3Backend.java (original)
+++ jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/S3Backend.java Tue Jul 19 03:48:23 2016
@@ -629,6 +629,25 @@ public class S3Backend implements Shared
}
}
+ @Override
+ public void addMetadataRecord(File input, String name) throws DataStoreException {
+ ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
+ try {
+ Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
+
+ Upload upload = tmx.upload(s3ReqDecorator
+ .decorate(new PutObjectRequest(bucket, addMetaKeyPrefix(name), input)));
+ upload.waitForUploadResult();
+ } catch (InterruptedException e) {
+ LOG.error("Exception in uploading metadata file {}", new Object[] {input, e});
+ throw new DataStoreException("Error in uploading metadata file", e);
+ } finally {
+ if (contextClassLoader != null) {
+ Thread.currentThread().setContextClassLoader(contextClassLoader);
+ }
+ }
+ }
+
public DataRecord getMetadataRecord(String name) {
ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
try {
Modified: jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/SharedS3Backend.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/SharedS3Backend.java?rev=1753331&r1=1753330&r2=1753331&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/SharedS3Backend.java (original)
+++ jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/SharedS3Backend.java Tue Jul 19 03:48:23 2016
@@ -23,6 +23,7 @@ import org.apache.jackrabbit.core.data.B
import org.apache.jackrabbit.core.data.DataRecord;
import org.apache.jackrabbit.core.data.DataStoreException;
+import java.io.File;
import java.io.InputStream;
import java.util.Iterator;
import java.util.List;
@@ -42,6 +43,15 @@ public interface SharedS3Backend extends
void addMetadataRecord(final InputStream input, final String name) throws DataStoreException;
/**
+ * Adds a metadata record with the specified name
+ *
+ * @param input the record file
+ * @param name the name
+ * @throws org.apache.jackrabbit.core.data.DataStoreException
+ */
+ void addMetadataRecord(final File input, final String name) throws DataStoreException;
+
+ /**
* Gets the metadata of the specified name.
*
* @param name the name of the record
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java?rev=1753331&r1=1753330&r2=1753331&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java Tue Jul 19 03:48:23 2016
@@ -742,13 +742,8 @@ public class MarkSweepGarbageCollector i
@Override
void addMarked(GarbageCollectableBlobStore blobStore, GarbageCollectorFileState fs,
String repoId) throws DataStoreException, IOException {
- InputStream is = new FileInputStream(fs.getMarkedRefs());
- try {
- ((SharedDataStore) blobStore)
- .addMetadataRecord(is, SharedStoreRecordType.REFERENCES.getNameFromId(repoId));
- } finally {
- Closeables.close(is, false);
- }
+ ((SharedDataStore) blobStore)
+ .addMetadataRecord(fs.getMarkedRefs(), SharedStoreRecordType.REFERENCES.getNameFromId(repoId));
}
@Override
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStore.java?rev=1753331&r1=1753330&r2=1753331&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStore.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStore.java Tue Jul 19 03:48:23 2016
@@ -16,6 +16,7 @@
*/
package org.apache.jackrabbit.oak.plugins.blob;
+import java.io.File;
import java.io.InputStream;
import java.util.Iterator;
import java.util.List;
@@ -45,6 +46,16 @@ public interface SharedDataStore {
throws DataStoreException;
/**
+ * Adds the root record.
+ *
+ * @param f the file
+ * @param name the name of the root record
+ * @throws DataStoreException the data store exception
+ */
+ void addMetadataRecord(File f, String name)
+ throws DataStoreException;
+
+ /**
* Retrieves the metadata record with the given name
*
* @param name the name of the record
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java?rev=1753331&r1=1753330&r2=1753331&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java Tue Jul 19 03:48:23 2016
@@ -426,6 +426,13 @@ public class DataStoreBlobStore implemen
}
}
+ @Override
+ public void addMetadataRecord(File f, String name) throws DataStoreException {
+ if (delegate instanceof SharedDataStore) {
+ ((SharedDataStore) delegate).addMetadataRecord(f, name);
+ }
+ }
+
@Override public DataRecord getMetadataRecord(String name) {
if (delegate instanceof SharedDataStore) {
return ((SharedDataStore) delegate).getMetadataRecord(name);
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/OakFileDataStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/OakFileDataStore.java?rev=1753331&r1=1753330&r2=1753331&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/OakFileDataStore.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/OakFileDataStore.java Tue Jul 19 03:48:23 2016
@@ -38,6 +38,7 @@ import com.google.common.io.BaseEncoding
import com.google.common.io.Closeables;
import com.google.common.io.Files;
+import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.io.filefilter.FileFilterUtils;
import org.apache.jackrabbit.core.data.DataIdentifier;
@@ -150,11 +151,23 @@ public class OakFileDataStore extends Fi
Closeables.close(input, true);
}
} catch (IOException e) {
- LOG.error("Exception while adding root record with name {}, {}",
+ LOG.error("Exception while adding metadata record with name {}, {}",
new Object[] {name, e});
throw new DataStoreException("Could not add root record", e);
}
}
+
+ @Override
+ public void addMetadataRecord(File input, String name) throws DataStoreException {
+ try {
+ File file = new File(getPath(), name);
+ FileUtils.copyFile(input, file);
+ } catch (IOException e) {
+ LOG.error("Exception while adding metadata record file {} with name {}, {}",
+ new Object[] {input, name, e});
+ throw new DataStoreException("Could not add root record", e);
+ }
+ }
@Override
public DataRecord getMetadataRecord(String name) {
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/SharedS3DataStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/SharedS3DataStore.java?rev=1753331&r1=1753330&r2=1753331&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/SharedS3DataStore.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/SharedS3DataStore.java Tue Jul 19 03:48:23 2016
@@ -25,6 +25,7 @@ import org.apache.jackrabbit.oak.blob.cl
import org.apache.jackrabbit.oak.blob.cloud.aws.s3.S3DataStore;
import org.apache.jackrabbit.oak.plugins.blob.SharedDataStore;
+import java.io.File;
import java.io.InputStream;
import java.util.Iterator;
import java.util.List;
@@ -50,6 +51,11 @@ public class SharedS3DataStore extends S
}
@Override
+ public void addMetadataRecord(File f, String name) throws DataStoreException {
+ backend.addMetadataRecord(f, name);
+ }
+
+ @Override
public DataRecord getMetadataRecord(String name) {
return backend.getMetadataRecord(name);
}
Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStoreUtilsTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStoreUtilsTest.java?rev=1753331&r1=1753330&r2=1753331&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStoreUtilsTest.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStoreUtilsTest.java Tue Jul 19 03:48:23 2016
@@ -26,6 +26,7 @@ import static org.hamcrest.CoreMatchers.
import java.io.ByteArrayInputStream;
import java.io.File;
+import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Date;
@@ -38,10 +39,12 @@ import javax.annotation.Nullable;
import com.google.common.base.Function;
import com.google.common.collect.Iterables;
+import com.google.common.collect.Sets;
import junit.framework.Assert;
import org.apache.commons.io.FileUtils;
import org.apache.jackrabbit.core.data.DataRecord;
+import org.apache.jackrabbit.oak.commons.FileIOUtils;
import org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore;
import org.apache.jackrabbit.oak.plugins.blob.datastore.SharedDataStoreUtils;
import org.apache.jackrabbit.oak.plugins.blob.datastore.SharedDataStoreUtils.SharedStoreRecordType;
@@ -49,7 +52,9 @@ import org.apache.jackrabbit.oak.plugins
import org.junit.After;
import org.junit.Assume;
import org.junit.BeforeClass;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -59,6 +64,9 @@ import org.slf4j.LoggerFactory;
public class SharedDataStoreUtilsTest {
private static final Logger log = LoggerFactory.getLogger(SharedDataStoreUtilsTest.class);
+ @Rule
+ public TemporaryFolder folder = new TemporaryFolder(new File("target"));
+
DataStoreBlobStore dataStore;
@BeforeClass
@@ -150,6 +158,32 @@ public class SharedDataStoreUtilsTest {
}
@Test
+ public void testAddMetadata() throws Exception {
+ dataStore = getBlobStore();
+ String repoId = UUID.randomUUID().toString();
+ Set<String> refs = Sets.newHashSet("1_1", "1_2");
+ File f = folder.newFile();
+ FileIOUtils.writeStrings(refs.iterator(), f, false);
+
+ dataStore.addMetadataRecord(new FileInputStream(f),
+ SharedStoreRecordType.REFERENCES.getNameFromId(repoId));
+ DataRecord rec = dataStore.getMetadataRecord(SharedStoreRecordType.REFERENCES.getNameFromId(repoId));
+ Set<String> refsReturned = FileIOUtils.readStringsAsSet(rec.getStream(), false);
+ Assert.assertEquals(refs, refsReturned);
+ dataStore.deleteAllMetadataRecords(SharedStoreRecordType.REFERENCES.getType());
+
+ dataStore.addMetadataRecord(f,
+ SharedStoreRecordType.REFERENCES.getNameFromId(repoId));
+ rec = dataStore.getMetadataRecord(SharedStoreRecordType.REFERENCES.getNameFromId(repoId));
+ refsReturned = FileIOUtils.readStringsAsSet(rec.getStream(), false);
+ Assert.assertEquals(refs, refsReturned);
+ assertEquals(
+ SharedStoreRecordType.REFERENCES.getIdFromName(rec.getIdentifier().toString()),
+ repoId);
+ dataStore.deleteAllMetadataRecords(SharedStoreRecordType.REFERENCES.getType());
+ }
+
+ @Test
public void testGetAllChunkIds() throws Exception {
dataStore = getBlobStore();
int number = 1010;