You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by am...@apache.org on 2016/08/02 04:37:59 UTC
svn commit: r1754814 - in /jackrabbit/oak/branches/1.4: ./
oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/
oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/
oak-core/src/main/java/org/apache/jackrabbit/oak/plugins...
Author: amitj
Date: Tue Aug 2 04:37:59 2016
New Revision: 1754814
URL: http://svn.apache.org/viewvc?rev=1754814&view=rev
Log:
OAK-4565: S3Backend fails to upload large metadata records
Merged r1753331 from trunk
Modified:
jackrabbit/oak/branches/1.4/ (props changed)
jackrabbit/oak/branches/1.4/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/S3Backend.java
jackrabbit/oak/branches/1.4/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/SharedS3Backend.java
jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStore.java
jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java
jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/OakFileDataStore.java
jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/SharedS3DataStore.java
jackrabbit/oak/branches/1.4/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStoreUtilsTest.java
Propchange: jackrabbit/oak/branches/1.4/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Aug 2 04:37:59 2016
@@ -1,3 +1,3 @@
/jackrabbit/oak/branches/1.0:1665962
-/jackrabbit/oak/trunk:1733615,1733875,1733913,1733929,1734230,1734254,1734279,1734941,1735052,1735405,1735484,1735549,1735564,1735588,1735622,1735638,1735919,1735983,1736176,1737309-1737310,1737334,1737349,1737998,1738004,1738775,1738795,1738833,1738950,1738957,1738963,1739894,1740116,1740625-1740626,1740971,1741032,1741339,1741343,1742520,1742888,1742916,1743097,1743172,1743343,1744265,1744959,1745038,1745197,1745368,1746086,1746117,1746342,1746345,1746696,1746981,1747341-1747342,1747492,1747512,1748505,1748553,1748722,1748870,1749275,1749350,1749464,1749475,1749645,1749662,1749815,1749872,1749875,1749899,1750052,1750076-1750077,1750287,1750457,1750462,1750465,1750495,1750626,1750809,1750886,1751410,1751478,1751755,1751871,1752273-1752274,1752438,1752447,1752508,1752616,1752659,1752672,1753262,1753444,1754117,1754239
+/jackrabbit/oak/trunk:1733615,1733875,1733913,1733929,1734230,1734254,1734279,1734941,1735052,1735405,1735484,1735549,1735564,1735588,1735622,1735638,1735919,1735983,1736176,1737309-1737310,1737334,1737349,1737998,1738004,1738775,1738795,1738833,1738950,1738957,1738963,1739894,1740116,1740625-1740626,1740971,1741032,1741339,1741343,1742520,1742888,1742916,1743097,1743172,1743343,1744265,1744959,1745038,1745197,1745368,1746086,1746117,1746342,1746345,1746696,1746981,1747341-1747342,1747492,1747512,1748505,1748553,1748722,1748870,1749275,1749350,1749464,1749475,1749645,1749662,1749815,1749872,1749875,1749899,1750052,1750076-1750077,1750287,1750457,1750462,1750465,1750495,1750626,1750809,1750886,1751410,1751478,1751755,1751871,1752273-1752274,1752438,1752447,1752508,1752616,1752659,1752672,1753262,1753331,1753444,1754117,1754239
/jackrabbit/trunk:1345480
Modified: jackrabbit/oak/branches/1.4/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/S3Backend.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.4/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/S3Backend.java?rev=1754814&r1=1754813&r2=1754814&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.4/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/S3Backend.java (original)
+++ jackrabbit/oak/branches/1.4/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/S3Backend.java Tue Aug 2 04:37:59 2016
@@ -616,6 +616,25 @@ public class S3Backend implements Shared
}
}
+ @Override
+ public void addMetadataRecord(File input, String name) throws DataStoreException {
+ ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
+ try {
+ Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
+
+ Upload upload = tmx.upload(s3ReqDecorator
+ .decorate(new PutObjectRequest(bucket, addMetaKeyPrefix(name), input)));
+ upload.waitForUploadResult();
+ } catch (InterruptedException e) {
+ LOG.error("Exception in uploading metadata file {}", new Object[] {input, e});
+ throw new DataStoreException("Error in uploading metadata file", e);
+ } finally {
+ if (contextClassLoader != null) {
+ Thread.currentThread().setContextClassLoader(contextClassLoader);
+ }
+ }
+ }
+
public DataRecord getMetadataRecord(String name) {
ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
try {
Modified: jackrabbit/oak/branches/1.4/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/SharedS3Backend.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.4/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/SharedS3Backend.java?rev=1754814&r1=1754813&r2=1754814&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.4/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/SharedS3Backend.java (original)
+++ jackrabbit/oak/branches/1.4/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/SharedS3Backend.java Tue Aug 2 04:37:59 2016
@@ -23,6 +23,7 @@ import org.apache.jackrabbit.core.data.B
import org.apache.jackrabbit.core.data.DataRecord;
import org.apache.jackrabbit.core.data.DataStoreException;
+import java.io.File;
import java.io.InputStream;
import java.util.Iterator;
import java.util.List;
@@ -42,6 +43,15 @@ public interface SharedS3Backend extends
void addMetadataRecord(final InputStream input, final String name) throws DataStoreException;
/**
+ * Adds a metadata record with the specified name
+ *
+ * @param input the record file
+ * @param name the name
+ * @throws org.apache.jackrabbit.core.data.DataStoreException
+ */
+ void addMetadataRecord(final File input, final String name) throws DataStoreException;
+
+ /**
* Gets the metadata of the specified name.
*
* @param name the name of the record
Modified: jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java?rev=1754814&r1=1754813&r2=1754814&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java (original)
+++ jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java Tue Aug 2 04:37:59 2016
@@ -805,13 +805,8 @@ public class MarkSweepGarbageCollector i
@Override
void addMarked(GarbageCollectableBlobStore blobStore, GarbageCollectorFileState fs,
String repoId) throws DataStoreException, IOException {
- InputStream is = new FileInputStream(fs.getMarkedRefs());
- try {
- ((SharedDataStore) blobStore)
- .addMetadataRecord(is, SharedStoreRecordType.REFERENCES.getNameFromId(repoId));
- } finally {
- Closeables.close(is, false);
- }
+ ((SharedDataStore) blobStore)
+ .addMetadataRecord(fs.getMarkedRefs(), SharedStoreRecordType.REFERENCES.getNameFromId(repoId));
}
@Override
Modified: jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStore.java?rev=1754814&r1=1754813&r2=1754814&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStore.java (original)
+++ jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStore.java Tue Aug 2 04:37:59 2016
@@ -16,6 +16,7 @@
*/
package org.apache.jackrabbit.oak.plugins.blob;
+import java.io.File;
import java.io.InputStream;
import java.util.Iterator;
import java.util.List;
@@ -46,6 +47,16 @@ public interface SharedDataStore {
throws DataStoreException;
/**
+ * Adds the root record.
+ *
+ * @param f the file
+ * @param name the name of the root record
+ * @throws DataStoreException the data store exception
+ */
+ void addMetadataRecord(File f, String name)
+ throws DataStoreException;
+
+ /**
* Retrieves the metadata record with the given name
*
* @param name the name of the record
Modified: jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java?rev=1754814&r1=1754813&r2=1754814&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java (original)
+++ jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java Tue Aug 2 04:37:59 2016
@@ -426,6 +426,13 @@ public class DataStoreBlobStore implemen
}
}
+ @Override
+ public void addMetadataRecord(File f, String name) throws DataStoreException {
+ if (delegate instanceof SharedDataStore) {
+ ((SharedDataStore) delegate).addMetadataRecord(f, name);
+ }
+ }
+
@Override public DataRecord getMetadataRecord(String name) {
if (delegate instanceof SharedDataStore) {
return ((SharedDataStore) delegate).getMetadataRecord(name);
Modified: jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/OakFileDataStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/OakFileDataStore.java?rev=1754814&r1=1754813&r2=1754814&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/OakFileDataStore.java (original)
+++ jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/OakFileDataStore.java Tue Aug 2 04:37:59 2016
@@ -38,6 +38,7 @@ import com.google.common.io.BaseEncoding
import com.google.common.io.Closeables;
import com.google.common.io.Files;
+import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.io.filefilter.FileFilterUtils;
import org.apache.jackrabbit.core.data.DataIdentifier;
@@ -150,11 +151,23 @@ public class OakFileDataStore extends Fi
Closeables.close(input, true);
}
} catch (IOException e) {
- LOG.error("Exception while adding root record with name {}, {}",
+ LOG.error("Exception while adding metadata record with name {}, {}",
new Object[] {name, e});
throw new DataStoreException("Could not add root record", e);
}
}
+
+ @Override
+ public void addMetadataRecord(File input, String name) throws DataStoreException {
+ try {
+ File file = new File(getPath(), name);
+ FileUtils.copyFile(input, file);
+ } catch (IOException e) {
+ LOG.error("Exception while adding metadata record file {} with name {}, {}",
+ new Object[] {input, name, e});
+ throw new DataStoreException("Could not add root record", e);
+ }
+ }
@Override
public DataRecord getMetadataRecord(String name) {
Modified: jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/SharedS3DataStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/SharedS3DataStore.java?rev=1754814&r1=1754813&r2=1754814&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/SharedS3DataStore.java (original)
+++ jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/SharedS3DataStore.java Tue Aug 2 04:37:59 2016
@@ -25,6 +25,7 @@ import org.apache.jackrabbit.oak.blob.cl
import org.apache.jackrabbit.oak.blob.cloud.aws.s3.S3DataStore;
import org.apache.jackrabbit.oak.plugins.blob.SharedDataStore;
+import java.io.File;
import java.io.InputStream;
import java.util.Iterator;
import java.util.List;
@@ -50,6 +51,11 @@ public class SharedS3DataStore extends S
}
@Override
+ public void addMetadataRecord(File f, String name) throws DataStoreException {
+ backend.addMetadataRecord(f, name);
+ }
+
+ @Override
public DataRecord getMetadataRecord(String name) {
return backend.getMetadataRecord(name);
}
Modified: jackrabbit/oak/branches/1.4/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStoreUtilsTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.4/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStoreUtilsTest.java?rev=1754814&r1=1754813&r2=1754814&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.4/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStoreUtilsTest.java (original)
+++ jackrabbit/oak/branches/1.4/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStoreUtilsTest.java Tue Aug 2 04:37:59 2016
@@ -26,6 +26,7 @@ import static org.hamcrest.CoreMatchers.
import java.io.ByteArrayInputStream;
import java.io.File;
+import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Date;
@@ -38,10 +39,12 @@ import javax.annotation.Nullable;
import com.google.common.base.Function;
import com.google.common.collect.Iterables;
+import com.google.common.collect.Sets;
import junit.framework.Assert;
import org.apache.commons.io.FileUtils;
import org.apache.jackrabbit.core.data.DataRecord;
+import org.apache.jackrabbit.oak.commons.FileIOUtils;
import org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore;
import org.apache.jackrabbit.oak.plugins.blob.datastore.SharedDataStoreUtils;
import org.apache.jackrabbit.oak.plugins.blob.datastore.SharedDataStoreUtils.SharedStoreRecordType;
@@ -49,7 +52,9 @@ import org.apache.jackrabbit.oak.plugins
import org.junit.After;
import org.junit.Assume;
import org.junit.BeforeClass;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -59,6 +64,9 @@ import org.slf4j.LoggerFactory;
public class SharedDataStoreUtilsTest {
private static final Logger log = LoggerFactory.getLogger(SharedDataStoreUtilsTest.class);
+ @Rule
+ public TemporaryFolder folder = new TemporaryFolder(new File("target"));
+
DataStoreBlobStore dataStore;
@BeforeClass
@@ -150,6 +158,32 @@ public class SharedDataStoreUtilsTest {
}
@Test
+ public void testAddMetadata() throws Exception {
+ dataStore = getBlobStore();
+ String repoId = UUID.randomUUID().toString();
+ Set<String> refs = Sets.newHashSet("1_1", "1_2");
+ File f = folder.newFile();
+ FileIOUtils.writeStrings(refs.iterator(), f, false);
+
+ dataStore.addMetadataRecord(new FileInputStream(f),
+ SharedStoreRecordType.REFERENCES.getNameFromId(repoId));
+ DataRecord rec = dataStore.getMetadataRecord(SharedStoreRecordType.REFERENCES.getNameFromId(repoId));
+ Set<String> refsReturned = FileIOUtils.readStringsAsSet(rec.getStream(), false);
+ Assert.assertEquals(refs, refsReturned);
+ dataStore.deleteAllMetadataRecords(SharedStoreRecordType.REFERENCES.getType());
+
+ dataStore.addMetadataRecord(f,
+ SharedStoreRecordType.REFERENCES.getNameFromId(repoId));
+ rec = dataStore.getMetadataRecord(SharedStoreRecordType.REFERENCES.getNameFromId(repoId));
+ refsReturned = FileIOUtils.readStringsAsSet(rec.getStream(), false);
+ Assert.assertEquals(refs, refsReturned);
+ assertEquals(
+ SharedStoreRecordType.REFERENCES.getIdFromName(rec.getIdentifier().toString()),
+ repoId);
+ dataStore.deleteAllMetadataRecords(SharedStoreRecordType.REFERENCES.getType());
+ }
+
+ @Test
public void testGetAllChunkIds() throws Exception {
dataStore = getBlobStore();
int number = 1010;