You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by am...@apache.org on 2016/08/02 05:19:48 UTC
svn commit: r1754819 - in /jackrabbit/oak/branches/1.2/oak-core/src:
main/java/org/apache/jackrabbit/oak/plugins/blob/
main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/
test/java/org/apache/jackrabbit/oak/plugins/blob/
Author: amitj
Date: Tue Aug 2 05:19:48 2016
New Revision: 1754819
URL: http://svn.apache.org/viewvc?rev=1754819&view=rev
Log:
OAK-4565: S3Backend fails to upload large metadata records
Merged r1753331 from trunk
Modified:
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStore.java
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/OakFileDataStore.java
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/SharedS3DataStore.java
jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStoreUtilsTest.java
Modified: jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java?rev=1754819&r1=1754818&r2=1754819&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java (original)
+++ jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java Tue Aug 2 05:19:48 2016
@@ -805,13 +805,8 @@ public class MarkSweepGarbageCollector i
@Override
void addMarked(GarbageCollectableBlobStore blobStore, GarbageCollectorFileState fs,
String repoId) throws DataStoreException, IOException {
- InputStream is = new FileInputStream(fs.getMarkedRefs());
- try {
- ((SharedDataStore) blobStore)
- .addMetadataRecord(is, SharedStoreRecordType.REFERENCES.getNameFromId(repoId));
- } finally {
- Closeables.close(is, false);
- }
+ ((SharedDataStore) blobStore)
+ .addMetadataRecord(fs.getMarkedRefs(), SharedStoreRecordType.REFERENCES.getNameFromId(repoId));
}
@Override
Modified: jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStore.java?rev=1754819&r1=1754818&r2=1754819&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStore.java (original)
+++ jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStore.java Tue Aug 2 05:19:48 2016
@@ -16,6 +16,7 @@
*/
package org.apache.jackrabbit.oak.plugins.blob;
+import java.io.File;
import java.io.InputStream;
import java.util.Iterator;
import java.util.List;
@@ -46,6 +47,16 @@ public interface SharedDataStore {
throws DataStoreException;
/**
+ * Adds the root record.
+ *
+ * @param f the file
+ * @param name the name of the root record
+ * @throws DataStoreException the data store exception
+ */
+ void addMetadataRecord(File f, String name)
+ throws DataStoreException;
+
+ /**
* Retrieves the metadata record with the given name
*
* @param name the name of the record
Modified: jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java?rev=1754819&r1=1754818&r2=1754819&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java (original)
+++ jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java Tue Aug 2 05:19:48 2016
@@ -408,6 +408,13 @@ public class DataStoreBlobStore implemen
}
}
+ @Override
+ public void addMetadataRecord(File f, String name) throws DataStoreException {
+ if (delegate instanceof SharedDataStore) {
+ ((SharedDataStore) delegate).addMetadataRecord(f, name);
+ }
+ }
+
@Override public DataRecord getMetadataRecord(String name) {
if (delegate instanceof SharedDataStore) {
return ((SharedDataStore) delegate).getMetadataRecord(name);
Modified: jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/OakFileDataStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/OakFileDataStore.java?rev=1754819&r1=1754818&r2=1754819&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/OakFileDataStore.java (original)
+++ jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/OakFileDataStore.java Tue Aug 2 05:19:48 2016
@@ -38,6 +38,7 @@ import com.google.common.io.BaseEncoding
import com.google.common.io.Closeables;
import com.google.common.io.Files;
+import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.io.filefilter.FileFilterUtils;
import org.apache.jackrabbit.core.data.DataIdentifier;
@@ -150,11 +151,23 @@ public class OakFileDataStore extends Fi
Closeables.close(input, true);
}
} catch (IOException e) {
- LOG.error("Exception while adding root record with name {}, {}",
+ LOG.error("Exception while adding metadata record with name {}, {}",
new Object[] {name, e});
throw new DataStoreException("Could not add root record", e);
}
}
+
+ @Override
+ public void addMetadataRecord(File input, String name) throws DataStoreException {
+ try {
+ File file = new File(getPath(), name);
+ FileUtils.copyFile(input, file);
+ } catch (IOException e) {
+ LOG.error("Exception while adding metadata record file {} with name {}, {}",
+ new Object[] {input, name, e});
+ throw new DataStoreException("Could not add root record", e);
+ }
+ }
@Override
public DataRecord getMetadataRecord(String name) {
Modified: jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/SharedS3DataStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/SharedS3DataStore.java?rev=1754819&r1=1754818&r2=1754819&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/SharedS3DataStore.java (original)
+++ jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/SharedS3DataStore.java Tue Aug 2 05:19:48 2016
@@ -25,6 +25,7 @@ import org.apache.jackrabbit.oak.blob.cl
import org.apache.jackrabbit.oak.blob.cloud.aws.s3.S3DataStore;
import org.apache.jackrabbit.oak.plugins.blob.SharedDataStore;
+import java.io.File;
import java.io.InputStream;
import java.util.Iterator;
import java.util.List;
@@ -50,6 +51,11 @@ public class SharedS3DataStore extends S
}
@Override
+ public void addMetadataRecord(File f, String name) throws DataStoreException {
+ backend.addMetadataRecord(f, name);
+ }
+
+ @Override
public DataRecord getMetadataRecord(String name) {
return backend.getMetadataRecord(name);
}
Modified: jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStoreUtilsTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStoreUtilsTest.java?rev=1754819&r1=1754818&r2=1754819&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStoreUtilsTest.java (original)
+++ jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStoreUtilsTest.java Tue Aug 2 05:19:48 2016
@@ -26,6 +26,7 @@ import static org.hamcrest.CoreMatchers.
import java.io.ByteArrayInputStream;
import java.io.File;
+import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Date;
@@ -38,10 +39,12 @@ import javax.annotation.Nullable;
import com.google.common.base.Function;
import com.google.common.collect.Iterables;
+import com.google.common.collect.Sets;
import junit.framework.Assert;
import org.apache.commons.io.FileUtils;
import org.apache.jackrabbit.core.data.DataRecord;
+import org.apache.jackrabbit.oak.commons.FileIOUtils;
import org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore;
import org.apache.jackrabbit.oak.plugins.blob.datastore.SharedDataStoreUtils;
import org.apache.jackrabbit.oak.plugins.blob.datastore.SharedDataStoreUtils.SharedStoreRecordType;
@@ -49,7 +52,9 @@ import org.apache.jackrabbit.oak.plugins
import org.junit.After;
import org.junit.Assume;
import org.junit.BeforeClass;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -59,6 +64,9 @@ import org.slf4j.LoggerFactory;
public class SharedDataStoreUtilsTest {
private static final Logger log = LoggerFactory.getLogger(SharedDataStoreUtilsTest.class);
+ @Rule
+ public TemporaryFolder folder = new TemporaryFolder(new File("target"));
+
DataStoreBlobStore dataStore;
@BeforeClass
@@ -150,6 +158,32 @@ public class SharedDataStoreUtilsTest {
}
@Test
+ public void testAddMetadata() throws Exception {
+ dataStore = getBlobStore();
+ String repoId = UUID.randomUUID().toString();
+ Set<String> refs = Sets.newHashSet("1_1", "1_2");
+ File f = folder.newFile();
+ FileIOUtils.writeStrings(refs.iterator(), f, false);
+
+ dataStore.addMetadataRecord(new FileInputStream(f),
+ SharedStoreRecordType.REFERENCES.getNameFromId(repoId));
+ DataRecord rec = dataStore.getMetadataRecord(SharedStoreRecordType.REFERENCES.getNameFromId(repoId));
+ Set<String> refsReturned = FileIOUtils.readStringsAsSet(rec.getStream(), false);
+ Assert.assertEquals(refs, refsReturned);
+ dataStore.deleteAllMetadataRecords(SharedStoreRecordType.REFERENCES.getType());
+
+ dataStore.addMetadataRecord(f,
+ SharedStoreRecordType.REFERENCES.getNameFromId(repoId));
+ rec = dataStore.getMetadataRecord(SharedStoreRecordType.REFERENCES.getNameFromId(repoId));
+ refsReturned = FileIOUtils.readStringsAsSet(rec.getStream(), false);
+ Assert.assertEquals(refs, refsReturned);
+ assertEquals(
+ SharedStoreRecordType.REFERENCES.getIdFromName(rec.getIdentifier().toString()),
+ repoId);
+ dataStore.deleteAllMetadataRecords(SharedStoreRecordType.REFERENCES.getType());
+ }
+
+ @Test
public void testGetAllChunkIds() throws Exception {
dataStore = getBlobStore();
int number = 1010;