You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by ch...@apache.org on 2014/04/16 06:56:36 UTC
svn commit: r1587800 - in /jackrabbit/oak/trunk/oak-core/src:
main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/
test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/
Author: chetanm
Date: Wed Apr 16 04:56:36 2014
New Revision: 1587800
URL: http://svn.apache.org/r1587800
Log:
OAK-1667 - Encode Blob length as part of blobId in DataStoreBlobStore
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/AbstractDataStoreService.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStoreTest.java
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/AbstractDataStoreService.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/AbstractDataStoreService.java?rev=1587800&r1=1587799&r2=1587800&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/AbstractDataStoreService.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/AbstractDataStoreService.java Wed Apr 16 04:56:36 2014
@@ -39,6 +39,8 @@ import org.slf4j.LoggerFactory;
public abstract class AbstractDataStoreService {
private static final String PROP_HOME = "repository.home";
+ public static final String PROP_ENCODE_LENGTH = "encodeLengthInId";
+
private ServiceRegistration reg;
private Logger log = LoggerFactory.getLogger(getClass());
@@ -47,14 +49,14 @@ public abstract class AbstractDataStoreS
protected void activate(ComponentContext context, Map<String, Object> config) throws RepositoryException {
DataStore ds = createDataStore(context, config);
-
+ boolean encodeLengthInId = PropertiesUtil.toBoolean(config.get(PROP_ENCODE_LENGTH), true);
String homeDir = lookup(context, PROP_HOME);
if (homeDir != null) {
log.info("Initializing the DataStore with homeDir [{}]", homeDir);
}
PropertiesUtil.populate(ds, config, false);
ds.init(homeDir);
- this.dataStore = new DataStoreBlobStore(ds);
+ this.dataStore = new DataStoreBlobStore(ds, encodeLengthInId);
Dictionary<String, String> props = new Hashtable<String, String>();
props.put(Constants.SERVICE_PID, ds.getClass().getName());
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java?rev=1587800&r1=1587799&r2=1587800&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java Wed Apr 16 04:56:36 2014
@@ -61,8 +61,25 @@ public class DataStoreBlobStore implemen
private final DataStore delegate;
+ /**
+ * If set to true then the blob length information would be encoded as part of blobId
+ * and thus no extra call would be made to DataStore to determine the length
+ *
+ * <b>Implementation Note</b>If enabled the length would be encoded in blobid by appending it at the end.
+ * This would be done for the methods which are part of BlobStore and GarbageCollectableBlobStore interface
+ *
+ * DataIdentifiers which are part of DataStore would not be affected by this as DataStore interface
+ * is not used in Oak and all access is via BlobStore interface
+ */
+ private final boolean encodeLengthInId;
+
public DataStoreBlobStore(DataStore delegate) {
+ this(delegate, true);
+ }
+
+ public DataStoreBlobStore(DataStore delegate, boolean encodeLengthInId) {
this.delegate = delegate;
+ this.encodeLengthInId = encodeLengthInId;
}
//~----------------------------------< DataStore >
@@ -134,7 +151,8 @@ public class DataStoreBlobStore implemen
boolean threw = true;
try {
checkNotNull(stream);
- String id = writeStream(stream).getIdentifier().toString();
+ DataRecord dr = writeStream(stream);
+ String id = getBlobId(dr);
threw = false;
return id;
} catch (DataStoreException e) {
@@ -147,11 +165,11 @@ public class DataStoreBlobStore implemen
}
@Override
- public int readBlob(String blobId, long pos, byte[] buff, int off, int length) throws IOException {
+ public int readBlob(String encodedBlobId, long pos, byte[] buff, int off, int length) throws IOException {
//This is inefficient as repeated calls for same blobId would involve opening new Stream
//instead clients should directly access the stream from DataRecord by special casing for
//BlobStore which implements DataStore
- InputStream stream = getStream(blobId);
+ InputStream stream = getInputStream(encodedBlobId);
boolean threw = true;
try {
ByteStreams.skipFully(stream, pos);
@@ -164,10 +182,14 @@ public class DataStoreBlobStore implemen
}
@Override
- public long getBlobLength(String blobId) throws IOException {
+ public long getBlobLength(String encodedBlobId) throws IOException {
try {
- checkNotNull(blobId, "BlobId must be specified");
- return getDataRecord(blobId).getLength();
+ checkNotNull(encodedBlobId, "BlobId must be specified");
+ BlobId id = BlobId.of(encodedBlobId);
+ if(encodeLengthInId && id.hasLengthInfo()){
+ return id.length;
+ }
+ return getDataRecord(id.blobId).getLength();
} catch (DataStoreException e) {
throw new IOException(e);
}
@@ -180,7 +202,7 @@ public class DataStoreBlobStore implemen
try {
record = delegate.getRecordFromReference(reference);
if (record != null) {
- return record.getIdentifier().toString();
+ return getBlobId(record);
}
} catch (DataStoreException e) {
log.warn("Unable to access the blobId for [{}]", reference, e);
@@ -189,8 +211,9 @@ public class DataStoreBlobStore implemen
}
@Override
- public String getReference(String blobId) {
- checkNotNull(blobId);
+ public String getReference(String encodedBlobId) {
+ checkNotNull(encodedBlobId);
+ String blobId = extractBlobId(encodedBlobId);
//Reference are not created for in memory record
if(InMemoryDataRecord.isInstance(blobId)){
return null;
@@ -211,8 +234,8 @@ public class DataStoreBlobStore implemen
}
@Override
- public InputStream getInputStream(String blobId) throws IOException {
- return getStream(blobId);
+ public InputStream getInputStream(String encodedBlobId) throws IOException {
+ return getStream(extractBlobId(encodedBlobId));
}
//~-------------------------------------------< GarbageCollectableBlobStore >
@@ -262,24 +285,33 @@ public class DataStoreBlobStore implemen
@Override
public Iterator<String> getAllChunkIds(final long maxLastModifiedTime) throws Exception {
- return transform(filter(delegate.getAllIdentifiers(), new Predicate<DataIdentifier>() {
+ return transform(filter(transform(delegate.getAllIdentifiers(), new Function<DataIdentifier, DataRecord>() {
+ @Nullable
@Override
- public boolean apply(DataIdentifier input) {
+ public DataRecord apply(@Nullable DataIdentifier input) {
try {
- DataRecord dr = delegate.getRecord(input);
- if(dr != null && (maxLastModifiedTime <=0
- || dr.getLastModified() < maxLastModifiedTime)){
- return true;
- }
+ return delegate.getRecord(input);
} catch (DataStoreException e) {
- log.warn("Error occurred while fetching DataRecord for identifier {}",input, e);
+ log.warn("Error occurred while fetching DataRecord for identifier {}", input, e);
+ }
+ return null;
+ }
+ }), new Predicate<DataRecord>() {
+ @Override
+ public boolean apply(@Nullable DataRecord input) {
+ if (input != null && (maxLastModifiedTime <= 0
+ || input.getLastModified() < maxLastModifiedTime)) {
+ return true;
}
return false;
}
- }),new Function<DataIdentifier, String>() {
+ }),new Function<DataRecord, String>() {
@Override
- public String apply(DataIdentifier input) {
- return input.toString();
+ public String apply(DataRecord input) {
+ if(encodeLengthInId) {
+ return BlobId.of(input).encodedValue();
+ }
+ return input.getIdentifier().toString();
}
});
}
@@ -288,7 +320,8 @@ public class DataStoreBlobStore implemen
public boolean deleteChunks(List<String> chunkIds, long maxLastModifiedTime) throws Exception {
if (delegate instanceof MultiDataStoreAware) {
for (String chunkId : chunkIds) {
- DataIdentifier identifier = new DataIdentifier(chunkId);
+ String blobId = extractBlobId(chunkId);
+ DataIdentifier identifier = new DataIdentifier(blobId);
DataRecord dataRecord = delegate.getRecord(identifier);
boolean success = (maxLastModifiedTime <= 0)
|| dataRecord.getLastModified() <= maxLastModifiedTime;
@@ -330,7 +363,7 @@ public class DataStoreBlobStore implemen
}else{
id = delegate.getRecord(new DataIdentifier(blobId));
}
- checkNotNull(id, "No DataRecord found for blodId [%s]", blobId);
+ checkNotNull(id, "No DataRecord found for blobId [%s]", blobId);
return id;
}
@@ -370,4 +403,76 @@ public class DataStoreBlobStore implemen
}
return record;
}
+
+ private String getBlobId(DataRecord dr){
+ if(encodeLengthInId){
+ return BlobId.of(dr).encodedValue();
+ }
+ return dr.getIdentifier().toString();
+ }
+
+ private String extractBlobId(String encodedBlobId){
+ if(encodeLengthInId){
+ return BlobId.of(encodedBlobId).blobId;
+ }
+ return encodedBlobId;
+ }
+
+ static class BlobId {
+ static final String SEP = "#";
+ final String blobId;
+ final long length;
+
+ BlobId(String blobId, long length) {
+ this.blobId = blobId;
+ this.length = length;
+ }
+
+ BlobId(DataRecord dr) {
+ this.blobId = dr.getIdentifier().toString();
+ long len;
+ try {
+ len = dr.getLength();
+ } catch (DataStoreException e) {
+ //Cannot determine length
+ len = -1;
+ }
+ this.length = len;
+ }
+
+ BlobId(String encodedBlobId) {
+ int indexOfSep = encodedBlobId.lastIndexOf(SEP);
+ if(indexOfSep != -1){
+ this.blobId = encodedBlobId.substring(0, indexOfSep);
+ this.length = Long.valueOf(encodedBlobId.substring(indexOfSep+SEP.length()));
+ }else{
+ this.blobId = encodedBlobId;
+ this.length = -1;
+ }
+ }
+
+ String encodedValue(){
+ if(hasLengthInfo()){
+ return blobId + SEP + String.valueOf(length);
+ } else{
+ return blobId;
+ }
+ }
+
+ boolean hasLengthInfo(){
+ return length != -1;
+ }
+
+ static boolean isEncoded(String encodedBlobId){
+ return encodedBlobId.contains(SEP);
+ }
+
+ static BlobId of(String encodedValue){
+ return new BlobId(encodedValue);
+ }
+
+ static BlobId of(DataRecord dr){
+ return new BlobId(dr);
+ }
+ }
}
Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStoreTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStoreTest.java?rev=1587800&r1=1587799&r2=1587800&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStoreTest.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStoreTest.java Wed Apr 16 04:56:36 2014
@@ -37,6 +37,7 @@ import org.apache.jackrabbit.core.data.D
import org.apache.jackrabbit.oak.spi.blob.BlobStoreInputStream;
import org.junit.Test;
+import static org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore.BlobId;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull;
@@ -69,7 +70,7 @@ public class DataStoreBlobStoreTest {
//Check for BlobStore methods
assertEquals(maxInlineSize, ds.getBlobLength(dr.getIdentifier().toString()));
- assertEquals(dr.getIdentifier().toString(), ds.writeBlob(new ByteArrayInputStream(data)));
+ assertEquals(dr.getIdentifier().toString(), BlobId.of(ds.writeBlob(new ByteArrayInputStream(data))).blobId);
}
@Test
@@ -102,7 +103,7 @@ public class DataStoreBlobStoreTest {
assertEquals(dr, ds.getRecord(dr.getIdentifier()));
assertEquals(actualSize, ds.getBlobLength(dr.getIdentifier().toString()));
- assertEquals(testDI.toString(), ds.writeBlob(new ByteArrayInputStream(data)));
+ assertEquals(testDI.toString(), BlobId.of(ds.writeBlob(new ByteArrayInputStream(data))).blobId);
}
@Test
@@ -118,7 +119,8 @@ public class DataStoreBlobStoreTest {
DataStoreBlobStore ds = new DataStoreBlobStore(mockedDS);
assertEquals(reference,ds.getReference(blobId));
- assertEquals(blobId, ds.getBlobId(reference));
+ assertEquals(blobId, BlobId.of(ds.getBlobId(reference)).blobId);
+ assertEquals(BlobId.of(testDR).encodedValue(),ds.getBlobId(reference));
String inMemBlobId = InMemoryDataRecord.getInstance("foo".getBytes())
.getIdentifier().toString();
@@ -147,6 +149,23 @@ public class DataStoreBlobStoreTest {
}
+ @Test
+ public void testEncodedBlobId() throws Exception{
+ BlobId blobId = new BlobId("abc"+BlobId.SEP+"123");
+ assertEquals("abc", blobId.blobId);
+ assertEquals(123, blobId.length);
+
+ blobId = new BlobId("abc"+BlobId.SEP+"abc"+BlobId.SEP+"123");
+ assertEquals("abc"+BlobId.SEP+"abc", blobId.blobId);
+ assertEquals(123, blobId.length);
+
+ blobId = new BlobId("abc",123);
+ assertEquals("abc"+BlobId.SEP+"123", blobId.encodedValue());
+
+ assertTrue(BlobId.isEncoded("abc"+BlobId.SEP+"123"));
+ assertFalse(BlobId.isEncoded("abc"));
+ }
+
private static class ByteArrayDataRecord implements DataRecord {
private final byte[] data;
private final DataIdentifier identifier;
@@ -203,7 +222,7 @@ public class DataStoreBlobStoreTest {
@Override
public long getLength() throws DataStoreException {
- throw new UnsupportedOperationException();
+ throw new DataStoreException(new UnsupportedOperationException());
}
@Override