You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by th...@apache.org on 2012/08/02 17:17:26 UTC
svn commit: r1368520 - in /jackrabbit/oak/trunk/oak-mk/src:
main/java/org/apache/jackrabbit/mk/blobs/
test/java/org/apache/jackrabbit/mk/blobs/
Author: thomasm
Date: Thu Aug 2 15:17:26 2012
New Revision: 1368520
URL: http://svn.apache.org/viewvc?rev=1368520&view=rev
Log:
OAK-209 BlobStore: use SHA-256 instead of SHA-1, and use two directory levels for FileBlobStore
Modified:
jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/blobs/AbstractBlobStore.java
jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/blobs/BlobStore.java
jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/blobs/FileBlobStore.java
jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/blobs/MemoryBlobStore.java
jackrabbit/oak/trunk/oak-mk/src/test/java/org/apache/jackrabbit/mk/blobs/AbstractBlobStoreTest.java
jackrabbit/oak/trunk/oak-mk/src/test/java/org/apache/jackrabbit/mk/blobs/DbBlobStoreTest.java
jackrabbit/oak/trunk/oak-mk/src/test/java/org/apache/jackrabbit/mk/blobs/FileBlobStoreTest.java
jackrabbit/oak/trunk/oak-mk/src/test/java/org/apache/jackrabbit/mk/blobs/MemoryBlobStoreTest.java
Modified: jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/blobs/AbstractBlobStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/blobs/AbstractBlobStore.java?rev=1368520&r1=1368519&r2=1368520&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/blobs/AbstractBlobStore.java (original)
+++ jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/blobs/AbstractBlobStore.java Thu Aug 2 15:17:26 2012
@@ -65,11 +65,13 @@ import java.util.WeakHashMap;
*/
public abstract class AbstractBlobStore implements Closeable, BlobStore, Cache.Backend<AbstractBlobStore.BlockId, AbstractBlobStore.Data> {
- protected static final String HASH_ALGORITHM = "SHA-1";
+ protected static final String HASH_ALGORITHM = "SHA-256";
protected static final int TYPE_DATA = 0;
protected static final int TYPE_HASH = 1;
protected static final int TYPE_HASH_COMPRESSED = 2;
+
+ protected static final int BLOCK_SIZE_LIMIT = 48;
protected Map<String, WeakReference<String>> inUse =
Collections.synchronizedMap(new WeakHashMap<String, WeakReference<String>>());
@@ -78,7 +80,7 @@ public abstract class AbstractBlobStore
* The minimum size of a block. Smaller blocks are inlined (the data store id
* is the data itself).
*/
- private int blockSizeMin = 256;
+ private int blockSizeMin = 4096;
/**
* The size of a block. 128 KB has been found to be as fast as larger
@@ -89,6 +91,7 @@ public abstract class AbstractBlobStore
private Cache<AbstractBlobStore.BlockId, Data> cache = Cache.newInstance(this, 8 * 1024 * 1024);
public void setBlockSizeMin(int x) {
+ validateBlockSize(x);
this.blockSizeMin = x;
}
@@ -97,14 +100,23 @@ public abstract class AbstractBlobStore
}
public void setBlockSize(int x) {
+ validateBlockSize(x);
this.blockSize = x;
}
+
+ private static void validateBlockSize(int x) {
+ if (x < BLOCK_SIZE_LIMIT) {
+ throw new IllegalArgumentException(
+ "The minimum size must be bigger " +
+ "than a content hash itself; limit = " + BLOCK_SIZE_LIMIT);
+ }
+ }
public int getBlockSize() {
return blockSize;
}
- public String addBlob(String tempFilePath) throws Exception {
+ public String writeBlob(String tempFilePath) throws Exception {
File file = new File(tempFilePath);
InputStream in = null;
try {
@@ -273,13 +285,25 @@ public abstract class AbstractBlobStore
}
public Data load(BlockId id) {
+ byte[] data;
try {
- return new Data(readBlockFromBackend(id));
+ data = readBlockFromBackend(id);
} catch (Exception e) {
- throw new RuntimeException("failed to read block from backend", e);
+ throw new RuntimeException("failed to read block from backend, id " + id, e);
+ }
+ if (data == null) {
+ throw new IllegalArgumentException("The block with id " + id + " was not found");
}
+ return new Data(data);
}
+ /**
+ * Load the block from the storage backend. Returns null if the block was
+ * not found.
+ *
+ * @param id the block id
+ * @return the block data, or null
+ */
protected abstract byte[] readBlockFromBackend(BlockId id) throws Exception;
public long getBlobLength(String blobId) throws IOException {
Modified: jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/blobs/BlobStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/blobs/BlobStore.java?rev=1368520&r1=1368519&r2=1368520&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/blobs/BlobStore.java (original)
+++ jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/blobs/BlobStore.java Thu Aug 2 15:17:26 2012
@@ -31,7 +31,7 @@ public interface BlobStore {
* @param tempFilePath the temporary file
* @return the blob id
*/
- String addBlob(String tempFilePath) throws Exception;
+ String writeBlob(String tempFilePath) throws Exception;
/**
* Write a blob from an input stream.
@@ -42,10 +42,31 @@ public interface BlobStore {
*/
String writeBlob(InputStream in) throws Exception;
+ /**
+ * Read a number of bytes from a blob.
+ *
+ * @param blobId the blob id
+ * @param pos the position within the blob
+ * @param buff the target byte array
+ * @param off the offset within the target array
+ * @param length the number of bytes to read
+ * @return the number of bytes read
+ */
int readBlob(String blobId, long pos, byte[] buff, int off, int length) throws Exception;
+ /**
+ * Get the length of the blob.
+ *
+ * @param blobId the blob id
+ * @return the length
+ */
long getBlobLength(String blobId) throws Exception;
+ /**
+ * Close all internally used resources, such as file handles. This method
+ * should be called at the end of the components lifecycle. After calling
+ * this method, the blob store should no longer be used.
+ */
void close();
}
Modified: jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/blobs/FileBlobStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/blobs/FileBlobStore.java?rev=1368520&r1=1368519&r2=1368520&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/blobs/FileBlobStore.java (original)
+++ jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/blobs/FileBlobStore.java Thu Aug 2 15:17:26 2012
@@ -48,7 +48,7 @@ public class FileBlobStore extends Abstr
}
@Override
- public String addBlob(String tempFilePath) throws Exception {
+ public String writeBlob(String tempFilePath) throws Exception {
File file = new File(tempFilePath);
InputStream in = new FileInputStream(file);
MessageDigest messageDigest = MessageDigest.getInstance(HASH_ALGORITHM);
@@ -106,11 +106,12 @@ public class FileBlobStore extends Abstr
private File getFile(byte[] digest, boolean old) {
String id = StringUtils.convertBytesToHex(digest);
- String sub = id.substring(id.length() - 2);
+ String sub1 = id.substring(id.length() - 2);
+ String sub2 = id.substring(id.length() - 4, id.length() - 2);
if (old) {
- sub += OLD_SUFFIX;
+ sub2 += OLD_SUFFIX;
}
- return new File(new File(baseDir, sub), id + ".dat");
+ return new File(new File(new File(baseDir, sub1), sub2), id + ".dat");
}
@Override
@@ -137,19 +138,23 @@ public class FileBlobStore extends Abstr
@Override
public void startMark() throws Exception {
mark = true;
- for (int i = 0; i < 256; i++) {
- String sub = StringUtils.convertBytesToHex(new byte[] { (byte) i });
- File d = new File(baseDir, sub);
- File old = new File(baseDir, sub + OLD_SUFFIX);
- if (d.exists()) {
- if (old.exists()) {
- for (File p : d.listFiles()) {
- String name = p.getName();
- File newName = new File(old, name);
- p.renameTo(newName);
+ for (int j = 0; j < 256; j++) {
+ String sub1 = StringUtils.convertBytesToHex(new byte[] { (byte) j });
+ File x = new File(baseDir, sub1);
+ for (int i = 0; i < 256; i++) {
+ String sub2 = StringUtils.convertBytesToHex(new byte[] { (byte) i });
+ File d = new File(x, sub2);
+ File old = new File(x, sub2 + OLD_SUFFIX);
+ if (d.exists()) {
+ if (old.exists()) {
+ for (File p : d.listFiles()) {
+ String name = p.getName();
+ File newName = new File(old, name);
+ p.renameTo(newName);
+ }
+ } else {
+ d.renameTo(old);
}
- } else {
- d.renameTo(old);
}
}
}
@@ -175,17 +180,21 @@ public class FileBlobStore extends Abstr
@Override
public int sweep() throws IOException {
int count = 0;
- for (int i = 0; i < 256; i++) {
- String sub = StringUtils.convertBytesToHex(new byte[] { (byte) i });
- File old = new File(baseDir, sub + OLD_SUFFIX);
- if (old.exists()) {
- for (File p : old.listFiles()) {
- String name = p.getName();
- File file = new File(old, name);
- file.delete();
- count++;
+ for (int j = 0; j < 256; j++) {
+ String sub1 = StringUtils.convertBytesToHex(new byte[] { (byte) j });
+ File x = new File(baseDir, sub1);
+ for (int i = 0; i < 256; i++) {
+ String sub = StringUtils.convertBytesToHex(new byte[] { (byte) i });
+ File old = new File(x, sub + OLD_SUFFIX);
+ if (old.exists()) {
+ for (File p : old.listFiles()) {
+ String name = p.getName();
+ File file = new File(old, name);
+ file.delete();
+ count++;
+ }
+ old.delete();
}
- old.delete();
}
}
mark = false;
Modified: jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/blobs/MemoryBlobStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/blobs/MemoryBlobStore.java?rev=1368520&r1=1368519&r2=1368520&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/blobs/MemoryBlobStore.java (original)
+++ jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/blobs/MemoryBlobStore.java Thu Aug 2 15:17:26 2012
@@ -29,7 +29,11 @@ public class MemoryBlobStore extends Abs
@Override
protected byte[] readBlockFromBackend(BlockId id) {
- return map.get(id);
+ byte[] result = map.get(id);
+ if (result == null) {
+ result = old.get(id);
+ }
+ return result;
}
@Override
Modified: jackrabbit/oak/trunk/oak-mk/src/test/java/org/apache/jackrabbit/mk/blobs/AbstractBlobStoreTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mk/src/test/java/org/apache/jackrabbit/mk/blobs/AbstractBlobStoreTest.java?rev=1368520&r1=1368519&r2=1368520&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-mk/src/test/java/org/apache/jackrabbit/mk/blobs/AbstractBlobStoreTest.java (original)
+++ jackrabbit/oak/trunk/oak-mk/src/test/java/org/apache/jackrabbit/mk/blobs/AbstractBlobStoreTest.java Thu Aug 2 15:17:26 2012
@@ -61,7 +61,7 @@ public abstract class AbstractBlobStoreT
OutputStream out = new FileOutputStream(tempFile, false);
out.write(data);
out.close();
- String s = store.addBlob(tempFileName);
+ String s = store.writeBlob(tempFileName);
assertEquals(data.length, store.getBlobLength(s));
byte[] buff = new byte[1];
for (int i = 0; i < data.length; i += 1024) {
@@ -69,7 +69,7 @@ public abstract class AbstractBlobStoreT
assertEquals(data[i], buff[0]);
}
try {
- store.addBlob(tempFileName + "_wrong");
+ store.writeBlob(tempFileName + "_wrong");
fail();
} catch (Exception e) {
// expected
@@ -162,7 +162,7 @@ public abstract class AbstractBlobStoreT
HashMap<String, byte[]> map = new HashMap<String, byte[]>();
ArrayList<String> mem = new ArrayList<String>();
int count;
- for (int i = 1; i < 10000; i += (i + 1) * 10) {
+ for (int i = 1; i <= 1000; i *= 10) {
byte[] data = new byte[i];
String id;
id = store.writeBlob(new ByteArrayInputStream(data));
@@ -194,7 +194,7 @@ public abstract class AbstractBlobStoreT
}
store.mark(id);
}
- store.sweep();
+ count = store.sweep();
store.clearInUse();
store.clearCache();
Modified: jackrabbit/oak/trunk/oak-mk/src/test/java/org/apache/jackrabbit/mk/blobs/DbBlobStoreTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mk/src/test/java/org/apache/jackrabbit/mk/blobs/DbBlobStoreTest.java?rev=1368520&r1=1368519&r2=1368520&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-mk/src/test/java/org/apache/jackrabbit/mk/blobs/DbBlobStoreTest.java (original)
+++ jackrabbit/oak/trunk/oak-mk/src/test/java/org/apache/jackrabbit/mk/blobs/DbBlobStoreTest.java Thu Aug 2 15:17:26 2012
@@ -35,7 +35,7 @@ public class DbBlobStoreTest extends Abs
DbBlobStore blobStore = new DbBlobStore();
blobStore.setConnectionPool(cp);
blobStore.setBlockSize(128);
- blobStore.setBlockSizeMin(32);
+ blobStore.setBlockSizeMin(48);
this.store = blobStore;
}
Modified: jackrabbit/oak/trunk/oak-mk/src/test/java/org/apache/jackrabbit/mk/blobs/FileBlobStoreTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mk/src/test/java/org/apache/jackrabbit/mk/blobs/FileBlobStoreTest.java?rev=1368520&r1=1368519&r2=1368520&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-mk/src/test/java/org/apache/jackrabbit/mk/blobs/FileBlobStoreTest.java (original)
+++ jackrabbit/oak/trunk/oak-mk/src/test/java/org/apache/jackrabbit/mk/blobs/FileBlobStoreTest.java Thu Aug 2 15:17:26 2012
@@ -24,7 +24,7 @@ public class FileBlobStoreTest extends A
public void setUp() throws Exception {
FileBlobStore store = new FileBlobStore("target/temp");
store.setBlockSize(128);
- store.setBlockSizeMin(32);
+ store.setBlockSizeMin(48);
this.store = store;
}
Modified: jackrabbit/oak/trunk/oak-mk/src/test/java/org/apache/jackrabbit/mk/blobs/MemoryBlobStoreTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mk/src/test/java/org/apache/jackrabbit/mk/blobs/MemoryBlobStoreTest.java?rev=1368520&r1=1368519&r2=1368520&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-mk/src/test/java/org/apache/jackrabbit/mk/blobs/MemoryBlobStoreTest.java (original)
+++ jackrabbit/oak/trunk/oak-mk/src/test/java/org/apache/jackrabbit/mk/blobs/MemoryBlobStoreTest.java Thu Aug 2 15:17:26 2012
@@ -23,6 +23,8 @@ public class MemoryBlobStoreTest extends
public void setUp() throws Exception {
store = new MemoryBlobStore();
- }
+ store.setBlockSize(128);
+ store.setBlockSizeMin(48);
+ }
}