You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by re...@apache.org on 2018/09/19 10:35:20 UTC
svn commit: r1841311 - in /jackrabbit/oak/branches/1.0:
oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/
oak-blob/src/test/java/org/apache/jackrabbit/oak/spi/blob/
oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/
oak-c...
Author: reschke
Date: Wed Sep 19 10:35:19 2018
New Revision: 1841311
URL: http://svn.apache.org/viewvc?rev=1841311&view=rev
Log:
OAK-7389: Mongo/FileBlobStore does not update timestamp for already existing blobs
Merge r1828502 from trunk and align codebase with 1.2
Modified:
jackrabbit/oak/branches/1.0/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/FileBlobStore.java
jackrabbit/oak/branches/1.0/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/MemoryBlobStore.java
jackrabbit/oak/branches/1.0/oak-blob/src/test/java/org/apache/jackrabbit/oak/spi/blob/AbstractBlobStoreTest.java
jackrabbit/oak/branches/1.0/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java
jackrabbit/oak/branches/1.0/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoBlobStore.java
Modified: jackrabbit/oak/branches/1.0/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/FileBlobStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.0/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/FileBlobStore.java?rev=1841311&r1=1841310&r2=1841311&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.0/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/FileBlobStore.java (original)
+++ jackrabbit/oak/branches/1.0/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/FileBlobStore.java Wed Sep 19 10:35:19 2018
@@ -109,6 +109,7 @@ public class FileBlobStore extends Abstr
protected synchronized void storeBlock(byte[] digest, int level, byte[] data) throws IOException {
File f = getFile(digest, false);
if (f.exists()) {
+ FileUtils.touch(f);
return;
}
File parent = f.getParentFile();
@@ -221,6 +222,12 @@ public class FileBlobStore extends Abstr
@Override
public boolean deleteChunks(List<String> chunkIds, long maxLastModifiedTime) throws Exception {
+ return (chunkIds.size() == countDeleteChunks(chunkIds, maxLastModifiedTime));
+ }
+
+ // @Override OAK-2973
+ public long countDeleteChunks(List<String> chunkIds, long maxLastModifiedTime) throws Exception {
+ int count = 0;
for (String chunkId : chunkIds) {
byte[] digest = StringUtils.convertHexToBytes(chunkId);
File f = getFile(digest, false);
@@ -233,9 +240,10 @@ public class FileBlobStore extends Abstr
if ((maxLastModifiedTime <= 0)
|| FileUtils.isFileOlder(f, maxLastModifiedTime)) {
f.delete();
+ count++;
}
}
- return true;
+ return count;
}
@Override
@@ -270,4 +278,4 @@ public class FileBlobStore extends Abstr
public void clearCache() {
// no cache
}
-}
\ No newline at end of file
+}
Modified: jackrabbit/oak/branches/1.0/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/MemoryBlobStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.0/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/MemoryBlobStore.java?rev=1841311&r1=1841310&r2=1841311&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.0/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/MemoryBlobStore.java (original)
+++ jackrabbit/oak/branches/1.0/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/MemoryBlobStore.java Wed Sep 19 10:35:19 2018
@@ -33,6 +33,7 @@ public class MemoryBlobStore extends Abs
private HashMap<BlockId, byte[]> map = new HashMap<BlockId, byte[]>();
private HashMap<BlockId, byte[]> old = new HashMap<BlockId, byte[]>();
+ private HashMap<BlockId, Long> timestamps = new HashMap<BlockId, Long>();
private boolean mark;
@Override
@@ -46,7 +47,9 @@ public class MemoryBlobStore extends Abs
@Override
protected synchronized void storeBlock(byte[] digest, int level, byte[] data) {
- map.put(new BlockId(digest, 0), data);
+ BlockId id = new BlockId(digest, 0);
+ map.put(id, data);
+ timestamps.put(id, System.currentTimeMillis());
}
@Override
@@ -81,20 +84,31 @@ public class MemoryBlobStore extends Abs
return count;
}
+ @Override
+ public boolean deleteChunks(List<String> chunkIds, long maxLastModifiedTime) throws Exception {
+ return (chunkIds.size() == countDeleteChunks(chunkIds, maxLastModifiedTime));
+ }
+
/**
* Ignores the maxlastModifiedTime
*/
- @Override
- public boolean deleteChunks(List<String> chunkIds, long maxLastModifiedTime) throws Exception {
+ // @Override OAK-2973
+ public long countDeleteChunks(List<String> chunkIds, long maxLastModifiedTime) throws Exception {
+ int count = 0;
for (String chunkId : chunkIds) {
BlockId id = new BlockId(StringUtils.convertHexToBytes(chunkId), 0);
if (map.containsKey(id)) {
- map.remove(id);
+ if (maxLastModifiedTime == 0 || (maxLastModifiedTime > 0 && maxLastModifiedTime > timestamps.get(id))) {
+ map.remove(id);
+ timestamps.remove(id);
+ count++;
+ }
} else if (old.containsKey(id)) {
old.remove(id);
+ count++;
}
}
- return true;
+ return count;
}
/**
Modified: jackrabbit/oak/branches/1.0/oak-blob/src/test/java/org/apache/jackrabbit/oak/spi/blob/AbstractBlobStoreTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.0/oak-blob/src/test/java/org/apache/jackrabbit/oak/spi/blob/AbstractBlobStoreTest.java?rev=1841311&r1=1841310&r2=1841311&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.0/oak-blob/src/test/java/org/apache/jackrabbit/oak/spi/blob/AbstractBlobStoreTest.java (original)
+++ jackrabbit/oak/branches/1.0/oak-blob/src/test/java/org/apache/jackrabbit/oak/spi/blob/AbstractBlobStoreTest.java Wed Sep 19 10:35:19 2018
@@ -18,6 +18,7 @@ package org.apache.jackrabbit.oak.spi.bl
import static org.hamcrest.core.IsInstanceOf.instanceOf;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.junit.Assume.assumeThat;
@@ -405,7 +406,7 @@ public abstract class AbstractBlobStoreT
ids.remove(iter.next());
}
- assertTrue(ids.isEmpty());
+ assertTrue("unexpected ids in store: " + ids, ids.isEmpty());
}
@Test
@@ -423,6 +424,28 @@ public abstract class AbstractBlobStoreT
assertTrue(ret.toString(), ret.isEmpty());
}
+ @Test
+ public void deleteUpdatedBlob() throws Exception {
+ String id = store.writeBlob(randomStream(0, getArtifactSize()));
+ Thread.sleep(100);
+
+ long beforeUpdateTime = System.currentTimeMillis();
+
+ Thread.sleep(1000);
+
+ // Should update the timestamp
+ String id2 = store.writeBlob(randomStream(0, getArtifactSize()));
+ assertEquals(id, id2);
+
+ Set<String> chunks = Sets.newHashSet();
+ Iterator<String> iter = store.resolveChunks(id.toString());
+ while (iter.hasNext()) {
+ chunks.add(iter.next());
+ }
+ boolean deleted = store.deleteChunks(Lists.newArrayList(chunks), beforeUpdateTime);
+ assertFalse("Deleted updated blobs", deleted);
+ }
+
private Set<String> createArtifacts() throws Exception {
Set<String> ids = Sets.newHashSet();
int number = 10;
Modified: jackrabbit/oak/branches/1.0/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.0/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java?rev=1841311&r1=1841310&r2=1841311&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.0/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java (original)
+++ jackrabbit/oak/branches/1.0/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java Wed Sep 19 10:35:19 2018
@@ -377,6 +377,12 @@ public class DataStoreBlobStore implemen
@Override
public boolean deleteChunks(List<String> chunkIds, long maxLastModifiedTime) throws Exception {
+ return (chunkIds.size() == countDeleteChunks(chunkIds, maxLastModifiedTime));
+ }
+
+ // @Override OAK-2973
+ public long countDeleteChunks(List<String> chunkIds, long maxLastModifiedTime) throws Exception {
+ int count = 0;
if (delegate instanceof MultiDataStoreAware) {
for (String chunkId : chunkIds) {
String blobId = extractBlobId(chunkId);
@@ -384,15 +390,16 @@ public class DataStoreBlobStore implemen
DataRecord dataRecord = delegate.getRecord(identifier);
boolean success = (maxLastModifiedTime <= 0)
|| dataRecord.getLastModified() <= maxLastModifiedTime;
- log.debug("Deleting blob [{}] with last modified date [{}] : [{}]", blobId,
+ log.trace("Deleting blob [{}] with last modified date [{}] : [{}]", blobId,
dataRecord.getLastModified(), success);
if (success) {
((MultiDataStoreAware) delegate).deleteRecord(identifier);
log.info("Deleted blob [{}]", blobId);
+ count++;
}
}
}
- return true;
+ return count;
}
@Override
Modified: jackrabbit/oak/branches/1.0/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoBlobStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.0/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoBlobStore.java?rev=1841311&r1=1841310&r2=1841311&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.0/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoBlobStore.java (original)
+++ jackrabbit/oak/branches/1.0/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoBlobStore.java Wed Sep 19 10:35:19 2018
@@ -77,18 +77,29 @@ public class MongoBlobStore extends Cach
protected void storeBlock(byte[] digest, int level, byte[] data) throws IOException {
String id = StringUtils.convertBytesToHex(digest);
cache.put(id, data);
- // Check if it already exists?
- MongoBlob mongoBlob = new MongoBlob();
- mongoBlob.setId(id);
- mongoBlob.setData(data);
- mongoBlob.setLevel(level);
- mongoBlob.setLastMod(System.currentTimeMillis());
- // TODO check the return value
- // TODO verify insert is fast if the entry already exists
+
+ // Create the mongo blob object
+ BasicDBObject mongoBlob = new BasicDBObject(MongoBlob.KEY_ID, id);
+ mongoBlob.append(MongoBlob.KEY_DATA, data);
+ mongoBlob.append(MongoBlob.KEY_LEVEL, level);
+
+ // If update only the lastMod needs to be modified
+ BasicDBObject updateBlob = new BasicDBObject(MongoBlob.KEY_LAST_MOD, System.currentTimeMillis());
+
+ BasicDBObject upsert = new BasicDBObject();
+ upsert.append("$setOnInsert", mongoBlob)
+ .append("$set", updateBlob);
+
try {
- getBlobCollection().insert(mongoBlob);
- } catch (MongoException.DuplicateKey e) {
- // the same block was already stored before: ignore
+ DBObject query = getBlobQuery(id, -1);
+ WriteResult update = getBlobCollection().update(query, upsert, true, false);
+ if (update != null && update.isUpdateOfExisting()) {
+ LOG.debug("Block with id [{}] updated", id);
+ } else {
+ LOG.trace("Block with id [{}] created", id);
+ }
+ } catch (MongoException e) {
+ throw new IOException(e.getMessage(), e);
}
}
@@ -200,6 +211,11 @@ public class MongoBlobStore extends Cach
@Override
public boolean deleteChunks(List<String> chunkIds, long maxLastModifiedTime) throws Exception {
+ return (chunkIds.size() == countDeleteChunks(chunkIds, maxLastModifiedTime));
+ }
+
+ // @Override OAK-2973
+ public long countDeleteChunks(List<String> chunkIds, long maxLastModifiedTime) throws Exception {
DBCollection collection = getBlobCollection();
QueryBuilder queryBuilder = new QueryBuilder();
if (chunkIds != null) {
@@ -211,11 +227,7 @@ public class MongoBlobStore extends Cach
}
WriteResult result = collection.remove(queryBuilder.get());
- if (result.getN() == chunkIds.size()) {
- return true;
- }
-
- return false;
+ return result.getN();
}
@Override