You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by th...@apache.org on 2014/02/24 13:53:41 UTC
svn commit: r1571268 - in /jackrabbit/oak/trunk:
oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/
oak-blob/src/test/java/org/apache/jackrabbit/oak/spi/blob/
oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/sort/
oak-core/src/main/j...
Author: thomasm
Date: Mon Feb 24 12:53:40 2014
New Revision: 1571268
URL: http://svn.apache.org/r1571268
Log:
OAK-377 Data store garbage collection
Modified:
jackrabbit/oak/trunk/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/FileBlobStore.java
jackrabbit/oak/trunk/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/GarbageCollectableBlobStore.java
jackrabbit/oak/trunk/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/MemoryBlobStore.java
jackrabbit/oak/trunk/oak-blob/src/test/java/org/apache/jackrabbit/oak/spi/blob/AbstractBlobStoreTest.java
jackrabbit/oak/trunk/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/sort/ExternalSort.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/cloud/CloudBlobStore.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/db/DbBlobStore.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoBlobStore.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBBlobStore.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoBlobGCTest.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/blob/AbstractBlobStoreTest.java
Modified: jackrabbit/oak/trunk/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/FileBlobStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/FileBlobStore.java?rev=1571268&r1=1571267&r2=1571268&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/FileBlobStore.java (original)
+++ jackrabbit/oak/trunk/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/FileBlobStore.java Mon Feb 24 12:53:40 2014
@@ -219,7 +219,7 @@ public class FileBlobStore extends Abstr
}
@Override
- public boolean deleteChunk(String chunkId) throws Exception {
+ public boolean deleteChunk(String chunkId, long maxLastModifiedTime) throws Exception {
byte[] digest = StringUtils.convertHexToBytes(chunkId);
File f = getFile(digest, false);
if (!f.exists()) {
@@ -228,8 +228,11 @@ public class FileBlobStore extends Abstr
old.renameTo(f);
f = getFile(digest, false);
}
- f.delete();
- return mark;
+ if ((maxLastModifiedTime <= 0)
+ || FileUtils.isFileOlder(f, maxLastModifiedTime)) {
+ return f.delete();
+ }
+ return false;
}
@Override
@@ -241,8 +244,8 @@ public class FileBlobStore extends Abstr
@Override
public boolean apply(@Nullable File input) {
if (!input.isDirectory() && (
- (maxLastModifiedTime == 0 || maxLastModifiedTime == -1) ||
- FileUtils.isFileOlder(input, maxLastModifiedTime))) {
+ (maxLastModifiedTime <= 0)
+ || FileUtils.isFileOlder(input, maxLastModifiedTime))) {
return true;
}
return false;
Modified: jackrabbit/oak/trunk/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/GarbageCollectableBlobStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/GarbageCollectableBlobStore.java?rev=1571268&r1=1571267&r2=1571268&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/GarbageCollectableBlobStore.java (original)
+++ jackrabbit/oak/trunk/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/GarbageCollectableBlobStore.java Mon Feb 24 12:53:40 2014
@@ -86,11 +86,13 @@ public interface GarbageCollectableBlobS
* Delete the blob with the given id.
*
* @param chunkId the chunk id
+ * @param maxLastModifiedTime
+ * the max last modified time to consider for retrieval
* @return true, if successful
* @throws Exception
* the exception
*/
- boolean deleteChunk(String chunkId) throws Exception;
+ boolean deleteChunk(String chunkId, long maxLastModifiedTime) throws Exception;
/**
* Resolve chunks from the given Id.
Modified: jackrabbit/oak/trunk/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/MemoryBlobStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/MemoryBlobStore.java?rev=1571268&r1=1571267&r2=1571268&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/MemoryBlobStore.java (original)
+++ jackrabbit/oak/trunk/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/MemoryBlobStore.java Mon Feb 24 12:53:40 2014
@@ -80,8 +80,11 @@ public class MemoryBlobStore extends Abs
return count;
}
+ /**
+ * Ignores the maxlastModifiedTime
+ */
@Override
- public boolean deleteChunk(String chunkId) throws Exception {
+ public boolean deleteChunk(String chunkId, long maxLastModifiedTime) throws Exception {
BlockId id = new BlockId(StringUtils.convertHexToBytes(chunkId), 0);
if (map.containsKey(id)) {
map.remove(id);
Modified: jackrabbit/oak/trunk/oak-blob/src/test/java/org/apache/jackrabbit/oak/spi/blob/AbstractBlobStoreTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob/src/test/java/org/apache/jackrabbit/oak/spi/blob/AbstractBlobStoreTest.java?rev=1571268&r1=1571267&r2=1571268&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-blob/src/test/java/org/apache/jackrabbit/oak/spi/blob/AbstractBlobStoreTest.java (original)
+++ jackrabbit/oak/trunk/oak-blob/src/test/java/org/apache/jackrabbit/oak/spi/blob/AbstractBlobStoreTest.java Mon Feb 24 12:53:40 2014
@@ -373,7 +373,7 @@ public abstract class AbstractBlobStoreT
Set<String> ids = createArtifacts();
for (String id : ids) {
- store.deleteChunk(id);
+ store.deleteChunk(id, 0);
}
Iterator<String> iter = store.getAllChunkIds(0);
Modified: jackrabbit/oak/trunk/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/sort/ExternalSort.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/sort/ExternalSort.java?rev=1571268&r1=1571267&r2=1571268&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/sort/ExternalSort.java (original)
+++ jackrabbit/oak/trunk/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/sort/ExternalSort.java Mon Feb 24 12:53:40 2014
@@ -53,12 +53,16 @@ import java.util.zip.GZIPOutputStream;
* You can change the default maximal number of temporary files with the -t flag: java
* org/apache/oak/commons/sort/ExternalSort somefile.txt out.txt -t 3
*
- * For very large files, you might want to use an appropriate flag to allocate more memory to the
- * Java VM: java -Xms2G org/apache/oak/commons/sort/ExternalSort somefile.txt out.txt
+ * You can change the default maximum memory available with the -m flag: java
+ * org/apache/oak/commons/sort/ExternalSort somefile.txt out.txt -m 8192
*
- * By (in alphabetical order) Philippe Beaudoin, Eleftherios Chetzakis, Jon Elsas, Christan Grant,
- * Daniel Haran, Daniel Lemire, Sugumaran Harikrishnan, Jerry Yang, First published: April 2010
- * originally posted at http://lemire.me/blog/archives/2010/04/01/external-memory-sorting-in-java/
+ * For very large files, you might want to use an appropriate flag to allocate more memory to
+ * the Java VM: java -Xms2G org/apache/oak/commons/sort/ExternalSort somefile.txt out.txt
+ *
+ * By (in alphabetical order) Philippe Beaudoin, Eleftherios Chetzakis, Jon Elsas, Christan
+ * Grant, Daniel Haran, Daniel Lemire, Sugumaran Harikrishnan, Jerry Yang, First published:
+ * April 2010 originally posted at
+ * http://lemire.me/blog/archives/2010/04/01/external-memory-sorting-in-java/
*/
public class ExternalSort {
@@ -74,12 +78,17 @@ public class ExternalSort {
}
static int DEFAULTMAXTEMPFILES = 1024;
-
+
+ /**
+ * Defines the default maximum memory to be used while sorting (8 MB)
+ */
+ static long DEFAULT_MAX_MEM_BYTES = 8388608L;
+
// we divide the file into small blocks. If the blocks
// are too small, we shall create too many temporary files.
// If they are too big, we shall be using too much memory.
public static long estimateBestSizeOfBlocks(File filetobesorted,
- int maxtmpfiles) {
+ int maxtmpfiles, long maxMemory) {
long sizeoffile = filetobesorted.length() * 2;
/**
* We multiply by two because later on someone insisted on counting the memory usage as 2
@@ -93,11 +102,11 @@ public class ExternalSort {
// on the other hand, we don't want to create many temporary
// files
- // for naught. If blocksize is smaller than half the free
- // memory, grow it.
- long freemem = Runtime.getRuntime().freeMemory();
- if (blocksize < freemem / 2) {
- blocksize = freemem / 2;
+ // for naught. If blocksize is less than maximum allowed memory,
+ // scale the blocksize to be equal to the maxMemory parameter
+
+ if (blocksize < maxMemory) {
+ blocksize = maxMemory;
}
return blocksize;
}
@@ -114,7 +123,7 @@ public class ExternalSort {
*/
public static List<File> sortInBatch(File file)
throws IOException {
- return sortInBatch(file, defaultcomparator, DEFAULTMAXTEMPFILES,
+ return sortInBatch(file, defaultcomparator, DEFAULTMAXTEMPFILES, DEFAULT_MAX_MEM_BYTES,
Charset.defaultCharset(), null, false);
}
@@ -130,7 +139,7 @@ public class ExternalSort {
*/
public static List<File> sortInBatch(File file, Comparator<String> cmp)
throws IOException {
- return sortInBatch(file, cmp, DEFAULTMAXTEMPFILES,
+ return sortInBatch(file, cmp, DEFAULTMAXTEMPFILES, DEFAULT_MAX_MEM_BYTES,
Charset.defaultCharset(), null, false);
}
@@ -148,7 +157,7 @@ public class ExternalSort {
*/
public static List<File> sortInBatch(File file, Comparator<String> cmp,
boolean distinct) throws IOException {
- return sortInBatch(file, cmp, DEFAULTMAXTEMPFILES,
+ return sortInBatch(file, cmp, DEFAULTMAXTEMPFILES, DEFAULT_MAX_MEM_BYTES,
Charset.defaultCharset(), null, distinct);
}
@@ -175,14 +184,14 @@ public class ExternalSort {
* @return a list of temporary flat files
*/
public static List<File> sortInBatch(File file, Comparator<String> cmp,
- int maxtmpfiles, Charset cs, File tmpdirectory,
+ int maxtmpfiles, long maxMemory, Charset cs, File tmpdirectory,
boolean distinct, int numHeader, boolean usegzip)
throws IOException {
List<File> files = new ArrayList<File>();
BufferedReader fbr = new BufferedReader(new InputStreamReader(
new FileInputStream(file), cs));
- long blocksize = estimateBestSizeOfBlocks(file, maxtmpfiles);// in
- // bytes
+ long blocksize = estimateBestSizeOfBlocks(file, maxtmpfiles, maxMemory);// in
+ // bytes
try {
List<String> tmplist = new ArrayList<String>();
@@ -243,9 +252,9 @@ public class ExternalSort {
* @return a list of temporary flat files
*/
public static List<File> sortInBatch(File file, Comparator<String> cmp,
- int maxtmpfiles, Charset cs, File tmpdirectory, boolean distinct)
+ int maxtmpfiles, long maxMemory, Charset cs, File tmpdirectory, boolean distinct)
throws IOException {
- return sortInBatch(file, cmp, maxtmpfiles, cs, tmpdirectory,
+ return sortInBatch(file, cmp, maxtmpfiles, maxMemory, cs, tmpdirectory,
distinct, 0, false);
}
@@ -520,6 +529,8 @@ public class ExternalSort {
System.out
.println("-t or --maxtmpfiles (followed by an integer): specify an upper bound on the number of temporary files");
System.out
+ .println("-m or --maxmembytes (followed by a long): specify an upper bound on the memory");
+ System.out
.println("-c or --charset (followed by a charset code): specify the character set to use (for sorting)");
System.out
.println("-z or --gzip: use compression for the temporary files");
@@ -534,6 +545,7 @@ public class ExternalSort {
boolean verbose = false;
boolean distinct = false;
int maxtmpfiles = DEFAULTMAXTEMPFILES;
+ long maxMemory = DEFAULT_MAX_MEM_BYTES;
Charset cs = Charset.defaultCharset();
String inputfile = null, outputfile = null;
File tempFileStore = null;
@@ -559,6 +571,15 @@ public class ExternalSort {
System.err
.println("maxtmpfiles should be positive");
}
+ } else if ((args[param].equals("-m") || args[param]
+ .equals("--maxmembytes"))
+ && args.length > param + 1) {
+ param++;
+ maxMemory = Long.parseLong(args[param]);
+ if (headersize < 0) {
+ System.err
+ .println("maxmembytes should be positive");
+ }
} else if ((args[param].equals("-c") || args[param]
.equals("--charset"))
&& args.length > param + 1) {
@@ -597,7 +618,7 @@ public class ExternalSort {
}
Comparator<String> comparator = defaultcomparator;
List<File> l = sortInBatch(new File(inputfile), comparator,
- maxtmpfiles, cs, tempFileStore, distinct, headersize,
+ maxtmpfiles, maxMemory, cs, tempFileStore, distinct, headersize,
usegzip);
if (verbose)
System.out
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java?rev=1571268&r1=1571267&r2=1571268&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java Mon Feb 24 12:53:40 2014
@@ -70,17 +70,20 @@ public class MarkSweepGarbageCollector i
public static final int DEFAULT_BATCH_COUNT = 2048;
+ /** The max last modified time of blobs to consider for garbage collection. */
+ private long maxLastModifiedTime;
+
/** Run concurrently when possible. */
- boolean runConcurrently = true;
+ private boolean runConcurrently = true;
/** The number of sweeper threads to use. */
- int numSweepers = 1;
+ private int numSweepers = 1;
/** The node store. */
- DocumentNodeStore nodeStore;
+ private DocumentNodeStore nodeStore;
/** The garbage collector file state */
- GarbageCollectorFileState fs;
+ private GarbageCollectorFileState fs;
/** The configured root to store gc process files. */
private String root = TEMP_DIR;
@@ -89,6 +92,24 @@ public class MarkSweepGarbageCollector i
private int batchCount = DEFAULT_BATCH_COUNT;
/**
+ * Gets the max last modified time considered for garbage collection.
+ *
+ * @return the max last modified time
+ */
+ protected long getMaxLastModifiedTime() {
+ return maxLastModifiedTime;
+ }
+
+ /**
+ * Sets the max last modified time considered for garbage collection.
+ *
+ * @param maxLastModifiedTime the new max last modified time
+ */
+ protected void setMaxLastModifiedTime(long maxLastModifiedTime) {
+ this.maxLastModifiedTime = maxLastModifiedTime;
+ }
+
+ /**
* Gets the root.
*
* @return the root
@@ -125,32 +146,27 @@ public class MarkSweepGarbageCollector i
}
/**
- * Instantiates a new blob garbage collector.
- *
- * @param nodeStore
- * the node store
- * @param root
- * the root
- * @param batchCount
- * the batch count
- * @param runBackendConcurrently
- * - run the backend iterate concurrently
- * @param maxSweeperThreads
- * the max sweeper threads
- * @throws IOException
- * Signals that an I/O exception has occurred.
+ * @param nodeStore the node store
+ * @param root the root
+ * @param batchCount the batch count
+ * @param runBackendConcurrently - run the backend iterate concurrently
+ * @param maxSweeperThreads the max sweeper threads
+ * @param maxLastModifiedTime the max last modified time
+ * @throws IOException Signals that an I/O exception has occurred.
*/
public void init(
NodeStore nodeStore,
String root,
int batchCount,
boolean runBackendConcurrently,
- int maxSweeperThreads)
+ int maxSweeperThreads,
+ long maxLastModifiedTime)
throws IOException {
this.batchCount = batchCount;
this.root = root;
this.runConcurrently = runBackendConcurrently;
this.numSweepers = maxSweeperThreads;
+ this.maxLastModifiedTime = maxLastModifiedTime;
init(nodeStore);
}
@@ -356,10 +372,10 @@ public class MarkSweepGarbageCollector i
class Sweeper implements Runnable {
/** The exception queue. */
- ConcurrentLinkedQueue<String> exceptionQueue;
+ private ConcurrentLinkedQueue<String> exceptionQueue;
/** The ids to sweep. */
- List<String> ids;
+ private List<String> ids;
/**
* Instantiates a new sweeper.
@@ -379,7 +395,7 @@ public class MarkSweepGarbageCollector i
for (String id : ids) {
try {
boolean deleted = ((GarbageCollectableBlobStore) nodeStore.getBlobStore())
- .deleteChunk(id);
+ .deleteChunk(id, maxLastModifiedTime);
if (!deleted) {
exceptionQueue.add(id);
}
@@ -394,8 +410,6 @@ public class MarkSweepGarbageCollector i
/**
* Iterates the complete node tree.
*
- * @param writer
- * the writer
* @return the list
* @throws Exception
* the exception
@@ -446,9 +460,6 @@ public class MarkSweepGarbageCollector i
* BlobIdRetriever class to retrieve all blob ids.
*/
class BlobIdRetriever implements Runnable {
-
- boolean finished;
-
@Override
public void run() {
retrieve();
@@ -465,7 +476,7 @@ public class MarkSweepGarbageCollector i
bufferWriter = new BufferedWriter(
new FileWriter(fs.getAvailableRefs()));
Iterator<String> idsIter = ((GarbageCollectableBlobStore) nodeStore.getBlobStore())
- .getAllChunkIds(0);
+ .getAllChunkIds(maxLastModifiedTime);
List<String> ids = Lists.newArrayList();
int blobsCount = 0;
while (idsIter.hasNext()) {
@@ -483,8 +494,6 @@ public class MarkSweepGarbageCollector i
// sort the file
fs.sort(fs.getAvailableRefs());
-
- finished = true;
LOG.debug("Ending retrieve of all blobs : " + blobsCount);
} catch (Exception e) {
e.printStackTrace();
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/cloud/CloudBlobStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/cloud/CloudBlobStore.java?rev=1571268&r1=1571267&r2=1571268&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/cloud/CloudBlobStore.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/cloud/CloudBlobStore.java Mon Feb 24 12:53:40 2014
@@ -227,23 +227,27 @@ public class CloudBlobStore extends Abst
}
@Override
- public boolean deleteChunk(String chunkId) throws Exception {
+ public boolean deleteChunk(String chunkId, long maxLastModifiedTime) throws Exception {
Preconditions.checkNotNull(context);
final org.jclouds.blobstore.BlobStore blobStore = context.getBlobStore();
- blobStore.removeBlob(cloudContainer, chunkId);
-
+ StorageMetadata metadata = blobStore.blobMetadata(cloudContainer, chunkId);
+ if ((maxLastModifiedTime <= 0)
+ || (metadata.getLastModified().getTime() <= maxLastModifiedTime)) {
+ blobStore.removeBlob(cloudContainer, chunkId);
+ return true;
+ }
return true;
}
class CloudStoreIterator implements Iterator<String> {
- static final int BATCH = 1000;
+ private static final int BATCH = 1000;
- org.jclouds.blobstore.BlobStore store;
- long maxLastModifiedTime;
+ private org.jclouds.blobstore.BlobStore store;
+ private long maxLastModifiedTime;
- PageSet<? extends StorageMetadata> set;
- ArrayDeque<String> queue;
+ private PageSet<? extends StorageMetadata> set;
+ private ArrayDeque<String> queue;
public CloudStoreIterator(org.jclouds.blobstore.BlobStore store,
long maxLastModifiedTime) {
@@ -255,7 +259,7 @@ public class CloudBlobStore extends Abst
@Override
public boolean hasNext() {
if ((set == null) || (queue == null)) {
- set = store.list(cloudContainer, maxResults(1000));
+ set = store.list(cloudContainer, maxResults(BATCH));
loadElements(set);
}
@@ -278,8 +282,8 @@ public class CloudBlobStore extends Abst
Iterator<? extends StorageMetadata> iter = set.iterator();
while (iter.hasNext()) {
StorageMetadata metadata = iter.next();
- if ((maxLastModifiedTime == 0 || maxLastModifiedTime == -1) ||
- (metadata.getLastModified().getTime() <= maxLastModifiedTime)) {
+ if ((maxLastModifiedTime <= 0)
+ || (metadata.getLastModified().getTime() <= maxLastModifiedTime)) {
queue.add(metadata.getName());
} else {
queue.add(metadata.getName());
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java?rev=1571268&r1=1571267&r2=1571268&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java Mon Feb 24 12:53:40 2014
@@ -522,9 +522,17 @@ public class DataStoreBlobStore implemen
}
@Override
- public boolean deleteChunk(String blobId) throws Exception {
- ((MultiDataStoreAware) dataStore).deleteRecord(new DataIdentifier(blobId));
- return true;
+ public boolean deleteChunk(String blobId, long maxLastModifiedTime) throws Exception {
+ if (dataStore instanceof MultiDataStoreAware) {
+ DataIdentifier identifier = new DataIdentifier(blobId);
+ DataRecord dataRecord = dataStore.getRecord(identifier);
+ if ((maxLastModifiedTime <= 0)
+ || dataRecord.getLastModified() <= maxLastModifiedTime) {
+ ((MultiDataStoreAware) dataStore).deleteRecord(identifier);
+ return true;
+ }
+ }
+ return false;
}
@Override
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/db/DbBlobStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/db/DbBlobStore.java?rev=1571268&r1=1571267&r2=1571268&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/db/DbBlobStore.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/db/DbBlobStore.java Mon Feb 24 12:53:40 2014
@@ -215,16 +215,27 @@ public class DbBlobStore extends Abstrac
}
@Override
- public boolean deleteChunk(String chunkId) throws Exception {
+ public boolean deleteChunk(String chunkId, long maxLastModifiedTime) throws Exception {
Connection conn = cp.getConnection();
try {
- PreparedStatement prep = conn
- .prepareStatement("delete from datastore_meta where id = ?");
- ;;;
- // TODO and lastMod <= ?
- ;;;
- PreparedStatement prepData = conn
- .prepareStatement("delete from datastore_data where id = ?");
+ PreparedStatement prep = null;
+ PreparedStatement prepData = null;
+
+ if (maxLastModifiedTime > 0) {
+ prep = conn.prepareStatement(
+ "delete from datastore_meta where id = ? and lastMod <= ?");
+ prep.setLong(2, maxLastModifiedTime);
+
+ prepData = conn.prepareStatement(
+ "delete from datastore_data where id = ? and lastMod <= ?");
+ prepData.setLong(2, maxLastModifiedTime);
+ } else {
+ prep = conn.prepareStatement(
+ "delete from datastore_meta where id = ?");
+
+ prepData = conn.prepareStatement(
+ "delete from datastore_data where id = ?");
+ }
prep.setString(1, chunkId);
prep.execute();
prepData.setString(1, chunkId);
@@ -245,7 +256,7 @@ public class DbBlobStore extends Abstrac
final Connection conn = cp.getConnection();
PreparedStatement prep = null;
- if ((maxLastModifiedTime != 0) && (maxLastModifiedTime != -1)) {
+ if (maxLastModifiedTime > 0) {
prep = conn.prepareStatement(
"select id from datastore_meta where lastMod <= ?");
prep.setLong(1, maxLastModifiedTime);
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoBlobStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoBlobStore.java?rev=1571268&r1=1571267&r2=1571268&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoBlobStore.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoBlobStore.java Mon Feb 24 12:53:40 2014
@@ -192,12 +192,10 @@ public class MongoBlobStore extends Abst
}
@Override
- public boolean deleteChunk(String chunkId) throws Exception {
+ public boolean deleteChunk(String chunkId, long maxLastModifiedTime) throws Exception {
DBCollection collection = getBlobCollection();
- BasicDBObject removeObj = new BasicDBObject();
- removeObj.append(MongoBlob.KEY_ID, chunkId);
+ WriteResult result = collection.remove(getBlobQuery(chunkId, maxLastModifiedTime));
- WriteResult result = collection.remove(removeObj);
if (result.getN() == 1) {
return true;
}
@@ -222,6 +220,7 @@ public class MongoBlobStore extends Abst
.addOption(Bytes.QUERYOPTION_SLAVEOK);
return new AbstractIterator<String>() {
+ @Override
protected String computeNext() {
if (cur.hasNext()) {
MongoBlob blob = (MongoBlob) cur.next();
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBBlobStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBBlobStore.java?rev=1571268&r1=1571267&r2=1571268&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBBlobStore.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBBlobStore.java Mon Feb 24 12:53:40 2014
@@ -286,12 +286,25 @@ public class RDBBlobStore extends Abstra
}
@Override
- public boolean deleteChunk(String chunkId) throws Exception {
+ public boolean deleteChunk(String chunkId, long maxLastModifiedTime) throws Exception {
try {
- PreparedStatement prep = connection.prepareStatement(
- "delete from datastore_meta where id = ?");
- PreparedStatement prepData = connection.prepareStatement(
- "delete from datastore_data where id = ?");
+ PreparedStatement prep = null;
+ PreparedStatement prepData = null;
+
+ if (maxLastModifiedTime > 0) {
+ prep = connection.prepareStatement(
+ "delete from datastore_meta where id = ? and lastMod <= ?");
+ prep.setLong(2, maxLastModifiedTime);
+
+ prepData = connection.prepareStatement(
+ "delete from datastore_data where id = ? and lastMod <= ?");
+ prepData.setLong(2, maxLastModifiedTime);
+ } else {
+ prep = connection.prepareStatement(
+ "delete from datastore_meta where id = ?");
+ prepData = connection.prepareStatement(
+ "delete from datastore_data where id = ?");
+ }
prep.setString(1, chunkId);
prep.execute();
prepData.setString(1, chunkId);
@@ -301,7 +314,7 @@ public class RDBBlobStore extends Abstra
prepData.close();
} finally {
connection.commit();
-}
+ }
return true;
}
@@ -310,7 +323,7 @@ public class RDBBlobStore extends Abstra
public Iterator<String> getAllChunkIds(long maxLastModifiedTime) throws Exception {
PreparedStatement prep = null;
- if ((maxLastModifiedTime != 0) && (maxLastModifiedTime != -1)) {
+ if (maxLastModifiedTime > 0) {
prep = connection.prepareStatement(
"select id from datastore_meta where lastMod <= ?");
prep.setLong(1, maxLastModifiedTime);
Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoBlobGCTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoBlobGCTest.java?rev=1571268&r1=1571267&r2=1571268&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoBlobGCTest.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoBlobGCTest.java Mon Feb 24 12:53:40 2014
@@ -94,7 +94,7 @@ public class MongoBlobGCTest extends Abs
DocumentNodeStore s = mk.getNodeStore();
MarkSweepGarbageCollector gc = new MarkSweepGarbageCollector();
- gc.init(s, "./target", 2048, true, 2);
+ gc.init(s, "./target", 2048, true, 2, 0);
gc.collectGarbage();
Set<String> existing = iterate();
Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/blob/AbstractBlobStoreTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/blob/AbstractBlobStoreTest.java?rev=1571268&r1=1571267&r2=1571268&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/blob/AbstractBlobStoreTest.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/blob/AbstractBlobStoreTest.java Mon Feb 24 12:53:40 2014
@@ -383,7 +383,7 @@ public abstract class AbstractBlobStoreT
Set<String> ids = createArtifacts();
for (String id : ids) {
- store.deleteChunk(id);
+ store.deleteChunk(id, 0);
}
Iterator<String> iter = store.getAllChunkIds(0);