You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by am...@apache.org on 2017/08/10 03:52:24 UTC
svn commit: r1804626 - in /jackrabbit/oak/trunk/oak-lucene/src:
main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/
test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/
Author: amitj
Date: Thu Aug 10 03:52:24 2017
New Revision: 1804626
URL: http://svn.apache.org/viewvc?rev=1804626&view=rev
Log:
OAK-6504: Active deletion of blobs needs to indicate information about purged blobs to mark-sweep collector
- Synchronizing the blob tracker with the deleted blob ids
- Added an integration test with FDS
Added:
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/AbstractActiveDeletedBlobTest.java (with props)
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/ActiveDeletedBlobSyncTrackerTest.java (with props)
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/ActiveDeletedBlobCollectorFactory.java
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/ActiveDeletedBlobCollectionIT.java
Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/ActiveDeletedBlobCollectorFactory.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/ActiveDeletedBlobCollectorFactory.java?rev=1804626&r1=1804625&r2=1804626&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/ActiveDeletedBlobCollectorFactory.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/ActiveDeletedBlobCollectorFactory.java Thu Aug 10 03:52:24 2017
@@ -16,12 +16,18 @@
*/
package org.apache.jackrabbit.oak.plugins.index.lucene.directory;
+import com.google.common.base.Charsets;
import com.google.common.base.Joiner;
import com.google.common.collect.Lists;
+import com.google.common.io.Closeables;
+import com.google.common.io.Files;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.filefilter.IOFileFilter;
import org.apache.commons.io.filefilter.RegexFileFilter;
+import org.apache.jackrabbit.oak.commons.FileIOUtils;
import org.apache.jackrabbit.oak.commons.benchmark.PerfLogger;
+import org.apache.jackrabbit.oak.plugins.blob.BlobTrackingStore;
+import org.apache.jackrabbit.oak.plugins.blob.datastore.BlobTracker;
import org.apache.jackrabbit.oak.plugins.index.IndexCommitCallback;
import org.apache.jackrabbit.oak.spi.blob.GarbageCollectableBlobStore;
import org.apache.jackrabbit.oak.stats.Clock;
@@ -31,6 +37,7 @@ import org.slf4j.LoggerFactory;
import javax.annotation.Nonnull;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
+import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
@@ -161,6 +168,21 @@ public class ActiveDeletedBlobCollectorF
long numBlobsDeleted = 0;
long numChunksDeleted = 0;
+ File idTempDeleteFile = null;
+ BufferedWriter idTempDeleteWriter = null;
+ // If blob store support blob tracking
+ boolean blobIdsTracked = blobStore instanceof BlobTrackingStore;
+
+ if (blobIdsTracked) {
+ try {
+ idTempDeleteFile = File.createTempFile("idTempDelete", null, rootDirectory);
+ idTempDeleteWriter = Files.newWriter(idTempDeleteFile, Charsets.UTF_8);
+ } catch (Exception e) {
+ LOG.warn("Unable to open a writer to a temp file, will ignore tracker sync");
+ blobIdsTracked = false;
+ }
+ }
+
long lastCheckedBlobTimestamp = readLastCheckedBlobTimestamp();
String currInUseFileName = deletedBlobsFileWriter.inUseFileName;
deletedBlobsFileWriter.releaseInUseFile();
@@ -204,6 +226,13 @@ public class ActiveDeletedBlobCollectorF
} else {
numBlobsDeleted++;
numChunksDeleted += deleted;
+
+ if (blobIdsTracked) {
+ // Save deleted chunkIds to a temporary file
+ for (String id : chunkIds) {
+ FileIOUtils.writeAsLine(idTempDeleteWriter, id, true);
+ }
+ }
}
}
} catch (NumberFormatException nfe) {
@@ -230,6 +259,21 @@ public class ActiveDeletedBlobCollectorF
LOG.debug("Skipping {} as its timestamp is newer than {}", deletedBlobListFile.getName(), before);
}
}
+
+ // Synchronize deleted blob ids with the blob id tracker
+ try {
+ Closeables.close(idTempDeleteWriter, true);
+
+ if (blobIdsTracked && numBlobsDeleted > 0) {
+ BlobTracker tracker = ((BlobTrackingStore) blobStore).getTracker();
+ if (tracker != null) {
+ tracker.remove(idTempDeleteFile);
+ }
+ }
+ } catch(Exception e) {
+ LOG.warn("Error refreshing tracked blob ids", e);
+ }
+
LOG.info("Deleted {} blobs contained in {} chunks", numBlobsDeleted, numChunksDeleted);
writeOutLastCheckedBlobTimestamp(before);
}
Added: jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/AbstractActiveDeletedBlobTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/AbstractActiveDeletedBlobTest.java?rev=1804626&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/AbstractActiveDeletedBlobTest.java (added)
+++ jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/AbstractActiveDeletedBlobTest.java Thu Aug 10 03:52:24 2017
@@ -0,0 +1,290 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.index.lucene.directory;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+import javax.annotation.CheckForNull;
+import javax.annotation.Nonnull;
+
+import com.google.common.collect.Iterators;
+import org.apache.jackrabbit.JcrConstants;
+import org.apache.jackrabbit.core.data.DataIdentifier;
+import org.apache.jackrabbit.core.data.DataRecord;
+import org.apache.jackrabbit.core.data.DataStoreException;
+import org.apache.jackrabbit.oak.api.CommitFailedException;
+import org.apache.jackrabbit.oak.api.Tree;
+import org.apache.jackrabbit.oak.api.Type;
+import org.apache.jackrabbit.oak.commons.concurrent.ExecutorCloser;
+import org.apache.jackrabbit.oak.plugins.blob.BlobTrackingStore;
+import org.apache.jackrabbit.oak.plugins.blob.datastore.BlobTracker;
+import org.apache.jackrabbit.oak.plugins.index.AsyncIndexUpdate;
+import org.apache.jackrabbit.oak.plugins.index.lucene.IndexCopier;
+import org.apache.jackrabbit.oak.plugins.index.lucene.IndexDefinition;
+import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants;
+import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexEditorProvider;
+import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexProvider;
+import org.apache.jackrabbit.oak.plugins.memory.PropertyStates;
+import org.apache.jackrabbit.oak.query.AbstractQueryTest;
+import org.apache.jackrabbit.oak.spi.blob.BlobOptions;
+import org.apache.jackrabbit.oak.spi.blob.GarbageCollectableBlobStore;
+import org.apache.jackrabbit.oak.spi.state.NodeStore;
+import org.apache.jackrabbit.oak.stats.Clock;
+import org.junit.After;
+import org.junit.Rule;
+import org.junit.rules.TemporaryFolder;
+
+import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.ASYNC_PROPERTY_NAME;
+import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.INDEX_DEFINITIONS_NAME;
+import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.INDEX_DEFINITIONS_NODE_TYPE;
+import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.REINDEX_PROPERTY_NAME;
+import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.TYPE_PROPERTY_NAME;
+
+public abstract class AbstractActiveDeletedBlobTest extends AbstractQueryTest {
+ @Rule
+ public TemporaryFolder temporaryFolder = new TemporaryFolder(new File("target"));
+ @Rule
+ public TemporaryFolder blobCollectionRoot = new TemporaryFolder(new File("target"));
+ @Rule
+ public TemporaryFolder fileDataStoreRoot = new TemporaryFolder(new File("target"));
+
+ protected ExecutorService executorService = Executors.newFixedThreadPool(2);
+
+ protected CountingBlobStore blobStore = null;
+
+ protected Clock clock = new Clock.Virtual();
+
+ protected ActiveDeletedBlobCollectorFactory.ActiveDeletedBlobCollectorImpl adbc = null;
+
+ protected AsyncIndexUpdate asyncIndexUpdate;
+
+ protected LuceneIndexEditorProvider editorProvider;
+
+ protected NodeStore nodeStore;
+
+ protected LuceneIndexProvider provider;
+
+ @After
+ public void after() {
+ new ExecutorCloser(executorService).close();
+ executorService.shutdown();
+ IndexDefinition.setDisableStoredIndexDefinition(false);
+ }
+
+ public static Tree createIndex(Tree index, String name, Set<String> propNames) throws CommitFailedException {
+ Tree def = index.addChild(INDEX_DEFINITIONS_NAME).addChild(name);
+ def.setProperty(JcrConstants.JCR_PRIMARYTYPE,
+ INDEX_DEFINITIONS_NODE_TYPE, Type.NAME);
+ def.setProperty(TYPE_PROPERTY_NAME, LuceneIndexConstants.TYPE_LUCENE);
+ def.setProperty(REINDEX_PROPERTY_NAME, true);
+ def.setProperty(ASYNC_PROPERTY_NAME, "async");
+ def.setProperty(LuceneIndexConstants.FULL_TEXT_ENABLED, false);
+ def.setProperty(PropertyStates.createProperty(LuceneIndexConstants.INCLUDE_PROPERTY_NAMES, propNames, Type.STRINGS));
+ def.setProperty(LuceneIndexConstants.SAVE_DIR_LISTING, true);
+ return index.getChild(INDEX_DEFINITIONS_NAME).getChild(name);
+ }
+
+ @Override
+ protected void createTestIndexNode() throws Exception {
+ setTraversalEnabled(false);
+ }
+
+ protected IndexCopier createIndexCopier() {
+ try {
+ return new IndexCopier(executorService, temporaryFolder.getRoot());
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ protected Tree createIndex(String name, Set<String> propNames) throws CommitFailedException {
+ Tree index = root.getTree("/");
+ return createIndex(index, name, propNames);
+ }
+
+ class CountingBlobStore implements GarbageCollectableBlobStore, BlobTrackingStore {
+
+ private final GarbageCollectableBlobStore delegate;
+ protected long numChunks = 0;
+
+ CountingBlobStore(GarbageCollectableBlobStore delegate) {
+ this.delegate = delegate;
+ }
+
+ @Override public String writeBlob(InputStream in) throws IOException {
+ String blobId = delegate.writeBlob(in);
+ numChunks += Iterators.size(delegate.resolveChunks(blobId));
+ return blobId;
+ }
+
+ @Override public void setBlockSize(int x) {
+ delegate.setBlockSize(x);
+ }
+
+ @Override public String writeBlob(InputStream in, BlobOptions options) throws IOException {
+ String blobId = delegate.writeBlob(in, options);
+ numChunks += Iterators.size(delegate.resolveChunks(blobId));
+ return blobId;
+ }
+
+ @Override public String writeBlob(String tempFileName) throws IOException {
+ String blobId = delegate.writeBlob(tempFileName);
+ numChunks += Iterators.size(delegate.resolveChunks(blobId));
+ return blobId;
+ }
+
+ @Override public int sweep() throws IOException {
+ return delegate.sweep();
+ }
+
+ @Override public void startMark() throws IOException {
+ delegate.startMark();
+ }
+
+ @Override public int readBlob(String blobId, long pos, byte[] buff, int off, int length) throws IOException {
+ return delegate.readBlob(blobId, pos, buff, off, length);
+ }
+
+ @Override public void clearInUse() {
+ delegate.clearInUse();
+ }
+
+ @Override public void clearCache() {
+ delegate.clearCache();
+ }
+
+ @Override public long getBlobLength(String blobId) throws IOException {
+ return delegate.getBlobLength(blobId);
+ }
+
+ @Override public long getBlockSizeMin() {
+ return delegate.getBlockSizeMin();
+ }
+
+ @Override public Iterator<String> getAllChunkIds(long maxLastModifiedTime) throws Exception {
+ return delegate.getAllChunkIds(maxLastModifiedTime);
+ }
+
+ @Override public InputStream getInputStream(String blobId) throws IOException {
+ return delegate.getInputStream(blobId);
+ }
+
+ @Override @Deprecated public boolean deleteChunks(List<String> chunkIds, long maxLastModifiedTime)
+ throws Exception {
+ numChunks -= chunkIds.size();
+ return delegate.deleteChunks(chunkIds, maxLastModifiedTime);
+ }
+
+ @Override @CheckForNull public String getBlobId(@Nonnull String reference) {
+ return delegate.getBlobId(reference);
+ }
+
+ @Override @CheckForNull public String getReference(@Nonnull String blobId) {
+ return delegate.getReference(blobId);
+ }
+
+ @Override public long countDeleteChunks(List<String> chunkIds, long maxLastModifiedTime) throws Exception {
+ long numDeleted = delegate.countDeleteChunks(chunkIds, maxLastModifiedTime);
+ numChunks -= numDeleted;
+ return numDeleted;
+ }
+
+ @Override public Iterator<String> resolveChunks(String blobId) throws IOException {
+ return delegate.resolveChunks(blobId);
+ }
+
+ @Override public void addTracker(BlobTracker tracker) {
+ if (delegate instanceof BlobTrackingStore) {
+ ((BlobTrackingStore) delegate).addTracker(tracker);
+ }
+ }
+
+ @Override public BlobTracker getTracker() {
+ if (delegate instanceof BlobTrackingStore) {
+ return ((BlobTrackingStore) delegate).getTracker();
+ }
+ return null;
+ }
+
+ @Override public void addMetadataRecord(InputStream stream, String name) throws DataStoreException {
+ if (delegate instanceof BlobTrackingStore) {
+ ((BlobTrackingStore) delegate).addMetadataRecord(stream, name);
+ }
+ }
+
+ @Override public void addMetadataRecord(File f, String name) throws DataStoreException {
+ if (delegate instanceof BlobTrackingStore) {
+ ((BlobTrackingStore) delegate).addMetadataRecord(f, name);
+ }
+ }
+
+ @Override public DataRecord getMetadataRecord(String name) {
+ if (delegate instanceof BlobTrackingStore) {
+ return ((BlobTrackingStore) delegate).getMetadataRecord(name);
+ }
+ return null;
+ }
+
+ @Override public List<DataRecord> getAllMetadataRecords(String prefix) {
+ if (delegate instanceof BlobTrackingStore) {
+ return ((BlobTrackingStore) delegate).getAllMetadataRecords(prefix);
+ }
+ return null;
+ }
+
+ @Override public boolean deleteMetadataRecord(String name) {
+ if (delegate instanceof BlobTrackingStore) {
+ ((BlobTrackingStore) delegate).deleteMetadataRecord(name);
+ }
+ return false;
+ }
+
+ @Override public void deleteAllMetadataRecords(String prefix) {
+ if (delegate instanceof BlobTrackingStore) {
+ ((BlobTrackingStore) delegate).deleteAllMetadataRecords(prefix);
+ }
+ }
+
+ @Override public Iterator<DataRecord> getAllRecords() throws DataStoreException {
+ if (delegate instanceof BlobTrackingStore) {
+ return ((BlobTrackingStore) delegate).getAllRecords();
+ }
+ return Iterators.emptyIterator();
+ }
+
+ @Override public DataRecord getRecordForId(DataIdentifier id) throws DataStoreException {
+ if (delegate instanceof BlobTrackingStore) {
+ return ((BlobTrackingStore) delegate).getRecordForId(id);
+ }
+ return null;
+ }
+
+ @Override public Type getType() {
+ if (delegate instanceof BlobTrackingStore) {
+ ((BlobTrackingStore) delegate).getType();
+ }
+ return Type.DEFAULT;
+ }
+ }
+}
Propchange: jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/AbstractActiveDeletedBlobTest.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/ActiveDeletedBlobCollectionIT.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/ActiveDeletedBlobCollectionIT.java?rev=1804626&r1=1804625&r2=1804626&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/ActiveDeletedBlobCollectionIT.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/ActiveDeletedBlobCollectionIT.java Thu Aug 10 03:52:24 2017
@@ -16,18 +16,12 @@
*/
package org.apache.jackrabbit.oak.plugins.index.lucene.directory;
-import com.google.common.collect.Iterators;
import com.google.common.collect.Lists;
import org.apache.commons.io.FileUtils;
-import org.apache.jackrabbit.JcrConstants;
import org.apache.jackrabbit.core.data.FileDataStore;
import org.apache.jackrabbit.oak.InitialContent;
import org.apache.jackrabbit.oak.Oak;
-import org.apache.jackrabbit.oak.api.CommitFailedException;
import org.apache.jackrabbit.oak.api.ContentRepository;
-import org.apache.jackrabbit.oak.api.Tree;
-import org.apache.jackrabbit.oak.api.Type;
-import org.apache.jackrabbit.oak.commons.concurrent.ExecutorCloser;
import org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore;
import org.apache.jackrabbit.oak.plugins.document.DocumentMK;
import org.apache.jackrabbit.oak.plugins.document.MongoConnectionFactory;
@@ -37,76 +31,35 @@ import org.apache.jackrabbit.oak.plugins
import org.apache.jackrabbit.oak.plugins.index.AsyncIndexUpdate;
import org.apache.jackrabbit.oak.plugins.index.lucene.ExtractedTextCache;
import org.apache.jackrabbit.oak.plugins.index.lucene.IndexCopier;
-import org.apache.jackrabbit.oak.plugins.index.lucene.IndexDefinition;
-import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants;
import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexEditorProvider;
import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexProvider;
import org.apache.jackrabbit.oak.plugins.index.lucene.directory.ActiveDeletedBlobCollectorFactory.ActiveDeletedBlobCollectorImpl;
-import org.apache.jackrabbit.oak.plugins.memory.PropertyStates;
-import org.apache.jackrabbit.oak.query.AbstractQueryTest;
-import org.apache.jackrabbit.oak.spi.blob.BlobOptions;
-import org.apache.jackrabbit.oak.spi.blob.GarbageCollectableBlobStore;
+
import org.apache.jackrabbit.oak.spi.commit.Observer;
import org.apache.jackrabbit.oak.spi.mount.Mounts;
import org.apache.jackrabbit.oak.spi.query.QueryIndexProvider;
import org.apache.jackrabbit.oak.spi.security.OpenSecurityProvider;
-import org.apache.jackrabbit.oak.spi.state.NodeStore;
-import org.apache.jackrabbit.oak.stats.Clock;
-import org.junit.After;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
-import javax.annotation.CheckForNull;
-import javax.annotation.Nonnull;
import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
import java.util.Collection;
-import java.util.Iterator;
import java.util.List;
-import java.util.Set;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
import static com.google.common.collect.ImmutableSet.of;
-import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.ASYNC_PROPERTY_NAME;
-import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.INDEX_DEFINITIONS_NAME;
-import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.INDEX_DEFINITIONS_NODE_TYPE;
-import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.REINDEX_PROPERTY_NAME;
-import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.TYPE_PROPERTY_NAME;
import static org.junit.Assume.assumeTrue;
@RunWith(Parameterized.class)
-public class ActiveDeletedBlobCollectionIT extends AbstractQueryTest {
- private ExecutorService executorService = Executors.newFixedThreadPool(2);
+public class ActiveDeletedBlobCollectionIT extends AbstractActiveDeletedBlobTest {
@Rule
- public TemporaryFolder temporaryFolder = new TemporaryFolder(new File("target"));
- @Rule
- public TemporaryFolder blobCollectionRoot = new TemporaryFolder(new File("target"));
- @Rule
- public TemporaryFolder fileDataStoreRoot = new TemporaryFolder(new File("target"));
- @Rule
public MongoConnectionFactory connectionFactory = new MongoConnectionFactory();
private MongoConnection mongoConnection = null;
- private CountingBlobStore blobStore = null;
-
- private Clock clock = new Clock.Virtual();
- private ActiveDeletedBlobCollectorImpl adbc = null;
-
- private AsyncIndexUpdate asyncIndexUpdate;
-
- private LuceneIndexEditorProvider editorProvider;
-
- private NodeStore nodeStore;
-
- private LuceneIndexProvider provider;
private final DataStoreType dataStoreType;
@@ -115,13 +68,6 @@ public class ActiveDeletedBlobCollection
assumeTrue(MongoUtils.isAvailable());
}
- @After
- public void after() {
- new ExecutorCloser(executorService).close();
- executorService.shutdown();
- IndexDefinition.setDisableStoredIndexDefinition(false);
- }
-
enum DataStoreType {
WITH_FDS,
WITHOUT_FDS
@@ -140,11 +86,6 @@ public class ActiveDeletedBlobCollection
}
@Override
- protected void createTestIndexNode() throws Exception {
- setTraversalEnabled(false);
- }
-
- @Override
protected ContentRepository createRepository() {
adbc = new ActiveDeletedBlobCollectorImpl(clock,
new File(blobCollectionRoot.getRoot(), "deleted-blobs"), executorService);
@@ -182,14 +123,6 @@ public class ActiveDeletedBlobCollection
.createContentRepository();
}
- private IndexCopier createIndexCopier() {
- try {
- return new IndexCopier(executorService, temporaryFolder.getRoot());
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
@Test
public void simpleAsyncIndexUpdateBasedBlobCollection() throws Exception {
createIndex("test1", of("propa"));
@@ -228,133 +161,4 @@ public class ActiveDeletedBlobCollection
Assert.assertTrue("Second GC should delete some chunks too", secondGCNumChunks < firstGCNumChunks);
}
- private Tree createIndex(String name, Set<String> propNames) throws CommitFailedException {
- Tree index = root.getTree("/");
- return createIndex(index, name, propNames);
- }
-
- public static Tree createIndex(Tree index, String name, Set<String> propNames) throws CommitFailedException {
- Tree def = index.addChild(INDEX_DEFINITIONS_NAME).addChild(name);
- def.setProperty(JcrConstants.JCR_PRIMARYTYPE,
- INDEX_DEFINITIONS_NODE_TYPE, Type.NAME);
- def.setProperty(TYPE_PROPERTY_NAME, LuceneIndexConstants.TYPE_LUCENE);
- def.setProperty(REINDEX_PROPERTY_NAME, true);
- def.setProperty(ASYNC_PROPERTY_NAME, "async");
- def.setProperty(LuceneIndexConstants.FULL_TEXT_ENABLED, false);
- def.setProperty(PropertyStates.createProperty(LuceneIndexConstants.INCLUDE_PROPERTY_NAMES, propNames, Type.STRINGS));
- def.setProperty(LuceneIndexConstants.SAVE_DIR_LISTING, true);
- return index.getChild(INDEX_DEFINITIONS_NAME).getChild(name);
- }
-
- class CountingBlobStore implements GarbageCollectableBlobStore {
-
- private final GarbageCollectableBlobStore delegate;
- private long numChunks = 0;
-
- CountingBlobStore(GarbageCollectableBlobStore delegate) {
- this.delegate = delegate;
- }
-
- @Override
- public String writeBlob(InputStream in) throws IOException {
- String blobId = delegate.writeBlob(in);
- numChunks += Iterators.size(delegate.resolveChunks(blobId));
- return blobId;
- }
-
- @Override
- public void setBlockSize(int x) {
- delegate.setBlockSize(x);
- }
-
- @Override
- public String writeBlob(InputStream in, BlobOptions options) throws IOException {
- String blobId = delegate.writeBlob(in, options);
- numChunks += Iterators.size(delegate.resolveChunks(blobId));
- return blobId;
- }
-
- @Override
- public String writeBlob(String tempFileName) throws IOException {
- String blobId = delegate.writeBlob(tempFileName);
- numChunks += Iterators.size(delegate.resolveChunks(blobId));
- return blobId;
- }
-
- @Override
- public int sweep() throws IOException {
- return delegate.sweep();
- }
-
- @Override
- public void startMark() throws IOException {
- delegate.startMark();
- }
-
- @Override
- public int readBlob(String blobId, long pos, byte[] buff, int off, int length) throws IOException {
- return delegate.readBlob(blobId, pos, buff, off, length);
- }
-
- @Override
- public void clearInUse() {
- delegate.clearInUse();
- }
-
- @Override
- public void clearCache() {
- delegate.clearCache();
- }
-
- @Override
- public long getBlobLength(String blobId) throws IOException {
- return delegate.getBlobLength(blobId);
- }
-
- @Override
- public long getBlockSizeMin() {
- return delegate.getBlockSizeMin();
- }
-
- @Override
- public Iterator<String> getAllChunkIds(long maxLastModifiedTime) throws Exception {
- return delegate.getAllChunkIds(maxLastModifiedTime);
- }
-
- @Override
- public InputStream getInputStream(String blobId) throws IOException {
- return delegate.getInputStream(blobId);
- }
-
- @Override
- @Deprecated
- public boolean deleteChunks(List<String> chunkIds, long maxLastModifiedTime) throws Exception {
- numChunks -= chunkIds.size();
- return delegate.deleteChunks(chunkIds, maxLastModifiedTime);
- }
-
- @Override
- @CheckForNull
- public String getBlobId(@Nonnull String reference) {
- return delegate.getBlobId(reference);
- }
-
- @Override
- @CheckForNull
- public String getReference(@Nonnull String blobId) {
- return delegate.getReference(blobId);
- }
-
- @Override
- public long countDeleteChunks(List<String> chunkIds, long maxLastModifiedTime) throws Exception {
- long numDeleted = delegate.countDeleteChunks(chunkIds, maxLastModifiedTime);
- numChunks -= numDeleted;
- return numDeleted;
- }
-
- @Override
- public Iterator<String> resolveChunks(String blobId) throws IOException {
- return delegate.resolveChunks(blobId);
- }
- }
}
Added: jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/ActiveDeletedBlobSyncTrackerTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/ActiveDeletedBlobSyncTrackerTest.java?rev=1804626&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/ActiveDeletedBlobSyncTrackerTest.java (added)
+++ jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/ActiveDeletedBlobSyncTrackerTest.java Thu Aug 10 03:52:24 2017
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.index.lucene.directory;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.jackrabbit.oak.InitialContent;
+import org.apache.jackrabbit.oak.Oak;
+import org.apache.jackrabbit.oak.api.ContentRepository;
+import org.apache.jackrabbit.oak.plugins.blob.BlobTrackingStore;
+import org.apache.jackrabbit.oak.plugins.blob.datastore.BlobIdTracker;
+import org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore;
+import org.apache.jackrabbit.oak.plugins.blob.datastore.OakFileDataStore;
+import org.apache.jackrabbit.oak.plugins.index.AsyncIndexUpdate;
+import org.apache.jackrabbit.oak.plugins.index.lucene.ExtractedTextCache;
+import org.apache.jackrabbit.oak.plugins.index.lucene.IndexCopier;
+import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexEditorProvider;
+import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexProvider;
+import org.apache.jackrabbit.oak.plugins.index.lucene.directory.ActiveDeletedBlobCollectorFactory
+ .ActiveDeletedBlobCollectorImpl;
+import org.apache.jackrabbit.oak.segment.SegmentNodeStoreBuilders;
+import org.apache.jackrabbit.oak.segment.file.FileStore;
+import org.apache.jackrabbit.oak.segment.file.FileStoreBuilder;
+import org.apache.jackrabbit.oak.spi.blob.GarbageCollectableBlobStore;
+import org.apache.jackrabbit.oak.spi.commit.Observer;
+import org.apache.jackrabbit.oak.spi.mount.Mounts;
+import org.apache.jackrabbit.oak.spi.query.QueryIndexProvider;
+import org.apache.jackrabbit.oak.spi.security.OpenSecurityProvider;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import static com.google.common.collect.ImmutableSet.of;
+import static com.google.common.collect.Lists.newArrayList;
+import static com.google.common.collect.Sets.intersection;
+import static com.google.common.collect.Sets.newHashSet;
+import static org.apache.jackrabbit.oak.spi.cluster.ClusterRepositoryInfo.getOrCreateId;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class ActiveDeletedBlobSyncTrackerTest extends AbstractActiveDeletedBlobTest {
+ @Rule
+ public TemporaryFolder blobTrackerRoot = new TemporaryFolder(new File("target"));
+
+ @Override
+ protected ContentRepository createRepository() {
+ try {
+ File blobCollectorDeleted = new File(blobCollectionRoot.getRoot(), "deleted-blobs");
+ blobCollectorDeleted.mkdirs();
+ adbc = new ActiveDeletedBlobCollectorImpl(clock, new File(blobCollectionRoot.getRoot(), "deleted-blobs"),
+ executorService);
+
+ IndexCopier copier = createIndexCopier();
+ editorProvider =
+ new LuceneIndexEditorProvider(copier, null, new ExtractedTextCache(10 * FileUtils.ONE_MB,
+ 100), null,
+ Mounts.defaultMountInfoProvider(), adbc);
+ provider = new LuceneIndexProvider(copier);
+
+ OakFileDataStore ds = new OakFileDataStore();
+ ds.setMinRecordLength(10);
+ ds.init(fileDataStoreRoot.getRoot().getAbsolutePath());
+ DataStoreBlobStore dsbs = new DataStoreBlobStore(ds);
+ this.blobStore = new AbstractActiveDeletedBlobTest.CountingBlobStore(dsbs);
+
+ FileStore store = FileStoreBuilder.fileStoreBuilder(temporaryFolder.getRoot()).withMemoryMapping(false)
+ .withBlobStore(blobStore).build();
+ nodeStore = SegmentNodeStoreBuilders.builder(store).build();
+ BlobTrackingStore trackingStore = (BlobTrackingStore) blobStore;
+ trackingStore.addTracker(
+ new BlobIdTracker(blobTrackerRoot.getRoot().getAbsolutePath(), getOrCreateId(nodeStore), 600,
+ dsbs));
+ // set the blob store to skip writing blobs through the node store
+ editorProvider.setBlobStore(blobStore);
+
+ asyncIndexUpdate = new AsyncIndexUpdate("async", nodeStore, editorProvider);
+ return new Oak(nodeStore).with(new InitialContent()).with(new OpenSecurityProvider())
+ .with((QueryIndexProvider) provider).with((Observer) provider).with(editorProvider)
+ .createContentRepository();
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ // OAK-6504
+ @Test
+ public void syncActiveDeletionWithBlobTracker() throws Exception {
+ createIndex("test1", of("propa"));
+ root.getTree("/oak:index/counter").remove();
+ root.commit();
+ asyncIndexUpdate.run();
+ long initialNumChunks = blobStore.numChunks;
+
+ root.getTree("/").addChild("test").setProperty("propa", "foo");
+ root.commit();
+ asyncIndexUpdate.run();
+ long firstCommitNumChunks = blobStore.numChunks;
+ adbc.purgeBlobsDeleted(0, blobStore);//hack to purge file
+ long time = clock.getTimeIncreasing();
+ long hackPurgeNumChunks = blobStore.numChunks;
+ assertEquals("Hack purge must not purge any blob (first commit)",
+ firstCommitNumChunks, hackPurgeNumChunks);
+
+ root.getTree("/").addChild("test").setProperty("propa", "foo1");
+ root.commit();
+ asyncIndexUpdate.run();
+ long secondCommitNumChunks = blobStore.numChunks;
+ adbc.purgeBlobsDeleted(0, blobStore);//hack to purge file
+ hackPurgeNumChunks = blobStore.numChunks;
+ assertEquals("Hack purge must not purge any blob (second commit)",
+ secondCommitNumChunks, hackPurgeNumChunks);
+
+ List<String> beforeDeletionIds = newArrayList(blobStore.getAllChunkIds(-1));
+
+ adbc.purgeBlobsDeleted(time, blobStore);
+ adbc.getBlobDeletionCallback();
+ long firstGCNumChunks = blobStore.numChunks;
+
+ assertTrue("First commit must create some chunks", firstCommitNumChunks > initialNumChunks);
+ assertTrue("First commit must create some chunks", secondCommitNumChunks > firstCommitNumChunks);
+ assertTrue("First GC should delete some chunks", firstGCNumChunks < secondCommitNumChunks);
+
+ List<String> afterDeletionIds = newArrayList(blobStore.getAllChunkIds(-1));
+
+ assertTrackedDeleted(beforeDeletionIds, afterDeletionIds, blobStore);
+ }
+
+ private static void assertTrackedDeleted(List<String> beforeDeletionIds,
+ List<String> afterDeletionIds,
+ GarbageCollectableBlobStore blobStore) throws IOException {
+
+ // get the currently tracked ones
+ ArrayList<String> trackedIds = newArrayList(((BlobTrackingStore) blobStore).getTracker().get());
+ assertTrue("Tracked ids different from current blob list",
+ newHashSet(trackedIds).equals(newHashSet(afterDeletionIds)));
+
+ // get the deleted ones
+ beforeDeletionIds.removeAll(afterDeletionIds);
+ // tracked should not contain the deleted
+ assertTrue("Blob tracker not synced",
+ intersection(newHashSet(beforeDeletionIds), newHashSet(trackedIds)).isEmpty());
+ }
+}
Propchange: jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/ActiveDeletedBlobSyncTrackerTest.java
------------------------------------------------------------------------------
svn:eol-style = native