You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by rn...@apache.org on 2024/10/25 14:44:49 UTC
(couchdb) 01/01: untidy wip
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a commit to branch nouveau-snapshots
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 431c06644590d4648635faeb73d804fe91b4b35a
Author: Robert Newson <rn...@apache.org>
AuthorDate: Thu Oct 24 19:52:42 2024 +0100
untidy wip
---
.../apache/couchdb/nouveau/NouveauApplication.java | 2 +
.../org/apache/couchdb/nouveau/core/Index.java | 2 +
.../apache/couchdb/nouveau/core/IndexManager.java | 6 +++
.../couchdb/nouveau/lucene9/Lucene9Index.java | 8 ++++
.../apache/couchdb/nouveau/tasks/SnapshotTask.java | 47 ++++++++++++++++++++++
.../couchdb/nouveau/lucene9/Lucene9IndexTest.java | 19 +++++++++
6 files changed, 84 insertions(+)
diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/NouveauApplication.java b/nouveau/src/main/java/org/apache/couchdb/nouveau/NouveauApplication.java
index 7179eadc0..1e2163d2a 100644
--- a/nouveau/src/main/java/org/apache/couchdb/nouveau/NouveauApplication.java
+++ b/nouveau/src/main/java/org/apache/couchdb/nouveau/NouveauApplication.java
@@ -29,6 +29,7 @@ import org.apache.couchdb.nouveau.lucene9.ParallelSearcherFactory;
import org.apache.couchdb.nouveau.resources.AnalyzeResource;
import org.apache.couchdb.nouveau.resources.IndexResource;
import org.apache.couchdb.nouveau.tasks.CloseAllIndexesTask;
+import org.apache.couchdb.nouveau.tasks.SnapshotTask;
public class NouveauApplication extends Application<NouveauApplicationConfiguration> {
@@ -81,6 +82,7 @@ public class NouveauApplication extends Application<NouveauApplicationConfigurat
// configure tasks
environment.admin().addTask(new CloseAllIndexesTask(indexManager));
+ environment.admin().addTask(new SnapshotTask(indexManager));
// Swagger
environment.jersey().register(new OpenApiResource());
diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/core/Index.java b/nouveau/src/main/java/org/apache/couchdb/nouveau/core/Index.java
index 69c9bd48f..3c070caf9 100644
--- a/nouveau/src/main/java/org/apache/couchdb/nouveau/core/Index.java
+++ b/nouveau/src/main/java/org/apache/couchdb/nouveau/core/Index.java
@@ -98,6 +98,8 @@ public abstract class Index implements Closeable {
protected abstract boolean doCommit(final long updateSeq, final long purgeSeq) throws IOException;
+ public abstract java.util.Collection<String> snapshot() throws IOException;
+
public final synchronized void setUpdateSeq(final long matchSeq, final long updateSeq) throws IOException {
if (updateSeq < this.updateSeq) {
throw new WebApplicationException(
diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/core/IndexManager.java b/nouveau/src/main/java/org/apache/couchdb/nouveau/core/IndexManager.java
index d4a0e8f10..5a928f13f 100644
--- a/nouveau/src/main/java/org/apache/couchdb/nouveau/core/IndexManager.java
+++ b/nouveau/src/main/java/org/apache/couchdb/nouveau/core/IndexManager.java
@@ -38,8 +38,11 @@ import org.apache.couchdb.nouveau.api.IndexDefinition;
import org.apache.couchdb.nouveau.lucene9.Lucene9AnalyzerFactory;
import org.apache.couchdb.nouveau.lucene9.Lucene9Index;
import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.index.IndexDeletionPolicy;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.KeepOnlyLastCommitDeletionPolicy;
+import org.apache.lucene.index.PersistentSnapshotDeletionPolicy;
import org.apache.lucene.misc.store.DirectIODirectory;
import org.apache.lucene.search.SearcherFactory;
import org.apache.lucene.search.SearcherManager;
@@ -373,6 +376,9 @@ public final class IndexManager implements Managed {
final Directory dir = new DirectIODirectory(FSDirectory.open(path.resolve("9")));
final IndexWriterConfig config = new IndexWriterConfig(analyzer);
config.setUseCompoundFile(false);
+ final IndexDeletionPolicy indexDeletionPolicy =
+ new PersistentSnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy(), dir);
+ config.setIndexDeletionPolicy(indexDeletionPolicy);
final IndexWriter writer = new IndexWriter(dir, config);
final long updateSeq = getSeq(writer, "update_seq");
final long purgeSeq = getSeq(writer, "purge_seq");
diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/Lucene9Index.java b/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/Lucene9Index.java
index e4e8402e7..9eff6c849 100644
--- a/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/Lucene9Index.java
+++ b/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/Lucene9Index.java
@@ -25,6 +25,7 @@ import java.nio.charset.CharsetDecoder;
import java.nio.file.NoSuchFileException;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
@@ -70,6 +71,7 @@ import org.apache.lucene.facet.range.DoubleRangeFacetCounts;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.PersistentSnapshotDeletionPolicy;
import org.apache.lucene.index.StoredFields;
import org.apache.lucene.index.Term;
import org.apache.lucene.queryparser.flexible.core.QueryNodeException;
@@ -216,6 +218,12 @@ public class Lucene9Index extends Index {
}
}
+ public Collection<String> snapshot() throws IOException {
+ final PersistentSnapshotDeletionPolicy policy =
+ (PersistentSnapshotDeletionPolicy) writer.getConfig().getIndexDeletionPolicy();
+ return policy.snapshot().getFileNames();
+ }
+
private CollectorManager<?, ? extends TopDocs> hitCollector(final SearchRequest searchRequest) {
final Sort sort = toSort(searchRequest);
diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/tasks/SnapshotTask.java b/nouveau/src/main/java/org/apache/couchdb/nouveau/tasks/SnapshotTask.java
new file mode 100644
index 000000000..1b24cc216
--- /dev/null
+++ b/nouveau/src/main/java/org/apache/couchdb/nouveau/tasks/SnapshotTask.java
@@ -0,0 +1,47 @@
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.apache.couchdb.nouveau.tasks;
+
+import io.dropwizard.servlets.tasks.Task;
+import java.io.PrintWriter;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import org.apache.couchdb.nouveau.core.IndexManager;
+
+public class SnapshotTask extends Task {
+
+ private final IndexManager indexManager;
+
+ public SnapshotTask(final IndexManager indexManager) {
+ super("snapshot");
+ this.indexManager = indexManager;
+ }
+
+ @Override
+ public void execute(Map<String, List<String>> parameters, PrintWriter output) throws Exception {
+ if (!parameters.containsKey("indexes")) {
+ throw new IllegalArgumentException("'indexes' parameter is required");
+ }
+ for (final String name : parameters.get("indexes")) {
+ output.format("%s\n", name);
+ final Collection<String> files = indexManager.with(name, (index) -> {
+ return index.snapshot();
+ });
+ for (final String file : files) {
+ output.format("%s\n", file);
+ }
+ }
+ }
+}
diff --git a/nouveau/src/test/java/org/apache/couchdb/nouveau/lucene9/Lucene9IndexTest.java b/nouveau/src/test/java/org/apache/couchdb/nouveau/lucene9/Lucene9IndexTest.java
index 428e3eb6a..6bdfcf555 100644
--- a/nouveau/src/test/java/org/apache/couchdb/nouveau/lucene9/Lucene9IndexTest.java
+++ b/nouveau/src/test/java/org/apache/couchdb/nouveau/lucene9/Lucene9IndexTest.java
@@ -36,6 +36,7 @@ import org.apache.couchdb.nouveau.core.Index;
import org.apache.couchdb.nouveau.core.StaleIndexException;
import org.apache.couchdb.nouveau.core.UpdatesOutOfOrderException;
import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.index.*;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.misc.store.DirectIODirectory;
@@ -54,6 +55,8 @@ public class Lucene9IndexTest {
final Directory dir = new DirectIODirectory(FSDirectory.open(path));
final IndexWriterConfig config = new IndexWriterConfig(analyzer);
config.setUseCompoundFile(false);
+ config.setIndexDeletionPolicy(
+ new PersistentSnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy(), dir));
final IndexWriter writer = new IndexWriter(dir, config);
final SearcherManager searcherManager = new SearcherManager(writer, null);
return new Lucene9Index(analyzer, writer, 0L, 0L, searcherManager);
@@ -290,4 +293,20 @@ public class Lucene9IndexTest {
cleanup(index);
}
}
+
+ @Test
+ public void testSnapshot(@TempDir Path path) throws IOException {
+ Index index = setup(path);
+ try {
+ final Collection<Field> fields = List.of(new DoubleField("bar", 12.0, false));
+ index.update("foo", new DocumentUpdateRequest(0, 2, null, fields));
+ index.commit();
+
+ var files = index.snapshot();
+
+ assertThat(files).isEqualTo(1);
+ } finally {
+ cleanup(index);
+ }
+ }
}