You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@solr.apache.org by br...@apache.org on 2023/08/14 09:25:27 UTC
[solr-sandbox] branch main updated: Cleanup EncryptionUpdateHandler by upgrading to solr-core:9.3.0. (#64)
This is an automated email from the ASF dual-hosted git repository.
broustant pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/solr-sandbox.git
The following commit(s) were added to refs/heads/main by this push:
new bb18cca Cleanup EncryptionUpdateHandler by upgrading to solr-core:9.3.0. (#64)
bb18cca is described below
commit bb18cca7178d22714a2338a8ee940e5e31526388
Author: Bruno Roustant <33...@users.noreply.github.com>
AuthorDate: Mon Aug 14 11:25:22 2023 +0200
Cleanup EncryptionUpdateHandler by upgrading to solr-core:9.3.0. (#64)
---
encryption/build.gradle | 11 +-
.../solr/encryption/EncryptionUpdateHandler.java | 5 +-
.../solr/update/DirectUpdateHandler2Copy.java | 1110 --------------------
3 files changed, 6 insertions(+), 1120 deletions(-)
diff --git a/encryption/build.gradle b/encryption/build.gradle
index d6db4e3..1e5ccdb 100644
--- a/encryption/build.gradle
+++ b/encryption/build.gradle
@@ -33,15 +33,12 @@ sourceSets {
}
dependencies {
- implementation 'org.apache.solr:solr-core:9.2.1'
- implementation 'org.apache.lucene:lucene-core:9.5.0'
+ implementation 'org.apache.solr:solr-core:9.3.0'
+ implementation 'org.apache.lucene:lucene-core:9.7.0'
implementation 'com.google.code.findbugs:jsr305:3.0.2'
- // Remove when removing DirectUpdateHandler2Copy.
- implementation 'javax.servlet:javax.servlet-api:3.1.0'
-
- testImplementation 'org.apache.solr:solr-test-framework:9.2.1'
- testImplementation 'org.apache.lucene:lucene-test-framework:9.5.0'
+ testImplementation 'org.apache.solr:solr-test-framework:9.3.0'
+ testImplementation 'org.apache.lucene:lucene-test-framework:9.7.0'
}
test {
diff --git a/encryption/src/main/java/org/apache/solr/encryption/EncryptionUpdateHandler.java b/encryption/src/main/java/org/apache/solr/encryption/EncryptionUpdateHandler.java
index 95d1c99..8b86d5f 100644
--- a/encryption/src/main/java/org/apache/solr/encryption/EncryptionUpdateHandler.java
+++ b/encryption/src/main/java/org/apache/solr/encryption/EncryptionUpdateHandler.java
@@ -19,7 +19,7 @@ package org.apache.solr.encryption;
import org.apache.lucene.index.IndexWriter;
import org.apache.solr.core.SolrCore;
import org.apache.solr.update.CommitUpdateCommand;
-import org.apache.solr.update.DirectUpdateHandler2Copy;
+import org.apache.solr.update.DirectUpdateHandler2;
import org.apache.solr.update.UpdateHandler;
import java.io.IOException;
@@ -32,8 +32,7 @@ import static org.apache.solr.encryption.CommitUtil.readLatestCommit;
* Extends {@link org.apache.solr.update.DirectUpdateHandler2} and adds the capability
* to transfer some transferable user data from the previous commit to the next one.
*/
-//TODO: extend DirectUpdateHandler2 once Solr 9.2 is available
-public class EncryptionUpdateHandler extends DirectUpdateHandler2Copy {
+public class EncryptionUpdateHandler extends DirectUpdateHandler2 {
/**
* Parameter prefix to state that this parameter should be transferred from a commit
diff --git a/encryption/src/main/java/org/apache/solr/update/DirectUpdateHandler2Copy.java b/encryption/src/main/java/org/apache/solr/update/DirectUpdateHandler2Copy.java
deleted file mode 100644
index a186db6..0000000
--- a/encryption/src/main/java/org/apache/solr/update/DirectUpdateHandler2Copy.java
+++ /dev/null
@@ -1,1110 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.update;
-
-import com.codahale.metrics.Meter;
-import java.io.IOException;
-import java.lang.invoke.MethodHandles;
-import java.lang.reflect.Array;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Locale;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Future;
-import java.util.concurrent.atomic.LongAdder;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.index.CodecReader;
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexableField;
-import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.index.SlowCodecReaderWrapper;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.queries.function.ValueSource;
-import org.apache.lucene.search.BooleanClause.Occur;
-import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.MatchAllDocsQuery;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.store.AlreadyClosedException;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.BytesRefHash;
-import org.apache.solr.cloud.ZkController;
-import org.apache.solr.common.SolrException;
-import org.apache.solr.common.cloud.Replica;
-import org.apache.solr.common.params.ModifiableSolrParams;
-import org.apache.solr.common.util.NamedList;
-import org.apache.solr.core.SolrConfig.UpdateHandlerInfo;
-import org.apache.solr.core.SolrCore;
-import org.apache.solr.metrics.SolrMetricProducer;
-import org.apache.solr.metrics.SolrMetricsContext;
-import org.apache.solr.request.LocalSolrQueryRequest;
-import org.apache.solr.request.SolrQueryRequest;
-import org.apache.solr.request.SolrRequestInfo;
-import org.apache.solr.response.SolrQueryResponse;
-import org.apache.solr.schema.IndexSchema;
-import org.apache.solr.schema.SchemaField;
-import org.apache.solr.search.FunctionRangeQuery;
-import org.apache.solr.search.QParser;
-import org.apache.solr.search.QueryUtils;
-import org.apache.solr.search.SolrIndexSearcher;
-import org.apache.solr.search.SyntaxError;
-import org.apache.solr.search.function.ValueSourceRangeFilter;
-import org.apache.solr.util.RefCounted;
-import org.apache.solr.util.TestInjection;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-//TODO: remove this DirectUpdateHandler2 copy once Solr 9.2 is available.
-// Its only modification is the shouldCommit() method that will be available in 9.2.
-/**
- * <code>DirectUpdateHandler2</code> implements an UpdateHandler where documents are added directly
- * to the main Lucene index as opposed to adding to a separate smaller index.
- */
-public class DirectUpdateHandler2Copy extends UpdateHandler
- implements SolrCoreState.IndexWriterCloser, SolrMetricProducer {
-
- private static final int NO_FILE_SIZE_UPPER_BOUND_PLACEHOLDER = -1;
-
- protected final SolrCoreState solrCoreState;
-
- // stats
- LongAdder addCommands = new LongAdder();
- Meter addCommandsCumulative;
- LongAdder deleteByIdCommands = new LongAdder();
- Meter deleteByIdCommandsCumulative;
- LongAdder deleteByQueryCommands = new LongAdder();
- Meter deleteByQueryCommandsCumulative;
- Meter expungeDeleteCommands;
- Meter mergeIndexesCommands;
- Meter commitCommands;
- Meter splitCommands;
- Meter optimizeCommands;
- Meter rollbackCommands;
- LongAdder numDocsPending = new LongAdder();
- LongAdder numErrors = new LongAdder();
- Meter numErrorsCumulative;
-
- // tracks when auto-commit should occur
- protected final CommitTracker commitTracker;
- protected final CommitTracker softCommitTracker;
-
- protected boolean commitWithinSoftCommit;
- /**
- * package access for testing
- *
- * @lucene.internal
- */
- void setCommitWithinSoftCommit(boolean value) {
- this.commitWithinSoftCommit = value;
- }
-
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- public DirectUpdateHandler2Copy(SolrCore core) {
- super(core);
-
- solrCoreState = core.getSolrCoreState();
-
- UpdateHandlerInfo updateHandlerInfo = core.getSolrConfig().getUpdateHandlerInfo();
- int docsUpperBound = updateHandlerInfo.autoCommmitMaxDocs;
- int timeUpperBound = updateHandlerInfo.autoCommmitMaxTime;
- long fileSizeUpperBound = updateHandlerInfo.autoCommitMaxSizeBytes;
- commitTracker =
- new CommitTracker(
- "Hard",
- core,
- docsUpperBound,
- timeUpperBound,
- fileSizeUpperBound,
- updateHandlerInfo.openSearcher,
- false);
-
- int softCommitDocsUpperBound = updateHandlerInfo.autoSoftCommmitMaxDocs;
- int softCommitTimeUpperBound = updateHandlerInfo.autoSoftCommmitMaxTime;
- softCommitTracker =
- new CommitTracker(
- "Soft",
- core,
- softCommitDocsUpperBound,
- softCommitTimeUpperBound,
- NO_FILE_SIZE_UPPER_BOUND_PLACEHOLDER,
- true,
- true);
-
- commitWithinSoftCommit = updateHandlerInfo.commitWithinSoftCommit;
-
- ZkController zkController = core.getCoreContainer().getZkController();
- if (zkController != null
- && core.getCoreDescriptor().getCloudDescriptor().getReplicaType() == Replica.Type.TLOG) {
- commitWithinSoftCommit = false;
- commitTracker.setOpenSearcher(true);
- }
- }
-
- public DirectUpdateHandler2Copy(SolrCore core, UpdateHandler updateHandler) {
- super(core, updateHandler.getUpdateLog());
- solrCoreState = core.getSolrCoreState();
-
- UpdateHandlerInfo updateHandlerInfo = core.getSolrConfig().getUpdateHandlerInfo();
- int docsUpperBound = updateHandlerInfo.autoCommmitMaxDocs;
- int timeUpperBound = updateHandlerInfo.autoCommmitMaxTime;
- long fileSizeUpperBound = updateHandlerInfo.autoCommitMaxSizeBytes;
- commitTracker =
- new CommitTracker(
- "Hard",
- core,
- docsUpperBound,
- timeUpperBound,
- fileSizeUpperBound,
- updateHandlerInfo.openSearcher,
- false);
-
- int softCommitDocsUpperBound = updateHandlerInfo.autoSoftCommmitMaxDocs;
- int softCommitTimeUpperBound = updateHandlerInfo.autoSoftCommmitMaxTime;
- softCommitTracker =
- new CommitTracker(
- "Soft",
- core,
- softCommitDocsUpperBound,
- softCommitTimeUpperBound,
- NO_FILE_SIZE_UPPER_BOUND_PLACEHOLDER,
- updateHandlerInfo.openSearcher,
- true);
-
- commitWithinSoftCommit = updateHandlerInfo.commitWithinSoftCommit;
-
- UpdateLog existingLog = updateHandler.getUpdateLog();
- if (this.ulog != null && this.ulog == existingLog) {
- // If we are reusing the existing update log, inform the log that its update handler has
- // changed. We do this as late as possible.
- this.ulog.init(this, core);
- }
- }
-
- @Override
- public void initializeMetrics(SolrMetricsContext parentContext, String scope) {
- solrMetricsContext = parentContext.getChildContext(this);
- commitCommands = solrMetricsContext.meter("commits", getCategory().toString(), scope);
- solrMetricsContext.gauge(
- () -> commitTracker.getCommitCount(), true, "autoCommits", getCategory().toString(), scope);
- solrMetricsContext.gauge(
- () -> softCommitTracker.getCommitCount(),
- true,
- "softAutoCommits",
- getCategory().toString(),
- scope);
- if (commitTracker.getDocsUpperBound() > 0) {
- solrMetricsContext.gauge(
- () -> commitTracker.getDocsUpperBound(),
- true,
- "autoCommitMaxDocs",
- getCategory().toString(),
- scope);
- }
- if (commitTracker.getTimeUpperBound() > 0) {
- solrMetricsContext.gauge(
- () -> "" + commitTracker.getTimeUpperBound() + "ms",
- true,
- "autoCommitMaxTime",
- getCategory().toString(),
- scope);
- }
- if (commitTracker.getTLogFileSizeUpperBound() > 0) {
- solrMetricsContext.gauge(
- () -> commitTracker.getTLogFileSizeUpperBound(),
- true,
- "autoCommitMaxSize",
- getCategory().toString(),
- scope);
- }
- if (softCommitTracker.getDocsUpperBound() > 0) {
- solrMetricsContext.gauge(
- () -> softCommitTracker.getDocsUpperBound(),
- true,
- "softAutoCommitMaxDocs",
- getCategory().toString(),
- scope);
- }
- if (softCommitTracker.getTimeUpperBound() > 0) {
- solrMetricsContext.gauge(
- () -> "" + softCommitTracker.getTimeUpperBound() + "ms",
- true,
- "softAutoCommitMaxTime",
- getCategory().toString(),
- scope);
- }
- optimizeCommands = solrMetricsContext.meter("optimizes", getCategory().toString(), scope);
- rollbackCommands = solrMetricsContext.meter("rollbacks", getCategory().toString(), scope);
- splitCommands = solrMetricsContext.meter("splits", getCategory().toString(), scope);
- mergeIndexesCommands = solrMetricsContext.meter("merges", getCategory().toString(), scope);
- expungeDeleteCommands =
- solrMetricsContext.meter("expungeDeletes", getCategory().toString(), scope);
- solrMetricsContext.gauge(
- () -> numDocsPending.longValue(), true, "docsPending", getCategory().toString(), scope);
- solrMetricsContext.gauge(
- () -> addCommands.longValue(), true, "adds", getCategory().toString(), scope);
- solrMetricsContext.gauge(
- () -> deleteByIdCommands.longValue(), true, "deletesById", getCategory().toString(), scope);
- solrMetricsContext.gauge(
- () -> deleteByQueryCommands.longValue(),
- true,
- "deletesByQuery",
- getCategory().toString(),
- scope);
- solrMetricsContext.gauge(
- () -> numErrors.longValue(), true, "errors", getCategory().toString(), scope);
-
- addCommandsCumulative =
- solrMetricsContext.meter("cumulativeAdds", getCategory().toString(), scope);
- deleteByIdCommandsCumulative =
- solrMetricsContext.meter("cumulativeDeletesById", getCategory().toString(), scope);
- deleteByQueryCommandsCumulative =
- solrMetricsContext.meter("cumulativeDeletesByQuery", getCategory().toString(), scope);
- numErrorsCumulative =
- solrMetricsContext.meter("cumulativeErrors", getCategory().toString(), scope);
- }
-
- private void deleteAll() throws IOException {
- log.info("REMOVING ALL DOCUMENTS FROM INDEX");
- RefCounted<IndexWriter> iw = solrCoreState.getIndexWriter(core);
- try {
- iw.get().deleteAll();
- } finally {
- iw.decref();
- }
- }
-
- protected void rollbackWriter() throws IOException {
- numDocsPending.reset();
- solrCoreState.rollbackIndexWriter(core);
- }
-
- @Override
- public int addDoc(AddUpdateCommand cmd) throws IOException {
- TestInjection.injectDirectUpdateLatch();
- try {
- return addDoc0(cmd);
- } catch (SolrException e) {
- throw e;
- } catch (AlreadyClosedException e) {
- throw new SolrException(
- SolrException.ErrorCode.SERVER_ERROR,
- String.format(
- Locale.ROOT,
- "Server error writing document id %s to the index",
- cmd.getPrintableId()),
- e);
- } catch (IllegalArgumentException iae) {
- throw new SolrException(
- SolrException.ErrorCode.BAD_REQUEST,
- String.format(
- Locale.ROOT,
- "Exception writing document id %s to the index; possible analysis error: "
- + iae.getMessage()
- + (iae.getCause() instanceof BytesRefHash.MaxBytesLengthExceededException
- ? ". Perhaps the document has an indexed string field (solr.StrField) which is too large"
- : ""),
- cmd.getPrintableId()),
- iae);
- } catch (RuntimeException t) {
- throw new SolrException(
- SolrException.ErrorCode.BAD_REQUEST,
- String.format(
- Locale.ROOT,
- "Exception writing document id %s to the index; possible analysis error.",
- cmd.getPrintableId()),
- t);
- }
- }
-
- /**
- * This is the implementation of {@link #addDoc(AddUpdateCommand)}. It is factored out to allow an
- * exception handler to decorate RuntimeExceptions with information about the document being
- * handled.
- *
- * @param cmd the command.
- * @return the count.
- */
- private int addDoc0(AddUpdateCommand cmd) throws IOException {
- int rc = -1;
-
- addCommands.increment();
- addCommandsCumulative.mark();
-
- // if there is no ID field, don't overwrite
- if (idField == null) {
- cmd.overwrite = false;
- }
- try {
- if ((cmd.getFlags() & UpdateCommand.IGNORE_INDEXWRITER) != 0) {
- if (ulog != null) ulog.add(cmd);
- return 1;
- }
-
- if (cmd.overwrite) {
- // Check for delete by query commands newer (i.e. reordered). This
- // should always be null on a leader
- List<UpdateLog.DBQ> deletesAfter = null;
- if (ulog != null && cmd.version > 0) {
- deletesAfter = ulog.getDBQNewer(cmd.version);
- }
-
- if (deletesAfter != null) {
- addAndDelete(cmd, deletesAfter);
- } else {
- doNormalUpdate(cmd);
- }
- } else {
- allowDuplicateUpdate(cmd);
- }
-
- if ((cmd.getFlags() & UpdateCommand.IGNORE_AUTOCOMMIT) == 0) {
- long currentTlogSize = getCurrentTLogSize();
- if (commitWithinSoftCommit) {
- commitTracker.addedDocument(-1, currentTlogSize);
- softCommitTracker.addedDocument(cmd.commitWithin);
- } else {
- softCommitTracker.addedDocument(-1);
- commitTracker.addedDocument(cmd.commitWithin, currentTlogSize);
- }
- }
-
- rc = 1;
- } finally {
- if (rc != 1) {
- numErrors.increment();
- numErrorsCumulative.mark();
- } else {
- numDocsPending.increment();
- }
- }
-
- return rc;
- }
-
- private void allowDuplicateUpdate(AddUpdateCommand cmd) throws IOException {
- RefCounted<IndexWriter> iw = solrCoreState.getIndexWriter(core);
- try {
- IndexWriter writer = iw.get();
- writer.addDocuments(cmd.makeLuceneDocs());
- if (ulog != null) ulog.add(cmd);
-
- } finally {
- iw.decref();
- }
- }
-
- private void doNormalUpdate(AddUpdateCommand cmd) throws IOException {
- RefCounted<IndexWriter> iw = solrCoreState.getIndexWriter(core);
- try {
- IndexWriter writer = iw.get();
-
- updateDocOrDocValues(cmd, writer);
-
- // Add to the transaction log *after* successfully adding to the
- // index, if there was no error.
- // This ordering ensures that if we log it, it's definitely been
- // added to the the index.
- // This also ensures that if a commit sneaks in-between, that we
- // know everything in a particular
- // log version was definitely committed.
- if (ulog != null) ulog.add(cmd);
-
- } finally {
- iw.decref();
- }
- }
-
- private void addAndDelete(AddUpdateCommand cmd, List<UpdateLog.DBQ> deletesAfter)
- throws IOException {
- // this logic is different enough from doNormalUpdate that it's separate
- log.info("Reordered DBQs detected. Update={} DBQs={}", cmd, deletesAfter);
- List<Query> dbqList = new ArrayList<>(deletesAfter.size());
- for (UpdateLog.DBQ dbq : deletesAfter) {
- try {
- DeleteUpdateCommand tmpDel = new DeleteUpdateCommand(cmd.req);
- tmpDel.query = dbq.q;
- tmpDel.version = -dbq.version;
- dbqList.add(getQuery(tmpDel));
- } catch (Exception e) {
- log.error("Exception parsing reordered query : {}", dbq, e);
- }
- }
-
- RefCounted<IndexWriter> iw = solrCoreState.getIndexWriter(core);
- try {
- IndexWriter writer = iw.get();
-
- // see comment in deleteByQuery
- synchronized (solrCoreState.getUpdateLock()) {
- updateDocOrDocValues(cmd, writer);
-
- if (cmd.isInPlaceUpdate() && ulog != null) {
- ulog.openRealtimeSearcher(); // This is needed due to LUCENE-7344.
- }
- for (Query q : dbqList) {
- writer.deleteDocuments(new DeleteByQueryWrapper(q, core.getLatestSchema()));
- }
- if (ulog != null) ulog.add(cmd, true); // this needs to be protected by update lock
- }
- } finally {
- iw.decref();
- }
- }
-
- private void updateDeleteTrackers(DeleteUpdateCommand cmd) {
- if ((cmd.getFlags() & UpdateCommand.IGNORE_AUTOCOMMIT) == 0) {
- if (commitWithinSoftCommit) {
- softCommitTracker.deletedDocument(cmd.commitWithin);
- } else {
- commitTracker.deletedDocument(cmd.commitWithin);
- }
-
- if (commitTracker.getTimeUpperBound() > 0) {
- commitTracker.scheduleCommitWithin(commitTracker.getTimeUpperBound());
- }
-
- long currentTlogSize = getCurrentTLogSize();
- commitTracker.scheduleMaxSizeTriggeredCommitIfNeeded(currentTlogSize);
-
- if (softCommitTracker.getTimeUpperBound() > 0) {
- softCommitTracker.scheduleCommitWithin(softCommitTracker.getTimeUpperBound());
- }
- }
- }
-
- // we don't return the number of docs deleted because it's not always possible to quickly know
- // that info.
- @Override
- public void delete(DeleteUpdateCommand cmd) throws IOException {
- TestInjection.injectDirectUpdateLatch();
- deleteByIdCommands.increment();
- deleteByIdCommandsCumulative.mark();
-
- if ((cmd.getFlags() & UpdateCommand.IGNORE_INDEXWRITER) != 0) {
- if (ulog != null) ulog.delete(cmd);
- return;
- }
-
- Term deleteTerm = getIdTerm(cmd.getIndexedId());
- // SolrCore.verbose("deleteDocuments",deleteTerm,writer);
- RefCounted<IndexWriter> iw = solrCoreState.getIndexWriter(core);
- try {
- iw.get().deleteDocuments(deleteTerm);
- } finally {
- iw.decref();
- }
- // SolrCore.verbose("deleteDocuments",deleteTerm,"DONE");
-
- if (ulog != null) ulog.delete(cmd);
-
- updateDeleteTrackers(cmd);
- }
-
- public void clearIndex() throws IOException {
- deleteAll();
- if (ulog != null) {
- ulog.deleteAll();
- }
- }
-
- private Query getQuery(DeleteUpdateCommand cmd) {
- Query q;
- try {
- // move this higher in the stack?
- QParser parser = QParser.getParser(cmd.getQuery(), cmd.req);
- q = parser.getQuery();
- q = QueryUtils.makeQueryable(q);
-
- // Make sure not to delete newer versions
- if (ulog != null && cmd.getVersion() != 0 && cmd.getVersion() != -Long.MAX_VALUE) {
- BooleanQuery.Builder bq = new BooleanQuery.Builder();
- bq.add(q, Occur.MUST);
- SchemaField sf = ulog.getVersionInfo().getVersionField();
- ValueSource vs = sf.getType().getValueSource(sf, null);
- ValueSourceRangeFilter filt =
- new ValueSourceRangeFilter(
- vs, Long.toString(Math.abs(cmd.getVersion())), null, true, true);
- FunctionRangeQuery range = new FunctionRangeQuery(filt);
- // formulated in the "MUST_NOT" sense so we can delete docs w/o a version (some tests depend
- // on this...)
- bq.add(range, Occur.MUST_NOT);
- q = bq.build();
- }
-
- return q;
-
- } catch (SyntaxError e) {
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
- }
- }
-
- // we don't return the number of docs deleted because it's not always possible to quickly know
- // that info.
- @Override
- public void deleteByQuery(DeleteUpdateCommand cmd) throws IOException {
- TestInjection.injectDirectUpdateLatch();
- deleteByQueryCommands.increment();
- deleteByQueryCommandsCumulative.mark();
- boolean madeIt = false;
- try {
- if ((cmd.getFlags() & UpdateCommand.IGNORE_INDEXWRITER) != 0) {
- if (ulog != null) ulog.deleteByQuery(cmd);
- madeIt = true;
- return;
- }
- Query q = getQuery(cmd);
-
- boolean delAll = MatchAllDocsQuery.class == q.getClass();
-
- // currently for testing purposes. Do a delete of complete index w/o worrying about versions,
- // don't log, clean up most state in update log, etc
- if (delAll && cmd.getVersion() == -Long.MAX_VALUE) {
- synchronized (solrCoreState.getUpdateLock()) {
- deleteAll();
- ulog.deleteAll();
- return;
- }
- }
-
- //
- // synchronized to prevent deleteByQuery from running during the "open new searcher"
- // part of a commit. DBQ needs to signal that a fresh reader will be needed for
- // a realtime view of the index. When a new searcher is opened after a DBQ, that
- // flag can be cleared. If those thing happen concurrently, it's not thread safe.
- // Also, ulog.deleteByQuery clears caches and is thus not safe to be called between
- // preSoftCommit/postSoftCommit and thus we use the updateLock to prevent this (just
- // as we use around ulog.preCommit... also see comments in ulog.postSoftCommit)
- //
- synchronized (solrCoreState.getUpdateLock()) {
-
- // We are reopening a searcher before applying the deletes to overcome LUCENE-7344.
- // Once LUCENE-7344 is resolved, we can consider removing this.
- if (ulog != null) ulog.openRealtimeSearcher();
-
- if (delAll) {
- deleteAll();
- } else {
- RefCounted<IndexWriter> iw = solrCoreState.getIndexWriter(core);
- try {
- iw.get().deleteDocuments(new DeleteByQueryWrapper(q, core.getLatestSchema()));
- } finally {
- iw.decref();
- }
- }
-
- if (ulog != null) ulog.deleteByQuery(cmd); // this needs to be protected by the update lock
- }
-
- madeIt = true;
-
- updateDeleteTrackers(cmd);
-
- } finally {
- if (!madeIt) {
- numErrors.increment();
- numErrorsCumulative.mark();
- }
- }
- }
-
- @Override
- public int mergeIndexes(MergeIndexesCommand cmd) throws IOException {
- TestInjection.injectDirectUpdateLatch();
- mergeIndexesCommands.mark();
- int rc;
-
- log.info("start {}", cmd);
-
- List<DirectoryReader> readers = cmd.readers;
- if (readers != null && readers.size() > 0) {
- List<CodecReader> mergeReaders = new ArrayList<>();
- for (DirectoryReader reader : readers) {
- for (LeafReaderContext leaf : reader.leaves()) {
- mergeReaders.add(SlowCodecReaderWrapper.wrap(leaf.reader()));
- }
- }
- RefCounted<IndexWriter> iw = solrCoreState.getIndexWriter(core);
- try {
- iw.get().addIndexes(mergeReaders.toArray(new CodecReader[mergeReaders.size()]));
- } finally {
- iw.decref();
- }
- rc = 1;
- } else {
- rc = 0;
- }
- log.info("end_mergeIndexes");
-
- // TODO: consider soft commit issues
- if (rc == 1 && commitTracker.getTimeUpperBound() > 0) {
- commitTracker.scheduleCommitWithin(commitTracker.getTimeUpperBound());
- } else if (rc == 1 && softCommitTracker.getTimeUpperBound() > 0) {
- softCommitTracker.scheduleCommitWithin(softCommitTracker.getTimeUpperBound());
- }
-
- return rc;
- }
-
- public void prepareCommit(CommitUpdateCommand cmd) throws IOException {
-
- boolean error = true;
-
- try {
- log.debug("start {}", cmd);
- RefCounted<IndexWriter> iw = solrCoreState.getIndexWriter(core);
- try {
- SolrIndexWriter.setCommitData(iw.get(), cmd.getVersion(), cmd.commitData);
- iw.get().prepareCommit();
- } finally {
- iw.decref();
- }
-
- log.debug("end_prepareCommit");
-
- error = false;
- } finally {
- if (error) {
- numErrors.increment();
- numErrorsCumulative.mark();
- }
- }
- }
-
- @Override
- public void commit(CommitUpdateCommand cmd) throws IOException {
- TestInjection.injectDirectUpdateLatch();
- if (cmd.prepareCommit) {
- prepareCommit(cmd);
- return;
- }
-
- if (cmd.optimize) {
- optimizeCommands.mark();
- } else {
- commitCommands.mark();
- if (cmd.expungeDeletes) expungeDeleteCommands.mark();
- }
-
- @SuppressWarnings("unchecked")
- Future<Void>[] waitSearcher =
- cmd.waitSearcher ? (Future<Void>[]) Array.newInstance(Future.class, 1) : null;
-
- boolean error = true;
- try {
- // only allow one hard commit to proceed at once
- if (!cmd.softCommit) {
- solrCoreState.getCommitLock().lock();
- }
-
- log.debug("start {}", cmd);
-
- // We must cancel pending commits *before* we actually execute the commit.
-
- if (cmd.openSearcher) {
- // we can cancel any pending soft commits if this commit will open a new searcher
- softCommitTracker.cancelPendingCommit();
- }
- if (!cmd.softCommit && (cmd.openSearcher || !commitTracker.getOpenSearcher())) {
- // cancel a pending hard commit if this commit is of equal or greater "strength"...
- // If the autoCommit has openSearcher=true, then this commit must have openSearcher=true
- // to cancel.
- commitTracker.cancelPendingCommit();
- }
-
- RefCounted<IndexWriter> iw = solrCoreState.getIndexWriter(core);
- try {
- IndexWriter writer = iw.get();
- if (cmd.optimize) {
- writer.forceMerge(cmd.maxOptimizeSegments);
- } else if (cmd.expungeDeletes) {
- writer.forceMergeDeletes();
- }
-
- if (!cmd.softCommit) {
- synchronized (solrCoreState.getUpdateLock()) {
- // sync is currently needed to prevent preCommit from being called between preSoft and
- // postSoft... see postSoft comments.
- if (ulog != null) ulog.preCommit(cmd);
- }
-
- // SolrCore.verbose("writer.commit() start writer=",writer);
-
- if (shouldCommit(cmd, writer)) {
- SolrIndexWriter.setCommitData(writer, cmd.getVersion(), cmd.commitData);
- writer.commit();
- } else {
- log.debug("No uncommitted changes. Skipping IW.commit.");
- }
-
- // SolrCore.verbose("writer.commit() end");
- numDocsPending.reset();
- callPostCommitCallbacks();
- }
- } finally {
- iw.decref();
- }
-
- if (cmd.optimize) {
- callPostOptimizeCallbacks();
- }
-
- if (cmd.softCommit) {
- // ulog.preSoftCommit();
- synchronized (solrCoreState.getUpdateLock()) {
- if (ulog != null) ulog.preSoftCommit(cmd);
- core.getSearcher(true, false, waitSearcher, true);
- if (ulog != null) ulog.postSoftCommit(cmd);
- }
- callPostSoftCommitCallbacks();
- } else {
- synchronized (solrCoreState.getUpdateLock()) {
- if (ulog != null) ulog.preSoftCommit(cmd);
- if (cmd.openSearcher) {
- core.getSearcher(true, false, waitSearcher);
- } else {
- // force open a new realtime searcher so realtime-get and versioning code can see the
- // latest
- RefCounted<SolrIndexSearcher> searchHolder = core.openNewSearcher(true, true);
- searchHolder.decref();
- }
- if (ulog != null) ulog.postSoftCommit(cmd);
- }
- if (ulog != null) ulog.postCommit(cmd); // postCommit currently means new searcher has
- // also been opened
- }
-
- // reset commit tracking
-
- if (cmd.softCommit) {
- softCommitTracker.didCommit();
- } else {
- commitTracker.didCommit();
- }
-
- log.debug("end_commit_flush");
-
- error = false;
- } finally {
- if (!cmd.softCommit) {
- solrCoreState.getCommitLock().unlock();
- }
-
- addCommands.reset();
- deleteByIdCommands.reset();
- deleteByQueryCommands.reset();
- if (error) {
- numErrors.increment();
- numErrorsCumulative.mark();
- }
- }
-
- // if we are supposed to wait for the searcher to be registered, then we should do it
- // outside any synchronized block so that other update operations can proceed.
- if (waitSearcher != null && waitSearcher[0] != null) {
- try {
- waitSearcher[0].get();
- } catch (InterruptedException | ExecutionException e) {
- SolrException.log(log, e);
- }
- }
- }
-
- /**
- * Determines whether the commit command should effectively trigger a commit on the index writer.
- * This method is called with the commit lock and is the last step before effectively calling
- * {@link IndexWriter#commit()}.
- */
- protected boolean shouldCommit(CommitUpdateCommand cmd, IndexWriter writer) throws IOException {
- return writer.hasUncommittedChanges() || (cmd.commitData != null && !cmd.commitData.isEmpty());
- }
-
- @Override
- public void newIndexWriter(boolean rollback) throws IOException {
- solrCoreState.newIndexWriter(core, rollback);
- }
-
- /**
- * @since Solr 1.4
- */
- @Override
- public void rollback(RollbackUpdateCommand cmd) throws IOException {
- TestInjection.injectDirectUpdateLatch();
- if (core.getCoreContainer().isZooKeeperAware()) {
- throw new UnsupportedOperationException(
- "Rollback is currently not supported in SolrCloud mode. (SOLR-4895)");
- }
-
- rollbackCommands.mark();
-
- boolean error = true;
-
- try {
- log.info("start {}", cmd);
-
- rollbackWriter();
-
- // callPostRollbackCallbacks();
-
- // reset commit tracking
- commitTracker.didRollback();
- softCommitTracker.didRollback();
-
- log.info("end_rollback");
-
- error = false;
- } finally {
- addCommandsCumulative.mark(-addCommands.sumThenReset());
- deleteByIdCommandsCumulative.mark(-deleteByIdCommands.sumThenReset());
- deleteByQueryCommandsCumulative.mark(-deleteByQueryCommands.sumThenReset());
- if (error) {
- numErrors.increment();
- numErrorsCumulative.mark();
- }
- }
- }
-
- @Override
- public UpdateLog getUpdateLog() {
- return ulog;
- }
-
- @Override
- public void close() throws IOException {
- log.debug("closing {}", this);
-
- commitTracker.close();
- softCommitTracker.close();
-
- numDocsPending.reset();
- try {
- super.close();
- } catch (Exception e) {
- throw new IOException("Error closing", e);
- }
- }
-
- // IndexWriterCloser interface method - called from solrCoreState.decref(this)
- @Override
- public void closeWriter(IndexWriter writer) throws IOException {
- log.trace("closeWriter({}): ulog={}", writer, ulog);
-
- assert TestInjection.injectNonGracefullClose(core.getCoreContainer());
-
- boolean clearRequestInfo = false;
-
- SolrQueryRequest req = new LocalSolrQueryRequest(core, new ModifiableSolrParams());
- SolrQueryResponse rsp = new SolrQueryResponse();
- if (SolrRequestInfo.getRequestInfo() == null) {
- clearRequestInfo = true;
- SolrRequestInfo.setRequestInfo(new SolrRequestInfo(req, rsp)); // important for debugging
- }
- try {
-
- if (TestInjection.injectSkipIndexWriterCommitOnClose(writer)) {
- // if this TestInjection triggers, we do some simple rollback()
- // (which closes the underlying IndexWriter) and then return immediately
- log.warn("Skipping commit for IndexWriter.close() due to TestInjection");
- if (writer != null) {
- writer.rollback();
- }
- // we shouldn't close the transaction logs either, but leaving them open
- // means we can't delete them on windows (needed for tests)
- if (ulog != null) ulog.close(false);
-
- return;
- }
-
- // do a commit before we quit?
- boolean tryToCommit =
- writer != null
- && ulog != null
- && ulog.hasUncommittedChanges()
- && ulog.getState() == UpdateLog.State.ACTIVE;
-
- // be tactical with this lock! closing the updatelog can deadlock when it tries to commit
- solrCoreState.getCommitLock().lock();
- try {
- try {
- if (log.isInfoEnabled()) {
- log.info(
- "Committing on IndexWriter.close() {}.",
- (tryToCommit ? "" : " ... SKIPPED (unnecessary)"));
- }
- if (tryToCommit) {
- CommitUpdateCommand cmd = new CommitUpdateCommand(req, false);
- cmd.openSearcher = false;
- cmd.waitSearcher = false;
- cmd.softCommit = false;
-
- // TODO: keep other commit callbacks from being called?
- // this.commit(cmd); // too many test failures using this method... is it because of
- // callbacks?
-
- synchronized (solrCoreState.getUpdateLock()) {
- ulog.preCommit(cmd);
- }
-
- // todo: refactor this shared code (or figure out why a real CommitUpdateCommand can't
- // be used)
- SolrIndexWriter.setCommitData(writer, cmd.getVersion(), null);
- writer.commit();
-
- synchronized (solrCoreState.getUpdateLock()) {
- ulog.postCommit(cmd);
- }
- }
- } catch (Throwable th) {
- log.error("Error in final commit", th);
- if (th instanceof OutOfMemoryError) {
- throw (OutOfMemoryError) th;
- }
- }
-
- } finally {
- solrCoreState.getCommitLock().unlock();
- }
- } finally {
- if (clearRequestInfo) SolrRequestInfo.clearRequestInfo();
- }
- // we went through the normal process to commit, so we don't have to artificially
- // cap any ulog files.
- try {
- if (ulog != null) ulog.close(false);
- } catch (Throwable th) {
- log.error("Error closing log files", th);
- if (th instanceof OutOfMemoryError) {
- throw (OutOfMemoryError) th;
- }
- }
-
- if (writer != null) {
- writer.close();
- }
- }
-
- @Override
- public void split(SplitIndexCommand cmd) throws IOException {
- commit(new CommitUpdateCommand(cmd.req, false));
- SolrIndexSplitter splitter = new SolrIndexSplitter(cmd);
- splitCommands.mark();
- NamedList<Object> results = new NamedList<>();
- try {
- splitter.split(results);
- cmd.rsp.addResponse(results);
- } catch (IOException e) {
- numErrors.increment();
- numErrorsCumulative.mark();
- throw e;
- }
- }
-
- /**
- * Calls either {@link IndexWriter#updateDocValues} or <code>IndexWriter#updateDocument</code>(s)
- * as needed based on {@link AddUpdateCommand#isInPlaceUpdate}.
- *
- * <p>If the this is an UPDATE_INPLACE cmd, then all fields included in {@link
- * AddUpdateCommand#makeLuceneDocForInPlaceUpdate} must either be the uniqueKey field, or be
- * DocValue only fields.
- *
- * @param cmd - cmd apply to IndexWriter
- * @param writer - IndexWriter to use
- */
- private void updateDocOrDocValues(AddUpdateCommand cmd, IndexWriter writer) throws IOException {
- // this code path requires an idField in order to potentially replace a doc
- assert idField != null;
- boolean hasUpdateTerm = cmd.updateTerm != null; // AKA dedupe
-
- if (cmd.isInPlaceUpdate()) {
- if (hasUpdateTerm) {
- throw new IllegalStateException(
- "cmd.updateTerm/dedupe is not compatible with in-place updates");
- }
- // we don't support the solrInputDoc with nested child docs either but we'll throw an
- // exception if attempted
-
- // can't use cmd.getIndexedId because it will be a root doc if this doc is a child
- Term updateTerm =
- new Term(
- idField.getName(),
- core.getLatestSchema().indexableUniqueKey(cmd.getSelfOrNestedDocIdStr()));
- // skips uniqueKey and _root_
- List<IndexableField> fields = cmd.makeLuceneDocForInPlaceUpdate().getFields();
- log.debug("updateDocValues({})", cmd);
- writer.updateDocValues(updateTerm, fields.toArray(new Field[fields.size()]));
-
- } else { // more normal path
-
- Iterable<Document> nestedDocs = cmd.makeLuceneDocs();
- Term idTerm = getIdTerm(cmd.getIndexedId());
- Term updateTerm = hasUpdateTerm ? cmd.updateTerm : idTerm;
-
- log.debug("updateDocuments({})", cmd);
- writer.updateDocuments(updateTerm, nestedDocs);
-
- // If hasUpdateTerm, then delete any existing documents with the same ID other than the one
- // added above (used in near-duplicate replacement)
- if (hasUpdateTerm) { // rare
- BooleanQuery.Builder bq = new BooleanQuery.Builder();
- // don't want the one we added above (will be unique)
- bq.add(new TermQuery(updateTerm), Occur.MUST_NOT);
- bq.add(new TermQuery(idTerm), Occur.MUST); // same ID
- writer.deleteDocuments(new DeleteByQueryWrapper(bq.build(), core.getLatestSchema()));
- }
- }
- }
-
- private Term getIdTerm(BytesRef termVal) {
- boolean useRootId = core.getLatestSchema().isUsableForChildDocs();
- return new Term(useRootId ? IndexSchema.ROOT_FIELD_NAME : idField.getName(), termVal);
- }
-
- /////////////////////////////////////////////////////////////////////
- // SolrInfoBean stuff: Statistics and Module Info
- /////////////////////////////////////////////////////////////////////
-
- @Override
- public String getName() {
- return DirectUpdateHandler2.class.getName();
- }
-
- @Override
- public String getDescription() {
- return "Update handler that efficiently directly updates the on-disk main lucene index";
- }
-
- @Override
- public SolrCoreState getSolrCoreState() {
- return solrCoreState;
- }
-
- private long getCurrentTLogSize() {
- return ulog != null && ulog.hasUncommittedChanges() ? ulog.getCurrentLogSizeFromStream() : -1;
- }
-
- // allow access for tests
- public CommitTracker getCommitTracker() {
- return commitTracker;
- }
-
- // allow access for tests
- public CommitTracker getSoftCommitTracker() {
- return softCommitTracker;
- }
-}