You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by si...@apache.org on 2020/06/24 16:50:29 UTC
[lucene-solr] branch jira/lucene-8962 updated (0a096f7 -> 2aa937e)
This is an automated email from the ASF dual-hosted git repository.
simonw pushed a change to branch jira/lucene-8962
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git.
from 0a096f7 fix javadoc
add cfae052 SOLR-14577: Return BAD REQUEST when field is missing in terms QP (#1588)
add 0ea0358 SOLR-14532: Add *.iml files to gitignore
add 936b9d7 SOLR-14561 CoreAdminAPI's parameters instanceDir and dataDir are now validated (#1572)
add 84729ed SOLR-14574: Fix or suppress warnings in solr/core/src/test (part 2)
add 4db1e38 LUCENE-9402: Let MultiCollector handle minCompetitiveScore (#1567)
add 56febf0 Replace DWPT.DocState with simple method parameters (#1594)
add 972c840 LUCENE-8962: add ability to selectively merge on commit (#1552)
add 591d844 LUCENE-9412 Do not validate jenkins HTTPS cert
add 541fc98 LUCENE-9409: Disable TestAllFilesDetectTruncation temporarily.
add aa5b26f Remove some needless toAbsolutePath calls (#1582)
add 62dd94d SOLR-12823: fix TestZKPropertiesWriter (#1575)
add 5d43e73 Revert "LUCENE-8962: add ability to selectively merge on commit (#1552)"
add 4774c6f Include delegate in AssertingSimilarity toString (#1596)
add a1be1de SOLR-14554: Let Solr use WAND algorithm when scores are requested (#1566)
add 3e42286 SOLR-14546: OverseerTaskProcessor can process messages out of order (#1561)
add dc1e359 fixes SOLR-14584
add 3f4f5e2 SOLR-14584: CHANGES.txt entry
add 6b55d79 SOLR-14702: increase bin/solr's post kill sleep (from 1s to 10s)
add b4b641e Revert "SOLR-14702: increase bin/solr's post kill sleep (from 1s to 10s)"
add 710fe3a SOLR-14502: increase bin/solr's post kill sleep (from 1s to 10s)
add 3c6e092 SOLR-14546: add a Bug Fixes section for Solr 9.0.0 in CHANGES.txt (#1607)
add ea1bb9f SOLR-13268: Clean up any test failures resulting from defaulting to async logging
add b0333ab LUCENE-9280: Collectors to skip noncompetitive documents (#1351)
add 9c1772f LUCENE-9411: Fail complation on warnings, 9x gradle-only
add db98608 SOLR-14541: Remove IntelliJ //noinspection comments
add 7030bb5 SOLR-14591: Move JoinQuery To Its Own File (#1609)
new ea750da Merge branch 'master' into jira/lucene-8962
new c29875f add failing test for carry-over deletes
new a533094 pimp test to randomly use soft-deletes which also check if DV updates work accordingly
new 189244b LUCENE-8962: Ensure we don't include fully deleted segments in a commit
new ff17e87 add prototype to preserve atomicity
new d9b12a0 ensure we actually apply the updates in the test
new 2aa937e apply comments from #1601
The 7 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.gitignore | 1 +
build.gradle | 2 +
dev-tools/scripts/reproduceJenkinsFailures.py | 5 +-
gradle/defaults-java.gradle | 1 +
gradle/hacks/findbugs.gradle | 45 ++
gradle/validation/jar-checks.gradle | 4 +-
help/ant.txt | 4 +-
lucene/CHANGES.txt | 13 +
lucene/MIGRATE.md | 6 +
.../apache/lucene/index/DefaultIndexingChain.java | 100 ++--
.../java/org/apache/lucene/index/DocConsumer.java | 2 +-
.../org/apache/lucene/index/DocumentsWriter.java | 2 +-
.../lucene/index/DocumentsWriterPerThread.java | 51 +-
.../java/org/apache/lucene/index/IndexWriter.java | 107 +++--
.../java/org/apache/lucene/index/MergePolicy.java | 63 ++-
.../org/apache/lucene/index/ReadersAndUpdates.java | 14 +-
.../apache/lucene/index/TermVectorsConsumer.java | 6 +-
.../java/org/apache/lucene/index/TermsHash.java | 10 +-
.../apache/lucene/search/ConstantScoreQuery.java | 2 +-
.../org/apache/lucene/search/FieldComparator.java | 16 +-
.../apache/lucene/search/FieldValueHitQueue.java | 29 +-
.../lucene/search/FilteringFieldComparator.java | 93 ++++
...ches.java => FilteringLeafFieldComparator.java} | 30 +-
.../lucene/search/FilteringNumericComparator.java | 52 +++
.../search/FilteringNumericLeafComparator.java | 336 ++++++++++++++
.../org/apache/lucene/search/LeafCollector.java | 12 +
.../apache/lucene/search/MatchAllDocsQuery.java | 2 +-
.../org/apache/lucene/search/MultiCollector.java | 111 +++--
.../java/org/apache/lucene/search/ScoreMode.java | 54 ++-
.../java/org/apache/lucene/search/SortField.java | 4 +
.../apache/lucene/search/TopFieldCollector.java | 69 ++-
.../src/java/org/apache/lucene/search/Weight.java | 18 +-
.../lucene/index/TestAllFilesDetectTruncation.java | 4 +-
.../lucene/index/TestDemoParallelLeafReader.java | 2 +-
.../org/apache/lucene/index/TestIndexWriter.java | 2 +-
.../lucene/index/TestIndexWriterMergePolicy.java | 71 +++
.../org/apache/lucene/index/TestMergePolicy.java | 10 +-
.../apache/lucene/search/MultiCollectorTest.java | 172 +++++++
.../search/TestFieldSortOptimizationSkipping.java | 294 ++++++++++++
.../lucene/search/AssertingLeafCollector.java | 6 +-
.../search/similarities/AssertingSimilarity.java | 2 +-
solr/CHANGES.txt | 34 +-
solr/bin/solr | 2 +-
solr/bin/solr.in.cmd | 8 +-
solr/bin/solr.in.sh | 9 +-
.../analytics/legacy/LegacyNoFacetCloudTest.java | 32 +-
.../solr/analytics/legacy/LegacyNoFacetTest.java | 32 +-
.../LegacyAbstractAnalyticsFacetCloudTest.java | 2 +-
.../facet/LegacyAbstractAnalyticsFacetTest.java | 4 +-
.../legacy/facet/LegacyFieldFacetTest.java | 1 +
.../dataimporthandler/src/test-files/log4j2.xml | 42 ++
.../src/test-files/solr/collection1/README | 1 +
.../solr/configsets/dihconfigset/conf/README | 2 +
.../configsets/dihconfigset/conf/schema.xml} | 0
.../configsets/dihconfigset/conf/solrconfig.xml} | 0
.../src/test-files/{dih => }/solr/solr.xml | 0
.../handler/dataimport/TestZKPropertiesWriter.java | 207 +++++++--
.../extraction/ExtractingRequestHandlerTest.java | 3 +
.../apache/solr/jaeger/TestJaegerConfigurator.java | 4 +
.../TestFeatureExtractionFromMultipleSegments.java | 2 +
.../solr/ltr/feature/TestNoMatchSolrFeature.java | 4 +
.../solr/ltr/feature/TestOriginalScoreFeature.java | 6 +
.../solr/ltr/model/TestNeuralNetworkModel.java | 1 +
.../store/rest/TestModelManagerPersistence.java | 4 +
.../src/java/org/apache/solr/cloud/LockTree.java | 54 ++-
.../cloud/OverseerConfigSetMessageHandler.java | 2 +-
.../apache/solr/cloud/OverseerMessageHandler.java | 7 +-
.../apache/solr/cloud/OverseerTaskProcessor.java | 180 +++-----
.../OverseerCollectionMessageHandler.java | 18 +-
.../solr/cloud/api/collections/RoutedAlias.java | 1 -
.../java/org/apache/solr/core/CoreContainer.java | 51 +-
.../java/org/apache/solr/core/CoreDescriptor.java | 8 +-
.../src/java/org/apache/solr/core/NodeConfig.java | 20 +-
.../src/java/org/apache/solr/core/SolrPaths.java | 41 +-
.../org/apache/solr/core/SolrResourceLoader.java | 18 +-
.../java/org/apache/solr/core/SolrXmlConfig.java | 16 +
.../apache/solr/filestore/DistribPackageStore.java | 2 +-
.../handler/component/QueryElevationComponent.java | 1 -
.../component/ShardFieldSortedHitQueue.java | 1 -
.../solr/highlight/DefaultSolrHighlighter.java | 511 +++++++++++----------
.../org/apache/solr/search/JoinQParserPlugin.java | 484 -------------------
.../{JoinQParserPlugin.java => JoinQuery.java} | 151 +-----
.../org/apache/solr/search/MaxScoreCollector.java | 6 +-
.../org/apache/solr/search/TermsQParserPlugin.java | 3 +
solr/core/src/test-files/log4j2.xml | 44 +-
solr/core/src/test-files/solr/solr-50-all.xml | 1 +
.../core/src/test-files/solr/solr-solrreporter.xml | 2 +
solr/core/src/test-files/solr/solr.xml | 1 +
.../src/test/org/apache/hadoop/fs/FileUtil.java | 2 +
.../org/apache/solr/BasicFunctionalityTest.java | 7 +
.../src/test/org/apache/solr/CursorPagingTest.java | 8 +
.../org/apache/solr/TestDistributedSearch.java | 10 +-
.../test/org/apache/solr/TestGroupingSearch.java | 7 +-
solr/core/src/test/org/apache/solr/TestJoin.java | 12 +-
.../apache/solr/cloud/AliasIntegrationTest.java | 1 -
.../solr/cloud/ChaosMonkeyNothingIsSafeTest.java | 1 +
.../org/apache/solr/cloud/CollectionPropsTest.java | 1 +
.../apache/solr/cloud/CollectionsAPISolrJTest.java | 8 +-
.../apache/solr/cloud/DistribCursorPagingTest.java | 1 +
.../DistribDocExpirationUpdateProcessorTest.java | 1 +
.../solr/cloud/FullSolrCloudDistribCmdsTest.java | 2 +-
.../solr/cloud/MetricsHistoryIntegrationTest.java | 4 +
.../test/org/apache/solr/cloud/MockSolrSource.java | 2 +-
.../apache/solr/cloud/MultiThreadedOCPTest.java | 1 +
.../solr/cloud/NestedShardedAtomicUpdateTest.java | 4 +
.../OverseerCollectionConfigSetProcessorTest.java | 1 +
.../solr/cloud/OverseerModifyCollectionTest.java | 1 +
.../test/org/apache/solr/cloud/OverseerTest.java | 1 +
.../apache/solr/cloud/ReindexCollectionTest.java | 2 +
.../apache/solr/cloud/ReplaceNodeNoTargetTest.java | 1 +
.../cloud/RoutingToNodesWithPropertiesTest.java | 1 +
.../apache/solr/cloud/SolrCloudExampleTest.java | 2 +
.../test/org/apache/solr/cloud/SyncSliceTest.java | 1 +
.../solr/cloud/SystemCollectionCompatTest.java | 1 +
.../apache/solr/cloud/TestBaseStatsCacheCloud.java | 1 +
.../apache/solr/cloud/TestCloudConsistency.java | 2 +
.../TestCloudPhrasesIdentificationComponent.java | 4 +
.../solr/cloud/TestCloudSearcherWarming.java | 2 +-
.../org/apache/solr/cloud/TestConfigSetsAPI.java | 16 +-
.../solr/cloud/TestConfigSetsAPIExclusivity.java | 4 +
.../org/apache/solr/cloud/TestHashPartitioner.java | 1 +
.../org/apache/solr/cloud/TestPullReplica.java | 4 +-
.../apache/solr/cloud/TestRebalanceLeaders.java | 1 +
.../apache/solr/cloud/TestSSLRandomization.java | 12 +-
.../cloud/TestSolrCloudWithDelegationTokens.java | 12 +-
.../TestSolrCloudWithSecureImpersonation.java | 3 +
.../cloud/TestStressCloudBlindAtomicUpdates.java | 16 +-
.../org/apache/solr/cloud/TestWithCollection.java | 6 +
.../apache/solr/cloud/UnloadDistributedZkTest.java | 2 +
.../org/apache/solr/cloud/ZkSolrClientTest.java | 12 +-
.../solr/cloud/api/collections/AssignTest.java | 1 +
.../CollectionsAPIDistributedZkTest.java | 6 +
.../ConcurrentCreateCollectionTest.java | 1 +
.../SimpleCollectionCreateDeleteTest.java | 1 +
.../cloud/api/collections/TestCollectionAPI.java | 54 +++
.../api/collections/TestReplicaProperties.java | 2 +
.../TestRequestStatusCollectionAPI.java | 2 +
.../AutoAddReplicasIntegrationTest.java | 4 +
.../autoscaling/AutoAddReplicasPlanActionTest.java | 10 +-
.../cloud/autoscaling/AutoScalingHandlerTest.java | 26 +-
.../cloud/autoscaling/ComputePlanActionTest.java | 27 +-
.../cloud/autoscaling/ExecutePlanActionTest.java | 5 +
.../cloud/autoscaling/HttpTriggerListenerTest.java | 1 +
.../IndexSizeTriggerMixedBoundsTest.java | 2 +
.../IndexSizeTriggerSizeEstimationTest.java | 3 +
.../cloud/autoscaling/IndexSizeTriggerTest.java | 15 +
.../autoscaling/MetricTriggerIntegrationTest.java | 1 +
.../NodeAddedTriggerIntegrationTest.java | 2 +
.../cloud/autoscaling/NodeAddedTriggerTest.java | 3 +
.../NodeLostTriggerIntegrationTest.java | 2 +
.../cloud/autoscaling/NodeLostTriggerTest.java | 3 +
.../autoscaling/NodeMarkersRegistrationTest.java | 2 +
.../cloud/autoscaling/RestoreTriggerStateTest.java | 2 +
.../ScheduledMaintenanceTriggerTest.java | 5 +
.../ScheduledTriggerIntegrationTest.java | 4 +-
.../SearchRateTriggerIntegrationTest.java | 4 +
.../cloud/autoscaling/SearchRateTriggerTest.java | 3 +
.../cloud/autoscaling/SystemLogListenerTest.java | 3 +
.../solr/cloud/autoscaling/TestPolicyCloud.java | 2 +
.../TriggerCooldownIntegrationTest.java | 1 +
.../cloud/autoscaling/TriggerEventQueueTest.java | 1 +
.../cloud/autoscaling/TriggerIntegrationTest.java | 5 +
.../autoscaling/sim/SimSolrCloudTestCase.java | 1 +
.../autoscaling/sim/TestSimComputePlanAction.java | 16 +-
.../autoscaling/sim/TestSimExecutePlanAction.java | 3 +
.../autoscaling/sim/TestSimExtremeIndexing.java | 1 +
.../cloud/autoscaling/sim/TestSimLargeCluster.java | 1 +
.../autoscaling/sim/TestSimNodeAddedTrigger.java | 2 +
.../autoscaling/sim/TestSimNodeLostTrigger.java | 3 +
.../cloud/autoscaling/sim/TestSimPolicyCloud.java | 2 +
.../cloud/autoscaling/sim/TestSimScenario.java | 4 +
.../autoscaling/sim/TestSimTriggerIntegration.java | 7 +
.../autoscaling/sim/TestSnapshotCloudManager.java | 1 +
.../solr/cloud/cdcr/BaseCdcrDistributedZkTest.java | 11 +
.../solr/cloud/cdcr/CdcrOpsAndBoundariesTest.java | 5 +-
.../solr/cloud/cdcr/CdcrRequestHandlerTest.java | 5 +
.../org/apache/solr/cloud/hdfs/StressHdfsTest.java | 2 +
.../solr/cloud/overseer/ZkStateWriterTest.java | 1 +
.../apache/solr/cloud/rule/ImplicitSnitchTest.java | 3 +
.../org/apache/solr/cloud/rule/RuleEngineTest.java | 8 +
.../test/org/apache/solr/cloud/rule/RulesTest.java | 7 +
.../org/apache/solr/core/DirectoryFactoryTest.java | 6 +-
.../org/apache/solr/core/ResourceLoaderTest.java | 4 +-
.../org/apache/solr/core/TestCoreContainer.java | 85 ++++
.../src/test/org/apache/solr/core/TestSolrXml.java | 10 +
.../solr/handler/admin/CoreAdminHandlerTest.java | 2 +
.../handler/admin/CoreAdminRequestStatusTest.java | 1 +
.../component/DistributedFacetPivotSmallTest.java | 12 +-
.../org/apache/solr/highlight/HighlighterTest.java | 2 +-
.../apache/solr/index/hdfs/CheckHdfsIndexTest.java | 2 +
.../org/apache/solr/logging/TestLogWatcher.java | 1 +
.../reporters/solr/SolrShardReporterTest.java | 12 +
.../src/test/org/apache/solr/pkg/TestPackages.java | 7 +-
.../org/apache/solr/request/SimpleFacetsTest.java | 1 +
.../apache/solr/request/TestIntervalFaceting.java | 2 +-
.../solr/request/macro/TestMacroExpander.java | 2 +
.../apache/solr/rest/schema/TestBulkSchemaAPI.java | 2 +-
.../apache/solr/search/MaxScoreCollectorTest.java | 91 ++++
.../apache/solr/search/SolrIndexSearcherTest.java | 8 +-
.../apache/solr/search/TestTermsQParserPlugin.java | 6 +
.../solr/security/AuditLoggerIntegrationTest.java | 8 +
.../solr/spelling/DirectSolrSpellCheckerTest.java | 3 +
.../solr/spelling/FileBasedSpellCheckerTest.java | 7 +
.../solr/spelling/IndexBasedSpellCheckerTest.java | 8 +
.../solr/spelling/SpellCheckCollatorTest.java | 9 +
.../solr/spelling/SpellingQueryConverterTest.java | 6 +
.../solr/spelling/suggest/SuggesterTest.java | 2 +
.../solr/store/blockcache/BlockCacheTest.java | 4 +-
.../apache/solr/store/hdfs/HdfsDirectoryTest.java | 2 +-
.../apache/solr/update/SolrIndexSplitterTest.java | 3 +
.../org/apache/solr/util/MockCoreContainer.java | 2 +-
solr/server/resources/log4j2-console.xml | 28 +-
solr/server/resources/log4j2.xml | 67 +--
solr/server/solr/solr.xml | 1 +
solr/solr-ref-guide/src/coreadmin-api.adoc | 4 +-
solr/solr-ref-guide/src/format-of-solr-xml.adoc | 4 +
.../solr/common/params/CollectionParams.java | 33 +-
.../apache/solr/common/util/CommandOperation.java | 1 -
.../solr/common/util/JsonSchemaValidator.java | 11 +-
solr/solrj/src/test-files/log4j2.xml | 45 +-
.../solr/client/solrj/request/TestCoreAdmin.java | 4 +-
.../apache/solr/BaseDistributedSearchTestCase.java | 2 +-
.../solr/cloud/AbstractDistribZkTestBase.java | 2 +-
.../solr/cloud/AbstractFullDistribZkTestBase.java | 4 +-
versions.lock | 4 +-
225 files changed, 3143 insertions(+), 1694 deletions(-)
create mode 100644 gradle/hacks/findbugs.gradle
create mode 100644 lucene/core/src/java/org/apache/lucene/search/FilteringFieldComparator.java
copy lucene/core/src/java/org/apache/lucene/search/{Matches.java => FilteringLeafFieldComparator.java} (51%)
create mode 100644 lucene/core/src/java/org/apache/lucene/search/FilteringNumericComparator.java
create mode 100644 lucene/core/src/java/org/apache/lucene/search/FilteringNumericLeafComparator.java
create mode 100644 lucene/core/src/test/org/apache/lucene/search/TestFieldSortOptimizationSkipping.java
create mode 100644 solr/contrib/dataimporthandler/src/test-files/log4j2.xml
create mode 100644 solr/contrib/dataimporthandler/src/test-files/solr/collection1/README
create mode 100644 solr/contrib/dataimporthandler/src/test-files/solr/configsets/dihconfigset/conf/README
copy solr/contrib/dataimporthandler/src/test-files/{dih/solr/collection1/conf/dataimport-schema.xml => solr/configsets/dihconfigset/conf/schema.xml} (100%)
copy solr/contrib/dataimporthandler/src/test-files/{dih/solr/collection1/conf/dataimport-solrconfig.xml => solr/configsets/dihconfigset/conf/solrconfig.xml} (100%)
copy solr/contrib/dataimporthandler/src/test-files/{dih => }/solr/solr.xml (100%)
copy solr/core/src/java/org/apache/solr/search/{JoinQParserPlugin.java => JoinQuery.java} (76%)
create mode 100644 solr/core/src/test/org/apache/solr/search/MaxScoreCollectorTest.java
[lucene-solr] 05/07: add prototype to preserve atomicity
Posted by si...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
simonw pushed a commit to branch jira/lucene-8962
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit ff17e876c61ba5ecfa4a791527c8a73e1d3f8d5c
Author: Simon Willnauer <si...@apache.org>
AuthorDate: Wed Jun 24 16:34:47 2020 +0200
add prototype to preserve atomicity
---
.../java/org/apache/lucene/index/IndexWriter.java | 57 ++++++++++------------
.../java/org/apache/lucene/index/MergePolicy.java | 26 +++++++++-
.../lucene/index/TestIndexWriterMergePolicy.java | 4 +-
3 files changed, 54 insertions(+), 33 deletions(-)
diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
index 88dc399..ff4817b 100644
--- a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
@@ -18,6 +18,7 @@ package org.apache.lucene.index;
import java.io.Closeable;
import java.io.IOException;
+import java.io.UncheckedIOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -39,6 +40,7 @@ import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.locks.ReentrantLock;
+import java.util.function.Function;
import java.util.function.IntPredicate;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;
@@ -3245,13 +3247,15 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable,
SegmentInfos committingSegmentInfos = toCommit;
onCommitMerges = updatePendingMerges(new OneMergeWrappingMergePolicy(config.getMergePolicy(), toWrap ->
new MergePolicy.OneMerge(toWrap.segments) {
+ SegmentCommitInfo origInfo;
+ AtomicBoolean onlyOnce = new AtomicBoolean(false);
@Override
public void mergeFinished(boolean committed, boolean segmentDropped) throws IOException {
assert Thread.holdsLock(IndexWriter.this);
if (segmentDropped == false
&& committed
&& includeInCommit.get()) {
- deleter.incRef(info.files());
+ deleter.incRef(origInfo.files());
Set<String> mergedSegmentNames = new HashSet<>();
for (SegmentCommitInfo sci : segments) {
mergedSegmentNames.add(sci.info.name);
@@ -3265,8 +3269,8 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable,
}
// Construct a OneMerge that applies to toCommit
MergePolicy.OneMerge applicableMerge = new MergePolicy.OneMerge(toCommitMergedAwaySegments);
- applicableMerge.info = info.clone();
- long segmentCounter = Long.parseLong(info.info.name.substring(1), Character.MAX_RADIX);
+ applicableMerge.info = origInfo;
+ long segmentCounter = Long.parseLong(origInfo.info.name.substring(1), Character.MAX_RADIX);
committingSegmentInfos.counter = Math.max(committingSegmentInfos.counter, segmentCounter + 1);
committingSegmentInfos.applyMergeChanges(applicableMerge, false);
}
@@ -3275,14 +3279,30 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable,
}
@Override
+ void onMergeCommit() {
+ origInfo = this.info.clone();
+ }
+
+ @Override
+ void setMergeReaders(IOContext mergeContext, ReaderPool readerPool) throws IOException {
+ if (onlyOnce.compareAndSet(false, true)) {
+ super.setMergeReaders(mergeContext, readerPool);
+ }
+ }
+
+ @Override
public CodecReader wrapForMerge(CodecReader reader) throws IOException {
return toWrap.wrapForMerge(reader);
}
}
), MergeTrigger.COMMIT, UNBOUNDED_MAX_MERGE_SEGMENTS);
+ if (onCommitMerges != null) {
+ for (MergePolicy.OneMerge merge : onCommitMerges.merges) {
+ // TODO we need to release these readers in the case of an aborted merge
+ merge.setMergeReaders(IOContext.DEFAULT, readerPool);
+ }
+ }
}
-
-
}
success = true;
} finally {
@@ -3907,6 +3927,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable,
@SuppressWarnings("try")
private synchronized boolean commitMerge(MergePolicy.OneMerge merge, MergeState mergeState) throws IOException {
+ merge.onMergeCommit();
testPoint("startCommitMerge");
if (tragedy.get() != null) {
@@ -4419,35 +4440,11 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable,
infoStream.message("IW", "merging " + segString(merge.segments));
}
- merge.readers = new ArrayList<>(sourceSegments.size());
- merge.hardLiveDocs = new ArrayList<>(sourceSegments.size());
-
// This is try/finally to make sure merger's readers are
// closed:
boolean success = false;
try {
- int segUpto = 0;
- while(segUpto < sourceSegments.size()) {
-
- final SegmentCommitInfo info = sourceSegments.get(segUpto);
-
- // Hold onto the "live" reader; we will use this to
- // commit merged deletes
- final ReadersAndUpdates rld = getPooledInstance(info, true);
- rld.setIsMerging();
-
- ReadersAndUpdates.MergeReader mr = rld.getReaderForMerge(context);
- SegmentReader reader = mr.reader;
-
- if (infoStream.isEnabled("IW")) {
- infoStream.message("IW", "seg=" + segString(info) + " reader=" + reader);
- }
-
- merge.hardLiveDocs.add(mr.hardLiveDocs);
- merge.readers.add(reader);
- segUpto++;
- }
-
+ merge.setMergeReaders(context, readerPool);
// Let the merge wrap readers
List<CodecReader> mergeReaders = new ArrayList<>();
Counter softDeleteCount = Counter.newCounter(false);
diff --git a/lucene/core/src/java/org/apache/lucene/index/MergePolicy.java b/lucene/core/src/java/org/apache/lucene/index/MergePolicy.java
index 80c7b0d..8d5c456 100644
--- a/lucene/core/src/java/org/apache/lucene/index/MergePolicy.java
+++ b/lucene/core/src/java/org/apache/lucene/index/MergePolicy.java
@@ -33,11 +33,14 @@ import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.ReentrantLock;
import java.util.function.BooleanSupplier;
+import java.util.function.Function;
+import java.util.function.Supplier;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;
import org.apache.lucene.document.Field;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.MergeInfo;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.IOSupplier;
@@ -215,8 +218,8 @@ public abstract class MergePolicy {
// Sum of sizeInBytes of all SegmentInfos; set by IW.mergeInit
volatile long totalMergeBytes;
- List<SegmentReader> readers; // used by IndexWriter
- List<Bits> hardLiveDocs; // used by IndexWriter
+ final List<SegmentReader> readers; // used by IndexWriter
+ final List<Bits> hardLiveDocs; // used by IndexWriter
/** Segments to be merged. */
public final List<SegmentCommitInfo> segments;
@@ -243,6 +246,9 @@ public abstract class MergePolicy {
this.segments = List.copyOf(segments);
totalMaxDoc = segments.stream().mapToInt(i -> i.info.maxDoc()).sum();
mergeProgress = new OneMergeProgress();
+ hardLiveDocs = new ArrayList<>(segments.size());
+ readers = new ArrayList<>(segments.size());
+
}
/**
@@ -403,6 +409,22 @@ public abstract class MergePolicy {
Optional<Boolean> hasCompletedSuccessfully() {
return Optional.ofNullable(mergeCompleted.getNow(null));
}
+
+ void onMergeCommit() {
+ }
+
+ void setMergeReaders(IOContext mergeContext, ReaderPool readerPool) throws IOException {
+ for (final SegmentCommitInfo info : segments) {
+ // Hold onto the "live" reader; we will use this to
+ // commit merged deletes
+ final ReadersAndUpdates rld = readerPool.get(info, true);
+ rld.setIsMerging();
+ ReadersAndUpdates.MergeReader mr = rld.getReaderForMerge(mergeContext);
+ SegmentReader reader = mr.reader;
+ hardLiveDocs.add(mr.hardLiveDocs);
+ readers.add(reader);
+ }
+ }
}
/**
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java
index 654c6a9..1d78497 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java
@@ -404,10 +404,12 @@ public class TestIndexWriterMergePolicy extends LuceneTestCase {
writer.updateDocument(new Term("id", "2"), d2);
}
writer.flush();
- waitForUpdate.countDown();
} catch (Exception e) {
throw new AssertionError(e);
+ } finally {
+ waitForUpdate.countDown();
}
+
});
t.start();
writer.commit();
[lucene-solr] 07/07: apply comments from #1601
Posted by si...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
simonw pushed a commit to branch jira/lucene-8962
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 2aa937ef156bd461b423d50abcdb4c01c2af9d79
Author: Simon Willnauer <si...@apache.org>
AuthorDate: Wed Jun 24 18:50:04 2020 +0200
apply comments from #1601
---
.../java/org/apache/lucene/index/IndexWriter.java | 53 ++++++++++-----------
.../java/org/apache/lucene/index/MergePolicy.java | 55 +++++++++++++++++-----
.../org/apache/lucene/index/ReadersAndUpdates.java | 14 +-----
.../lucene/index/TestIndexWriterMergePolicy.java | 16 +++++--
4 files changed, 81 insertions(+), 57 deletions(-)
diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
index ff4817b..911852a 100644
--- a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
@@ -18,7 +18,6 @@ package org.apache.lucene.index;
import java.io.Closeable;
import java.io.IOException;
-import java.io.UncheckedIOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -3284,9 +3283,9 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable,
}
@Override
- void setMergeReaders(IOContext mergeContext, ReaderPool readerPool) throws IOException {
+ void initMergeReaders(IOContext mergeContext, Function<SegmentCommitInfo, ReadersAndUpdates> readerFactory) throws IOException {
if (onlyOnce.compareAndSet(false, true)) {
- super.setMergeReaders(mergeContext, readerPool);
+ super.initMergeReaders(mergeContext, readerFactory);
}
}
@@ -3299,7 +3298,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable,
if (onCommitMerges != null) {
for (MergePolicy.OneMerge merge : onCommitMerges.merges) {
// TODO we need to release these readers in the case of an aborted merge
- merge.setMergeReaders(IOContext.DEFAULT, readerPool);
+ merge.initMergeReaders(IOContext.DEFAULT, sci -> getPooledInstance(sci, true));
}
}
}
@@ -3773,7 +3772,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable,
MergeState.DocMap segDocMap = mergeState.docMaps[i];
MergeState.DocMap segLeafDocMap = mergeState.leafDocMaps[i];
- carryOverHardDeletes(mergedDeletesAndUpdates, maxDoc, mergeState.liveDocs[i], merge.hardLiveDocs.get(i), rld.getHardLiveDocs(),
+ carryOverHardDeletes(mergedDeletesAndUpdates, maxDoc, mergeState.liveDocs[i], merge.getMergeReader().get(i).hardLiveDocs, rld.getHardLiveDocs(),
segDocMap, segLeafDocMap);
// Now carry over all doc values updates that were resolved while we were merging, remapping the docIDs to the newly merged docIDs.
@@ -3926,7 +3925,6 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable,
@SuppressWarnings("try")
private synchronized boolean commitMerge(MergePolicy.OneMerge merge, MergeState mergeState) throws IOException {
-
merge.onMergeCommit();
testPoint("startCommitMerge");
@@ -4369,25 +4367,22 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable,
// first call mergeFinished before we potentially drop the reader and the last reference.
merge.mergeFinished(suppressExceptions == false, droppedSegment);
} finally {
- try {
- IOUtils.applyToAll(merge.readers, sr -> {
- final ReadersAndUpdates rld = getPooledInstance(sr.getOriginalSegmentInfo(), false);
- // We still hold a ref so it should not have been removed:
- assert rld != null;
- if (drop) {
- rld.dropChanges();
- } else {
- rld.dropMergingUpdates();
- }
- rld.release(sr);
- release(rld);
- if (drop) {
- readerPool.drop(rld.info);
- }
- });
- } finally {
- Collections.fill(merge.readers, null);
- }
+ IOUtils.applyToAll(merge.clearMergeReader(), mr -> {
+ final SegmentReader sr = mr.reader;
+ final ReadersAndUpdates rld = getPooledInstance(sr.getOriginalSegmentInfo(), false);
+ // We still hold a ref so it should not have been removed:
+ assert rld != null;
+ if (drop) {
+ rld.dropChanges();
+ } else {
+ rld.dropMergingUpdates();
+ }
+ rld.release(sr);
+ release(rld);
+ if (drop) {
+ readerPool.drop(rld.info);
+ }
+ });
}
}
@@ -4444,17 +4439,17 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable,
// closed:
boolean success = false;
try {
- merge.setMergeReaders(context, readerPool);
+ merge.initMergeReaders(context, sci -> getPooledInstance(sci, true));
// Let the merge wrap readers
List<CodecReader> mergeReaders = new ArrayList<>();
Counter softDeleteCount = Counter.newCounter(false);
- for (int r = 0; r < merge.readers.size(); r++) {
- SegmentReader reader = merge.readers.get(r);
+ for (MergePolicy.MergeReader mergeReader : merge.getMergeReader()) {
+ SegmentReader reader = mergeReader.reader;
CodecReader wrappedReader = merge.wrapForMerge(reader);
validateMergeReader(wrappedReader);
if (softDeletesEnabled) {
if (reader != wrappedReader) { // if we don't have a wrapped reader we won't preserve any soft-deletes
- Bits hardLiveDocs = merge.hardLiveDocs.get(r);
+ Bits hardLiveDocs = mergeReader.hardLiveDocs;
if (hardLiveDocs != null) { // we only need to do this accounting if we have mixed deletes
Bits wrappedLiveDocs = wrappedReader.getLiveDocs();
Counter hardDeleteCounter = Counter.newCounter(false);
diff --git a/lucene/core/src/java/org/apache/lucene/index/MergePolicy.java b/lucene/core/src/java/org/apache/lucene/index/MergePolicy.java
index 8d5c456..16decf6 100644
--- a/lucene/core/src/java/org/apache/lucene/index/MergePolicy.java
+++ b/lucene/core/src/java/org/apache/lucene/index/MergePolicy.java
@@ -34,7 +34,6 @@ import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.ReentrantLock;
import java.util.function.BooleanSupplier;
import java.util.function.Function;
-import java.util.function.Supplier;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;
@@ -218,8 +217,7 @@ public abstract class MergePolicy {
// Sum of sizeInBytes of all SegmentInfos; set by IW.mergeInit
volatile long totalMergeBytes;
- final List<SegmentReader> readers; // used by IndexWriter
- final List<Bits> hardLiveDocs; // used by IndexWriter
+ private List<MergeReader> mergeReaders; // used by IndexWriter
/** Segments to be merged. */
public final List<SegmentCommitInfo> segments;
@@ -246,9 +244,7 @@ public abstract class MergePolicy {
this.segments = List.copyOf(segments);
totalMaxDoc = segments.stream().mapToInt(i -> i.info.maxDoc()).sum();
mergeProgress = new OneMergeProgress();
- hardLiveDocs = new ArrayList<>(segments.size());
- readers = new ArrayList<>(segments.size());
-
+ mergeReaders = List.of();
}
/**
@@ -410,20 +406,45 @@ public abstract class MergePolicy {
return Optional.ofNullable(mergeCompleted.getNow(null));
}
+
+ /**
+ * Called before the merge is committed
+ */
void onMergeCommit() {
}
- void setMergeReaders(IOContext mergeContext, ReaderPool readerPool) throws IOException {
+ /**
+ * Sets the merge readers for this merge.
+ */
+ void initMergeReaders(IOContext mergeContext, Function<SegmentCommitInfo, ReadersAndUpdates> readerFactory) throws IOException {
+ assert mergeReaders.isEmpty() : "merge readers must be empty";
+ assert mergeCompleted.isDone() == false : "merge is already done";
+ ArrayList<MergeReader> readers = new ArrayList<>(segments.size());
for (final SegmentCommitInfo info : segments) {
// Hold onto the "live" reader; we will use this to
// commit merged deletes
- final ReadersAndUpdates rld = readerPool.get(info, true);
+ final ReadersAndUpdates rld = readerFactory.apply(info);
rld.setIsMerging();
- ReadersAndUpdates.MergeReader mr = rld.getReaderForMerge(mergeContext);
- SegmentReader reader = mr.reader;
- hardLiveDocs.add(mr.hardLiveDocs);
- readers.add(reader);
+ readers.add(rld.getReaderForMerge(mergeContext));
}
+ this.mergeReaders = List.copyOf(readers);
+ }
+
+ /**
+ * Returns the merge readers or an empty list if the readers were not initialized yet.
+ */
+ List<MergeReader> getMergeReader() {
+ return mergeReaders;
+ }
+
+ /**
+ * Clears the list of merge readers;
+ */
+ List<MergeReader> clearMergeReader() {
+ assert mergeCompleted.isDone();
+ List<MergeReader> readers = mergeReaders;
+ mergeReaders = List.of();
+ return readers;
}
}
@@ -793,4 +814,14 @@ public abstract class MergePolicy {
*/
Set<SegmentCommitInfo> getMergingSegments();
}
+
+ final static class MergeReader {
+ final SegmentReader reader;
+ final Bits hardLiveDocs;
+
+ MergeReader(SegmentReader reader, Bits hardLiveDocs) {
+ this.reader = reader;
+ this.hardLiveDocs = hardLiveDocs;
+ }
+ }
}
diff --git a/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java b/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java
index b0ee8d68..505f08f 100644
--- a/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java
+++ b/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java
@@ -695,18 +695,8 @@ final class ReadersAndUpdates {
return isMerging;
}
- final static class MergeReader {
- final SegmentReader reader;
- final Bits hardLiveDocs;
-
- MergeReader(SegmentReader reader, Bits hardLiveDocs) {
- this.reader = reader;
- this.hardLiveDocs = hardLiveDocs;
- }
- }
-
/** Returns a reader for merge, with the latest doc values updates and deletions. */
- synchronized MergeReader getReaderForMerge(IOContext context) throws IOException {
+ synchronized MergePolicy.MergeReader getReaderForMerge(IOContext context) throws IOException {
// We must carry over any still-pending DV updates because they were not
// successfully written, e.g. because there was a hole in the delGens,
@@ -728,7 +718,7 @@ final class ReadersAndUpdates {
reader = createNewReaderWithLatestLiveDocs(reader);
}
assert pendingDeletes.verifyDocCounts(reader);
- return new MergeReader(reader, pendingDeletes.getHardLiveDocs());
+ return new MergePolicy.MergeReader(reader, pendingDeletes.getHardLiveDocs());
}
/**
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java
index 1c6ba26..56eccc3 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java
@@ -392,9 +392,17 @@ public class TestIndexWriterMergePolicy extends LuceneTestCase {
d1.add(new StringField("id", "1", Field.Store.NO));
Document d2 = new Document();
d2.add(new StringField("id", "2", Field.Store.NO));
+ Document d3 = new Document();
+ d3.add(new StringField("id", "3", Field.Store.NO));
writer.addDocument(d1);
writer.flush();
writer.addDocument(d2);
+ boolean addThreeDocs = random().nextBoolean();
+ int expectedNumDocs = 2;
+ if (addThreeDocs) { // sometimes add another doc to ensure we don't have a fully deleted segment
+ expectedNumDocs = 3;
+ writer.addDocument(d3);
+ }
Thread t = new Thread(() -> {
try {
waitForMerge.await();
@@ -415,13 +423,13 @@ public class TestIndexWriterMergePolicy extends LuceneTestCase {
writer.commit();
t.join();
try (DirectoryReader open = new SoftDeletesDirectoryReaderWrapper(DirectoryReader.open(directory), "soft_delete")) {
- assertEquals(2, open.numDocs());
- assertEquals("we should not have any deletes", 2, open.maxDoc());
+ assertEquals(expectedNumDocs, open.numDocs());
+ assertEquals("we should not have any deletes", expectedNumDocs, open.maxDoc());
}
try (DirectoryReader open = DirectoryReader.open(writer)) {
- assertEquals(2, open.numDocs());
- assertEquals("we should not have one delete", 3, open.maxDoc());
+ assertEquals(expectedNumDocs, open.numDocs());
+ assertEquals("we should not have one delete", expectedNumDocs+1, open.maxDoc());
}
}
}
[lucene-solr] 04/07: LUCENE-8962: Ensure we don't include fully
deleted segments in a commit
Posted by si...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
simonw pushed a commit to branch jira/lucene-8962
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 189244b60a137626d5029e1e56899da3acb2f340
Author: Simon Willnauer <si...@apache.org>
AuthorDate: Mon Jun 22 19:44:06 2020 +0200
LUCENE-8962: Ensure we don't include fully deleted segments in a commit
IW might drop segments that are merged into a fully deleted segment on the floor
and deletes the newly created files right away. We should not include these segments
in a commit since we can't guarantee valid ref-counts on these files.
---
.../java/org/apache/lucene/index/IndexWriter.java | 71 ++++++++++++----------
.../java/org/apache/lucene/index/MergePolicy.java | 6 +-
.../lucene/index/TestDemoParallelLeafReader.java | 2 +-
.../org/apache/lucene/index/TestIndexWriter.java | 2 +-
.../org/apache/lucene/index/TestMergePolicy.java | 10 +--
5 files changed, 50 insertions(+), 41 deletions(-)
diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
index bd273a1..88dc399 100644
--- a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
@@ -3234,15 +3234,23 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable,
// corresponding add from an updateDocument) can
// sneak into the commit point:
toCommit = segmentInfos.clone();
-
+ pendingCommitChangeCount = changeCount.get();
+ // This protects the segmentInfos we are now going
+ // to commit. This is important in case, eg, while
+ // we are trying to sync all referenced files, a
+ // merge completes which would otherwise have
+ // removed the files we are now syncing.
+ deleter.incRef(toCommit.files(false));
if (anyChanges && maxCommitMergeWaitSeconds > 0) {
SegmentInfos committingSegmentInfos = toCommit;
onCommitMerges = updatePendingMerges(new OneMergeWrappingMergePolicy(config.getMergePolicy(), toWrap ->
new MergePolicy.OneMerge(toWrap.segments) {
@Override
- public void mergeFinished(boolean committed) throws IOException {
+ public void mergeFinished(boolean committed, boolean segmentDropped) throws IOException {
assert Thread.holdsLock(IndexWriter.this);
- if (committed && includeInCommit.get()) {
+ if (segmentDropped == false
+ && committed
+ && includeInCommit.get()) {
deleter.incRef(info.files());
Set<String> mergedSegmentNames = new HashSet<>();
for (SegmentCommitInfo sci : segments) {
@@ -3262,8 +3270,8 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable,
committingSegmentInfos.counter = Math.max(committingSegmentInfos.counter, segmentCounter + 1);
committingSegmentInfos.applyMergeChanges(applicableMerge, false);
}
- toWrap.mergeFinished(committed);
- super.mergeFinished(committed);
+ toWrap.mergeFinished(committed, false);
+ super.mergeFinished(committed, segmentDropped);
}
@Override
@@ -3274,14 +3282,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable,
), MergeTrigger.COMMIT, UNBOUNDED_MAX_MERGE_SEGMENTS);
}
- pendingCommitChangeCount = changeCount.get();
- // This protects the segmentInfos we are now going
- // to commit. This is important in case, eg, while
- // we are trying to sync all referenced files, a
- // merge completes which would otherwise have
- // removed the files we are now syncing.
- deleter.incRef(toCommit.files(false));
}
success = true;
} finally {
@@ -4019,7 +4020,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable,
// Must close before checkpoint, otherwise IFD won't be
// able to delete the held-open files from the merge
// readers:
- closeMergeReaders(merge, false);
+ closeMergeReaders(merge, false, dropSegment);
}
if (infoStream.isEnabled("IW")) {
@@ -4341,27 +4342,33 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable,
}
@SuppressWarnings("try")
- private synchronized void closeMergeReaders(MergePolicy.OneMerge merge, boolean suppressExceptions) throws IOException {
+ private synchronized void closeMergeReaders(MergePolicy.OneMerge merge, boolean suppressExceptions, boolean droppedSegment) throws IOException {
final boolean drop = suppressExceptions == false;
- try (Closeable finalizer = () -> merge.mergeFinished(suppressExceptions == false)) {
- IOUtils.applyToAll(merge.readers, sr -> {
- final ReadersAndUpdates rld = getPooledInstance(sr.getOriginalSegmentInfo(), false);
- // We still hold a ref so it should not have been removed:
- assert rld != null;
- if (drop) {
- rld.dropChanges();
- } else {
- rld.dropMergingUpdates();
- }
- rld.release(sr);
- release(rld);
- if (drop) {
- readerPool.drop(rld.info);
- }
- });
+ try {
+ // first call mergeFinished before we potentially drop the reader and the last reference.
+ merge.mergeFinished(suppressExceptions == false, droppedSegment);
} finally {
- Collections.fill(merge.readers, null);
+ try {
+ IOUtils.applyToAll(merge.readers, sr -> {
+ final ReadersAndUpdates rld = getPooledInstance(sr.getOriginalSegmentInfo(), false);
+ // We still hold a ref so it should not have been removed:
+ assert rld != null;
+ if (drop) {
+ rld.dropChanges();
+ } else {
+ rld.dropMergingUpdates();
+ }
+ rld.release(sr);
+ release(rld);
+ if (drop) {
+ readerPool.drop(rld.info);
+ }
+ });
+ } finally {
+ Collections.fill(merge.readers, null);
+ }
}
+
}
private void countSoftDeletes(CodecReader reader, Bits wrappedLiveDocs, Bits hardLiveDocs, Counter softDeleteCounter,
@@ -4661,7 +4668,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable,
// Readers are already closed in commitMerge if we didn't hit
// an exc:
if (success == false) {
- closeMergeReaders(merge, true);
+ closeMergeReaders(merge, true, false);
}
}
diff --git a/lucene/core/src/java/org/apache/lucene/index/MergePolicy.java b/lucene/core/src/java/org/apache/lucene/index/MergePolicy.java
index 5a090da..80c7b0d 100644
--- a/lucene/core/src/java/org/apache/lucene/index/MergePolicy.java
+++ b/lucene/core/src/java/org/apache/lucene/index/MergePolicy.java
@@ -254,8 +254,10 @@ public abstract class MergePolicy {
}
/** Called by {@link IndexWriter} after the merge is done and all readers have been closed.
- * @param success true iff the merge finished successfully ie. was committed */
- public void mergeFinished(boolean success) throws IOException {
+ * @param success true iff the merge finished successfully ie. was committed
+ * @param segmentDropped true iff the merged segment was dropped since it was fully deleted
+ */
+ public void mergeFinished(boolean success, boolean segmentDropped) throws IOException {
mergeCompleted.complete(success);
// https://issues.apache.org/jira/browse/LUCENE-9408
// if (mergeCompleted.complete(success) == false) {
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDemoParallelLeafReader.java b/lucene/core/src/test/org/apache/lucene/index/TestDemoParallelLeafReader.java
index 7fdad3b..2ab2c92 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestDemoParallelLeafReader.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestDemoParallelLeafReader.java
@@ -538,7 +538,7 @@ public class TestDemoParallelLeafReader extends LuceneTestCase {
}
@Override
- public void mergeFinished(boolean success) throws IOException {
+ public void mergeFinished(boolean success, boolean segmentDropped) throws IOException {
Throwable th = null;
for (ParallelLeafReader r : parallelReaders) {
try {
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
index 5c6164a..41bd607 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
@@ -4181,7 +4181,7 @@ public class TestIndexWriter extends LuceneTestCase {
SetOnce<Boolean> onlyFinishOnce = new SetOnce<>();
return new MergePolicy.OneMerge(merge.segments) {
@Override
- public void mergeFinished(boolean success) {
+ public void mergeFinished(boolean success, boolean segmentDropped) {
onlyFinishOnce.set(true);
}
};
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestMergePolicy.java b/lucene/core/src/test/org/apache/lucene/index/TestMergePolicy.java
index e5f5635..0ff32e6 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestMergePolicy.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestMergePolicy.java
@@ -43,7 +43,7 @@ public class TestMergePolicy extends LuceneTestCase {
Thread t = new Thread(() -> {
try {
for (MergePolicy.OneMerge m : ms.merges) {
- m.mergeFinished(true);
+ m.mergeFinished(true, false);
}
} catch (IOException e) {
throw new AssertionError(e);
@@ -66,7 +66,7 @@ public class TestMergePolicy extends LuceneTestCase {
}
Thread t = new Thread(() -> {
try {
- ms.merges.get(0).mergeFinished(true);
+ ms.merges.get(0).mergeFinished(true, false);
} catch (IOException e) {
throw new AssertionError(e);
}
@@ -89,7 +89,7 @@ public class TestMergePolicy extends LuceneTestCase {
Thread t = new Thread(() -> {
while (stop.get() == false) {
try {
- ms.merges.get(i.getAndIncrement()).mergeFinished(true);
+ ms.merges.get(i.getAndIncrement()).mergeFinished(true, false);
Thread.sleep(1);
} catch (IOException | InterruptedException e) {
throw new AssertionError(e);
@@ -115,8 +115,8 @@ public class TestMergePolicy extends LuceneTestCase {
try (Directory dir = newDirectory()) {
MergePolicy.MergeSpecification spec = createRandomMergeSpecification(dir, 1);
MergePolicy.OneMerge oneMerge = spec.merges.get(0);
- oneMerge.mergeFinished(true);
- expectThrows(IllegalStateException.class, () -> oneMerge.mergeFinished(false));
+ oneMerge.mergeFinished(true, false);
+ expectThrows(IllegalStateException.class, () -> oneMerge.mergeFinished(false, false));
}
}
[lucene-solr] 02/07: add failing test for carry-over deletes
Posted by si...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
simonw pushed a commit to branch jira/lucene-8962
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit c29875f964deb4cd04d88ac9b357f0fa0e615237
Author: Simon Willnauer <si...@apache.org>
AuthorDate: Tue Jun 23 08:56:42 2020 +0200
add failing test for carry-over deletes
---
.../lucene/index/TestIndexWriterMergePolicy.java | 47 ++++++++++++++++++++++
1 file changed, 47 insertions(+)
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java
index 2577f6b..08222ef 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java
@@ -20,10 +20,12 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
+import java.util.concurrent.CountDownLatch;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
+import org.apache.lucene.document.StringField;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.MatchAllDocsQuery;
@@ -363,4 +365,49 @@ public class TestIndexWriterMergePolicy extends LuceneTestCase {
// TODO: Add more checks for other non-double setters!
}
+
+ public void testCarryOverNewDeletes() throws IOException, InterruptedException {
+ try (Directory directory = newDirectory()) {
+ CountDownLatch waitForMerge = new CountDownLatch(1);
+ CountDownLatch waitForUpdate = new CountDownLatch(1);
+ try (IndexWriter writer = new IndexWriter(directory, newIndexWriterConfig()
+ .setMergePolicy(MERGE_ON_COMMIT_POLICY).setMaxCommitMergeWaitSeconds(30)
+ .setMergeScheduler(new ConcurrentMergeScheduler())) {
+ @Override
+ protected void merge(MergePolicy.OneMerge merge) throws IOException {
+ waitForMerge.countDown();
+ try {
+ waitForUpdate.await();
+ } catch (InterruptedException e) {
+ throw new AssertionError(e);
+ }
+ super.merge(merge);
+ }
+ }) {
+ Document d1 = new Document();
+ d1.add(new StringField("id", "1", Field.Store.NO));
+ Document d2 = new Document();
+ d2.add(new StringField("id", "2", Field.Store.NO));
+ writer.addDocument(d1);
+ writer.flush();
+ writer.addDocument(d2);
+ Thread t = new Thread(() -> {
+ try {
+ waitForMerge.await();
+ writer.updateDocument(new Term("id", "2"), d2);
+ writer.flush();
+ waitForUpdate.countDown();
+ } catch (Exception e) {
+ throw new AssertionError(e);
+ }
+ });
+ t.start();
+ writer.commit();
+ t.join();
+ try (DirectoryReader open = DirectoryReader.open(directory)) {
+ assertEquals(2, open.numDocs());
+ }
+ }
+ }
+ }
}
[lucene-solr] 01/07: Merge branch 'master' into jira/lucene-8962
Posted by si...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
simonw pushed a commit to branch jira/lucene-8962
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit ea750da32128aaf0b5bdf5c62c7bfa2655b87b4f
Merge: 0a096f7 7030bb5
Author: Simon Willnauer <si...@apache.org>
AuthorDate: Wed Jun 24 18:12:37 2020 +0200
Merge branch 'master' into jira/lucene-8962
.gitignore | 1 +
build.gradle | 2 +
dev-tools/scripts/reproduceJenkinsFailures.py | 5 +-
gradle/defaults-java.gradle | 1 +
gradle/hacks/findbugs.gradle | 45 ++
gradle/validation/jar-checks.gradle | 4 +-
help/ant.txt | 4 +-
lucene/CHANGES.txt | 13 +
lucene/MIGRATE.md | 6 +
.../apache/lucene/index/DefaultIndexingChain.java | 100 ++--
.../java/org/apache/lucene/index/DocConsumer.java | 2 +-
.../org/apache/lucene/index/DocumentsWriter.java | 2 +-
.../lucene/index/DocumentsWriterPerThread.java | 51 +-
.../apache/lucene/index/TermVectorsConsumer.java | 6 +-
.../java/org/apache/lucene/index/TermsHash.java | 10 +-
.../apache/lucene/search/ConstantScoreQuery.java | 2 +-
.../org/apache/lucene/search/FieldComparator.java | 16 +-
.../apache/lucene/search/FieldValueHitQueue.java | 29 +-
.../lucene/search/FilteringFieldComparator.java | 93 ++++
.../search/FilteringLeafFieldComparator.java | 39 ++
.../lucene/search/FilteringNumericComparator.java | 52 +++
.../search/FilteringNumericLeafComparator.java | 336 ++++++++++++++
.../org/apache/lucene/search/LeafCollector.java | 12 +
.../apache/lucene/search/MatchAllDocsQuery.java | 2 +-
.../org/apache/lucene/search/MultiCollector.java | 111 +++--
.../java/org/apache/lucene/search/ScoreMode.java | 54 ++-
.../java/org/apache/lucene/search/SortField.java | 4 +
.../apache/lucene/search/TopFieldCollector.java | 69 ++-
.../src/java/org/apache/lucene/search/Weight.java | 18 +-
.../lucene/index/TestAllFilesDetectTruncation.java | 4 +-
.../apache/lucene/search/MultiCollectorTest.java | 172 +++++++
.../search/TestFieldSortOptimizationSkipping.java | 294 ++++++++++++
.../lucene/search/AssertingLeafCollector.java | 6 +-
.../search/similarities/AssertingSimilarity.java | 2 +-
solr/CHANGES.txt | 34 +-
solr/bin/solr | 2 +-
solr/bin/solr.in.cmd | 8 +-
solr/bin/solr.in.sh | 9 +-
.../analytics/legacy/LegacyNoFacetCloudTest.java | 32 +-
.../solr/analytics/legacy/LegacyNoFacetTest.java | 32 +-
.../LegacyAbstractAnalyticsFacetCloudTest.java | 2 +-
.../facet/LegacyAbstractAnalyticsFacetTest.java | 4 +-
.../legacy/facet/LegacyFieldFacetTest.java | 1 +
.../dataimporthandler/src/test-files/log4j2.xml | 42 ++
.../src/test-files/solr/collection1/README | 1 +
.../solr/configsets/dihconfigset/conf/README | 2 +
.../solr/configsets/dihconfigset/conf/schema.xml | 70 +++
.../configsets/dihconfigset/conf/solrconfig.xml | 287 ++++++++++++
.../dataimporthandler/src/test-files/solr/solr.xml | 27 ++
.../handler/dataimport/TestZKPropertiesWriter.java | 207 +++++++--
.../extraction/ExtractingRequestHandlerTest.java | 3 +
.../apache/solr/jaeger/TestJaegerConfigurator.java | 4 +
.../TestFeatureExtractionFromMultipleSegments.java | 2 +
.../solr/ltr/feature/TestNoMatchSolrFeature.java | 4 +
.../solr/ltr/feature/TestOriginalScoreFeature.java | 6 +
.../solr/ltr/model/TestNeuralNetworkModel.java | 1 +
.../store/rest/TestModelManagerPersistence.java | 4 +
.../src/java/org/apache/solr/cloud/LockTree.java | 54 ++-
.../cloud/OverseerConfigSetMessageHandler.java | 2 +-
.../apache/solr/cloud/OverseerMessageHandler.java | 7 +-
.../apache/solr/cloud/OverseerTaskProcessor.java | 180 +++-----
.../OverseerCollectionMessageHandler.java | 18 +-
.../solr/cloud/api/collections/RoutedAlias.java | 1 -
.../java/org/apache/solr/core/CoreContainer.java | 51 +-
.../java/org/apache/solr/core/CoreDescriptor.java | 8 +-
.../src/java/org/apache/solr/core/NodeConfig.java | 20 +-
.../src/java/org/apache/solr/core/SolrPaths.java | 41 +-
.../org/apache/solr/core/SolrResourceLoader.java | 18 +-
.../java/org/apache/solr/core/SolrXmlConfig.java | 16 +
.../apache/solr/filestore/DistribPackageStore.java | 2 +-
.../handler/component/QueryElevationComponent.java | 1 -
.../component/ShardFieldSortedHitQueue.java | 1 -
.../solr/highlight/DefaultSolrHighlighter.java | 511 +++++++++++----------
.../org/apache/solr/search/JoinQParserPlugin.java | 484 -------------------
.../{JoinQParserPlugin.java => JoinQuery.java} | 151 +-----
.../org/apache/solr/search/MaxScoreCollector.java | 6 +-
.../org/apache/solr/search/TermsQParserPlugin.java | 3 +
solr/core/src/test-files/log4j2.xml | 44 +-
solr/core/src/test-files/solr/solr-50-all.xml | 1 +
.../core/src/test-files/solr/solr-solrreporter.xml | 2 +
solr/core/src/test-files/solr/solr.xml | 1 +
.../src/test/org/apache/hadoop/fs/FileUtil.java | 2 +
.../org/apache/solr/BasicFunctionalityTest.java | 7 +
.../src/test/org/apache/solr/CursorPagingTest.java | 8 +
.../org/apache/solr/TestDistributedSearch.java | 10 +-
.../test/org/apache/solr/TestGroupingSearch.java | 7 +-
solr/core/src/test/org/apache/solr/TestJoin.java | 12 +-
.../apache/solr/cloud/AliasIntegrationTest.java | 1 -
.../solr/cloud/ChaosMonkeyNothingIsSafeTest.java | 1 +
.../org/apache/solr/cloud/CollectionPropsTest.java | 1 +
.../apache/solr/cloud/CollectionsAPISolrJTest.java | 8 +-
.../apache/solr/cloud/DistribCursorPagingTest.java | 1 +
.../DistribDocExpirationUpdateProcessorTest.java | 1 +
.../solr/cloud/FullSolrCloudDistribCmdsTest.java | 2 +-
.../solr/cloud/MetricsHistoryIntegrationTest.java | 4 +
.../test/org/apache/solr/cloud/MockSolrSource.java | 2 +-
.../apache/solr/cloud/MultiThreadedOCPTest.java | 1 +
.../solr/cloud/NestedShardedAtomicUpdateTest.java | 4 +
.../OverseerCollectionConfigSetProcessorTest.java | 1 +
.../solr/cloud/OverseerModifyCollectionTest.java | 1 +
.../test/org/apache/solr/cloud/OverseerTest.java | 1 +
.../apache/solr/cloud/ReindexCollectionTest.java | 2 +
.../apache/solr/cloud/ReplaceNodeNoTargetTest.java | 1 +
.../cloud/RoutingToNodesWithPropertiesTest.java | 1 +
.../apache/solr/cloud/SolrCloudExampleTest.java | 2 +
.../test/org/apache/solr/cloud/SyncSliceTest.java | 1 +
.../solr/cloud/SystemCollectionCompatTest.java | 1 +
.../apache/solr/cloud/TestBaseStatsCacheCloud.java | 1 +
.../apache/solr/cloud/TestCloudConsistency.java | 2 +
.../TestCloudPhrasesIdentificationComponent.java | 4 +
.../solr/cloud/TestCloudSearcherWarming.java | 2 +-
.../org/apache/solr/cloud/TestConfigSetsAPI.java | 16 +-
.../solr/cloud/TestConfigSetsAPIExclusivity.java | 4 +
.../org/apache/solr/cloud/TestHashPartitioner.java | 1 +
.../org/apache/solr/cloud/TestPullReplica.java | 4 +-
.../apache/solr/cloud/TestRebalanceLeaders.java | 1 +
.../apache/solr/cloud/TestSSLRandomization.java | 12 +-
.../cloud/TestSolrCloudWithDelegationTokens.java | 12 +-
.../TestSolrCloudWithSecureImpersonation.java | 3 +
.../cloud/TestStressCloudBlindAtomicUpdates.java | 16 +-
.../org/apache/solr/cloud/TestWithCollection.java | 6 +
.../apache/solr/cloud/UnloadDistributedZkTest.java | 2 +
.../org/apache/solr/cloud/ZkSolrClientTest.java | 12 +-
.../solr/cloud/api/collections/AssignTest.java | 1 +
.../CollectionsAPIDistributedZkTest.java | 6 +
.../ConcurrentCreateCollectionTest.java | 1 +
.../SimpleCollectionCreateDeleteTest.java | 1 +
.../cloud/api/collections/TestCollectionAPI.java | 54 +++
.../api/collections/TestReplicaProperties.java | 2 +
.../TestRequestStatusCollectionAPI.java | 2 +
.../AutoAddReplicasIntegrationTest.java | 4 +
.../autoscaling/AutoAddReplicasPlanActionTest.java | 10 +-
.../cloud/autoscaling/AutoScalingHandlerTest.java | 26 +-
.../cloud/autoscaling/ComputePlanActionTest.java | 27 +-
.../cloud/autoscaling/ExecutePlanActionTest.java | 5 +
.../cloud/autoscaling/HttpTriggerListenerTest.java | 1 +
.../IndexSizeTriggerMixedBoundsTest.java | 2 +
.../IndexSizeTriggerSizeEstimationTest.java | 3 +
.../cloud/autoscaling/IndexSizeTriggerTest.java | 15 +
.../autoscaling/MetricTriggerIntegrationTest.java | 1 +
.../NodeAddedTriggerIntegrationTest.java | 2 +
.../cloud/autoscaling/NodeAddedTriggerTest.java | 3 +
.../NodeLostTriggerIntegrationTest.java | 2 +
.../cloud/autoscaling/NodeLostTriggerTest.java | 3 +
.../autoscaling/NodeMarkersRegistrationTest.java | 2 +
.../cloud/autoscaling/RestoreTriggerStateTest.java | 2 +
.../ScheduledMaintenanceTriggerTest.java | 5 +
.../ScheduledTriggerIntegrationTest.java | 4 +-
.../SearchRateTriggerIntegrationTest.java | 4 +
.../cloud/autoscaling/SearchRateTriggerTest.java | 3 +
.../cloud/autoscaling/SystemLogListenerTest.java | 3 +
.../solr/cloud/autoscaling/TestPolicyCloud.java | 2 +
.../TriggerCooldownIntegrationTest.java | 1 +
.../cloud/autoscaling/TriggerEventQueueTest.java | 1 +
.../cloud/autoscaling/TriggerIntegrationTest.java | 5 +
.../autoscaling/sim/SimSolrCloudTestCase.java | 1 +
.../autoscaling/sim/TestSimComputePlanAction.java | 16 +-
.../autoscaling/sim/TestSimExecutePlanAction.java | 3 +
.../autoscaling/sim/TestSimExtremeIndexing.java | 1 +
.../cloud/autoscaling/sim/TestSimLargeCluster.java | 1 +
.../autoscaling/sim/TestSimNodeAddedTrigger.java | 2 +
.../autoscaling/sim/TestSimNodeLostTrigger.java | 3 +
.../cloud/autoscaling/sim/TestSimPolicyCloud.java | 2 +
.../cloud/autoscaling/sim/TestSimScenario.java | 4 +
.../autoscaling/sim/TestSimTriggerIntegration.java | 7 +
.../autoscaling/sim/TestSnapshotCloudManager.java | 1 +
.../solr/cloud/cdcr/BaseCdcrDistributedZkTest.java | 11 +
.../solr/cloud/cdcr/CdcrOpsAndBoundariesTest.java | 5 +-
.../solr/cloud/cdcr/CdcrRequestHandlerTest.java | 5 +
.../org/apache/solr/cloud/hdfs/StressHdfsTest.java | 2 +
.../solr/cloud/overseer/ZkStateWriterTest.java | 1 +
.../apache/solr/cloud/rule/ImplicitSnitchTest.java | 3 +
.../org/apache/solr/cloud/rule/RuleEngineTest.java | 8 +
.../test/org/apache/solr/cloud/rule/RulesTest.java | 7 +
.../org/apache/solr/core/DirectoryFactoryTest.java | 6 +-
.../org/apache/solr/core/ResourceLoaderTest.java | 4 +-
.../org/apache/solr/core/TestCoreContainer.java | 85 ++++
.../src/test/org/apache/solr/core/TestSolrXml.java | 10 +
.../solr/handler/admin/CoreAdminHandlerTest.java | 2 +
.../handler/admin/CoreAdminRequestStatusTest.java | 1 +
.../component/DistributedFacetPivotSmallTest.java | 12 +-
.../org/apache/solr/highlight/HighlighterTest.java | 2 +-
.../apache/solr/index/hdfs/CheckHdfsIndexTest.java | 2 +
.../org/apache/solr/logging/TestLogWatcher.java | 1 +
.../reporters/solr/SolrShardReporterTest.java | 12 +
.../src/test/org/apache/solr/pkg/TestPackages.java | 7 +-
.../org/apache/solr/request/SimpleFacetsTest.java | 1 +
.../apache/solr/request/TestIntervalFaceting.java | 2 +-
.../solr/request/macro/TestMacroExpander.java | 2 +
.../apache/solr/rest/schema/TestBulkSchemaAPI.java | 2 +-
.../apache/solr/search/MaxScoreCollectorTest.java | 91 ++++
.../apache/solr/search/SolrIndexSearcherTest.java | 8 +-
.../apache/solr/search/TestTermsQParserPlugin.java | 6 +
.../solr/security/AuditLoggerIntegrationTest.java | 8 +
.../solr/spelling/DirectSolrSpellCheckerTest.java | 3 +
.../solr/spelling/FileBasedSpellCheckerTest.java | 7 +
.../solr/spelling/IndexBasedSpellCheckerTest.java | 8 +
.../solr/spelling/SpellCheckCollatorTest.java | 9 +
.../solr/spelling/SpellingQueryConverterTest.java | 6 +
.../solr/spelling/suggest/SuggesterTest.java | 2 +
.../solr/store/blockcache/BlockCacheTest.java | 4 +-
.../apache/solr/store/hdfs/HdfsDirectoryTest.java | 2 +-
.../apache/solr/update/SolrIndexSplitterTest.java | 3 +
.../org/apache/solr/util/MockCoreContainer.java | 2 +-
solr/server/resources/log4j2-console.xml | 28 +-
solr/server/resources/log4j2.xml | 67 +--
solr/server/solr/solr.xml | 1 +
solr/solr-ref-guide/src/coreadmin-api.adoc | 4 +-
solr/solr-ref-guide/src/format-of-solr-xml.adoc | 4 +
.../solr/common/params/CollectionParams.java | 33 +-
.../apache/solr/common/util/CommandOperation.java | 1 -
.../solr/common/util/JsonSchemaValidator.java | 11 +-
solr/solrj/src/test-files/log4j2.xml | 45 +-
.../solr/client/solrj/request/TestCoreAdmin.java | 4 +-
.../apache/solr/BaseDistributedSearchTestCase.java | 2 +-
.../solr/cloud/AbstractDistribZkTestBase.java | 2 +-
.../solr/cloud/AbstractFullDistribZkTestBase.java | 4 +-
versions.lock | 4 +-
218 files changed, 3363 insertions(+), 1598 deletions(-)
[lucene-solr] 06/07: ensure we actually apply the updates in the
test
Posted by si...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
simonw pushed a commit to branch jira/lucene-8962
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit d9b12a00ab82a993683788913092bf918367876e
Author: Simon Willnauer <si...@apache.org>
AuthorDate: Wed Jun 24 18:28:57 2020 +0200
ensure we actually apply the updates in the test
---
.../test/org/apache/lucene/index/TestIndexWriterMergePolicy.java | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java
index 1d78497..1c6ba26 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java
@@ -416,6 +416,12 @@ public class TestIndexWriterMergePolicy extends LuceneTestCase {
t.join();
try (DirectoryReader open = new SoftDeletesDirectoryReaderWrapper(DirectoryReader.open(directory), "soft_delete")) {
assertEquals(2, open.numDocs());
+ assertEquals("we should not have any deletes", 2, open.maxDoc());
+ }
+
+ try (DirectoryReader open = DirectoryReader.open(writer)) {
+ assertEquals(2, open.numDocs());
+ assertEquals("we should not have one delete", 3, open.maxDoc());
}
}
}
[lucene-solr] 03/07: pimp test to randomly use soft-deletes which
also check if DV updates work accordingly
Posted by si...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
simonw pushed a commit to branch jira/lucene-8962
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit a533094c1479e9e350ad7f364e64f6ec362a62d0
Author: Simon Willnauer <si...@apache.org>
AuthorDate: Wed Jun 24 18:23:50 2020 +0200
pimp test to randomly use soft-deletes which also check if DV updates work accordingly
---
.../org/apache/lucene/index/TestIndexWriterMergePolicy.java | 12 ++++++++++--
1 file changed, 10 insertions(+), 2 deletions(-)
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java
index 08222ef..654c6a9 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java
@@ -25,6 +25,7 @@ import java.util.concurrent.CountDownLatch;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
+import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.search.IndexSearcher;
@@ -368,10 +369,12 @@ public class TestIndexWriterMergePolicy extends LuceneTestCase {
public void testCarryOverNewDeletes() throws IOException, InterruptedException {
try (Directory directory = newDirectory()) {
+ boolean useSoftDeletes = random().nextBoolean();
CountDownLatch waitForMerge = new CountDownLatch(1);
CountDownLatch waitForUpdate = new CountDownLatch(1);
try (IndexWriter writer = new IndexWriter(directory, newIndexWriterConfig()
.setMergePolicy(MERGE_ON_COMMIT_POLICY).setMaxCommitMergeWaitSeconds(30)
+ .setSoftDeletesField("soft_delete")
.setMergeScheduler(new ConcurrentMergeScheduler())) {
@Override
protected void merge(MergePolicy.OneMerge merge) throws IOException {
@@ -384,6 +387,7 @@ public class TestIndexWriterMergePolicy extends LuceneTestCase {
super.merge(merge);
}
}) {
+
Document d1 = new Document();
d1.add(new StringField("id", "1", Field.Store.NO));
Document d2 = new Document();
@@ -394,7 +398,11 @@ public class TestIndexWriterMergePolicy extends LuceneTestCase {
Thread t = new Thread(() -> {
try {
waitForMerge.await();
- writer.updateDocument(new Term("id", "2"), d2);
+ if (useSoftDeletes) {
+ writer.softUpdateDocument(new Term("id", "2"), d2, new NumericDocValuesField("soft_delete", 1));
+ } else {
+ writer.updateDocument(new Term("id", "2"), d2);
+ }
writer.flush();
waitForUpdate.countDown();
} catch (Exception e) {
@@ -404,7 +412,7 @@ public class TestIndexWriterMergePolicy extends LuceneTestCase {
t.start();
writer.commit();
t.join();
- try (DirectoryReader open = DirectoryReader.open(directory)) {
+ try (DirectoryReader open = new SoftDeletesDirectoryReaderWrapper(DirectoryReader.open(directory), "soft_delete")) {
assertEquals(2, open.numDocs());
}
}