You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ab...@apache.org on 2019/03/12 18:35:30 UTC
[lucene-solr] branch jira/solr-11127-2 updated (1a01235 -> 116c6d7)
This is an automated email from the ASF dual-hosted git repository.
ab pushed a change to branch jira/solr-11127-2
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git.
from 1a01235 Merge branch 'master' into jira/solr-11127-2
add aeb0774 Adding 7.7.1 version
add de03017 SOLR-9882: picking results for mergeIds even after partial ones occurred
add 558c515 SOLR-13295: Reproducible failure in TestDistributedGrouping
add 5f91670 Clean up any test failures resulting from defaulting to async logging, reenable shutdown hook
add 3800535 SOLR-13287: Remove System.out debugging
add e827dc2 SOLR-13237: AwaitsFix LeaderTragicEventTest until we can make the underlying corruption detection reliable
new 675dd0b Merge branch 'master' into jira/solr-11127-2
add 4eb1c6f3c SOLR-12313: Removing TestInjection#waitForInSyncWithLeader
add 583ae97 SOLR-12313: Fix precommit
add 002a4aa SOLR-13302: Adding HttpPartitionTest and ForceLeaderTest for TLOG replicas
add d2d3942 SOLR-13296 fix doc example so that it can be accepted by Solr (previously caused error due to decreasing offsets)
add db5f1af Fix Windows startup script to disable HTTP/2 if TLS is enabled on Java 8.
add 1e09268 SOLR-13261: Make SortableTextField work with export/streaming, now requires useDocValuesAsStored='true'
add 683aa3d SOLR-13254: Correct message that is logged in solrj's ConnectionManager when an exception occurred while reconnecting to ZooKeeper. (hu xiaodong via Christine Poerschke)
add 20de3d2 SOLR-12923: harden TestSimLargeCluster
add 4335ad9 Delete OpenCloseCoreStressTest
add 9eabaf4 SOLR-12121: Move CHANGES entry from Improvements to New Features section
add 3b2d836 Merge remote-tracking branch 'refs/remotes/origin/master'
add 8c6e305 SOLR-12732: TestLogWatcher failure on Jenkins. Added more logging
add 27df824 SOLR-13292: Provide extended per-segment status of a collection.
add 9272c29 SOLR-13268: Clean up any test failures resulting from defaulting to async logging
add e353dcf SOLR-13284: fallback to json instead of NPE and 500 code, when wt is omitted or wrong.
add d8cfeba LUCENE-8652: SynonymQuery can now deboost the document frequency of each term when blending the score of the synonym
new 3f0b1c2 Merge branch 'master' into jira/solr-11127-2
new fb37b7c SOLR-11127: Improved error handling, bug fixes, more tests.
add 6777af0 SOLR-13307: Ensure HDFS tests clear System properties they set (Kevin Risden)
add 7eb728a SOLR-13300: DistributedFacetExistsSmallTest should not attempt to compare results of a query that is known to differ in behavior in distributed mode
add 3a1603d SOLR-13268: Clean up any test failures resulting from defaulting to async logging. Fix precommit test to not include the baseDir
add 28c8171 SOLR-12732: TestLogWatcher failure on Jenkins. Deal with superclass logging bleeding into first watcher in the test
add b893548 SOLR-13268: Clean up any test failures resulting from defaulting to async logging. Put TestXmlQParser back
add 4582053 LUCENE-8712: Polygon2D does not detect crossings in some cases (#598)
add fd8d9d5 LUCENE-8713: Add Line2D tests
add c87e761 LUCENE-8652: ensure that the norm doesn't influence the score in TestSynonymQuery#testBoosts
add b1f870a LUCENE-8631: The Korean user dictionary now picks the longest-matching word and discards the other matches.
add b2c83de LUCENE-8652: remove unused import
add 9edc557 SOLR-12891 MacroExpander will no longer will expand URL parameters by default inside of the 'expr' parameter, add InjectionDefense class for safer handling of untrusted data in streaming expressions and add -DStreamingExpressionMacros system property to revert to legacy behavior
add c1bea96 LUCENE-8720: fix int overflow in NameIntCacheLRU
new 841938f Merge branch 'master' into jira/solr-11127-2
new 116c6d7 SOLR-11127: Add proper shutdown of daemon streams.
The 5 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
lucene/CHANGES.txt | 19 +
.../apache/lucene/analysis/ko/KoreanTokenizer.java | 23 +-
.../lucene/analysis/ko/TestKoreanTokenizer.java | 14 +
.../org/apache/lucene/analysis/ko/userdict.txt | 7 +-
.../src/java/org/apache/lucene/geo/EdgeTree.java | 47 +-
.../src/java/org/apache/lucene/geo/GeoUtils.java | 23 +-
.../src/java/org/apache/lucene/geo/Polygon2D.java | 4 +-
.../org/apache/lucene/search/SynonymQuery.java | 261 ++++++++--
.../java/org/apache/lucene/util/QueryBuilder.java | 8 +-
.../test/org/apache/lucene/geo/TestPolygon2D.java | 30 ++
.../apache/lucene/search/TestMatchesIterator.java | 6 +-
.../lucene/search/TestSimpleExplanations.java | 5 +-
.../lucene/search/TestSimpleSearchEquivalence.java | 2 +-
.../org/apache/lucene/search/TestSynonymQuery.java | 162 ++++++-
.../org/apache/lucene/util/TestQueryBuilder.java | 40 +-
.../writercache/LruTaxonomyWriterCache.java | 2 +-
.../taxonomy/writercache/NameIntCacheLRU.java | 4 +-
.../lucene/search/highlight/HighlighterTest.java | 5 +-
.../vectorhighlight/FastVectorHighlighterTest.java | 5 +-
.../queryparser/classic/TestQueryParser.java | 52 +-
.../src/java/org/apache/lucene/geo/Line2D.java | 20 +-
.../java/org/apache/lucene/search/BM25FQuery.java | 6 +-
.../lucene/document/BaseLatLonShapeTestCase.java | 2 +-
.../apache/lucene/document/TestLatLonShape.java | 27 +-
.../src/test/org/apache/lucene/geo/TestLine2D.java | 85 ++++
.../org/apache/lucene/search/TestBM25FQuery.java | 6 +-
.../tools/src/groovy/check-source-patterns.groovy | 9 +
solr/CHANGES.txt | 100 +++-
solr/bin/solr.cmd | 7 +-
.../apache/solr/ltr/TestLTRReRankingPipeline.java | 4 +-
.../org/apache/solr/ltr/TestLTRScoringQuery.java | 4 +-
.../solr/ltr/feature/TestOriginalScoreScorer.java | 4 +-
.../src/java/org/apache/solr/api/V2HttpCall.java | 28 +-
.../OverseerCollectionMessageHandler.java | 2 +-
.../api/collections/ReindexCollectionCmd.java | 259 ++++++++--
.../java/org/apache/solr/core/CoreContainer.java | 6 +
.../org/apache/solr/handler/StreamHandler.java | 4 +-
.../org/apache/solr/handler/admin/ColStatus.java | 197 ++++++++
.../solr/handler/admin/CollectionsHandler.java | 23 +-
.../handler/admin/SegmentsInfoRequestHandler.java | 350 +++++++++++++-
.../solr/handler/component/QueryComponent.java | 18 +-
.../solr/handler/component/ResponseBuilder.java | 2 +-
.../apache/solr/handler/export/ExportWriter.java | 10 +-
.../apache/solr/request/macro/MacroExpander.java | 13 +-
.../java/org/apache/solr/servlet/HttpSolrCall.java | 18 +-
.../org/apache/solr/update/SolrIndexWriter.java | 19 +-
.../processor/DistributedUpdateProcessor.java | 3 -
.../org/apache/solr/util/StartupLoggingUtils.java | 11 +-
.../java/org/apache/solr/util/TestInjection.java | 44 +-
.../collection1/conf/schema-sortingresponse.xml | 3 +
.../test/org/apache/solr/TestDocumentBuilder.java | 3 +-
.../test/org/apache/solr/TestTolerantSearch.java | 10 +-
...aosMonkeyNothingIsSafeWithPullReplicasTest.java | 1 -
.../ChaosMonkeySafeLeaderWithPullReplicasTest.java | 1 -
.../apache/solr/cloud/CollectionsAPISolrJTest.java | 30 ++
.../org/apache/solr/cloud/ForceLeaderTest.java | 30 +-
.../cloud/ForceLeaderWithTlogReplicasTest.java} | 11 +-
.../org/apache/solr/cloud/HttpPartitionTest.java | 13 +-
.../cloud/HttpPartitionWithTlogReplicasTest.java} | 19 +-
.../apache/solr/cloud/LeaderTragicEventTest.java | 2 +
.../solr/cloud/MoveReplicaHDFSFailoverTest.java | 15 +-
.../org/apache/solr/cloud/MoveReplicaHDFSTest.java | 4 +-
.../solr/cloud/RecoveryAfterSoftCommitTest.java | 1 -
.../apache/solr/cloud/ReindexCollectionTest.java | 150 +++++-
.../org/apache/solr/cloud/TestPullReplica.java | 1 -
.../solr/cloud/TestPullReplicaErrorHandling.java | 3 +-
.../org/apache/solr/cloud/TestTlogReplica.java | 3 +-
.../collections/TestHdfsCloudBackupRestore.java | 20 +-
.../HdfsAutoAddReplicasIntegrationTest.java | 2 +
.../cloud/autoscaling/sim/TestSimLargeCluster.java | 156 +++---
.../solr/cloud/hdfs/HDFSCollectionsAPITest.java | 14 +-
.../cloud/hdfs/HdfsBasicDistributedZkTest.java | 13 +-
.../hdfs/HdfsChaosMonkeyNothingIsSafeTest.java | 14 +-
.../cloud/hdfs/HdfsChaosMonkeySafeLeaderTest.java | 12 +-
.../apache/solr/cloud/hdfs/HdfsNNFailoverTest.java | 11 +-
.../apache/solr/cloud/hdfs/HdfsRecoveryZkTest.java | 14 +-
.../cloud/hdfs/HdfsRestartWhileUpdatingTest.java | 12 +-
.../org/apache/solr/cloud/hdfs/HdfsTestUtil.java | 3 +-
.../HdfsTlogReplayBufferedWhileIndexingTest.java | 19 +-
.../cloud/hdfs/HdfsUnloadDistributedZkTest.java | 10 +-
.../hdfs/HdfsWriteToMultipleCollectionsTest.java | 12 +-
.../org/apache/solr/cloud/hdfs/StressHdfsTest.java | 36 +-
.../org/apache/solr/core/DirectoryFactoryTest.java | 4 +-
.../apache/solr/core/HdfsDirectoryFactoryTest.java | 251 +++++-----
.../apache/solr/core/OpenCloseCoreStressTest.java | 525 ---------------------
.../apache/solr/core/RAMDirectoryFactoryTest.java | 4 +-
.../org/apache/solr/core/TestConfigOverlay.java | 4 +-
.../apache/solr/handler/BackupRestoreUtils.java | 4 +-
.../solr/handler/TestHdfsBackupRestoreCore.java | 16 +-
.../apache/solr/handler/V2ApiIntegrationTest.java | 29 ++
.../admin/SegmentsInfoRequestHandlerTest.java | 27 ++
.../solr/handler/admin/SystemInfoHandlerTest.java | 4 +-
.../component/DistributedFacetExistsSmallTest.java | 10 +-
.../component/ReplicaListTransformerTest.java | 5 +-
.../ShufflingReplicaListTransformerTest.java | 4 +-
.../solr/handler/export/TestExportWriter.java | 34 +-
.../solr/index/TestSlowCompositeReaderWrapper.java | 4 +-
.../org/apache/solr/legacy/TestLegacyField.java | 4 +-
.../apache/solr/legacy/TestLegacyNumericUtils.java | 4 +-
.../org/apache/solr/legacy/TestLegacyTerms.java | 4 +-
.../legacy/TestMultiValuedNumericRangeQuery.java | 4 +-
.../solr/legacy/TestNumericRangeQuery32.java | 4 +-
.../solr/legacy/TestNumericRangeQuery64.java | 4 +-
.../org/apache/solr/logging/TestLogWatcher.java | 56 ++-
.../solr/request/RegexBytesRefFilterTest.java | 4 +-
.../solr/request/SubstringBytesRefFilterTest.java | 4 +-
.../org/apache/solr/request/TestFacetMethods.java | 4 +-
.../solr/request/macro/TestMacroExpander.java | 30 +-
.../solr/response/TestSolrQueryResponse.java | 4 +-
.../test/org/apache/solr/search/TestDocSet.java | 4 +-
.../org/apache/solr/search/TestFastLRUCache.java | 4 +-
.../apache/solr/search/TestFilteredDocIdSet.java | 4 +-
.../test/org/apache/solr/search/TestLRUCache.java | 4 +-
.../search/TestLegacyNumericRangeQueryBuilder.java | 4 +-
.../org/apache/solr/search/TestRecoveryHdfs.java | 52 +-
.../org/apache/solr/search/TestSolrCoreParser.java | 4 +-
.../apache/solr/search/TestStandardQParsers.java | 4 +-
.../org/apache/solr/search/TestXmlQParser.java | 13 +
.../apache/solr/search/function/TestOrdValues.java | 4 +-
.../spelling/ConjunctionSolrSpellCheckerTest.java | 4 +-
.../solr/spelling/SpellingQueryConverterTest.java | 4 +-
.../solr/store/blockcache/BlockCacheTest.java | 3 +-
.../solr/store/blockcache/BufferStoreTest.java | 4 +-
.../apache/solr/uninverting/TestDocTermOrds.java | 4 +-
.../uninverting/TestDocTermOrdsUninvertLimit.java | 4 +-
.../apache/solr/uninverting/TestFieldCache.java | 4 +-
.../solr/uninverting/TestFieldCacheReopen.java | 4 +-
.../solr/uninverting/TestFieldCacheSort.java | 4 +-
.../solr/uninverting/TestFieldCacheSortRandom.java | 4 +-
.../uninverting/TestFieldCacheVsDocValues.java | 4 +-
.../uninverting/TestFieldCacheWithThreads.java | 4 +-
.../solr/uninverting/TestLegacyFieldCache.java | 4 +-
.../solr/uninverting/TestNumericTerms32.java | 4 +-
.../solr/uninverting/TestNumericTerms64.java | 4 +-
.../solr/uninverting/TestUninvertingReader.java | 4 +-
.../org/apache/solr/update/TestHdfsUpdateLog.java | 31 +-
.../org/apache/solr/update/TransactionLogTest.java | 4 +-
.../IgnoreLargeDocumentProcessorFactoryTest.java | 4 +-
.../solr/update/processor/ScriptEngineTest.java | 4 +-
.../org/apache/solr/util/CircularListTest.java | 4 +-
.../org/apache/solr/util/DistanceUnitsTest.java | 4 +-
.../test/org/apache/solr/util/FileUtilsTest.java | 4 +-
.../org/apache/solr/util/OrderedExecutorTest.java | 4 +-
.../test/org/apache/solr/util/PrimUtilsTest.java | 4 +-
.../org/apache/solr/util/TestFastOutputStream.java | 4 +-
.../test/org/apache/solr/util/TestRTimerTree.java | 4 +-
.../org/apache/solr/util/TestSafeXMLParsing.java | 4 +-
.../org/apache/solr/util/TestTestInjection.java | 4 +-
.../org/apache/solr/util/TimeZoneUtilsTest.java | 4 +-
.../BigEndianAscendingWordDeserializerTest.java | 5 +-
.../hll/BigEndianAscendingWordSerializerTest.java | 4 +-
.../org/apache/solr/util/hll/BitVectorTest.java | 4 +-
.../org/apache/solr/util/hll/ExplicitHLLTest.java | 4 +-
.../test/org/apache/solr/util/hll/FullHLLTest.java | 4 +-
.../apache/solr/util/hll/HLLSerializationTest.java | 4 +-
.../test/org/apache/solr/util/hll/HLLUtilTest.java | 4 +-
.../org/apache/solr/util/hll/SparseHLLTest.java | 4 +-
solr/solr-ref-guide/src/collections-api.adoc | 186 ++++++++
solr/solr-ref-guide/src/response-writers.adoc | 6 +-
solr/solr-ref-guide/src/streaming-expressions.adoc | 12 +-
.../working-with-external-files-and-processes.adoc | 4 +-
.../java/org/apache/solr/client/solrj/io/Lang.java | 2 +-
.../{CalculatorStream.java => NoOpStream.java} | 19 +-
.../solr/client/solrj/io/stream/ZplotStream.java | 4 -
...meter.java => InjectedExpressionException.java} | 10 +-
.../solrj/io/stream/expr/InjectionDefense.java | 199 ++++++++
.../io/stream/expr/StreamExpressionParser.java | 116 +++--
.../solrj/request/CollectionAdminRequest.java | 80 +++-
.../solr/common/cloud/ConnectionManager.java | 2 +-
.../org/apache/solr/common/luke/FieldFlag.java | 4 +
.../solr/common/params/CollectionParams.java | 4 +-
.../CollectionAdminRequestRequiredParamsTest.java | 4 +-
.../solr/client/solrj/SolrExceptionTest.java | 4 +-
.../apache/solr/client/solrj/SolrQueryTest.java | 4 +-
.../solrj/beans/TestDocumentObjectBinder.java | 5 +-
.../impl/CloudHttp2SolrClientBuilderTest.java | 4 +-
.../CloudHttp2SolrClientMultiConstructorTest.java | 4 +-
.../solrj/impl/CloudSolrClientBuilderTest.java | 4 +-
.../impl/CloudSolrClientMultiConstructorTest.java | 4 +-
.../ConcurrentUpdateSolrClientBuilderTest.java | 4 +-
.../solr/client/solrj/impl/HttpClientUtilTest.java | 8 +-
.../solrj/impl/HttpSolrClientBuilderTest.java | 4 +-
.../solrj/impl/LBHttpSolrClientBuilderTest.java | 4 +-
.../org/apache/solr/client/solrj/io/TestLang.java | 5 +-
.../solrj/io/stream/StreamExpressionTest.java | 109 +++--
.../io/stream/StreamExpressionToExpessionTest.java | 4 +-
.../stream/StreamExpressionToExplanationTest.java | 4 +-
.../io/stream/eval/AbsoluteValueEvaluatorTest.java | 4 +-
.../solrj/io/stream/eval/AddEvaluatorTest.java | 4 +-
.../solrj/io/stream/eval/AndEvaluatorTest.java | 4 +-
.../solrj/io/stream/eval/AppendEvaluatorTest.java | 4 +-
.../io/stream/eval/ArcCosineEvaluatorTest.java | 4 +-
.../solrj/io/stream/eval/ArcSineEvaluatorTest.java | 4 +-
.../io/stream/eval/ArcTangentEvaluatorTest.java | 4 +-
.../solrj/io/stream/eval/ArrayEvaluatorTest.java | 4 +-
.../solrj/io/stream/eval/AscEvaluatorTest.java | 4 +-
.../solrj/io/stream/eval/CeilingEvaluatorTest.java | 4 +-
.../io/stream/eval/CoalesceEvaluatorTest.java | 4 +-
.../io/stream/eval/CorrelationEvaluatorTest.java | 4 +-
.../solrj/io/stream/eval/CosineEvaluatorTest.java | 4 +-
.../io/stream/eval/CubedRootEvaluatorTest.java | 4 +-
.../eval/CumulativeProbabilityEvaluatorTest.java | 4 +-
.../solrj/io/stream/eval/DivideEvaluatorTest.java | 4 +-
.../eval/EmpiricalDistributionEvaluatorTest.java | 4 +-
.../solrj/io/stream/eval/EqualToEvaluatorTest.java | 4 +-
.../io/stream/eval/ExclusiveOrEvaluatorTest.java | 4 +-
.../io/stream/eval/FieldValueEvaluatorTest.java | 4 +-
.../solrj/io/stream/eval/FloorEvaluatorTest.java | 4 +-
.../eval/GreaterThanEqualToEvaluatorTest.java | 4 +-
.../io/stream/eval/GreaterThanEvaluatorTest.java | 4 +-
.../stream/eval/HyperbolicCosineEvaluatorTest.java | 4 +-
.../stream/eval/HyperbolicSineEvaluatorTest.java | 4 +-
.../eval/HyperbolicTangentEvaluatorTest.java | 4 +-
.../solrj/io/stream/eval/LengthEvaluatorTest.java | 4 +-
.../stream/eval/LessThanEqualToEvaluatorTest.java | 4 +-
.../io/stream/eval/LessThanEvaluatorTest.java | 4 +-
.../solrj/io/stream/eval/ModuloEvaluatorTest.java | 4 +-
.../io/stream/eval/MultiplyEvaluatorTest.java | 4 +-
.../io/stream/eval/NaturalLogEvaluatorTest.java | 4 +-
.../eval/NormalDistributionEvaluatorTest.java | 4 +-
.../io/stream/eval/NormalizeEvaluatorTest.java | 4 +-
.../solrj/io/stream/eval/NotEvaluatorTest.java | 4 +-
.../solrj/io/stream/eval/OrEvaluatorTest.java | 4 +-
.../solrj/io/stream/eval/PowerEvaluatorTest.java | 4 +-
.../io/stream/eval/RawValueEvaluatorTest.java | 4 +-
.../io/stream/eval/RecursiveEvaluatorTest.java | 4 +-
.../io/stream/eval/RegressionEvaluatorTest.java | 4 +-
.../solrj/io/stream/eval/ReverseEvaluatorTest.java | 4 +-
.../solrj/io/stream/eval/RoundEvaluatorTest.java | 4 +-
.../solrj/io/stream/eval/SineEvaluatorTest.java | 4 +-
.../io/stream/eval/SquareRootEvaluatorTest.java | 4 +-
.../io/stream/eval/SubtractEvaluatorTest.java | 4 +-
.../solrj/io/stream/eval/TangentEvaluatorTest.java | 4 +-
.../eval/UniformDistributionEvaluatorTest.java | 4 +-
.../solrj/io/stream/eval/UuidEvaluatorTest.java | 4 +-
.../solrj/io/stream/expr/InjectionDefenseTest.java | 115 +++++
.../io/stream/expr/StreamExpressionParserTest.java | 5 +-
.../solrj/io/stream/ops/ConcatOperationTest.java | 4 +-
.../client/solrj/io/stream/ops/OperationsTest.java | 4 +-
.../solrj/request/TestCollectionAdminRequest.java | 4 +-
.../solrj/request/TestDelegationTokenRequest.java | 5 +-
.../solrj/request/TestUpdateRequestCodec.java | 4 +-
.../client/solrj/request/TestV1toV2ApiMapper.java | 4 +-
.../request/json/JsonQueryRequestUnitTest.java | 4 +-
.../solrj/response/AnlysisResponseBaseTest.java | 4 +-
.../response/DocumentAnalysisResponseTest.java | 4 +-
.../solr/client/solrj/response/FacetFieldTest.java | 4 +-
.../solrj/response/FieldAnalysisResponseTest.java | 4 +-
.../client/solrj/response/QueryResponseTest.java | 4 +-
.../response/TestDelegationTokenResponse.java | 4 +-
.../solr/client/solrj/util/ClientUtilsTest.java | 4 +-
.../org/apache/solr/common/SolrDocumentTest.java | 4 +-
.../solr/common/TestToleratedUpdateError.java | 4 +-
.../solr/common/params/CommonAdminParamsTest.java | 4 +-
.../solr/common/params/CommonParamsTest.java | 4 +-
.../common/params/ModifiableSolrParamsTest.java | 4 +-
.../apache/solr/common/params/ShardParamsTest.java | 4 +-
.../apache/solr/common/params/SolrParamTest.java | 4 +-
.../org/apache/solr/common/util/NamedListTest.java | 4 +-
.../solr/common/util/TestFastInputStream.java | 4 +-
.../apache/solr/common/util/TestFastWriter.java | 4 +-
.../test/org/apache/solr/common/util/TestHash.java | 4 +-
.../solr/common/util/TestNamedListCodec.java | 4 +-
.../apache/solr/common/util/TestXMLEscaping.java | 4 +-
.../RandomMergePolicy.java => SolrTestCase.java} | 35 +-
.../src/java/org/apache/solr/SolrTestCaseJ4.java | 11 +-
.../java/org/apache/solr/util/DOMUtilTestBase.java | 4 +-
.../solr/cloud/MiniSolrCloudClusterTest.java | 3 +-
268 files changed, 3492 insertions(+), 1780 deletions(-)
create mode 100644 lucene/sandbox/src/test/org/apache/lucene/geo/TestLine2D.java
create mode 100644 solr/core/src/java/org/apache/solr/handler/admin/ColStatus.java
copy solr/core/src/{java/org/apache/solr/cloud/package-info.java => test/org/apache/solr/cloud/ForceLeaderWithTlogReplicasTest.java} (85%)
copy solr/core/src/{java/org/apache/solr/search/grouping/endresulttransformer/package-info.java => test/org/apache/solr/cloud/HttpPartitionWithTlogReplicasTest.java} (73%)
delete mode 100644 solr/core/src/test/org/apache/solr/core/OpenCloseCoreStressTest.java
mode change 100755 => 100644 solr/core/src/test/org/apache/solr/search/TestXmlQParser.java
copy solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/{CalculatorStream.java => NoOpStream.java} (88%)
copy solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/expr/{StreamExpressionParameter.java => InjectedExpressionException.java} (87%)
create mode 100644 solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/expr/InjectionDefense.java
create mode 100644 solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/expr/InjectionDefenseTest.java
copy solr/test-framework/src/java/org/apache/solr/{util/RandomMergePolicy.java => SolrTestCase.java} (56%)
[lucene-solr] 05/05: SOLR-11127: Add proper shutdown of daemon
streams.
Posted by ab...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ab pushed a commit to branch jira/solr-11127-2
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 116c6d7bebb7fad869c41a471b6219571c0c0b64
Author: Andrzej Bialecki <ab...@apache.org>
AuthorDate: Tue Mar 12 19:34:54 2019 +0100
SOLR-11127: Add proper shutdown of daemon streams.
---
.../api/collections/ReindexCollectionCmd.java | 85 ++++++++++++++++++++--
.../org/apache/solr/handler/StreamHandler.java | 4 +-
2 files changed, 81 insertions(+), 8 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/ReindexCollectionCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/ReindexCollectionCmd.java
index 452e394..f1b31d7 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/ReindexCollectionCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/ReindexCollectionCmd.java
@@ -378,8 +378,9 @@ public class ReindexCollectionCmd implements OverseerCollectionMessageHandler.Cm
aborted = true;
return;
}
- TestInjection.injectReindexFailure();
log.debug("- finished copying from " + collection + " to " + targetCollection);
+ // fail here or earlier during daemon run
+ TestInjection.injectReindexFailure();
// 5. if (sameTarget) set up an alias to use targetCollection as the source name
if (sameTarget) {
@@ -544,11 +545,11 @@ public class ReindexCollectionCmd implements OverseerCollectionMessageHandler.Cm
NamedList<Object> rsp = solrClient.request(req);
Map<String, Object> rs = (Map<String, Object>)rsp.get("result-set");
if (rs == null || rs.isEmpty()) {
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Missing result-set: " + Utils.toJSONString(rsp));
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Can't find daemon list: missing result-set: " + Utils.toJSONString(rsp));
}
List<Object> list = (List<Object>)rs.get("docs");
if (list == null) {
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Missing result-set: " + Utils.toJSONString(rsp));
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Can't find daemon list: missing result-set: " + Utils.toJSONString(rsp));
}
if (list.isEmpty()) { // finished?
break;
@@ -558,11 +559,14 @@ public class ReindexCollectionCmd implements OverseerCollectionMessageHandler.Cm
String id = (String)map.get("id");
if (daemonName.equals(id)) {
isRunning = true;
+ // fail here
+ TestInjection.injectReindexFailure();
break;
}
}
} catch (Exception e) {
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Exception copying the documents", e);
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Exception waiting for daemon " +
+ daemonName + " at " + daemonUrl, e);
}
ocmh.cloudManager.getTimeSource().sleep(2000);
} while (isRunning && !maybeAbort(collection));
@@ -577,14 +581,81 @@ public class ReindexCollectionCmd implements OverseerCollectionMessageHandler.Cm
.withBaseSolrUrl(daemonUrl).build()) {
ModifiableSolrParams q = new ModifiableSolrParams();
q.set(CommonParams.QT, "/stream");
- q.set("action", "kill");
+ // we should really use 'kill' here, but then we will never
+ // know when the daemon actually finishes running - 'kill' only
+ // sets a flag that may be noticed much later
+ q.set("action", "stop");
q.set(CommonParams.ID, daemonName);
q.set(CommonParams.DISTRIB, false);
QueryRequest req = new QueryRequest(q);
NamedList<Object> rsp = solrClient.request(req);
// /result-set/docs/[0]/DaemonOp : Deamon:id killed on coreName
-
- // nocommit error checking
+ log.debug(" -- stop daemon response: " + Utils.toJSONString(rsp));
+ Map<String, Object> rs = (Map<String, Object>) rsp.get("result-set");
+ if (rs == null || rs.isEmpty()) {
+ log.warn("Problem killing daemon " + daemonName + ": missing result-set: " + Utils.toJSONString(rsp));
+ return;
+ }
+ List<Object> list = (List<Object>) rs.get("docs");
+ if (list == null) {
+ log.warn("Problem killing daemon " + daemonName + ": missing result-set: " + Utils.toJSONString(rsp));
+ return;
+ }
+ if (list.isEmpty()) { // already finished?
+ return;
+ }
+ for (Object o : list) {
+ Map<String, Object> map = (Map<String, Object>) o;
+ String op = (String) map.get("DaemonOp");
+ if (op == null) {
+ continue;
+ }
+ if (op.contains(daemonName) && op.contains("stopped")) {
+ // now wait for the daemon to really stop
+ q.set("action", "list");
+ req = new QueryRequest(q);
+ TimeOut timeOut = new TimeOut(60, TimeUnit.SECONDS, ocmh.timeSource);
+ while (!timeOut.hasTimedOut()) {
+ rsp = solrClient.request(req);
+ rs = (Map<String, Object>) rsp.get("result-set");
+ if (rs == null || rs.isEmpty()) {
+ log.warn("Problem killing daemon " + daemonName + ": missing result-set: " + Utils.toJSONString(rsp));
+ break;
+ }
+ List<Object> list2 = (List<Object>) rs.get("docs");
+ if (list2 == null) {
+ log.warn("Problem killing daemon " + daemonName + ": missing result-set: " + Utils.toJSONString(rsp));
+ break;
+ }
+ if (list2.isEmpty()) { // already finished?
+ break;
+ }
+ Map<String, Object> status2 = null;
+ for (Object o2 : list2) {
+ Map<String, Object> map2 = (Map<String, Object>)o2;
+ if (daemonName.equals(map2.get("id"))) {
+ status2 = map2;
+ break;
+ }
+ }
+ if (status2 == null) { // finished?
+ break;
+ }
+ Number stopTime = (Number)status2.get("stopTime");
+ if (stopTime.longValue() > 0) {
+ break;
+ }
+ }
+ if (timeOut.hasTimedOut()) {
+ log.warn("Problem killing daemon " + daemonName + ": timed out waiting for daemon to stop.");
+ // proceed anyway
+ }
+ }
+ }
+ // now kill it - it's already stopped, this simply removes its status
+ q.set("action", "kill");
+ req = new QueryRequest(q);
+ solrClient.request(req);
}
}
diff --git a/solr/core/src/java/org/apache/solr/handler/StreamHandler.java b/solr/core/src/java/org/apache/solr/handler/StreamHandler.java
index a447093..545e2b3 100644
--- a/solr/core/src/java/org/apache/solr/handler/StreamHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/StreamHandler.java
@@ -216,8 +216,10 @@ public class StreamHandler extends RequestHandlerBase implements SolrCoreAware,
DaemonStream d = daemons.remove(id);
if (d != null) {
d.close();
+ rsp.add("result-set", new DaemonResponseStream("Deamon:" + id + " killed on " + coreName));
+ } else {
+ rsp.add("result-set", new DaemonResponseStream("Deamon:" + id + " not found on " + coreName));
}
- rsp.add("result-set", new DaemonResponseStream("Deamon:" + id + " killed on " + coreName));
}
}
}
[lucene-solr] 02/05: Merge branch 'master' into jira/solr-11127-2
Posted by ab...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ab pushed a commit to branch jira/solr-11127-2
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 3f0b1c2681cd8d9c4860795150250fd7da2dc76d
Merge: 675dd0b d8cfeba
Author: Andrzej Bialecki <ab...@apache.org>
AuthorDate: Mon Mar 11 11:07:36 2019 +0100
Merge branch 'master' into jira/solr-11127-2
lucene/CHANGES.txt | 3 +
.../org/apache/lucene/search/SynonymQuery.java | 261 ++++++++--
.../java/org/apache/lucene/util/QueryBuilder.java | 8 +-
.../apache/lucene/search/TestMatchesIterator.java | 6 +-
.../lucene/search/TestSimpleExplanations.java | 5 +-
.../lucene/search/TestSimpleSearchEquivalence.java | 2 +-
.../org/apache/lucene/search/TestSynonymQuery.java | 161 ++++++-
.../org/apache/lucene/util/TestQueryBuilder.java | 40 +-
.../lucene/search/highlight/HighlighterTest.java | 5 +-
.../vectorhighlight/FastVectorHighlighterTest.java | 5 +-
.../queryparser/classic/TestQueryParser.java | 52 +-
.../java/org/apache/lucene/search/BM25FQuery.java | 6 +-
.../org/apache/lucene/search/TestBM25FQuery.java | 6 +-
.../tools/src/groovy/check-source-patterns.groovy | 6 +
solr/CHANGES.txt | 19 +-
solr/bin/solr.cmd | 7 +-
.../apache/solr/ltr/TestLTRReRankingPipeline.java | 4 +-
.../org/apache/solr/ltr/TestLTRScoringQuery.java | 4 +-
.../solr/ltr/feature/TestOriginalScoreScorer.java | 4 +-
.../src/java/org/apache/solr/api/V2HttpCall.java | 28 +-
.../org/apache/solr/handler/admin/ColStatus.java | 197 ++++++++
.../solr/handler/admin/CollectionsHandler.java | 16 +
.../handler/admin/SegmentsInfoRequestHandler.java | 350 +++++++++++++-
.../apache/solr/handler/export/ExportWriter.java | 10 +-
.../java/org/apache/solr/servlet/HttpSolrCall.java | 18 +-
.../org/apache/solr/update/SolrIndexWriter.java | 19 +-
.../processor/DistributedUpdateProcessor.java | 3 -
.../java/org/apache/solr/util/TestInjection.java | 13 -
.../collection1/conf/schema-sortingresponse.xml | 3 +
.../test/org/apache/solr/TestDocumentBuilder.java | 3 +-
...aosMonkeyNothingIsSafeWithPullReplicasTest.java | 1 -
.../ChaosMonkeySafeLeaderWithPullReplicasTest.java | 1 -
.../apache/solr/cloud/CollectionsAPISolrJTest.java | 30 ++
.../org/apache/solr/cloud/ForceLeaderTest.java | 30 +-
.../cloud/ForceLeaderWithTlogReplicasTest.java} | 16 +-
.../org/apache/solr/cloud/HttpPartitionTest.java | 13 +-
.../cloud/HttpPartitionWithTlogReplicasTest.java} | 22 +-
.../solr/cloud/RecoveryAfterSoftCommitTest.java | 1 -
.../org/apache/solr/cloud/TestPullReplica.java | 1 -
.../solr/cloud/TestPullReplicaErrorHandling.java | 3 +-
.../org/apache/solr/cloud/TestTlogReplica.java | 3 +-
.../cloud/autoscaling/sim/TestSimLargeCluster.java | 156 +++---
.../org/apache/solr/core/DirectoryFactoryTest.java | 4 +-
.../apache/solr/core/OpenCloseCoreStressTest.java | 525 ---------------------
.../apache/solr/core/RAMDirectoryFactoryTest.java | 4 +-
.../org/apache/solr/core/TestConfigOverlay.java | 4 +-
.../apache/solr/handler/BackupRestoreUtils.java | 4 +-
.../apache/solr/handler/V2ApiIntegrationTest.java | 29 ++
.../admin/SegmentsInfoRequestHandlerTest.java | 27 ++
.../solr/handler/admin/SystemInfoHandlerTest.java | 4 +-
.../component/ReplicaListTransformerTest.java | 5 +-
.../ShufflingReplicaListTransformerTest.java | 4 +-
.../solr/handler/export/TestExportWriter.java | 34 +-
.../solr/index/TestSlowCompositeReaderWrapper.java | 4 +-
.../org/apache/solr/legacy/TestLegacyField.java | 4 +-
.../apache/solr/legacy/TestLegacyNumericUtils.java | 4 +-
.../org/apache/solr/legacy/TestLegacyTerms.java | 4 +-
.../legacy/TestMultiValuedNumericRangeQuery.java | 4 +-
.../solr/legacy/TestNumericRangeQuery32.java | 4 +-
.../solr/legacy/TestNumericRangeQuery64.java | 4 +-
.../org/apache/solr/logging/TestLogWatcher.java | 21 +-
.../solr/request/RegexBytesRefFilterTest.java | 4 +-
.../solr/request/SubstringBytesRefFilterTest.java | 4 +-
.../org/apache/solr/request/TestFacetMethods.java | 4 +-
.../solr/request/macro/TestMacroExpander.java | 5 +-
.../solr/response/TestSolrQueryResponse.java | 4 +-
.../test/org/apache/solr/search/TestDocSet.java | 4 +-
.../org/apache/solr/search/TestFastLRUCache.java | 4 +-
.../apache/solr/search/TestFilteredDocIdSet.java | 4 +-
.../test/org/apache/solr/search/TestLRUCache.java | 4 +-
.../search/TestLegacyNumericRangeQueryBuilder.java | 4 +-
.../org/apache/solr/search/TestSolrCoreParser.java | 4 +-
.../apache/solr/search/TestStandardQParsers.java | 4 +-
.../org/apache/solr/search/TestXmlQParser.java | 42 --
.../apache/solr/search/function/TestOrdValues.java | 4 +-
.../spelling/ConjunctionSolrSpellCheckerTest.java | 4 +-
.../solr/spelling/SpellingQueryConverterTest.java | 4 +-
.../solr/store/blockcache/BlockCacheTest.java | 3 +-
.../solr/store/blockcache/BufferStoreTest.java | 4 +-
.../apache/solr/uninverting/TestDocTermOrds.java | 4 +-
.../uninverting/TestDocTermOrdsUninvertLimit.java | 4 +-
.../apache/solr/uninverting/TestFieldCache.java | 4 +-
.../solr/uninverting/TestFieldCacheReopen.java | 4 +-
.../solr/uninverting/TestFieldCacheSort.java | 4 +-
.../solr/uninverting/TestFieldCacheSortRandom.java | 4 +-
.../uninverting/TestFieldCacheVsDocValues.java | 4 +-
.../uninverting/TestFieldCacheWithThreads.java | 4 +-
.../solr/uninverting/TestLegacyFieldCache.java | 4 +-
.../solr/uninverting/TestNumericTerms32.java | 4 +-
.../solr/uninverting/TestNumericTerms64.java | 4 +-
.../solr/uninverting/TestUninvertingReader.java | 4 +-
.../org/apache/solr/update/TransactionLogTest.java | 4 +-
.../IgnoreLargeDocumentProcessorFactoryTest.java | 4 +-
.../solr/update/processor/ScriptEngineTest.java | 4 +-
.../org/apache/solr/util/CircularListTest.java | 4 +-
.../org/apache/solr/util/DistanceUnitsTest.java | 4 +-
.../test/org/apache/solr/util/FileUtilsTest.java | 4 +-
.../org/apache/solr/util/OrderedExecutorTest.java | 4 +-
.../test/org/apache/solr/util/PrimUtilsTest.java | 4 +-
.../org/apache/solr/util/TestFastOutputStream.java | 4 +-
.../test/org/apache/solr/util/TestRTimerTree.java | 4 +-
.../org/apache/solr/util/TestSafeXMLParsing.java | 4 +-
.../org/apache/solr/util/TestTestInjection.java | 4 +-
.../org/apache/solr/util/TimeZoneUtilsTest.java | 4 +-
.../BigEndianAscendingWordDeserializerTest.java | 5 +-
.../hll/BigEndianAscendingWordSerializerTest.java | 4 +-
.../org/apache/solr/util/hll/BitVectorTest.java | 4 +-
.../org/apache/solr/util/hll/ExplicitHLLTest.java | 4 +-
.../test/org/apache/solr/util/hll/FullHLLTest.java | 4 +-
.../apache/solr/util/hll/HLLSerializationTest.java | 4 +-
.../test/org/apache/solr/util/hll/HLLUtilTest.java | 4 +-
.../org/apache/solr/util/hll/SparseHLLTest.java | 4 +-
solr/solr-ref-guide/src/collections-api.adoc | 186 ++++++++
solr/solr-ref-guide/src/response-writers.adoc | 6 +-
.../working-with-external-files-and-processes.adoc | 4 +-
.../solrj/request/CollectionAdminRequest.java | 48 ++
.../solr/common/cloud/ConnectionManager.java | 2 +-
.../org/apache/solr/common/luke/FieldFlag.java | 4 +
.../solr/common/params/CollectionParams.java | 1 +
.../CollectionAdminRequestRequiredParamsTest.java | 4 +-
.../solr/client/solrj/SolrExceptionTest.java | 4 +-
.../apache/solr/client/solrj/SolrQueryTest.java | 4 +-
.../solrj/beans/TestDocumentObjectBinder.java | 5 +-
.../impl/CloudHttp2SolrClientBuilderTest.java | 4 +-
.../CloudHttp2SolrClientMultiConstructorTest.java | 4 +-
.../solrj/impl/CloudSolrClientBuilderTest.java | 4 +-
.../impl/CloudSolrClientMultiConstructorTest.java | 4 +-
.../ConcurrentUpdateSolrClientBuilderTest.java | 4 +-
.../solr/client/solrj/impl/HttpClientUtilTest.java | 8 +-
.../solrj/impl/HttpSolrClientBuilderTest.java | 4 +-
.../solrj/impl/LBHttpSolrClientBuilderTest.java | 4 +-
.../org/apache/solr/client/solrj/io/TestLang.java | 3 +-
.../io/stream/StreamExpressionToExpessionTest.java | 4 +-
.../stream/StreamExpressionToExplanationTest.java | 4 +-
.../io/stream/eval/AbsoluteValueEvaluatorTest.java | 4 +-
.../solrj/io/stream/eval/AddEvaluatorTest.java | 4 +-
.../solrj/io/stream/eval/AndEvaluatorTest.java | 4 +-
.../solrj/io/stream/eval/AppendEvaluatorTest.java | 4 +-
.../io/stream/eval/ArcCosineEvaluatorTest.java | 4 +-
.../solrj/io/stream/eval/ArcSineEvaluatorTest.java | 4 +-
.../io/stream/eval/ArcTangentEvaluatorTest.java | 4 +-
.../solrj/io/stream/eval/ArrayEvaluatorTest.java | 4 +-
.../solrj/io/stream/eval/AscEvaluatorTest.java | 4 +-
.../solrj/io/stream/eval/CeilingEvaluatorTest.java | 4 +-
.../io/stream/eval/CoalesceEvaluatorTest.java | 4 +-
.../io/stream/eval/CorrelationEvaluatorTest.java | 4 +-
.../solrj/io/stream/eval/CosineEvaluatorTest.java | 4 +-
.../io/stream/eval/CubedRootEvaluatorTest.java | 4 +-
.../eval/CumulativeProbabilityEvaluatorTest.java | 4 +-
.../solrj/io/stream/eval/DivideEvaluatorTest.java | 4 +-
.../eval/EmpiricalDistributionEvaluatorTest.java | 4 +-
.../solrj/io/stream/eval/EqualToEvaluatorTest.java | 4 +-
.../io/stream/eval/ExclusiveOrEvaluatorTest.java | 4 +-
.../io/stream/eval/FieldValueEvaluatorTest.java | 4 +-
.../solrj/io/stream/eval/FloorEvaluatorTest.java | 4 +-
.../eval/GreaterThanEqualToEvaluatorTest.java | 4 +-
.../io/stream/eval/GreaterThanEvaluatorTest.java | 4 +-
.../stream/eval/HyperbolicCosineEvaluatorTest.java | 4 +-
.../stream/eval/HyperbolicSineEvaluatorTest.java | 4 +-
.../eval/HyperbolicTangentEvaluatorTest.java | 4 +-
.../solrj/io/stream/eval/LengthEvaluatorTest.java | 4 +-
.../stream/eval/LessThanEqualToEvaluatorTest.java | 4 +-
.../io/stream/eval/LessThanEvaluatorTest.java | 4 +-
.../solrj/io/stream/eval/ModuloEvaluatorTest.java | 4 +-
.../io/stream/eval/MultiplyEvaluatorTest.java | 4 +-
.../io/stream/eval/NaturalLogEvaluatorTest.java | 4 +-
.../eval/NormalDistributionEvaluatorTest.java | 4 +-
.../io/stream/eval/NormalizeEvaluatorTest.java | 4 +-
.../solrj/io/stream/eval/NotEvaluatorTest.java | 4 +-
.../solrj/io/stream/eval/OrEvaluatorTest.java | 4 +-
.../solrj/io/stream/eval/PowerEvaluatorTest.java | 4 +-
.../io/stream/eval/RawValueEvaluatorTest.java | 4 +-
.../io/stream/eval/RecursiveEvaluatorTest.java | 4 +-
.../io/stream/eval/RegressionEvaluatorTest.java | 4 +-
.../solrj/io/stream/eval/ReverseEvaluatorTest.java | 4 +-
.../solrj/io/stream/eval/RoundEvaluatorTest.java | 4 +-
.../solrj/io/stream/eval/SineEvaluatorTest.java | 4 +-
.../io/stream/eval/SquareRootEvaluatorTest.java | 4 +-
.../io/stream/eval/SubtractEvaluatorTest.java | 4 +-
.../solrj/io/stream/eval/TangentEvaluatorTest.java | 4 +-
.../eval/UniformDistributionEvaluatorTest.java | 4 +-
.../solrj/io/stream/eval/UuidEvaluatorTest.java | 4 +-
.../io/stream/expr/StreamExpressionParserTest.java | 5 +-
.../solrj/io/stream/ops/ConcatOperationTest.java | 4 +-
.../client/solrj/io/stream/ops/OperationsTest.java | 4 +-
.../solrj/request/TestCollectionAdminRequest.java | 4 +-
.../solrj/request/TestDelegationTokenRequest.java | 5 +-
.../solrj/request/TestUpdateRequestCodec.java | 4 +-
.../client/solrj/request/TestV1toV2ApiMapper.java | 4 +-
.../request/json/JsonQueryRequestUnitTest.java | 4 +-
.../solrj/response/AnlysisResponseBaseTest.java | 4 +-
.../response/DocumentAnalysisResponseTest.java | 4 +-
.../solr/client/solrj/response/FacetFieldTest.java | 4 +-
.../solrj/response/FieldAnalysisResponseTest.java | 4 +-
.../client/solrj/response/QueryResponseTest.java | 4 +-
.../response/TestDelegationTokenResponse.java | 4 +-
.../solr/client/solrj/util/ClientUtilsTest.java | 4 +-
.../org/apache/solr/common/SolrDocumentTest.java | 4 +-
.../solr/common/TestToleratedUpdateError.java | 4 +-
.../solr/common/params/CommonAdminParamsTest.java | 4 +-
.../solr/common/params/CommonParamsTest.java | 4 +-
.../common/params/ModifiableSolrParamsTest.java | 4 +-
.../apache/solr/common/params/ShardParamsTest.java | 4 +-
.../apache/solr/common/params/SolrParamTest.java | 4 +-
.../org/apache/solr/common/util/NamedListTest.java | 4 +-
.../solr/common/util/TestFastInputStream.java | 4 +-
.../apache/solr/common/util/TestFastWriter.java | 4 +-
.../test/org/apache/solr/common/util/TestHash.java | 4 +-
.../solr/common/util/TestNamedListCodec.java | 4 +-
.../apache/solr/common/util/TestXMLEscaping.java | 4 +-
.../src/java/org/apache/solr/SolrTestCase.java | 48 ++
.../src/java/org/apache/solr/SolrTestCaseJ4.java | 11 +-
.../java/org/apache/solr/util/DOMUtilTestBase.java | 4 +-
.../solr/cloud/MiniSolrCloudClusterTest.java | 3 +-
214 files changed, 1964 insertions(+), 1200 deletions(-)
diff --cc solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
index dee2d1d,cbff8d4..ddfe90e
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
@@@ -784,68 -784,54 +784,116 @@@ public abstract class CollectionAdminRe
}
/**
+ * Returns a SolrRequest to reindex a collection
+ */
+ public static ReindexCollection reindexCollection(String collection) {
+ return new ReindexCollection(collection);
+ }
+
+ public static class ReindexCollection extends AsyncCollectionSpecificAdminRequest {
+ String target;
+ String query;
+ String configName;
+ Boolean keepSource;
+ Integer batchSize;
+ Map<String, Object> collectionParams = new HashMap<>();
+
+ private ReindexCollection(String collection) {
+ super(CollectionAction.REINDEX_COLLECTION, collection);
+ }
+
+ public ReindexCollection setTarget(String target) {
+ this.target = target;
+ return this;
+ }
+
+ public ReindexCollection setQuery(String query) {
+ this.query = query;
+ return this;
+ }
+
+ public ReindexCollection setKeepSource(boolean keepSource) {
+ this.keepSource = keepSource;
+ return this;
+ }
+
+ public ReindexCollection setBatchSize(int batchSize) {
+ this.batchSize = batchSize;
+ return this;
+ }
+
+ public ReindexCollection setConfigName(String configName) {
+ this.configName = configName;
+ return this;
+ }
+
+ public ReindexCollection setCollectionParam(String key, Object value) {
+ this.collectionParams.put(key, value);
+ return this;
+ }
+
+ @Override
+ public SolrParams getParams() {
+ ModifiableSolrParams params = (ModifiableSolrParams) super.getParams();
+ params.setNonNull("target", target);
+ params.setNonNull(ZkStateReader.CONFIGNAME_PROP, configName);
+ params.setNonNull(CommonParams.Q, query);
+ params.setNonNull("keepSource", keepSource);
+ params.setNonNull(CommonParams.ROWS, batchSize);
+ collectionParams.forEach((k, v) -> params.setNonNull(k, v));
+ return params;
+ }
+ }
+
+ /**
+ * Return a SolrRequest for low-level detailed status of the collection.
+ */
+ public static ColStatus collectionStatus(String collection) {
+ return new ColStatus(collection);
+ }
+
+ public static class ColStatus extends AsyncCollectionSpecificAdminRequest {
+ protected Boolean withSegments = null;
+ protected Boolean withFieldInfo = null;
+ protected Boolean withCoreInfo = null;
+ protected Boolean withSizeInfo = null;
+
+ private ColStatus(String collection) {
+ super(CollectionAction.COLSTATUS, collection);
+ }
+
+ public ColStatus setWithSegments(boolean withSegments) {
+ this.withSegments = withSegments;
+ return this;
+ }
+
+ public ColStatus setWithFieldInfo(boolean withFieldInfo) {
+ this.withFieldInfo = withFieldInfo;
+ return this;
+ }
+
+ public ColStatus setWithCoreInfo(boolean withCoreInfo) {
+ this.withCoreInfo = withCoreInfo;
+ return this;
+ }
+
+ public ColStatus setWithSizeInfo(boolean withSizeInfo) {
+ this.withSizeInfo = withSizeInfo;
+ return this;
+ }
+
+ @Override
+ public SolrParams getParams() {
+ ModifiableSolrParams params = (ModifiableSolrParams)super.getParams();
+ params.setNonNull("segments", withSegments.toString());
+ params.setNonNull("fieldInfo", withFieldInfo.toString());
+ params.setNonNull("coreInfo", withCoreInfo.toString());
+ params.setNonNull("sizeInfo", withSizeInfo.toString());
+ return params;
+ }
+ }
+
+ /**
* Returns a SolrRequest to delete a collection
*/
public static Delete deleteCollection(String collection) {
diff --cc solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java
index 6176263,a9e00e9..df97c35
--- a/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java
+++ b/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java
@@@ -122,7 -122,7 +122,8 @@@ public interface CollectionParams
NONE(false, LockLevel.NONE),
// TODO: not implemented yet
MERGESHARDS(true, LockLevel.SHARD),
- COLSTATUS(true, LockLevel.NONE)
++ COLSTATUS(true, LockLevel.NONE),
+ REINDEX_COLLECTION(true, LockLevel.COLLECTION)
;
public final boolean isWrite;
[lucene-solr] 04/05: Merge branch 'master' into jira/solr-11127-2
Posted by ab...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ab pushed a commit to branch jira/solr-11127-2
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 841938fecd57019d9cde86ea6eceed45a983b770
Merge: fb37b7c c1bea96
Author: Andrzej Bialecki <ab...@apache.org>
AuthorDate: Tue Mar 12 18:03:28 2019 +0100
Merge branch 'master' into jira/solr-11127-2
lucene/CHANGES.txt | 13 ++
.../apache/lucene/analysis/ko/KoreanTokenizer.java | 23 +-
.../lucene/analysis/ko/TestKoreanTokenizer.java | 14 ++
.../org/apache/lucene/analysis/ko/userdict.txt | 7 +-
.../src/java/org/apache/lucene/geo/EdgeTree.java | 47 ++--
.../src/java/org/apache/lucene/geo/GeoUtils.java | 23 +-
.../src/java/org/apache/lucene/geo/Polygon2D.java | 4 +-
.../test/org/apache/lucene/geo/TestPolygon2D.java | 30 +++
.../org/apache/lucene/search/TestSynonymQuery.java | 9 +-
.../writercache/LruTaxonomyWriterCache.java | 2 +-
.../taxonomy/writercache/NameIntCacheLRU.java | 4 +-
.../src/java/org/apache/lucene/geo/Line2D.java | 20 +-
.../lucene/document/BaseLatLonShapeTestCase.java | 2 +-
.../apache/lucene/document/TestLatLonShape.java | 27 ++-
.../src/test/org/apache/lucene/geo/TestLine2D.java | 85 +++++++
.../tools/src/groovy/check-source-patterns.groovy | 5 +-
solr/CHANGES.txt | 37 +--
.../apache/solr/request/macro/MacroExpander.java | 13 +-
.../solr/cloud/MoveReplicaHDFSFailoverTest.java | 15 +-
.../org/apache/solr/cloud/MoveReplicaHDFSTest.java | 4 +-
.../collections/TestHdfsCloudBackupRestore.java | 20 +-
.../HdfsAutoAddReplicasIntegrationTest.java | 2 +
.../solr/cloud/hdfs/HDFSCollectionsAPITest.java | 14 +-
.../cloud/hdfs/HdfsBasicDistributedZkTest.java | 13 +-
.../hdfs/HdfsChaosMonkeyNothingIsSafeTest.java | 14 +-
.../cloud/hdfs/HdfsChaosMonkeySafeLeaderTest.java | 12 +-
.../apache/solr/cloud/hdfs/HdfsNNFailoverTest.java | 11 +-
.../apache/solr/cloud/hdfs/HdfsRecoveryZkTest.java | 14 +-
.../cloud/hdfs/HdfsRestartWhileUpdatingTest.java | 12 +-
.../org/apache/solr/cloud/hdfs/HdfsTestUtil.java | 3 +-
.../HdfsTlogReplayBufferedWhileIndexingTest.java | 19 +-
.../cloud/hdfs/HdfsUnloadDistributedZkTest.java | 10 +-
.../hdfs/HdfsWriteToMultipleCollectionsTest.java | 12 +-
.../org/apache/solr/cloud/hdfs/StressHdfsTest.java | 36 ++-
.../apache/solr/core/HdfsDirectoryFactoryTest.java | 251 ++++++++++-----------
.../solr/handler/TestHdfsBackupRestoreCore.java | 16 +-
.../component/DistributedFacetExistsSmallTest.java | 10 +-
.../org/apache/solr/logging/TestLogWatcher.java | 63 ++++--
.../solr/request/macro/TestMacroExpander.java | 25 ++
.../org/apache/solr/search/TestRecoveryHdfs.java | 52 ++---
.../org/apache/solr/search/TestXmlQParser.java | 55 +++++
.../org/apache/solr/update/TestHdfsUpdateLog.java | 31 ++-
solr/solr-ref-guide/src/streaming-expressions.adoc | 12 +-
.../java/org/apache/solr/client/solrj/io/Lang.java | 2 +-
.../solr/client/solrj/io/stream/NoOpStream.java | 107 +++++++++
.../stream/expr/InjectedExpressionException.java | 24 ++
.../solrj/io/stream/expr/InjectionDefense.java | 199 ++++++++++++++++
.../io/stream/expr/StreamExpressionParser.java | 116 +++++-----
.../org/apache/solr/client/solrj/io/TestLang.java | 2 +-
.../solrj/io/stream/StreamExpressionTest.java | 109 ++++-----
.../solrj/io/stream/expr/InjectionDefenseTest.java | 115 ++++++++++
51 files changed, 1233 insertions(+), 532 deletions(-)
[lucene-solr] 01/05: Merge branch 'master' into jira/solr-11127-2
Posted by ab...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ab pushed a commit to branch jira/solr-11127-2
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 675dd0bcf8d624a4b9ee5ff2a0a26d4c0e88ec72
Merge: 1a01235 e827dc2
Author: Andrzej Bialecki <ab...@apache.org>
AuthorDate: Thu Mar 7 10:47:35 2019 +0100
Merge branch 'master' into jira/solr-11127-2
lucene/CHANGES.txt | 3 ++
solr/CHANGES.txt | 46 ++++++++++++++++++++++
.../solr/handler/component/QueryComponent.java | 18 +++++----
.../solr/handler/component/ResponseBuilder.java | 2 +-
.../org/apache/solr/util/StartupLoggingUtils.java | 11 +++---
.../test/org/apache/solr/TestTolerantSearch.java | 10 ++---
.../apache/solr/cloud/LeaderTragicEventTest.java | 2 +
.../solr/client/solrj/io/stream/ZplotStream.java | 4 --
8 files changed, 73 insertions(+), 23 deletions(-)
[lucene-solr] 03/05: SOLR-11127: Improved error handling, bug fixes,
more tests.
Posted by ab...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ab pushed a commit to branch jira/solr-11127-2
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit fb37b7c730e01e2b3b246cb32f34c7001269196e
Author: Andrzej Bialecki <ab...@apache.org>
AuthorDate: Tue Mar 12 18:02:43 2019 +0100
SOLR-11127: Improved error handling, bug fixes, more tests.
---
.../OverseerCollectionMessageHandler.java | 2 +-
.../api/collections/ReindexCollectionCmd.java | 176 ++++++++++++++++-----
.../java/org/apache/solr/core/CoreContainer.java | 6 +
.../solr/handler/admin/CollectionsHandler.java | 7 +-
.../java/org/apache/solr/util/TestInjection.java | 35 ++++
.../apache/solr/cloud/ReindexCollectionTest.java | 150 ++++++++++++++++--
.../solrj/request/CollectionAdminRequest.java | 32 +++-
.../solr/common/params/CollectionParams.java | 3 +-
8 files changed, 349 insertions(+), 62 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java
index 26818c4..86b8570 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java
@@ -241,7 +241,7 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler,
.put(DELETEREPLICA, new DeleteReplicaCmd(this))
.put(ADDREPLICA, new AddReplicaCmd(this))
.put(MOVEREPLICA, new MoveReplicaCmd(this))
- .put(REINDEX_COLLECTION, new ReindexCollectionCmd(this))
+ .put(REINDEXCOLLECTION, new ReindexCollectionCmd(this))
.put(UTILIZENODE, new UtilizeNodeCmd(this))
.build()
;
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/ReindexCollectionCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/ReindexCollectionCmd.java
index d676791..452e394 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/ReindexCollectionCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/ReindexCollectionCmd.java
@@ -38,6 +38,7 @@ import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.request.QueryRequest;
import org.apache.solr.cloud.Overseer;
import org.apache.solr.common.SolrException;
+import org.apache.solr.common.cloud.Aliases;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.DocRouter;
@@ -52,6 +53,7 @@ import org.apache.solr.common.params.CoreAdminParams;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.Utils;
+import org.apache.solr.util.TestInjection;
import org.apache.solr.util.TimeOut;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -84,12 +86,11 @@ public class ReindexCollectionCmd implements OverseerCollectionMessageHandler.Cm
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
public static final String ABORT = "abort";
- public static final String KEEP_SOURCE = "keepSource";
+ public static final String REMOVE_SOURCE = "removeSource";
public static final String TARGET = "target";
- public static final String TARGET_COL_PREFIX = ".reindex_";
- public static final String CHK_COL_PREFIX = ".reindex_ck_";
- public static final String REINDEXING_PROP = CollectionAdminRequest.PROPERTY_PREFIX + "reindexing";
- public static final String REINDEX_PHASE_PROP = CollectionAdminRequest.PROPERTY_PREFIX + "reindex_phase";
+ public static final String TARGET_COL_PREFIX = ".rx_";
+ public static final String CHK_COL_PREFIX = ".rx_ck_";
+ public static final String REINDEXING_PROP = CollectionAdminRequest.PROPERTY_PREFIX + "rx";
private static final List<String> COLLECTION_PARAMS = Arrays.asList(
ZkStateReader.CONFIGNAME_PROP,
@@ -141,21 +142,39 @@ public class ReindexCollectionCmd implements OverseerCollectionMessageHandler.Cm
@Override
public void call(ClusterState clusterState, ZkNodeProps message, NamedList results) throws Exception {
- log.info("*** called: {}", message);
+ log.debug("*** called: {}", message);
String collection = message.getStr(CommonParams.NAME);
+ // before resolving aliases
+ String originalCollection = collection;
+ Aliases aliases = ocmh.zkStateReader.getAliases();
+ if (collection != null) {
+ // resolve aliases - the source may be an alias
+ List<String> aliasList = aliases.resolveAliases(collection);
+ if (aliasList != null && !aliasList.isEmpty()) {
+ collection = aliasList.get(0);
+ }
+ }
+
if (collection == null || clusterState.getCollectionOrNull(collection) == null) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Collection name must be specified and must exist");
}
String target = message.getStr(TARGET);
if (target == null) {
target = collection;
+ } else {
+ // resolve aliases
+ List<String> aliasList = aliases.resolveAliases(target);
+ if (aliasList != null && !aliasList.isEmpty()) {
+ target = aliasList.get(0);
+ }
}
- boolean sameTarget = target.equals(collection);
- boolean keepSource = message.getBool(KEEP_SOURCE, true);
+ boolean sameTarget = target.equals(collection) || target.equals(originalCollection);
+ boolean removeSource = message.getBool(REMOVE_SOURCE, false);
boolean abort = message.getBool(ABORT, false);
DocCollection coll = clusterState.getCollection(collection);
if (abort) {
+ log.info("Abort requested for collection " + collection + ", setting the state to ABORTED.");
ZkNodeProps props = new ZkNodeProps(
Overseer.QUEUE_OPERATION, CollectionParams.CollectionAction.MODIFYCOLLECTION.toLower(),
ZkStateReader.COLLECTION_PROP, collection,
@@ -181,6 +200,7 @@ public class ReindexCollectionCmd implements OverseerCollectionMessageHandler.Cm
boolean aborted = false;
int batchSize = message.getInt(CommonParams.ROWS, 100);
String query = message.getStr(CommonParams.Q, "*:*");
+ String fl = message.getStr(CommonParams.FL, "*");
Integer rf = message.getInt(ZkStateReader.REPLICATION_FACTOR, coll.getReplicationFactor());
Integer numNrt = message.getInt(ZkStateReader.NRT_REPLICAS, coll.getNumNrtReplicas());
Integer numTlog = message.getInt(ZkStateReader.TLOG_REPLICAS, coll.getNumTlogReplicas());
@@ -193,12 +213,23 @@ public class ReindexCollectionCmd implements OverseerCollectionMessageHandler.Cm
}
String configName = message.getStr(ZkStateReader.CONFIGNAME_PROP, ocmh.zkStateReader.readConfigName(collection));
+ String targetCollection;
int seq = tmpCollectionSeq.getAndIncrement();
- String targetCollection = sameTarget ?
- TARGET_COL_PREFIX + collection + "_" + seq : target;
- String chkCollection = CHK_COL_PREFIX + collection + "_" + seq;
+ if (sameTarget) {
+ do {
+ targetCollection = TARGET_COL_PREFIX + originalCollection + "_" + seq;
+ if (!clusterState.hasCollection(targetCollection)) {
+ break;
+ }
+ seq = tmpCollectionSeq.getAndIncrement();
+ } while (clusterState.hasCollection(targetCollection));
+ } else {
+ targetCollection = target;
+ }
+ String chkCollection = CHK_COL_PREFIX + originalCollection + "_" + seq;
String daemonUrl = null;
Exception exc = null;
+ boolean createdTarget = false;
try {
// 0. set up target and checkpoint collections
NamedList<Object> cmdResults = new NamedList<>();
@@ -214,7 +245,7 @@ public class ReindexCollectionCmd implements OverseerCollectionMessageHandler.Cm
CoreAdminParams.DELETE_METRICS_HISTORY, "true"
);
ocmh.commandMap.get(CollectionParams.CollectionAction.DELETE).call(clusterState, cmd, cmdResults);
- // nocommit error checking
+ checkResults("deleting old checkpoint collection " + chkCollection, cmdResults, true);
}
if (maybeAbort(collection)) {
@@ -259,8 +290,10 @@ public class ReindexCollectionCmd implements OverseerCollectionMessageHandler.Cm
}
// create the target collection
cmd = new ZkNodeProps(propMap);
+ cmdResults = new NamedList<>();
ocmh.commandMap.get(CollectionParams.CollectionAction.CREATE).call(clusterState, cmd, cmdResults);
- // nocommit error checking
+ createdTarget = true;
+ checkResults("creating target collection " + targetCollection, cmdResults, true);
// create the checkpoint collection - use RF=1 and 1 shard
cmd = new ZkNodeProps(
@@ -271,8 +304,9 @@ public class ReindexCollectionCmd implements OverseerCollectionMessageHandler.Cm
CollectionAdminParams.COLL_CONF, "_default",
CommonAdminParams.WAIT_FOR_FINAL_STATE, "true"
);
+ cmdResults = new NamedList<>();
ocmh.commandMap.get(CollectionParams.CollectionAction.CREATE).call(clusterState, cmd, cmdResults);
- // nocommit error checking
+ checkResults("creating checkpoint collection " + chkCollection, cmdResults, true);
// wait for a while until we see both collections
TimeOut waitUntil = new TimeOut(30, TimeUnit.SECONDS, ocmh.timeSource);
boolean created = false;
@@ -296,7 +330,14 @@ public class ReindexCollectionCmd implements OverseerCollectionMessageHandler.Cm
Overseer.QUEUE_OPERATION, CollectionParams.CollectionAction.MODIFYCOLLECTION.toLower(),
ZkStateReader.COLLECTION_PROP, collection,
ZkStateReader.READ_ONLY, "true");
- ocmh.overseer.offerStateUpdate(Utils.toJSON(props));
+ ocmh.overseer.offerStateUpdate(Utils.toJSON(cmd));
+
+ TestInjection.injectReindexLatch();
+
+ if (maybeAbort(collection)) {
+ aborted = true;
+ return;
+ }
// 2. copy the documents to target
// Recipe taken from: http://joelsolr.blogspot.com/2016/10/solr-63-batch-jobs-parallel-etl-and.html
@@ -312,12 +353,12 @@ public class ReindexCollectionCmd implements OverseerCollectionMessageHandler.Cm
"topic(" + chkCollection + "," +
collection + "," +
"q=\"" + query + "\"," +
- "fl=\"*\"," +
+ "fl=\"" + fl + "\"," +
"id=\"topic_" + targetCollection + "\"," +
// some of the documents eg. in .system contain large blobs
"rows=\"" + batchSize + "\"," +
"initialCheckpoint=\"0\"))))");
- log.info("- starting copying documents from " + collection + " to " + targetCollection);
+ log.debug("- starting copying documents from " + collection + " to " + targetCollection);
SolrResponse rsp = null;
try {
rsp = ocmh.cloudManager.request(new QueryRequest(q));
@@ -337,14 +378,18 @@ public class ReindexCollectionCmd implements OverseerCollectionMessageHandler.Cm
aborted = true;
return;
}
- log.info("- finished copying from " + collection + " to " + targetCollection);
+ TestInjection.injectReindexFailure();
+ log.debug("- finished copying from " + collection + " to " + targetCollection);
// 5. if (sameTarget) set up an alias to use targetCollection as the source name
if (sameTarget) {
+ log.debug("- setting up alias from " + originalCollection + " to " + targetCollection);
cmd = new ZkNodeProps(
- CommonParams.NAME, collection,
+ CommonParams.NAME, originalCollection,
"collections", targetCollection);
+ cmdResults = new NamedList<>();
ocmh.commandMap.get(CollectionParams.CollectionAction.CREATEALIAS).call(clusterState, cmd, results);
+ checkResults("setting up alias " + originalCollection + " -> " + targetCollection, cmdResults, true);
}
if (maybeAbort(collection)) {
@@ -352,42 +397,51 @@ public class ReindexCollectionCmd implements OverseerCollectionMessageHandler.Cm
return;
}
// 6. delete the checkpoint collection
+ log.debug("- deleting " + chkCollection);
cmd = new ZkNodeProps(
Overseer.QUEUE_OPERATION, CollectionParams.CollectionAction.DELETE.toLower(),
CommonParams.NAME, chkCollection,
CoreAdminParams.DELETE_METRICS_HISTORY, "true"
);
+ cmdResults = new NamedList<>();
ocmh.commandMap.get(CollectionParams.CollectionAction.DELETE).call(clusterState, cmd, cmdResults);
-
- // nocommit error checking
+ checkResults("deleting checkpoint collection " + chkCollection, cmdResults, true);
// 7. optionally delete the source collection
- if (keepSource) {
- // 8. set the FINISHED state and clear readOnly
- props = new ZkNodeProps(
- Overseer.QUEUE_OPERATION, CollectionParams.CollectionAction.MODIFYCOLLECTION.toLower(),
- ZkStateReader.COLLECTION_PROP, collection,
- REINDEXING_PROP, State.FINISHED.toLower(),
- ZkStateReader.READ_ONLY, "");
- ocmh.overseer.offerStateUpdate(Utils.toJSON(props));
- } else {
+ if (removeSource) {
+ log.debug("- deleting source collection");
cmd = new ZkNodeProps(
Overseer.QUEUE_OPERATION, CollectionParams.CollectionAction.DELETE.toLower(),
CommonParams.NAME, collection,
CoreAdminParams.DELETE_METRICS_HISTORY, "true"
);
+ cmdResults = new NamedList<>();
ocmh.commandMap.get(CollectionParams.CollectionAction.DELETE).call(clusterState, cmd, cmdResults);
+ checkResults("deleting source collection " + collection, cmdResults, true);
+ } else {
+ // 8. clear readOnly on source
+ props = new ZkNodeProps(
+ Overseer.QUEUE_OPERATION, CollectionParams.CollectionAction.MODIFYCOLLECTION.toLower(),
+ ZkStateReader.COLLECTION_PROP, collection,
+ ZkStateReader.READ_ONLY, null);
+ ocmh.overseer.offerStateUpdate(Utils.toJSON(props));
}
+ // 9. set FINISHED state on the target
+ props = new ZkNodeProps(
+ Overseer.QUEUE_OPERATION, CollectionParams.CollectionAction.MODIFYCOLLECTION.toLower(),
+ ZkStateReader.COLLECTION_PROP, targetCollection,
+ REINDEXING_PROP, State.FINISHED.toLower());
+ ocmh.overseer.offerStateUpdate(Utils.toJSON(props));
- results.add(State.FINISHED.toLower(), collection);
+ results.add(State.FINISHED.toLower(), originalCollection);
} catch (Exception e) {
- log.warn("Error during reindexing of " + collection, e);
+ log.warn("Error during reindexing of " + originalCollection, e);
exc = e;
aborted = true;
throw e;
} finally {
if (aborted) {
- cleanup(collection, targetCollection, chkCollection, daemonUrl, targetCollection);
+ cleanup(collection, targetCollection, chkCollection, daemonUrl, targetCollection, createdTarget);
results.add(State.ABORTED.toLower(), collection);
if (exc != null) {
results.add("error", exc.toString());
@@ -396,16 +450,33 @@ public class ReindexCollectionCmd implements OverseerCollectionMessageHandler.Cm
}
}
+ private void checkResults(String label, NamedList<Object> results, boolean failureIsFatal) throws Exception {
+ Object failure = results.get("failure");
+ if (failure == null) {
+ failure = results.get("error");
+ }
+ if (failure != null) {
+ String msg = "Error: " + label + ": " + Utils.toJSONString(results);
+ if (failureIsFatal) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, msg);
+ } else {
+ log.error(msg);
+ }
+ }
+ }
+
private boolean maybeAbort(String collection) throws Exception {
DocCollection coll = ocmh.cloudManager.getClusterStateProvider().getClusterState().getCollectionOrNull(collection);
if (coll == null) {
// collection no longer present - abort
+ log.info("## Aborting - collection {} no longer present.", collection);
return true;
}
State state = State.get(coll.getStr(REINDEXING_PROP, State.RUNNING.toLower()));
if (state != State.ABORTED) {
return false;
}
+ log.info("## Aborting - collection {} state is {}", collection, state);
return true;
}
@@ -413,11 +484,11 @@ public class ReindexCollectionCmd implements OverseerCollectionMessageHandler.Cm
private String getDaemonUrl(SolrResponse rsp, DocCollection coll) {
Map<String, Object> rs = (Map<String, Object>)rsp.getResponse().get("result-set");
if (rs == null || rs.isEmpty()) {
- log.debug("Missing daemon information in response: " + Utils.toJSONString(rsp));
+ log.debug(" -- Missing daemon information in response: " + Utils.toJSONString(rsp));
}
List<Object> list = (List<Object>)rs.get("docs");
if (list == null) {
- log.debug("Missing daemon information in response: " + Utils.toJSONString(rsp));
+ log.debug(" -- Missing daemon information in response: " + Utils.toJSONString(rsp));
return null;
}
String replicaName = null;
@@ -429,7 +500,7 @@ public class ReindexCollectionCmd implements OverseerCollectionMessageHandler.Cm
}
String[] parts = op.split("\\s+");
if (parts.length != 4) {
- log.debug("Invalid daemon location info, expected 4 tokens: " + op);
+ log.debug(" -- Invalid daemon location info, expected 4 tokens: " + op);
return null;
}
// check if it's plausible
@@ -437,7 +508,7 @@ public class ReindexCollectionCmd implements OverseerCollectionMessageHandler.Cm
replicaName = parts[3];
break;
} else {
- log.debug("daemon location info likely invalid: " + op);
+ log.debug(" -- daemon location info likely invalid: " + op);
return null;
}
}
@@ -493,12 +564,13 @@ public class ReindexCollectionCmd implements OverseerCollectionMessageHandler.Cm
} catch (Exception e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Exception copying the documents", e);
}
- ocmh.cloudManager.getTimeSource().sleep(5000);
+ ocmh.cloudManager.getTimeSource().sleep(2000);
} while (isRunning && !maybeAbort(collection));
}
}
private void killDaemon(String daemonName, String daemonUrl) throws Exception {
+ log.debug("-- killing daemon " + daemonName + " at " + daemonUrl);
HttpClient client = ocmh.overseer.getCoreContainer().getUpdateShardHandler().getDefaultHttpClient();
try (HttpSolrClient solrClient = new HttpSolrClient.Builder()
.withHttpClient(client)
@@ -516,7 +588,9 @@ public class ReindexCollectionCmd implements OverseerCollectionMessageHandler.Cm
}
}
- private void cleanup(String collection, String targetCollection, String chkCollection, String daemonUrl, String daemonName) throws Exception {
+ private void cleanup(String collection, String targetCollection, String chkCollection,
+ String daemonUrl, String daemonName, boolean createdTarget) throws Exception {
+ log.info("## Cleaning up after abort or error");
// 1. kill the daemon
// 2. cleanup target / chk collections IFF the source collection still exists and is not empty
// 3. cleanup collection state
@@ -526,20 +600,36 @@ public class ReindexCollectionCmd implements OverseerCollectionMessageHandler.Cm
}
ClusterState clusterState = ocmh.cloudManager.getClusterStateProvider().getClusterState();
NamedList<Object> cmdResults = new NamedList<>();
- if (!collection.equals(targetCollection) && clusterState.hasCollection(targetCollection)) {
+ if (createdTarget && !collection.equals(targetCollection) && clusterState.hasCollection(targetCollection)) {
+ log.debug(" -- removing " + targetCollection);
ZkNodeProps cmd = new ZkNodeProps(
Overseer.QUEUE_OPERATION, CollectionParams.CollectionAction.DELETE.toLower(),
CommonParams.NAME, targetCollection,
CoreAdminParams.DELETE_METRICS_HISTORY, "true"
);
ocmh.commandMap.get(CollectionParams.CollectionAction.DELETE).call(clusterState, cmd, cmdResults);
- // nocommit error checking
+ checkResults("CLEANUP: deleting target collection " + targetCollection, cmdResults, false);
+
+ }
+ // remove chk collection
+ if (clusterState.hasCollection(chkCollection)) {
+ log.debug(" -- removing " + chkCollection);
+ ZkNodeProps cmd = new ZkNodeProps(
+ Overseer.QUEUE_OPERATION, CollectionParams.CollectionAction.DELETE.toLower(),
+ CommonParams.NAME, chkCollection,
+ CoreAdminParams.DELETE_METRICS_HISTORY, "true"
+ );
+ cmdResults = new NamedList<>();
+ ocmh.commandMap.get(CollectionParams.CollectionAction.DELETE).call(clusterState, cmd, cmdResults);
+ checkResults("CLEANUP: deleting checkpoint collection " + chkCollection, cmdResults, false);
}
+ log.debug(" -- turning readOnly mode off for " + collection);
ZkNodeProps props = new ZkNodeProps(
Overseer.QUEUE_OPERATION, CollectionParams.CollectionAction.MODIFYCOLLECTION.toLower(),
ZkStateReader.COLLECTION_PROP, collection,
- REINDEXING_PROP, State.ABORTED.toLower(),
- ZkStateReader.READ_ONLY, "");
+ // remove the rx flag, we already aborted
+ REINDEXING_PROP, null,
+ ZkStateReader.READ_ONLY, null);
ocmh.overseer.offerStateUpdate(Utils.toJSON(props));
}
}
diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
index 8c5e227..58c9f69 100644
--- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java
+++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
@@ -747,10 +747,16 @@ public class CoreContainer {
containerHandlers.put(AutoScalingHandler.HANDLER_PATH, autoScalingHandler);
autoScalingHandler.initializeMetrics(metricManager, SolrInfoBean.Group.node.toString(), metricTag, AutoScalingHandler.HANDLER_PATH);
}
+ // verify .system compatibility
+ systemCollCompatCheck();
// This is a bit redundant but these are two distinct concepts for all they're accomplished at the same time.
status |= LOAD_COMPLETE | INITIAL_CORE_LOAD_COMPLETE;
}
+ private void systemCollCompatCheck() {
+
+ }
+
// MetricsHistoryHandler supports both cloud and standalone configs
private void createMetricsHistoryHandler() {
PluginInfo plugin = cfg.getMetricsConfig().getHistoryHandler();
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
index 6b53c00..106eece 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
@@ -542,11 +542,11 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission
RELOAD_OP(RELOAD, (req, rsp, h) -> copy(req.getParams().required(), null, NAME)),
- REINDEX_COLLECTION_OP(REINDEX_COLLECTION, (req, rsp, h) -> {
+ REINDEXCOLLECTION_OP(REINDEXCOLLECTION, (req, rsp, h) -> {
Map<String, Object> m = copy(req.getParams().required(), null, NAME);
copy(req.getParams(), m,
ReindexCollectionCmd.ABORT,
- ReindexCollectionCmd.KEEP_SOURCE,
+ ReindexCollectionCmd.REMOVE_SOURCE,
ReindexCollectionCmd.TARGET,
ZkStateReader.CONFIGNAME_PROP,
NUM_SLICES,
@@ -561,7 +561,8 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission
AUTO_ADD_REPLICAS,
"shards",
CommonParams.ROWS,
- CommonParams.Q);
+ CommonParams.Q,
+ CommonParams.FL);
copyPropertiesWithPrefix(req.getParams(), m, "router.");
return m;
}),
diff --git a/solr/core/src/java/org/apache/solr/util/TestInjection.java b/solr/core/src/java/org/apache/solr/util/TestInjection.java
index cf7681e..7a49ba4 100644
--- a/solr/core/src/java/org/apache/solr/util/TestInjection.java
+++ b/solr/core/src/java/org/apache/solr/util/TestInjection.java
@@ -126,6 +126,10 @@ public class TestInjection {
public volatile static CountDownLatch splitLatch = null;
+ public volatile static CountDownLatch reindexLatch = null;
+
+ public volatile static String reindexFailure = null;
+
public volatile static String failIndexFingerprintRequests = null;
public volatile static String wrongIndexFingerprint = null;
@@ -156,6 +160,8 @@ public class TestInjection {
splitFailureBeforeReplicaCreation = null;
splitFailureAfterReplicaCreation = null;
splitLatch = null;
+ reindexLatch = null;
+ reindexFailure = null;
prepRecoveryOpPauseForever = null;
countPrepRecoveryOpPauseForever = new AtomicInteger(0);
failIndexFingerprintRequests = null;
@@ -423,6 +429,35 @@ public class TestInjection {
return true;
}
+ public static boolean injectReindexFailure() {
+ if (reindexFailure != null) {
+ Random rand = random();
+ if (null == rand) return true;
+
+ Pair<Boolean,Integer> pair = parseValue(reindexFailure);
+ boolean enabled = pair.first();
+ int chanceIn100 = pair.second();
+ if (enabled && rand.nextInt(100) >= (100 - chanceIn100)) {
+ log.info("Test injection failure");
+ throw new SolrException(ErrorCode.SERVER_ERROR, "Test injection failure");
+ }
+ }
+ return true;
+ }
+
+
+ public static boolean injectReindexLatch() {
+ if (reindexLatch != null) {
+ try {
+ log.info("Waiting in ReindexCollectionCmd for up to 60s");
+ return reindexLatch.await(60, TimeUnit.SECONDS);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ return true;
+ }
+
private static Pair<Boolean,Integer> parseValue(final String raw) {
if (raw == null) return new Pair<>(false, 0);
Matcher m = ENABLED_PERCENT.matcher(raw);
diff --git a/solr/core/src/test/org/apache/solr/cloud/ReindexCollectionTest.java b/solr/core/src/test/org/apache/solr/cloud/ReindexCollectionTest.java
index 2a9cfd4..218476c 100644
--- a/solr/core/src/test/org/apache/solr/cloud/ReindexCollectionTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/ReindexCollectionTest.java
@@ -21,21 +21,27 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
import java.util.function.Function;
import org.apache.solr.client.solrj.cloud.SolrCloudManager;
+import org.apache.solr.client.solrj.impl.BaseHttpSolrClient;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.cloud.api.collections.ReindexCollectionCmd;
import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.ImplicitDocRouter;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.CommonParams;
-import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.util.LogLevel;
+import org.apache.solr.util.TestInjection;
+import org.apache.solr.util.TimeOut;
import org.junit.After;
import org.junit.Before;
import org.junit.BeforeClass;
@@ -44,6 +50,7 @@ import org.junit.Test;
/**
*
*/
+@LogLevel("org.apache.solr.cloud.api.collections.ReindexCollectionCmd=DEBUG")
public class ReindexCollectionTest extends SolrCloudTestCase {
@BeforeClass
@@ -74,6 +81,8 @@ public class ReindexCollectionTest extends SolrCloudTestCase {
cluster.deleteAllCollections(); // deletes aliases too
solrClient.close();
+
+ TestInjection.reset();
}
private static final int NUM_DOCS = 200; // at least two batches, default batchSize=100
@@ -93,7 +102,7 @@ public class ReindexCollectionTest extends SolrCloudTestCase {
.setTarget(targetCollection);
req.process(solrClient);
- CloudTestUtils.waitForState(cloudManager, "did not finish copying in time", sourceCollection, (liveNodes, coll) -> {
+ CloudTestUtils.waitForState(cloudManager, "did not finish copying in time", targetCollection, (liveNodes, coll) -> {
ReindexCollectionCmd.State state = ReindexCollectionCmd.State.get(coll.getStr(ReindexCollectionCmd.REINDEXING_PROP));
return ReindexCollectionCmd.State.FINISHED == state;
});
@@ -114,7 +123,24 @@ public class ReindexCollectionTest extends SolrCloudTestCase {
.setTarget(targetCollection);
req.process(solrClient);
- CloudTestUtils.waitForState(cloudManager, "did not finish copying in time", sourceCollection, (liveNodes, coll) -> {
+ String realTargetCollection = null;
+ TimeOut timeOut = new TimeOut(30, TimeUnit.SECONDS, cloudManager.getTimeSource());
+ String prefix = ReindexCollectionCmd.TARGET_COL_PREFIX + targetCollection;
+ while (!timeOut.hasTimedOut()) {
+ timeOut.sleep(500);
+ for (String name : cloudManager.getClusterStateProvider().getClusterState().getCollectionsMap().keySet()) {
+ if (name.startsWith(prefix)) {
+ realTargetCollection = name;
+ break;
+ }
+ }
+ if (realTargetCollection != null) {
+ break;
+ }
+ }
+ assertNotNull("target collection not present after 30s", realTargetCollection);
+
+ CloudTestUtils.waitForState(cloudManager, "did not finish copying in time", realTargetCollection, (liveNodes, coll) -> {
ReindexCollectionCmd.State state = ReindexCollectionCmd.State.get(coll.getStr(ReindexCollectionCmd.REINDEXING_PROP));
return ReindexCollectionCmd.State.FINISHED == state;
});
@@ -142,7 +168,7 @@ public class ReindexCollectionTest extends SolrCloudTestCase {
.setConfigName("conf3");
req.process(solrClient);
- CloudTestUtils.waitForState(cloudManager, "did not finish copying in time", sourceCollection, (liveNodes, coll) -> {
+ CloudTestUtils.waitForState(cloudManager, "did not finish copying in time", targetCollection, (liveNodes, coll) -> {
ReindexCollectionCmd.State state = ReindexCollectionCmd.State.get(coll.getStr(ReindexCollectionCmd.REINDEXING_PROP));
return ReindexCollectionCmd.State.FINISHED == state;
});
@@ -159,26 +185,38 @@ public class ReindexCollectionTest extends SolrCloudTestCase {
@Test
public void testReshapeReindexing() throws Exception {
final String sourceCollection = "reshapeReindexing";
- final String targetCollection = sourceCollection;
+ final String targetCollection = "reshapeReindexingTarget";
createCollection(sourceCollection, "conf1", 2, 2);
indexDocs(sourceCollection, NUM_DOCS,
- i -> new SolrInputDocument("id", String.valueOf(i), "string_s", String.valueOf(i)));
+ i -> new SolrInputDocument(
+ "id", String.valueOf(i),
+ "string_s", String.valueOf(i),
+ "remove_s", String.valueOf(i)));
CollectionAdminRequest.ReindexCollection req = CollectionAdminRequest.reindexCollection(sourceCollection)
.setTarget(targetCollection)
.setCollectionParam(ZkStateReader.NUM_SHARDS_PROP, 3)
.setCollectionParam(ZkStateReader.REPLICATION_FACTOR, 1)
.setCollectionParam("router.name", ImplicitDocRouter.NAME)
- .setCollectionParam("shards", "foo,bar,baz");
+ .setCollectionParam("shards", "foo,bar,baz")
+ .setCollectionParam("fl", "id,string_s")
+ .setCollectionParam("q", "id:10*");
req.process(solrClient);
- CloudTestUtils.waitForState(cloudManager, "did not finish copying in time", sourceCollection, (liveNodes, coll) -> {
+ CloudTestUtils.waitForState(cloudManager, "did not finish copying in time", targetCollection, (liveNodes, coll) -> {
ReindexCollectionCmd.State state = ReindexCollectionCmd.State.get(coll.getStr(ReindexCollectionCmd.REINDEXING_PROP));
return ReindexCollectionCmd.State.FINISHED == state;
});
// verify the target docs exist
QueryResponse rsp = solrClient.query(targetCollection, params(CommonParams.Q, "*:*"));
- assertEquals("copied num docs", NUM_DOCS, rsp.getResults().getNumFound());
+ // 10 and 100-109
+ assertEquals("copied num docs", 11, rsp.getResults().getNumFound());
+ // verify the correct fields exist
+ for (SolrDocument doc : rsp.getResults()) {
+ assertNotNull(doc.getFieldValue("id"));
+ assertNotNull(doc.getFieldValue("string_s"));
+ assertNull(doc.getFieldValue("remove_s"));
+ }
// check the shape of the new collection
ClusterState clusterState = solrClient.getClusterStateProvider().getClusterState();
@@ -195,6 +233,100 @@ public class ReindexCollectionTest extends SolrCloudTestCase {
assertEquals(ImplicitDocRouter.NAME, coll.getRouter().getName());
}
+ @Test
+ public void testFailure() throws Exception {
+ final String sourceCollection = "failReindexing";
+ final String targetCollection = "failReindexingTarget";
+ final String aliasTarget = "failAlias";
+ createCollection(sourceCollection, "conf1", 2, 2);
+ createCollection(targetCollection, "conf1", 1, 1);
+ CollectionAdminRequest.createAlias(aliasTarget, targetCollection).process(solrClient);
+ indexDocs(sourceCollection, NUM_DOCS,
+ i -> new SolrInputDocument(
+ "id", String.valueOf(i),
+ "string_s", String.valueOf(i)));
+
+ CollectionAdminRequest.ReindexCollection req = CollectionAdminRequest.reindexCollection(sourceCollection)
+ .setTarget(targetCollection);
+ try {
+ req.process(solrClient);
+ fail("succeeded but expected reindexing to fail due to the target collection already present");
+ } catch (Exception e) {
+ assertTrue(e instanceof BaseHttpSolrClient.RemoteSolrException);
+ BaseHttpSolrClient.RemoteSolrException rse = (BaseHttpSolrClient.RemoteSolrException)e;
+ assertEquals(SolrException.ErrorCode.SERVER_ERROR.code, rse.code());
+ }
+ req = CollectionAdminRequest.reindexCollection(sourceCollection)
+ .setTarget(aliasTarget);
+ try {
+ req.process(solrClient);
+ fail("succeeded but expected reindexing to fail due to the target collection already present");
+ } catch (Exception e) {
+ assertTrue(e instanceof BaseHttpSolrClient.RemoteSolrException);
+ BaseHttpSolrClient.RemoteSolrException rse = (BaseHttpSolrClient.RemoteSolrException)e;
+ assertEquals(SolrException.ErrorCode.SERVER_ERROR.code, rse.code());
+ }
+
+ CollectionAdminRequest.deleteAlias(aliasTarget).process(solrClient);
+ CollectionAdminRequest.deleteCollection(targetCollection).process(solrClient);
+
+ req = CollectionAdminRequest.reindexCollection(sourceCollection)
+ .setTarget(targetCollection);
+
+ TestInjection.reindexFailure = "true:100";
+ try {
+ req.process(solrClient);
+ fail("succeeded but expected reindexing to fail due to a test-injected failure");
+ } catch (Exception e) {
+ assertTrue(e instanceof BaseHttpSolrClient.RemoteSolrException);
+ BaseHttpSolrClient.RemoteSolrException rse = (BaseHttpSolrClient.RemoteSolrException)e;
+ assertEquals(SolrException.ErrorCode.SERVER_ERROR.code, rse.code());
+ }
+ // verify that the target and checkpoint collections don't exist
+ cloudManager.getClusterStateProvider().getClusterState().forEachCollection(coll -> {
+ assertFalse(coll.getName() + " still exists", coll.getName().startsWith(ReindexCollectionCmd.TARGET_COL_PREFIX));
+ assertFalse(coll.getName() + " still exists", coll.getName().startsWith(ReindexCollectionCmd.CHK_COL_PREFIX));
+ });
+ // verify that the source collection is read-write and has no reindexing flags
+ CloudTestUtils.waitForState(cloudManager, "collection state is incorrect", sourceCollection,
+ ((liveNodes, collectionState) ->
+ !collectionState.isReadOnly() &&
+ collectionState.getStr(ReindexCollectionCmd.REINDEXING_PROP) == null));
+ }
+
+ @Test
+ public void testAbort() throws Exception {
+ final String sourceCollection = "abortReindexing";
+ final String targetCollection = "abortReindexingTarget";
+ createCollection(sourceCollection, "conf1", 2, 1);
+
+ TestInjection.reindexLatch = new CountDownLatch(1);
+ CollectionAdminRequest.ReindexCollection req = CollectionAdminRequest.reindexCollection(sourceCollection)
+ .setTarget(targetCollection);
+ String asyncId = req.processAsync(solrClient);
+ // wait for the source collection to be put in readOnly mode
+ CloudTestUtils.waitForState(cloudManager, "source collection didn't become readOnly",
+ sourceCollection, (liveNodes, coll) -> coll.isReadOnly());
+
+ req = CollectionAdminRequest.reindexCollection(sourceCollection);
+ req.setAbort(true);
+ req.process(solrClient);
+ CloudTestUtils.waitForState(cloudManager, "incorrect collection state", sourceCollection,
+ ((liveNodes, collectionState) ->
+ collectionState.isReadOnly() &&
+ ReindexCollectionCmd.State.ABORTED.toLower().equals(collectionState.getStr(ReindexCollectionCmd.REINDEXING_PROP))));
+ // let the process continue
+ TestInjection.reindexLatch.countDown();
+ CloudTestUtils.waitForState(cloudManager, "source collection is in wrong state",
+ sourceCollection, (liveNodes, docCollection) -> {
+ System.err.println("-- coll " + docCollection);
+ return !docCollection.isReadOnly() && docCollection.getStr(ReindexCollectionCmd.REINDEXING_PROP) == null;
+ });
+ // verify the response
+ CollectionAdminRequest.RequestStatusResponse rsp = CollectionAdminRequest.requestStatus(asyncId).process(solrClient);
+ rsp.getRequestStatus();
+ }
+
private void createCollection(String name, String config, int numShards, int numReplicas) throws Exception {
CollectionAdminRequest.createCollection(name, config, numShards, numReplicas)
.setMaxShardsPerNode(-1)
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
index ddfe90e..d0ae477 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
@@ -793,40 +793,60 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>
public static class ReindexCollection extends AsyncCollectionSpecificAdminRequest {
String target;
String query;
+ String fields;
String configName;
- Boolean keepSource;
+ Boolean removeSource;
+ Boolean abort;
Integer batchSize;
Map<String, Object> collectionParams = new HashMap<>();
private ReindexCollection(String collection) {
- super(CollectionAction.REINDEX_COLLECTION, collection);
+ super(CollectionAction.REINDEXCOLLECTION, collection);
}
+ /** Target collection name (null if the same). */
public ReindexCollection setTarget(String target) {
this.target = target;
return this;
}
+ /** Set to true to abort already running requests. */
+ public ReindexCollection setAbort(boolean abort) {
+ this.abort = abort;
+ return this;
+ }
+
+ /** Query matching the documents to reindex (default is '*:*'). */
public ReindexCollection setQuery(String query) {
this.query = query;
return this;
}
- public ReindexCollection setKeepSource(boolean keepSource) {
- this.keepSource = keepSource;
+ /** Fields to reindex (the same syntax as {@link CommonParams#FL}), default is '*'. */
+ public ReindexCollection setFields(String fields) {
+ this.fields = fields;
+ return this;
+ }
+
+ /** Remove source collection after success. Default is false. */
+ public ReindexCollection setRemoveSource(boolean removeSource) {
+ this.removeSource = removeSource;
return this;
}
+ /** Copy documents in batches of this size. Default is 100. */
public ReindexCollection setBatchSize(int batchSize) {
this.batchSize = batchSize;
return this;
}
+ /** Config name for the target collection. Default is the same as source. */
public ReindexCollection setConfigName(String configName) {
this.configName = configName;
return this;
}
+ /** Set other supported collection CREATE parameters. */
public ReindexCollection setCollectionParam(String key, Object value) {
this.collectionParams.put(key, value);
return this;
@@ -836,9 +856,11 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>
public SolrParams getParams() {
ModifiableSolrParams params = (ModifiableSolrParams) super.getParams();
params.setNonNull("target", target);
+ params.setNonNull("abort", abort);
params.setNonNull(ZkStateReader.CONFIGNAME_PROP, configName);
params.setNonNull(CommonParams.Q, query);
- params.setNonNull("keepSource", keepSource);
+ params.setNonNull(CommonParams.FL, fields);
+ params.setNonNull("removeSource", removeSource);
params.setNonNull(CommonParams.ROWS, batchSize);
collectionParams.forEach((k, v) -> params.setNonNull(k, v));
return params;
diff --git a/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java b/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java
index df97c35..1485e34 100644
--- a/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java
+++ b/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java
@@ -123,7 +123,8 @@ public interface CollectionParams {
// TODO: not implemented yet
MERGESHARDS(true, LockLevel.SHARD),
COLSTATUS(true, LockLevel.NONE),
- REINDEX_COLLECTION(true, LockLevel.COLLECTION)
+ // this command implements its own locking
+ REINDEXCOLLECTION(true, LockLevel.NONE)
;
public final boolean isWrite;