You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ab...@apache.org on 2017/12/07 12:45:43 UTC
[49/50] [abbrv] lucene-solr:jira/solr-11285-sim: Merge branch
'master' into jira/solr-11285-sim
Merge branch 'master' into jira/solr-11285-sim
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/5ec9b64b
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/5ec9b64b
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/5ec9b64b
Branch: refs/heads/jira/solr-11285-sim
Commit: 5ec9b64b270c8c4d4388fcfdc9074785d7567389
Parents: 5d2524e 4fc5a87
Author: Andrzej Bialecki <ab...@apache.org>
Authored: Thu Dec 7 13:08:37 2017 +0100
Committer: Andrzej Bialecki <ab...@apache.org>
Committed: Thu Dec 7 13:08:37 2017 +0100
----------------------------------------------------------------------
lucene/CHANGES.txt | 35 +-
lucene/MIGRATE.txt | 9 +
.../apache/lucene/document/RangeFieldQuery.java | 3 +-
.../SortedNumericDocValuesRangeQuery.java | 3 +-
.../document/SortedSetDocValuesRangeQuery.java | 3 +-
.../apache/lucene/index/DocumentsWriter.java | 20 +-
.../index/DocumentsWriterFlushControl.java | 88 ++-
.../lucene/index/DocumentsWriterPerThread.java | 4 +-
.../org/apache/lucene/index/FlushPolicy.java | 27 +-
.../lucene/index/FrozenBufferedUpdates.java | 3 +-
.../org/apache/lucene/index/IndexWriter.java | 105 ++-
.../org/apache/lucene/index/SegmentInfos.java | 5 +-
.../java/org/apache/lucene/index/Sorter.java | 5 +
.../lucene/search/Boolean2ScorerSupplier.java | 30 +-
.../org/apache/lucene/search/BooleanQuery.java | 6 +-
.../org/apache/lucene/search/BooleanWeight.java | 25 +-
.../org/apache/lucene/search/BoostQuery.java | 7 +-
.../apache/lucene/search/CachingCollector.java | 13 +-
.../org/apache/lucene/search/Collector.java | 6 +-
.../apache/lucene/search/ConjunctionScorer.java | 20 +
.../lucene/search/ConstantScoreQuery.java | 14 +-
.../lucene/search/ConstantScoreScorer.java | 5 +
.../org/apache/lucene/search/DisiWrapper.java | 3 +
.../lucene/search/DisjunctionMaxQuery.java | 14 +-
.../lucene/search/DisjunctionMaxScorer.java | 6 +
.../lucene/search/DisjunctionSumScorer.java | 7 +
.../search/DocValuesFieldExistsQuery.java | 2 +-
.../lucene/search/DocValuesRewriteMethod.java | 2 +-
.../lucene/search/DoubleValuesSource.java | 4 +-
.../apache/lucene/search/ExactPhraseScorer.java | 5 +
.../org/apache/lucene/search/FakeScorer.java | 5 +
.../apache/lucene/search/FilterCollector.java | 4 +-
.../org/apache/lucene/search/FilterScorer.java | 3 +
.../org/apache/lucene/search/FilterWeight.java | 2 +-
.../lucene/search/IndexOrDocValuesQuery.java | 6 +-
.../org/apache/lucene/search/IndexSearcher.java | 34 +-
.../apache/lucene/search/MatchAllDocsQuery.java | 2 +-
.../apache/lucene/search/MatchNoDocsQuery.java | 2 +-
.../lucene/search/MinShouldMatchSumScorer.java | 6 +
.../apache/lucene/search/MultiCollector.java | 13 +-
.../lucene/search/MultiCollectorManager.java | 18 +-
.../apache/lucene/search/MultiPhraseQuery.java | 4 +-
.../MultiTermQueryConstantScoreWrapper.java | 4 +-
.../lucene/search/NormsFieldExistsQuery.java | 2 +-
.../org/apache/lucene/search/PhraseQuery.java | 4 +-
.../apache/lucene/search/PointInSetQuery.java | 2 +-
.../apache/lucene/search/PointRangeQuery.java | 2 +-
.../java/org/apache/lucene/search/Query.java | 4 +-
.../org/apache/lucene/search/QueryCache.java | 2 +-
.../org/apache/lucene/search/QueryRescorer.java | 2 +-
.../org/apache/lucene/search/ReqExclScorer.java | 11 +
.../apache/lucene/search/ReqOptSumScorer.java | 5 +
.../search/ScoreCachingWrappingScorer.java | 12 +-
.../org/apache/lucene/search/ScoreMode.java | 60 ++
.../java/org/apache/lucene/search/Scorer.java | 17 +
.../lucene/search/SloppyPhraseScorer.java | 5 +
.../org/apache/lucene/search/SynonymQuery.java | 49 +-
.../apache/lucene/search/TermInSetQuery.java | 4 +-
.../org/apache/lucene/search/TermQuery.java | 25 +-
.../org/apache/lucene/search/TermScorer.java | 11 +-
.../lucene/search/TimeLimitingCollector.java | 4 +-
.../apache/lucene/search/TopDocsCollector.java | 2 +-
.../apache/lucene/search/TopFieldCollector.java | 4 +-
.../lucene/search/TopScoreDocCollector.java | 79 ++-
.../lucene/search/TotalHitCountCollector.java | 4 +-
.../search/UsageTrackingQueryCachingPolicy.java | 2 +-
.../org/apache/lucene/search/WANDScorer.java | 478 +++++++++++++
.../java/org/apache/lucene/search/Weight.java | 2 +-
.../org/apache/lucene/search/package-info.java | 6 +-
.../lucene/search/similarities/AfterEffect.java | 27 +-
.../search/similarities/AfterEffectB.java | 6 +-
.../search/similarities/AfterEffectL.java | 6 +-
.../lucene/search/similarities/Axiomatic.java | 6 +
.../search/similarities/BM25Similarity.java | 8 +-
.../lucene/search/similarities/BasicModel.java | 10 +-
.../search/similarities/BasicModelBE.java | 55 --
.../lucene/search/similarities/BasicModelD.java | 56 --
.../lucene/search/similarities/BasicModelG.java | 12 +-
.../search/similarities/BasicModelIF.java | 11 +-
.../search/similarities/BasicModelIn.java | 15 +-
.../search/similarities/BasicModelIne.java | 11 +-
.../lucene/search/similarities/BasicModelP.java | 49 --
.../search/similarities/BooleanSimilarity.java | 5 +
.../search/similarities/DFISimilarity.java | 6 +
.../search/similarities/DFRSimilarity.java | 30 +-
.../search/similarities/IBSimilarity.java | 6 +
.../similarities/LMDirichletSimilarity.java | 8 +-
.../similarities/LMJelinekMercerSimilarity.java | 8 +-
.../search/similarities/MultiSimilarity.java | 9 +
.../lucene/search/similarities/Similarity.java | 7 +
.../search/similarities/SimilarityBase.java | 14 +-
.../search/similarities/TFIDFSimilarity.java | 14 +
.../search/spans/FieldMaskingSpanQuery.java | 5 +-
.../lucene/search/spans/SpanBoostQuery.java | 5 +-
.../search/spans/SpanContainingQuery.java | 9 +-
.../search/spans/SpanMultiTermQueryWrapper.java | 3 +-
.../lucene/search/spans/SpanNearQuery.java | 9 +-
.../lucene/search/spans/SpanNotQuery.java | 9 +-
.../apache/lucene/search/spans/SpanOrQuery.java | 7 +-
.../search/spans/SpanPositionCheckQuery.java | 7 +-
.../apache/lucene/search/spans/SpanQuery.java | 3 +-
.../apache/lucene/search/spans/SpanScorer.java | 5 +
.../lucene/search/spans/SpanTermQuery.java | 5 +-
.../lucene/search/spans/SpanWithinQuery.java | 9 +-
.../org/apache/lucene/util/QueryBuilder.java | 2 +-
.../java/org/apache/lucene/util/Version.java | 7 +
.../src/test/org/apache/lucene/TestSearch.java | 42 --
.../lucene/index/TestIndexTooManyDocs.java | 114 +++
.../apache/lucene/index/TestIndexWriter.java | 129 ++++
.../lucene/index/TestIndexWriterExceptions.java | 28 +-
.../lucene/index/TestMaxTermFrequency.java | 4 +
.../org/apache/lucene/index/TestOmitTf.java | 21 +-
.../apache/lucene/search/JustCompileSearch.java | 9 +-
.../lucene/search/MultiCollectorTest.java | 24 +-
.../search/TestBoolean2ScorerSupplier.java | 62 +-
.../search/TestBooleanMinShouldMatch.java | 10 +-
.../org/apache/lucene/search/TestBooleanOr.java | 6 +-
.../apache/lucene/search/TestBooleanQuery.java | 26 +-
.../search/TestBooleanQueryVisitSubscorers.java | 14 +-
.../lucene/search/TestBooleanRewrites.java | 2 +-
.../apache/lucene/search/TestBooleanScorer.java | 14 +-
.../apache/lucene/search/TestBoostQuery.java | 18 +-
.../lucene/search/TestCachingCollector.java | 11 +-
.../lucene/search/TestComplexExplanations.java | 2 +-
.../lucene/search/TestConjunctionDISI.java | 4 +
.../apache/lucene/search/TestConjunctions.java | 9 +-
.../lucene/search/TestConstantScoreQuery.java | 14 +-
.../lucene/search/TestDisjunctionMaxQuery.java | 20 +-
.../lucene/search/TestDocValuesQueries.java | 2 +-
.../lucene/search/TestDocValuesScoring.java | 5 +
.../lucene/search/TestDoubleValuesSource.java | 8 +-
.../lucene/search/TestEarlyTermination.java | 4 +-
.../search/TestIndexOrDocValuesQuery.java | 4 +-
.../apache/lucene/search/TestLRUQueryCache.java | 14 +-
.../lucene/search/TestMinShouldMatch2.java | 6 +-
.../apache/lucene/search/TestNeedsScores.java | 22 +-
.../apache/lucene/search/TestPointQueries.java | 12 +-
.../lucene/search/TestPositionIncrement.java | 4 +-
.../search/TestPositiveScoresOnlyCollector.java | 7 +-
.../apache/lucene/search/TestQueryRescorer.java | 9 +-
.../search/TestScoreCachingWrappingScorer.java | 11 +-
.../apache/lucene/search/TestScorerPerf.java | 6 +-
.../apache/lucene/search/TestSimilarity.java | 16 +-
.../lucene/search/TestSimilarityProvider.java | 10 +-
.../lucene/search/TestSloppyPhraseQuery.java | 8 +-
.../apache/lucene/search/TestSortRandom.java | 2 +-
.../lucene/search/TestSubScorerFreqs.java | 5 +
.../org/apache/lucene/search/TestTermQuery.java | 4 +-
.../apache/lucene/search/TestTermScorer.java | 14 +-
.../search/TestTimeLimitingCollector.java | 4 +-
.../lucene/search/TestTopDocsCollector.java | 206 +++++-
.../apache/lucene/search/TestTopDocsMerge.java | 3 +-
.../lucene/search/TestTopFieldCollector.java | 9 +-
.../TestTopFieldCollectorEarlyTermination.java | 2 +-
.../TestUsageTrackingFilterCachingPolicy.java | 2 +-
.../apache/lucene/search/TestWANDScorer.java | 394 +++++++++++
.../search/similarities/BasicModelTestCase.java | 5 +-
.../search/similarities/TestBasicModelBE.java | 30 -
.../search/similarities/TestBasicModelD.java | 30 -
.../search/similarities/TestBasicModelP.java | 30 -
.../search/similarities/TestSimilarityBase.java | 62 +-
.../search/spans/JustCompileSearchSpans.java | 3 +-
.../search/spans/TestFieldMaskingSpanQuery.java | 15 +-
.../search/spans/TestNearSpansOrdered.java | 29 +-
.../lucene/search/spans/TestSpanCollection.java | 7 +-
.../search/spans/TestSpanContainQuery.java | 3 +-
.../apache/lucene/search/spans/TestSpans.java | 11 +-
.../apache/lucene/expressions/FakeScorer.java | 5 +
.../org/apache/lucene/facet/DrillSideways.java | 11 +-
.../apache/lucene/facet/DrillSidewaysQuery.java | 7 +-
.../lucene/facet/DrillSidewaysScorer.java | 5 +
.../apache/lucene/facet/FacetsCollector.java | 7 +-
.../apache/lucene/facet/range/DoubleRange.java | 5 +-
.../facet/range/DoubleRangeFacetCounts.java | 3 +-
.../apache/lucene/facet/range/LongRange.java | 5 +-
.../facet/range/LongRangeFacetCounts.java | 3 +-
.../facet/AssertingSubDocsAtOnceCollector.java | 5 +-
.../apache/lucene/facet/TestDrillSideways.java | 7 +-
.../facet/range/TestRangeFacetCounts.java | 5 +-
.../search/grouping/AllGroupHeadsCollector.java | 5 +-
.../search/grouping/AllGroupsCollector.java | 5 +-
.../search/grouping/BlockGroupingCollector.java | 5 +-
.../grouping/DistinctValuesCollector.java | 5 +-
.../lucene/search/grouping/FakeScorer.java | 5 +
.../grouping/FirstPassGroupingCollector.java | 5 +-
.../search/grouping/GroupFacetCollector.java | 5 +-
.../lucene/search/grouping/GroupingSearch.java | 3 +-
.../grouping/SecondPassGroupingCollector.java | 5 +-
.../lucene/search/grouping/TestGrouping.java | 5 +-
.../search/highlight/QueryTermExtractor.java | 3 +-
.../highlight/WeightedSpanTermExtractor.java | 9 +-
.../lucene/search/uhighlight/PhraseHelper.java | 7 +-
.../search/uhighlight/UnifiedHighlighter.java | 3 +-
.../search/highlight/HighlighterPhraseTest.java | 5 +-
.../uhighlight/TestUnifiedHighlighterMTQ.java | 5 +-
.../TestUnifiedHighlighterStrictPhrases.java | 5 +-
.../search/join/BaseGlobalOrdinalScorer.java | 5 +
.../apache/lucene/search/join/FakeScorer.java | 5 +
.../search/join/GenericTermsCollector.java | 4 +-
.../search/join/GlobalOrdinalsCollector.java | 4 +-
.../lucene/search/join/GlobalOrdinalsQuery.java | 4 +-
.../join/GlobalOrdinalsWithScoreCollector.java | 8 +-
.../join/GlobalOrdinalsWithScoreQuery.java | 8 +-
.../org/apache/lucene/search/join/JoinUtil.java | 8 +-
.../join/ParentChildrenBlockJoinQuery.java | 8 +-
.../join/PointInSetIncludingScoreQuery.java | 7 +-
.../lucene/search/join/QueryBitSetProducer.java | 2 +-
.../lucene/search/join/TermsCollector.java | 4 +-
.../search/join/TermsIncludingScoreQuery.java | 11 +-
.../search/join/TermsWithScoreCollector.java | 4 +-
.../search/join/ToChildBlockJoinQuery.java | 9 +-
.../search/join/ToParentBlockJoinQuery.java | 15 +-
.../lucene/search/join/TestBlockJoin.java | 21 +-
.../search/join/TestBlockJoinValidation.java | 2 +-
.../apache/lucene/search/join/TestJoinUtil.java | 28 +-
.../apache/lucene/index/memory/MemoryIndex.java | 5 +-
.../apache/lucene/index/PKIndexSplitter.java | 4 +-
.../search/DiversifiedTopDocsCollector.java | 4 +-
.../lucene/search/DocValuesStatsCollector.java | 4 +-
.../search/TestDiversifiedTopDocsCollector.java | 5 +
.../apache/lucene/queries/BoostingQuery.java | 23 +-
.../apache/lucene/queries/CustomScoreQuery.java | 16 +-
.../lucene/queries/function/BoostedQuery.java | 51 +-
.../queries/function/FunctionMatchQuery.java | 3 +-
.../lucene/queries/function/FunctionQuery.java | 29 +-
.../queries/function/FunctionRangeQuery.java | 3 +-
.../queries/function/FunctionScoreQuery.java | 66 +-
.../lucene/queries/function/ValueSource.java | 5 +
.../queries/function/ValueSourceScorer.java | 5 +
.../function/valuesource/QueryValueSource.java | 3 +-
.../queries/payloads/PayloadScoreQuery.java | 22 +-
.../queries/payloads/SpanPayloadCheckQuery.java | 7 +-
.../function/TestFunctionScoreExplanations.java | 10 +-
.../function/TestFunctionScoreQuery.java | 41 ++
.../function/TestIndexReaderFunctions.java | 5 +-
.../queries/function/TestValueSources.java | 12 +-
.../queries/payloads/TestPayloadSpans.java | 31 +-
.../queries/payloads/TestPayloadTermQuery.java | 5 +-
.../surround/query/BooleanQueryTst.java | 5 +-
.../document/LatLonDocValuesBoxQuery.java | 3 +-
.../document/LatLonDocValuesDistanceQuery.java | 3 +-
.../document/LatLonPointDistanceQuery.java | 3 +-
.../document/LatLonPointInPolygonQuery.java | 3 +-
.../apache/lucene/payloads/PayloadSpanUtil.java | 3 +-
.../org/apache/lucene/search/CoveringQuery.java | 4 +-
.../apache/lucene/search/CoveringScorer.java | 6 +
.../lucene/search/DocValuesNumbersQuery.java | 2 +-
.../lucene/search/DocValuesTermsQuery.java | 2 +-
.../lucene/search/TermAutomatonQuery.java | 2 +-
.../lucene/search/TermAutomatonScorer.java | 6 +-
.../sandbox/queries/FuzzyLikeThisQueryTest.java | 9 +-
.../lucene/search/TestTermAutomatonQuery.java | 2 +-
.../spatial/composite/CompositeVerifyQuery.java | 5 +-
.../composite/IntersectsRPTVerifyQuery.java | 3 +-
.../spatial/prefix/AbstractPrefixTreeQuery.java | 3 +-
.../serialized/SerializedDVStrategy.java | 3 +-
.../spatial/vector/PointVectorStrategy.java | 5 +-
.../spatial/prefix/NumberRangeFacetsTest.java | 5 +-
.../spatial3d/PointInGeo3DShapeQuery.java | 3 +-
.../lucene/spatial3d/geom/GeoExactCircle.java | 67 +-
.../lucene/spatial3d/geom/PlanetModel.java | 20 +-
.../apache/lucene/spatial3d/TestGeo3DPoint.java | 5 +-
.../spatial3d/geom/GeoExactCircleTest.java | 45 ++
.../search/spell/LevenshteinDistance.java | 125 ++++
.../lucene/search/spell/LevensteinDistance.java | 125 ----
.../lucene/search/spell/SpellChecker.java | 4 +-
.../search/suggest/document/ContextQuery.java | 5 +-
.../suggest/document/FuzzyCompletionQuery.java | 3 +-
.../suggest/document/PrefixCompletionQuery.java | 3 +-
.../suggest/document/RegexCompletionQuery.java | 3 +-
.../suggest/document/SuggestIndexSearcher.java | 2 +-
.../document/TopSuggestDocsCollector.java | 5 +-
.../search/spell/TestLevenshteinDistance.java | 2 +-
.../lucene/search/spell/TestSpellChecker.java | 4 +-
.../apache/lucene/geo/BaseGeoPointTestCase.java | 17 +-
.../apache/lucene/index/RandomIndexWriter.java | 11 +
.../lucene/search/AssertingBulkScorer.java | 12 +-
.../lucene/search/AssertingCollector.java | 2 +-
.../lucene/search/AssertingIndexSearcher.java | 4 +-
.../lucene/search/AssertingLeafCollector.java | 6 +-
.../apache/lucene/search/AssertingQuery.java | 5 +-
.../apache/lucene/search/AssertingScorer.java | 31 +-
.../apache/lucene/search/AssertingWeight.java | 12 +-
.../search/BaseRangeFieldQueryTestCase.java | 2 +-
.../lucene/search/BulkScorerWrapperScorer.java | 5 +
.../org/apache/lucene/search/CheckHits.java | 10 +-
.../org/apache/lucene/search/QueryUtils.java | 22 +-
.../lucene/search/RandomApproximationQuery.java | 9 +-
.../lucene/search/ShardSearchingTestBase.java | 2 +-
.../similarities/AssertingSimilarity.java | 12 +-
.../similarities/BaseSimilarityTestCase.java | 155 ++++-
.../search/similarities/RandomSimilarity.java | 6 +-
.../lucene/search/spans/AssertingSpanQuery.java | 5 +-
.../search/TestBaseExplanationTestCase.java | 5 +-
solr/CHANGES.txt | 87 ++-
solr/README.txt | 33 +-
solr/bin/solr | 2 +-
solr/bin/solr.cmd | 4 +-
.../analytics/facet/AbstractSolrQueryFacet.java | 5 +-
.../java/org/apache/solr/ltr/LTRRescorer.java | 5 +-
.../org/apache/solr/ltr/LTRScoringQuery.java | 22 +-
.../org/apache/solr/ltr/feature/Feature.java | 4 +
.../solr/ltr/feature/FieldLengthFeature.java | 5 +
.../solr/ltr/feature/FieldValueFeature.java | 5 +
.../solr/ltr/feature/OriginalScoreFeature.java | 8 +-
.../apache/solr/ltr/feature/SolrFeature.java | 8 +-
.../LTRFeatureLoggerTransformerFactory.java | 3 +-
.../solr/ltr/TestLTRReRankingPipeline.java | 5 +-
.../apache/solr/ltr/TestLTRScoringQuery.java | 3 +-
.../solr/ltr/TestSelectiveWeightCreation.java | 3 +-
.../org/apache/solr/cloud/AddReplicaCmd.java | 22 +-
.../src/java/org/apache/solr/cloud/Assign.java | 14 +-
.../apache/solr/cloud/CreateCollectionCmd.java | 11 +-
.../org/apache/solr/cloud/CreateShardCmd.java | 9 +-
.../org/apache/solr/cloud/MoveReplicaCmd.java | 85 ++-
.../cloud/OverseerCollectionMessageHandler.java | 2 +-
.../java/org/apache/solr/cloud/RestoreCmd.java | 4 +-
.../org/apache/solr/cloud/SplitShardCmd.java | 5 +-
.../org/apache/solr/cloud/UtilizeNodeCmd.java | 12 +-
.../org/apache/solr/cloud/ZkController.java | 1 +
.../cloud/autoscaling/ComputePlanAction.java | 54 +-
.../cloud/autoscaling/ExecutePlanAction.java | 4 +-
.../cloud/autoscaling/ScheduledTriggers.java | 7 +-
.../apache/solr/core/HdfsDirectoryFactory.java | 12 +-
.../src/java/org/apache/solr/core/SolrCore.java | 113 +--
.../apache/solr/handler/ReplicationHandler.java | 42 +-
.../org/apache/solr/handler/SnapShooter.java | 75 +-
.../org/apache/solr/handler/StreamHandler.java | 6 +-
.../apache/solr/handler/admin/BackupCoreOp.java | 6 -
.../solr/handler/admin/CollectionsHandler.java | 2 +
.../solr/handler/component/ExpandComponent.java | 9 +-
.../solr/handler/component/QueryComponent.java | 5 +
.../handler/component/RealTimeGetComponent.java | 3 +-
.../org/apache/solr/parser/QueryParser.java | 5 +-
.../apache/solr/parser/SolrQueryParserBase.java | 100 ++-
.../java/org/apache/solr/query/FilterQuery.java | 7 +-
.../org/apache/solr/query/SolrRangeQuery.java | 7 +-
.../java/org/apache/solr/schema/FieldType.java | 2 +
.../java/org/apache/solr/schema/LatLonType.java | 8 +-
.../java/org/apache/solr/schema/TextField.java | 11 +
.../apache/solr/search/AbstractReRankQuery.java | 6 +-
.../solr/search/CollapsingQParserPlugin.java | 14 +-
.../apache/solr/search/DelegatingCollector.java | 5 +-
.../org/apache/solr/search/DocSetCollector.java | 5 +-
.../apache/solr/search/ExportQParserPlugin.java | 6 +-
.../solr/search/ExtendedDismaxQParser.java | 5 +-
.../src/java/org/apache/solr/search/Filter.java | 3 +-
.../solr/search/GraphTermsQParserPlugin.java | 5 +-
.../apache/solr/search/HashQParserPlugin.java | 3 +-
.../apache/solr/search/JoinQParserPlugin.java | 3 +-
.../apache/solr/search/QueryWrapperFilter.java | 3 +-
.../org/apache/solr/search/ReRankCollector.java | 5 +-
.../solr/search/SolrConstantScoreQuery.java | 3 +-
.../apache/solr/search/SolrIndexSearcher.java | 14 +-
.../apache/solr/search/ValueSourceParser.java | 6 +-
.../org/apache/solr/search/WrappedQuery.java | 5 +-
.../facet/FacetFieldProcessorByHashDV.java | 7 +-
.../org/apache/solr/search/facet/HLLAgg.java | 2 +
.../org/apache/solr/search/facet/MinMaxAgg.java | 8 +-
.../org/apache/solr/search/facet/SlotAcc.java | 22 +-
.../solr/search/facet/UniqueMultiDvSlotAcc.java | 6 +-
.../search/facet/UniqueSinglevaluedSlotAcc.java | 12 +-
.../apache/solr/search/facet/UniqueSlotAcc.java | 9 +-
.../org/apache/solr/search/join/GraphQuery.java | 8 +-
.../solr/search/join/GraphTermsCollector.java | 5 +-
.../search/join/ScoreJoinQParserPlugin.java | 12 +-
.../similarities/DFRSimilarityFactory.java | 18 +-
.../solr/search/stats/ExactStatsCache.java | 3 +-
.../spelling/AbstractLuceneSpellChecker.java | 4 +-
.../apache/solr/spelling/SolrSpellChecker.java | 6 +-
.../solr/update/DeleteByQueryWrapper.java | 5 +-
.../DistributedUpdateProcessorFactory.java | 2 +-
.../TimePartitionedUpdateProcessor.java | 294 --------
.../TimeRoutedAliasUpdateProcessor.java | 294 ++++++++
.../solr/collection1/conf/schema-dfr.xml | 2 +-
.../solr/collection1/conf/schema12.xml | 65 ++
.../solr/collection1/conf/synonyms.txt | 8 +-
.../cloud/CollectionTooManyReplicasTest.java | 8 +-
.../apache/solr/cloud/MoveReplicaHDFSTest.java | 13 +
.../org/apache/solr/cloud/MoveReplicaTest.java | 38 +-
.../autoscaling/TriggerIntegrationTest.java | 40 +-
.../sim/SimClusterStateProvider.java | 23 +-
.../cloud/autoscaling/sim/TestLargeCluster.java | 1 +
.../org/apache/solr/handler/TestCoreBackup.java | 61 ++
.../admin/AutoscalingHistoryHandlerTest.java | 2 +-
.../solr/search/TestExtendedDismaxParser.java | 8 +-
.../solr/search/TestQueryWrapperFilter.java | 5 +-
.../apache/solr/search/TestRankQueryPlugin.java | 18 +-
.../apache/solr/search/TestSolrQueryParser.java | 26 +-
.../org/apache/solr/search/facet/DebugAgg.java | 5 +
.../solr/search/facet/TestJsonFacets.java | 29 +-
.../solr/search/function/TestFunctionQuery.java | 30 +-
.../similarities/TestDFRSimilarityFactory.java | 4 +-
.../ConjunctionSolrSpellCheckerTest.java | 6 +-
.../uninverting/TestFieldCacheSortRandom.java | 3 +-
.../apache/solr/update/CdcrUpdateLogTest.java | 4 +-
.../TimePartitionedUpdateProcessorTest.java | 275 --------
.../TimeRoutedAliasUpdateProcessorTest.java | 275 ++++++++
solr/solr-ref-guide/src/about-this-guide.adoc | 33 +-
...adding-custom-plugins-in-solrcloud-mode.adoc | 74 +-
.../src/basic-authentication-plugin.adoc | 70 +-
solr/solr-ref-guide/src/blob-store-api.adoc | 20 +-
solr/solr-ref-guide/src/cdcr-api.adoc | 313 +++++++++
solr/solr-ref-guide/src/cdcr-architecture.adoc | 157 +++++
solr/solr-ref-guide/src/cdcr-config.adoc | 342 +++++++++
solr/solr-ref-guide/src/cdcr-operations.adoc | 41 ++
solr/solr-ref-guide/src/config-api.adoc | 2 +-
solr/solr-ref-guide/src/config-sets.adoc | 21 +-
solr/solr-ref-guide/src/content-streams.adoc | 34 +-
.../src/cross-data-center-replication-cdcr.adoc | 692 +------------------
solr/solr-ref-guide/src/css/customstyles.css | 6 +-
.../src/field-properties-by-use-case.adoc | 2 +-
.../field-type-definitions-and-properties.adoc | 7 +
.../src/field-types-included-with-solr.adoc | 5 +
.../solr-ref-guide/src/filter-descriptions.adoc | 4 +-
solr/solr-ref-guide/src/function-queries.adoc | 2 +-
.../src/getting-started-with-solrcloud.adoc | 23 +-
solr/solr-ref-guide/src/highlighting.adoc | 24 +-
.../CDCR_bidir.png | Bin 0 -> 70828 bytes
solr/solr-ref-guide/src/json-facet-api.adoc | 11 +-
solr/solr-ref-guide/src/learning-to-rank.adoc | 2 +-
.../major-changes-from-solr-5-to-solr-6.adoc | 2 +-
.../src/major-changes-in-solr-7.adoc | 18 +
solr/solr-ref-guide/src/other-parsers.adoc | 4 +-
.../src/other-schema-elements.adoc | 2 +-
solr/solr-ref-guide/src/ping.adoc | 8 +-
solr/solr-ref-guide/src/realtime-get.adoc | 143 +++-
.../src/request-parameters-api.adoc | 2 +-
.../src/requestdispatcher-in-solrconfig.adoc | 30 +-
.../src/running-solr-on-hdfs.adoc | 40 +-
solr/solr-ref-guide/src/schema-api.adoc | 248 ++++++-
.../src/solrcloud-autoscaling-overview.adoc | 2 +-
solr/solr-ref-guide/src/spell-checking.adoc | 6 +-
.../src/stream-evaluator-reference.adoc | 2 +-
.../transforming-and-indexing-custom-json.adoc | 688 ++++++++++++++++--
.../src/transforming-result-documents.adoc | 4 +-
.../src/updating-parts-of-documents.adoc | 4 +-
...-the-solr-administration-user-interface.adoc | 3 +-
.../autoscaling/DelegatingCloudManager.java | 3 +-
.../DelegatingDistribStateManager.java | 106 +++
.../solrj/cloud/autoscaling/PolicyHelper.java | 297 ++++++--
.../solrj/impl/ConcurrentUpdateSolrClient.java | 4 +-
.../solrj/io/eval/AbsoluteValueEvaluator.java | 4 +-
.../client/solrj/io/eval/AnovaEvaluator.java | 3 +-
.../solrj/io/eval/ArcCosineEvaluator.java | 4 +-
.../client/solrj/io/eval/ArcSineEvaluator.java | 4 +-
.../solrj/io/eval/ArcTangentEvaluator.java | 3 +-
.../client/solrj/io/eval/CeilingEvaluator.java | 4 +-
.../solrj/io/eval/ConvolutionEvaluator.java | 5 +-
.../solrj/io/eval/CorrelationEvaluator.java | 13 +-
.../client/solrj/io/eval/CosineEvaluator.java | 4 +-
.../io/eval/CosineSimilarityEvaluator.java | 5 +-
.../solrj/io/eval/CovarianceEvaluator.java | 5 +-
.../solrj/io/eval/CubedRootEvaluator.java | 4 +-
.../solrj/io/eval/DerivativeEvaluator.java | 63 ++
.../client/solrj/io/eval/DescribeEvaluator.java | 3 +-
.../client/solrj/io/eval/DistanceEvaluator.java | 17 +-
.../solrj/io/eval/DotProductEvaluator.java | 5 +-
.../client/solrj/io/eval/EBEAddEvaluator.java | 5 +-
.../solrj/io/eval/EBEDivideEvaluator.java | 5 +-
.../solrj/io/eval/EBEMultiplyEvaluator.java | 5 +-
.../solrj/io/eval/EBESubtractEvaluator.java | 5 +-
.../io/eval/EmpiricalDistributionEvaluator.java | 3 +-
.../eval/EnumeratedDistributionEvaluator.java | 7 +-
.../solrj/io/eval/FactorialEvaluator.java | 3 +-
.../solrj/io/eval/FieldValueEvaluator.java | 11 +-
.../solrj/io/eval/FindDelayEvaluator.java | 5 +-
.../client/solrj/io/eval/FloorEvaluator.java | 4 +-
.../solrj/io/eval/HarmonicFitEvaluator.java | 7 +-
.../io/eval/HyperbolicCosineEvaluator.java | 4 +-
.../solrj/io/eval/HyperbolicSineEvaluator.java | 4 +-
.../client/solrj/io/eval/LoessEvaluator.java | 34 +-
.../solrj/io/eval/MeanDifferenceEvaluator.java | 5 +-
.../solrj/io/eval/NaturalLogEvaluator.java | 4 +-
.../solrj/io/eval/OLSRegressionEvaluator.java | 123 ++++
.../io/eval/PolyFitDerivativeEvaluator.java | 13 +-
.../client/solrj/io/eval/PolyFitEvaluator.java | 13 +-
.../client/solrj/io/eval/PredictEvaluator.java | 73 +-
.../client/solrj/io/eval/RankEvaluator.java | 3 +-
.../solrj/io/eval/RecursiveEvaluator.java | 7 +-
.../io/eval/RecursiveNumericEvaluator.java | 7 +-
.../solrj/io/eval/RegressionEvaluator.java | 14 +-
.../solrj/io/eval/ResidualsEvaluator.java | 83 ---
.../client/solrj/io/eval/RoundEvaluator.java | 4 +-
.../client/solrj/io/eval/SineEvaluator.java | 4 +-
.../client/solrj/io/eval/SplineEvaluator.java | 71 ++
.../solrj/io/eval/SquareRootEvaluator.java | 4 +-
.../solrj/io/eval/SumDifferenceEvaluator.java | 5 +-
.../client/solrj/io/eval/TangentEvaluator.java | 4 +-
.../io/eval/TimeDifferencingEvaluator.java | 3 +-
.../client/solrj/io/eval/VectorFunction.java | 55 ++
.../solrj/request/CollectionAdminRequest.java | 14 +
.../solr/common/cloud/ConnectionManager.java | 5 +-
.../apache/solr/common/cloud/SolrZkClient.java | 17 +-
.../apache/solr/common/cloud/ZkStateReader.java | 1 +
.../solr/common/params/CommonAdminParams.java | 7 +-
.../java/org/apache/solr/common/util/Utils.java | 10 +
.../collections.collection.Commands.json | 24 +-
.../solrj/cloud/autoscaling/TestPolicy.java | 97 ++-
.../ConcurrentUpdateSolrClientBuilderTest.java | 8 +
.../solrj/io/stream/StreamExpressionTest.java | 157 +++--
.../stream/eval/AbsoluteValueEvaluatorTest.java | 7 +-
.../solrj/io/stream/eval/AddEvaluatorTest.java | 11 +-
.../io/stream/eval/ArcCosineEvaluatorTest.java | 8 +-
.../io/stream/eval/ArcSineEvaluatorTest.java | 7 +-
.../io/stream/eval/ArcTangentEvaluatorTest.java | 8 +-
.../io/stream/eval/CeilingEvaluatorTest.java | 7 +-
.../io/stream/eval/CoalesceEvaluatorTest.java | 16 +-
.../io/stream/eval/CosineEvaluatorTest.java | 6 +-
.../io/stream/eval/CubedRootEvaluatorTest.java | 8 +-
.../io/stream/eval/FloorEvaluatorTest.java | 8 +-
.../eval/HyperbolicCosineEvaluatorTest.java | 7 +-
.../eval/HyperbolicSineEvaluatorTest.java | 7 +-
.../eval/HyperbolicTangentEvaluatorTest.java | 7 +-
.../io/stream/eval/MultiplyEvaluatorTest.java | 11 +-
.../io/stream/eval/NaturalLogEvaluatorTest.java | 5 +-
.../io/stream/eval/PowerEvaluatorTest.java | 6 +-
.../io/stream/eval/RoundEvaluatorTest.java | 8 +-
.../solrj/io/stream/eval/SineEvaluatorTest.java | 7 +-
.../io/stream/eval/SquareRootEvaluatorTest.java | 7 +-
.../io/stream/eval/SubtractEvaluatorTest.java | 10 +-
.../io/stream/eval/TangentEvaluatorTest.java | 7 +-
.../io/stream/eval/TemporalEvaluatorsTest.java | 3 -
.../java/org/apache/solr/cloud/ChaosMonkey.java | 5 +-
524 files changed, 8531 insertions(+), 3776 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5ec9b64b/solr/CHANGES.txt
----------------------------------------------------------------------
diff --cc solr/CHANGES.txt
index 8eda1a5,80d8557..2ccf62b
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@@ -102,8 -119,44 +119,46 @@@ New Feature
* SOLR-11250: A new DefaultWrapperModel class for loading of large and/or externally stored
LTRScoringModel definitions. (Yuki Yano, shalin, Christine Poerschke)
+ * SOLR-11662: New synonymQueryStyle option to configure whether SynonymQuery, a DisjunctionMaxQuery, or BooleanQuery
+ occurs over query terms that overlap their position. (Doug Turnbull, David Smiley)
+
+ * SOLR-11429: Add loess Stream Evaluator to support local regression (Joel Bernstein)
+
+ * SOLR-11568: Add matrix Stream Evaluator to support efficient matrix operations (Joel Bernstein)
+
+ * SOLR-11566: Add transpose Stream Evaluator to support transposing of matrices (Joel Bernstein)
+
+ * SOLR-11565: Add unit Stream Evaluator to support unitizing of vectors (Joel Bernstein)
+
+ * SOLR-11567: Add triangularDistribution Stream Evaluator (Joel Bernstein)
+
+ * SOLR-11593: Add support for covariance matrices to the cov Stream Evaluator (Joel Bernstein)
+
+ * SOLR-10680: Add minMaxScale Stream Evaluator (Joel Bernstein)
+
+ * SOLR-11599: Change normalize function to standardize and make it work with matrices (Joel Bernstein)
+
+ * SOLR-11602: Add Markov Chain Stream Evaluator (Joel Bernstein)
+
+ * SOLR-11607: Add grandSum, sumRows, sumColumns, scalarDivide, scalarMultiply, scalarAdd,
+ scalarSubtract Stream Evaluators (Joel Bernstein)
+
+ * SOLR-11571: Add diff Stream Evaluator to support time series differencing (Mathew Skaria, Joel Bernstein)
+
+ * SOLR-11570: Add support for correlation matrices to the corr Stream Evaluator (Joel Bernstein)
+
+ * SOLR-11569: Add support for distance matrices to the distance Stream Evaluator (Joel Bernstein)
+
+ * SOLR-11674: Support ranges in the probability Stream Evaluator (Joel Bernstein)
+
+ * SOLR-11680: Add normalizeSum Stream Evaluator (Joel Bernstein)
+
+ * SOLR-11697: Add geometricDistribution Stream Evaluator (Joel Bernstein)
+
+ * SOLR-11485: Add olsRegress, spline and derivative Stream Evaluators (Joel Bernstein)
+
+* SOLR-11285: Simulation framework for autoscaling. (ab)
+
Bug Fixes
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5ec9b64b/solr/core/src/java/org/apache/solr/cloud/AddReplicaCmd.java
----------------------------------------------------------------------
diff --cc solr/core/src/java/org/apache/solr/cloud/AddReplicaCmd.java
index 5887b78,c785f9f..b7a89d6
--- a/solr/core/src/java/org/apache/solr/cloud/AddReplicaCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/AddReplicaCmd.java
@@@ -56,6 -56,6 +57,7 @@@ import static org.apache.solr.common.cl
import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICA;
import static org.apache.solr.common.params.CommonAdminParams.ASYNC;
++import static org.apache.solr.common.params.CommonAdminParams.TIMEOUT;
import static org.apache.solr.common.params.CommonAdminParams.WAIT_FOR_FINAL_STATE;
public class AddReplicaCmd implements OverseerCollectionMessageHandler.Cmd {
@@@ -75,24 -75,75 +77,24 @@@
ZkNodeProps addReplica(ClusterState clusterState, ZkNodeProps message, NamedList results, Runnable onComplete)
throws IOException, InterruptedException {
log.debug("addReplica() : {}", Utils.toJSONString(message));
+ boolean waitForFinalState = message.getBool(WAIT_FOR_FINAL_STATE, false);
+ boolean skipCreateReplicaInClusterState = message.getBool(SKIP_CREATE_REPLICA_IN_CLUSTER_STATE, false);
+ final String asyncId = message.getStr(ASYNC);
+
- AtomicLong policyVersionAfter = new AtomicLong(-1);
- message = assignReplicaDetails(ocmh.cloudManager, clusterState, message, policyVersionAfter);
++ AtomicReference<PolicyHelper.SessionWrapper> sessionWrapper = new AtomicReference<>();
++ message = assignReplicaDetails(ocmh.cloudManager, clusterState, message, sessionWrapper);
+
String collection = message.getStr(COLLECTION_PROP);
+ DocCollection coll = clusterState.getCollection(collection);
+
String node = message.getStr(CoreAdminParams.NODE);
String shard = message.getStr(SHARD_ID_PROP);
String coreName = message.getStr(CoreAdminParams.NAME);
String coreNodeName = message.getStr(CoreAdminParams.CORE_NODE_NAME);
-- int timeout = message.getInt("timeout", 10 * 60); // 10 minutes
++ int timeout = message.getInt(TIMEOUT, 10 * 60); // 10 minutes
Replica.Type replicaType = Replica.Type.valueOf(message.getStr(ZkStateReader.REPLICA_TYPE, Replica.Type.NRT.name()).toUpperCase(Locale.ROOT));
boolean parallel = message.getBool("parallel", false);
- boolean waitForFinalState = message.getBool(WAIT_FOR_FINAL_STATE, false);
- if (StringUtils.isBlank(coreName)) {
- coreName = message.getStr(CoreAdminParams.PROPERTY_PREFIX + CoreAdminParams.NAME);
- }
-
- final String asyncId = message.getStr(ASYNC);
-
- DocCollection coll = clusterState.getCollection(collection);
- if (coll == null) {
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Collection: " + collection + " does not exist");
- }
- if (coll.getSlice(shard) == null) {
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
- "Collection: " + collection + " shard: " + shard + " does not exist");
- }
- ShardHandler shardHandler = ocmh.shardHandlerFactory.getShardHandler();
- boolean skipCreateReplicaInClusterState = message.getBool(SKIP_CREATE_REPLICA_IN_CLUSTER_STATE, false);
-
- AtomicReference<PolicyHelper.SessionWrapper> sessionWrapper = new AtomicReference<>();
- // Kind of unnecessary, but it does put the logic of whether to override maxShardsPerNode in one place.
- if (!skipCreateReplicaInClusterState) {
- if (CreateShardCmd.usePolicyFramework(coll, ocmh)) {
- if (node == null) {
- if(coll.getPolicyName() != null) message.getProperties().put(Policy.POLICY, coll.getPolicyName());
- node = Assign.identifyNodes(ocmh,
- clusterState,
- Collections.emptyList(),
- collection,
- message,
- Collections.singletonList(shard),
- replicaType == Replica.Type.NRT ? 0 : 1,
- replicaType == Replica.Type.TLOG ? 0 : 1,
- replicaType == Replica.Type.PULL ? 0 : 1
- ).get(0).node;
- sessionWrapper.set(PolicyHelper.getLastSessionWrapper(true));
- }
- } else {
- node = getNodesForNewReplicas(clusterState, collection, shard, 1, node,
- ocmh.overseer.getSolrCloudManager()).get(0).nodeName;// TODO: use replica type in this logic too
- }
- }
- log.info("Node Identified {} for creating new replica", node);
-
- if (!clusterState.liveNodesContain(node)) {
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Node: " + node + " is not live");
- }
- if (coreName == null) {
- coreName = Assign.buildCoreName(ocmh.overseer.getSolrCloudManager().getDistribStateManager(), coll, shard, replicaType);
- } else if (!skipCreateReplicaInClusterState) {
- //Validate that the core name is unique in that collection
- for (Slice slice : coll.getSlices()) {
- for (Replica replica : slice.getReplicas()) {
- String replicaCoreName = replica.getStr(CORE_NAME_PROP);
- if (coreName.equals(replicaCoreName)) {
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Another replica with the same core name already exists" +
- " for this collection");
- }
- }
- }
- }
ModifiableSolrParams params = new ModifiableSolrParams();
ZkStateReader zkStateReader = ocmh.zkStateReader;
@@@ -205,78 -253,4 +206,75 @@@
);
}
+ public static ZkNodeProps assignReplicaDetails(SolrCloudManager cloudManager, ClusterState clusterState,
- ZkNodeProps message, AtomicLong policyVersionAfter) throws IOException, InterruptedException {
++ ZkNodeProps message, AtomicReference<PolicyHelper.SessionWrapper> sessionWrapper) throws IOException, InterruptedException {
+ boolean skipCreateReplicaInClusterState = message.getBool(SKIP_CREATE_REPLICA_IN_CLUSTER_STATE, false);
+
+ String collection = message.getStr(COLLECTION_PROP);
+ String node = message.getStr(CoreAdminParams.NODE);
+ String shard = message.getStr(SHARD_ID_PROP);
+ String coreName = message.getStr(CoreAdminParams.NAME);
+ String coreNodeName = message.getStr(CoreAdminParams.CORE_NODE_NAME);
+ Replica.Type replicaType = Replica.Type.valueOf(message.getStr(ZkStateReader.REPLICA_TYPE, Replica.Type.NRT.name()).toUpperCase(Locale.ROOT));
+ if (StringUtils.isBlank(coreName)) {
+ coreName = message.getStr(CoreAdminParams.PROPERTY_PREFIX + CoreAdminParams.NAME);
+ }
+
+ DocCollection coll = clusterState.getCollection(collection);
+ if (coll == null) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Collection: " + collection + " does not exist");
+ }
+ if (coll.getSlice(shard) == null) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ "Collection: " + collection + " shard: " + shard + " does not exist");
+ }
+
- final Long policyVersionBefore = PolicyHelper.REF_VERSION.get();
+ // Kind of unnecessary, but it does put the logic of whether to override maxShardsPerNode in one place.
+ if (!skipCreateReplicaInClusterState) {
+ if (CloudUtil.usePolicyFramework(coll, cloudManager)) {
+ if (node == null) {
+ if(coll.getPolicyName() != null) message.getProperties().put(Policy.POLICY, coll.getPolicyName());
+ node = Assign.identifyNodes(cloudManager,
+ clusterState,
+ Collections.emptyList(),
+ collection,
+ message,
+ Collections.singletonList(shard),
+ replicaType == Replica.Type.NRT ? 0 : 1,
+ replicaType == Replica.Type.TLOG ? 0 : 1,
+ replicaType == Replica.Type.PULL ? 0 : 1
+ ).get(0).node;
- if (policyVersionBefore == null && PolicyHelper.REF_VERSION.get() != null) {
- policyVersionAfter.set(PolicyHelper.REF_VERSION.get());
- }
++ sessionWrapper.set(PolicyHelper.getLastSessionWrapper(true));
+ }
+ } else {
+ node = Assign.getNodesForNewReplicas(clusterState, collection, shard, 1, node,
+ cloudManager).get(0).nodeName;// TODO: use replica type in this logic too
+ }
+ }
+ log.info("Node Identified {} for creating new replica", node);
+
+ if (!clusterState.liveNodesContain(node)) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Node: " + node + " is not live");
+ }
+ if (coreName == null) {
+ coreName = Assign.buildSolrCoreName(cloudManager.getDistribStateManager(), coll, shard, replicaType);
+ } else if (!skipCreateReplicaInClusterState) {
+ //Validate that the core name is unique in that collection
+ for (Slice slice : coll.getSlices()) {
+ for (Replica replica : slice.getReplicas()) {
+ String replicaCoreName = replica.getStr(CORE_NAME_PROP);
+ if (coreName.equals(replicaCoreName)) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Another replica with the same core name already exists" +
+ " for this collection");
+ }
+ }
+ }
+ }
+ if (coreNodeName != null) {
+ message = message.plus(CoreAdminParams.CORE_NODE_NAME, coreNodeName);
+ }
+ message = message.plus(CoreAdminParams.NAME, coreName);
+ message = message.plus(CoreAdminParams.NODE, node);
+ return message;
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5ec9b64b/solr/core/src/java/org/apache/solr/cloud/Assign.java
----------------------------------------------------------------------
diff --cc solr/core/src/java/org/apache/solr/cloud/Assign.java
index 613469f,fd0738f..c746c94
--- a/solr/core/src/java/org/apache/solr/cloud/Assign.java
+++ b/solr/core/src/java/org/apache/solr/cloud/Assign.java
@@@ -293,15 -293,8 +293,8 @@@ public class Assign
} else {
if (message.getStr(CREATE_NODE_SET) == null)
nodeList = Collections.emptyList();// unless explicitly specified do not pass node list to Policy
- synchronized (PolicyHelper.class) {
- PolicyHelper.SESSION_REF.set(PolicyHelper.getPolicySessionRef(cloudManager));
- try {
- return getPositionsUsingPolicy(collectionName,
- shardNames, numNrtReplicas, numTlogReplicas, numPullReplicas, policyName, cloudManager, nodeList);
- } finally {
- PolicyHelper.SESSION_REF.remove();
- }
- }
+ return getPositionsUsingPolicy(collectionName,
- shardNames, numNrtReplicas, numTlogReplicas, numPullReplicas, policyName, ocmh.overseer.getSolrCloudManager(), nodeList);
++ shardNames, numNrtReplicas, numTlogReplicas, numPullReplicas, policyName, cloudManager, nodeList);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5ec9b64b/solr/core/src/java/org/apache/solr/cloud/CreateCollectionCmd.java
----------------------------------------------------------------------
diff --cc solr/core/src/java/org/apache/solr/cloud/CreateCollectionCmd.java
index 4d1953f,2c4f01e..2171c60
--- a/solr/core/src/java/org/apache/solr/cloud/CreateCollectionCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/CreateCollectionCmd.java
@@@ -27,17 -26,11 +27,18 @@@ import java.util.HashSet
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.NoSuchElementException;
import java.util.Properties;
import java.util.concurrent.TimeUnit;
++import java.util.concurrent.atomic.AtomicReference;
+import org.apache.solr.client.solrj.cloud.autoscaling.AlreadyExistsException;
+import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig;
+import org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager;
import org.apache.solr.client.solrj.cloud.autoscaling.Policy;
import org.apache.solr.client.solrj.cloud.autoscaling.PolicyHelper;
+import org.apache.solr.client.solrj.cloud.autoscaling.SolrCloudManager;
+import org.apache.solr.client.solrj.cloud.autoscaling.VersionedData;
import org.apache.solr.cloud.OverseerCollectionMessageHandler.Cmd;
import org.apache.solr.cloud.overseer.ClusterStateMutator;
import org.apache.solr.common.SolrException;
@@@ -111,15 -103,91 +112,16 @@@ public class CreateCollectionCmd implem
}
ocmh.validateConfigOrThrowSolrException(configName);
- PolicyHelper.SessionWrapper sessionWrapper = null;
++ AtomicReference<PolicyHelper.SessionWrapper> sessionWrapper = new AtomicReference<>();
try {
- // look at the replication factor and see if it matches reality
- // if it does not, find best nodes to create more cores
- int numTlogReplicas = message.getInt(TLOG_REPLICAS, 0);
- int numNrtReplicas = message.getInt(NRT_REPLICAS, message.getInt(REPLICATION_FACTOR, numTlogReplicas>0?0:1));
- int numPullReplicas = message.getInt(PULL_REPLICAS, 0);
- Map autoScalingJson = Utils.getJson(ocmh.zkStateReader.getZkClient(), SOLR_AUTOSCALING_CONF_PATH, true);
- String policy = message.getStr(Policy.POLICY);
- boolean usePolicyFramework = autoScalingJson.get(Policy.CLUSTER_POLICY) != null || policy != null;
-
- ShardHandler shardHandler = ocmh.shardHandlerFactory.getShardHandler();
final String async = message.getStr(ASYNC);
- Integer numSlices = message.getInt(NUM_SLICES, null);
- String router = message.getStr("router.name", DocRouter.DEFAULT_NAME);
+ List<String> nodeList = new ArrayList<>();
List<String> shardNames = new ArrayList<>();
- if(ImplicitDocRouter.NAME.equals(router)){
- ClusterStateMutator.getShardNames(shardNames, message.getStr("shards", null));
- numSlices = shardNames.size();
- } else {
- if (numSlices == null ) {
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, NUM_SLICES + " is a required param (when using CompositeId router).");
- }
- ClusterStateMutator.getShardNames(numSlices, shardNames);
- }
-
- int maxShardsPerNode = message.getInt(MAX_SHARDS_PER_NODE, 1);
- if (usePolicyFramework && message.getStr(MAX_SHARDS_PER_NODE) != null && maxShardsPerNode > 0) {
- throw new SolrException(ErrorCode.BAD_REQUEST, "'maxShardsPerNode>0' is not supported when autoScaling policies are used");
- }
- if (maxShardsPerNode == -1 || usePolicyFramework) maxShardsPerNode = Integer.MAX_VALUE;
- if (numNrtReplicas + numTlogReplicas <= 0) {
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, NRT_REPLICAS + " + " + TLOG_REPLICAS + " must be greater than 0");
- }
-
- if (numSlices <= 0) {
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, NUM_SLICES + " must be > 0");
- }
-
- // we need to look at every node and see how many cores it serves
- // add our new cores to existing nodes serving the least number of cores
- // but (for now) require that each core goes on a distinct node.
-
- final List<String> nodeList = Assign.getLiveOrLiveAndCreateNodeSetList(clusterState.getLiveNodes(), message, RANDOM);
- List<ReplicaPosition> replicaPositions;
- if (nodeList.isEmpty()) {
- log.warn("It is unusual to create a collection ("+collectionName+") without cores.");
-
- replicaPositions = new ArrayList<>();
- } else {
- int totalNumReplicas = numNrtReplicas + numTlogReplicas + numPullReplicas;
- if (totalNumReplicas > nodeList.size()) {
- log.warn("Specified number of replicas of "
- + totalNumReplicas
- + " on collection "
- + collectionName
- + " is higher than the number of Solr instances currently live or live and part of your " + CREATE_NODE_SET + "("
- + nodeList.size()
- + "). It's unusual to run two replica of the same slice on the same Solr-instance.");
- }
-
- int maxShardsAllowedToCreate = maxShardsPerNode == Integer.MAX_VALUE ?
- Integer.MAX_VALUE :
- maxShardsPerNode * nodeList.size();
- int requestedShardsToCreate = numSlices * totalNumReplicas;
- if (maxShardsAllowedToCreate < requestedShardsToCreate) {
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Cannot create collection " + collectionName + ". Value of "
- + MAX_SHARDS_PER_NODE + " is " + maxShardsPerNode
- + ", and the number of nodes currently live or live and part of your "+CREATE_NODE_SET+" is " + nodeList.size()
- + ". This allows a maximum of " + maxShardsAllowedToCreate
- + " to be created. Value of " + NUM_SLICES + " is " + numSlices
- + ", value of " + NRT_REPLICAS + " is " + numNrtReplicas
- + ", value of " + TLOG_REPLICAS + " is " + numTlogReplicas
- + " and value of " + PULL_REPLICAS + " is " + numPullReplicas
- + ". This requires " + requestedShardsToCreate
- + " shards to be created (higher than the allowed number)");
- }
- replicaPositions = Assign.identifyNodes(ocmh
- , clusterState, nodeList, collectionName, message, shardNames, numNrtReplicas, numTlogReplicas, numPullReplicas);
- sessionWrapper = PolicyHelper.getLastSessionWrapper(true);
- }
-
+ List<ReplicaPosition> replicaPositions = buildReplicaPositions(ocmh.cloudManager, clusterState, message,
- nodeList, shardNames);
++ nodeList, shardNames, sessionWrapper);
ZkStateReader zkStateReader = ocmh.zkStateReader;
boolean isLegacyCloud = Overseer.isLegacy(zkStateReader);
@@@ -254,90 -320,10 +256,93 @@@
} catch (Exception ex) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, null, ex);
} finally {
- PolicyHelper.clearFlagAndDecref(PolicyHelper.getPolicySessionRef(ocmh.overseer.getSolrCloudManager()));
- if(sessionWrapper != null) sessionWrapper.release();
++ if (sessionWrapper.get() != null) sessionWrapper.get().release();
}
}
+ public static List<ReplicaPosition> buildReplicaPositions(SolrCloudManager cloudManager, ClusterState clusterState,
- ZkNodeProps message, List<String> nodeList, List<String> shardNames) throws IOException, InterruptedException {
++ ZkNodeProps message,
++ List<String> nodeList, List<String> shardNames,
++ AtomicReference<PolicyHelper.SessionWrapper> sessionWrapper) throws IOException, InterruptedException {
+ final String collectionName = message.getStr(NAME);
+ // look at the replication factor and see if it matches reality
+ // if it does not, find best nodes to create more cores
+ int numTlogReplicas = message.getInt(TLOG_REPLICAS, 0);
+ int numNrtReplicas = message.getInt(NRT_REPLICAS, message.getInt(REPLICATION_FACTOR, numTlogReplicas>0?0:1));
+ int numPullReplicas = message.getInt(PULL_REPLICAS, 0);
+ AutoScalingConfig autoScalingConfig = cloudManager.getDistribStateManager().getAutoScalingConfig();
+ String policy = message.getStr(Policy.POLICY);
+ boolean usePolicyFramework = !autoScalingConfig.getPolicy().getClusterPolicy().isEmpty() || policy != null;
+
+ Integer numSlices = message.getInt(NUM_SLICES, null);
+ String router = message.getStr("router.name", DocRouter.DEFAULT_NAME);
+ if(ImplicitDocRouter.NAME.equals(router)){
+ ClusterStateMutator.getShardNames(shardNames, message.getStr("shards", null));
+ numSlices = shardNames.size();
+ } else {
+ if (numSlices == null ) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, NUM_SLICES + " is a required param (when using CompositeId router).");
+ }
+ if (numSlices <= 0) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, NUM_SLICES + " must be > 0");
+ }
+ ClusterStateMutator.getShardNames(numSlices, shardNames);
+ }
+
+ int maxShardsPerNode = message.getInt(MAX_SHARDS_PER_NODE, 1);
+ if (usePolicyFramework && message.getStr(MAX_SHARDS_PER_NODE) != null && maxShardsPerNode > 0) {
+ throw new SolrException(ErrorCode.BAD_REQUEST, "'maxShardsPerNode>0' is not supported when autoScaling policies are used");
+ }
+ if (maxShardsPerNode == -1 || usePolicyFramework) maxShardsPerNode = Integer.MAX_VALUE;
+ if (numNrtReplicas + numTlogReplicas <= 0) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, NRT_REPLICAS + " + " + TLOG_REPLICAS + " must be greater than 0");
+ }
+
+ // we need to look at every node and see how many cores it serves
+ // add our new cores to existing nodes serving the least number of cores
+ // but (for now) require that each core goes on a distinct node.
+
+ List<ReplicaPosition> replicaPositions;
+ nodeList.addAll(Assign.getLiveOrLiveAndCreateNodeSetList(clusterState.getLiveNodes(), message, RANDOM));
+ if (nodeList.isEmpty()) {
+ log.warn("It is unusual to create a collection ("+collectionName+") without cores.");
+
+ replicaPositions = new ArrayList<>();
+ } else {
+ int totalNumReplicas = numNrtReplicas + numTlogReplicas + numPullReplicas;
+ if (totalNumReplicas > nodeList.size()) {
+ log.warn("Specified number of replicas of "
+ + totalNumReplicas
+ + " on collection "
+ + collectionName
+ + " is higher than the number of Solr instances currently live or live and part of your " + CREATE_NODE_SET + "("
+ + nodeList.size()
+ + "). It's unusual to run two replica of the same slice on the same Solr-instance.");
+ }
+
+ int maxShardsAllowedToCreate = maxShardsPerNode == Integer.MAX_VALUE ?
+ Integer.MAX_VALUE :
+ maxShardsPerNode * nodeList.size();
+ int requestedShardsToCreate = numSlices * totalNumReplicas;
+ if (maxShardsAllowedToCreate < requestedShardsToCreate) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Cannot create collection " + collectionName + ". Value of "
+ + MAX_SHARDS_PER_NODE + " is " + maxShardsPerNode
+ + ", and the number of nodes currently live or live and part of your "+CREATE_NODE_SET+" is " + nodeList.size()
+ + ". This allows a maximum of " + maxShardsAllowedToCreate
+ + " to be created. Value of " + NUM_SLICES + " is " + numSlices
+ + ", value of " + NRT_REPLICAS + " is " + numNrtReplicas
+ + ", value of " + TLOG_REPLICAS + " is " + numTlogReplicas
+ + " and value of " + PULL_REPLICAS + " is " + numPullReplicas
+ + ". This requires " + requestedShardsToCreate
+ + " shards to be created (higher than the allowed number)");
+ }
+ replicaPositions = Assign.identifyNodes(cloudManager
+ , clusterState, nodeList, collectionName, message, shardNames, numNrtReplicas, numTlogReplicas, numPullReplicas);
++ sessionWrapper.set(PolicyHelper.getLastSessionWrapper(true));
+ }
+ return replicaPositions;
+ }
+
String getConfigName(String coll, ZkNodeProps message) throws KeeperException, InterruptedException {
String configName = message.getStr(COLL_CONF);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5ec9b64b/solr/core/src/java/org/apache/solr/cloud/CreateShardCmd.java
----------------------------------------------------------------------
diff --cc solr/core/src/java/org/apache/solr/cloud/CreateShardCmd.java
index 4ccf993,18b0b63..c6afdcc
--- a/solr/core/src/java/org/apache/solr/cloud/CreateShardCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/CreateShardCmd.java
@@@ -25,11 -25,11 +25,12 @@@ import java.util.HashMap
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
++import java.util.concurrent.atomic.AtomicReference;
import com.google.common.collect.ImmutableMap;
-import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig;
import org.apache.solr.client.solrj.cloud.autoscaling.Policy;
import org.apache.solr.client.solrj.cloud.autoscaling.PolicyHelper;
+import org.apache.solr.client.solrj.cloud.autoscaling.SolrCloudManager;
import org.apache.solr.cloud.OverseerCollectionMessageHandler.Cmd;
import org.apache.solr.common.SolrCloseableLatch;
import org.apache.solr.common.SolrException;
@@@ -76,11 -76,50 +77,12 @@@ public class CreateShardCmd implements
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "'collection' and 'shard' are required parameters");
DocCollection collection = clusterState.getCollection(collectionName);
- int numNrtReplicas = message.getInt(NRT_REPLICAS, message.getInt(REPLICATION_FACTOR, collection.getInt(NRT_REPLICAS, collection.getInt(REPLICATION_FACTOR, 1))));
- int numPullReplicas = message.getInt(PULL_REPLICAS, collection.getInt(PULL_REPLICAS, 0));
- int numTlogReplicas = message.getInt(TLOG_REPLICAS, collection.getInt(TLOG_REPLICAS, 0));
- int totalReplicas = numNrtReplicas + numPullReplicas + numTlogReplicas;
-
- if (numNrtReplicas + numTlogReplicas <= 0) {
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, NRT_REPLICAS + " + " + TLOG_REPLICAS + " must be greater than 0");
- }
-
- Object createNodeSetStr = message.get(OverseerCollectionMessageHandler.CREATE_NODE_SET);
ZkStateReader zkStateReader = ocmh.zkStateReader;
- PolicyHelper.SessionWrapper sessionWrapper = null;
- boolean usePolicyFramework = usePolicyFramework(collection,ocmh);
- List<ReplicaPosition> positions = null;
++ AtomicReference<PolicyHelper.SessionWrapper> sessionWrapper = new AtomicReference<>();
SolrCloseableLatch countDownLatch;
try {
- List<ReplicaPosition> positions = buildReplicaPositions(ocmh.cloudManager, clusterState, collectionName, message);
- if (usePolicyFramework) {
- if (collection.getPolicyName() != null) message.getProperties().put(Policy.POLICY, collection.getPolicyName());
- positions = Assign.identifyNodes(ocmh,
- clusterState,
- Assign.getLiveOrLiveAndCreateNodeSetList(clusterState.getLiveNodes(), message, RANDOM),
- collectionName,
- message,
- Collections.singletonList(sliceName),
- numNrtReplicas,
- numTlogReplicas,
- numPullReplicas);
- sessionWrapper = PolicyHelper.getLastSessionWrapper(true);
- } else {
- List<Assign.ReplicaCount> sortedNodeList = getNodesForNewReplicas(clusterState, collectionName, sliceName, totalReplicas,
- createNodeSetStr, ocmh.overseer.getSolrCloudManager());
- int i = 0;
- positions = new ArrayList<>();
- for (Map.Entry<Replica.Type, Integer> e : ImmutableMap.of(Replica.Type.NRT, numNrtReplicas,
- Replica.Type.TLOG, numTlogReplicas,
- Replica.Type.PULL, numPullReplicas
- ).entrySet()) {
- for (int j = 0; j < e.getValue(); j++) {
- positions.add(new ReplicaPosition(sliceName, j + 1, e.getKey(), sortedNodeList.get(i % sortedNodeList.size()).nodeName));
- i++;
- }
- }
- }
++ List<ReplicaPosition> positions = buildReplicaPositions(ocmh.cloudManager, clusterState, collectionName, message, sessionWrapper);
Overseer.getStateUpdateQueue(zkStateReader.getZkClient()).offer(Utils.toJSON(message));
// wait for a while until we see the shard
ocmh.waitForNewShard(collectionName, sliceName);
@@@ -127,7 -166,7 +129,7 @@@
});
}
} finally {
- PolicyHelper.clearFlagAndDecref(PolicyHelper.getPolicySessionRef(ocmh.overseer.getSolrCloudManager()));
- if(sessionWrapper != null) sessionWrapper.release();
++ if (sessionWrapper.get() != null) sessionWrapper.get().release();
}
log.debug("Waiting for create shard action to complete");
@@@ -138,51 -177,9 +140,52 @@@
}
- static boolean usePolicyFramework(DocCollection collection, OverseerCollectionMessageHandler ocmh)
- throws IOException, InterruptedException {
- AutoScalingConfig autoScalingConfig = ocmh.overseer.getSolrCloudManager().getDistribStateManager().getAutoScalingConfig();
- return !autoScalingConfig.getPolicy().getClusterPolicy().isEmpty() || collection.getPolicyName() != null;
+ public static List<ReplicaPosition> buildReplicaPositions(SolrCloudManager cloudManager, ClusterState clusterState,
- String collectionName, ZkNodeProps message) throws IOException, InterruptedException {
++ String collectionName, ZkNodeProps message, AtomicReference< PolicyHelper.SessionWrapper> sessionWrapper) throws IOException, InterruptedException {
+ String sliceName = message.getStr(SHARD_ID_PROP);
+ DocCollection collection = clusterState.getCollection(collectionName);
+
+ int numNrtReplicas = message.getInt(NRT_REPLICAS, message.getInt(REPLICATION_FACTOR, collection.getInt(NRT_REPLICAS, collection.getInt(REPLICATION_FACTOR, 1))));
+ int numPullReplicas = message.getInt(PULL_REPLICAS, collection.getInt(PULL_REPLICAS, 0));
+ int numTlogReplicas = message.getInt(TLOG_REPLICAS, collection.getInt(TLOG_REPLICAS, 0));
+ int totalReplicas = numNrtReplicas + numPullReplicas + numTlogReplicas;
+
+ if (numNrtReplicas + numTlogReplicas <= 0) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, NRT_REPLICAS + " + " + TLOG_REPLICAS + " must be greater than 0");
+ }
+
+ Object createNodeSetStr = message.get(OverseerCollectionMessageHandler.CREATE_NODE_SET);
+
+ boolean usePolicyFramework = CloudUtil.usePolicyFramework(collection, cloudManager);
+ List<ReplicaPosition> positions;
+ if (usePolicyFramework) {
+ if (collection.getPolicyName() != null) message.getProperties().put(Policy.POLICY, collection.getPolicyName());
+ positions = Assign.identifyNodes(cloudManager,
+ clusterState,
+ Assign.getLiveOrLiveAndCreateNodeSetList(clusterState.getLiveNodes(), message, RANDOM),
+ collection.getName(),
+ message,
+ Collections.singletonList(sliceName),
+ numNrtReplicas,
+ numTlogReplicas,
+ numPullReplicas);
++ sessionWrapper.set(PolicyHelper.getLastSessionWrapper(true));
+ } else {
+ List<Assign.ReplicaCount> sortedNodeList = getNodesForNewReplicas(clusterState, collection.getName(), sliceName, totalReplicas,
+ createNodeSetStr, cloudManager);
+ int i = 0;
+ positions = new ArrayList<>();
+ for (Map.Entry<Replica.Type, Integer> e : ImmutableMap.of(Replica.Type.NRT, numNrtReplicas,
+ Replica.Type.TLOG, numTlogReplicas,
+ Replica.Type.PULL, numPullReplicas
+ ).entrySet()) {
+ for (int j = 0; j < e.getValue(); j++) {
+ positions.add(new ReplicaPosition(sliceName, j + 1, e.getKey(), sortedNodeList.get(i % sortedNodeList.size()).nodeName));
+ i++;
+ }
+ }
+ }
+ return positions;
}
+
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5ec9b64b/solr/core/src/java/org/apache/solr/cloud/MoveReplicaCmd.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5ec9b64b/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5ec9b64b/solr/core/src/java/org/apache/solr/cloud/RestoreCmd.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5ec9b64b/solr/core/src/java/org/apache/solr/cloud/SplitShardCmd.java
----------------------------------------------------------------------
diff --cc solr/core/src/java/org/apache/solr/cloud/SplitShardCmd.java
index a3970a2,8f65255..9732616
--- a/solr/core/src/java/org/apache/solr/cloud/SplitShardCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/SplitShardCmd.java
@@@ -87,13 -86,44 +87,15 @@@ public class SplitShardCmd implements C
log.info("Split shard invoked");
ZkStateReader zkStateReader = ocmh.zkStateReader;
zkStateReader.forceUpdateCollection(collectionName);
+ AtomicReference<String> slice = new AtomicReference<>();
+ slice.set(message.getStr(ZkStateReader.SHARD_ID_PROP));
String splitKey = message.getStr("split.key");
- ShardHandler shardHandler = ocmh.shardHandlerFactory.getShardHandler();
-
DocCollection collection = clusterState.getCollection(collectionName);
- DocRouter router = collection.getRouter() != null ? collection.getRouter() : DocRouter.DEFAULT;
- PolicyHelper.SessionWrapper sessionWrapper = null;
-
-
- Slice parentSlice;
- if (slice == null) {
- if (router instanceof CompositeIdRouter) {
- Collection<Slice> searchSlices = router.getSearchSlicesSingle(splitKey, new ModifiableSolrParams(), collection);
- if (searchSlices.isEmpty()) {
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Unable to find an active shard for split.key: " + splitKey);
- }
- if (searchSlices.size() > 1) {
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
- "Splitting a split.key: " + splitKey + " which spans multiple shards is not supported");
- }
- parentSlice = searchSlices.iterator().next();
- slice = parentSlice.getName();
- log.info("Split by route.key: {}, parent shard is: {} ", splitKey, slice);
- } else {
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
- "Split by route key can only be used with CompositeIdRouter or subclass. Found router: "
- + router.getClass().getName());
- }
- } else {
- parentSlice = collection.getSlice(slice);
- }
++ PolicyHelper.SessionWrapper sessionWrapper = null;
+
- if (parentSlice == null) {
- // no chance of the collection being null because ClusterState#getCollection(String) would have thrown
- // an exception already
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No shard with the specified name exists: " + slice);
- }
+ Slice parentSlice = getParentSlice(clusterState, collectionName, slice, splitKey);
// find the leader for the shard
Replica parentShardLeader = null;
@@@ -421,119 -516,7 +424,119 @@@
log.error("Error executing split operation for collection: " + collectionName + " parent shard: " + slice, e);
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, null, e);
} finally {
- PolicyHelper.clearFlagAndDecref(PolicyHelper.getPolicySessionRef(ocmh.overseer.getSolrCloudManager()));
+ if (sessionWrapper != null) sessionWrapper.release();
}
}
+
+ public static Slice getParentSlice(ClusterState clusterState, String collectionName, AtomicReference<String> slice, String splitKey) {
+ DocCollection collection = clusterState.getCollection(collectionName);
+ DocRouter router = collection.getRouter() != null ? collection.getRouter() : DocRouter.DEFAULT;
+
+ Slice parentSlice;
+
+ if (slice.get() == null) {
+ if (router instanceof CompositeIdRouter) {
+ Collection<Slice> searchSlices = router.getSearchSlicesSingle(splitKey, new ModifiableSolrParams(), collection);
+ if (searchSlices.isEmpty()) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Unable to find an active shard for split.key: " + splitKey);
+ }
+ if (searchSlices.size() > 1) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ "Splitting a split.key: " + splitKey + " which spans multiple shards is not supported");
+ }
+ parentSlice = searchSlices.iterator().next();
+ slice.set(parentSlice.getName());
+ log.info("Split by route.key: {}, parent shard is: {} ", splitKey, slice);
+ } else {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ "Split by route key can only be used with CompositeIdRouter or subclass. Found router: "
+ + router.getClass().getName());
+ }
+ } else {
+ parentSlice = collection.getSlice(slice.get());
+ }
+
+ if (parentSlice == null) {
+ // no chance of the collection being null because ClusterState#getCollection(String) would have thrown
+ // an exception already
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No shard with the specified name exists: " + slice);
+ }
+ return parentSlice;
+ }
+
+ public static String fillRanges(SolrCloudManager cloudManager, ZkNodeProps message, DocCollection collection, Slice parentSlice,
+ List<DocRouter.Range> subRanges, List<String> subSlices, List<String> subShardNames) {
+ String splitKey = message.getStr("split.key");
+ DocRouter.Range range = parentSlice.getRange();
+ if (range == null) {
+ range = new PlainIdRouter().fullRange();
+ }
+ DocRouter router = collection.getRouter() != null ? collection.getRouter() : DocRouter.DEFAULT;
+
+ String rangesStr = message.getStr(CoreAdminParams.RANGES);
+ if (rangesStr != null) {
+ String[] ranges = rangesStr.split(",");
+ if (ranges.length == 0 || ranges.length == 1) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "There must be at least two ranges specified to split a shard");
+ } else {
+ for (int i = 0; i < ranges.length; i++) {
+ String r = ranges[i];
+ try {
+ subRanges.add(DocRouter.DEFAULT.fromString(r));
+ } catch (Exception e) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Exception in parsing hexadecimal hash range: " + r, e);
+ }
+ if (!subRanges.get(i).isSubsetOf(range)) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ "Specified hash range: " + r + " is not a subset of parent shard's range: " + range.toString());
+ }
+ }
+ List<DocRouter.Range> temp = new ArrayList<>(subRanges); // copy to preserve original order
+ Collections.sort(temp);
+ if (!range.equals(new DocRouter.Range(temp.get(0).min, temp.get(temp.size() - 1).max))) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ "Specified hash ranges: " + rangesStr + " do not cover the entire range of parent shard: " + range);
+ }
+ for (int i = 1; i < temp.size(); i++) {
+ if (temp.get(i - 1).max + 1 != temp.get(i).min) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Specified hash ranges: " + rangesStr
+ + " either overlap with each other or " + "do not cover the entire range of parent shard: " + range);
+ }
+ }
+ }
+ } else if (splitKey != null) {
+ if (router instanceof CompositeIdRouter) {
+ CompositeIdRouter compositeIdRouter = (CompositeIdRouter) router;
+ List<DocRouter.Range> tmpSubRanges = compositeIdRouter.partitionRangeByKey(splitKey, range);
+ if (tmpSubRanges.size() == 1) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "The split.key: " + splitKey
+ + " has a hash range that is exactly equal to hash range of shard: " + parentSlice.getName());
+ }
+ for (DocRouter.Range subRange : tmpSubRanges) {
+ if (subRange.min == subRange.max) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "The split.key: " + splitKey + " must be a compositeId");
+ }
+ }
+ subRanges.addAll(tmpSubRanges);
+ log.info("Partitioning parent shard " + parentSlice.getName() + " range: " + parentSlice.getRange() + " yields: " + subRanges);
+ rangesStr = "";
+ for (int i = 0; i < subRanges.size(); i++) {
+ DocRouter.Range subRange = subRanges.get(i);
+ rangesStr += subRange.toString();
+ if (i < subRanges.size() - 1) rangesStr += ',';
+ }
+ }
+ } else {
+ // todo: fixed to two partitions?
+ subRanges.addAll(router.partitionRange(2, range));
+ }
+
+ for (int i = 0; i < subRanges.size(); i++) {
+ String subSlice = parentSlice.getName() + "_" + i;
+ subSlices.add(subSlice);
+ String subShardName = Assign.buildSolrCoreName(cloudManager.getDistribStateManager(), collection, subSlice, Replica.Type.NRT);
+ subShardNames.add(subShardName);
+ }
+ return rangesStr;
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5ec9b64b/solr/core/src/java/org/apache/solr/cloud/autoscaling/ComputePlanAction.java
----------------------------------------------------------------------
diff --cc solr/core/src/java/org/apache/solr/cloud/autoscaling/ComputePlanAction.java
index c770cac,ccffea7..f404721
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/ComputePlanAction.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/ComputePlanAction.java
@@@ -29,13 -28,13 +28,14 @@@ import org.apache.solr.client.solrj.Sol
import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig;
import org.apache.solr.client.solrj.cloud.autoscaling.NoneSuggester;
import org.apache.solr.client.solrj.cloud.autoscaling.Policy;
+ import org.apache.solr.client.solrj.cloud.autoscaling.PolicyHelper;
import org.apache.solr.client.solrj.cloud.autoscaling.ReplicaInfo;
- import org.apache.solr.common.SolrException;
- import org.apache.solr.common.params.AutoScalingParams;
import org.apache.solr.client.solrj.cloud.autoscaling.SolrCloudManager;
import org.apache.solr.client.solrj.cloud.autoscaling.Suggester;
+ import org.apache.solr.common.SolrException;
+ import org.apache.solr.common.params.AutoScalingParams;
import org.apache.solr.common.params.CollectionParams;
+import org.apache.solr.common.util.Pair;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@@ -58,25 -57,30 +58,29 @@@ public class ComputePlanAction extends
if (autoScalingConf.isEmpty()) {
throw new Exception("Action: " + getName() + " executed but no policy is configured");
}
- Policy policy = autoScalingConf.getPolicy();
- Policy.Session session = policy.createSession(cloudManager);
- Suggester suggester = getSuggester(session, event, cloudManager);
- log.debug("{} Created suggester with node(s): {}, event {}", event.eventType, event.getProperty(TriggerEvent.NODE_NAMES), event.id);
- while (true) {
- long start = cloudManager.getTimeSource().getTime();
- SolrRequest operation = suggester.getSuggestion();
- if (operation == null) break;
- log.info("Computed Plan ({} ms): {}",
- TimeUnit.MILLISECONDS.convert(cloudManager.getTimeSource().getTime() - start, TimeUnit.NANOSECONDS), operation.getParams());
- Map<String, Object> props = context.getProperties();
- props.compute("operations", (k, v) -> {
- List<SolrRequest> operations = (List<SolrRequest>) v;
- if (operations == null) operations = new ArrayList<>();
- operations.add(operation);
- return operations;
- });
- session = suggester.getSession();
- suggester = getSuggester(session, event, cloudManager);
-
+ // Policy.Session session = cloudManager.getDistribStateManager().getAutoScalingConfig().getPolicy().createSession(cloudManager);
+ // return new PolicyHelper.SessionWrapper(session, null);
+ PolicyHelper.SessionWrapper sessionWrapper = PolicyHelper.getSession(cloudManager);
+ Policy.Session session = sessionWrapper.get();
+ // Policy policy = autoScalingConf.getPolicy();
+ try {
+ Suggester suggester = getSuggester(session, event, cloudManager);
+ while (true) {
+ SolrRequest operation = suggester.getSuggestion();
+ if (operation == null) break;
+ log.info("Computed Plan: {}", operation.getParams());
+ Map<String, Object> props = context.getProperties();
+ props.compute("operations", (k, v) -> {
+ List<SolrRequest> operations = (List<SolrRequest>) v;
+ if (operations == null) operations = new ArrayList<>();
+ operations.add(operation);
+ return operations;
+ });
+ session = suggester.getSession();
+ suggester = getSuggester(session, event, cloudManager);
+ }
+ } finally {
+ releasePolicySession(sessionWrapper, session);
}
} catch (Exception e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5ec9b64b/solr/core/src/java/org/apache/solr/cloud/autoscaling/ExecutePlanAction.java
----------------------------------------------------------------------
diff --cc solr/core/src/java/org/apache/solr/cloud/autoscaling/ExecutePlanAction.java
index f3965d6,ebe0660..841856e
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/ExecutePlanAction.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/ExecutePlanAction.java
@@@ -82,8 -82,8 +82,8 @@@ public class ExecutePlanAction extends
throw new IOException("" + asyncResponse.getResponse().get("error"));
}
asyncId = (String)asyncResponse.getResponse().get("requestid");
- CollectionAdminRequest.RequestStatusResponse statusResponse = waitForTaskToFinish(dataProvider, asyncId,
+ CollectionAdminRequest.RequestStatusResponse statusResponse = waitForTaskToFinish(cloudManager, asyncId,
- DEFAULT_TASK_TIMEOUT, TimeUnit.SECONDS);
+ DEFAULT_TASK_TIMEOUT_SECONDS, TimeUnit.SECONDS);
if (statusResponse != null) {
RequestStatusState state = statusResponse.getRequestStatus();
if (state == RequestStatusState.COMPLETED || state == RequestStatusState.FAILED || state == RequestStatusState.NOT_FOUND) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5ec9b64b/solr/core/src/java/org/apache/solr/cloud/autoscaling/ScheduledTriggers.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5ec9b64b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5ec9b64b/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerIntegrationTest.java
----------------------------------------------------------------------