You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ho...@apache.org on 2017/01/16 00:11:07 UTC
[49/50] [abbrv] lucene-solr:jira/solr-5944: SOLR-5944: Merge branch
'master' into jira/solr-5944
SOLR-5944: Merge branch 'master' into jira/solr-5944
merged master into branch to pick up some bug fixes that we're waiting for...
* resolved conflicts / new-failures in TestRecovery
- simple import collisions
- divergent test changes resulted in a metrics assertion not expecting as many docs as it should (SOLR-9950)
* fixed (new) SkipExistingDocumentsProcessorFactory to use newer syntax for getInputDocumentFromTlog (SOLR-9918)
* SOLR-9941 allows TestRecovery.testLogReplayWithInPlaceUpdatesAndDeletes to start passing
* eliminated workaround for SOLR-9934 in TestInPlaceUpdatesStandalone now that the mainline fix is on this branch
Conflicts:
solr/core/src/test/org/apache/solr/search/TestRecovery.java
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/027a92a4
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/027a92a4
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/027a92a4
Branch: refs/heads/jira/solr-5944
Commit: 027a92a400c60f23a81e9bf7ff9c32009583fb90
Parents: bb5dfb6 9aa78dc
Author: Chris Hostetter <ho...@apache.org>
Authored: Sun Jan 15 14:18:29 2017 -0700
Committer: Chris Hostetter <ho...@apache.org>
Committed: Sun Jan 15 14:18:29 2017 -0700
----------------------------------------------------------------------
build.xml | 42 +-
.../dot.settings/org.eclipse.jdt.core.prefs | 1 +
dev-tools/idea/lucene/suggest/suggest.iml | 2 -
.../idea/solr/contrib/analytics/analytics.iml | 1 +
.../lucene/analysis/common/pom.xml.template | 36 +-
.../maven/lucene/analysis/icu/pom.xml.template | 42 +-
.../lucene/analysis/kuromoji/pom.xml.template | 38 +-
.../lucene/analysis/morfologik/pom.xml.template | 38 +-
.../lucene/analysis/phonetic/pom.xml.template | 38 +-
.../maven/lucene/analysis/pom.xml.template | 36 +-
.../lucene/analysis/smartcn/pom.xml.template | 36 +-
.../lucene/analysis/stempel/pom.xml.template | 36 +-
.../maven/lucene/analysis/uima/pom.xml.template | 36 +-
.../lucene/backward-codecs/pom.xml.template | 36 +-
.../maven/lucene/benchmark/pom.xml.template | 36 +-
.../lucene/classification/pom.xml.template | 36 +-
dev-tools/maven/lucene/codecs/pom.xml.template | 36 +-
.../lucene/codecs/src/java/pom.xml.template | 36 +-
.../lucene/codecs/src/test/pom.xml.template | 36 +-
dev-tools/maven/lucene/core/pom.xml.template | 36 +-
.../maven/lucene/core/src/java/pom.xml.template | 36 +-
.../maven/lucene/core/src/test/pom.xml.template | 36 +-
dev-tools/maven/lucene/demo/pom.xml.template | 36 +-
.../maven/lucene/expressions/pom.xml.template | 36 +-
dev-tools/maven/lucene/facet/pom.xml.template | 36 +-
.../maven/lucene/grouping/pom.xml.template | 36 +-
.../maven/lucene/highlighter/pom.xml.template | 36 +-
dev-tools/maven/lucene/join/pom.xml.template | 36 +-
dev-tools/maven/lucene/memory/pom.xml.template | 36 +-
dev-tools/maven/lucene/misc/pom.xml.template | 36 +-
dev-tools/maven/lucene/pom.xml.template | 36 +-
dev-tools/maven/lucene/queries/pom.xml.template | 36 +-
.../maven/lucene/queryparser/pom.xml.template | 36 +-
.../maven/lucene/replicator/pom.xml.template | 38 +-
dev-tools/maven/lucene/sandbox/pom.xml.template | 36 +-
.../lucene/spatial-extras/pom.xml.template | 36 +-
dev-tools/maven/lucene/spatial/pom.xml.template | 38 +-
.../maven/lucene/spatial3d/pom.xml.template | 36 +-
dev-tools/maven/lucene/suggest/pom.xml.template | 36 +-
.../lucene/test-framework/pom.xml.template | 36 +-
dev-tools/maven/pom.xml.template | 36 +-
.../contrib/analysis-extras/pom.xml.template | 36 +-
.../solr/contrib/analytics/pom.xml.template | 36 +-
.../solr/contrib/clustering/pom.xml.template | 36 +-
.../dataimporthandler-extras/pom.xml.template | 36 +-
.../contrib/dataimporthandler/pom.xml.template | 36 +-
.../solr/contrib/extraction/pom.xml.template | 36 +-
.../maven/solr/contrib/langid/pom.xml.template | 36 +-
.../maven/solr/contrib/ltr/pom.xml.template | 36 +-
.../solr/contrib/map-reduce/pom.xml.template | 36 +-
.../contrib/morphlines-cell/pom.xml.template | 36 +-
.../contrib/morphlines-core/pom.xml.template | 36 +-
dev-tools/maven/solr/contrib/pom.xml.template | 36 +-
.../maven/solr/contrib/uima/pom.xml.template | 36 +-
.../solr/contrib/velocity/pom.xml.template | 36 +-
dev-tools/maven/solr/core/pom.xml.template | 36 +-
.../maven/solr/core/src/java/pom.xml.template | 36 +-
.../maven/solr/core/src/test/pom.xml.template | 36 +-
dev-tools/maven/solr/pom.xml.template | 36 +-
dev-tools/maven/solr/solrj/pom.xml.template | 36 +-
.../maven/solr/solrj/src/java/pom.xml.template | 36 +-
.../maven/solr/solrj/src/test/pom.xml.template | 36 +-
.../maven/solr/test-framework/pom.xml.template | 36 +-
lucene/CHANGES.txt | 115 +-
.../lucene/analysis/custom/CustomAnalyzer.java | 2 +-
.../miscellaneous/FixBrokenOffsetsFilter.java | 78 +
.../FixBrokenOffsetsFilterFactory.java | 39 +
.../analysis/synonym/FlattenGraphFilter.java | 417 ++++
.../synonym/FlattenGraphFilterFactory.java | 44 +
.../lucene/analysis/synonym/SynonymFilter.java | 4 +
.../analysis/synonym/SynonymFilterFactory.java | 4 +
.../analysis/synonym/SynonymGraphFilter.java | 586 ++++++
.../synonym/SynonymGraphFilterFactory.java | 204 ++
.../lucene/analysis/synonym/SynonymMap.java | 7 +-
.../lucene/analysis/util/CharTokenizer.java | 6 +-
...ache.lucene.analysis.util.TokenFilterFactory | 3 +
.../lucene/analysis/core/TestRandomChains.java | 177 +-
.../analysis/custom/TestCustomAnalyzer.java | 21 +
.../apache/lucene/analysis/custom/mapping1.txt | 1 +
.../apache/lucene/analysis/custom/mapping2.txt | 1 +
.../TestFixBrokenOffsetsFilter.java | 50 +
.../miscellaneous/TestWordDelimiterFilter.java | 56 +-
.../synonym/TestFlattenGraphFilter.java | 284 +++
.../synonym/TestSynonymGraphFilter.java | 1956 ++++++++++++++++++
.../apache/lucene/index/FixBrokenOffsets.java | 125 ++
.../java/org/apache/lucene/index/package.html | 27 +
.../lucene/index/TestFixBrokenOffsets.java | 115 +
.../lucene/index/index.630.brokenoffsets.zip | Bin 0 -> 3203 bytes
lucene/common-build.xml | 2 +-
.../lucene/analysis/TokenStreamToAutomaton.java | 11 +-
.../PackedTokenAttributeImpl.java | 9 +-
.../apache/lucene/codecs/DocValuesConsumer.java | 10 +-
.../org/apache/lucene/codecs/NormsConsumer.java | 2 +-
.../lucene/codecs/StoredFieldsWriter.java | 2 +-
.../apache/lucene/codecs/TermVectorsWriter.java | 2 +-
.../CompressingStoredFieldsWriter.java | 68 +-
.../lucene50/Lucene50StoredFieldsFormat.java | 2 +-
.../codecs/lucene60/Lucene60PointsWriter.java | 11 +-
.../lucene70/Lucene70DocValuesConsumer.java | 163 +-
.../lucene70/Lucene70DocValuesFormat.java | 5 +-
.../lucene70/Lucene70DocValuesProducer.java | 220 +-
.../org/apache/lucene/document/DoublePoint.java | 30 +-
.../org/apache/lucene/document/FloatPoint.java | 30 +-
.../apache/lucene/index/AutomatonTermsEnum.java | 3 +
.../lucene/index/BinaryDocValuesWriter.java | 39 +-
.../org/apache/lucene/index/CheckIndex.java | 29 +-
.../lucene/index/DefaultIndexingChain.java | 136 +-
.../org/apache/lucene/index/DocConsumer.java | 2 +-
.../org/apache/lucene/index/DocIDMerger.java | 211 +-
.../apache/lucene/index/DocValuesWriter.java | 4 +-
.../lucene/index/DocumentsWriterPerThread.java | 31 +-
.../lucene/index/FreqProxTermsWriter.java | 8 +-
.../org/apache/lucene/index/IndexWriter.java | 8 +-
.../lucene/index/MappingMultiPostingsEnum.java | 2 +-
.../org/apache/lucene/index/MergeState.java | 8 +-
.../apache/lucene/index/NormValuesWriter.java | 17 +-
.../lucene/index/NumericDocValuesWriter.java | 51 +-
.../apache/lucene/index/PointValuesWriter.java | 96 +-
.../apache/lucene/index/PrefixCodedTerms.java | 4 +-
.../org/apache/lucene/index/SegmentInfos.java | 7 +-
.../apache/lucene/index/SortedDocValues.java | 22 +
.../lucene/index/SortedDocValuesWriter.java | 76 +-
.../index/SortedNumericDocValuesWriter.java | 64 +-
.../apache/lucene/index/SortedSetDocValues.java | 22 +
.../lucene/index/SortedSetDocValuesWriter.java | 98 +-
.../java/org/apache/lucene/index/Sorter.java | 57 +-
.../apache/lucene/index/SortingLeafReader.java | 21 +-
.../index/SortingStoredFieldsConsumer.java | 206 ++
.../index/SortingTermVectorsConsumer.java | 181 ++
.../lucene/index/StoredFieldsConsumer.java | 85 +
.../lucene/index/TermVectorsConsumer.java | 7 +-
.../java/org/apache/lucene/index/TermsHash.java | 4 +-
.../TrackingTmpOutputDirectoryWrapper.java | 53 +
.../org/apache/lucene/search/DoubleValues.java | 38 +
.../lucene/search/DoubleValuesSource.java | 340 +++
.../org/apache/lucene/search/GraphQuery.java | 136 ++
.../lucene/search/LeafFieldComparator.java | 2 +-
.../org/apache/lucene/search/LongValues.java | 38 +
.../apache/lucene/search/LongValuesSource.java | 244 +++
.../lucene/search/MultiCollectorManager.java | 107 +
.../lucene/search/SimpleFieldComparator.java | 2 +-
.../apache/lucene/search/TermInSetQuery.java | 369 ++++
.../search/UsageTrackingQueryCachingPolicy.java | 5 +-
.../org/apache/lucene/store/MMapDirectory.java | 126 +-
.../java/org/apache/lucene/util/LongValues.java | 9 +
.../org/apache/lucene/util/QueryBuilder.java | 113 +-
.../apache/lucene/util/automaton/Automaton.java | 5 +-
.../lucene/util/automaton/Operations.java | 6 +-
.../apache/lucene/util/automaton/StatePair.java | 4 +-
.../org/apache/lucene/util/bkd/BKDWriter.java | 53 +-
.../graph/GraphTokenStreamFiniteStrings.java | 230 ++
.../apache/lucene/util/graph/package-info.java | 21 +
.../apache/lucene/util/packed/DirectWriter.java | 8 +-
.../lucene/analysis/TestGraphTokenizers.java | 600 ++++++
.../lucene70/TestLucene70DocValuesFormat.java | 152 ++
.../lucene/index/Test4GBStoredFields.java | 2 +
.../org/apache/lucene/index/TestCheckIndex.java | 5 -
.../apache/lucene/index/TestDocIDMerger.java | 4 +-
.../apache/lucene/index/TestIndexSorting.java | 2 -
.../apache/lucene/index/TestIndexWriter.java | 3 +-
.../org/apache/lucene/index/TestTermsEnum.java | 8 +
.../lucene/search/TermInSetQueryTest.java | 328 +++
.../lucene/search/TestDoubleValuesSource.java | 167 ++
.../apache/lucene/search/TestGraphQuery.java | 79 +
.../apache/lucene/search/TestLRUQueryCache.java | 7 +
.../lucene/search/TestLongValuesSource.java | 149 ++
.../apache/lucene/search/TestPointQueries.java | 28 +
.../org/apache/lucene/store/TestDirectory.java | 13 +-
.../apache/lucene/store/TestMmapDirectory.java | 3 +-
.../org/apache/lucene/store/TestMultiMMap.java | 3 +-
.../apache/lucene/util/TestQueryBuilder.java | 15 +-
.../org/apache/lucene/util/bkd/TestBKD.java | 48 +-
.../org/apache/lucene/util/fst/Test2BFST.java | 2 +
.../TestGraphTokenStreamFiniteStrings.java | 217 ++
.../demo/facet/DistanceFacetsExample.java | 18 +-
.../ExpressionAggregationFacetsExample.java | 2 +-
lucene/expressions/build.xml | 11 -
.../org/apache/lucene/expressions/Bindings.java | 14 +-
.../apache/lucene/expressions/Expression.java | 21 +-
.../expressions/ExpressionComparator.java | 100 -
.../expressions/ExpressionFunctionValues.java | 35 +-
.../lucene/expressions/ExpressionRescorer.java | 33 +-
.../lucene/expressions/ExpressionSortField.java | 77 -
.../expressions/ExpressionValueSource.java | 70 +-
.../lucene/expressions/ScoreFunctionValues.java | 46 -
.../lucene/expressions/ScoreValueSource.java | 61 -
.../lucene/expressions/SimpleBindings.java | 30 +-
.../expressions/js/JavascriptCompiler.java | 15 +-
.../apache/lucene/expressions/package-info.java | 5 +-
.../lucene/expressions/TestDemoExpressions.java | 44 +-
.../expressions/TestExpressionSortField.java | 2 +-
.../expressions/TestExpressionValueSource.java | 111 +-
.../expressions/js/TestCustomFunctions.java | 20 +-
.../expressions/js/TestJavascriptFunction.java | 2 +-
.../js/TestJavascriptOperations.java | 2 +-
.../org/apache/lucene/facet/DrillSideways.java | 331 ++-
.../org/apache/lucene/facet/FacetQuery.java | 52 +
.../lucene/facet/FacetsCollectorManager.java | 55 +
.../apache/lucene/facet/MultiFacetQuery.java | 61 +
.../org/apache/lucene/facet/package-info.java | 2 +-
.../apache/lucene/facet/range/DoubleRange.java | 28 +-
.../facet/range/DoubleRangeFacetCounts.java | 63 +-
.../apache/lucene/facet/range/LongRange.java | 28 +-
.../facet/range/LongRangeFacetCounts.java | 26 +-
.../org/apache/lucene/facet/range/Range.java | 31 -
.../lucene/facet/taxonomy/FakeScorer.java | 53 -
.../taxonomy/TaxonomyFacetSumValueSource.java | 114 +-
.../apache/lucene/facet/TestDrillSideways.java | 315 +--
.../org/apache/lucene/facet/TestFacetQuery.java | 103 +
.../lucene/facet/TestParallelDrillSideways.java | 90 +
.../facet/range/TestRangeFacetCounts.java | 61 +-
.../TestTaxonomyFacetSumValueSource.java | 54 +-
.../AbstractAllGroupHeadsCollector.java | 176 --
.../grouping/AbstractAllGroupsCollector.java | 67 -
.../AbstractDistinctValuesCollector.java | 59 -
.../AbstractFirstPassGroupingCollector.java | 354 ----
.../grouping/AbstractGroupFacetCollector.java | 319 ---
.../AbstractSecondPassGroupingCollector.java | 162 --
.../search/grouping/AllGroupHeadsCollector.java | 176 ++
.../search/grouping/AllGroupsCollector.java | 67 +
.../search/grouping/CollectedSearchGroup.java | 2 +-
.../grouping/DistinctValuesCollector.java | 59 +
.../grouping/FirstPassGroupingCollector.java | 363 ++++
.../lucene/search/grouping/GroupDocs.java | 8 +-
.../search/grouping/GroupFacetCollector.java | 324 +++
.../apache/lucene/search/grouping/Grouper.java | 56 +
.../lucene/search/grouping/GroupingSearch.java | 130 +-
.../lucene/search/grouping/SearchGroup.java | 23 +-
.../grouping/SecondPassGroupingCollector.java | 169 ++
.../lucene/search/grouping/TopGroups.java | 12 +-
.../FunctionAllGroupHeadsCollector.java | 32 +-
.../function/FunctionAllGroupsCollector.java | 4 +-
.../FunctionDistinctValuesCollector.java | 35 +-
.../FunctionFirstPassGroupingCollector.java | 6 +-
.../grouping/function/FunctionGrouper.java | 69 +
.../FunctionSecondPassGroupingCollector.java | 6 +-
.../term/TermAllGroupHeadsCollector.java | 26 +-
.../grouping/term/TermAllGroupsCollector.java | 8 +-
.../term/TermDistinctValuesCollector.java | 26 +-
.../term/TermFirstPassGroupingCollector.java | 6 +-
.../grouping/term/TermGroupFacetCollector.java | 10 +-
.../search/grouping/term/TermGrouper.java | 81 +
.../term/TermSecondPassGroupingCollector.java | 6 +-
.../grouping/AllGroupHeadsCollectorTest.java | 14 +-
.../search/grouping/AllGroupsCollectorTest.java | 6 +-
.../grouping/DistinctValuesCollectorTest.java | 46 +-
.../grouping/GroupFacetCollectorTest.java | 10 +-
.../lucene/search/grouping/TestGrouping.java | 79 +-
.../search/uhighlight/FieldHighlighter.java | 4 +-
.../uhighlight/LengthGoalBreakIterator.java | 185 ++
.../lucene/search/uhighlight/OffsetsEnum.java | 45 +-
.../lucene/search/uhighlight/Passage.java | 1 +
.../search/highlight/TokenSourcesTest.java | 2 +-
.../uhighlight/LengthGoalBreakIteratorTest.java | 104 +
.../TestUnifiedHighlighterExtensibility.java | 57 +-
lucene/ivy-versions.properties | 10 +-
.../apache/lucene/search/DocValuesStats.java | 254 ++-
.../search/TestDocValuesStatsCollector.java | 333 ++-
.../org/apache/lucene/queries/TermsQuery.java | 381 ----
.../lucene/queries/function/ValueSource.java | 111 +
.../apache/lucene/queries/TermsQueryTest.java | 339 ---
.../queryparser/classic/QueryParserBase.java | 42 +-
.../complexPhrase/ComplexPhraseQueryParser.java | 4 +-
.../classic/TestMultiFieldQueryParser.java | 11 +-
.../queryparser/classic/TestQueryParser.java | 131 +-
.../complexPhrase/TestComplexPhraseQuery.java | 6 +
.../lucene/search/DocValuesNumbersQuery.java | 26 +-
.../lucene/search/DocValuesTermsQuery.java | 49 +-
.../org/apache/lucene/search/LongHashSet.java | 156 ++
.../lucene/document/TestHalfFloatPoint.java | 2 +
.../apache/lucene/search/LongHashSetTests.java | 100 +
.../lucene/search/TestDocValuesTermsQuery.java | 1 +
.../lucene/search/TestTermAutomatonQuery.java | 3 +
.../prefix/TermQueryPrefixTreeStrategy.java | 10 +-
.../spatial/prefix/NumberRangeFacetsTest.java | 6 +-
.../lucene/spatial3d/geom/GeoBBoxFactory.java | 34 +-
.../lucene/spatial3d/geom/GeoCircleFactory.java | 2 +-
.../spatial3d/geom/GeoPolygonFactory.java | 6 +-
.../apache/lucene/spatial3d/geom/Vector.java | 4 +
lucene/suggest/build.xml | 8 +-
.../suggest/DocumentValueSourceDictionary.java | 58 +-
.../analyzing/AnalyzingInfixSuggester.java | 137 +-
.../suggest/analyzing/FreeTextSuggester.java | 45 -
.../DocumentValueSourceDictionaryTest.java | 287 ++-
.../analysis/BaseTokenStreamTestCase.java | 37 +-
.../apache/lucene/analysis/MockTokenizer.java | 3 +-
.../asserting/AssertingLiveDocsFormat.java | 9 +-
.../index/BaseDocValuesFormatTestCase.java | 32 +
.../index/BaseStoredFieldsFormatTestCase.java | 2 +
.../index/BaseTermVectorsFormatTestCase.java | 17 +-
.../apache/lucene/index/BaseTestCheckIndex.java | 19 -
.../org/apache/lucene/index/RandomCodec.java | 2 +-
.../org/apache/lucene/search/QueryUtils.java | 4 -
.../org/apache/lucene/util/LuceneTestCase.java | 110 +-
.../org/apache/lucene/util/RamUsageTester.java | 102 +-
.../java/org/apache/lucene/util/TestUtil.java | 4 +-
.../lucene/analysis/TestGraphTokenizers.java | 588 ------
lucene/tools/javadoc/ecj.javadocs.prefs | 2 +-
lucene/tools/junit4/tests.policy | 1 -
solr/CHANGES.txt | 155 +-
solr/bin/solr | 26 +-
solr/bin/solr.cmd | 36 +-
.../plugin/AnalyticsStatisticsCollector.java | 4 +-
.../solr/collection1/conf/solrconfig.xml | 5 -
.../solr/handler/dataimport/JdbcDataSource.java | 4 +-
.../handler/dataimport/SolrEntityProcessor.java | 192 +-
.../dataimport/MockSolrEntityProcessor.java | 18 +-
.../handler/dataimport/TestJdbcDataSource.java | 50 +-
.../TestSolrEntityProcessorEndToEnd.java | 27 +-
.../dataimport/TestSolrEntityProcessorUnit.java | 70 +
solr/contrib/ltr/README.md | 406 +---
solr/contrib/ltr/example/README.md | 132 ++
solr/contrib/ltr/example/config.json | 13 +-
solr/contrib/ltr/example/exampleFeatures.json | 26 +
solr/contrib/ltr/example/libsvm_formatter.py | 12 +-
solr/contrib/ltr/example/solrconfig.xml | 1722 ---------------
.../ltr/example/techproducts-features.json | 26 -
.../contrib/ltr/example/techproducts-model.json | 18 -
.../ltr/example/train_and_upload_demo_model.py | 143 +-
solr/contrib/ltr/example/user_queries.txt | 12 +-
.../org/apache/solr/ltr/CSVFeatureLogger.java | 62 +
.../java/org/apache/solr/ltr/FeatureLogger.java | 134 +-
.../java/org/apache/solr/ltr/LTRRescorer.java | 2 +-
.../org/apache/solr/ltr/LTRScoringQuery.java | 2 +-
.../solr/ltr/SolrQueryRequestContextUtils.java | 6 +-
.../org/apache/solr/ltr/model/LinearModel.java | 10 +
.../ltr/model/MultipleAdditiveTreesModel.java | 5 +
.../LTRFeatureLoggerTransformerFactory.java | 100 +-
.../ltr/store/rest/ManagedFeatureStore.java | 3 -
.../solr/ltr/store/rest/ManagedModelStore.java | 6 +-
solr/contrib/ltr/src/java/overview.html | 2 +-
.../multipleadditivetreesmodel.json | 2 +-
.../solr/collection1/conf/solrconfig-ltr.xml | 6 +-
.../collection1/conf/solrconfig-ltr_Th10_10.xml | 5 +-
.../collection1/conf/solrconfig-multiseg.xml | 12 +-
.../apache/solr/ltr/FeatureLoggerTestUtils.java | 44 +
.../org/apache/solr/ltr/TestLTROnSolrCloud.java | 17 +-
.../apache/solr/ltr/TestLTRQParserExplain.java | 2 +-
.../apache/solr/ltr/TestLTRQParserPlugin.java | 5 +-
.../org/apache/solr/ltr/TestLTRWithFacet.java | 2 +-
.../org/apache/solr/ltr/TestLTRWithSort.java | 2 +-
.../org/apache/solr/ltr/TestRerankBase.java | 63 +-
.../solr/ltr/TestSelectiveWeightCreation.java | 13 +-
.../ltr/feature/TestEdisMaxSolrFeature.java | 2 +-
.../solr/ltr/feature/TestExternalFeatures.java | 39 +-
.../ltr/feature/TestExternalValueFeatures.java | 23 +-
.../solr/ltr/feature/TestFeatureLogging.java | 115 +-
.../ltr/feature/TestFeatureLtrScoringModel.java | 2 +-
.../solr/ltr/feature/TestFeatureStore.java | 2 +-
.../ltr/feature/TestFieldLengthFeature.java | 2 +-
.../solr/ltr/feature/TestFieldValueFeature.java | 15 +-
.../solr/ltr/feature/TestFilterSolrFeature.java | 8 +-
.../ltr/feature/TestNoMatchSolrFeature.java | 94 +-
.../ltr/feature/TestOriginalScoreFeature.java | 11 +-
.../solr/ltr/feature/TestRankingFeature.java | 2 +-
.../ltr/feature/TestUserTermScoreWithQ.java | 2 +-
.../ltr/feature/TestUserTermScorerQuery.java | 2 +-
.../ltr/feature/TestUserTermScorereQDF.java | 2 +-
.../solr/ltr/feature/TestValueFeature.java | 2 +-
.../apache/solr/ltr/model/TestLinearModel.java | 2 +-
.../model/TestMultipleAdditiveTreesModel.java | 11 +-
.../solr/ltr/store/rest/TestModelManager.java | 49 +-
.../store/rest/TestModelManagerPersistence.java | 6 +-
solr/contrib/morphlines-core/ivy.xml | 8 +-
.../solr/solrcloud/conf/solrconfig.xml | 11 -
.../solr/SolrMorphlineZkAliasTest.java | 7 -
solr/core/ivy.xml | 5 +-
.../org/apache/solr/cloud/CreateAliasCmd.java | 21 +-
.../java/org/apache/solr/cloud/Overseer.java | 2 +-
.../apache/solr/cloud/OverseerStatusCmd.java | 4 +-
.../apache/solr/cloud/rule/ReplicaAssigner.java | 31 +-
.../java/org/apache/solr/cloud/rule/Rule.java | 4 +-
.../org/apache/solr/core/CoreContainer.java | 84 +-
.../org/apache/solr/core/DirectoryFactory.java | 27 +
.../apache/solr/core/HdfsDirectoryFactory.java | 29 +
.../org/apache/solr/core/JmxMonitoredMap.java | 63 +-
.../solr/core/MetricsDirectoryFactory.java | 505 +++++
.../java/org/apache/solr/core/NodeConfig.java | 18 +-
.../java/org/apache/solr/core/PluginInfo.java | 4 +-
.../java/org/apache/solr/core/SolrConfig.java | 2 +-
.../src/java/org/apache/solr/core/SolrCore.java | 189 +-
.../apache/solr/core/SolrDeletionPolicy.java | 6 +
.../org/apache/solr/core/SolrInfoMBean.java | 11 +-
.../org/apache/solr/core/SolrXmlConfig.java | 23 +-
.../solr/core/StandardDirectoryFactory.java | 46 +-
.../apache/solr/handler/CdcrRequestHandler.java | 5 +
.../org/apache/solr/handler/GraphHandler.java | 4 +-
.../org/apache/solr/handler/IndexFetcher.java | 25 +-
.../apache/solr/handler/PingRequestHandler.java | 5 +
.../apache/solr/handler/ReplicationHandler.java | 21 +-
.../apache/solr/handler/RequestHandlerBase.java | 57 +-
.../org/apache/solr/handler/SchemaHandler.java | 5 +
.../org/apache/solr/handler/SnapShooter.java | 6 +-
.../apache/solr/handler/SolrConfigHandler.java | 2 +-
.../org/apache/solr/handler/StreamHandler.java | 24 +-
.../solr/handler/UpdateRequestHandler.java | 5 +
.../solr/handler/admin/CollectionsHandler.java | 7 +-
.../solr/handler/admin/ConfigSetsHandler.java | 5 +
.../solr/handler/admin/CoreAdminHandler.java | 16 +-
.../apache/solr/handler/admin/InfoHandler.java | 5 +
.../solr/handler/admin/LoggingHandler.java | 6 +
.../solr/handler/admin/LukeRequestHandler.java | 23 +-
.../solr/handler/admin/MetricsHandler.java | 207 ++
.../solr/handler/admin/PluginInfoHandler.java | 5 +
.../handler/admin/PropertiesRequestHandler.java | 5 +
.../solr/handler/admin/SecurityConfHandler.java | 5 +
.../admin/SegmentsInfoRequestHandler.java | 6 +
.../handler/admin/ShowFileRequestHandler.java | 4 +
.../handler/admin/SolrInfoMBeanHandler.java | 5 +
.../solr/handler/admin/SystemInfoHandler.java | 7 +-
.../solr/handler/admin/ThreadDumpHandler.java | 5 +
.../handler/admin/ZookeeperInfoHandler.java | 5 +
.../solr/handler/component/DebugComponent.java | 5 +
.../solr/handler/component/ExpandComponent.java | 30 +-
.../solr/handler/component/FacetComponent.java | 5 +
.../handler/component/HighlightComponent.java | 5 +
.../handler/component/HttpShardHandler.java | 48 +-
.../component/HttpShardHandlerFactory.java | 129 +-
.../component/MoreLikeThisComponent.java | 7 +-
.../solr/handler/component/QueryComponent.java | 23 +-
.../component/QueryElevationComponent.java | 4 +-
.../handler/component/RealTimeGetComponent.java | 5 +
.../solr/handler/component/ResponseBuilder.java | 1 -
.../solr/handler/component/SearchHandler.java | 2 +-
.../solr/handler/component/ShardHandler.java | 5 +-
.../handler/component/SpellCheckComponent.java | 5 +
.../handler/component/TermVectorComponent.java | 5 +
.../solr/handler/component/TermsComponent.java | 5 +
.../solr/highlight/HighlightingPluginBase.java | 2 +-
.../solr/highlight/LuceneRegexFragmenter.java | 217 ++
.../apache/solr/highlight/RegexFragmenter.java | 196 --
.../solr/highlight/UnifiedSolrHighlighter.java | 469 +++--
.../solr/metrics/OperatingSystemMetricSet.java | 92 +
.../solr/metrics/SolrCoreMetricManager.java | 148 ++
.../org/apache/solr/metrics/SolrMetricInfo.java | 104 +
.../apache/solr/metrics/SolrMetricManager.java | 675 ++++++
.../apache/solr/metrics/SolrMetricProducer.java | 32 +
.../apache/solr/metrics/SolrMetricReporter.java | 83 +
.../org/apache/solr/metrics/package-info.java | 23 +
.../metrics/reporters/JmxObjectNameFactory.java | 155 ++
.../metrics/reporters/SolrGangliaReporter.java | 144 ++
.../metrics/reporters/SolrGraphiteReporter.java | 129 ++
.../solr/metrics/reporters/SolrJmxReporter.java | 179 ++
.../metrics/reporters/SolrSlf4jReporter.java | 127 ++
.../solr/metrics/reporters/package-info.java | 22 +
.../apache/solr/parser/SolrQueryParserBase.java | 16 +-
.../request/PerSegmentSingleValuedFaceting.java | 9 +-
.../org/apache/solr/request/SimpleFacets.java | 41 +-
.../solr/response/JSONResponseWriter.java | 86 +-
.../java/org/apache/solr/schema/FieldType.java | 4 +-
.../apache/solr/schema/PreAnalyzedField.java | 11 +
.../org/apache/solr/schema/SchemaManager.java | 99 +-
.../org/apache/solr/search/CacheConfig.java | 2 +-
.../solr/search/CollapsingQParserPlugin.java | 14 +-
.../solr/search/ComplexPhraseQParserPlugin.java | 70 +-
.../org/apache/solr/search/FastLRUCache.java | 2 +-
.../java/org/apache/solr/search/Grouping.java | 14 +-
.../apache/solr/search/TermsQParserPlugin.java | 4 +-
.../apache/solr/search/facet/FacetModule.java | 5 +
.../org/apache/solr/search/facet/HLLAgg.java | 13 +-
.../apache/solr/search/facet/PercentileAgg.java | 7 +-
.../org/apache/solr/search/facet/SlotAcc.java | 5 +-
.../apache/solr/search/facet/UniqueSlotAcc.java | 2 +-
.../solr/search/grouping/CommandHandler.java | 4 +-
.../search/grouping/GroupingSpecification.java | 18 +-
.../command/SearchGroupsFieldCommand.java | 8 +-
.../command/TopGroupsFieldCommand.java | 4 +-
.../org/apache/solr/search/join/GraphQuery.java | 4 +-
.../apache/solr/search/mlt/CloudMLTQParser.java | 49 +-
.../solr/search/mlt/SimpleMLTQParser.java | 30 +-
.../security/AttributeOnlyServletContext.java | 291 +++
.../ConfigurableInternodeAuthHadoopPlugin.java | 68 +
.../security/DelegationTokenKerberosFilter.java | 6 +-
.../solr/security/GenericHadoopAuthPlugin.java | 266 +++
.../apache/solr/security/HadoopAuthFilter.java | 198 ++
.../apache/solr/security/HadoopAuthPlugin.java | 241 +++
.../apache/solr/security/KerberosFilter.java | 6 +-
.../apache/solr/security/KerberosPlugin.java | 314 +--
.../solr/security/PKIAuthenticationPlugin.java | 6 +
.../solr/security/PermissionNameProvider.java | 1 +
...tContinuesRecorderAuthenticationHandler.java | 71 +
.../apache/solr/servlet/SolrDispatchFilter.java | 27 +
.../solr/spelling/DirectSolrSpellChecker.java | 18 +-
.../DocumentExpressionDictionaryFactory.java | 6 +-
.../solr/update/DirectUpdateHandler2.java | 5 -
.../java/org/apache/solr/update/PeerSync.java | 50 +-
.../org/apache/solr/update/SolrIndexConfig.java | 16 +-
.../org/apache/solr/update/SolrIndexWriter.java | 144 ++
.../org/apache/solr/update/UpdateHandler.java | 5 +
.../java/org/apache/solr/update/UpdateLog.java | 74 +-
.../apache/solr/update/UpdateShardHandler.java | 81 +-
.../solr/update/UpdateShardHandlerConfig.java | 14 +-
.../AddSchemaFieldsUpdateProcessorFactory.java | 3 +-
.../SkipExistingDocumentsProcessorFactory.java | 258 +++
.../src/java/org/apache/solr/util/DOMUtil.java | 5 +
.../src/java/org/apache/solr/util/JmxUtil.java | 78 +
.../src/java/org/apache/solr/util/SolrCLI.java | 65 +-
.../src/java/org/apache/solr/util/TimeOut.java | 13 +-
.../stats/HttpClientMetricNameStrategy.java | 28 +
.../stats/InstrumentedHttpRequestExecutor.java | 136 ++
...entedPoolingHttpClientConnectionManager.java | 72 +
.../org/apache/solr/util/stats/MetricUtils.java | 203 ++
.../org/apache/solr/util/stats/TimerUtils.java | 58 -
.../resources/EditableSolrConfigAttributes.json | 16 +-
.../schema-minimal-with-another-uniqkey.xml | 23 +
...dd-schema-fields-update-processor-chains.xml | 8 +-
.../conf/solrconfig-cache-enable-disable.xml | 80 +
.../conf/solrconfig-indexmetrics.xml | 57 +
.../collection1/conf/solrconfig-schemaless.xml | 45 +-
.../solr/collection1/conf/solrconfig_perf.xml | 1 -
.../solr/security/hadoop_kerberos_config.json | 16 +
.../hadoop_simple_auth_with_delegation.json | 29 +
.../test-files/solr/solr-gangliareporter.xml | 32 +
.../test-files/solr/solr-graphitereporter.xml | 31 +
.../src/test-files/solr/solr-metricreporter.xml | 57 +
.../src/test-files/solr/solr-slf4jreporter.xml | 35 +
.../TestReversedWildcardFilterFactory.java | 56 +-
.../apache/solr/cloud/AliasIntegrationTest.java | 50 +-
.../solr/cloud/BasicDistributedZkTest.java | 2 +-
.../apache/solr/cloud/CdcrBootstrapTest.java | 25 +-
.../org/apache/solr/cloud/ClusterStateTest.java | 8 +
.../solr/cloud/CollectionsAPISolrJTest.java | 470 ++---
.../DeleteLastCustomShardedReplicaTest.java | 104 +-
.../org/apache/solr/cloud/DeleteShardTest.java | 211 +-
.../solr/cloud/DocValuesNotIndexedTest.java | 10 -
.../cloud/LeaderFailoverAfterPartitionTest.java | 2 +-
.../cloud/LeaderFailureAfterFreshStartTest.java | 68 +-
...verseerCollectionConfigSetProcessorTest.java | 4 +-
.../cloud/OverseerModifyCollectionTest.java | 92 +-
.../apache/solr/cloud/OverseerRolesTest.java | 165 +-
.../apache/solr/cloud/OverseerStatusTest.java | 55 +-
.../solr/cloud/PeerSyncReplicationTest.java | 70 +-
.../apache/solr/cloud/RemoteQueryErrorTest.java | 53 +-
.../apache/solr/cloud/TestCloudRecovery.java | 26 +
.../solr/cloud/TestDownShardTolerantSearch.java | 40 +-
.../TestExclusionRuleCollectionAccess.java | 38 +-
.../TestSolrCloudWithSecureImpersonation.java | 8 +-
.../apache/solr/cloud/TestStressLiveNodes.java | 2 +-
.../apache/solr/cloud/rule/RuleEngineTest.java | 2 +-
.../solr/core/BlobRepositoryMockingTest.java | 9 +-
.../org/apache/solr/core/CoreSorterTest.java | 7 +
.../solr/core/MockShardHandlerFactory.java | 2 +-
.../test/org/apache/solr/core/TestConfig.java | 34 +
.../solr/handler/TestReplicationHandler.java | 20 +-
.../admin/CoreMergeIndexesAdminHandlerTest.java | 10 +-
.../solr/handler/admin/MBeansHandlerTest.java | 4 +-
.../solr/handler/admin/MetricsHandlerTest.java | 149 ++
.../admin/SegmentsInfoRequestHandlerTest.java | 17 +-
.../solr/handler/admin/StatsReloadRaceTest.java | 82 +-
.../highlight/TestUnifiedSolrHighlighter.java | 34 +-
.../solr/index/hdfs/CheckHdfsIndexTest.java | 5 -
.../org/apache/solr/metrics/JvmMetricsTest.java | 68 +
.../solr/metrics/SolrCoreMetricManagerTest.java | 171 ++
.../solr/metrics/SolrMetricManagerTest.java | 273 +++
.../solr/metrics/SolrMetricReporterTest.java | 69 +
.../solr/metrics/SolrMetricTestUtils.java | 103 +
.../metrics/SolrMetricsIntegrationTest.java | 143 ++
.../metrics/reporters/MockMetricReporter.java | 80 +
.../reporters/SolrGangliaReporterTest.java | 82 +
.../reporters/SolrGraphiteReporterTest.java | 116 ++
.../metrics/reporters/SolrJmxReporterTest.java | 156 ++
.../reporters/SolrSlf4jReporterTest.java | 77 +
.../apache/solr/request/SimpleFacetsTest.java | 12 +-
.../apache/solr/response/JSONWriterTest.java | 20 +-
.../TestSubQueryTransformerDistrib.java | 55 +-
.../schema/TestSchemalessBufferedUpdates.java | 160 ++
.../ApacheLuceneSolrNearQueryBuilder.java | 51 +
.../TestComplexPhraseLeadingWildcard.java | 113 +
.../org/apache/solr/search/TestRTGBase.java | 12 -
.../org/apache/solr/search/TestRealTimeGet.java | 2 +-
.../org/apache/solr/search/TestRecovery.java | 183 +-
.../apache/solr/search/TestRecoveryHdfs.java | 11 -
.../apache/solr/search/TestSolrCoreParser.java | 110 +
.../apache/solr/search/TestSolrQueryParser.java | 12 +-
.../apache/solr/search/TestStressLucene.java | 2 +-
.../apache/solr/search/TestStressRecovery.java | 2 +-
.../apache/solr/search/TestStressReorder.java | 5 +-
.../solr/search/TestStressUserVersions.java | 4 +-
.../apache/solr/search/TestStressVersions.java | 2 +-
.../solr/search/facet/TestJsonFacets.java | 71 +-
.../solr/search/mlt/CloudMLTQParserTest.java | 23 +-
.../solr/search/mlt/SimpleMLTQParserTest.java | 33 +-
.../PKIAuthenticationIntegrationTest.java | 40 +-
.../security/TestPKIAuthenticationPlugin.java | 8 +
.../solr/security/hadoop/ImpersonationUtil.java | 73 +
.../hadoop/ImpersonatorCollectionsHandler.java | 60 +
.../hadoop/TestDelegationWithHadoopAuth.java | 400 ++++
.../hadoop/TestImpersonationWithHadoopAuth.java | 215 ++
.../TestSolrCloudWithHadoopAuthPlugin.java | 138 ++
.../solr/servlet/SolrRequestParserTest.java | 3 +
.../org/apache/solr/update/AutoCommitTest.java | 2 +-
.../apache/solr/update/CdcrUpdateLogTest.java | 17 -
.../apache/solr/update/SoftAutoCommitTest.java | 2 +-
.../apache/solr/update/SolrIndexConfigTest.java | 1 +
.../solr/update/SolrIndexMetricsTest.java | 94 +
.../update/TestInPlaceUpdatesStandalone.java | 14 +-
...ipExistingDocumentsProcessorFactoryTest.java | 335 +++
.../apache/solr/util/stats/MetricUtilsTest.java | 56 +
.../apache/solr/util/stats/TimerUtilsTest.java | 58 -
.../example-DIH/solr/db/conf/solrconfig.xml | 11 -
.../example-DIH/solr/mail/conf/solrconfig.xml | 11 -
.../example-DIH/solr/rss/conf/solrconfig.xml | 11 -
.../example-DIH/solr/solr/conf/solrconfig.xml | 11 -
.../example-DIH/solr/tika/conf/solrconfig.xml | 11 -
solr/example/files/conf/solrconfig.xml | 16 +-
solr/licenses/byte-buddy-1.6.2.jar.sha1 | 1 +
solr/licenses/byte-buddy-LICENSE-ASL.txt | 202 ++
solr/licenses/byte-buddy-NOTICE.txt | 4 +
solr/licenses/gmetric4j-1.0.7.jar.sha1 | 1 +
solr/licenses/gmetric4j-LICENSE-BSD.txt | 31 +
solr/licenses/gmetric4j-NOTICE.txt | 0
solr/licenses/metrics-ganglia-3.1.2.jar.sha1 | 1 +
solr/licenses/metrics-ganglia-LICENSE-ASL.txt | 203 ++
solr/licenses/metrics-ganglia-NOTICE.txt | 12 +
solr/licenses/metrics-graphite-3.1.2.jar.sha1 | 1 +
solr/licenses/metrics-graphite-LICENSE-ASL.txt | 203 ++
solr/licenses/metrics-graphite-NOTICE.txt | 12 +
solr/licenses/metrics-jetty-LICENSE-ASL.txt | 203 ++
solr/licenses/metrics-jetty-NOTICE.txt | 12 +
solr/licenses/metrics-jetty9-3.1.2.jar.sha1 | 1 +
solr/licenses/metrics-json-LICENSE-ASL.txt | 203 ++
solr/licenses/metrics-json-NOTICE.txt | 12 +
solr/licenses/metrics-jvm-3.1.2.jar.sha1 | 1 +
solr/licenses/metrics-jvm-LICENSE-ASL.txt | 203 ++
solr/licenses/metrics-jvm-NOTICE.txt | 12 +
solr/licenses/metrics-servlets-LICENSE-ASL.txt | 203 ++
solr/licenses/metrics-servlets-NOTICE.txt | 12 +
solr/licenses/mockito-core-1.9.5.jar.sha1 | 1 -
solr/licenses/mockito-core-2.6.2.jar.sha1 | 1 +
solr/licenses/objenesis-1.2.jar.sha1 | 1 -
solr/licenses/objenesis-2.5.jar.sha1 | 1 +
solr/server/build.xml | 4 +-
solr/server/etc/jetty.xml | 20 +-
solr/server/ivy.xml | 14 +-
.../basic_configs/conf/solrconfig.xml | 17 +-
.../conf/solrconfig.xml | 16 +-
.../conf/solrconfig.xml | 65 +-
.../impl/DelegationTokenHttpSolrClient.java | 34 +-
.../solrj/impl/HttpClientBuilderFactory.java | 41 +
.../solr/client/solrj/impl/HttpClientUtil.java | 35 +-
.../solr/client/solrj/impl/HttpSolrClient.java | 56 +-
.../solrj/impl/Krb5HttpClientBuilder.java | 10 +-
.../solr/client/solrj/io/ops/AndOperation.java | 101 +
.../client/solrj/io/ops/BooleanOperation.java | 26 +
.../client/solrj/io/ops/EqualsOperation.java | 70 +
.../io/ops/GreaterThanEqualToOperation.java | 70 +
.../solrj/io/ops/GreaterThanOperation.java | 70 +
.../solr/client/solrj/io/ops/LeafOperation.java | 67 +
.../solrj/io/ops/LessThanEqualToOperation.java | 70 +
.../client/solrj/io/ops/LessThanOperation.java | 70 +
.../solr/client/solrj/io/ops/NotOperation.java | 87 +
.../solr/client/solrj/io/ops/OrOperation.java | 71 +
.../client/solrj/io/stream/CloudSolrStream.java | 4 +
.../client/solrj/io/stream/HavingStream.java | 173 ++
.../solr/client/solrj/io/stream/NullStream.java | 155 ++
.../client/solrj/io/stream/PriorityStream.java | 161 ++
.../solrj/io/stream/expr/StreamFactory.java | 2 +
.../solrj/request/CollectionAdminRequest.java | 2 +
.../apache/solr/common/params/CommonParams.java | 4 +-
.../solr/common/params/HighlightParams.java | 4 +-
.../solr/client/solrj/LargeVolumeTestBase.java | 3 +-
.../solr/client/solrj/SolrExampleTestBase.java | 1 -
.../solrj/impl/BasicHttpSolrClientTest.java | 31 +
.../solrj/impl/CloudSolrClientCacheTest.java | 7 +
.../client/solrj/impl/CloudSolrClientTest.java | 8 +-
.../solrj/io/graph/GraphExpressionTest.java | 3 +-
.../solr/client/solrj/io/sql/JdbcTest.java | 7 +-
.../client/solrj/io/stream/JDBCStreamTest.java | 9 +-
.../solrj/io/stream/StreamExpressionTest.java | 467 ++++-
.../client/solrj/io/stream/StreamingTest.java | 10 +-
.../solr/common/params/CommonParamsTest.java | 2 +
.../java/org/apache/solr/SolrJettyTestBase.java | 2 -
.../java/org/apache/solr/SolrTestCaseHS.java | 4 +
.../java/org/apache/solr/SolrTestCaseJ4.java | 24 +-
.../solr/cloud/AbstractDistribZkTestBase.java | 31 +-
.../java/org/apache/solr/cloud/ChaosMonkey.java | 2 -
.../apache/solr/cloud/MiniSolrCloudCluster.java | 29 +-
.../apache/solr/cloud/SolrCloudTestCase.java | 30 +-
.../component/TrackingShardHandlerFactory.java | 2 +-
.../java/org/apache/solr/util/TestHarness.java | 12 +-
solr/webapp/web/js/angular/services.js | 46 +-
681 files changed, 31988 insertions(+), 11353 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/027a92a4/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/027a92a4/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/027a92a4/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/027a92a4/solr/core/src/java/org/apache/solr/update/PeerSync.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/027a92a4/solr/core/src/java/org/apache/solr/update/UpdateLog.java
----------------------------------------------------------------------
diff --cc solr/core/src/java/org/apache/solr/update/UpdateLog.java
index 87833eb,7c2dae6..23a7670
--- a/solr/core/src/java/org/apache/solr/update/UpdateLog.java
+++ b/solr/core/src/java/org/apache/solr/update/UpdateLog.java
@@@ -42,9 -40,10 +42,11 @@@ import java.util.concurrent.Synchronous
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
+ import com.codahale.metrics.Gauge;
+ import com.codahale.metrics.Meter;
import org.apache.hadoop.fs.FileSystem;
import org.apache.lucene.util.BytesRef;
+import org.apache.solr.common.SolrDocumentBase;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.SolrInputDocument;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/027a92a4/solr/core/src/java/org/apache/solr/update/processor/SkipExistingDocumentsProcessorFactory.java
----------------------------------------------------------------------
diff --cc solr/core/src/java/org/apache/solr/update/processor/SkipExistingDocumentsProcessorFactory.java
index 0000000,ec637a4..2c58410
mode 000000,100644..100644
--- a/solr/core/src/java/org/apache/solr/update/processor/SkipExistingDocumentsProcessorFactory.java
+++ b/solr/core/src/java/org/apache/solr/update/processor/SkipExistingDocumentsProcessorFactory.java
@@@ -1,0 -1,255 +1,258 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+ package org.apache.solr.update.processor;
+
+ import org.apache.lucene.util.BytesRef;
+ import org.apache.solr.common.SolrException;
+ import org.apache.solr.common.SolrInputDocument;
+ import org.apache.solr.common.util.NamedList;
+ import org.apache.solr.core.SolrCore;
+ import org.apache.solr.handler.component.RealTimeGetComponent;
+ import org.apache.solr.request.SolrQueryRequest;
+ import org.apache.solr.response.SolrQueryResponse;
+ import org.apache.solr.search.SolrIndexSearcher;
+ import org.apache.solr.update.AddUpdateCommand;
+ import org.apache.solr.update.UpdateCommand;
+ import org.apache.solr.util.RefCounted;
+ import org.apache.solr.util.plugin.SolrCoreAware;
+ import org.slf4j.Logger;
+ import org.slf4j.LoggerFactory;
+
+ import java.io.IOException;
+ import java.lang.invoke.MethodHandles;
++import java.util.Collections;
+
+ import static org.apache.solr.common.SolrException.ErrorCode.SERVER_ERROR;
+ import static org.apache.solr.update.processor.DistributingUpdateProcessorFactory.DISTRIB_UPDATE_PARAM;
+
+ /**
+ * <p>
+ * This Factory generates an UpdateProcessor that will (by default) skip inserting new documents
+ * if there already exists a document with the same uniqueKey value in the index. It will also
+ * skip Atomic Updates to a document if that document does not already exist. This behaviour is applied
+ * to each document in turn, so adding a batch of documents can result in some being added and some
+ * ignored, depending on what is already in the index. If all of the documents are skipped, no changes
+ * to the index will occur.
+ * </p>
+ * These two forms of skipping can be switched on or off independently, by using init params:
+ * <ul>
+ * <li><code>skipInsertIfExists</code> - This boolean parameter defaults to
+ * <code>true</code>, but if set to <code>false</code> then inserts (i.e. not Atomic Updates)
+ * will be passed through unchanged even if the document already exists.</li>
+ * <li><code>skipUpdateIfMissing</code> - This boolean parameter defaults to
+ * <code>true</code>, but if set to <code>false</code> then Atomic Updates
+ * will be passed through unchanged regardless of whether the document exists.</li>
+ * </ul>
+ * <p>
+ * These params can also be specified per-request, to override the configured behaviour
+ * for specific updates e.g. <code>/update?skipUpdateIfMissing=true</code>
+ * </p>
+ * <p>
+ * This implementation is a simpler alternative to {@link DocBasedVersionConstraintsProcessorFactory}
+ * when you are not concerned with versioning, and just want to quietly ignore duplicate documents and/or
+ * silently skip updates to non-existent documents (in the same way a database <code>UPDATE</code> would).
+ *
+ * If your documents do have an explicit version field, and you want to ensure older versions are
+ * skipped instead of replacing the indexed document, you should consider {@link DocBasedVersionConstraintsProcessorFactory}
+ * instead.
+ * </p>
+ * <p>
+ * An example chain configuration to use this for skipping duplicate inserts, but not skipping updates to
+ * missing documents by default, is:
+ * </p>
+ * <pre class="prettyprint">
+ * <updateRequestProcessorChain name="skipexisting">
+ * <processor class="solr.LogUpdateProcessorFactory" />
+ * <processor class="solr.SkipExistingDocumentsProcessorFactory">
+ * <bool name="skipInsertIfExists">true</bool>
+ * <bool name="skipUpdateIfMissing">false</bool> <!-- Can override this per-request -->
+ * </processor>
+ * <processor class="solr.DistributedUpdateProcessorFactory" />
+ * <processor class="solr.RunUpdateProcessorFactory" />
+ * </updateRequestProcessorChain>
+ * </pre>
+ */
+ public class SkipExistingDocumentsProcessorFactory extends UpdateRequestProcessorFactory implements SolrCoreAware, UpdateRequestProcessorFactory.RunAlways {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ private static final String PARAM_SKIP_INSERT_IF_EXISTS = "skipInsertIfExists";
+ private static final String PARAM_SKIP_UPDATE_IF_MISSING = "skipUpdateIfMissing";
+
+ private boolean skipInsertIfExists = true;
+ private boolean skipUpdateIfMissing = true;
+
+ @Override
+ public void init(NamedList args) {
+ Object tmp = args.remove(PARAM_SKIP_INSERT_IF_EXISTS);
+ if (null != tmp) {
+ if (! (tmp instanceof Boolean) ) {
+ throw new SolrException(SERVER_ERROR, "'" + PARAM_SKIP_INSERT_IF_EXISTS + "' must be configured as a <bool>");
+ }
+ skipInsertIfExists = (Boolean)tmp;
+ }
+ tmp = args.remove(PARAM_SKIP_UPDATE_IF_MISSING);
+ if (null != tmp) {
+ if (! (tmp instanceof Boolean) ) {
+ throw new SolrException(SERVER_ERROR, "'" + PARAM_SKIP_UPDATE_IF_MISSING + "' must be configured as a <bool>");
+ }
+ skipUpdateIfMissing = (Boolean)tmp;
+ }
+
+ super.init(args);
+ }
+
+ @Override
+ public SkipExistingDocumentsUpdateProcessor getInstance(SolrQueryRequest req,
+ SolrQueryResponse rsp,
+ UpdateRequestProcessor next) {
+ // Ensure the parameters are forwarded to the leader
+ DistributedUpdateProcessorFactory.addParamToDistributedRequestWhitelist(req, PARAM_SKIP_INSERT_IF_EXISTS, PARAM_SKIP_UPDATE_IF_MISSING);
+
+ // Allow the particular request to override the plugin's configured behaviour
+ boolean skipInsertForRequest = req.getOriginalParams().getBool(PARAM_SKIP_INSERT_IF_EXISTS, this.skipInsertIfExists);
+ boolean skipUpdateForRequest = req.getOriginalParams().getBool(PARAM_SKIP_UPDATE_IF_MISSING, this.skipUpdateIfMissing);
+
+ return new SkipExistingDocumentsUpdateProcessor(req, next, skipInsertForRequest, skipUpdateForRequest);
+ }
+
+ @Override
+ public void inform(SolrCore core) {
+
+ if (core.getUpdateHandler().getUpdateLog() == null) {
+ throw new SolrException(SERVER_ERROR, "updateLog must be enabled.");
+ }
+
+ if (core.getLatestSchema().getUniqueKeyField() == null) {
+ throw new SolrException(SERVER_ERROR, "schema must have uniqueKey defined.");
+ }
+ }
+
+ static class SkipExistingDocumentsUpdateProcessor extends UpdateRequestProcessor {
+
+ private final boolean skipInsertIfExists;
+ private final boolean skipUpdateIfMissing;
+ private final SolrCore core;
+
+ private DistributedUpdateProcessor distribProc; // the distributed update processor following us
+ private DistributedUpdateProcessor.DistribPhase phase;
+
+ SkipExistingDocumentsUpdateProcessor(SolrQueryRequest req,
+ UpdateRequestProcessor next,
+ boolean skipInsertIfExists,
+ boolean skipUpdateIfMissing) {
+ super(next);
+ this.skipInsertIfExists = skipInsertIfExists;
+ this.skipUpdateIfMissing = skipUpdateIfMissing;
+ this.core = req.getCore();
+
+ for (UpdateRequestProcessor proc = next ;proc != null; proc = proc.next) {
+ if (proc instanceof DistributedUpdateProcessor) {
+ distribProc = (DistributedUpdateProcessor)proc;
+ break;
+ }
+ }
+
+ if (distribProc == null) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "DistributedUpdateProcessor must follow SkipExistingDocumentsUpdateProcessor");
+ }
+
+ phase = DistributedUpdateProcessor.DistribPhase.parseParam(req.getParams().get(DISTRIB_UPDATE_PARAM));
+ }
+
+ boolean isSkipInsertIfExists() {
+ return this.skipInsertIfExists;
+ }
+
+ boolean isSkipUpdateIfMissing() {
+ return this.skipUpdateIfMissing;
+ }
+
+ boolean doesDocumentExist(BytesRef indexedDocId) throws IOException {
+ assert null != indexedDocId;
+
- SolrInputDocument oldDoc = RealTimeGetComponent.getInputDocumentFromTlog(core, indexedDocId);
++ // we don't need any fields populated, we just need to know if the doc is in the tlog...
++ SolrInputDocument oldDoc = RealTimeGetComponent.getInputDocumentFromTlog(core, indexedDocId, null,
++ Collections.<String>emptySet(), false);
+ if (oldDoc == RealTimeGetComponent.DELETED) {
+ return false;
+ }
+ if (oldDoc != null) {
+ return true;
+ }
+
+ // need to look up in index now...
+ RefCounted<SolrIndexSearcher> newestSearcher = core.getRealtimeSearcher();
+ try {
+ SolrIndexSearcher searcher = newestSearcher.get();
+ return searcher.lookupId(indexedDocId) >= 0L;
+ } catch (IOException e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error reading document from index", e);
+ } finally {
+ if (newestSearcher != null) {
+ newestSearcher.decref();
+ }
+ }
+ }
+
+ boolean isLeader(UpdateCommand cmd) {
+ if ((cmd.getFlags() & (UpdateCommand.REPLAY | UpdateCommand.PEER_SYNC)) != 0) {
+ return false;
+ }
+ if (phase == DistributedUpdateProcessor.DistribPhase.FROMLEADER) {
+ return false;
+ }
+ return distribProc.isLeader(cmd);
+ }
+
+ @Override
+ public void processAdd(AddUpdateCommand cmd) throws IOException {
+ BytesRef indexedDocId = cmd.getIndexedId();
+
+ boolean isUpdate = AtomicUpdateDocumentMerger.isAtomicUpdate(cmd);
+
+ // boolean existsByLookup = (RealTimeGetComponent.getInputDocument(core, indexedDocId) != null);
+ // if (docExists != existsByLookup) {
+ // log.error("Found docExists {} but existsByLookup {} for doc {}", docExists, existsByLookup, indexedDocId.utf8ToString());
+ // }
+
+ if (log.isDebugEnabled()) {
+ log.debug("Document ID {} ... exists already? {} ... isAtomicUpdate? {} ... isLeader? {}",
+ indexedDocId.utf8ToString(), doesDocumentExist(indexedDocId), isUpdate, isLeader(cmd));
+ }
+
+ if (skipInsertIfExists && !isUpdate && isLeader(cmd) && doesDocumentExist(indexedDocId)) {
+ if (log.isDebugEnabled()) {
+ log.debug("Skipping insert for pre-existing document ID {}", indexedDocId.utf8ToString());
+ }
+ return;
+ }
+
+ if (skipUpdateIfMissing && isUpdate && isLeader(cmd) && !doesDocumentExist(indexedDocId)) {
+ if (log.isDebugEnabled()) {
+ log.debug("Skipping update to non-existent document ID {}", indexedDocId.utf8ToString());
+ }
+ return;
+ }
+
+ if (log.isDebugEnabled()) {
+ log.debug("Passing on document ID {}", indexedDocId.utf8ToString());
+ }
+
+ super.processAdd(cmd);
+ }
+ }
+ }
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/027a92a4/solr/core/src/test/org/apache/solr/search/TestRecovery.java
----------------------------------------------------------------------
diff --cc solr/core/src/test/org/apache/solr/search/TestRecovery.java
index 2782ef9,15aed5d..29efa52
--- a/solr/core/src/test/org/apache/solr/search/TestRecovery.java
+++ b/solr/core/src/test/org/apache/solr/search/TestRecovery.java
@@@ -19,12 -19,15 +19,20 @@@ package org.apache.solr.search
import static org.apache.solr.update.processor.DistributingUpdateProcessorFactory.DISTRIB_UPDATE_PARAM;
+ import com.codahale.metrics.Gauge;
+ import com.codahale.metrics.Meter;
+ import com.codahale.metrics.Metric;
+ import com.codahale.metrics.MetricRegistry;
+ import org.apache.solr.metrics.SolrMetricManager;
import org.noggit.ObjectBuilder;
++
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
++
+ import org.apache.lucene.util.TestUtil;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.schema.IndexSchema;
import org.apache.solr.update.DirectUpdateHandler2;
import org.apache.solr.update.UpdateLog;
import org.apache.solr.update.UpdateHandler;
@@@ -83,22 -78,14 +91,15 @@@ public class TestRecovery extends SolrT
}
}
-
- // since we make up fake versions in these tests, we can get messed up by a DBQ with a real version
- // since Solr can think following updates were reordered.
- @Override
- public void clearIndex() {
- try {
- deleteByQueryAndGetVersion("*:*", params("_version_", Long.toString(-Long.MAX_VALUE), DISTRIB_UPDATE_PARAM,FROM_LEADER));
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
+ private Map<String, Metric> getMetrics() {
+ SolrMetricManager manager = h.getCoreContainer().getMetricManager();
+ MetricRegistry registry = manager.registry(h.getCore().getCoreMetricManager().getRegistryName());
+ return registry.getMetrics();
}
-
@Test
public void testLogReplay() throws Exception {
+
try {
DirectUpdateHandler2.commitOnClose = false;
@@@ -155,6 -154,9 +170,9 @@@
assertJQ(req("q","*:*") ,"/response/numFound==3");
- assertEquals(5L, replayDocs.getCount() - initialOps);
++ assertEquals(7L, replayDocs.getCount() - initialOps);
+ assertEquals(UpdateLog.State.ACTIVE.ordinal(), state.getValue().intValue());
+
// make sure we can still access versions after recovery
assertJQ(req("qt","/get", "getVersions",""+versions.size()) ,"/versions==" + versions);
@@@ -1119,133 -1258,6 +1276,133 @@@
}
}
+ @Test
+ public void testLogReplayWithInPlaceUpdatesAndDeletes() throws Exception {
+
+ try {
+
+ DirectUpdateHandler2.commitOnClose = false;
+ final Semaphore logReplay = new Semaphore(0);
+ final Semaphore logReplayFinish = new Semaphore(0);
+
+ UpdateLog.testing_logReplayHook = () -> {
+ try {
+ assertTrue(logReplay.tryAcquire(timeout, TimeUnit.SECONDS));
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ };
+
+ UpdateLog.testing_logReplayFinishHook = () -> logReplayFinish.release();
+
+
+ clearIndex();
+ assertU(commit());
+
+ Deque<Long> versions = new ArrayDeque<>();
+ versions.addFirst(addAndGetVersion(sdoc("id", "A1"), null));
+
+ // DBQ of updated document using id
+ versions.addFirst(addAndGetVersion(sdoc("id", "A2", "val_i_dvo", "1"), null));
+ versions.addFirst(addAndGetVersion(sdoc("id", "A2", "val_i_dvo", map("set", 2)), null)); // in-place update
+ versions.addFirst(deleteByQueryAndGetVersion("id:A2", null));
+
+ // DBQ of updated document using updated value
+ versions.addFirst(addAndGetVersion(sdoc("id", "A3", "val_i_dvo", "101"), null));
+ versions.addFirst(addAndGetVersion(sdoc("id", "A3", "val_i_dvo", map("set", 102)), null)); // in-place update
+ versions.addFirst(deleteByQueryAndGetVersion("val_i_dvo:102", null));
+
+ // DBQ using an intermediate update value (shouldn't delete anything)
+ versions.addFirst(addAndGetVersion(sdoc("id", "A4", "val_i_dvo", "200"), null));
+ versions.addFirst(addAndGetVersion(sdoc("id", "A4", "val_i_dvo", map("inc", "1")), null)); // in-place update
+ versions.addFirst(addAndGetVersion(sdoc("id", "A4", "val_i_dvo", map("inc", "1")), null)); // in-place update
+ versions.addFirst(deleteByQueryAndGetVersion("val_i_dvo:201", null));
+
+ // DBI of updated document
+ versions.addFirst(addAndGetVersion(sdoc("id", "A5", "val_i_dvo", "300"), null));
+ versions.addFirst(addAndGetVersion(sdoc("id", "A5", "val_i_dvo", map("inc", "1")), null)); // in-place update
+ versions.addFirst(addAndGetVersion(sdoc("id", "A5", "val_i_dvo", map("inc", "1")), null)); // in-place update
+ versions.addFirst(deleteAndGetVersion("A5", null));
+
+ assertJQ(req("q","*:*"),"/response/numFound==0");
+
+
+ assertJQ(req("qt","/get", "getVersions",""+versions.size()) ,"/versions==" + versions);
+
+ h.close();
+ createCore();
+
+ // Solr should kick this off now
+ // h.getCore().getUpdateHandler().getUpdateLog().recoverFromLog();
+
+ // verify that previous close didn't do a commit
+ // recovery should be blocked by our hook
+ assertJQ(req("q","*:*") ,"/response/numFound==0");
+
+ // make sure we can still access versions after a restart
+ assertJQ(req("qt","/get", "getVersions",""+versions.size()),"/versions==" + versions);
+
+ // unblock recovery
+ logReplay.release(1000);
+
+ // make sure we can still access versions during recovery
+ assertJQ(req("qt","/get", "getVersions",""+versions.size()),"/versions==" + versions);
+
+ // wait until recovery has finished
+ assertTrue(logReplayFinish.tryAcquire(timeout, TimeUnit.SECONDS));
+ assertJQ(req("q","val_i_dvo:202") ,"/response/numFound==1"); // assert that in-place update is retained
+
- assertJQ(req("q","*:*") ,"/response/numFound==2"); // nocommit: Fails due to SOLR-9941
++ assertJQ(req("q","*:*") ,"/response/numFound==2");
+ assertJQ(req("q","id:A2") ,"/response/numFound==0");
+ assertJQ(req("q","id:A3") ,"/response/numFound==0");
+ assertJQ(req("q","id:A4") ,"/response/numFound==1");
+ assertJQ(req("q","id:A5") ,"/response/numFound==0");
+
+ // make sure we can still access versions after recovery
+ assertJQ(req("qt","/get", "getVersions",""+versions.size()) ,"/versions==" + versions);
+
+ assertU(adoc("id","A10"));
+
+ h.close();
+ createCore();
+ // Solr should kick this off now
+ // h.getCore().getUpdateHandler().getUpdateLog().recoverFromLog();
+
+ // wait until recovery has finished
+ assertTrue(logReplayFinish.tryAcquire(timeout, TimeUnit.SECONDS));
+ assertJQ(req("q","*:*") ,"/response/numFound==3");
+ assertJQ(req("q","id:A2") ,"/response/numFound==0");
+ assertJQ(req("q","id:A3") ,"/response/numFound==0");
+ assertJQ(req("q","id:A4") ,"/response/numFound==1");
+ assertJQ(req("q","id:A5") ,"/response/numFound==0");
+ assertJQ(req("q","id:A10"),"/response/numFound==1");
+
+ // no updates, so insure that recovery does not run
+ h.close();
+ int permits = logReplay.availablePermits();
+ createCore();
+ // Solr should kick this off now
+ // h.getCore().getUpdateHandler().getUpdateLog().recoverFromLog();
+
+ assertJQ(req("q","*:*") ,"/response/numFound==3");
+ assertJQ(req("q","val_i_dvo:202") ,"/response/numFound==1"); // assert that in-place update is retained
+ assertJQ(req("q","id:A2") ,"/response/numFound==0");
+ assertJQ(req("q","id:A3") ,"/response/numFound==0");
+ assertJQ(req("q","id:A4") ,"/response/numFound==1");
+ assertJQ(req("q","id:A5") ,"/response/numFound==0");
+ assertJQ(req("q","id:A10"),"/response/numFound==1");
+ Thread.sleep(100);
+ assertEquals(permits, logReplay.availablePermits()); // no updates, so insure that recovery didn't run
+
+ assertEquals(UpdateLog.State.ACTIVE, h.getCore().getUpdateHandler().getUpdateLog().getState());
+
+ } finally {
+ DirectUpdateHandler2.commitOnClose = true;
+ UpdateLog.testing_logReplayHook = null;
+ UpdateLog.testing_logReplayFinishHook = null;
+ }
+
+ }
// NOTE: replacement must currently be same size
private static void findReplace(byte[] from, byte[] to, byte[] data) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/027a92a4/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java
----------------------------------------------------------------------