You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sh...@apache.org on 2017/09/18 16:24:03 UTC
[01/50] lucene-solr:feature/autoscaling: Merge remote-tracking branch
'origin/master'
Repository: lucene-solr
Updated Branches:
refs/heads/feature/autoscaling 6fb2803bc -> de3107c70
Merge remote-tracking branch 'origin/master'
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/1fe611d0
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/1fe611d0
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/1fe611d0
Branch: refs/heads/feature/autoscaling
Commit: 1fe611d094fae0ed5e93a5dce9c6d076e2ee2e79
Parents: 2d8aaa3 f49de60
Author: Noble Paul <no...@apache.org>
Authored: Wed Sep 6 18:52:00 2017 +0930
Committer: Noble Paul <no...@apache.org>
Committed: Wed Sep 6 18:52:00 2017 +0930
----------------------------------------------------------------------
lucene/CHANGES.txt | 16 ++
lucene/NOTICE.txt | 5 +-
.../lucene/analysis/bn/BengaliAnalyzer.java | 132 +++++++++++++
.../analysis/bn/BengaliNormalizationFilter.java | 59 ++++++
.../bn/BengaliNormalizationFilterFactory.java | 55 ++++++
.../lucene/analysis/bn/BengaliNormalizer.java | 155 +++++++++++++++
.../lucene/analysis/bn/BengaliStemFilter.java | 49 +++++
.../analysis/bn/BengaliStemFilterFactory.java | 48 +++++
.../lucene/analysis/bn/BengaliStemmer.java | 183 ++++++++++++++++++
.../apache/lucene/analysis/bn/package-info.java | 21 +++
...ache.lucene.analysis.util.TokenFilterFactory | 2 +
.../org/apache/lucene/analysis/bn/stopwords.txt | 121 ++++++++++++
.../lucene/analysis/bn/TestBengaliAnalyzer.java | 53 ++++++
.../lucene/analysis/bn/TestBengaliFilters.java | 80 ++++++++
.../analysis/bn/TestBengaliNormalizer.java | 110 +++++++++++
.../lucene/analysis/bn/TestBengaliStemmer.java | 79 ++++++++
.../analysis/icu/ICUNormalizer2CharFilter.java | 35 ++--
.../icu/TestICUNormalizer2CharFilter.java | 21 +++
.../SimpleNaiveBayesDocumentClassifier.java | 30 +--
.../tokenattributes/CharTermAttributeImpl.java | 3 +
.../java/org/apache/lucene/util/LongBitSet.java | 8 +-
.../TestCharTermAttributeImpl.java | 10 +
.../org/apache/lucene/util/TestLongBitSet.java | 24 ++-
.../writercache/LruTaxonomyWriterCache.java | 20 +-
.../writercache/TestLruTaxonomyWriterCache.java | 50 +++++
lucene/ivy-versions.properties | 2 +-
.../randomizedtesting-runner-2.5.2.jar.sha1 | 1 -
.../randomizedtesting-runner-2.5.3.jar.sha1 | 1 +
.../lucene/replicator/nrt/PrimaryNode.java | 12 --
.../lucene/replicator/nrt/ReplicaNode.java | 10 -
solr/CHANGES.txt | 21 ++-
solr/core/build.xml | 5 +
.../org/apache/solr/handler/StreamHandler.java | 85 +--------
.../org/apache/solr/parser/QueryParser.java | 2 +-
.../solr/request/json/JsonQueryConverter.java | 110 +++++++++++
.../apache/solr/request/json/RequestUtil.java | 10 +-
.../org/apache/solr/schema/DateRangeField.java | 4 +-
.../apache/solr/search/BoolQParserPlugin.java | 58 ++++++
.../org/apache/solr/search/FunctionQParser.java | 11 +-
.../org/apache/solr/search/QParserPlugin.java | 1 +
.../apache/solr/search/ValueSourceParser.java | 4 +-
.../org/apache/solr/search/facet/MinMaxAgg.java | 145 ++++++++++++--
.../org/apache/solr/search/facet/SlotAcc.java | 84 ++++++++-
.../search/function/FieldNameValueSource.java | 60 ++++++
.../FieldLengthUpdateProcessorFactory.java | 6 +-
.../FieldMutatingUpdateProcessorFactory.java | 4 +-
.../ParseDateFieldUpdateProcessorFactory.java | 4 +-
.../ParseDoubleFieldUpdateProcessorFactory.java | 4 +-
.../ParseFloatFieldUpdateProcessorFactory.java | 4 +-
.../ParseIntFieldUpdateProcessorFactory.java | 4 +-
.../ParseLongFieldUpdateProcessorFactory.java | 4 +-
.../org/apache/solr/BasicFunctionalityTest.java | 2 +-
.../apache/solr/TestDistributedMissingSort.java | 8 +-
.../apache/solr/cloud/CdcrBootstrapTest.java | 1 +
.../org/apache/solr/core/TestLazyCores.java | 4 +-
.../org/apache/solr/handler/TestSQLHandler.java | 1 -
.../org/apache/solr/search/CursorMarkTest.java | 10 +-
.../apache/solr/search/QueryEqualityTest.java | 21 +++
.../apache/solr/search/TestSmileRequest.java | 2 +-
.../apache/solr/search/TestSolrQueryParser.java | 2 +-
.../solr/search/facet/TestJsonFacets.java | 86 +++++++--
.../solr/search/json/TestJsonRequest.java | 179 +++++++++++++++++-
.../apache/solr/update/SoftAutoCommitTest.java | 6 -
.../example-DIH/solr/db/conf/managed-schema | 2 +-
.../example-DIH/solr/mail/conf/managed-schema | 2 +-
.../example-DIH/solr/solr/conf/managed-schema | 2 +-
solr/example/files/conf/managed-schema | 8 +-
solr/example/films/README.txt | 6 +-
solr/licenses/httpmime-4.5.3.jar.sha1 | 2 +-
solr/licenses/junit4-ant-2.5.2.jar.sha1 | 1 -
solr/licenses/junit4-ant-2.5.3.jar.sha1 | 1 +
.../randomizedtesting-runner-2.5.2.jar.sha1 | 1 -
.../randomizedtesting-runner-2.5.3.jar.sha1 | 1 +
solr/solr-ref-guide/README.adoc | 32 +++-
.../src/cross-data-center-replication-cdcr.adoc | 1 -
solr/solr-ref-guide/src/docvalues.adoc | 24 +--
.../field-type-definitions-and-properties.adoc | 2 +-
.../src/field-types-included-with-solr.adoc | 81 +++++---
solr/solr-ref-guide/src/function-queries.adoc | 2 +-
.../solr-ref-guide/src/how-solrcloud-works.adoc | 3 +-
solr/solr-ref-guide/src/how-to-contribute.adoc | 3 +-
solr/solr-ref-guide/src/index.adoc | 2 +-
.../src/indexconfig-in-solrconfig.adoc | 2 +-
.../src/major-changes-in-solr-7.adoc | 187 +++++++++++++++++++
solr/solr-ref-guide/src/other-parsers.adoc | 2 +-
.../src/putting-the-pieces-together.adoc | 4 +-
.../read-and-write-side-fault-tolerance.adoc | 109 -----------
solr/solr-ref-guide/src/schema-api.adoc | 7 +-
solr/solr-ref-guide/src/schemaless-mode.adoc | 62 ++++--
solr/solr-ref-guide/src/solr-upgrade-notes.adoc | 56 +-----
...olrcloud-autoscaling-policy-preferences.adoc | 4 +-
...rcloud-query-routing-and-read-tolerance.adoc | 85 +++++++++
...olrcloud-recoveries-and-write-tolerance.adoc | 43 +++++
.../src/solrcloud-resilience.adoc | 28 +++
solr/solr-ref-guide/src/solrcloud.adoc | 6 +-
solr/solr-ref-guide/src/spatial-search.adoc | 5 +-
.../src/the-extended-dismax-query-parser.adoc | 21 +--
.../src/the-query-elevation-component.adoc | 12 +-
.../src/the-standard-query-parser.adoc | 6 +-
.../src/update-request-processors.adoc | 2 +-
.../src/updatehandlers-in-solrconfig.adoc | 2 +-
solr/solr-ref-guide/src/working-with-dates.adoc | 6 +-
...rking-with-external-files-and-processes.adoc | 6 +-
.../io/eval/CosineSimilarityEvaluator.java | 67 +++++++
.../solrj/io/eval/DotProductEvaluator.java | 58 ++++++
.../client/solrj/io/eval/EBEAddEvaluator.java | 63 +++++++
.../solrj/io/eval/EBEDivideEvaluator.java | 63 +++++++
.../solrj/io/eval/EBEMultiplyEvaluator.java | 63 +++++++
.../solrj/io/eval/EBESubtractEvaluator.java | 63 +++++++
.../client/solrj/io/stream/FacetStream.java | 10 +-
.../solrj/io/stream/StreamExpressionTest.java | 124 ++++++++++++
111 files changed, 3159 insertions(+), 530 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1fe611d0/solr/solr-ref-guide/src/solrcloud-autoscaling-policy-preferences.adoc
----------------------------------------------------------------------
[50/50] lucene-solr:feature/autoscaling: Merge branch 'master' into
feature/autoscaling
Posted by sh...@apache.org.
Merge branch 'master' into feature/autoscaling
# Conflicts:
# solr/CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/de3107c7
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/de3107c7
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/de3107c7
Branch: refs/heads/feature/autoscaling
Commit: de3107c70f0ca35840496a990c7939f31c8095aa
Parents: 6fb2803 2e5f9a4
Author: Shalin Shekhar Mangar <sh...@apache.org>
Authored: Mon Sep 18 09:23:41 2017 -0700
Committer: Shalin Shekhar Mangar <sh...@apache.org>
Committed: Mon Sep 18 09:23:41 2017 -0700
----------------------------------------------------------------------
build.xml | 18 +-
dev-tools/doap/lucene.rdf | 9 +-
dev-tools/doap/solr.rdf | 7 +
lucene/CHANGES.txt | 20 +
lucene/NOTICE.txt | 5 +-
lucene/analysis/build.xml | 4 +
.../lucene/analysis/bn/BengaliAnalyzer.java | 132 +++
.../analysis/bn/BengaliNormalizationFilter.java | 59 +
.../bn/BengaliNormalizationFilterFactory.java | 55 +
.../lucene/analysis/bn/BengaliNormalizer.java | 155 +++
.../lucene/analysis/bn/BengaliStemFilter.java | 49 +
.../analysis/bn/BengaliStemFilterFactory.java | 48 +
.../lucene/analysis/bn/BengaliStemmer.java | 183 ++++
.../apache/lucene/analysis/bn/package-info.java | 21 +
...ache.lucene.analysis.util.TokenFilterFactory | 2 +
.../org/apache/lucene/analysis/bn/stopwords.txt | 121 +++
.../lucene/analysis/bn/TestBengaliAnalyzer.java | 53 +
.../lucene/analysis/bn/TestBengaliFilters.java | 80 ++
.../analysis/bn/TestBengaliNormalizer.java | 110 ++
.../lucene/analysis/bn/TestBengaliStemmer.java | 79 ++
.../analysis/icu/ICUNormalizer2CharFilter.java | 35 +-
.../icu/TestICUNormalizer2CharFilter.java | 21 +
.../index/TestBackwardsCompatibility.java | 4 +-
.../lucene/index/unsupported.6.6.1-cfs.zip | Bin 0 -> 15836 bytes
.../lucene/index/unsupported.6.6.1-nocfs.zip | Bin 0 -> 15830 bytes
lucene/build.xml | 13 +
lucene/codecs/build.xml | 2 +
lucene/common-build.xml | 53 +-
lucene/core/build.xml | 2 +
.../lucene/index/DefaultIndexingChain.java | 2 -
.../lucene/store/NativeFSLockFactory.java | 14 +-
.../lucene/store/TestNativeFSLockFactory.java | 42 +
.../writercache/LruTaxonomyWriterCache.java | 20 +-
.../writercache/TestLruTaxonomyWriterCache.java | 50 +
lucene/ivy-versions.properties | 2 +-
.../lucene/spatial3d/geom/GeoBBoxFactory.java | 9 +-
.../spatial3d/geom/GeoDegeneratePath.java | 788 ++++++++++++++
.../spatial3d/geom/GeoDegeneratePoint.java | 2 +-
.../apache/lucene/spatial3d/geom/GeoPath.java | 58 +
.../lucene/spatial3d/geom/GeoPathFactory.java | 3 +
.../lucene/spatial3d/geom/GeoStandardPath.java | 170 +++
.../lucene/spatial3d/geom/StandardObjects.java | 1 +
.../lucene/spatial3d/geom/GeoPathTest.java | 75 ++
.../spatial3d/geom/RandomBinaryCodecTest.java | 10 +-
.../geom/RandomGeo3dShapeGenerator.java | 1013 ++++++++++++++++++
.../spatial3d/geom/RandomGeoShapeGenerator.java | 944 ----------------
.../geom/RandomGeoShapeRelationshipTest.java | 65 +-
.../geom/SimpleGeoPolygonRelationshipsTest.java | 24 +
.../suggest/analyzing/AnalyzingSuggester.java | 6 +-
.../analyzing/AnalyzingSuggesterTest.java | 60 ++
solr/CHANGES.txt | 32 +-
solr/build.xml | 16 +-
.../apache/solr/ltr/model/LTRScoringModel.java | 4 +-
.../org/apache/solr/ltr/TestRerankBase.java | 12 +-
solr/core/build.xml | 7 +
.../org/apache/solr/handler/StreamHandler.java | 107 +-
.../solr/handler/component/QueryComponent.java | 460 ++++----
.../org/apache/solr/parser/FastCharStream.java | 7 +-
.../org/apache/solr/parser/QueryParser.java | 2 +-
.../apache/solr/schema/AbstractEnumField.java | 11 +-
.../org/apache/solr/schema/DatePointField.java | 15 +-
.../org/apache/solr/schema/DateRangeField.java | 4 +-
.../apache/solr/schema/DoublePointField.java | 15 +-
.../java/org/apache/solr/schema/FieldType.java | 65 +-
.../org/apache/solr/schema/FloatPointField.java | 15 +-
.../org/apache/solr/schema/IntPointField.java | 15 +-
.../org/apache/solr/schema/LongPointField.java | 15 +-
.../apache/solr/schema/PreAnalyzedField.java | 6 +-
.../java/org/apache/solr/schema/TextField.java | 5 +-
.../java/org/apache/solr/schema/TrieField.java | 44 +-
.../org/apache/solr/search/FunctionQParser.java | 11 +-
.../apache/solr/search/LuceneQParserPlugin.java | 54 -
.../solr/search/OldLuceneQParserPlugin.java | 37 -
.../java/org/apache/solr/search/Sorting.java | 12 +-
.../apache/solr/search/ValueSourceParser.java | 4 +-
.../org/apache/solr/search/facet/MinMaxAgg.java | 145 ++-
.../org/apache/solr/search/facet/SlotAcc.java | 84 +-
.../search/function/FieldNameValueSource.java | 60 ++
.../FieldLengthUpdateProcessorFactory.java | 6 +-
.../FieldMutatingUpdateProcessorFactory.java | 4 +-
.../ParseDateFieldUpdateProcessorFactory.java | 4 +-
.../ParseDoubleFieldUpdateProcessorFactory.java | 4 +-
.../ParseFloatFieldUpdateProcessorFactory.java | 4 +-
.../ParseIntFieldUpdateProcessorFactory.java | 4 +-
.../ParseLongFieldUpdateProcessorFactory.java | 4 +-
.../configuration/providers/package-info.java | 23 +
.../solr/collection1/conf/schema-enums.xml | 10 +
.../solr/collection1/conf/solrconfig.xml | 3 -
.../org/apache/solr/BasicFunctionalityTest.java | 2 +-
.../org/apache/solr/ConvertedLegacyTest.java | 70 +-
.../apache/solr/TestDistributedMissingSort.java | 8 +-
.../apache/solr/cloud/CdcrBootstrapTest.java | 1 +
.../org/apache/solr/core/TestLazyCores.java | 4 +-
.../apache/solr/handler/SearchHandlerTest.java | 4 +-
.../org/apache/solr/handler/TestSQLHandler.java | 1 -
.../org/apache/solr/schema/EnumFieldTest.java | 16 +
.../org/apache/solr/schema/TestPointFields.java | 9 +-
.../org/apache/solr/search/CursorMarkTest.java | 10 +-
.../apache/solr/search/QueryEqualityTest.java | 8 -
.../apache/solr/search/QueryParsingTest.java | 1 -
.../apache/solr/search/TestSolrQueryParser.java | 2 +-
.../test/org/apache/solr/search/TestSort.java | 21 +-
.../solr/search/facet/TestJsonFacets.java | 86 +-
solr/example/example-DIH/.gitignore | 1 +
solr/example/example-DIH/hsqldb/.gitignore | 5 +
solr/example/example-DIH/hsqldb/ex.backup | Bin 11354 -> 0 bytes
solr/example/example-DIH/hsqldb/ex.data | Bin 1048576 -> 0 bytes
solr/example/example-DIH/hsqldb/ex.script | 173 ++-
.../example-DIH/solr/db/conf/managed-schema | 2 +-
.../example-DIH/solr/mail/conf/managed-schema | 2 +-
.../example-DIH/solr/solr/conf/managed-schema | 2 +-
solr/example/files/conf/managed-schema | 8 +-
solr/example/films/README.txt | 6 +-
solr/licenses/commons-math3-3.4.1.jar.sha1 | 1 -
solr/licenses/commons-math3-3.6.1.jar.sha1 | 1 +
solr/solr-ref-guide/README.adoc | 32 +-
solr/solr-ref-guide/src/schema-api.adoc | 4 +-
solr/solr-ref-guide/src/schemaless-mode.adoc | 62 +-
.../src/solrcloud-autoscaling-overview.adoc | 2 +-
...olrcloud-autoscaling-policy-preferences.adoc | 1 +
solr/solr-ref-guide/src/spatial-search.adoc | 5 +-
solr/solr-ref-guide/src/working-with-dates.adoc | 4 +-
solr/solrj/build.xml | 2 +
.../io/eval/BinomialDistributionEvaluator.java | 48 +
.../io/eval/CanberraDistanceEvaluator.java | 56 +
.../io/eval/ChebyshevDistanceEvaluator.java | 56 +
.../solrj/io/eval/CopyOfRangeEvaluator.java | 4 +-
.../io/eval/CosineSimilarityEvaluator.java | 67 ++
.../io/eval/CumulativeProbabilityEvaluator.java | 21 +-
.../solrj/io/eval/DotProductEvaluator.java | 58 +
.../client/solrj/io/eval/EBEAddEvaluator.java | 63 ++
.../solrj/io/eval/EBEDivideEvaluator.java | 63 ++
.../solrj/io/eval/EBEMultiplyEvaluator.java | 63 ++
.../solrj/io/eval/EBESubtractEvaluator.java | 63 ++
.../io/eval/EarthMoversDistanceEvaluator.java | 56 +
.../eval/EnumeratedDistributionEvaluator.java | 45 +
.../solrj/io/eval/FrequencyTableEvaluator.java | 84 ++
.../solrj/io/eval/HistogramEvaluator.java | 7 +-
.../io/eval/KendallsCorrelationEvaluator.java | 57 +
.../io/eval/ManhattanDistanceEvaluator.java | 56 +
.../solrj/io/eval/MeanDifferenceEvaluator.java | 55 +
.../io/eval/PoissonDistributionEvaluator.java | 44 +
.../client/solrj/io/eval/PrimesEvaluator.java | 58 +
.../solrj/io/eval/ProbabilityEvaluator.java | 52 +
.../client/solrj/io/eval/SampleEvaluator.java | 18 +-
.../io/eval/SpearmansCorrelationEvaluator.java | 57 +
.../solrj/io/eval/SumDifferenceEvaluator.java | 55 +
.../UniformIntegerDistributionEvaluator.java | 48 +
.../client/solrj/io/stream/FacetStream.java | 10 +-
.../solrj/io/stream/StreamExpressionTest.java | 357 +++++-
solr/webapp/build.xml | 3 +
151 files changed, 6488 insertions(+), 1720 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/de3107c7/solr/CHANGES.txt
----------------------------------------------------------------------
diff --cc solr/CHANGES.txt
index 50ec4ed,5d035c3..a1a23e6
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@@ -98,8 -83,13 +98,15 @@@ New Feature
* SOLR-11244: Query DSL for Solr (Cao Manh Dat)
+* SOLR-11072: Implement trigger for searchRate event type. (ab)
+
+ * SOLR-11317: JSON Facet API: min/max aggregations on numeric fields are now typed better so int/long
+ fields return an appropriate integral type rather than a double. (yonik)
+
+ * SOLR-11316: JSON Facet API: min/max aggregations are now supported on single-valued date fields.
+ (yonik)
+
+
Bug Fixes
----------------------
@@@ -148,12 -139,12 +161,16 @@@ Optimization
* SOLR-10985: Remove unnecessary toString() calls in solr-core's search package's debug logging.
(Michael Braun via Christine Poerschke)
+* SOLR-11000: Changes made via AutoScalingHandler should be atomic. (ab, shalin)
+
* SOLR-11124: MoveReplicaCmd should skip deleting old replica in case of its node is not live (Cao Manh Dat)
+* SOLR-10769: Allow nodeAdded / nodeLost events to report multiple nodes in one event. (ab)
+
+ * SOLR-11242: QueryParser: re-use the LookaheadSuccess exception. (Michael Braun via David Smiley)
+
+ * SOLR-11314: FastCharStream: re-use the READ_PAST_EOF exception. (Michael Braun via David Smiley)
+
Other Changes
----------------------
@@@ -941,7 -907,9 +967,9 @@@ Other Change
* SOLR-11261, SOLR-10966: Upgrade to Hadoop 2.7.4 to fix incompatibility with Java 9.
(Uwe Schindler)
-
+
+ * SOLR-11324: Clean up mention of trie fields in documentation and source comments. (Steve Rowe)
+
================== 6.6.1 ==================
Bug Fixes
[24/50] lucene-solr:feature/autoscaling: SOLR-11342: Add
sumDifference and meanDifference Stream Evaluators
Posted by sh...@apache.org.
SOLR-11342: Add sumDifference and meanDifference Stream Evaluators
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/c229dc32
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/c229dc32
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/c229dc32
Branch: refs/heads/feature/autoscaling
Commit: c229dc328fb7a68696372fd5e45ef1b404a8e621
Parents: e8dee89
Author: Joel Bernstein <jb...@apache.org>
Authored: Fri Sep 8 21:08:29 2017 -0400
Committer: Joel Bernstein <jb...@apache.org>
Committed: Fri Sep 8 21:08:29 2017 -0400
----------------------------------------------------------------------
.../org/apache/solr/handler/StreamHandler.java | 2 +
.../solrj/io/eval/MeanDifferenceEvaluator.java | 55 ++++++++++++++++++++
.../solrj/io/eval/SumDifferenceEvaluator.java | 55 ++++++++++++++++++++
.../solrj/io/stream/StreamExpressionTest.java | 34 ++++++++++++
4 files changed, 146 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c229dc32/solr/core/src/java/org/apache/solr/handler/StreamHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/StreamHandler.java b/solr/core/src/java/org/apache/solr/handler/StreamHandler.java
index 252def2..f7e59c3 100644
--- a/solr/core/src/java/org/apache/solr/handler/StreamHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/StreamHandler.java
@@ -294,6 +294,8 @@ public class StreamHandler extends RequestHandlerBase implements SolrCoreAware,
.withFunctionName("poissonDistribution", PoissonDistributionEvaluator.class)
.withFunctionName("enumeratedDistribution", EnumeratedDistributionEvaluator.class)
.withFunctionName("probability", ProbabilityEvaluator.class)
+ .withFunctionName("sumDifference", SumDifferenceEvaluator.class)
+ .withFunctionName("meanDifference", MeanDifferenceEvaluator.class)
// Boolean Stream Evaluators
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c229dc32/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/MeanDifferenceEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/MeanDifferenceEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/MeanDifferenceEvaluator.java
new file mode 100644
index 0000000..c93c97a
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/MeanDifferenceEvaluator.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.client.solrj.io.eval;
+
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.util.List;
+import java.util.Locale;
+
+import org.apache.commons.math3.stat.StatUtils;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+
+public class MeanDifferenceEvaluator extends RecursiveNumericEvaluator implements TwoValueWorker {
+ protected static final long serialVersionUID = 1L;
+
+ public MeanDifferenceEvaluator(StreamExpression expression, StreamFactory factory) throws IOException{
+ super(expression, factory);
+ }
+
+ @Override
+ public Object doWork(Object first, Object second) throws IOException{
+ if(null == first){
+ throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - null found for the first value",toExpression(constructingFactory)));
+ }
+ if(null == second){
+ throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - null found for the second value",toExpression(constructingFactory)));
+ }
+ if(!(first instanceof List<?>)){
+ throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - found type %s for the first value, expecting a list of numbers",toExpression(constructingFactory), first.getClass().getSimpleName()));
+ }
+ if(!(second instanceof List<?>)){
+ throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - found type %s for the second value, expecting a list of numbers",toExpression(constructingFactory), first.getClass().getSimpleName()));
+ }
+
+ return StatUtils.meanDifference(
+ ((List) first).stream().mapToDouble(value -> ((BigDecimal) value).doubleValue()).toArray(),
+ ((List) second).stream().mapToDouble(value -> ((BigDecimal) value).doubleValue()).toArray()
+ );
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c229dc32/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/SumDifferenceEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/SumDifferenceEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/SumDifferenceEvaluator.java
new file mode 100644
index 0000000..7402888
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/SumDifferenceEvaluator.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.client.solrj.io.eval;
+
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.util.List;
+import java.util.Locale;
+
+import org.apache.commons.math3.stat.StatUtils;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+
+public class SumDifferenceEvaluator extends RecursiveNumericEvaluator implements TwoValueWorker {
+ protected static final long serialVersionUID = 1L;
+
+ public SumDifferenceEvaluator(StreamExpression expression, StreamFactory factory) throws IOException{
+ super(expression, factory);
+ }
+
+ @Override
+ public Object doWork(Object first, Object second) throws IOException{
+ if(null == first){
+ throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - null found for the first value",toExpression(constructingFactory)));
+ }
+ if(null == second){
+ throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - null found for the second value",toExpression(constructingFactory)));
+ }
+ if(!(first instanceof List<?>)){
+ throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - found type %s for the first value, expecting a list of numbers",toExpression(constructingFactory), first.getClass().getSimpleName()));
+ }
+ if(!(second instanceof List<?>)){
+ throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - found type %s for the second value, expecting a list of numbers",toExpression(constructingFactory), first.getClass().getSimpleName()));
+ }
+
+ return StatUtils.sumDifference(
+ ((List) first).stream().mapToDouble(value -> ((BigDecimal) value).doubleValue()).toArray(),
+ ((List) second).stream().mapToDouble(value -> ((BigDecimal) value).doubleValue()).toArray()
+ );
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c229dc32/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
index 01d6577..2ee881e 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
@@ -6073,6 +6073,40 @@ public class StreamExpressionTest extends SolrCloudTestCase {
}
}
+
+ @Test
+ public void testSumDifference() throws Exception {
+ String cexpr = "sumDifference(array(2,4,6,8,10,12),array(1,2,3,4,5,6))";
+ ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", cexpr);
+ paramsLoc.set("qt", "/stream");
+ String url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString()+"/"+COLLECTIONORALIAS;
+ TupleStream solrStream = new SolrStream(url, paramsLoc);
+ StreamContext context = new StreamContext();
+ solrStream.setStreamContext(context);
+ List<Tuple> tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+ double sd = tuples.get(0).getDouble("return-value");
+ assertEquals(sd, 21.0D, 0.0);
+ }
+
+ @Test
+ public void testMeanDifference() throws Exception {
+ String cexpr = "meanDifference(array(2,4,6,8,10,12),array(1,2,3,4,5,6))";
+ ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", cexpr);
+ paramsLoc.set("qt", "/stream");
+ String url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString()+"/"+COLLECTIONORALIAS;
+ TupleStream solrStream = new SolrStream(url, paramsLoc);
+ StreamContext context = new StreamContext();
+ solrStream.setStreamContext(context);
+ List<Tuple> tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+ double sd = tuples.get(0).getDouble("return-value");
+ assertEquals(sd, 3.5, 0.0);
+ }
+
+
@Test
public void testEBESubtract() throws Exception {
String cexpr = "ebeSubtract(array(2,4,6,8,10,12),array(1,2,3,4,5,6))";
[44/50] lucene-solr:feature/autoscaling: LUCENE-7906: Fix random
shape generator. Committed on behalf of Ignacio Vera.
Posted by sh...@apache.org.
LUCENE-7906: Fix random shape generator. Committed on behalf of Ignacio Vera.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/12b0acdf
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/12b0acdf
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/12b0acdf
Branch: refs/heads/feature/autoscaling
Commit: 12b0acdf7c03a905151300c9b635c23e9453e26d
Parents: b125107
Author: Karl Wright <Da...@gmail.com>
Authored: Thu Sep 14 03:51:54 2017 -0400
Committer: Karl Wright <Da...@gmail.com>
Committed: Thu Sep 14 03:51:54 2017 -0400
----------------------------------------------------------------------
.../geom/RandomGeo3dShapeGenerator.java | 20 +++++++-------------
1 file changed, 7 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/12b0acdf/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/RandomGeo3dShapeGenerator.java
----------------------------------------------------------------------
diff --git a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/RandomGeo3dShapeGenerator.java b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/RandomGeo3dShapeGenerator.java
index 72b7109..d41bf1b 100644
--- a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/RandomGeo3dShapeGenerator.java
+++ b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/RandomGeo3dShapeGenerator.java
@@ -174,13 +174,13 @@ public class RandomGeo3dShapeGenerator extends LuceneTestCase {
public GeoPoint randomGeoPoint(PlanetModel planetModel, Constraints constraints) {
int iterations = 0;
while (iterations < MAX_POINT_ITERATIONS) {
- double lat = randomDouble();
- if (Math.PI/2 - Math.abs(lat) <0){
- continue;
+ double lat = randomDouble() * Math.PI/2;
+ if (random().nextBoolean()) {
+ lat = (-1)*lat;
}
- double lon = randomDouble();
- if (Math.PI - Math.abs(lat) <0){
- continue;
+ double lon = randomDouble() * Math.PI;
+ if (random().nextBoolean()) {
+ lon = (-1)*lon;
}
iterations++;
GeoPoint point = new GeoPoint(planetModel, lat, lon);
@@ -810,13 +810,7 @@ public class RandomGeo3dShapeGenerator extends LuceneTestCase {
* @return the cutoff angle.
*/
private double randomCutoffAngle() {
- while(true) {
- double radius = randomDouble();
- if (radius <0 || radius > Math.PI){
- continue;
- }
- return radius;
- }
+ return randomDouble() * Math.PI;
}
/**
[27/50] lucene-solr:feature/autoscaling: LUCENE-7959: Improve
NativeFSLockFactory's NoSuchFileException
Posted by sh...@apache.org.
LUCENE-7959: Improve NativeFSLockFactory's NoSuchFileException
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/0283d3e2
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/0283d3e2
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/0283d3e2
Branch: refs/heads/feature/autoscaling
Commit: 0283d3e28a6dea083060cb6932bca82a9b8fefb2
Parents: 8452edb
Author: Robert Muir <rm...@apache.org>
Authored: Sat Sep 9 02:00:47 2017 -0400
Committer: Robert Muir <rm...@apache.org>
Committed: Sat Sep 9 02:02:19 2017 -0400
----------------------------------------------------------------------
lucene/CHANGES.txt | 4 ++
.../lucene/store/NativeFSLockFactory.java | 14 ++++++-
.../lucene/store/TestNativeFSLockFactory.java | 42 ++++++++++++++++++++
3 files changed, 59 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0283d3e2/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index a2fd223..555c97d 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -57,6 +57,10 @@ Bug Fixes
* LUCENE-7891: Lucene's taxonomy facets now uses a non-buggy LRU cache
by default. (Jan-Willem van den Broek via Mike McCandless)
+* LUCENE-7959: Improve NativeFSLockFactory's exception message if it cannot create
+ write.lock for an empty index due to bad permissions/read-only filesystem/etc.
+ (Erick Erickson, Shawn Heisey, Robert Muir)
+
Build
* SOLR-11181: Switch order of maven artifact publishing procedure: deploy first
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0283d3e2/lucene/core/src/java/org/apache/lucene/store/NativeFSLockFactory.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/store/NativeFSLockFactory.java b/lucene/core/src/java/org/apache/lucene/store/NativeFSLockFactory.java
index 4f17d95..106b02e 100644
--- a/lucene/core/src/java/org/apache/lucene/store/NativeFSLockFactory.java
+++ b/lucene/core/src/java/org/apache/lucene/store/NativeFSLockFactory.java
@@ -93,15 +93,27 @@ public final class NativeFSLockFactory extends FSLockFactory {
Path lockFile = lockDir.resolve(lockName);
+ IOException creationException = null;
try {
Files.createFile(lockFile);
} catch (IOException ignore) {
// we must create the file to have a truly canonical path.
// if it's already created, we don't care. if it cant be created, it will fail below.
+ creationException = ignore;
}
// fails if the lock file does not exist
- final Path realPath = lockFile.toRealPath();
+ final Path realPath;
+ try {
+ realPath = lockFile.toRealPath();
+ } catch (IOException e) {
+ // if we couldn't resolve the lock file, it might be because we couldn't create it.
+ // so append any exception from createFile as a suppressed exception, in case its useful
+ if (creationException != null) {
+ e.addSuppressed(creationException);
+ }
+ throw e;
+ }
// used as a best-effort check, to see if the underlying file has changed
final FileTime creationTime = Files.readAttributes(realPath, BasicFileAttributes.class).creationTime();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0283d3e2/lucene/core/src/test/org/apache/lucene/store/TestNativeFSLockFactory.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/store/TestNativeFSLockFactory.java b/lucene/core/src/test/org/apache/lucene/store/TestNativeFSLockFactory.java
index 5d202bf..dc77178 100644
--- a/lucene/core/src/test/org/apache/lucene/store/TestNativeFSLockFactory.java
+++ b/lucene/core/src/test/org/apache/lucene/store/TestNativeFSLockFactory.java
@@ -18,9 +18,17 @@ package org.apache.lucene.store;
import java.io.IOException;
+import java.nio.channels.SeekableByteChannel;
+import java.nio.file.AccessDeniedException;
+import java.nio.file.FileSystem;
import java.nio.file.Files;
+import java.nio.file.OpenOption;
import java.nio.file.Path;
+import java.nio.file.attribute.FileAttribute;
+import java.util.Set;
+import org.apache.lucene.mockfile.FilterFileSystemProvider;
+import org.apache.lucene.mockfile.FilterPath;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.TestUtil;
@@ -89,4 +97,38 @@ public class TestNativeFSLockFactory extends BaseLockFactoryTestCase {
IOUtils.closeWhileHandlingException(lock);
}
}
+
+ /** MockFileSystem that throws AccessDeniedException on creating test.lock */
+ static class MockBadPermissionsFileSystem extends FilterFileSystemProvider {
+ public MockBadPermissionsFileSystem(FileSystem delegateInstance) {
+ super("mockbadpermissions://", delegateInstance);
+ }
+
+ @Override
+ public SeekableByteChannel newByteChannel(Path path, Set<? extends OpenOption> options, FileAttribute<?>... attrs) throws IOException {
+ if (path.getFileName().toString().equals("test.lock")) {
+ throw new AccessDeniedException(path.toString(), null, "fake access denied");
+ }
+ return super.newByteChannel(path, options, attrs);
+ }
+ }
+
+ public void testBadPermissions() throws IOException {
+ // create a mock filesystem that will throw exc on creating test.lock
+ Path tmpDir = createTempDir();
+ tmpDir = FilterPath.unwrap(tmpDir).toRealPath();
+ FileSystem mock = new MockBadPermissionsFileSystem(tmpDir.getFileSystem()).getFileSystem(null);
+ Path mockPath = mock.getPath(tmpDir.toString());
+
+ // we should get an IOException (typically NoSuchFileException but no guarantee) with
+ // our fake AccessDenied added as suppressed.
+ Directory dir = getDirectory(mockPath.resolve("indexDir"));
+ IOException expected = expectThrows(IOException.class, () -> {
+ dir.obtainLock("test.lock");
+ });
+ AccessDeniedException suppressed = (AccessDeniedException) expected.getSuppressed()[0];
+ assertTrue(suppressed.getMessage().contains("fake access denied"));
+
+ dir.close();
+ }
}
[06/50] lucene-solr:feature/autoscaling: SOLR-10783 Add support for
Hadoop Credential Provider as SSL/TLS store password source.
Posted by sh...@apache.org.
SOLR-10783 Add support for Hadoop Credential Provider as SSL/TLS store password source.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/b4d280f3
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/b4d280f3
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/b4d280f3
Branch: refs/heads/feature/autoscaling
Commit: b4d280f369023a179e98154535ed4b06ea096f5f
Parents: ba9504d
Author: Mark Miller <ma...@apache.org>
Authored: Wed Sep 6 22:33:12 2017 -0500
Committer: Mark Miller <ma...@apache.org>
Committed: Wed Sep 6 22:33:12 2017 -0500
----------------------------------------------------------------------
solr/CHANGES.txt | 2 +
solr/bin/solr | 6 +
solr/bin/solr.cmd | 15 +-
solr/bin/solr.in.cmd | 11 ++
solr/bin/solr.in.sh | 11 ++
.../util/configuration/SSLConfigurations.java | 102 +++++++---
.../configuration/SSLConfigurationsFactory.java | 2 +-
.../configuration/SSLCredentialProvider.java | 36 ++++
.../SSLCredentialProviderFactory.java | 93 +++++++++
.../AbstractSSLCredentialProvider.java | 56 ++++++
.../providers/EnvSSLCredentialProvider.java | 68 +++++++
.../providers/HadoopSSLCredentialProvider.java | 66 +++++++
.../providers/SysPropSSLCredentialProvider.java | 38 ++++
.../configuration/SSLConfigurationsTest.java | 193 ++++++++++++++++---
.../SSLCredentialProviderFactoryTest.java | 90 +++++++++
.../providers/EnvSSLCredentialProviderTest.java | 52 +++++
.../HadoopSSLCredentialProviderTest.java | 72 +++++++
.../SysPropSSLCredentialProviderTest.java | 55 ++++++
solr/server/etc/jetty-ssl.xml | 8 +-
19 files changed, 920 insertions(+), 56 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b4d280f3/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index be89fb1..dcfb15e 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -89,6 +89,8 @@ New Features
* SOLR-11316: JSON Facet API: min/max aggregations are now supported on single-valued date fields.
(yonik)
+* SOLR-10783 Add support for Hadoop Credential Provider as SSL/TLS store password source.
+ (Mano Kovacs via Mark Miller)
Bug Fixes
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b4d280f3/solr/bin/solr
----------------------------------------------------------------------
diff --git a/solr/bin/solr b/solr/bin/solr
index bc2abca..29af3f4 100755
--- a/solr/bin/solr
+++ b/solr/bin/solr
@@ -163,6 +163,11 @@ fi
SOLR_URL_SCHEME=http
SOLR_JETTY_CONFIG=()
SOLR_SSL_OPTS=""
+
+if [ -n "$SOLR_HADOOP_CREDENTIAL_PROVIDER_PATH" ]; then
+ SOLR_SSL_OPTS+=" -Dhadoop.security.credential.provider.path=$SOLR_HADOOP_CREDENTIAL_PROVIDER_PATH"
+fi
+
if [ -z "$SOLR_SSL_ENABLED" ]; then
if [ -n "$SOLR_SSL_KEY_STORE" ]; then
SOLR_SSL_ENABLED="true" # implicitly from earlier behaviour
@@ -172,6 +177,7 @@ if [ -z "$SOLR_SSL_ENABLED" ]; then
fi
if [ "$SOLR_SSL_ENABLED" == "true" ]; then
SOLR_JETTY_CONFIG+=("--module=https")
+ SOLR_JETTY_CONFIG+=("--lib=$DEFAULT_SERVER_DIR/solr-webapp/webapp/WEB-INF/lib/*")
SOLR_URL_SCHEME=https
if [ -n "$SOLR_SSL_KEY_STORE" ]; then
SOLR_SSL_OPTS+=" -Dsolr.jetty.keystore=$SOLR_SSL_KEY_STORE"
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b4d280f3/solr/bin/solr.cmd
----------------------------------------------------------------------
diff --git a/solr/bin/solr.cmd b/solr/bin/solr.cmd
index 57e2e62..b5b1d78 100644
--- a/solr/bin/solr.cmd
+++ b/solr/bin/solr.cmd
@@ -38,11 +38,17 @@ REM command line args
IF "%SOLR_INCLUDE%"=="" set "SOLR_INCLUDE=%SOLR_TIP%\bin\solr.in.cmd"
IF EXIST "%SOLR_INCLUDE%" CALL "%SOLR_INCLUDE%"
+set "DEFAULT_SERVER_DIR=%SOLR_TIP%\server"
+
REM Select HTTP OR HTTPS related configurations
set SOLR_URL_SCHEME=http
set "SOLR_JETTY_CONFIG=--module=http"
set "SOLR_SSL_OPTS= "
+IF DEFINED SOLR_HADOOP_CREDENTIAL_PROVIDER_PATH (
+ set "SOLR_SSL_OPTS=!SOLR_SSL_OPTS! -Dhadoop.security.credential.provider.path=%SOLR_HADOOP_CREDENTIAL_PROVIDER_PATH%"
+)
+
IF NOT DEFINED SOLR_SSL_ENABLED (
IF DEFINED SOLR_SSL_KEY_STORE (
set "SOLR_SSL_ENABLED=true"
@@ -50,8 +56,9 @@ IF NOT DEFINED SOLR_SSL_ENABLED (
set "SOLR_SSL_ENABLED=false"
)
)
+
IF "%SOLR_SSL_ENABLED%"=="true" (
- set "SOLR_JETTY_CONFIG=--module=https"
+ set "SOLR_JETTY_CONFIG=--lib="%DEFAULT_SERVER_DIR%\solr-webapp\webapp\WEB-INF\lib\*" --module=https"
set SOLR_URL_SCHEME=https
IF DEFINED SOLR_SSL_KEY_STORE (
set "SOLR_SSL_OPTS=!SOLR_SSL_OPTS! -Dsolr.jetty.keystore=%SOLR_SSL_KEY_STORE%"
@@ -182,8 +189,6 @@ IF !JAVA_MAJOR_VERSION! LSS 8 (
goto err
)
-set "DEFAULT_SERVER_DIR=%SOLR_TIP%\server"
-
set FIRST_ARG=%1
IF [%1]==[] goto usage
@@ -995,7 +1000,7 @@ IF "%SCRIPT_CMD%"=="stop" (
set found_it=1
@echo Stopping Solr process %%N running on port %SOLR_PORT%
set /A STOP_PORT=%SOLR_PORT% - 1000
- "%JAVA%" %SOLR_SSL_OPTS% -Djetty.home="%SOLR_SERVER_DIR%" -jar "%SOLR_SERVER_DIR%\start.jar" "%SOLR_JETTY_CONFIG%" STOP.PORT=!STOP_PORT! STOP.KEY=%STOP_KEY% --stop
+ "%JAVA%" %SOLR_SSL_OPTS% -Djetty.home="%SOLR_SERVER_DIR%" -jar "%SOLR_SERVER_DIR%\start.jar" %SOLR_JETTY_CONFIG% STOP.PORT=!STOP_PORT! STOP.KEY=%STOP_KEY% --stop
del "%SOLR_TIP%"\bin\solr-%SOLR_PORT%.port
timeout /T 5
REM Kill it if it is still running after the graceful shutdown
@@ -1243,7 +1248,7 @@ IF "%FG%"=="1" (
-Dlog4j.configuration="%LOG4J_CONFIG%" -DSTOP.PORT=!STOP_PORT! -DSTOP.KEY=%STOP_KEY% ^
-Dsolr.solr.home="%SOLR_HOME%" -Dsolr.install.dir="%SOLR_TIP%" -Dsolr.default.confdir="%DEFAULT_CONFDIR%" ^
-Djetty.host=%SOLR_JETTY_HOST% -Djetty.port=%SOLR_PORT% -Djetty.home="%SOLR_SERVER_DIR%" ^
- -Djava.io.tmpdir="%SOLR_SERVER_DIR%\tmp" -jar start.jar "%SOLR_JETTY_CONFIG%" "%SOLR_JETTY_ADDL_CONFIG%"
+ -Djava.io.tmpdir="%SOLR_SERVER_DIR%\tmp" -jar start.jar %SOLR_JETTY_CONFIG% "%SOLR_JETTY_ADDL_CONFIG%"
) ELSE (
START /B "Solr-%SOLR_PORT%" /D "%SOLR_SERVER_DIR%" ^
"%JAVA%" %SERVEROPT% %SOLR_JAVA_MEM% %START_OPTS% ^
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b4d280f3/solr/bin/solr.in.cmd
----------------------------------------------------------------------
diff --git a/solr/bin/solr.in.cmd b/solr/bin/solr.in.cmd
index afba4f7..99a6d79 100644
--- a/solr/bin/solr.in.cmd
+++ b/solr/bin/solr.in.cmd
@@ -116,6 +116,17 @@ REM set SOLR_SSL_CLIENT_TRUST_STORE=
REM set SOLR_SSL_CLIENT_TRUST_STORE_PASSWORD=
REM set SOLR_SSL_CLIENT_TRUST_STORE_TYPE=
+REM Sets path of Hadoop credential provider (hadoop.security.credential.provider.path property) and
+REM enables usage of credential store.
+REM Credential provider should store the following keys:
+REM * solr.jetty.keystore.password
+REM * solr.jetty.truststore.password
+REM Set the two below if you want to set specific store passwords for HTTP client
+REM * javax.net.ssl.keyStorePassword
+REM * javax.net.ssl.trustStorePassword
+REM More info: https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/CredentialProviderAPI.html
+REM set SOLR_HADOOP_CREDENTIAL_PROVIDER_PATH=localjceks://file/home/solr/hadoop-credential-provider.jceks
+
REM Settings for authentication
REM Please configure only one of SOLR_AUTHENTICATION_CLIENT_BUILDER or SOLR_AUTH_TYPE parameters
REM set SOLR_AUTHENTICATION_CLIENT_BUILDER=org.apache.solr.client.solrj.impl.PreemptiveBasicAuthClientBuilderFactory
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b4d280f3/solr/bin/solr.in.sh
----------------------------------------------------------------------
diff --git a/solr/bin/solr.in.sh b/solr/bin/solr.in.sh
index 514efb1..22c5bf7 100644
--- a/solr/bin/solr.in.sh
+++ b/solr/bin/solr.in.sh
@@ -133,6 +133,17 @@
#SOLR_SSL_CLIENT_TRUST_STORE_PASSWORD=
#SOLR_SSL_CLIENT_TRUST_STORE_TYPE=
+# Sets path of Hadoop credential provider (hadoop.security.credential.provider.path property) and
+# enables usage of credential store.
+# Credential provider should store the following keys:
+# * solr.jetty.keystore.password
+# * solr.jetty.truststore.password
+# Set the two below if you want to set specific store passwords for HTTP client
+# * javax.net.ssl.keyStorePassword
+# * javax.net.ssl.trustStorePassword
+# More info: https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/CredentialProviderAPI.html
+#SOLR_HADOOP_CREDENTIAL_PROVIDER_PATH=localjceks://file/home/solr/hadoop-credential-provider.jceks
+
# Settings for authentication
# Please configure only one of SOLR_AUTHENTICATION_CLIENT_BUILDER or SOLR_AUTH_TYPE parameters
#SOLR_AUTHENTICATION_CLIENT_BUILDER="org.apache.solr.client.solrj.impl.PreemptiveBasicAuthClientBuilderFactory"
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b4d280f3/solr/core/src/java/org/apache/solr/util/configuration/SSLConfigurations.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/util/configuration/SSLConfigurations.java b/solr/core/src/java/org/apache/solr/util/configuration/SSLConfigurations.java
index bfa5186..30edf57 100644
--- a/solr/core/src/java/org/apache/solr/util/configuration/SSLConfigurations.java
+++ b/solr/core/src/java/org/apache/solr/util/configuration/SSLConfigurations.java
@@ -18,7 +18,7 @@
package org.apache.solr.util.configuration;
import java.lang.invoke.MethodHandles;
-import java.util.Map;
+import java.util.List;
import org.apache.solr.common.StringUtils;
import org.slf4j.Logger;
@@ -28,48 +28,102 @@ import org.slf4j.LoggerFactory;
* Dedicated object to handle Solr configurations
*/
public class SSLConfigurations {
- private final Map<String, String> envVars;
-
+ public static final String DEFAULT_STORE_PASSWORD = "secret";
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ protected final List<SSLCredentialProvider> credentialProviders;
+
public static class SysProps {
- public static final String SSL_KEY_STORE_PASSWORD = "javax.net.ssl.keyStorePassword";
- public static final String SSL_TRUST_STORE_PASSWORD = "javax.net.ssl.trustStorePassword";
+ public static final String SSL_KEY_STORE_PASSWORD = "solr.jetty.keystore.password";
+ public static final String SSL_TRUST_STORE_PASSWORD = "solr.jetty.truststore.password";
+ public static final String SSL_CLIENT_KEY_STORE_PASSWORD = "javax.net.ssl.keyStorePassword";
+ public static final String SSL_CLIENT_TRUST_STORE_PASSWORD = "javax.net.ssl.trustStorePassword";
}
- public static class EnvVars {
- public static final String SOLR_SSL_CLIENT_KEY_STORE_PASSWORD = "SOLR_SSL_CLIENT_KEY_STORE_PASSWORD";
- public static final String SOLR_SSL_KEY_STORE_PASSWORD = "SOLR_SSL_KEY_STORE_PASSWORD";
- public static final String SOLR_SSL_CLIENT_TRUST_STORE_PASSWORD = "SOLR_SSL_CLIENT_TRUST_STORE_PASSWORD";
- public static final String SOLR_SSL_TRUST_STORE_PASSWORD = "SOLR_SSL_TRUST_STORE_PASSWORD";
+ /**
+ * @param sslCredentialProviderFactory Credential provider factory to use for providers
+ */
+ public SSLConfigurations(SSLCredentialProviderFactory sslCredentialProviderFactory) {
+ credentialProviders = sslCredentialProviderFactory.getProviders();
}
/**
- * @param envVars Map of environment variables to use
+ * @param credentialProviders Explicit list of credential providers to use
*/
- public SSLConfigurations(Map<String, String> envVars) {
- this.envVars = envVars;
+ public SSLConfigurations(List<SSLCredentialProvider> credentialProviders) {
+ this.credentialProviders = credentialProviders;
}
- /** Initiates javax.net.ssl.* system properties from the proper sources. */
+ /**
+ * Initiates javax.net.ssl.* system properties from the proper sources.
+ */
public void init() {
- String clientKeystorePassword = envVars.get(EnvVars.SOLR_SSL_CLIENT_KEY_STORE_PASSWORD);
- String keystorePassword = envVars.get(EnvVars.SOLR_SSL_KEY_STORE_PASSWORD);
+ String clientKeystorePassword = getClientKeyStorePassword();
+ String keystorePassword = getKeyStorePassword();
- String clientTruststorePassword = envVars.get(EnvVars.SOLR_SSL_CLIENT_TRUST_STORE_PASSWORD);
- String truststorePassword = envVars.get(EnvVars.SOLR_SSL_TRUST_STORE_PASSWORD);
+ String clientTruststorePassword = getClientTrustStorePassword();
+ String truststorePassword = getTrustStorePassword();
- if (isEmpty(System.getProperty(SysProps.SSL_KEY_STORE_PASSWORD))
+ if (isEmpty(System.getProperty(SysProps.SSL_CLIENT_KEY_STORE_PASSWORD))
&& !(isEmpty(clientKeystorePassword) && isEmpty(keystorePassword))) {
- log.debug("Setting {} based on env var", SysProps.SSL_KEY_STORE_PASSWORD);
- System.setProperty(SysProps.SSL_KEY_STORE_PASSWORD, clientKeystorePassword != null ? clientKeystorePassword : keystorePassword);
+ log.info("Setting {}", SysProps.SSL_CLIENT_KEY_STORE_PASSWORD);
+ System.setProperty(SysProps.SSL_CLIENT_KEY_STORE_PASSWORD, clientKeystorePassword != null ? clientKeystorePassword : keystorePassword);
}
- if (isEmpty(System.getProperty(SysProps.SSL_TRUST_STORE_PASSWORD))
+ if (isEmpty(System.getProperty(SysProps.SSL_CLIENT_TRUST_STORE_PASSWORD))
&& !(isEmpty(clientTruststorePassword) && isEmpty(truststorePassword))) {
- log.debug("Setting {} based on env var", SysProps.SSL_TRUST_STORE_PASSWORD);
- System.setProperty(SysProps.SSL_TRUST_STORE_PASSWORD, clientTruststorePassword != null ? clientTruststorePassword : truststorePassword);
+ log.info("Setting {}", SysProps.SSL_CLIENT_TRUST_STORE_PASSWORD);
+ System.setProperty(SysProps.SSL_CLIENT_TRUST_STORE_PASSWORD, clientTruststorePassword != null ? clientTruststorePassword : truststorePassword);
+ }
+
+ }
+
+ /**
+ * @return password for keystore used for SSL connections
+ */
+ public String getKeyStorePassword() {
+ String keyStorePassword = getPassword(SSLCredentialProvider.CredentialType.SSL_KEY_STORE_PASSWORD);
+ if (isEmpty(keyStorePassword)) {
+ log.warn("No keystore password found, using default.");
+ keyStorePassword = DEFAULT_STORE_PASSWORD; // default value
+ }
+ return keyStorePassword;
+ }
+
+ /**
+ * @return password for truststore used for SSL connections
+ */
+ public String getTrustStorePassword() {
+ String trustStorePassword = getPassword(SSLCredentialProvider.CredentialType.SSL_TRUST_STORE_PASSWORD);
+ if (isEmpty(trustStorePassword)) {
+ log.warn("No truststore password found, using default.");
+ trustStorePassword = DEFAULT_STORE_PASSWORD; // default value
+ }
+ return trustStorePassword;
+ }
+
+ /**
+ * @return password for keystore used for SSL client connections
+ */
+ public String getClientKeyStorePassword() {
+ String keyStorePassword = getPassword(SSLCredentialProvider.CredentialType.SSL_CLIENT_KEY_STORE_PASSWORD);
+ return keyStorePassword;
+ }
+
+ /**
+ * @return password for truststore used for SSL client connections
+ */
+ public String getClientTrustStorePassword() {
+ String trustStorePassword = getPassword(SSLCredentialProvider.CredentialType.SSL_CLIENT_TRUST_STORE_PASSWORD);
+ return trustStorePassword;
+ }
+
+ protected String getPassword(SSLCredentialProvider.CredentialType type) {
+ for(SSLCredentialProvider provider: credentialProviders){
+ String credential = provider.getCredential(type);
+ if(credential != null) return credential;
}
+ return null;
}
private boolean isEmpty(String str) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b4d280f3/solr/core/src/java/org/apache/solr/util/configuration/SSLConfigurationsFactory.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/util/configuration/SSLConfigurationsFactory.java b/solr/core/src/java/org/apache/solr/util/configuration/SSLConfigurationsFactory.java
index 1da7c01..80571ef 100644
--- a/solr/core/src/java/org/apache/solr/util/configuration/SSLConfigurationsFactory.java
+++ b/solr/core/src/java/org/apache/solr/util/configuration/SSLConfigurationsFactory.java
@@ -39,7 +39,7 @@ public class SSLConfigurationsFactory {
}
private static SSLConfigurations getInstance() {
- return new SSLConfigurations(System.getenv());
+ return new SSLConfigurations(new SSLCredentialProviderFactory());
}
@VisibleForTesting
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b4d280f3/solr/core/src/java/org/apache/solr/util/configuration/SSLCredentialProvider.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/util/configuration/SSLCredentialProvider.java b/solr/core/src/java/org/apache/solr/util/configuration/SSLCredentialProvider.java
new file mode 100644
index 0000000..f714d28
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/util/configuration/SSLCredentialProvider.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.util.configuration;
+
+/**
+ * Interface for different source of SSL configurations.
+ */
+public interface SSLCredentialProvider {
+ enum CredentialType {
+ SSL_KEY_STORE_PASSWORD,
+ SSL_TRUST_STORE_PASSWORD,
+ SSL_CLIENT_KEY_STORE_PASSWORD,
+ SSL_CLIENT_TRUST_STORE_PASSWORD
+ }
+
+ /**
+ * @return Credential for the given credential type
+ */
+ String getCredential(CredentialType type);
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b4d280f3/solr/core/src/java/org/apache/solr/util/configuration/SSLCredentialProviderFactory.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/util/configuration/SSLCredentialProviderFactory.java b/solr/core/src/java/org/apache/solr/util/configuration/SSLCredentialProviderFactory.java
new file mode 100644
index 0000000..1fcfd96
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/util/configuration/SSLCredentialProviderFactory.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.util.configuration;
+
+import java.lang.invoke.MethodHandles;
+import java.lang.reflect.InvocationTargetException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+
+import com.google.common.collect.ImmutableMap;
+import org.apache.solr.util.configuration.providers.EnvSSLCredentialProvider;
+import org.apache.solr.util.configuration.providers.HadoopSSLCredentialProvider;
+import org.apache.solr.util.configuration.providers.SysPropSSLCredentialProvider;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Class responsible to build SSL credential providers
+ */
+public class SSLCredentialProviderFactory {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ public static final String DEFAULT_PROVIDER_CHAIN = "env;sysprop";
+ public static final String PROVIDER_CHAIN_KEY = "solr.ssl.credential.provider.chain";
+
+ private final static ImmutableMap<String, Class> defaultProviders = ImmutableMap.of(
+ "env", EnvSSLCredentialProvider.class,
+ "sysprop", SysPropSSLCredentialProvider.class,
+ "hadoop", HadoopSSLCredentialProvider.class
+ );
+
+ private String providerChain;
+
+ public SSLCredentialProviderFactory() {
+ this.providerChain = System.getProperty(PROVIDER_CHAIN_KEY, DEFAULT_PROVIDER_CHAIN);
+ }
+
+ public SSLCredentialProviderFactory(String providerChain) {
+ this.providerChain = providerChain;
+ }
+
+ public List<SSLCredentialProvider> getProviders() {
+ ArrayList<SSLCredentialProvider> providers = new ArrayList<>();
+ log.info(String.format(Locale.ROOT, "Processing SSL Credential Provider chain: %s", providerChain));
+ String classPrefix = "class://";
+ for (String provider : providerChain.split(";")) {
+ if (defaultProviders.containsKey(provider)) {
+ providers.add(getDefaultProvider(defaultProviders.get(provider)));
+ } else if (provider.startsWith(classPrefix)) {
+ providers.add(getProviderByClassName(provider.substring(classPrefix.length())));
+ } else {
+ throw new RuntimeException("Unable to parse credential provider: " + provider);
+ }
+ }
+ return providers;
+ }
+
+ private SSLCredentialProvider getProviderByClassName(String clazzName) {
+ try {
+ return (SSLCredentialProvider) Class.forName(clazzName).getConstructor().newInstance();
+ } catch (InstantiationException | ClassNotFoundException | IllegalAccessException | InvocationTargetException | NoSuchMethodException e) {
+ String msg = String.format(Locale.ROOT, "Could not instantiate %s credential provider", clazzName);
+ log.error(msg);
+ throw new RuntimeException(msg, e);
+ }
+ }
+
+ private SSLCredentialProvider getDefaultProvider(Class aClass) {
+ try {
+ return (SSLCredentialProvider) aClass.getConstructor().newInstance();
+ } catch (InstantiationException | IllegalAccessException | InvocationTargetException | NoSuchMethodException e) {
+ String msg = String.format(Locale.ROOT, "Could not instantiate %s credential provider", aClass.getName());
+ log.error(msg);
+ throw new RuntimeException(msg, e);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b4d280f3/solr/core/src/java/org/apache/solr/util/configuration/providers/AbstractSSLCredentialProvider.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/util/configuration/providers/AbstractSSLCredentialProvider.java b/solr/core/src/java/org/apache/solr/util/configuration/providers/AbstractSSLCredentialProvider.java
new file mode 100644
index 0000000..9825154
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/util/configuration/providers/AbstractSSLCredentialProvider.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.util.configuration.providers;
+
+import java.util.EnumMap;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Maps;
+import org.apache.solr.util.configuration.SSLCredentialProvider;
+
+import static org.apache.solr.util.configuration.SSLCredentialProvider.CredentialType.SSL_CLIENT_KEY_STORE_PASSWORD;
+import static org.apache.solr.util.configuration.SSLCredentialProvider.CredentialType.SSL_CLIENT_TRUST_STORE_PASSWORD;
+import static org.apache.solr.util.configuration.SSLCredentialProvider.CredentialType.SSL_KEY_STORE_PASSWORD;
+import static org.apache.solr.util.configuration.SSLCredentialProvider.CredentialType.SSL_TRUST_STORE_PASSWORD;
+
+/**
+ * Abstract provider with default implementation
+ */
+abstract public class AbstractSSLCredentialProvider implements SSLCredentialProvider {
+ public static final EnumMap<CredentialType, String> DEFAULT_CREDENTIAL_KEY_MAP = Maps.newEnumMap(ImmutableMap.of(
+ SSL_KEY_STORE_PASSWORD, "solr.jetty.keystore.password",
+ SSL_TRUST_STORE_PASSWORD, "solr.jetty.truststore.password",
+ SSL_CLIENT_KEY_STORE_PASSWORD, "javax.net.ssl.keyStorePassword",
+ SSL_CLIENT_TRUST_STORE_PASSWORD, "javax.net.ssl.trustStorePassword"
+ ));
+ private final EnumMap<CredentialType, String> credentialKeyMap;
+
+ public AbstractSSLCredentialProvider() {
+ credentialKeyMap = getCredentialKeyMap();
+ }
+
+ abstract protected String getCredential(String key);
+
+ abstract protected EnumMap<CredentialType, String> getCredentialKeyMap();
+
+ @Override
+ public String getCredential(CredentialType type) {
+ return getCredential(credentialKeyMap.get(type));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b4d280f3/solr/core/src/java/org/apache/solr/util/configuration/providers/EnvSSLCredentialProvider.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/util/configuration/providers/EnvSSLCredentialProvider.java b/solr/core/src/java/org/apache/solr/util/configuration/providers/EnvSSLCredentialProvider.java
new file mode 100644
index 0000000..ccfd807
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/util/configuration/providers/EnvSSLCredentialProvider.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.util.configuration.providers;
+
+import java.util.EnumMap;
+import java.util.Map;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Maps;
+
+import static org.apache.solr.util.configuration.SSLCredentialProvider.CredentialType.SSL_CLIENT_KEY_STORE_PASSWORD;
+import static org.apache.solr.util.configuration.SSLCredentialProvider.CredentialType.SSL_CLIENT_TRUST_STORE_PASSWORD;
+import static org.apache.solr.util.configuration.SSLCredentialProvider.CredentialType.SSL_KEY_STORE_PASSWORD;
+import static org.apache.solr.util.configuration.SSLCredentialProvider.CredentialType.SSL_TRUST_STORE_PASSWORD;
+
+
+/**
+ * Environment variable based SSL configuration provider
+ */
+public class EnvSSLCredentialProvider extends AbstractSSLCredentialProvider {
+
+ public static class EnvVars {
+ public static final String SOLR_SSL_CLIENT_KEY_STORE_PASSWORD = "SOLR_SSL_CLIENT_KEY_STORE_PASSWORD";
+ public static final String SOLR_SSL_KEY_STORE_PASSWORD = "SOLR_SSL_KEY_STORE_PASSWORD";
+ public static final String SOLR_SSL_CLIENT_TRUST_STORE_PASSWORD = "SOLR_SSL_CLIENT_TRUST_STORE_PASSWORD";
+ public static final String SOLR_SSL_TRUST_STORE_PASSWORD = "SOLR_SSL_TRUST_STORE_PASSWORD";
+ }
+
+ private Map<String, String> envVars;
+
+ protected EnumMap<CredentialType, String> getCredentialKeyMap() {
+ return Maps.newEnumMap(ImmutableMap.of(
+ SSL_KEY_STORE_PASSWORD, EnvVars.SOLR_SSL_KEY_STORE_PASSWORD,
+ SSL_TRUST_STORE_PASSWORD, EnvVars.SOLR_SSL_TRUST_STORE_PASSWORD,
+ SSL_CLIENT_KEY_STORE_PASSWORD, EnvVars.SOLR_SSL_CLIENT_KEY_STORE_PASSWORD,
+ SSL_CLIENT_TRUST_STORE_PASSWORD, EnvVars.SOLR_SSL_CLIENT_TRUST_STORE_PASSWORD
+ ));
+ }
+
+ protected String getCredential(String envKey) {
+ if (envVars.containsKey(envKey)) {
+ return envVars.get(envKey);
+ } else {
+ return null;
+ }
+ }
+
+ @VisibleForTesting
+ public void setEnvVars(Map<String, String> envVars) {
+ this.envVars = envVars;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b4d280f3/solr/core/src/java/org/apache/solr/util/configuration/providers/HadoopSSLCredentialProvider.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/util/configuration/providers/HadoopSSLCredentialProvider.java b/solr/core/src/java/org/apache/solr/util/configuration/providers/HadoopSSLCredentialProvider.java
new file mode 100644
index 0000000..37eb432
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/util/configuration/providers/HadoopSSLCredentialProvider.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.util.configuration.providers;
+
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.util.EnumMap;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.solr.common.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.hadoop.security.alias.CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH;
+
+/**
+ * System property based SSL configuration provider
+ */
+public class HadoopSSLCredentialProvider extends AbstractSSLCredentialProvider {
+
+ private Configuration hadoopConfigurationProvider;
+
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ public HadoopSSLCredentialProvider() {
+ this(new Configuration());
+ }
+
+ public HadoopSSLCredentialProvider(Configuration hadoopConfigurationProvider) {
+ if (StringUtils.isEmpty(System.getProperty(CREDENTIAL_PROVIDER_PATH))) {
+ throw new RuntimeException("Cannot initialize Hadoop configuration provider without credential provider path. Use " + CREDENTIAL_PROVIDER_PATH + " system property to configure.");
+ }
+ this.hadoopConfigurationProvider = hadoopConfigurationProvider;
+ hadoopConfigurationProvider.set(CREDENTIAL_PROVIDER_PATH, System.getProperty(CREDENTIAL_PROVIDER_PATH));
+ }
+
+ @Override
+ protected EnumMap<CredentialType, String> getCredentialKeyMap() {
+ return DEFAULT_CREDENTIAL_KEY_MAP;
+ }
+
+ protected String getCredential(String keystoreKey) {
+ try {
+ char[] password = hadoopConfigurationProvider.getPassword(keystoreKey);
+ return password == null ? null : String.valueOf(password);
+ } catch (IOException e) {
+ log.error("Could not read password from Hadoop Credential Store: " + keystoreKey, e);
+ return null;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b4d280f3/solr/core/src/java/org/apache/solr/util/configuration/providers/SysPropSSLCredentialProvider.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/util/configuration/providers/SysPropSSLCredentialProvider.java b/solr/core/src/java/org/apache/solr/util/configuration/providers/SysPropSSLCredentialProvider.java
new file mode 100644
index 0000000..0a15a69
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/util/configuration/providers/SysPropSSLCredentialProvider.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.util.configuration.providers;
+
+import java.util.EnumMap;
+
+/**
+ * System property based SSL configuration provider
+ */
+public class SysPropSSLCredentialProvider extends AbstractSSLCredentialProvider {
+ @Override
+ protected EnumMap<CredentialType, String> getCredentialKeyMap() {
+ return DEFAULT_CREDENTIAL_KEY_MAP;
+ }
+
+ protected String getCredential(String syspropKey) {
+ if (System.getProperty(syspropKey) != null) {
+ return System.getProperty(syspropKey);
+ } else {
+ return null;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b4d280f3/solr/core/src/test/org/apache/solr/util/configuration/SSLConfigurationsTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/util/configuration/SSLConfigurationsTest.java b/solr/core/src/test/org/apache/solr/util/configuration/SSLConfigurationsTest.java
index a1d4696..8629e2c 100644
--- a/solr/core/src/test/org/apache/solr/util/configuration/SSLConfigurationsTest.java
+++ b/solr/core/src/test/org/apache/solr/util/configuration/SSLConfigurationsTest.java
@@ -17,17 +17,26 @@
package org.apache.solr.util.configuration;
+import java.io.IOException;
+import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
+import org.apache.hadoop.conf.Configuration;
import org.apache.lucene.util.TestRuleRestoreSystemProperties;
+import org.apache.solr.util.configuration.providers.EnvSSLCredentialProvider;
+import org.apache.solr.util.configuration.providers.HadoopSSLCredentialProvider;
+import org.apache.solr.util.configuration.providers.SysPropSSLCredentialProvider;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TestRule;
+import org.mockito.Mockito;
+import static org.apache.hadoop.security.alias.CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH;
import static org.hamcrest.CoreMatchers.is;
import static org.junit.Assert.assertThat;
+import static org.mockito.Mockito.when;
public class SSLConfigurationsTest {
private Map<String, String> envs;
@@ -38,11 +47,16 @@ public class SSLConfigurationsTest {
public static final String SAMPLE_PW3 = "pw789";
public static final String KEY_STORE_PASSWORD = SSLConfigurations.SysProps.SSL_KEY_STORE_PASSWORD;
public static final String TRUST_STORE_PASSWORD = SSLConfigurations.SysProps.SSL_TRUST_STORE_PASSWORD;
+ public static final String CLIENT_KEY_STORE_PASSWORD = SSLConfigurations.SysProps.SSL_CLIENT_KEY_STORE_PASSWORD;
+ public static final String CLIENT_TRUST_STORE_PASSWORD = SSLConfigurations.SysProps.SSL_CLIENT_TRUST_STORE_PASSWORD;
@Rule
public TestRule syspropRestore = new TestRuleRestoreSystemProperties(
SSLConfigurations.SysProps.SSL_KEY_STORE_PASSWORD,
- SSLConfigurations.SysProps.SSL_TRUST_STORE_PASSWORD
+ SSLConfigurations.SysProps.SSL_TRUST_STORE_PASSWORD,
+ SSLConfigurations.SysProps.SSL_CLIENT_KEY_STORE_PASSWORD,
+ SSLConfigurations.SysProps.SSL_CLIENT_TRUST_STORE_PASSWORD,
+ CREDENTIAL_PROVIDER_PATH
);
@Before
@@ -50,72 +64,203 @@ public class SSLConfigurationsTest {
envs = new HashMap<>();
}
+ private SSLConfigurations createSut(Configuration mockHadoopConfiguration) {
+ EnvSSLCredentialProvider envSSLCredentialProvider = new EnvSSLCredentialProvider();
+ envSSLCredentialProvider.setEnvVars(envs);
+ sut = new SSLConfigurations(Arrays.asList(
+ new HadoopSSLCredentialProvider(mockHadoopConfiguration),
+ envSSLCredentialProvider,
+ new SysPropSSLCredentialProvider())
+ );
+ return sut;
+ }
+
private SSLConfigurations createSut() {
- sut = new SSLConfigurations(envs);
+ EnvSSLCredentialProvider envSSLCredentialProvider = new EnvSSLCredentialProvider();
+ envSSLCredentialProvider.setEnvVars(envs);
+ sut = new SSLConfigurations(Arrays.asList(envSSLCredentialProvider, new SysPropSSLCredentialProvider()));
return sut;
}
@Test
public void testSslConfigKeystorePwFromKeystoreEnvVar() {
- envs.put(SSLConfigurations.EnvVars.SOLR_SSL_KEY_STORE_PASSWORD, SAMPLE_PW1);
+ envs.put(EnvSSLCredentialProvider.EnvVars.SOLR_SSL_KEY_STORE_PASSWORD, SAMPLE_PW1);
createSut().init();
- assertThat(System.getProperty(KEY_STORE_PASSWORD), is(SAMPLE_PW1));
+ assertThat(System.getProperty(CLIENT_KEY_STORE_PASSWORD), is(SAMPLE_PW1));
}
@Test
public void testSslConfigKeystorePwFromClientKeystoreEnvVar() {
- envs.put(SSLConfigurations.EnvVars.SOLR_SSL_CLIENT_KEY_STORE_PASSWORD, SAMPLE_PW2);
+ envs.put(EnvSSLCredentialProvider.EnvVars.SOLR_SSL_CLIENT_KEY_STORE_PASSWORD, SAMPLE_PW2);
createSut().init();
- assertThat(System.getProperty(KEY_STORE_PASSWORD), is(SAMPLE_PW2));
+ assertThat(System.getProperty(CLIENT_KEY_STORE_PASSWORD), is(SAMPLE_PW2));
}
@Test
public void testSslConfigKeystorePwFromBothEnvVars() {
- envs.put(SSLConfigurations.EnvVars.SOLR_SSL_KEY_STORE_PASSWORD, SAMPLE_PW1);
- envs.put(SSLConfigurations.EnvVars.SOLR_SSL_CLIENT_KEY_STORE_PASSWORD, SAMPLE_PW2);
+ envs.put(EnvSSLCredentialProvider.EnvVars.SOLR_SSL_KEY_STORE_PASSWORD, SAMPLE_PW1);
+ envs.put(EnvSSLCredentialProvider.EnvVars.SOLR_SSL_CLIENT_KEY_STORE_PASSWORD, SAMPLE_PW2);
createSut().init();
- assertThat(System.getProperty(KEY_STORE_PASSWORD), is(SAMPLE_PW2));
+ assertThat(System.getProperty(CLIENT_KEY_STORE_PASSWORD), is(SAMPLE_PW2));
+ }
+
+ @Test
+ public void testSslConfigKeystorePwFromKeystoreHadoopCredentialProvider() throws IOException {
+ getSutWithMockedHadoopCredentialProvider(SSLConfigurations.SysProps.SSL_KEY_STORE_PASSWORD, SAMPLE_PW1)
+ .init();
+ assertThat(System.getProperty(CLIENT_KEY_STORE_PASSWORD), is(SAMPLE_PW1));
+ }
+
+ @Test
+ public void testSslConfigKeystorePwFromClientKeystoreHadoopCredentialProvider() throws IOException {
+ getSutWithMockedHadoopCredentialProvider(SSLConfigurations.SysProps.SSL_CLIENT_KEY_STORE_PASSWORD, SAMPLE_PW2)
+ .init();
+ assertThat(System.getProperty(CLIENT_KEY_STORE_PASSWORD), is(SAMPLE_PW2));
}
@Test
public void testSslConfigKeystorePwNotOverwrittenIfExists() {
- System.setProperty(KEY_STORE_PASSWORD, SAMPLE_PW3);
- envs.put(SSLConfigurations.EnvVars.SOLR_SSL_KEY_STORE_PASSWORD, SAMPLE_PW1);
- envs.put(SSLConfigurations.EnvVars.SOLR_SSL_CLIENT_KEY_STORE_PASSWORD, SAMPLE_PW2);
+ System.setProperty(CLIENT_KEY_STORE_PASSWORD, SAMPLE_PW3);
+ envs.put(EnvSSLCredentialProvider.EnvVars.SOLR_SSL_KEY_STORE_PASSWORD, SAMPLE_PW1);
+ envs.put(EnvSSLCredentialProvider.EnvVars.SOLR_SSL_CLIENT_KEY_STORE_PASSWORD, SAMPLE_PW2);
createSut().init();
- assertThat(System.getProperty(KEY_STORE_PASSWORD), is(SAMPLE_PW3)); // unchanged
+ assertThat(System.getProperty(CLIENT_KEY_STORE_PASSWORD), is(SAMPLE_PW3)); // unchanged
}
@Test
public void testSslConfigTruststorePwFromKeystoreEnvVar() {
- envs.put(SSLConfigurations.EnvVars.SOLR_SSL_TRUST_STORE_PASSWORD, SAMPLE_PW1);
+ envs.put(EnvSSLCredentialProvider.EnvVars.SOLR_SSL_TRUST_STORE_PASSWORD, SAMPLE_PW1);
createSut().init();
- assertThat(System.getProperty(TRUST_STORE_PASSWORD), is(SAMPLE_PW1));
+ assertThat(System.getProperty(CLIENT_TRUST_STORE_PASSWORD), is(SAMPLE_PW1));
}
@Test
public void testSslConfigTruststorePwFromClientKeystoreEnvVar() {
- envs.put(SSLConfigurations.EnvVars.SOLR_SSL_CLIENT_TRUST_STORE_PASSWORD, SAMPLE_PW2);
+ envs.put(EnvSSLCredentialProvider.EnvVars.SOLR_SSL_CLIENT_TRUST_STORE_PASSWORD, SAMPLE_PW2);
createSut().init();
- assertThat(System.getProperty(TRUST_STORE_PASSWORD), is(SAMPLE_PW2));
+ assertThat(System.getProperty(CLIENT_TRUST_STORE_PASSWORD), is(SAMPLE_PW2));
}
@Test
public void testSslConfigTruststorePwFromBothEnvVars() {
- envs.put(SSLConfigurations.EnvVars.SOLR_SSL_TRUST_STORE_PASSWORD, SAMPLE_PW1);
- envs.put(SSLConfigurations.EnvVars.SOLR_SSL_CLIENT_TRUST_STORE_PASSWORD, SAMPLE_PW2);
+ envs.put(EnvSSLCredentialProvider.EnvVars.SOLR_SSL_TRUST_STORE_PASSWORD, SAMPLE_PW1);
+ envs.put(EnvSSLCredentialProvider.EnvVars.SOLR_SSL_CLIENT_TRUST_STORE_PASSWORD, SAMPLE_PW2);
createSut().init();
- assertThat(System.getProperty(TRUST_STORE_PASSWORD), is(SAMPLE_PW2));
+ assertThat(System.getProperty(CLIENT_TRUST_STORE_PASSWORD), is(SAMPLE_PW2));
}
@Test
public void testSslConfigTruststorePwNotOverwrittenIfExists() {
- System.setProperty(TRUST_STORE_PASSWORD, SAMPLE_PW3);
- envs.put(SSLConfigurations.EnvVars.SOLR_SSL_TRUST_STORE_PASSWORD, SAMPLE_PW1);
- envs.put(SSLConfigurations.EnvVars.SOLR_SSL_CLIENT_TRUST_STORE_PASSWORD, SAMPLE_PW2);
+ System.setProperty(CLIENT_TRUST_STORE_PASSWORD, SAMPLE_PW3);
+ envs.put(EnvSSLCredentialProvider.EnvVars.SOLR_SSL_TRUST_STORE_PASSWORD, SAMPLE_PW1);
+ envs.put(EnvSSLCredentialProvider.EnvVars.SOLR_SSL_CLIENT_TRUST_STORE_PASSWORD, SAMPLE_PW2);
createSut().init();
- assertThat(System.getProperty(TRUST_STORE_PASSWORD), is(SAMPLE_PW3)); // unchanged
+ assertThat(System.getProperty(CLIENT_TRUST_STORE_PASSWORD), is(SAMPLE_PW3)); // unchanged
+ }
+
+ @Test
+ public void testGetKeyStorePasswordFromProperty() {
+ System.setProperty(KEY_STORE_PASSWORD, SAMPLE_PW1);
+ assertThat(createSut().getKeyStorePassword(), is(SAMPLE_PW1));
+ }
+
+ @Test
+ public void testGetKeyStorePasswordFromEnv() {
+ envs.put(EnvSSLCredentialProvider.EnvVars.SOLR_SSL_KEY_STORE_PASSWORD, SAMPLE_PW2);
+ assertThat(createSut().getKeyStorePassword(), is(SAMPLE_PW2));
+ }
+
+ @Test
+ public void testGetKeyStorePasswordFromHadoopCredentialProvider() throws IOException {
+ SSLConfigurations sut = getSutWithMockedHadoopCredentialProvider(SSLConfigurations.SysProps.SSL_KEY_STORE_PASSWORD, SAMPLE_PW3);
+ assertThat(sut.getKeyStorePassword(), is(SAMPLE_PW3));
+ }
+
+
+ @Test
+ public void testGetTrustStorePasswordFromProperty() {
+ System.setProperty(TRUST_STORE_PASSWORD, SAMPLE_PW1);
+ assertThat(createSut().getTrustStorePassword(), is(SAMPLE_PW1));
+ }
+
+ @Test
+ public void testGetTrustStorePasswordFromEnv() {
+ envs.put(EnvSSLCredentialProvider.EnvVars.SOLR_SSL_TRUST_STORE_PASSWORD, SAMPLE_PW2);
+ assertThat(createSut().getTrustStorePassword(), is(SAMPLE_PW2));
+ }
+
+ @Test
+ public void testGetTruststorePasswordFromHadoopCredentialProvider() throws IOException {
+ SSLConfigurations sut = getSutWithMockedHadoopCredentialProvider(SSLConfigurations.SysProps.SSL_TRUST_STORE_PASSWORD, SAMPLE_PW3);
+ assertThat(sut.getTrustStorePassword(), is(SAMPLE_PW3));
+ }
+
+ @Test
+ public void testGetClientKeyStorePasswordFromProperty() {
+ System.setProperty(CLIENT_KEY_STORE_PASSWORD, SAMPLE_PW1);
+ assertThat(createSut().getClientKeyStorePassword(), is(SAMPLE_PW1));
+ }
+
+ @Test
+ public void testGetClientKeyStorePasswordFromEnv() {
+ envs.put(EnvSSLCredentialProvider.EnvVars.SOLR_SSL_CLIENT_KEY_STORE_PASSWORD, SAMPLE_PW2);
+ assertThat(createSut().getClientKeyStorePassword(), is(SAMPLE_PW2));
+ }
+
+ @Test
+ public void testGetClientKeyStorePasswordFromHadoopCredentialProvider() throws IOException {
+ SSLConfigurations sut = getSutWithMockedHadoopCredentialProvider(SSLConfigurations.SysProps.SSL_CLIENT_KEY_STORE_PASSWORD, SAMPLE_PW3);
+ assertThat(sut.getClientKeyStorePassword(), is(SAMPLE_PW3));
+ }
+
+
+ @Test
+ public void testGetClientTrustStorePasswordFromProperty() {
+ System.setProperty(CLIENT_TRUST_STORE_PASSWORD, SAMPLE_PW1);
+ assertThat(createSut().getClientTrustStorePassword(), is(SAMPLE_PW1));
+ }
+
+ @Test
+ public void testGetClientTrustStorePasswordFromEnv() {
+ envs.put(EnvSSLCredentialProvider.EnvVars.SOLR_SSL_CLIENT_TRUST_STORE_PASSWORD, SAMPLE_PW2);
+ assertThat(createSut().getClientTrustStorePassword(), is(SAMPLE_PW2));
+ }
+
+ @Test
+ public void testGetClientTruststorePasswordFromHadoopCredentialProvider() throws IOException {
+ SSLConfigurations sut = getSutWithMockedHadoopCredentialProvider(SSLConfigurations.SysProps.SSL_CLIENT_TRUST_STORE_PASSWORD, SAMPLE_PW3);
+ assertThat(sut.getClientTrustStorePassword(), is(SAMPLE_PW3));
+ }
+
+ @Test
+ public void testSystemPropertyPriorityOverEnvVar() throws IOException {
+ envs.put(EnvSSLCredentialProvider.EnvVars.SOLR_SSL_KEY_STORE_PASSWORD, SAMPLE_PW2);
+ assertThat(createSut().getKeyStorePassword(), is(SAMPLE_PW2));
+ System.setProperty(KEY_STORE_PASSWORD, SAMPLE_PW1);
+ assertThat(createSut().getKeyStorePassword(), is(SAMPLE_PW2));
+ }
+
+ @Test
+ public void testHadoopCredentialProviderPrioritySysPropAndEnvVars() throws IOException {
+ envs.put(EnvSSLCredentialProvider.EnvVars.SOLR_SSL_KEY_STORE_PASSWORD, SAMPLE_PW2);
+ assertThat(createSut().getKeyStorePassword(), is(SAMPLE_PW2));
+ System.setProperty(KEY_STORE_PASSWORD, SAMPLE_PW1);
+ assertThat(createSut().getKeyStorePassword(), is(SAMPLE_PW2));
+ SSLConfigurations sut = getSutWithMockedHadoopCredentialProvider(KEY_STORE_PASSWORD, SAMPLE_PW3);
+ assertThat(sut.getKeyStorePassword(), is(SAMPLE_PW3));
+ }
+
+ private SSLConfigurations getSutWithMockedHadoopCredentialProvider(String key, String pw) throws IOException {
+ Configuration mockHadoopConfiguration = getMockHadoopConfiguration();
+ when(mockHadoopConfiguration.getPassword(key))
+ .then(invocationOnMock -> invocationOnMock.getArguments()[0].equals(key) ? pw.toCharArray() : null);
+ System.setProperty(CREDENTIAL_PROVIDER_PATH, "/some/path"); // enables HCP
+ return createSut(mockHadoopConfiguration);
+ }
+
+ private Configuration getMockHadoopConfiguration() {
+ return Mockito.mock(Configuration.class);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b4d280f3/solr/core/src/test/org/apache/solr/util/configuration/SSLCredentialProviderFactoryTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/util/configuration/SSLCredentialProviderFactoryTest.java b/solr/core/src/test/org/apache/solr/util/configuration/SSLCredentialProviderFactoryTest.java
new file mode 100644
index 0000000..21e7f8a
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/util/configuration/SSLCredentialProviderFactoryTest.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.util.configuration;
+
+import java.util.List;
+
+import org.apache.lucene.util.TestRuleRestoreSystemProperties;
+import org.apache.solr.util.configuration.providers.EnvSSLCredentialProvider;
+import org.apache.solr.util.configuration.providers.SysPropSSLCredentialProvider;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestRule;
+
+import static org.hamcrest.core.Is.is;
+import static org.junit.Assert.assertThat;
+
+/**
+ */
+public class SSLCredentialProviderFactoryTest {
+
+ @Rule
+ public TestRule syspropRestore = new TestRuleRestoreSystemProperties(
+ SSLCredentialProviderFactory.PROVIDER_CHAIN_KEY
+ );
+
+ @Test
+ public void testGetProvidersOrder() {
+ SSLCredentialProviderFactory sut = getSut("sysprop;env");
+ List<SSLCredentialProvider> providers = sut.getProviders();
+ assertThat(providers.get(0), is(SysPropSSLCredentialProvider.class));
+ assertThat(providers.get(1), is(EnvSSLCredentialProvider.class));
+
+ sut = getSut("env;sysprop");
+ providers = sut.getProviders();
+ assertThat(providers.get(0), is(EnvSSLCredentialProvider.class));
+ assertThat(providers.get(1), is(SysPropSSLCredentialProvider.class));
+ }
+
+ @Test
+ public void testGetProvidersWithCustomProvider() {
+ SSLCredentialProviderFactory sut = getSut("sysprop;class://" + CustomSSLCredentialProvider.class.getName() + ";env");
+ List<SSLCredentialProvider> providers = sut.getProviders();
+ assertThat(providers.get(0), is(SysPropSSLCredentialProvider.class));
+ assertThat(providers.get(1), is(CustomSSLCredentialProvider.class));
+ assertThat(providers.get(2), is(EnvSSLCredentialProvider.class));
+ }
+
+ @Test(expected = RuntimeException.class)
+ public void testGetProvidersInvalidProvider() {
+ getSut("sysprop;DoesNotExists").getProviders();
+ }
+
+ @Test
+ public void testGetProvidersBySysprop() {
+ String chain = "sysprop;class://" + CustomSSLCredentialProvider.class.getName() + ";env";
+ System.setProperty(SSLCredentialProviderFactory.PROVIDER_CHAIN_KEY, chain);
+ SSLCredentialProviderFactory sut = new SSLCredentialProviderFactory();
+ List<SSLCredentialProvider> providers = sut.getProviders();
+ assertThat(providers.get(0), is(SysPropSSLCredentialProvider.class));
+ assertThat(providers.get(1), is(CustomSSLCredentialProvider.class));
+ assertThat(providers.get(2), is(EnvSSLCredentialProvider.class));
+ }
+
+ private SSLCredentialProviderFactory getSut(String providerChain) {
+ return new SSLCredentialProviderFactory(providerChain);
+ }
+
+ static public class CustomSSLCredentialProvider implements SSLCredentialProvider {
+ @Override
+ public String getCredential(CredentialType type) {
+ return null;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b4d280f3/solr/core/src/test/org/apache/solr/util/configuration/providers/EnvSSLCredentialProviderTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/util/configuration/providers/EnvSSLCredentialProviderTest.java b/solr/core/src/test/org/apache/solr/util/configuration/providers/EnvSSLCredentialProviderTest.java
new file mode 100644
index 0000000..c355dce
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/util/configuration/providers/EnvSSLCredentialProviderTest.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.util.configuration.providers;
+
+import java.util.Map;
+
+import com.google.common.collect.ImmutableMap;
+import org.apache.solr.util.configuration.SSLCredentialProvider;
+import org.junit.Test;
+
+import static org.apache.solr.util.configuration.providers.AbstractSSLCredentialProvider.DEFAULT_CREDENTIAL_KEY_MAP;
+import static org.hamcrest.core.Is.is;
+import static org.junit.Assert.assertThat;
+
+/**
+ */
+public class EnvSSLCredentialProviderTest {
+
+ @Test
+ public void testGetCredentials() throws Exception {
+ int cnt = 0;
+ Map<String, String> envvars = ImmutableMap.of(
+ EnvSSLCredentialProvider.EnvVars.SOLR_SSL_KEY_STORE_PASSWORD, "pw" + ++cnt,
+ EnvSSLCredentialProvider.EnvVars.SOLR_SSL_TRUST_STORE_PASSWORD, "pw" + ++cnt,
+ EnvSSLCredentialProvider.EnvVars.SOLR_SSL_CLIENT_KEY_STORE_PASSWORD, "pw" + ++cnt,
+ EnvSSLCredentialProvider.EnvVars.SOLR_SSL_CLIENT_TRUST_STORE_PASSWORD, "pw" + ++cnt
+ );
+ EnvSSLCredentialProvider sut = new EnvSSLCredentialProvider();
+ sut.setEnvVars(envvars);
+ cnt = 0;
+ for (Map.Entry<SSLCredentialProvider.CredentialType, String> set : DEFAULT_CREDENTIAL_KEY_MAP.entrySet()) {
+ String expectedpw = "pw" + ++cnt;
+ assertThat(sut.getCredential(set.getKey()), is(expectedpw));
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b4d280f3/solr/core/src/test/org/apache/solr/util/configuration/providers/HadoopSSLCredentialProviderTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/util/configuration/providers/HadoopSSLCredentialProviderTest.java b/solr/core/src/test/org/apache/solr/util/configuration/providers/HadoopSSLCredentialProviderTest.java
new file mode 100644
index 0000000..bc0a1f4
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/util/configuration/providers/HadoopSSLCredentialProviderTest.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.util.configuration.providers;
+
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.lucene.util.TestRuleRestoreSystemProperties;
+import org.apache.solr.util.configuration.SSLCredentialProvider;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestRule;
+import org.mockito.Mockito;
+
+import static org.apache.hadoop.security.alias.CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH;
+import static org.apache.solr.util.configuration.providers.AbstractSSLCredentialProvider.DEFAULT_CREDENTIAL_KEY_MAP;
+import static org.hamcrest.core.Is.is;
+import static org.junit.Assert.assertThat;
+import static org.mockito.Mockito.when;
+
+/**
+ */
+public class HadoopSSLCredentialProviderTest {
+
+ @Rule
+ public TestRule syspropRestore = new TestRuleRestoreSystemProperties(
+ CREDENTIAL_PROVIDER_PATH
+ );
+
+ @Test(expected = RuntimeException.class)
+ public void testConstructorRequiresCredPath() {
+ new HadoopSSLCredentialProvider(getMockHadoopConfiguration());
+ }
+
+ @Test
+ public void testGetCredentials() throws Exception {
+ int cnt = 0;
+ for (Map.Entry<SSLCredentialProvider.CredentialType, String> set : DEFAULT_CREDENTIAL_KEY_MAP.entrySet()) {
+ String pw = "pw" + ++cnt;
+ HadoopSSLCredentialProvider sut = new HadoopSSLCredentialProvider(getMockedHadoopCredentialProvider(set.getValue(), pw));
+ assertThat(sut.getCredential(set.getKey()), is(pw));
+ }
+ }
+
+ private Configuration getMockedHadoopCredentialProvider(String key, String pw) throws IOException {
+ Configuration mockHadoopConfiguration = getMockHadoopConfiguration();
+ when(mockHadoopConfiguration.getPassword(key))
+ .then(invocationOnMock -> invocationOnMock.getArguments()[0].equals(key) ? pw.toCharArray() : null);
+ System.setProperty(CREDENTIAL_PROVIDER_PATH, "/some/path"); // enables HCP
+ return mockHadoopConfiguration;
+ }
+
+ private Configuration getMockHadoopConfiguration() {
+ return Mockito.mock(Configuration.class);
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b4d280f3/solr/core/src/test/org/apache/solr/util/configuration/providers/SysPropSSLCredentialProviderTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/util/configuration/providers/SysPropSSLCredentialProviderTest.java b/solr/core/src/test/org/apache/solr/util/configuration/providers/SysPropSSLCredentialProviderTest.java
new file mode 100644
index 0000000..e278cb5
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/util/configuration/providers/SysPropSSLCredentialProviderTest.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.util.configuration.providers;
+
+import java.util.Map;
+
+import org.apache.lucene.util.TestRuleRestoreSystemProperties;
+import org.apache.solr.util.configuration.SSLConfigurations;
+import org.apache.solr.util.configuration.SSLCredentialProvider;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestRule;
+
+import static org.apache.solr.util.configuration.providers.AbstractSSLCredentialProvider.DEFAULT_CREDENTIAL_KEY_MAP;
+import static org.hamcrest.core.Is.is;
+import static org.junit.Assert.assertThat;
+
+/**
+ */
+public class SysPropSSLCredentialProviderTest {
+
+ @Rule
+ public TestRule syspropRestore = new TestRuleRestoreSystemProperties(
+ SSLConfigurations.SysProps.SSL_KEY_STORE_PASSWORD,
+ SSLConfigurations.SysProps.SSL_TRUST_STORE_PASSWORD,
+ SSLConfigurations.SysProps.SSL_CLIENT_KEY_STORE_PASSWORD,
+ SSLConfigurations.SysProps.SSL_CLIENT_TRUST_STORE_PASSWORD
+ );
+
+ @Test
+ public void testGetCredentials() throws Exception {
+ int cnt = 0;
+ SysPropSSLCredentialProvider sut = new SysPropSSLCredentialProvider();
+ for (Map.Entry<SSLCredentialProvider.CredentialType, String> set : DEFAULT_CREDENTIAL_KEY_MAP.entrySet()) {
+ String pw = "pw" + ++cnt;
+ System.setProperty(set.getValue(), pw);
+ assertThat(sut.getCredential(set.getKey()), is(pw));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b4d280f3/solr/server/etc/jetty-ssl.xml
----------------------------------------------------------------------
diff --git a/solr/server/etc/jetty-ssl.xml b/solr/server/etc/jetty-ssl.xml
index 741ac48..cdbf57e 100644
--- a/solr/server/etc/jetty-ssl.xml
+++ b/solr/server/etc/jetty-ssl.xml
@@ -7,10 +7,14 @@
<!-- and either jetty-https.xml or jetty-spdy.xml (but not both) -->
<!-- ============================================================= -->
<Configure id="sslContextFactory" class="org.eclipse.jetty.util.ssl.SslContextFactory">
+ <Call class="org.apache.solr.util.configuration.SSLConfigurationsFactory" name="current">
+ <Get name="keyStorePassword" id="keyStorePassword"/>
+ <Get name="trustStorePassword" id="trustStorePassword"/>
+ </Call>
<Set name="KeyStorePath"><Property name="solr.jetty.keystore" default="./etc/solr-ssl.keystore.jks"/></Set>
- <Set name="KeyStorePassword"><Env name="SOLR_SSL_KEY_STORE_PASSWORD" default="secret"/></Set>
+ <Set name="KeyStorePassword"><Ref refid="keyStorePassword"/></Set>
<Set name="TrustStorePath"><Property name="solr.jetty.truststore" default="./etc/solr-ssl.keystore.jks"/></Set>
- <Set name="TrustStorePassword"><Env name="SOLR_SSL_TRUST_STORE_PASSWORD" default="secret"/></Set>
+ <Set name="TrustStorePassword"><Ref refid="trustStorePassword"/></Set>
<Set name="NeedClientAuth"><Property name="solr.jetty.ssl.needClientAuth" default="false"/></Set>
<Set name="WantClientAuth"><Property name="solr.jetty.ssl.wantClientAuth" default="false"/></Set>
<Set name="KeyStoreType"><Property name="solr.jetty.keystore.type" default="JKS"/></Set>
[38/50] lucene-solr:feature/autoscaling: SOLR-11350: Add primes
Stream Evaluator
Posted by sh...@apache.org.
SOLR-11350: Add primes Stream Evaluator
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/15448393
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/15448393
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/15448393
Branch: refs/heads/feature/autoscaling
Commit: 1544839309d6511b6136c7947b06d41e4c85e475
Parents: 31eab31
Author: Joel Bernstein <jb...@apache.org>
Authored: Mon Sep 11 15:00:06 2017 -0400
Committer: Joel Bernstein <jb...@apache.org>
Committed: Mon Sep 11 15:00:19 2017 -0400
----------------------------------------------------------------------
.../org/apache/solr/handler/StreamHandler.java | 1 +
.../client/solrj/io/eval/PrimesEvaluator.java | 58 ++++++++++++++++++++
.../solrj/io/stream/StreamExpressionTest.java | 29 ++++++++++
3 files changed, 88 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/15448393/solr/core/src/java/org/apache/solr/handler/StreamHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/StreamHandler.java b/solr/core/src/java/org/apache/solr/handler/StreamHandler.java
index f7e59c3..bde5f3b 100644
--- a/solr/core/src/java/org/apache/solr/handler/StreamHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/StreamHandler.java
@@ -296,6 +296,7 @@ public class StreamHandler extends RequestHandlerBase implements SolrCoreAware,
.withFunctionName("probability", ProbabilityEvaluator.class)
.withFunctionName("sumDifference", SumDifferenceEvaluator.class)
.withFunctionName("meanDifference", MeanDifferenceEvaluator.class)
+ .withFunctionName("primes", PrimesEvaluator.class)
// Boolean Stream Evaluators
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/15448393/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/PrimesEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/PrimesEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/PrimesEvaluator.java
new file mode 100644
index 0000000..94a3628
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/PrimesEvaluator.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.client.solrj.io.eval;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+
+import org.apache.commons.math3.primes.Primes;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+
+public class PrimesEvaluator extends RecursiveNumericEvaluator implements ManyValueWorker {
+ protected static final long serialVersionUID = 1L;
+
+ public PrimesEvaluator(StreamExpression expression, StreamFactory factory) throws IOException{
+ super(expression, factory);
+
+ if(2 != containedEvaluators.size()){
+ throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - expecting two values but found %d",expression, containedEvaluators.size()));
+ }
+ }
+
+ @Override
+ public Object doWork(Object... values) throws IOException {
+ if(2 != values.length){
+ throw new IOException(String.format(Locale.ROOT,"%s(...) only works with 2 values but %d were provided", constructingFactory.getFunctionName(getClass()), values.length));
+ }
+
+ int sizeNum = ((Number)values[0]).intValue();
+ int startNum = ((Number)values[1]).intValue();
+ List<Number> primes = new ArrayList();
+
+ for(int i=0; i< sizeNum; i++) {
+ int prime = Primes.nextPrime(startNum);
+ primes.add(prime);
+ startNum = prime;
+ ++startNum;
+ }
+
+ return primes;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/15448393/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
index 9b2b186..6fa5d67 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
@@ -5858,6 +5858,35 @@ public class StreamExpressionTest extends SolrCloudTestCase {
@Test
+ public void testPrimes() throws Exception {
+ String cexpr = "primes(10, 0)";
+ ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", cexpr);
+ paramsLoc.set("qt", "/stream");
+
+ String url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString()+"/"+COLLECTIONORALIAS;
+ TupleStream solrStream = new SolrStream(url, paramsLoc);
+
+ StreamContext context = new StreamContext();
+ solrStream.setStreamContext(context);
+ List<Tuple> tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+ Tuple tuple = tuples.get(0);
+ List<Number> asort = (List<Number>)tuple.get("return-value");
+ assertEquals(asort.size(), 10);
+ assertEquals(asort.get(0).intValue(), 2);
+ assertEquals(asort.get(1).intValue(), 3);
+ assertEquals(asort.get(2).intValue(), 5);
+ assertEquals(asort.get(3).intValue(), 7);
+ assertEquals(asort.get(4).intValue(), 11);
+ assertEquals(asort.get(5).intValue(), 13);
+ assertEquals(asort.get(6).intValue(), 17);
+ assertEquals(asort.get(7).intValue(), 19);
+ assertEquals(asort.get(8).intValue(), 23);
+ assertEquals(asort.get(9).intValue(), 29);
+ }
+
+ @Test
public void testAscend() throws Exception {
String cexpr = "asc(array(11.5, 12.3, 4, 3, 1, 0))";
ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
[35/50] lucene-solr:feature/autoscaling: SOLR-11345:
TestPointFields.doTestDoublePointFunctionQuery() helper: fix generated # of
values (was 10, should be 'numVals')
Posted by sh...@apache.org.
SOLR-11345: TestPointFields.doTestDoublePointFunctionQuery() helper: fix generated # of values (was 10, should be 'numVals')
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/0359db97
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/0359db97
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/0359db97
Branch: refs/heads/feature/autoscaling
Commit: 0359db97b5f3b045abfc12cbc118b99196911169
Parents: fe34e14
Author: Steve Rowe <sa...@gmail.com>
Authored: Mon Sep 11 12:35:08 2017 -0400
Committer: Steve Rowe <sa...@gmail.com>
Committed: Mon Sep 11 12:35:08 2017 -0400
----------------------------------------------------------------------
solr/core/src/test/org/apache/solr/schema/TestPointFields.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0359db97/solr/core/src/test/org/apache/solr/schema/TestPointFields.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/schema/TestPointFields.java b/solr/core/src/test/org/apache/solr/schema/TestPointFields.java
index 703ddd8..34c48e6 100644
--- a/solr/core/src/test/org/apache/solr/schema/TestPointFields.java
+++ b/solr/core/src/test/org/apache/solr/schema/TestPointFields.java
@@ -3331,7 +3331,7 @@ public class TestPointFields extends SolrTestCaseJ4 {
assertTrue(h.getCore().getLatestSchema().getField(field).getType() instanceof PointField);
int numVals = 10 * RANDOM_MULTIPLIER;
// Restrict values to float range; otherwise conversion to float will cause truncation -> undefined results
- List<Double> values = getRandomList(10, false, () -> {
+ List<Double> values = getRandomList(numVals, false, () -> {
Float f = Float.NaN;
while (f.isNaN()) {
f = Float.intBitsToFloat(random().nextInt());
[16/50] lucene-solr:feature/autoscaling: LUCENE-7962: Add path
support for computing distance along the path only.
Posted by sh...@apache.org.
LUCENE-7962: Add path support for computing distance along the path only.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/99ae6f87
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/99ae6f87
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/99ae6f87
Branch: refs/heads/feature/autoscaling
Commit: 99ae6f87c8a81129c61e53520ae236fb82069b53
Parents: 913a2c4
Author: Karl Wright <Da...@gmail.com>
Authored: Fri Sep 8 01:19:29 2017 -0400
Committer: Karl Wright <Da...@gmail.com>
Committed: Fri Sep 8 01:19:29 2017 -0400
----------------------------------------------------------------------
.../apache/lucene/spatial3d/geom/GeoPath.java | 30 +++++++
.../lucene/spatial3d/geom/GeoStandardPath.java | 89 ++++++++++++++++++++
.../lucene/spatial3d/geom/GeoPathTest.java | 2 +
3 files changed, 121 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/99ae6f87/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPath.java
----------------------------------------------------------------------
diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPath.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPath.java
index 48a613b..891c896 100644
--- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPath.java
+++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPath.java
@@ -22,4 +22,34 @@ package org.apache.lucene.spatial3d.geom;
* @lucene.experimental
*/
public interface GeoPath extends GeoDistanceShape {
+
+ // The following methods compute distances along the path from the shape to a point
+ // that doesn't need to be inside the shape. The perpendicular distance from the path
+ // itself to the point is not included in the calculation.
+
+ /**
+ * Compute the nearest path distance to the GeoPoint.
+ * The path distance will not include the distance from the path itself to the
+ * point, but just the distance along the path to the nearest point on the path.
+ *
+ * @param distanceStyle is the distance style.
+ * @param point is the point to compute the distance to.
+ * @return the distance to the nearest path point.
+ */
+ public default double computeNearestDistance(final DistanceStyle distanceStyle, final GeoPoint point) {
+ return computeNearestDistance(distanceStyle, point.x, point.y, point.z);
+ }
+
+ /**
+ * Compute the nearest path distance to the GeoPoint.
+ * The path distance will not include the distance from the path itself to the
+ * point, but just the distance along the path to the nearest point on the path.
+ *
+ * @param x is the point's unit x coordinate (using U.S. convention).
+ * @param y is the point's unit y coordinate (using U.S. convention).
+ * @param z is the point's unit z coordinate (using U.S. convention).
+ * @return the distance to the nearest path point.
+ */
+ public double computeNearestDistance(final DistanceStyle distanceStyle, final double x, final double y, final double z);
+
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/99ae6f87/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoStandardPath.java
----------------------------------------------------------------------
diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoStandardPath.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoStandardPath.java
index 4db74de..841b981 100755
--- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoStandardPath.java
+++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoStandardPath.java
@@ -215,6 +215,34 @@ class GeoStandardPath extends GeoBasePath {
}
@Override
+ public double computeNearestDistance(final DistanceStyle distanceStyle, final double x, final double y, final double z) {
+ // Algorithm:
+ // (1) If the point is within any of the segments along the path, return that value.
+ // (2) If the point is within any of the segment end circles along the path, return that value.
+ double currentDistance = 0.0;
+ for (PathSegment segment : segments) {
+ double distance = segment.nearestPathDistance(planetModel, distanceStyle, x,y,z);
+ if (distance != Double.POSITIVE_INFINITY)
+ return distanceStyle.fromAggregationForm(distanceStyle.aggregateDistances(currentDistance, distance));
+ currentDistance = distanceStyle.aggregateDistances(currentDistance, segment.fullPathDistance(distanceStyle));
+ }
+
+ int segmentIndex = 0;
+ currentDistance = 0.0;
+ for (SegmentEndpoint endpoint : endPoints) {
+ double distance = endpoint.nearestPathDistance(distanceStyle, x, y, z);
+ if (distance != Double.POSITIVE_INFINITY) {
+ return distanceStyle.fromAggregationForm(distanceStyle.aggregateDistances(currentDistance, distance));
+ }
+ if (segmentIndex < segments.size()) {
+ currentDistance = distanceStyle.aggregateDistances(currentDistance, segments.get(segmentIndex++).fullPathDistance(distanceStyle));
+ }
+ }
+
+ return Double.POSITIVE_INFINITY;
+ }
+
+ @Override
protected double distance(final DistanceStyle distanceStyle, final double x, final double y, final double z) {
// Algorithm:
// (1) If the point is within any of the segments along the path, return that value.
@@ -562,6 +590,23 @@ class GeoStandardPath extends GeoBasePath {
return distanceStyle.toAggregationForm(distanceStyle.computeDistance(this.point, x, y, z));
}
+ /** Compute nearest path distance.
+ *@param distanceStyle is the distance style.
+ *@param x is the point x.
+ *@param y is the point y.
+ *@param z is the point z.
+ *@return the distance metric (always value zero), in aggregation form, or POSITIVE_INFINITY
+ * if the point is not within the bounds of the endpoint.
+ */
+ public double nearestPathDistance(final DistanceStyle distanceStyle, final double x, final double y, final double z) {
+ for (final Membership m : cutoffPlanes) {
+ if (!m.isWithin(x,y,z)) {
+ return Double.POSITIVE_INFINITY;
+ }
+ }
+ return distanceStyle.toAggregationForm(0.0);
+ }
+
/** Compute external distance.
*@param distanceStyle is the distance style.
*@param x is the point x.
@@ -766,6 +811,50 @@ class GeoStandardPath extends GeoBasePath {
lowerConnectingPlane.isWithin(x, y, z);
}
+ /** Compute nearest path distance.
+ *@param planetModel is the planet model.
+ *@param distanceStyle is the distance style.
+ *@param x is the point x.
+ *@param y is the point y.
+ *@param z is the point z.
+ *@return the distance metric, in aggregation form, or Double.POSITIVE_INFINITY if outside this segment
+ */
+ public double nearestPathDistance(final PlanetModel planetModel, final DistanceStyle distanceStyle, final double x, final double y, final double z) {
+ // First, if this point is outside the endplanes of the segment, return POSITIVE_INFINITY.
+ if (!startCutoffPlane.isWithin(x, y, z) || !endCutoffPlane.isWithin(x, y, z)) {
+ return Double.POSITIVE_INFINITY;
+ }
+ // (1) Compute normalizedPerpPlane. If degenerate, then there is no such plane, which means that the point given
+ // is insufficient to distinguish between a family of such planes. This can happen only if the point is one of the
+ // "poles", imagining the normalized plane to be the "equator". In that case, the distance returned should be zero.
+ // Want no allocations or expensive operations! so we do this the hard way
+ final double perpX = normalizedConnectingPlane.y * z - normalizedConnectingPlane.z * y;
+ final double perpY = normalizedConnectingPlane.z * x - normalizedConnectingPlane.x * z;
+ final double perpZ = normalizedConnectingPlane.x * y - normalizedConnectingPlane.y * x;
+ final double magnitude = Math.sqrt(perpX * perpX + perpY * perpY + perpZ * perpZ);
+ if (Math.abs(magnitude) < Vector.MINIMUM_RESOLUTION)
+ return distanceStyle.toAggregationForm(0.0);
+ final double normFactor = 1.0/magnitude;
+ final Plane normalizedPerpPlane = new Plane(perpX * normFactor, perpY * normFactor, perpZ * normFactor, 0.0);
+
+ final GeoPoint[] intersectionPoints = normalizedConnectingPlane.findIntersections(planetModel, normalizedPerpPlane);
+ GeoPoint thePoint;
+ if (intersectionPoints.length == 0)
+ throw new RuntimeException("Can't find world intersection for point x="+x+" y="+y+" z="+z);
+ else if (intersectionPoints.length == 1)
+ thePoint = intersectionPoints[0];
+ else {
+ if (startCutoffPlane.isWithin(intersectionPoints[0]) && endCutoffPlane.isWithin(intersectionPoints[0]))
+ thePoint = intersectionPoints[0];
+ else if (startCutoffPlane.isWithin(intersectionPoints[1]) && endCutoffPlane.isWithin(intersectionPoints[1]))
+ thePoint = intersectionPoints[1];
+ else
+ throw new RuntimeException("Can't find world intersection for point x="+x+" y="+y+" z="+z);
+ }
+ return distanceStyle.toAggregationForm(distanceStyle.computeDistance(start, thePoint.x, thePoint.y, thePoint.z));
+ }
+
+
/** Compute interior path distance.
*@param planetModel is the planet model.
*@param distanceStyle is the distance style.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/99ae6f87/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPathTest.java
----------------------------------------------------------------------
diff --git a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPathTest.java b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPathTest.java
index 648854e..3aada9a 100755
--- a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPathTest.java
+++ b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPathTest.java
@@ -56,8 +56,10 @@ public class GeoPathTest {
p.done();
gp = new GeoPoint(PlanetModel.SPHERE, 0.05, 0.15);
assertEquals(0.15 + 0.05, p.computeDistance(DistanceStyle.ARC,gp), 0.000001);
+ assertEquals(0.15, p.computeNearestDistance(DistanceStyle.ARC,gp), 0.000001);
gp = new GeoPoint(PlanetModel.SPHERE, 0.0, 0.12);
assertEquals(0.12, p.computeDistance(DistanceStyle.ARC,gp), 0.000001);
+ assertEquals(0.12, p.computeNearestDistance(DistanceStyle.ARC,gp), 0.000001);
// Now try a vertical path, and make sure distances are as expected
p = new GeoStandardPath(PlanetModel.SPHERE, 0.1);
[13/50] lucene-solr:feature/autoscaling: SOLR-10783: Revert for now -
having run the tests a few times today,
one of them may be concerning (reverted from commit
b4d280f369023a179e98154535ed4b06ea096f5f)
Posted by sh...@apache.org.
SOLR-10783: Revert for now - having run the tests a few times today, one of them may be concerning (reverted from commit b4d280f369023a179e98154535ed4b06ea096f5f)
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/ce291247
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/ce291247
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/ce291247
Branch: refs/heads/feature/autoscaling
Commit: ce291247218d733dff12e19110dd9c8bef9d759f
Parents: 9388208
Author: Mark Miller <ma...@apache.org>
Authored: Thu Sep 7 16:35:11 2017 -0500
Committer: Mark Miller <ma...@apache.org>
Committed: Thu Sep 7 16:35:11 2017 -0500
----------------------------------------------------------------------
solr/CHANGES.txt | 2 -
solr/bin/solr | 6 -
solr/bin/solr.cmd | 15 +-
solr/bin/solr.in.cmd | 11 --
solr/bin/solr.in.sh | 11 --
.../util/configuration/SSLConfigurations.java | 102 +++-------
.../configuration/SSLConfigurationsFactory.java | 2 +-
.../configuration/SSLCredentialProvider.java | 36 ----
.../SSLCredentialProviderFactory.java | 93 ---------
.../AbstractSSLCredentialProvider.java | 56 ------
.../providers/EnvSSLCredentialProvider.java | 72 -------
.../providers/HadoopSSLCredentialProvider.java | 66 -------
.../providers/SysPropSSLCredentialProvider.java | 38 ----
.../configuration/SSLConfigurationsTest.java | 193 +++----------------
.../SSLCredentialProviderFactoryTest.java | 90 ---------
.../providers/EnvSSLCredentialProviderTest.java | 61 ------
.../HadoopSSLCredentialProviderTest.java | 72 -------
.../SysPropSSLCredentialProviderTest.java | 66 -------
solr/server/etc/jetty-ssl.xml | 8 +-
19 files changed, 56 insertions(+), 944 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ce291247/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index dcfb15e..be89fb1 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -89,8 +89,6 @@ New Features
* SOLR-11316: JSON Facet API: min/max aggregations are now supported on single-valued date fields.
(yonik)
-* SOLR-10783 Add support for Hadoop Credential Provider as SSL/TLS store password source.
- (Mano Kovacs via Mark Miller)
Bug Fixes
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ce291247/solr/bin/solr
----------------------------------------------------------------------
diff --git a/solr/bin/solr b/solr/bin/solr
index 29af3f4..bc2abca 100755
--- a/solr/bin/solr
+++ b/solr/bin/solr
@@ -163,11 +163,6 @@ fi
SOLR_URL_SCHEME=http
SOLR_JETTY_CONFIG=()
SOLR_SSL_OPTS=""
-
-if [ -n "$SOLR_HADOOP_CREDENTIAL_PROVIDER_PATH" ]; then
- SOLR_SSL_OPTS+=" -Dhadoop.security.credential.provider.path=$SOLR_HADOOP_CREDENTIAL_PROVIDER_PATH"
-fi
-
if [ -z "$SOLR_SSL_ENABLED" ]; then
if [ -n "$SOLR_SSL_KEY_STORE" ]; then
SOLR_SSL_ENABLED="true" # implicitly from earlier behaviour
@@ -177,7 +172,6 @@ if [ -z "$SOLR_SSL_ENABLED" ]; then
fi
if [ "$SOLR_SSL_ENABLED" == "true" ]; then
SOLR_JETTY_CONFIG+=("--module=https")
- SOLR_JETTY_CONFIG+=("--lib=$DEFAULT_SERVER_DIR/solr-webapp/webapp/WEB-INF/lib/*")
SOLR_URL_SCHEME=https
if [ -n "$SOLR_SSL_KEY_STORE" ]; then
SOLR_SSL_OPTS+=" -Dsolr.jetty.keystore=$SOLR_SSL_KEY_STORE"
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ce291247/solr/bin/solr.cmd
----------------------------------------------------------------------
diff --git a/solr/bin/solr.cmd b/solr/bin/solr.cmd
index b5b1d78..57e2e62 100644
--- a/solr/bin/solr.cmd
+++ b/solr/bin/solr.cmd
@@ -38,17 +38,11 @@ REM command line args
IF "%SOLR_INCLUDE%"=="" set "SOLR_INCLUDE=%SOLR_TIP%\bin\solr.in.cmd"
IF EXIST "%SOLR_INCLUDE%" CALL "%SOLR_INCLUDE%"
-set "DEFAULT_SERVER_DIR=%SOLR_TIP%\server"
-
REM Select HTTP OR HTTPS related configurations
set SOLR_URL_SCHEME=http
set "SOLR_JETTY_CONFIG=--module=http"
set "SOLR_SSL_OPTS= "
-IF DEFINED SOLR_HADOOP_CREDENTIAL_PROVIDER_PATH (
- set "SOLR_SSL_OPTS=!SOLR_SSL_OPTS! -Dhadoop.security.credential.provider.path=%SOLR_HADOOP_CREDENTIAL_PROVIDER_PATH%"
-)
-
IF NOT DEFINED SOLR_SSL_ENABLED (
IF DEFINED SOLR_SSL_KEY_STORE (
set "SOLR_SSL_ENABLED=true"
@@ -56,9 +50,8 @@ IF NOT DEFINED SOLR_SSL_ENABLED (
set "SOLR_SSL_ENABLED=false"
)
)
-
IF "%SOLR_SSL_ENABLED%"=="true" (
- set "SOLR_JETTY_CONFIG=--lib="%DEFAULT_SERVER_DIR%\solr-webapp\webapp\WEB-INF\lib\*" --module=https"
+ set "SOLR_JETTY_CONFIG=--module=https"
set SOLR_URL_SCHEME=https
IF DEFINED SOLR_SSL_KEY_STORE (
set "SOLR_SSL_OPTS=!SOLR_SSL_OPTS! -Dsolr.jetty.keystore=%SOLR_SSL_KEY_STORE%"
@@ -189,6 +182,8 @@ IF !JAVA_MAJOR_VERSION! LSS 8 (
goto err
)
+set "DEFAULT_SERVER_DIR=%SOLR_TIP%\server"
+
set FIRST_ARG=%1
IF [%1]==[] goto usage
@@ -1000,7 +995,7 @@ IF "%SCRIPT_CMD%"=="stop" (
set found_it=1
@echo Stopping Solr process %%N running on port %SOLR_PORT%
set /A STOP_PORT=%SOLR_PORT% - 1000
- "%JAVA%" %SOLR_SSL_OPTS% -Djetty.home="%SOLR_SERVER_DIR%" -jar "%SOLR_SERVER_DIR%\start.jar" %SOLR_JETTY_CONFIG% STOP.PORT=!STOP_PORT! STOP.KEY=%STOP_KEY% --stop
+ "%JAVA%" %SOLR_SSL_OPTS% -Djetty.home="%SOLR_SERVER_DIR%" -jar "%SOLR_SERVER_DIR%\start.jar" "%SOLR_JETTY_CONFIG%" STOP.PORT=!STOP_PORT! STOP.KEY=%STOP_KEY% --stop
del "%SOLR_TIP%"\bin\solr-%SOLR_PORT%.port
timeout /T 5
REM Kill it if it is still running after the graceful shutdown
@@ -1248,7 +1243,7 @@ IF "%FG%"=="1" (
-Dlog4j.configuration="%LOG4J_CONFIG%" -DSTOP.PORT=!STOP_PORT! -DSTOP.KEY=%STOP_KEY% ^
-Dsolr.solr.home="%SOLR_HOME%" -Dsolr.install.dir="%SOLR_TIP%" -Dsolr.default.confdir="%DEFAULT_CONFDIR%" ^
-Djetty.host=%SOLR_JETTY_HOST% -Djetty.port=%SOLR_PORT% -Djetty.home="%SOLR_SERVER_DIR%" ^
- -Djava.io.tmpdir="%SOLR_SERVER_DIR%\tmp" -jar start.jar %SOLR_JETTY_CONFIG% "%SOLR_JETTY_ADDL_CONFIG%"
+ -Djava.io.tmpdir="%SOLR_SERVER_DIR%\tmp" -jar start.jar "%SOLR_JETTY_CONFIG%" "%SOLR_JETTY_ADDL_CONFIG%"
) ELSE (
START /B "Solr-%SOLR_PORT%" /D "%SOLR_SERVER_DIR%" ^
"%JAVA%" %SERVEROPT% %SOLR_JAVA_MEM% %START_OPTS% ^
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ce291247/solr/bin/solr.in.cmd
----------------------------------------------------------------------
diff --git a/solr/bin/solr.in.cmd b/solr/bin/solr.in.cmd
index 99a6d79..afba4f7 100644
--- a/solr/bin/solr.in.cmd
+++ b/solr/bin/solr.in.cmd
@@ -116,17 +116,6 @@ REM set SOLR_SSL_CLIENT_TRUST_STORE=
REM set SOLR_SSL_CLIENT_TRUST_STORE_PASSWORD=
REM set SOLR_SSL_CLIENT_TRUST_STORE_TYPE=
-REM Sets path of Hadoop credential provider (hadoop.security.credential.provider.path property) and
-REM enables usage of credential store.
-REM Credential provider should store the following keys:
-REM * solr.jetty.keystore.password
-REM * solr.jetty.truststore.password
-REM Set the two below if you want to set specific store passwords for HTTP client
-REM * javax.net.ssl.keyStorePassword
-REM * javax.net.ssl.trustStorePassword
-REM More info: https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/CredentialProviderAPI.html
-REM set SOLR_HADOOP_CREDENTIAL_PROVIDER_PATH=localjceks://file/home/solr/hadoop-credential-provider.jceks
-
REM Settings for authentication
REM Please configure only one of SOLR_AUTHENTICATION_CLIENT_BUILDER or SOLR_AUTH_TYPE parameters
REM set SOLR_AUTHENTICATION_CLIENT_BUILDER=org.apache.solr.client.solrj.impl.PreemptiveBasicAuthClientBuilderFactory
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ce291247/solr/bin/solr.in.sh
----------------------------------------------------------------------
diff --git a/solr/bin/solr.in.sh b/solr/bin/solr.in.sh
index 22c5bf7..514efb1 100644
--- a/solr/bin/solr.in.sh
+++ b/solr/bin/solr.in.sh
@@ -133,17 +133,6 @@
#SOLR_SSL_CLIENT_TRUST_STORE_PASSWORD=
#SOLR_SSL_CLIENT_TRUST_STORE_TYPE=
-# Sets path of Hadoop credential provider (hadoop.security.credential.provider.path property) and
-# enables usage of credential store.
-# Credential provider should store the following keys:
-# * solr.jetty.keystore.password
-# * solr.jetty.truststore.password
-# Set the two below if you want to set specific store passwords for HTTP client
-# * javax.net.ssl.keyStorePassword
-# * javax.net.ssl.trustStorePassword
-# More info: https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/CredentialProviderAPI.html
-#SOLR_HADOOP_CREDENTIAL_PROVIDER_PATH=localjceks://file/home/solr/hadoop-credential-provider.jceks
-
# Settings for authentication
# Please configure only one of SOLR_AUTHENTICATION_CLIENT_BUILDER or SOLR_AUTH_TYPE parameters
#SOLR_AUTHENTICATION_CLIENT_BUILDER="org.apache.solr.client.solrj.impl.PreemptiveBasicAuthClientBuilderFactory"
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ce291247/solr/core/src/java/org/apache/solr/util/configuration/SSLConfigurations.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/util/configuration/SSLConfigurations.java b/solr/core/src/java/org/apache/solr/util/configuration/SSLConfigurations.java
index 30edf57..bfa5186 100644
--- a/solr/core/src/java/org/apache/solr/util/configuration/SSLConfigurations.java
+++ b/solr/core/src/java/org/apache/solr/util/configuration/SSLConfigurations.java
@@ -18,7 +18,7 @@
package org.apache.solr.util.configuration;
import java.lang.invoke.MethodHandles;
-import java.util.List;
+import java.util.Map;
import org.apache.solr.common.StringUtils;
import org.slf4j.Logger;
@@ -28,102 +28,48 @@ import org.slf4j.LoggerFactory;
* Dedicated object to handle Solr configurations
*/
public class SSLConfigurations {
- public static final String DEFAULT_STORE_PASSWORD = "secret";
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ private final Map<String, String> envVars;
- protected final List<SSLCredentialProvider> credentialProviders;
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
public static class SysProps {
- public static final String SSL_KEY_STORE_PASSWORD = "solr.jetty.keystore.password";
- public static final String SSL_TRUST_STORE_PASSWORD = "solr.jetty.truststore.password";
- public static final String SSL_CLIENT_KEY_STORE_PASSWORD = "javax.net.ssl.keyStorePassword";
- public static final String SSL_CLIENT_TRUST_STORE_PASSWORD = "javax.net.ssl.trustStorePassword";
+ public static final String SSL_KEY_STORE_PASSWORD = "javax.net.ssl.keyStorePassword";
+ public static final String SSL_TRUST_STORE_PASSWORD = "javax.net.ssl.trustStorePassword";
}
- /**
- * @param sslCredentialProviderFactory Credential provider factory to use for providers
- */
- public SSLConfigurations(SSLCredentialProviderFactory sslCredentialProviderFactory) {
- credentialProviders = sslCredentialProviderFactory.getProviders();
+ public static class EnvVars {
+ public static final String SOLR_SSL_CLIENT_KEY_STORE_PASSWORD = "SOLR_SSL_CLIENT_KEY_STORE_PASSWORD";
+ public static final String SOLR_SSL_KEY_STORE_PASSWORD = "SOLR_SSL_KEY_STORE_PASSWORD";
+ public static final String SOLR_SSL_CLIENT_TRUST_STORE_PASSWORD = "SOLR_SSL_CLIENT_TRUST_STORE_PASSWORD";
+ public static final String SOLR_SSL_TRUST_STORE_PASSWORD = "SOLR_SSL_TRUST_STORE_PASSWORD";
}
/**
- * @param credentialProviders Explicit list of credential providers to use
+ * @param envVars Map of environment variables to use
*/
- public SSLConfigurations(List<SSLCredentialProvider> credentialProviders) {
- this.credentialProviders = credentialProviders;
+ public SSLConfigurations(Map<String, String> envVars) {
+ this.envVars = envVars;
}
- /**
- * Initiates javax.net.ssl.* system properties from the proper sources.
- */
+ /** Initiates javax.net.ssl.* system properties from the proper sources. */
public void init() {
- String clientKeystorePassword = getClientKeyStorePassword();
- String keystorePassword = getKeyStorePassword();
+ String clientKeystorePassword = envVars.get(EnvVars.SOLR_SSL_CLIENT_KEY_STORE_PASSWORD);
+ String keystorePassword = envVars.get(EnvVars.SOLR_SSL_KEY_STORE_PASSWORD);
- String clientTruststorePassword = getClientTrustStorePassword();
- String truststorePassword = getTrustStorePassword();
+ String clientTruststorePassword = envVars.get(EnvVars.SOLR_SSL_CLIENT_TRUST_STORE_PASSWORD);
+ String truststorePassword = envVars.get(EnvVars.SOLR_SSL_TRUST_STORE_PASSWORD);
- if (isEmpty(System.getProperty(SysProps.SSL_CLIENT_KEY_STORE_PASSWORD))
+ if (isEmpty(System.getProperty(SysProps.SSL_KEY_STORE_PASSWORD))
&& !(isEmpty(clientKeystorePassword) && isEmpty(keystorePassword))) {
- log.info("Setting {}", SysProps.SSL_CLIENT_KEY_STORE_PASSWORD);
- System.setProperty(SysProps.SSL_CLIENT_KEY_STORE_PASSWORD, clientKeystorePassword != null ? clientKeystorePassword : keystorePassword);
+ log.debug("Setting {} based on env var", SysProps.SSL_KEY_STORE_PASSWORD);
+ System.setProperty(SysProps.SSL_KEY_STORE_PASSWORD, clientKeystorePassword != null ? clientKeystorePassword : keystorePassword);
}
- if (isEmpty(System.getProperty(SysProps.SSL_CLIENT_TRUST_STORE_PASSWORD))
+ if (isEmpty(System.getProperty(SysProps.SSL_TRUST_STORE_PASSWORD))
&& !(isEmpty(clientTruststorePassword) && isEmpty(truststorePassword))) {
- log.info("Setting {}", SysProps.SSL_CLIENT_TRUST_STORE_PASSWORD);
- System.setProperty(SysProps.SSL_CLIENT_TRUST_STORE_PASSWORD, clientTruststorePassword != null ? clientTruststorePassword : truststorePassword);
- }
-
- }
-
- /**
- * @return password for keystore used for SSL connections
- */
- public String getKeyStorePassword() {
- String keyStorePassword = getPassword(SSLCredentialProvider.CredentialType.SSL_KEY_STORE_PASSWORD);
- if (isEmpty(keyStorePassword)) {
- log.warn("No keystore password found, using default.");
- keyStorePassword = DEFAULT_STORE_PASSWORD; // default value
- }
- return keyStorePassword;
- }
-
- /**
- * @return password for truststore used for SSL connections
- */
- public String getTrustStorePassword() {
- String trustStorePassword = getPassword(SSLCredentialProvider.CredentialType.SSL_TRUST_STORE_PASSWORD);
- if (isEmpty(trustStorePassword)) {
- log.warn("No truststore password found, using default.");
- trustStorePassword = DEFAULT_STORE_PASSWORD; // default value
- }
- return trustStorePassword;
- }
-
- /**
- * @return password for keystore used for SSL client connections
- */
- public String getClientKeyStorePassword() {
- String keyStorePassword = getPassword(SSLCredentialProvider.CredentialType.SSL_CLIENT_KEY_STORE_PASSWORD);
- return keyStorePassword;
- }
-
- /**
- * @return password for truststore used for SSL client connections
- */
- public String getClientTrustStorePassword() {
- String trustStorePassword = getPassword(SSLCredentialProvider.CredentialType.SSL_CLIENT_TRUST_STORE_PASSWORD);
- return trustStorePassword;
- }
-
- protected String getPassword(SSLCredentialProvider.CredentialType type) {
- for(SSLCredentialProvider provider: credentialProviders){
- String credential = provider.getCredential(type);
- if(credential != null) return credential;
+ log.debug("Setting {} based on env var", SysProps.SSL_TRUST_STORE_PASSWORD);
+ System.setProperty(SysProps.SSL_TRUST_STORE_PASSWORD, clientTruststorePassword != null ? clientTruststorePassword : truststorePassword);
}
- return null;
}
private boolean isEmpty(String str) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ce291247/solr/core/src/java/org/apache/solr/util/configuration/SSLConfigurationsFactory.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/util/configuration/SSLConfigurationsFactory.java b/solr/core/src/java/org/apache/solr/util/configuration/SSLConfigurationsFactory.java
index 80571ef..1da7c01 100644
--- a/solr/core/src/java/org/apache/solr/util/configuration/SSLConfigurationsFactory.java
+++ b/solr/core/src/java/org/apache/solr/util/configuration/SSLConfigurationsFactory.java
@@ -39,7 +39,7 @@ public class SSLConfigurationsFactory {
}
private static SSLConfigurations getInstance() {
- return new SSLConfigurations(new SSLCredentialProviderFactory());
+ return new SSLConfigurations(System.getenv());
}
@VisibleForTesting
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ce291247/solr/core/src/java/org/apache/solr/util/configuration/SSLCredentialProvider.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/util/configuration/SSLCredentialProvider.java b/solr/core/src/java/org/apache/solr/util/configuration/SSLCredentialProvider.java
deleted file mode 100644
index f714d28..0000000
--- a/solr/core/src/java/org/apache/solr/util/configuration/SSLCredentialProvider.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.util.configuration;
-
-/**
- * Interface for different source of SSL configurations.
- */
-public interface SSLCredentialProvider {
- enum CredentialType {
- SSL_KEY_STORE_PASSWORD,
- SSL_TRUST_STORE_PASSWORD,
- SSL_CLIENT_KEY_STORE_PASSWORD,
- SSL_CLIENT_TRUST_STORE_PASSWORD
- }
-
- /**
- * @return Credential for the given credential type
- */
- String getCredential(CredentialType type);
-
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ce291247/solr/core/src/java/org/apache/solr/util/configuration/SSLCredentialProviderFactory.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/util/configuration/SSLCredentialProviderFactory.java b/solr/core/src/java/org/apache/solr/util/configuration/SSLCredentialProviderFactory.java
deleted file mode 100644
index 1fcfd96..0000000
--- a/solr/core/src/java/org/apache/solr/util/configuration/SSLCredentialProviderFactory.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.util.configuration;
-
-import java.lang.invoke.MethodHandles;
-import java.lang.reflect.InvocationTargetException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Locale;
-
-import com.google.common.collect.ImmutableMap;
-import org.apache.solr.util.configuration.providers.EnvSSLCredentialProvider;
-import org.apache.solr.util.configuration.providers.HadoopSSLCredentialProvider;
-import org.apache.solr.util.configuration.providers.SysPropSSLCredentialProvider;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Class responsible to build SSL credential providers
- */
-public class SSLCredentialProviderFactory {
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
- public static final String DEFAULT_PROVIDER_CHAIN = "env;sysprop";
- public static final String PROVIDER_CHAIN_KEY = "solr.ssl.credential.provider.chain";
-
- private final static ImmutableMap<String, Class> defaultProviders = ImmutableMap.of(
- "env", EnvSSLCredentialProvider.class,
- "sysprop", SysPropSSLCredentialProvider.class,
- "hadoop", HadoopSSLCredentialProvider.class
- );
-
- private String providerChain;
-
- public SSLCredentialProviderFactory() {
- this.providerChain = System.getProperty(PROVIDER_CHAIN_KEY, DEFAULT_PROVIDER_CHAIN);
- }
-
- public SSLCredentialProviderFactory(String providerChain) {
- this.providerChain = providerChain;
- }
-
- public List<SSLCredentialProvider> getProviders() {
- ArrayList<SSLCredentialProvider> providers = new ArrayList<>();
- log.info(String.format(Locale.ROOT, "Processing SSL Credential Provider chain: %s", providerChain));
- String classPrefix = "class://";
- for (String provider : providerChain.split(";")) {
- if (defaultProviders.containsKey(provider)) {
- providers.add(getDefaultProvider(defaultProviders.get(provider)));
- } else if (provider.startsWith(classPrefix)) {
- providers.add(getProviderByClassName(provider.substring(classPrefix.length())));
- } else {
- throw new RuntimeException("Unable to parse credential provider: " + provider);
- }
- }
- return providers;
- }
-
- private SSLCredentialProvider getProviderByClassName(String clazzName) {
- try {
- return (SSLCredentialProvider) Class.forName(clazzName).getConstructor().newInstance();
- } catch (InstantiationException | ClassNotFoundException | IllegalAccessException | InvocationTargetException | NoSuchMethodException e) {
- String msg = String.format(Locale.ROOT, "Could not instantiate %s credential provider", clazzName);
- log.error(msg);
- throw new RuntimeException(msg, e);
- }
- }
-
- private SSLCredentialProvider getDefaultProvider(Class aClass) {
- try {
- return (SSLCredentialProvider) aClass.getConstructor().newInstance();
- } catch (InstantiationException | IllegalAccessException | InvocationTargetException | NoSuchMethodException e) {
- String msg = String.format(Locale.ROOT, "Could not instantiate %s credential provider", aClass.getName());
- log.error(msg);
- throw new RuntimeException(msg, e);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ce291247/solr/core/src/java/org/apache/solr/util/configuration/providers/AbstractSSLCredentialProvider.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/util/configuration/providers/AbstractSSLCredentialProvider.java b/solr/core/src/java/org/apache/solr/util/configuration/providers/AbstractSSLCredentialProvider.java
deleted file mode 100644
index 9825154..0000000
--- a/solr/core/src/java/org/apache/solr/util/configuration/providers/AbstractSSLCredentialProvider.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.util.configuration.providers;
-
-import java.util.EnumMap;
-
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Maps;
-import org.apache.solr.util.configuration.SSLCredentialProvider;
-
-import static org.apache.solr.util.configuration.SSLCredentialProvider.CredentialType.SSL_CLIENT_KEY_STORE_PASSWORD;
-import static org.apache.solr.util.configuration.SSLCredentialProvider.CredentialType.SSL_CLIENT_TRUST_STORE_PASSWORD;
-import static org.apache.solr.util.configuration.SSLCredentialProvider.CredentialType.SSL_KEY_STORE_PASSWORD;
-import static org.apache.solr.util.configuration.SSLCredentialProvider.CredentialType.SSL_TRUST_STORE_PASSWORD;
-
-/**
- * Abstract provider with default implementation
- */
-abstract public class AbstractSSLCredentialProvider implements SSLCredentialProvider {
- public static final EnumMap<CredentialType, String> DEFAULT_CREDENTIAL_KEY_MAP = Maps.newEnumMap(ImmutableMap.of(
- SSL_KEY_STORE_PASSWORD, "solr.jetty.keystore.password",
- SSL_TRUST_STORE_PASSWORD, "solr.jetty.truststore.password",
- SSL_CLIENT_KEY_STORE_PASSWORD, "javax.net.ssl.keyStorePassword",
- SSL_CLIENT_TRUST_STORE_PASSWORD, "javax.net.ssl.trustStorePassword"
- ));
- private final EnumMap<CredentialType, String> credentialKeyMap;
-
- public AbstractSSLCredentialProvider() {
- credentialKeyMap = getCredentialKeyMap();
- }
-
- abstract protected String getCredential(String key);
-
- abstract protected EnumMap<CredentialType, String> getCredentialKeyMap();
-
- @Override
- public String getCredential(CredentialType type) {
- return getCredential(credentialKeyMap.get(type));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ce291247/solr/core/src/java/org/apache/solr/util/configuration/providers/EnvSSLCredentialProvider.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/util/configuration/providers/EnvSSLCredentialProvider.java b/solr/core/src/java/org/apache/solr/util/configuration/providers/EnvSSLCredentialProvider.java
deleted file mode 100644
index 6641cfe..0000000
--- a/solr/core/src/java/org/apache/solr/util/configuration/providers/EnvSSLCredentialProvider.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.util.configuration.providers;
-
-import java.util.EnumMap;
-import java.util.Map;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Maps;
-
-import static org.apache.solr.util.configuration.SSLCredentialProvider.CredentialType.SSL_CLIENT_KEY_STORE_PASSWORD;
-import static org.apache.solr.util.configuration.SSLCredentialProvider.CredentialType.SSL_CLIENT_TRUST_STORE_PASSWORD;
-import static org.apache.solr.util.configuration.SSLCredentialProvider.CredentialType.SSL_KEY_STORE_PASSWORD;
-import static org.apache.solr.util.configuration.SSLCredentialProvider.CredentialType.SSL_TRUST_STORE_PASSWORD;
-
-
-/**
- * Environment variable based SSL configuration provider
- */
-public class EnvSSLCredentialProvider extends AbstractSSLCredentialProvider {
-
- public static class EnvVars {
- public static final String SOLR_SSL_CLIENT_KEY_STORE_PASSWORD = "SOLR_SSL_CLIENT_KEY_STORE_PASSWORD";
- public static final String SOLR_SSL_KEY_STORE_PASSWORD = "SOLR_SSL_KEY_STORE_PASSWORD";
- public static final String SOLR_SSL_CLIENT_TRUST_STORE_PASSWORD = "SOLR_SSL_CLIENT_TRUST_STORE_PASSWORD";
- public static final String SOLR_SSL_TRUST_STORE_PASSWORD = "SOLR_SSL_TRUST_STORE_PASSWORD";
- }
-
- private Map<String, String> envVars;
-
- public EnvSSLCredentialProvider() {
- this.envVars = System.getenv();
- }
-
- protected EnumMap<CredentialType, String> getCredentialKeyMap() {
- return Maps.newEnumMap(ImmutableMap.of(
- SSL_KEY_STORE_PASSWORD, EnvVars.SOLR_SSL_KEY_STORE_PASSWORD,
- SSL_TRUST_STORE_PASSWORD, EnvVars.SOLR_SSL_TRUST_STORE_PASSWORD,
- SSL_CLIENT_KEY_STORE_PASSWORD, EnvVars.SOLR_SSL_CLIENT_KEY_STORE_PASSWORD,
- SSL_CLIENT_TRUST_STORE_PASSWORD, EnvVars.SOLR_SSL_CLIENT_TRUST_STORE_PASSWORD
- ));
- }
-
- protected String getCredential(String envKey) {
- if (envVars.containsKey(envKey)) {
- return envVars.get(envKey);
- } else {
- return null;
- }
- }
-
- @VisibleForTesting
- public void setEnvVars(Map<String, String> envVars) {
- this.envVars = envVars;
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ce291247/solr/core/src/java/org/apache/solr/util/configuration/providers/HadoopSSLCredentialProvider.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/util/configuration/providers/HadoopSSLCredentialProvider.java b/solr/core/src/java/org/apache/solr/util/configuration/providers/HadoopSSLCredentialProvider.java
deleted file mode 100644
index 37eb432..0000000
--- a/solr/core/src/java/org/apache/solr/util/configuration/providers/HadoopSSLCredentialProvider.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.util.configuration.providers;
-
-import java.io.IOException;
-import java.lang.invoke.MethodHandles;
-import java.util.EnumMap;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.solr.common.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static org.apache.hadoop.security.alias.CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH;
-
-/**
- * System property based SSL configuration provider
- */
-public class HadoopSSLCredentialProvider extends AbstractSSLCredentialProvider {
-
- private Configuration hadoopConfigurationProvider;
-
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- public HadoopSSLCredentialProvider() {
- this(new Configuration());
- }
-
- public HadoopSSLCredentialProvider(Configuration hadoopConfigurationProvider) {
- if (StringUtils.isEmpty(System.getProperty(CREDENTIAL_PROVIDER_PATH))) {
- throw new RuntimeException("Cannot initialize Hadoop configuration provider without credential provider path. Use " + CREDENTIAL_PROVIDER_PATH + " system property to configure.");
- }
- this.hadoopConfigurationProvider = hadoopConfigurationProvider;
- hadoopConfigurationProvider.set(CREDENTIAL_PROVIDER_PATH, System.getProperty(CREDENTIAL_PROVIDER_PATH));
- }
-
- @Override
- protected EnumMap<CredentialType, String> getCredentialKeyMap() {
- return DEFAULT_CREDENTIAL_KEY_MAP;
- }
-
- protected String getCredential(String keystoreKey) {
- try {
- char[] password = hadoopConfigurationProvider.getPassword(keystoreKey);
- return password == null ? null : String.valueOf(password);
- } catch (IOException e) {
- log.error("Could not read password from Hadoop Credential Store: " + keystoreKey, e);
- return null;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ce291247/solr/core/src/java/org/apache/solr/util/configuration/providers/SysPropSSLCredentialProvider.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/util/configuration/providers/SysPropSSLCredentialProvider.java b/solr/core/src/java/org/apache/solr/util/configuration/providers/SysPropSSLCredentialProvider.java
deleted file mode 100644
index 0a15a69..0000000
--- a/solr/core/src/java/org/apache/solr/util/configuration/providers/SysPropSSLCredentialProvider.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.util.configuration.providers;
-
-import java.util.EnumMap;
-
-/**
- * System property based SSL configuration provider
- */
-public class SysPropSSLCredentialProvider extends AbstractSSLCredentialProvider {
- @Override
- protected EnumMap<CredentialType, String> getCredentialKeyMap() {
- return DEFAULT_CREDENTIAL_KEY_MAP;
- }
-
- protected String getCredential(String syspropKey) {
- if (System.getProperty(syspropKey) != null) {
- return System.getProperty(syspropKey);
- } else {
- return null;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ce291247/solr/core/src/test/org/apache/solr/util/configuration/SSLConfigurationsTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/util/configuration/SSLConfigurationsTest.java b/solr/core/src/test/org/apache/solr/util/configuration/SSLConfigurationsTest.java
index 8629e2c..a1d4696 100644
--- a/solr/core/src/test/org/apache/solr/util/configuration/SSLConfigurationsTest.java
+++ b/solr/core/src/test/org/apache/solr/util/configuration/SSLConfigurationsTest.java
@@ -17,26 +17,17 @@
package org.apache.solr.util.configuration;
-import java.io.IOException;
-import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
-import org.apache.hadoop.conf.Configuration;
import org.apache.lucene.util.TestRuleRestoreSystemProperties;
-import org.apache.solr.util.configuration.providers.EnvSSLCredentialProvider;
-import org.apache.solr.util.configuration.providers.HadoopSSLCredentialProvider;
-import org.apache.solr.util.configuration.providers.SysPropSSLCredentialProvider;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TestRule;
-import org.mockito.Mockito;
-import static org.apache.hadoop.security.alias.CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH;
import static org.hamcrest.CoreMatchers.is;
import static org.junit.Assert.assertThat;
-import static org.mockito.Mockito.when;
public class SSLConfigurationsTest {
private Map<String, String> envs;
@@ -47,16 +38,11 @@ public class SSLConfigurationsTest {
public static final String SAMPLE_PW3 = "pw789";
public static final String KEY_STORE_PASSWORD = SSLConfigurations.SysProps.SSL_KEY_STORE_PASSWORD;
public static final String TRUST_STORE_PASSWORD = SSLConfigurations.SysProps.SSL_TRUST_STORE_PASSWORD;
- public static final String CLIENT_KEY_STORE_PASSWORD = SSLConfigurations.SysProps.SSL_CLIENT_KEY_STORE_PASSWORD;
- public static final String CLIENT_TRUST_STORE_PASSWORD = SSLConfigurations.SysProps.SSL_CLIENT_TRUST_STORE_PASSWORD;
@Rule
public TestRule syspropRestore = new TestRuleRestoreSystemProperties(
SSLConfigurations.SysProps.SSL_KEY_STORE_PASSWORD,
- SSLConfigurations.SysProps.SSL_TRUST_STORE_PASSWORD,
- SSLConfigurations.SysProps.SSL_CLIENT_KEY_STORE_PASSWORD,
- SSLConfigurations.SysProps.SSL_CLIENT_TRUST_STORE_PASSWORD,
- CREDENTIAL_PROVIDER_PATH
+ SSLConfigurations.SysProps.SSL_TRUST_STORE_PASSWORD
);
@Before
@@ -64,203 +50,72 @@ public class SSLConfigurationsTest {
envs = new HashMap<>();
}
- private SSLConfigurations createSut(Configuration mockHadoopConfiguration) {
- EnvSSLCredentialProvider envSSLCredentialProvider = new EnvSSLCredentialProvider();
- envSSLCredentialProvider.setEnvVars(envs);
- sut = new SSLConfigurations(Arrays.asList(
- new HadoopSSLCredentialProvider(mockHadoopConfiguration),
- envSSLCredentialProvider,
- new SysPropSSLCredentialProvider())
- );
- return sut;
- }
-
private SSLConfigurations createSut() {
- EnvSSLCredentialProvider envSSLCredentialProvider = new EnvSSLCredentialProvider();
- envSSLCredentialProvider.setEnvVars(envs);
- sut = new SSLConfigurations(Arrays.asList(envSSLCredentialProvider, new SysPropSSLCredentialProvider()));
+ sut = new SSLConfigurations(envs);
return sut;
}
@Test
public void testSslConfigKeystorePwFromKeystoreEnvVar() {
- envs.put(EnvSSLCredentialProvider.EnvVars.SOLR_SSL_KEY_STORE_PASSWORD, SAMPLE_PW1);
+ envs.put(SSLConfigurations.EnvVars.SOLR_SSL_KEY_STORE_PASSWORD, SAMPLE_PW1);
createSut().init();
- assertThat(System.getProperty(CLIENT_KEY_STORE_PASSWORD), is(SAMPLE_PW1));
+ assertThat(System.getProperty(KEY_STORE_PASSWORD), is(SAMPLE_PW1));
}
@Test
public void testSslConfigKeystorePwFromClientKeystoreEnvVar() {
- envs.put(EnvSSLCredentialProvider.EnvVars.SOLR_SSL_CLIENT_KEY_STORE_PASSWORD, SAMPLE_PW2);
+ envs.put(SSLConfigurations.EnvVars.SOLR_SSL_CLIENT_KEY_STORE_PASSWORD, SAMPLE_PW2);
createSut().init();
- assertThat(System.getProperty(CLIENT_KEY_STORE_PASSWORD), is(SAMPLE_PW2));
+ assertThat(System.getProperty(KEY_STORE_PASSWORD), is(SAMPLE_PW2));
}
@Test
public void testSslConfigKeystorePwFromBothEnvVars() {
- envs.put(EnvSSLCredentialProvider.EnvVars.SOLR_SSL_KEY_STORE_PASSWORD, SAMPLE_PW1);
- envs.put(EnvSSLCredentialProvider.EnvVars.SOLR_SSL_CLIENT_KEY_STORE_PASSWORD, SAMPLE_PW2);
+ envs.put(SSLConfigurations.EnvVars.SOLR_SSL_KEY_STORE_PASSWORD, SAMPLE_PW1);
+ envs.put(SSLConfigurations.EnvVars.SOLR_SSL_CLIENT_KEY_STORE_PASSWORD, SAMPLE_PW2);
createSut().init();
- assertThat(System.getProperty(CLIENT_KEY_STORE_PASSWORD), is(SAMPLE_PW2));
- }
-
- @Test
- public void testSslConfigKeystorePwFromKeystoreHadoopCredentialProvider() throws IOException {
- getSutWithMockedHadoopCredentialProvider(SSLConfigurations.SysProps.SSL_KEY_STORE_PASSWORD, SAMPLE_PW1)
- .init();
- assertThat(System.getProperty(CLIENT_KEY_STORE_PASSWORD), is(SAMPLE_PW1));
- }
-
- @Test
- public void testSslConfigKeystorePwFromClientKeystoreHadoopCredentialProvider() throws IOException {
- getSutWithMockedHadoopCredentialProvider(SSLConfigurations.SysProps.SSL_CLIENT_KEY_STORE_PASSWORD, SAMPLE_PW2)
- .init();
- assertThat(System.getProperty(CLIENT_KEY_STORE_PASSWORD), is(SAMPLE_PW2));
+ assertThat(System.getProperty(KEY_STORE_PASSWORD), is(SAMPLE_PW2));
}
@Test
public void testSslConfigKeystorePwNotOverwrittenIfExists() {
- System.setProperty(CLIENT_KEY_STORE_PASSWORD, SAMPLE_PW3);
- envs.put(EnvSSLCredentialProvider.EnvVars.SOLR_SSL_KEY_STORE_PASSWORD, SAMPLE_PW1);
- envs.put(EnvSSLCredentialProvider.EnvVars.SOLR_SSL_CLIENT_KEY_STORE_PASSWORD, SAMPLE_PW2);
+ System.setProperty(KEY_STORE_PASSWORD, SAMPLE_PW3);
+ envs.put(SSLConfigurations.EnvVars.SOLR_SSL_KEY_STORE_PASSWORD, SAMPLE_PW1);
+ envs.put(SSLConfigurations.EnvVars.SOLR_SSL_CLIENT_KEY_STORE_PASSWORD, SAMPLE_PW2);
createSut().init();
- assertThat(System.getProperty(CLIENT_KEY_STORE_PASSWORD), is(SAMPLE_PW3)); // unchanged
+ assertThat(System.getProperty(KEY_STORE_PASSWORD), is(SAMPLE_PW3)); // unchanged
}
@Test
public void testSslConfigTruststorePwFromKeystoreEnvVar() {
- envs.put(EnvSSLCredentialProvider.EnvVars.SOLR_SSL_TRUST_STORE_PASSWORD, SAMPLE_PW1);
+ envs.put(SSLConfigurations.EnvVars.SOLR_SSL_TRUST_STORE_PASSWORD, SAMPLE_PW1);
createSut().init();
- assertThat(System.getProperty(CLIENT_TRUST_STORE_PASSWORD), is(SAMPLE_PW1));
+ assertThat(System.getProperty(TRUST_STORE_PASSWORD), is(SAMPLE_PW1));
}
@Test
public void testSslConfigTruststorePwFromClientKeystoreEnvVar() {
- envs.put(EnvSSLCredentialProvider.EnvVars.SOLR_SSL_CLIENT_TRUST_STORE_PASSWORD, SAMPLE_PW2);
+ envs.put(SSLConfigurations.EnvVars.SOLR_SSL_CLIENT_TRUST_STORE_PASSWORD, SAMPLE_PW2);
createSut().init();
- assertThat(System.getProperty(CLIENT_TRUST_STORE_PASSWORD), is(SAMPLE_PW2));
+ assertThat(System.getProperty(TRUST_STORE_PASSWORD), is(SAMPLE_PW2));
}
@Test
public void testSslConfigTruststorePwFromBothEnvVars() {
- envs.put(EnvSSLCredentialProvider.EnvVars.SOLR_SSL_TRUST_STORE_PASSWORD, SAMPLE_PW1);
- envs.put(EnvSSLCredentialProvider.EnvVars.SOLR_SSL_CLIENT_TRUST_STORE_PASSWORD, SAMPLE_PW2);
+ envs.put(SSLConfigurations.EnvVars.SOLR_SSL_TRUST_STORE_PASSWORD, SAMPLE_PW1);
+ envs.put(SSLConfigurations.EnvVars.SOLR_SSL_CLIENT_TRUST_STORE_PASSWORD, SAMPLE_PW2);
createSut().init();
- assertThat(System.getProperty(CLIENT_TRUST_STORE_PASSWORD), is(SAMPLE_PW2));
+ assertThat(System.getProperty(TRUST_STORE_PASSWORD), is(SAMPLE_PW2));
}
@Test
public void testSslConfigTruststorePwNotOverwrittenIfExists() {
- System.setProperty(CLIENT_TRUST_STORE_PASSWORD, SAMPLE_PW3);
- envs.put(EnvSSLCredentialProvider.EnvVars.SOLR_SSL_TRUST_STORE_PASSWORD, SAMPLE_PW1);
- envs.put(EnvSSLCredentialProvider.EnvVars.SOLR_SSL_CLIENT_TRUST_STORE_PASSWORD, SAMPLE_PW2);
+ System.setProperty(TRUST_STORE_PASSWORD, SAMPLE_PW3);
+ envs.put(SSLConfigurations.EnvVars.SOLR_SSL_TRUST_STORE_PASSWORD, SAMPLE_PW1);
+ envs.put(SSLConfigurations.EnvVars.SOLR_SSL_CLIENT_TRUST_STORE_PASSWORD, SAMPLE_PW2);
createSut().init();
- assertThat(System.getProperty(CLIENT_TRUST_STORE_PASSWORD), is(SAMPLE_PW3)); // unchanged
- }
-
- @Test
- public void testGetKeyStorePasswordFromProperty() {
- System.setProperty(KEY_STORE_PASSWORD, SAMPLE_PW1);
- assertThat(createSut().getKeyStorePassword(), is(SAMPLE_PW1));
- }
-
- @Test
- public void testGetKeyStorePasswordFromEnv() {
- envs.put(EnvSSLCredentialProvider.EnvVars.SOLR_SSL_KEY_STORE_PASSWORD, SAMPLE_PW2);
- assertThat(createSut().getKeyStorePassword(), is(SAMPLE_PW2));
- }
-
- @Test
- public void testGetKeyStorePasswordFromHadoopCredentialProvider() throws IOException {
- SSLConfigurations sut = getSutWithMockedHadoopCredentialProvider(SSLConfigurations.SysProps.SSL_KEY_STORE_PASSWORD, SAMPLE_PW3);
- assertThat(sut.getKeyStorePassword(), is(SAMPLE_PW3));
- }
-
-
- @Test
- public void testGetTrustStorePasswordFromProperty() {
- System.setProperty(TRUST_STORE_PASSWORD, SAMPLE_PW1);
- assertThat(createSut().getTrustStorePassword(), is(SAMPLE_PW1));
- }
-
- @Test
- public void testGetTrustStorePasswordFromEnv() {
- envs.put(EnvSSLCredentialProvider.EnvVars.SOLR_SSL_TRUST_STORE_PASSWORD, SAMPLE_PW2);
- assertThat(createSut().getTrustStorePassword(), is(SAMPLE_PW2));
- }
-
- @Test
- public void testGetTruststorePasswordFromHadoopCredentialProvider() throws IOException {
- SSLConfigurations sut = getSutWithMockedHadoopCredentialProvider(SSLConfigurations.SysProps.SSL_TRUST_STORE_PASSWORD, SAMPLE_PW3);
- assertThat(sut.getTrustStorePassword(), is(SAMPLE_PW3));
- }
-
- @Test
- public void testGetClientKeyStorePasswordFromProperty() {
- System.setProperty(CLIENT_KEY_STORE_PASSWORD, SAMPLE_PW1);
- assertThat(createSut().getClientKeyStorePassword(), is(SAMPLE_PW1));
- }
-
- @Test
- public void testGetClientKeyStorePasswordFromEnv() {
- envs.put(EnvSSLCredentialProvider.EnvVars.SOLR_SSL_CLIENT_KEY_STORE_PASSWORD, SAMPLE_PW2);
- assertThat(createSut().getClientKeyStorePassword(), is(SAMPLE_PW2));
- }
-
- @Test
- public void testGetClientKeyStorePasswordFromHadoopCredentialProvider() throws IOException {
- SSLConfigurations sut = getSutWithMockedHadoopCredentialProvider(SSLConfigurations.SysProps.SSL_CLIENT_KEY_STORE_PASSWORD, SAMPLE_PW3);
- assertThat(sut.getClientKeyStorePassword(), is(SAMPLE_PW3));
- }
-
-
- @Test
- public void testGetClientTrustStorePasswordFromProperty() {
- System.setProperty(CLIENT_TRUST_STORE_PASSWORD, SAMPLE_PW1);
- assertThat(createSut().getClientTrustStorePassword(), is(SAMPLE_PW1));
- }
-
- @Test
- public void testGetClientTrustStorePasswordFromEnv() {
- envs.put(EnvSSLCredentialProvider.EnvVars.SOLR_SSL_CLIENT_TRUST_STORE_PASSWORD, SAMPLE_PW2);
- assertThat(createSut().getClientTrustStorePassword(), is(SAMPLE_PW2));
- }
-
- @Test
- public void testGetClientTruststorePasswordFromHadoopCredentialProvider() throws IOException {
- SSLConfigurations sut = getSutWithMockedHadoopCredentialProvider(SSLConfigurations.SysProps.SSL_CLIENT_TRUST_STORE_PASSWORD, SAMPLE_PW3);
- assertThat(sut.getClientTrustStorePassword(), is(SAMPLE_PW3));
- }
-
- @Test
- public void testSystemPropertyPriorityOverEnvVar() throws IOException {
- envs.put(EnvSSLCredentialProvider.EnvVars.SOLR_SSL_KEY_STORE_PASSWORD, SAMPLE_PW2);
- assertThat(createSut().getKeyStorePassword(), is(SAMPLE_PW2));
- System.setProperty(KEY_STORE_PASSWORD, SAMPLE_PW1);
- assertThat(createSut().getKeyStorePassword(), is(SAMPLE_PW2));
- }
-
- @Test
- public void testHadoopCredentialProviderPrioritySysPropAndEnvVars() throws IOException {
- envs.put(EnvSSLCredentialProvider.EnvVars.SOLR_SSL_KEY_STORE_PASSWORD, SAMPLE_PW2);
- assertThat(createSut().getKeyStorePassword(), is(SAMPLE_PW2));
- System.setProperty(KEY_STORE_PASSWORD, SAMPLE_PW1);
- assertThat(createSut().getKeyStorePassword(), is(SAMPLE_PW2));
- SSLConfigurations sut = getSutWithMockedHadoopCredentialProvider(KEY_STORE_PASSWORD, SAMPLE_PW3);
- assertThat(sut.getKeyStorePassword(), is(SAMPLE_PW3));
- }
-
- private SSLConfigurations getSutWithMockedHadoopCredentialProvider(String key, String pw) throws IOException {
- Configuration mockHadoopConfiguration = getMockHadoopConfiguration();
- when(mockHadoopConfiguration.getPassword(key))
- .then(invocationOnMock -> invocationOnMock.getArguments()[0].equals(key) ? pw.toCharArray() : null);
- System.setProperty(CREDENTIAL_PROVIDER_PATH, "/some/path"); // enables HCP
- return createSut(mockHadoopConfiguration);
- }
-
- private Configuration getMockHadoopConfiguration() {
- return Mockito.mock(Configuration.class);
+ assertThat(System.getProperty(TRUST_STORE_PASSWORD), is(SAMPLE_PW3)); // unchanged
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ce291247/solr/core/src/test/org/apache/solr/util/configuration/SSLCredentialProviderFactoryTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/util/configuration/SSLCredentialProviderFactoryTest.java b/solr/core/src/test/org/apache/solr/util/configuration/SSLCredentialProviderFactoryTest.java
deleted file mode 100644
index 21e7f8a..0000000
--- a/solr/core/src/test/org/apache/solr/util/configuration/SSLCredentialProviderFactoryTest.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.util.configuration;
-
-import java.util.List;
-
-import org.apache.lucene.util.TestRuleRestoreSystemProperties;
-import org.apache.solr.util.configuration.providers.EnvSSLCredentialProvider;
-import org.apache.solr.util.configuration.providers.SysPropSSLCredentialProvider;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TestRule;
-
-import static org.hamcrest.core.Is.is;
-import static org.junit.Assert.assertThat;
-
-/**
- */
-public class SSLCredentialProviderFactoryTest {
-
- @Rule
- public TestRule syspropRestore = new TestRuleRestoreSystemProperties(
- SSLCredentialProviderFactory.PROVIDER_CHAIN_KEY
- );
-
- @Test
- public void testGetProvidersOrder() {
- SSLCredentialProviderFactory sut = getSut("sysprop;env");
- List<SSLCredentialProvider> providers = sut.getProviders();
- assertThat(providers.get(0), is(SysPropSSLCredentialProvider.class));
- assertThat(providers.get(1), is(EnvSSLCredentialProvider.class));
-
- sut = getSut("env;sysprop");
- providers = sut.getProviders();
- assertThat(providers.get(0), is(EnvSSLCredentialProvider.class));
- assertThat(providers.get(1), is(SysPropSSLCredentialProvider.class));
- }
-
- @Test
- public void testGetProvidersWithCustomProvider() {
- SSLCredentialProviderFactory sut = getSut("sysprop;class://" + CustomSSLCredentialProvider.class.getName() + ";env");
- List<SSLCredentialProvider> providers = sut.getProviders();
- assertThat(providers.get(0), is(SysPropSSLCredentialProvider.class));
- assertThat(providers.get(1), is(CustomSSLCredentialProvider.class));
- assertThat(providers.get(2), is(EnvSSLCredentialProvider.class));
- }
-
- @Test(expected = RuntimeException.class)
- public void testGetProvidersInvalidProvider() {
- getSut("sysprop;DoesNotExists").getProviders();
- }
-
- @Test
- public void testGetProvidersBySysprop() {
- String chain = "sysprop;class://" + CustomSSLCredentialProvider.class.getName() + ";env";
- System.setProperty(SSLCredentialProviderFactory.PROVIDER_CHAIN_KEY, chain);
- SSLCredentialProviderFactory sut = new SSLCredentialProviderFactory();
- List<SSLCredentialProvider> providers = sut.getProviders();
- assertThat(providers.get(0), is(SysPropSSLCredentialProvider.class));
- assertThat(providers.get(1), is(CustomSSLCredentialProvider.class));
- assertThat(providers.get(2), is(EnvSSLCredentialProvider.class));
- }
-
- private SSLCredentialProviderFactory getSut(String providerChain) {
- return new SSLCredentialProviderFactory(providerChain);
- }
-
- static public class CustomSSLCredentialProvider implements SSLCredentialProvider {
- @Override
- public String getCredential(CredentialType type) {
- return null;
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ce291247/solr/core/src/test/org/apache/solr/util/configuration/providers/EnvSSLCredentialProviderTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/util/configuration/providers/EnvSSLCredentialProviderTest.java b/solr/core/src/test/org/apache/solr/util/configuration/providers/EnvSSLCredentialProviderTest.java
deleted file mode 100644
index 2b85e7d..0000000
--- a/solr/core/src/test/org/apache/solr/util/configuration/providers/EnvSSLCredentialProviderTest.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.util.configuration.providers;
-
-import java.util.Map;
-
-import com.google.common.collect.ImmutableMap;
-import org.apache.solr.util.configuration.SSLCredentialProvider;
-import org.junit.Test;
-
-import static org.apache.solr.util.configuration.providers.AbstractSSLCredentialProvider.DEFAULT_CREDENTIAL_KEY_MAP;
-import static org.hamcrest.core.Is.is;
-import static org.junit.Assert.assertThat;
-
-/**
- */
-public class EnvSSLCredentialProviderTest {
-
- @Test
- public void testGetCredentials() throws Exception {
- int cnt = 0;
- Map<String, String> envvars = ImmutableMap.of(
- EnvSSLCredentialProvider.EnvVars.SOLR_SSL_KEY_STORE_PASSWORD, "pw" + ++cnt,
- EnvSSLCredentialProvider.EnvVars.SOLR_SSL_TRUST_STORE_PASSWORD, "pw" + ++cnt,
- EnvSSLCredentialProvider.EnvVars.SOLR_SSL_CLIENT_KEY_STORE_PASSWORD, "pw" + ++cnt,
- EnvSSLCredentialProvider.EnvVars.SOLR_SSL_CLIENT_TRUST_STORE_PASSWORD, "pw" + ++cnt
- );
- EnvSSLCredentialProvider sut = new EnvSSLCredentialProvider();
- sut.setEnvVars(envvars);
- cnt = 0;
- for (Map.Entry<SSLCredentialProvider.CredentialType, String> set : DEFAULT_CREDENTIAL_KEY_MAP.entrySet()) {
- String expectedpw = "pw" + ++cnt;
- assertThat(sut.getCredential(set.getKey()), is(expectedpw));
- }
- }
-
- @Test
- public void testGetCredentialsWithEnvVars() throws Exception {
- EnvSSLCredentialProvider sut = new EnvSSLCredentialProvider();
- // assuming not to fail
- sut.getCredential(SSLCredentialProvider.CredentialType.SSL_KEY_STORE_PASSWORD);
- sut.getCredential(SSLCredentialProvider.CredentialType.SSL_CLIENT_KEY_STORE_PASSWORD);
- sut.getCredential(SSLCredentialProvider.CredentialType.SSL_TRUST_STORE_PASSWORD);
- sut.getCredential(SSLCredentialProvider.CredentialType.SSL_CLIENT_TRUST_STORE_PASSWORD);
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ce291247/solr/core/src/test/org/apache/solr/util/configuration/providers/HadoopSSLCredentialProviderTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/util/configuration/providers/HadoopSSLCredentialProviderTest.java b/solr/core/src/test/org/apache/solr/util/configuration/providers/HadoopSSLCredentialProviderTest.java
deleted file mode 100644
index bc0a1f4..0000000
--- a/solr/core/src/test/org/apache/solr/util/configuration/providers/HadoopSSLCredentialProviderTest.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.util.configuration.providers;
-
-import java.io.IOException;
-import java.util.Map;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.lucene.util.TestRuleRestoreSystemProperties;
-import org.apache.solr.util.configuration.SSLCredentialProvider;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TestRule;
-import org.mockito.Mockito;
-
-import static org.apache.hadoop.security.alias.CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH;
-import static org.apache.solr.util.configuration.providers.AbstractSSLCredentialProvider.DEFAULT_CREDENTIAL_KEY_MAP;
-import static org.hamcrest.core.Is.is;
-import static org.junit.Assert.assertThat;
-import static org.mockito.Mockito.when;
-
-/**
- */
-public class HadoopSSLCredentialProviderTest {
-
- @Rule
- public TestRule syspropRestore = new TestRuleRestoreSystemProperties(
- CREDENTIAL_PROVIDER_PATH
- );
-
- @Test(expected = RuntimeException.class)
- public void testConstructorRequiresCredPath() {
- new HadoopSSLCredentialProvider(getMockHadoopConfiguration());
- }
-
- @Test
- public void testGetCredentials() throws Exception {
- int cnt = 0;
- for (Map.Entry<SSLCredentialProvider.CredentialType, String> set : DEFAULT_CREDENTIAL_KEY_MAP.entrySet()) {
- String pw = "pw" + ++cnt;
- HadoopSSLCredentialProvider sut = new HadoopSSLCredentialProvider(getMockedHadoopCredentialProvider(set.getValue(), pw));
- assertThat(sut.getCredential(set.getKey()), is(pw));
- }
- }
-
- private Configuration getMockedHadoopCredentialProvider(String key, String pw) throws IOException {
- Configuration mockHadoopConfiguration = getMockHadoopConfiguration();
- when(mockHadoopConfiguration.getPassword(key))
- .then(invocationOnMock -> invocationOnMock.getArguments()[0].equals(key) ? pw.toCharArray() : null);
- System.setProperty(CREDENTIAL_PROVIDER_PATH, "/some/path"); // enables HCP
- return mockHadoopConfiguration;
- }
-
- private Configuration getMockHadoopConfiguration() {
- return Mockito.mock(Configuration.class);
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ce291247/solr/core/src/test/org/apache/solr/util/configuration/providers/SysPropSSLCredentialProviderTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/util/configuration/providers/SysPropSSLCredentialProviderTest.java b/solr/core/src/test/org/apache/solr/util/configuration/providers/SysPropSSLCredentialProviderTest.java
deleted file mode 100644
index 4a5894d..0000000
--- a/solr/core/src/test/org/apache/solr/util/configuration/providers/SysPropSSLCredentialProviderTest.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.util.configuration.providers;
-
-import java.util.Map;
-
-import org.apache.lucene.util.TestRuleRestoreSystemProperties;
-import org.apache.solr.util.configuration.SSLConfigurations;
-import org.apache.solr.util.configuration.SSLCredentialProvider;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TestRule;
-
-import static org.apache.solr.util.configuration.providers.AbstractSSLCredentialProvider.DEFAULT_CREDENTIAL_KEY_MAP;
-import static org.hamcrest.core.Is.is;
-import static org.junit.Assert.assertThat;
-
-/**
- */
-public class SysPropSSLCredentialProviderTest {
-
- @Rule
- public TestRule syspropRestore = new TestRuleRestoreSystemProperties(
- SSLConfigurations.SysProps.SSL_KEY_STORE_PASSWORD,
- SSLConfigurations.SysProps.SSL_TRUST_STORE_PASSWORD,
- SSLConfigurations.SysProps.SSL_CLIENT_KEY_STORE_PASSWORD,
- SSLConfigurations.SysProps.SSL_CLIENT_TRUST_STORE_PASSWORD
- );
-
- @Test
- public void testGetCredentials() throws Exception {
- int cnt = 0;
- SysPropSSLCredentialProvider sut = new SysPropSSLCredentialProvider();
- for (Map.Entry<SSLCredentialProvider.CredentialType, String> set : DEFAULT_CREDENTIAL_KEY_MAP.entrySet()) {
- String pw = "pw" + ++cnt;
- System.setProperty(set.getValue(), pw);
- assertThat(sut.getCredential(set.getKey()), is(pw));
- }
- }
-
-
- @Test
- public void testGetCredentialsWithoutSetup() throws Exception {
- SysPropSSLCredentialProvider sut = new SysPropSSLCredentialProvider();
- // assuming not to fail
- sut.getCredential(SSLCredentialProvider.CredentialType.SSL_KEY_STORE_PASSWORD);
- sut.getCredential(SSLCredentialProvider.CredentialType.SSL_CLIENT_KEY_STORE_PASSWORD);
- sut.getCredential(SSLCredentialProvider.CredentialType.SSL_TRUST_STORE_PASSWORD);
- sut.getCredential(SSLCredentialProvider.CredentialType.SSL_CLIENT_TRUST_STORE_PASSWORD);
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ce291247/solr/server/etc/jetty-ssl.xml
----------------------------------------------------------------------
diff --git a/solr/server/etc/jetty-ssl.xml b/solr/server/etc/jetty-ssl.xml
index cdbf57e..741ac48 100644
--- a/solr/server/etc/jetty-ssl.xml
+++ b/solr/server/etc/jetty-ssl.xml
@@ -7,14 +7,10 @@
<!-- and either jetty-https.xml or jetty-spdy.xml (but not both) -->
<!-- ============================================================= -->
<Configure id="sslContextFactory" class="org.eclipse.jetty.util.ssl.SslContextFactory">
- <Call class="org.apache.solr.util.configuration.SSLConfigurationsFactory" name="current">
- <Get name="keyStorePassword" id="keyStorePassword"/>
- <Get name="trustStorePassword" id="trustStorePassword"/>
- </Call>
<Set name="KeyStorePath"><Property name="solr.jetty.keystore" default="./etc/solr-ssl.keystore.jks"/></Set>
- <Set name="KeyStorePassword"><Ref refid="keyStorePassword"/></Set>
+ <Set name="KeyStorePassword"><Env name="SOLR_SSL_KEY_STORE_PASSWORD" default="secret"/></Set>
<Set name="TrustStorePath"><Property name="solr.jetty.truststore" default="./etc/solr-ssl.keystore.jks"/></Set>
- <Set name="TrustStorePassword"><Ref refid="trustStorePassword"/></Set>
+ <Set name="TrustStorePassword"><Env name="SOLR_SSL_TRUST_STORE_PASSWORD" default="secret"/></Set>
<Set name="NeedClientAuth"><Property name="solr.jetty.ssl.needClientAuth" default="false"/></Set>
<Set name="WantClientAuth"><Property name="solr.jetty.ssl.wantClientAuth" default="false"/></Set>
<Set name="KeyStoreType"><Property name="solr.jetty.keystore.type" default="JKS"/></Set>
[17/50] lucene-solr:feature/autoscaling: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/lucene-solr
Posted by sh...@apache.org.
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/lucene-solr
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/5fee9372
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/5fee9372
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/5fee9372
Branch: refs/heads/feature/autoscaling
Commit: 5fee937242430b4f90681968b91969a656fd6dcd
Parents: 99ae6f8 85ae963
Author: Karl Wright <Da...@gmail.com>
Authored: Fri Sep 8 01:20:00 2017 -0400
Committer: Karl Wright <Da...@gmail.com>
Committed: Fri Sep 8 01:20:00 2017 -0400
----------------------------------------------------------------------
build.xml | 18 +-
dev-tools/doap/lucene.rdf | 9 +-
dev-tools/doap/solr.rdf | 7 +
lucene/CHANGES.txt | 16 ++
lucene/NOTICE.txt | 5 +-
lucene/analysis/build.xml | 4 +
.../lucene/analysis/bn/BengaliAnalyzer.java | 132 +++++++++
.../analysis/bn/BengaliNormalizationFilter.java | 59 ++++
.../bn/BengaliNormalizationFilterFactory.java | 55 ++++
.../lucene/analysis/bn/BengaliNormalizer.java | 155 +++++++++++
.../lucene/analysis/bn/BengaliStemFilter.java | 49 ++++
.../analysis/bn/BengaliStemFilterFactory.java | 48 ++++
.../lucene/analysis/bn/BengaliStemmer.java | 183 +++++++++++++
.../apache/lucene/analysis/bn/package-info.java | 21 ++
...ache.lucene.analysis.util.TokenFilterFactory | 2 +
.../org/apache/lucene/analysis/bn/stopwords.txt | 121 +++++++++
.../lucene/analysis/bn/TestBengaliAnalyzer.java | 53 ++++
.../lucene/analysis/bn/TestBengaliFilters.java | 80 ++++++
.../analysis/bn/TestBengaliNormalizer.java | 110 ++++++++
.../lucene/analysis/bn/TestBengaliStemmer.java | 79 ++++++
.../analysis/icu/ICUNormalizer2CharFilter.java | 35 ++-
.../icu/TestICUNormalizer2CharFilter.java | 21 ++
.../index/TestBackwardsCompatibility.java | 4 +-
.../lucene/index/unsupported.6.6.1-cfs.zip | Bin 0 -> 15836 bytes
.../lucene/index/unsupported.6.6.1-nocfs.zip | Bin 0 -> 15830 bytes
lucene/build.xml | 13 +
.../SimpleNaiveBayesDocumentClassifier.java | 30 ++-
lucene/codecs/build.xml | 2 +
lucene/common-build.xml | 53 +++-
lucene/core/build.xml | 2 +
.../tokenattributes/CharTermAttributeImpl.java | 3 +
.../java/org/apache/lucene/util/LongBitSet.java | 8 +-
.../TestCharTermAttributeImpl.java | 10 +
.../org/apache/lucene/util/TestLongBitSet.java | 24 +-
.../writercache/LruTaxonomyWriterCache.java | 20 +-
.../writercache/TestLruTaxonomyWriterCache.java | 50 ++++
lucene/ivy-versions.properties | 4 +-
.../randomizedtesting-runner-2.5.2.jar.sha1 | 1 -
.../randomizedtesting-runner-2.5.3.jar.sha1 | 1 +
.../lucene/replicator/nrt/PrimaryNode.java | 12 -
.../lucene/replicator/nrt/ReplicaNode.java | 10 -
solr/CHANGES.txt | 25 +-
solr/build.xml | 16 +-
solr/core/build.xml | 7 +
.../apache/solr/cloud/ReplicateFromLeader.java | 1 -
.../org/apache/solr/cloud/ZkController.java | 2 +-
.../org/apache/solr/handler/StreamHandler.java | 98 ++-----
.../solr/handler/admin/MetricsHandler.java | 11 +-
.../org/apache/solr/parser/FastCharStream.java | 7 +-
.../org/apache/solr/parser/QueryParser.java | 2 +-
.../solr/request/json/JsonQueryConverter.java | 110 ++++++++
.../apache/solr/request/json/RequestUtil.java | 10 +-
.../org/apache/solr/schema/DateRangeField.java | 4 +-
.../apache/solr/search/BoolQParserPlugin.java | 58 ++++
.../org/apache/solr/search/FunctionQParser.java | 11 +-
.../apache/solr/search/LuceneQParserPlugin.java | 54 ----
.../solr/search/OldLuceneQParserPlugin.java | 37 ---
.../org/apache/solr/search/QParserPlugin.java | 1 +
.../apache/solr/search/ValueSourceParser.java | 4 +-
.../org/apache/solr/search/facet/MinMaxAgg.java | 145 +++++++++-
.../org/apache/solr/search/facet/SlotAcc.java | 84 +++++-
.../search/function/FieldNameValueSource.java | 60 +++++
.../java/org/apache/solr/update/UpdateLog.java | 13 +-
.../FieldLengthUpdateProcessorFactory.java | 6 +-
.../FieldMutatingUpdateProcessorFactory.java | 4 +-
.../ParseDateFieldUpdateProcessorFactory.java | 4 +-
.../ParseDoubleFieldUpdateProcessorFactory.java | 4 +-
.../ParseFloatFieldUpdateProcessorFactory.java | 4 +-
.../ParseIntFieldUpdateProcessorFactory.java | 4 +-
.../ParseLongFieldUpdateProcessorFactory.java | 4 +-
.../configuration/providers/package-info.java | 23 ++
.../solr/collection1/conf/solrconfig.xml | 3 -
.../org/apache/solr/BasicFunctionalityTest.java | 2 +-
.../org/apache/solr/ConvertedLegacyTest.java | 70 +++--
.../apache/solr/TestDistributedMissingSort.java | 8 +-
.../apache/solr/cloud/CdcrBootstrapTest.java | 1 +
.../apache/solr/cloud/HttpPartitionTest.java | 4 +-
.../org/apache/solr/core/TestLazyCores.java | 4 +-
.../apache/solr/handler/SearchHandlerTest.java | 4 +-
.../org/apache/solr/handler/TestSQLHandler.java | 1 -
.../solr/handler/admin/MetricsHandlerTest.java | 9 +
.../org/apache/solr/search/CursorMarkTest.java | 10 +-
.../apache/solr/search/QueryEqualityTest.java | 29 +-
.../apache/solr/search/QueryParsingTest.java | 1 -
.../apache/solr/search/TestSmileRequest.java | 2 +-
.../apache/solr/search/TestSolrQueryParser.java | 2 +-
.../solr/search/facet/TestJsonFacets.java | 86 ++++--
.../solr/search/json/TestJsonRequest.java | 179 ++++++++++++-
.../apache/solr/update/SoftAutoCommitTest.java | 6 -
.../example-DIH/solr/db/conf/managed-schema | 2 +-
.../example-DIH/solr/mail/conf/managed-schema | 2 +-
.../example-DIH/solr/solr/conf/managed-schema | 2 +-
solr/example/files/conf/managed-schema | 8 +-
solr/example/films/README.txt | 6 +-
solr/licenses/commons-math3-3.4.1.jar.sha1 | 1 -
solr/licenses/commons-math3-3.6.1.jar.sha1 | 1 +
solr/licenses/httpmime-4.5.3.jar.sha1 | 2 +-
solr/licenses/junit4-ant-2.5.2.jar.sha1 | 1 -
solr/licenses/junit4-ant-2.5.3.jar.sha1 | 1 +
.../randomizedtesting-runner-2.5.2.jar.sha1 | 1 -
.../randomizedtesting-runner-2.5.3.jar.sha1 | 1 +
solr/solr-ref-guide/README.adoc | 32 ++-
solr/solr-ref-guide/src/collections-api.adoc | 2 +
.../src/cross-data-center-replication-cdcr.adoc | 1 -
solr/solr-ref-guide/src/docvalues.adoc | 24 +-
.../field-type-definitions-and-properties.adoc | 2 +-
.../src/field-types-included-with-solr.adoc | 81 ++++--
solr/solr-ref-guide/src/function-queries.adoc | 2 +-
.../solr-ref-guide/src/how-solrcloud-works.adoc | 3 +-
solr/solr-ref-guide/src/how-to-contribute.adoc | 3 +-
solr/solr-ref-guide/src/index.adoc | 2 +-
.../src/indexconfig-in-solrconfig.adoc | 2 +-
.../src/major-changes-in-solr-7.adoc | 187 +++++++++++++
solr/solr-ref-guide/src/other-parsers.adoc | 2 +-
.../src/putting-the-pieces-together.adoc | 4 +-
.../read-and-write-side-fault-tolerance.adoc | 109 --------
solr/solr-ref-guide/src/schema-api.adoc | 7 +-
solr/solr-ref-guide/src/schemaless-mode.adoc | 62 +++--
solr/solr-ref-guide/src/solr-upgrade-notes.adoc | 56 +---
.../src/solrcloud-autoscaling-overview.adoc | 2 +-
...olrcloud-autoscaling-policy-preferences.adoc | 5 +-
...rcloud-query-routing-and-read-tolerance.adoc | 85 ++++++
...olrcloud-recoveries-and-write-tolerance.adoc | 43 +++
.../src/solrcloud-resilience.adoc | 28 ++
solr/solr-ref-guide/src/solrcloud.adoc | 6 +-
solr/solr-ref-guide/src/spatial-search.adoc | 5 +-
.../src/the-extended-dismax-query-parser.adoc | 21 +-
.../src/the-query-elevation-component.adoc | 12 +-
.../src/the-standard-query-parser.adoc | 6 +-
.../src/update-request-processors.adoc | 2 +-
.../src/updatehandlers-in-solrconfig.adoc | 2 +-
solr/solr-ref-guide/src/working-with-dates.adoc | 6 +-
...rking-with-external-files-and-processes.adoc | 6 +-
solr/solrj/build.xml | 2 +
.../io/eval/BinomialDistributionEvaluator.java | 48 ++++
.../io/eval/CosineSimilarityEvaluator.java | 67 +++++
.../io/eval/CumulativeProbabilityEvaluator.java | 21 +-
.../solrj/io/eval/DotProductEvaluator.java | 58 ++++
.../client/solrj/io/eval/EBEAddEvaluator.java | 63 +++++
.../solrj/io/eval/EBEDivideEvaluator.java | 63 +++++
.../solrj/io/eval/EBEMultiplyEvaluator.java | 63 +++++
.../solrj/io/eval/EBESubtractEvaluator.java | 63 +++++
.../eval/EnumeratedDistributionEvaluator.java | 45 ++++
.../solrj/io/eval/FrequencyTableEvaluator.java | 84 ++++++
.../solrj/io/eval/HistogramEvaluator.java | 7 +-
.../io/eval/PoissonDistributionEvaluator.java | 44 +++
.../solrj/io/eval/ProbabilityEvaluator.java | 52 ++++
.../client/solrj/io/eval/SampleEvaluator.java | 18 +-
.../UniformIntegerDistributionEvaluator.java | 48 ++++
.../client/solrj/io/stream/FacetStream.java | 10 +-
.../solrj/io/stream/StreamExpressionTest.java | 268 +++++++++++++++++++
solr/webapp/build.xml | 3 +
152 files changed, 3896 insertions(+), 712 deletions(-)
----------------------------------------------------------------------
[15/50] lucene-solr:feature/autoscaling: Adding 6.6.1 back-compat
indexes
Posted by sh...@apache.org.
Adding 6.6.1 back-compat indexes
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/85ae9633
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/85ae9633
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/85ae9633
Branch: refs/heads/feature/autoscaling
Commit: 85ae9633dafba8e3efc7ea7144da4667292ba0f3
Parents: 89feb15
Author: Varun Thacker <va...@apache.org>
Authored: Thu Sep 7 20:55:39 2017 -0700
Committer: Varun Thacker <va...@apache.org>
Committed: Thu Sep 7 20:55:39 2017 -0700
----------------------------------------------------------------------
.../lucene/index/TestBackwardsCompatibility.java | 4 +++-
.../apache/lucene/index/unsupported.6.6.1-cfs.zip | Bin 0 -> 15836 bytes
.../lucene/index/unsupported.6.6.1-nocfs.zip | Bin 0 -> 15830 bytes
3 files changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/85ae9633/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
----------------------------------------------------------------------
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java b/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
index 22b0ab6..abb189a 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
@@ -432,7 +432,9 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
"6.5.1-cfs",
"6.5.1-nocfs",
"6.6.0-cfs",
- "6.6.0-nocfs"
+ "6.6.0-nocfs",
+ "6.6.1-cfs",
+ "6.6.1-nocfs"
};
// TODO: on 6.0.0 release, gen the single segment indices and add here:
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/85ae9633/lucene/backward-codecs/src/test/org/apache/lucene/index/unsupported.6.6.1-cfs.zip
----------------------------------------------------------------------
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/index/unsupported.6.6.1-cfs.zip b/lucene/backward-codecs/src/test/org/apache/lucene/index/unsupported.6.6.1-cfs.zip
new file mode 100644
index 0000000..7a64c23
Binary files /dev/null and b/lucene/backward-codecs/src/test/org/apache/lucene/index/unsupported.6.6.1-cfs.zip differ
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/85ae9633/lucene/backward-codecs/src/test/org/apache/lucene/index/unsupported.6.6.1-nocfs.zip
----------------------------------------------------------------------
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/index/unsupported.6.6.1-nocfs.zip b/lucene/backward-codecs/src/test/org/apache/lucene/index/unsupported.6.6.1-nocfs.zip
new file mode 100644
index 0000000..8b624e3
Binary files /dev/null and b/lucene/backward-codecs/src/test/org/apache/lucene/index/unsupported.6.6.1-nocfs.zip differ
[20/50] lucene-solr:feature/autoscaling: LUCENE-7955: Add zero-width
path support.
Posted by sh...@apache.org.
LUCENE-7955: Add zero-width path support.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/c09678e5
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/c09678e5
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/c09678e5
Branch: refs/heads/feature/autoscaling
Commit: c09678e540c4e285312e11679e13328f1e84c1e6
Parents: 5fee937
Author: Karl Wright <Da...@gmail.com>
Authored: Fri Sep 8 12:18:39 2017 -0400
Committer: Karl Wright <Da...@gmail.com>
Committed: Fri Sep 8 12:18:39 2017 -0400
----------------------------------------------------------------------
.../spatial3d/geom/GeoDegeneratePath.java | 706 +++++++++++++++++++
.../lucene/spatial3d/geom/GeoPathFactory.java | 3 +
2 files changed, 709 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c09678e5/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoDegeneratePath.java
----------------------------------------------------------------------
diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoDegeneratePath.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoDegeneratePath.java
new file mode 100644
index 0000000..175ca33
--- /dev/null
+++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoDegeneratePath.java
@@ -0,0 +1,706 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.spatial3d.geom;
+
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.IOException;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * GeoShape representing a path across the surface of the globe,
+ * with a specified half-width. Path is described by a series of points.
+ * Distances are measured from the starting point along the path, and then at right
+ * angles to the path.
+ *
+ * @lucene.internal
+ */
+class GeoDegeneratePath extends GeoBasePath {
+
+ /** The original list of path points */
+ protected final List<GeoPoint> points = new ArrayList<GeoPoint>();
+
+ /** A list of SegmentEndpoints */
+ protected List<SegmentEndpoint> endPoints;
+ /** A list of PathSegments */
+ protected List<PathSegment> segments;
+
+ /** A point on the edge */
+ protected GeoPoint[] edgePoints;
+
+ /** Set to true if path has been completely constructed */
+ protected boolean isDone = false;
+
+ /** Constructor.
+ *@param planetModel is the planet model.
+ *@param pathPoints are the points in the path.
+ */
+ public GeoDegeneratePath(final PlanetModel planetModel, final GeoPoint[] pathPoints) {
+ this(planetModel);
+ Collections.addAll(points, pathPoints);
+ done();
+ }
+
+ /** Piece-wise constructor. Use in conjunction with addPoint() and done().
+ *@param planetModel is the planet model.
+ */
+ public GeoDegeneratePath(final PlanetModel planetModel) {
+ super(planetModel);
+ }
+
+ /** Add a point to the path.
+ *@param lat is the latitude of the point.
+ *@param lon is the longitude of the point.
+ */
+ public void addPoint(final double lat, final double lon) {
+ if (isDone)
+ throw new IllegalStateException("Can't call addPoint() if done() already called");
+ points.add(new GeoPoint(planetModel, lat, lon));
+ }
+
+ /** Complete the path.
+ */
+ public void done() {
+ if (isDone)
+ throw new IllegalStateException("Can't call done() twice");
+ if (points.size() == 0)
+ throw new IllegalArgumentException("Path must have at least one point");
+ isDone = true;
+
+ endPoints = new ArrayList<>(points.size());
+ segments = new ArrayList<>(points.size());
+
+ // First, build all segments. We'll then go back and build corresponding segment endpoints.
+ GeoPoint lastPoint = null;
+ for (final GeoPoint end : points) {
+ if (lastPoint != null) {
+ final Plane normalizedConnectingPlane = new Plane(lastPoint, end);
+ if (normalizedConnectingPlane == null) {
+ continue;
+ }
+ segments.add(new PathSegment(planetModel, lastPoint, end, normalizedConnectingPlane));
+ }
+ lastPoint = end;
+ }
+
+ if (segments.size() == 0) {
+ // Simple circle
+ final GeoPoint point = points.get(0);
+
+ final SegmentEndpoint onlyEndpoint = new SegmentEndpoint(point);
+ endPoints.add(onlyEndpoint);
+ this.edgePoints = new GeoPoint[]{point};
+ return;
+ }
+
+ // Create segment endpoints. Use an appropriate constructor for the start and end of the path.
+ for (int i = 0; i < segments.size(); i++) {
+ final PathSegment currentSegment = segments.get(i);
+
+ if (i == 0) {
+ // Starting endpoint
+ final SegmentEndpoint startEndpoint = new SegmentEndpoint(currentSegment.start,
+ new SidedPlane(currentSegment.startCutoffPlane));
+ endPoints.add(startEndpoint);
+ this.edgePoints = new GeoPoint[]{currentSegment.start};
+ continue;
+ }
+
+ endPoints.add(new SegmentEndpoint(currentSegment.start,
+ new SidedPlane(segments.get(i-1).endCutoffPlane),
+ new SidedPlane(currentSegment.startCutoffPlane)));
+ }
+ // Do final endpoint
+ final PathSegment lastSegment = segments.get(segments.size()-1);
+ endPoints.add(new SegmentEndpoint(lastSegment.end,
+ new SidedPlane(lastSegment.endCutoffPlane)));
+
+ }
+
+ /**
+ * Constructor for deserialization.
+ * @param planetModel is the planet model.
+ * @param inputStream is the input stream.
+ */
+ public GeoDegeneratePath(final PlanetModel planetModel, final InputStream inputStream) throws IOException {
+ this(planetModel,
+ SerializableObject.readPointArray(planetModel, inputStream));
+ }
+
+ @Override
+ public void write(final OutputStream outputStream) throws IOException {
+ SerializableObject.writePointArray(outputStream, points);
+ }
+
+ @Override
+ public double computeNearestDistance(final DistanceStyle distanceStyle, final double x, final double y, final double z) {
+ // Algorithm:
+ // (1) If the point is within any of the segments along the path, return that value.
+ // (2) If the point is within any of the segment end circles along the path, return that value.
+ double currentDistance = 0.0;
+ for (PathSegment segment : segments) {
+ double distance = segment.nearestPathDistance(planetModel, distanceStyle, x,y,z);
+ if (distance != Double.POSITIVE_INFINITY)
+ return distanceStyle.fromAggregationForm(distanceStyle.aggregateDistances(currentDistance, distance));
+ currentDistance = distanceStyle.aggregateDistances(currentDistance, segment.fullPathDistance(distanceStyle));
+ }
+
+ int segmentIndex = 0;
+ currentDistance = 0.0;
+ for (SegmentEndpoint endpoint : endPoints) {
+ double distance = endpoint.nearestPathDistance(distanceStyle, x, y, z);
+ if (distance != Double.POSITIVE_INFINITY) {
+ return distanceStyle.fromAggregationForm(distanceStyle.aggregateDistances(currentDistance, distance));
+ }
+ if (segmentIndex < segments.size()) {
+ currentDistance = distanceStyle.aggregateDistances(currentDistance, segments.get(segmentIndex++).fullPathDistance(distanceStyle));
+ }
+ }
+
+ return Double.POSITIVE_INFINITY;
+ }
+
+ @Override
+ protected double distance(final DistanceStyle distanceStyle, final double x, final double y, final double z) {
+ // Algorithm:
+ // (1) If the point is within any of the segments along the path, return that value.
+ // (2) If the point is within any of the segment end circles along the path, return that value.
+ double currentDistance = 0.0;
+ for (PathSegment segment : segments) {
+ double distance = segment.pathDistance(planetModel, distanceStyle, x,y,z);
+ if (distance != Double.POSITIVE_INFINITY)
+ return distanceStyle.fromAggregationForm(distanceStyle.aggregateDistances(currentDistance, distance));
+ currentDistance = distanceStyle.aggregateDistances(currentDistance, segment.fullPathDistance(distanceStyle));
+ }
+
+ int segmentIndex = 0;
+ currentDistance = 0.0;
+ for (SegmentEndpoint endpoint : endPoints) {
+ double distance = endpoint.pathDistance(distanceStyle, x, y, z);
+ if (distance != Double.POSITIVE_INFINITY)
+ return distanceStyle.fromAggregationForm(distanceStyle.aggregateDistances(currentDistance, distance));
+ if (segmentIndex < segments.size())
+ currentDistance = distanceStyle.aggregateDistances(currentDistance, segments.get(segmentIndex++).fullPathDistance(distanceStyle));
+ }
+
+ return Double.POSITIVE_INFINITY;
+ }
+
+ @Override
+ protected void distanceBounds(final Bounds bounds, final DistanceStyle distanceStyle, final double distanceValue) {
+ // TBD: Compute actual bounds based on distance
+ getBounds(bounds);
+ }
+
+ @Override
+ protected double outsideDistance(final DistanceStyle distanceStyle, final double x, final double y, final double z) {
+ double minDistance = Double.POSITIVE_INFINITY;
+ for (final SegmentEndpoint endpoint : endPoints) {
+ final double newDistance = endpoint.outsideDistance(distanceStyle, x,y,z);
+ if (newDistance < minDistance)
+ minDistance = newDistance;
+ }
+ for (final PathSegment segment : segments) {
+ final double newDistance = segment.outsideDistance(planetModel, distanceStyle, x, y, z);
+ if (newDistance < minDistance)
+ minDistance = newDistance;
+ }
+ return minDistance;
+ }
+
+ @Override
+ public boolean isWithin(final double x, final double y, final double z) {
+ for (SegmentEndpoint pathPoint : endPoints) {
+ if (pathPoint.isWithin(x, y, z)) {
+ return true;
+ }
+ }
+ for (PathSegment pathSegment : segments) {
+ if (pathSegment.isWithin(x, y, z)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ @Override
+ public GeoPoint[] getEdgePoints() {
+ return edgePoints;
+ }
+
+ @Override
+ public boolean intersects(final Plane plane, final GeoPoint[] notablePoints, final Membership... bounds) {
+ // We look for an intersection with any of the exterior edges of the path.
+ // We also have to look for intersections with the cones described by the endpoints.
+ // Return "true" if any such intersections are found.
+
+ // For plane intersections, the basic idea is to come up with an equation of the line that is
+ // the intersection (if any). Then, find the intersections with the unit sphere (if any). If
+ // any of the intersection points are within the bounds, then we've detected an intersection.
+ // Well, sort of. We can detect intersections also due to overlap of segments with each other.
+ // But that's an edge case and we won't be optimizing for it.
+ //System.err.println(" Looking for intersection of plane "+plane+" with path "+this);
+ for (final SegmentEndpoint pathPoint : endPoints) {
+ if (pathPoint.intersects(planetModel, plane, notablePoints, bounds)) {
+ return true;
+ }
+ }
+
+ for (final PathSegment pathSegment : segments) {
+ if (pathSegment.intersects(planetModel, plane, notablePoints, bounds)) {
+ return true;
+ }
+ }
+
+ return false;
+ }
+
+ @Override
+ public boolean intersects(GeoShape geoShape) {
+ for (final SegmentEndpoint pathPoint : endPoints) {
+ if (pathPoint.intersects(geoShape)) {
+ return true;
+ }
+ }
+
+ for (final PathSegment pathSegment : segments) {
+ if (pathSegment.intersects(geoShape)) {
+ return true;
+ }
+ }
+
+ return false;
+ }
+
+ @Override
+ public void getBounds(Bounds bounds) {
+ super.getBounds(bounds);
+ // For building bounds, order matters. We want to traverse
+ // never more than 180 degrees longitude at a pop or we risk having the
+ // bounds object get itself inverted. So do the edges first.
+ for (PathSegment pathSegment : segments) {
+ pathSegment.getBounds(planetModel, bounds);
+ }
+ for (SegmentEndpoint pathPoint : endPoints) {
+ pathPoint.getBounds(planetModel, bounds);
+ }
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (!(o instanceof GeoDegeneratePath))
+ return false;
+ GeoDegeneratePath p = (GeoDegeneratePath) o;
+ if (!super.equals(p))
+ return false;
+ return points.equals(p.points);
+ }
+
+ @Override
+ public int hashCode() {
+ int result = super.hashCode();
+ result = 31 * result + points.hashCode();
+ return result;
+ }
+
+ @Override
+ public String toString() {
+ return "GeoDegeneratePath: {planetmodel=" + planetModel+", points={" + points + "}}";
+ }
+
+ /**
+ * This is precalculated data for segment endpoint. Since the path is degenerate, there are several different cases:
+ * (1) The path consists of a single endpoint. In this case, the degenerate path consists of this one point.
+ * (2) This is the end of a path. There is a bounding plane passed in which describes the part of the world that is considered
+ * to belong to this endpoint.
+ * (3) Intersection. There are two cutoff planes, one for each end of the intersection.
+ */
+ private static class SegmentEndpoint {
+ /** The center point of the endpoint */
+ public final GeoPoint point;
+ /** Pertinent cutoff planes from adjoining segments */
+ public final Membership[] cutoffPlanes;
+ /** Notable points for this segment endpoint */
+ public final GeoPoint[] notablePoints;
+ /** No notable points from the circle itself */
+ public final static GeoPoint[] circlePoints = new GeoPoint[0];
+ /** Null membership */
+ public final static Membership[] NO_MEMBERSHIP = new Membership[0];
+
+ /** Constructor for case (1).
+ *@param point is the center point.
+ */
+ public SegmentEndpoint(final GeoPoint point) {
+ this.point = point;
+ this.cutoffPlanes = NO_MEMBERSHIP;
+ this.notablePoints = circlePoints;
+ }
+
+ /** Constructor for case (2).
+ * Generate an endpoint, given a single cutoff plane plus upper and lower edge points.
+ *@param point is the center point.
+ *@param cutoffPlane is the plane from the adjoining path segment marking the boundary between this endpoint and that segment.
+ */
+ public SegmentEndpoint(final GeoPoint point, final SidedPlane cutoffPlane) {
+ this.point = point;
+ this.cutoffPlanes = new Membership[]{new SidedPlane(cutoffPlane)};
+ this.notablePoints = new GeoPoint[]{point};
+ }
+
+ /** Constructor for case (3).
+ * Generate an endpoint, given two cutoff planes.
+ *@param point is the center.
+ *@param cutoffPlane1 is one adjoining path segment cutoff plane.
+ *@param cutoffPlane2 is another adjoining path segment cutoff plane.
+ */
+ public SegmentEndpoint(final GeoPoint point,
+ final SidedPlane cutoffPlane1, final SidedPlane cutoffPlane2) {
+ this.point = point;
+ this.cutoffPlanes = new Membership[]{new SidedPlane(cutoffPlane1), new SidedPlane(cutoffPlane2)};
+ this.notablePoints = new GeoPoint[]{point};
+ }
+
+ /** Check if point is within this endpoint.
+ *@param point is the point.
+ *@return true of within.
+ */
+ public boolean isWithin(final Vector point) {
+ return this.point.isIdentical(point.x, point.y, point.z);
+ }
+
+ /** Check if point is within this endpoint.
+ *@param x is the point x.
+ *@param y is the point y.
+ *@param z is the point z.
+ *@return true of within.
+ */
+ public boolean isWithin(final double x, final double y, final double z) {
+ return this.point.isIdentical(x, y, z);
+ }
+
+ /** Compute interior path distance.
+ *@param distanceStyle is the distance style.
+ *@param x is the point x.
+ *@param y is the point y.
+ *@param z is the point z.
+ *@return the distance metric, in aggregation form.
+ */
+ public double pathDistance(final DistanceStyle distanceStyle, final double x, final double y, final double z) {
+ if (!isWithin(x,y,z))
+ return Double.POSITIVE_INFINITY;
+ return distanceStyle.toAggregationForm(distanceStyle.computeDistance(this.point, x, y, z));
+ }
+
+ /** Compute nearest path distance.
+ *@param distanceStyle is the distance style.
+ *@param x is the point x.
+ *@param y is the point y.
+ *@param z is the point z.
+ *@return the distance metric (always value zero), in aggregation form, or POSITIVE_INFINITY
+ * if the point is not within the bounds of the endpoint.
+ */
+ public double nearestPathDistance(final DistanceStyle distanceStyle, final double x, final double y, final double z) {
+ for (final Membership m : cutoffPlanes) {
+ if (!m.isWithin(x,y,z)) {
+ return Double.POSITIVE_INFINITY;
+ }
+ }
+ return distanceStyle.toAggregationForm(0.0);
+ }
+
+ /** Compute external distance.
+ *@param distanceStyle is the distance style.
+ *@param x is the point x.
+ *@param y is the point y.
+ *@param z is the point z.
+ *@return the distance metric.
+ */
+ public double outsideDistance(final DistanceStyle distanceStyle, final double x, final double y, final double z) {
+ return distanceStyle.computeDistance(this.point, x, y, z);
+ }
+
+ /** Determine if this endpoint intersects a specified plane.
+ *@param planetModel is the planet model.
+ *@param p is the plane.
+ *@param notablePoints are the points associated with the plane.
+ *@param bounds are any bounds which the intersection must lie within.
+ *@return true if there is a matching intersection.
+ */
+ public boolean intersects(final PlanetModel planetModel, final Plane p, final GeoPoint[] notablePoints, final Membership[] bounds) {
+ // If not on the plane, no intersection
+ if (!p.evaluateIsZero(point))
+ return false;
+
+ for (Membership m : bounds) {
+ if (!m.isWithin(point))
+ return false;
+ }
+ return true;
+ }
+
+ /** Determine if this endpoint intersects a GeoShape.
+ *@param geoShape is the GeoShape.
+ *@return true if there is shape intersect this endpoint.
+ */
+ public boolean intersects(final GeoShape geoShape) {
+ return geoShape.isWithin(point);
+ }
+
+ /** Get the bounds for a segment endpoint.
+ *@param planetModel is the planet model.
+ *@param bounds are the bounds to be modified.
+ */
+ public void getBounds(final PlanetModel planetModel, Bounds bounds) {
+ bounds.addPoint(point);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (!(o instanceof SegmentEndpoint))
+ return false;
+ SegmentEndpoint other = (SegmentEndpoint) o;
+ return point.equals(other.point);
+ }
+
+ @Override
+ public int hashCode() {
+ return point.hashCode();
+ }
+
+ @Override
+ public String toString() {
+ return point.toString();
+ }
+ }
+
+ /**
+ * This is the pre-calculated data for a path segment.
+ */
+ private static class PathSegment {
+ /** Starting point of the segment */
+ public final GeoPoint start;
+ /** End point of the segment */
+ public final GeoPoint end;
+ /** Place to keep any complete segment distances we've calculated so far */
+ public final Map<DistanceStyle,Double> fullDistanceCache = new HashMap<DistanceStyle,Double>();
+ /** Normalized plane connecting the two points and going through world center */
+ public final Plane normalizedConnectingPlane;
+ /** Plane going through the center and start point, marking the start edge of the segment */
+ public final SidedPlane startCutoffPlane;
+ /** Plane going through the center and end point, marking the end edge of the segment */
+ public final SidedPlane endCutoffPlane;
+ /** Notable points for the connecting plane */
+ public final GeoPoint[] connectingPlanePoints;
+
+ /** Construct a path segment.
+ *@param planetModel is the planet model.
+ *@param start is the starting point.
+ *@param end is the ending point.
+ *@param normalizedConnectingPlane is the connecting plane.
+ */
+ public PathSegment(final PlanetModel planetModel, final GeoPoint start, final GeoPoint end,
+ final Plane normalizedConnectingPlane) {
+ this.start = start;
+ this.end = end;
+ this.normalizedConnectingPlane = normalizedConnectingPlane;
+
+ // Cutoff planes use opposite endpoints as correct side examples
+ startCutoffPlane = new SidedPlane(end, normalizedConnectingPlane, start);
+ endCutoffPlane = new SidedPlane(start, normalizedConnectingPlane, end);
+ connectingPlanePoints = new GeoPoint[]{start, end};
+ }
+
+ /** Compute the full distance along this path segment.
+ *@param distanceStyle is the distance style.
+ *@return the distance metric, in aggregation form.
+ */
+ public double fullPathDistance(final DistanceStyle distanceStyle) {
+ synchronized (fullDistanceCache) {
+ Double dist = fullDistanceCache.get(distanceStyle);
+ if (dist == null) {
+ dist = new Double(distanceStyle.toAggregationForm(distanceStyle.computeDistance(start, end.x, end.y, end.z)));
+ fullDistanceCache.put(distanceStyle, dist);
+ }
+ return dist.doubleValue();
+ }
+ }
+
+ /** Check if point is within this segment.
+ *@param point is the point.
+ *@return true of within.
+ */
+ public boolean isWithin(final Vector point) {
+ return startCutoffPlane.isWithin(point) &&
+ endCutoffPlane.isWithin(point) &&
+ normalizedConnectingPlane.evaluateIsZero(point);
+ }
+
+ /** Check if point is within this segment.
+ *@param x is the point x.
+ *@param y is the point y.
+ *@param z is the point z.
+ *@return true of within.
+ */
+ public boolean isWithin(final double x, final double y, final double z) {
+ return startCutoffPlane.isWithin(x, y, z) &&
+ endCutoffPlane.isWithin(x, y, z) &&
+ normalizedConnectingPlane.evaluateIsZero(x, y, z);
+ }
+
+ /** Compute nearest path distance.
+ *@param planetModel is the planet model.
+ *@param distanceStyle is the distance style.
+ *@param x is the point x.
+ *@param y is the point y.
+ *@param z is the point z.
+ *@return the distance metric, in aggregation form, or Double.POSITIVE_INFINITY if outside this segment
+ */
+ public double nearestPathDistance(final PlanetModel planetModel, final DistanceStyle distanceStyle, final double x, final double y, final double z) {
+ // First, if this point is outside the endplanes of the segment, return POSITIVE_INFINITY.
+ if (!startCutoffPlane.isWithin(x, y, z) || !endCutoffPlane.isWithin(x, y, z)) {
+ return Double.POSITIVE_INFINITY;
+ }
+ // (1) Compute normalizedPerpPlane. If degenerate, then there is no such plane, which means that the point given
+ // is insufficient to distinguish between a family of such planes. This can happen only if the point is one of the
+ // "poles", imagining the normalized plane to be the "equator". In that case, the distance returned should be zero.
+ // Want no allocations or expensive operations! so we do this the hard way
+ final double perpX = normalizedConnectingPlane.y * z - normalizedConnectingPlane.z * y;
+ final double perpY = normalizedConnectingPlane.z * x - normalizedConnectingPlane.x * z;
+ final double perpZ = normalizedConnectingPlane.x * y - normalizedConnectingPlane.y * x;
+ final double magnitude = Math.sqrt(perpX * perpX + perpY * perpY + perpZ * perpZ);
+ if (Math.abs(magnitude) < Vector.MINIMUM_RESOLUTION)
+ return distanceStyle.toAggregationForm(0.0);
+ final double normFactor = 1.0/magnitude;
+ final Plane normalizedPerpPlane = new Plane(perpX * normFactor, perpY * normFactor, perpZ * normFactor, 0.0);
+
+ final GeoPoint[] intersectionPoints = normalizedConnectingPlane.findIntersections(planetModel, normalizedPerpPlane);
+ GeoPoint thePoint;
+ if (intersectionPoints.length == 0)
+ throw new RuntimeException("Can't find world intersection for point x="+x+" y="+y+" z="+z);
+ else if (intersectionPoints.length == 1)
+ thePoint = intersectionPoints[0];
+ else {
+ if (startCutoffPlane.isWithin(intersectionPoints[0]) && endCutoffPlane.isWithin(intersectionPoints[0]))
+ thePoint = intersectionPoints[0];
+ else if (startCutoffPlane.isWithin(intersectionPoints[1]) && endCutoffPlane.isWithin(intersectionPoints[1]))
+ thePoint = intersectionPoints[1];
+ else
+ throw new RuntimeException("Can't find world intersection for point x="+x+" y="+y+" z="+z);
+ }
+ return distanceStyle.toAggregationForm(distanceStyle.computeDistance(start, thePoint.x, thePoint.y, thePoint.z));
+ }
+
+
+ /** Compute interior path distance.
+ *@param planetModel is the planet model.
+ *@param distanceStyle is the distance style.
+ *@param x is the point x.
+ *@param y is the point y.
+ *@param z is the point z.
+ *@return the distance metric, in aggregation form.
+ */
+ public double pathDistance(final PlanetModel planetModel, final DistanceStyle distanceStyle, final double x, final double y, final double z) {
+ if (!isWithin(x,y,z))
+ return Double.POSITIVE_INFINITY;
+
+ // (1) Compute normalizedPerpPlane. If degenerate, then return point distance from start to point.
+ // Want no allocations or expensive operations! so we do this the hard way
+ final double perpX = normalizedConnectingPlane.y * z - normalizedConnectingPlane.z * y;
+ final double perpY = normalizedConnectingPlane.z * x - normalizedConnectingPlane.x * z;
+ final double perpZ = normalizedConnectingPlane.x * y - normalizedConnectingPlane.y * x;
+ final double magnitude = Math.sqrt(perpX * perpX + perpY * perpY + perpZ * perpZ);
+ if (Math.abs(magnitude) < Vector.MINIMUM_RESOLUTION)
+ return distanceStyle.toAggregationForm(distanceStyle.computeDistance(start, x,y,z));
+ final double normFactor = 1.0/magnitude;
+ final Plane normalizedPerpPlane = new Plane(perpX * normFactor, perpY * normFactor, perpZ * normFactor, 0.0);
+
+ // Old computation: too expensive, because it calculates the intersection point twice.
+ //return distanceStyle.computeDistance(planetModel, normalizedConnectingPlane, x, y, z, startCutoffPlane, endCutoffPlane) +
+ // distanceStyle.computeDistance(planetModel, normalizedPerpPlane, start.x, start.y, start.z, upperConnectingPlane, lowerConnectingPlane);
+
+ final GeoPoint[] intersectionPoints = normalizedConnectingPlane.findIntersections(planetModel, normalizedPerpPlane);
+ GeoPoint thePoint;
+ if (intersectionPoints.length == 0)
+ throw new RuntimeException("Can't find world intersection for point x="+x+" y="+y+" z="+z);
+ else if (intersectionPoints.length == 1)
+ thePoint = intersectionPoints[0];
+ else {
+ if (startCutoffPlane.isWithin(intersectionPoints[0]) && endCutoffPlane.isWithin(intersectionPoints[0]))
+ thePoint = intersectionPoints[0];
+ else if (startCutoffPlane.isWithin(intersectionPoints[1]) && endCutoffPlane.isWithin(intersectionPoints[1]))
+ thePoint = intersectionPoints[1];
+ else
+ throw new RuntimeException("Can't find world intersection for point x="+x+" y="+y+" z="+z);
+ }
+ return distanceStyle.aggregateDistances(distanceStyle.toAggregationForm(distanceStyle.computeDistance(thePoint, x, y, z)),
+ distanceStyle.toAggregationForm(distanceStyle.computeDistance(start, thePoint.x, thePoint.y, thePoint.z)));
+ }
+
+ /** Compute external distance.
+ *@param planetModel is the planet model.
+ *@param distanceStyle is the distance style.
+ *@param x is the point x.
+ *@param y is the point y.
+ *@param z is the point z.
+ *@return the distance metric.
+ */
+ public double outsideDistance(final PlanetModel planetModel, final DistanceStyle distanceStyle, final double x, final double y, final double z) {
+ final double distance = distanceStyle.computeDistance(planetModel, normalizedConnectingPlane, x,y,z, startCutoffPlane, endCutoffPlane);
+ final double startDistance = distanceStyle.computeDistance(start, x,y,z);
+ final double endDistance = distanceStyle.computeDistance(end, x,y,z);
+ return Math.min(
+ Math.min(startDistance, endDistance),
+ distance);
+ }
+
+ /** Determine if this endpoint intersects a specified plane.
+ *@param planetModel is the planet model.
+ *@param p is the plane.
+ *@param notablePoints are the points associated with the plane.
+ *@param bounds are any bounds which the intersection must lie within.
+ *@return true if there is a matching intersection.
+ */
+ public boolean intersects(final PlanetModel planetModel, final Plane p, final GeoPoint[] notablePoints, final Membership[] bounds) {
+ return normalizedConnectingPlane.intersects(planetModel, p, connectingPlanePoints, notablePoints, bounds, startCutoffPlane, endCutoffPlane);
+ }
+
+ /** Determine if this endpoint intersects a specified GeoShape.
+ *@param geoShape is the GeoShape.
+ *@return true if there GeoShape intersects this endpoint.
+ */
+ public boolean intersects(final GeoShape geoShape) {
+ return geoShape.intersects(normalizedConnectingPlane, connectingPlanePoints, startCutoffPlane, endCutoffPlane);
+ }
+
+ /** Get the bounds for a segment endpoint.
+ *@param planetModel is the planet model.
+ *@param bounds are the bounds to be modified.
+ */
+ public void getBounds(final PlanetModel planetModel, Bounds bounds) {
+ // We need to do all bounding planes as well as corner points
+ bounds.addPoint(start).addPoint(end)
+ .addPlane(planetModel, normalizedConnectingPlane, startCutoffPlane, endCutoffPlane);
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c09678e5/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPathFactory.java
----------------------------------------------------------------------
diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPathFactory.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPathFactory.java
index 3fd7cf7..2ca132f 100644
--- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPathFactory.java
+++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPathFactory.java
@@ -33,6 +33,9 @@ public class GeoPathFactory {
* @return a GeoPath corresponding to what was specified.
*/
public static GeoPath makeGeoPath(final PlanetModel planetModel, final double maxCutoffAngle, final GeoPoint[] pathPoints) {
+ if (maxCutoffAngle < Vector.MINIMUM_ANGULAR_RESOLUTION) {
+ return new GeoDegeneratePath(planetModel, pathPoints);
+ }
return new GeoStandardPath(planetModel, maxCutoffAngle, pathPoints);
}
[37/50] lucene-solr:feature/autoscaling: SOLR-11332,
SOLR-11132: Fix sorting on 'enum' fieldTypes that use
sortMissingFirst or sortMissingLast and Refactor common getSortField logic in
various FieldTypes
Posted by sh...@apache.org.
SOLR-11332, SOLR-11132: Fix sorting on 'enum' fieldTypes that use sortMissingFirst or sortMissingLast and Refactor common getSortField logic in various FieldTypes
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/31eab319
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/31eab319
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/31eab319
Branch: refs/heads/feature/autoscaling
Commit: 31eab319f4b2632f1be488be3c1008b3567c6142
Parents: 37b8ac6
Author: Chris Hostetter <ho...@apache.org>
Authored: Mon Sep 11 10:38:30 2017 -0700
Committer: Chris Hostetter <ho...@apache.org>
Committed: Mon Sep 11 10:38:30 2017 -0700
----------------------------------------------------------------------
solr/CHANGES.txt | 4 ++
.../apache/solr/schema/AbstractEnumField.java | 11 ++--
.../org/apache/solr/schema/DatePointField.java | 15 +----
.../apache/solr/schema/DoublePointField.java | 15 +----
.../java/org/apache/solr/schema/FieldType.java | 65 +++++++++++++++++++-
.../org/apache/solr/schema/FloatPointField.java | 15 +----
.../org/apache/solr/schema/IntPointField.java | 15 +----
.../org/apache/solr/schema/LongPointField.java | 15 +----
.../apache/solr/schema/PreAnalyzedField.java | 6 +-
.../java/org/apache/solr/schema/TextField.java | 5 +-
.../java/org/apache/solr/schema/TrieField.java | 44 ++-----------
.../java/org/apache/solr/search/Sorting.java | 12 +++-
.../solr/collection1/conf/schema-enums.xml | 10 +++
.../org/apache/solr/schema/EnumFieldTest.java | 16 +++++
.../test/org/apache/solr/search/TestSort.java | 21 ++++++-
15 files changed, 139 insertions(+), 130 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/31eab319/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index d278821..519852e 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -129,6 +129,8 @@ Bug Fixes
* SOLR-10101: TestLazyCores hangs (Erick Erickson)
+* SOLR-11332: Fix sorting on 'enum' fieldTypes that use sortMissingFirst or sortMissingLast (hossman)
+
Optimizations
----------------------
@@ -199,6 +201,8 @@ Other Changes
* SOLR-10990: Breakup QueryComponent.process method for readability. (Christine Poerschke)
+* SOLR-11132: Refactor common getSortField logic in various FieldTypes (Jason Gerlowski, hossman)
+
================== 7.0.0 ==================
Versions of Major Components
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/31eab319/solr/core/src/java/org/apache/solr/schema/AbstractEnumField.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/schema/AbstractEnumField.java b/solr/core/src/java/org/apache/solr/schema/AbstractEnumField.java
index 1111cec..d4ce268 100644
--- a/solr/core/src/java/org/apache/solr/schema/AbstractEnumField.java
+++ b/solr/core/src/java/org/apache/solr/schema/AbstractEnumField.java
@@ -250,11 +250,12 @@ public abstract class AbstractEnumField extends PrimitiveFieldType {
@Override
public SortField getSortField(SchemaField field, boolean top) {
- field.checkSortability();
- final Object missingValue = Integer.MIN_VALUE;
- SortField sf = new SortField(field.getName(), SortField.Type.INT, top);
- sf.setMissingValue(missingValue);
- return sf;
+ SortField result = getSortField(field, SortField.Type.INT, top, Integer.MIN_VALUE, Integer.MAX_VALUE);
+ if (null == result.getMissingValue()) {
+ // special case default behavior: assume missing values are "below" all enum values
+ result.setMissingValue(Integer.MIN_VALUE);
+ }
+ return result;
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/31eab319/solr/core/src/java/org/apache/solr/schema/DatePointField.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/schema/DatePointField.java b/solr/core/src/java/org/apache/solr/schema/DatePointField.java
index e43f706..4861917 100644
--- a/solr/core/src/java/org/apache/solr/schema/DatePointField.java
+++ b/solr/core/src/java/org/apache/solr/schema/DatePointField.java
@@ -190,20 +190,7 @@ public class DatePointField extends PointField implements DateValueFieldType {
@Override
public SortField getSortField(SchemaField field, boolean top) {
- field.checkSortability();
-
- Object missingValue = null;
- boolean sortMissingLast = field.sortMissingLast();
- boolean sortMissingFirst = field.sortMissingFirst();
-
- if (sortMissingLast) {
- missingValue = top ? Long.MIN_VALUE : Long.MAX_VALUE;
- } else if (sortMissingFirst) {
- missingValue = top ? Long.MAX_VALUE : Long.MIN_VALUE;
- }
- SortField sf = new SortField(field.getName(), SortField.Type.LONG, top);
- sf.setMissingValue(missingValue);
- return sf;
+ return getSortField(field, SortField.Type.LONG, top, Long.MIN_VALUE, Long.MAX_VALUE);
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/31eab319/solr/core/src/java/org/apache/solr/schema/DoublePointField.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/schema/DoublePointField.java b/solr/core/src/java/org/apache/solr/schema/DoublePointField.java
index edc7c38..ba71a8a 100644
--- a/solr/core/src/java/org/apache/solr/schema/DoublePointField.java
+++ b/solr/core/src/java/org/apache/solr/schema/DoublePointField.java
@@ -134,20 +134,7 @@ public class DoublePointField extends PointField implements DoubleValueFieldType
@Override
public SortField getSortField(SchemaField field, boolean top) {
- field.checkSortability();
-
- Object missingValue = null;
- boolean sortMissingLast = field.sortMissingLast();
- boolean sortMissingFirst = field.sortMissingFirst();
-
- if (sortMissingLast) {
- missingValue = top ? Double.NEGATIVE_INFINITY : Double.POSITIVE_INFINITY;
- } else if (sortMissingFirst) {
- missingValue = top ? Double.POSITIVE_INFINITY : Double.NEGATIVE_INFINITY;
- }
- SortField sf = new SortField(field.getName(), SortField.Type.DOUBLE, top);
- sf.setMissingValue(missingValue);
- return sf;
+ return getSortField(field, SortField.Type.DOUBLE, top, Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY);
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/31eab319/solr/core/src/java/org/apache/solr/schema/FieldType.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/schema/FieldType.java b/solr/core/src/java/org/apache/solr/schema/FieldType.java
index 3d38844..5494cf1 100644
--- a/solr/core/src/java/org/apache/solr/schema/FieldType.java
+++ b/solr/core/src/java/org/apache/solr/schema/FieldType.java
@@ -48,6 +48,7 @@ import org.apache.lucene.search.MultiTermQuery;
import org.apache.lucene.search.PrefixQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.SortedSetSortField;
import org.apache.lucene.search.SortedNumericSelector;
import org.apache.lucene.search.SortedSetSelector;
import org.apache.lucene.search.TermInSetQuery;
@@ -69,7 +70,6 @@ import org.apache.solr.query.SolrRangeQuery;
import org.apache.solr.response.TextResponseWriter;
import org.apache.solr.search.QParser;
import org.apache.solr.search.QueryUtils;
-import org.apache.solr.search.Sorting;
import org.apache.solr.uninverting.UninvertingReader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -662,17 +662,76 @@ public abstract class FieldType extends FieldProperties {
* Returns the SortField instance that should be used to sort fields
* of this type.
* @see SchemaField#checkSortability
+ * @see #getSortField(SchemaField,SortField.Type,boolean,Object,Object)
*/
public abstract SortField getSortField(SchemaField field, boolean top);
/**
+ * <p>A Helper utility method for use by subclasses.</p>
+ * <p>This method deals with:</p>
+ * <ul>
+ * <li>{@link SchemaField#checkSortability}</li>
+ * <li>Creating a {@link SortField} on <code>field</code> with the specified
+ * <code>reverse</code> & <code>sortType</code></li>
+ * <li>Setting the {@link SortField#setMissingValue} to <code>missingLow</code> or <code>missingHigh</code>
+ * as appropriate based on the value of <code>reverse</code> and the
+ * <code>sortMissingFirst</code> & <code>sortMissingLast</code> properties of the
+ * <code>field</code></li>
+ * </ul>
+ *
+ * @param field The SchemaField to sort on. May use <code>sortMissingFirst</code> or <code>sortMissingLast</code> or neither.
+ * @param sortType The sort Type of the underlying values in the <code>field</code>
+ * @param reverse True if natural order of the <code>sortType</code> should be reversed
+ * @param missingLow The <code>missingValue</code> to be used if the other params indicate that docs w/o values should sort as "low" as possible.
+ * @param missingHigh The <code>missingValue</code> to be used if the other params indicate that docs w/o values should sort as "high" as possible.
+ * @see #getSortedSetSortField
+ */
+ protected static SortField getSortField(SchemaField field, SortField.Type sortType, boolean reverse,
+ Object missingLow, Object missingHigh) {
+ field.checkSortability();
+
+ SortField sf = new SortField(field.getName(), sortType, reverse);
+ applySetMissingValue(field, sf, missingLow, missingHigh);
+
+ return sf;
+ }
+
+ /**
+ * Same as {@link #getSortField} but using {@link SortedSetSortField}
+ */
+ protected static SortField getSortedSetSortField(SchemaField field, SortedSetSelector.Type selector,
+ boolean reverse, Object missingLow, Object missingHigh) {
+
+ field.checkSortability();
+
+ SortField sf = new SortedSetSortField(field.getName(), reverse, selector);
+ applySetMissingValue(field, sf, missingLow, missingHigh);
+
+ return sf;
+ }
+
+ /**
+ * @see #getSortField
+ * @see #getSortedSetSortField
+ */
+ private static void applySetMissingValue(SchemaField field, SortField sortField,
+ Object missingLow, Object missingHigh) {
+ final boolean reverse = sortField.getReverse();
+
+ if (field.sortMissingLast()) {
+ sortField.setMissingValue(reverse ? missingLow : missingHigh);
+ } else if (field.sortMissingFirst()) {
+ sortField.setMissingValue(reverse ? missingHigh : missingLow);
+ }
+ }
+
+ /**
* Utility usable by subclasses when they want to get basic String sorting
* using common checks.
* @see SchemaField#checkSortability
*/
protected SortField getStringSort(SchemaField field, boolean reverse) {
- field.checkSortability();
- return Sorting.getStringSortField(field.name, reverse, field.sortMissingLast(),field.sortMissingFirst());
+ return getSortField(field, SortField.Type.STRING, reverse, SortField.STRING_FIRST, SortField.STRING_LAST);
}
/** called to get the default value source (normally, from the
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/31eab319/solr/core/src/java/org/apache/solr/schema/FloatPointField.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/schema/FloatPointField.java b/solr/core/src/java/org/apache/solr/schema/FloatPointField.java
index c70655a..f69a1db 100644
--- a/solr/core/src/java/org/apache/solr/schema/FloatPointField.java
+++ b/solr/core/src/java/org/apache/solr/schema/FloatPointField.java
@@ -134,20 +134,7 @@ public class FloatPointField extends PointField implements FloatValueFieldType {
@Override
public SortField getSortField(SchemaField field, boolean top) {
- field.checkSortability();
-
- Object missingValue = null;
- boolean sortMissingLast = field.sortMissingLast();
- boolean sortMissingFirst = field.sortMissingFirst();
-
- if (sortMissingLast) {
- missingValue = top ? Float.NEGATIVE_INFINITY : Float.POSITIVE_INFINITY;
- } else if (sortMissingFirst) {
- missingValue = top ? Float.POSITIVE_INFINITY : Float.NEGATIVE_INFINITY;
- }
- SortField sf = new SortField(field.getName(), SortField.Type.FLOAT, top);
- sf.setMissingValue(missingValue);
- return sf;
+ return getSortField(field, SortField.Type.FLOAT, top, Float.NEGATIVE_INFINITY, Float.POSITIVE_INFINITY);
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/31eab319/solr/core/src/java/org/apache/solr/schema/IntPointField.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/schema/IntPointField.java b/solr/core/src/java/org/apache/solr/schema/IntPointField.java
index d5e4aec..b179c57 100644
--- a/solr/core/src/java/org/apache/solr/schema/IntPointField.java
+++ b/solr/core/src/java/org/apache/solr/schema/IntPointField.java
@@ -132,20 +132,7 @@ public class IntPointField extends PointField implements IntValueFieldType {
@Override
public SortField getSortField(SchemaField field, boolean top) {
- field.checkSortability();
-
- Object missingValue = null;
- boolean sortMissingLast = field.sortMissingLast();
- boolean sortMissingFirst = field.sortMissingFirst();
-
- if (sortMissingLast) {
- missingValue = top ? Integer.MIN_VALUE : Integer.MAX_VALUE;
- } else if (sortMissingFirst) {
- missingValue = top ? Integer.MAX_VALUE : Integer.MIN_VALUE;
- }
- SortField sf = new SortField(field.getName(), SortField.Type.INT, top);
- sf.setMissingValue(missingValue);
- return sf;
+ return getSortField(field, SortField.Type.INT, top, Integer.MIN_VALUE, Integer.MAX_VALUE);
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/31eab319/solr/core/src/java/org/apache/solr/schema/LongPointField.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/schema/LongPointField.java b/solr/core/src/java/org/apache/solr/schema/LongPointField.java
index 7e3f5e1..547725b 100644
--- a/solr/core/src/java/org/apache/solr/schema/LongPointField.java
+++ b/solr/core/src/java/org/apache/solr/schema/LongPointField.java
@@ -131,20 +131,7 @@ public class LongPointField extends PointField implements LongValueFieldType {
@Override
public SortField getSortField(SchemaField field, boolean top) {
- field.checkSortability();
-
- Object missingValue = null;
- boolean sortMissingLast = field.sortMissingLast();
- boolean sortMissingFirst = field.sortMissingFirst();
-
- if (sortMissingLast) {
- missingValue = top ? Long.MIN_VALUE : Long.MAX_VALUE;
- } else if (sortMissingFirst) {
- missingValue = top ? Long.MAX_VALUE : Long.MIN_VALUE;
- }
- SortField sf = new SortField(field.getName(), SortField.Type.LONG, top);
- sf.setMissingValue(missingValue);
- return sf;
+ return getSortField(field, SortField.Type.LONG, top, Long.MIN_VALUE, Long.MAX_VALUE);
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/31eab319/solr/core/src/java/org/apache/solr/schema/PreAnalyzedField.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/schema/PreAnalyzedField.java b/solr/core/src/java/org/apache/solr/schema/PreAnalyzedField.java
index d2dc811..f5affe1 100644
--- a/solr/core/src/java/org/apache/solr/schema/PreAnalyzedField.java
+++ b/solr/core/src/java/org/apache/solr/schema/PreAnalyzedField.java
@@ -34,13 +34,13 @@ import org.apache.lucene.index.IndexableField;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.valuesource.SortedSetFieldSource;
import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.SortedSetSelector;
import org.apache.lucene.util.AttributeFactory;
import org.apache.lucene.util.AttributeSource.State;
import org.apache.lucene.util.AttributeSource;
import org.apache.solr.analysis.SolrAnalyzer;
import org.apache.solr.response.TextResponseWriter;
import org.apache.solr.search.QParser;
-import org.apache.solr.search.Sorting;
import org.apache.solr.uninverting.UninvertingReader.Type;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -132,8 +132,8 @@ public class PreAnalyzedField extends TextField implements HasImplicitIndexAnaly
@Override
public SortField getSortField(SchemaField field, boolean top) {
- field.checkSortability();
- return Sorting.getTextSortField(field.getName(), top, field.sortMissingLast(), field.sortMissingFirst());
+ return getSortedSetSortField(field, SortedSetSelector.Type.MIN, top,
+ SortField.STRING_FIRST, SortField.STRING_LAST);
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/31eab319/solr/core/src/java/org/apache/solr/schema/TextField.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/schema/TextField.java b/solr/core/src/java/org/apache/solr/schema/TextField.java
index d8bae24..effadc4 100644
--- a/solr/core/src/java/org/apache/solr/schema/TextField.java
+++ b/solr/core/src/java/org/apache/solr/schema/TextField.java
@@ -32,7 +32,6 @@ import org.apache.solr.common.SolrException;
import org.apache.solr.query.SolrRangeQuery;
import org.apache.solr.response.TextResponseWriter;
import org.apache.solr.search.QParser;
-import org.apache.solr.search.Sorting;
import org.apache.solr.uninverting.UninvertingReader.Type;
/** <code>TextField</code> is the basic type for configurable text analysis.
@@ -108,8 +107,8 @@ public class TextField extends FieldType {
@Override
public SortField getSortField(SchemaField field, boolean reverse) {
/* :TODO: maybe warn if isTokenized(), but doesn't use LimitTokenCountFilter in its chain? */
- field.checkSortability();
- return Sorting.getTextSortField(field.getName(), reverse, field.sortMissingLast(), field.sortMissingFirst());
+ return getSortedSetSortField(field, SortedSetSelector.Type.MIN, reverse,
+ SortField.STRING_FIRST, SortField.STRING_LAST);
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/31eab319/solr/core/src/java/org/apache/solr/schema/TrieField.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/schema/TrieField.java b/solr/core/src/java/org/apache/solr/schema/TrieField.java
index 9e57159..ebe2103 100644
--- a/solr/core/src/java/org/apache/solr/schema/TrieField.java
+++ b/solr/core/src/java/org/apache/solr/schema/TrieField.java
@@ -171,50 +171,14 @@ public class TrieField extends NumericFieldType {
switch (type) {
case INTEGER:
- if( sortMissingLast ) {
- missingValue = top ? Integer.MIN_VALUE : Integer.MAX_VALUE;
- }
- else if( sortMissingFirst ) {
- missingValue = top ? Integer.MAX_VALUE : Integer.MIN_VALUE;
- }
- sf = new SortField( field.getName(), SortField.Type.INT, top);
- sf.setMissingValue(missingValue);
- return sf;
-
+ return getSortField(field, SortField.Type.INT, top, Integer.MIN_VALUE, Integer.MAX_VALUE);
case FLOAT:
- if( sortMissingLast ) {
- missingValue = top ? Float.NEGATIVE_INFINITY : Float.POSITIVE_INFINITY;
- }
- else if( sortMissingFirst ) {
- missingValue = top ? Float.POSITIVE_INFINITY : Float.NEGATIVE_INFINITY;
- }
- sf = new SortField( field.getName(), SortField.Type.FLOAT, top);
- sf.setMissingValue(missingValue);
- return sf;
-
+ return getSortField(field, SortField.Type.FLOAT, top, Float.NEGATIVE_INFINITY, Float.POSITIVE_INFINITY);
case DATE: // fallthrough
case LONG:
- if( sortMissingLast ) {
- missingValue = top ? Long.MIN_VALUE : Long.MAX_VALUE;
- }
- else if( sortMissingFirst ) {
- missingValue = top ? Long.MAX_VALUE : Long.MIN_VALUE;
- }
- sf = new SortField( field.getName(), SortField.Type.LONG, top);
- sf.setMissingValue(missingValue);
- return sf;
-
+ return getSortField(field, SortField.Type.LONG, top, Long.MIN_VALUE, Long.MAX_VALUE);
case DOUBLE:
- if( sortMissingLast ) {
- missingValue = top ? Double.NEGATIVE_INFINITY : Double.POSITIVE_INFINITY;
- }
- else if( sortMissingFirst ) {
- missingValue = top ? Double.POSITIVE_INFINITY : Double.NEGATIVE_INFINITY;
- }
- sf = new SortField( field.getName(), SortField.Type.DOUBLE, top);
- sf.setMissingValue(missingValue);
- return sf;
-
+ return getSortField(field, SortField.Type.DOUBLE, top, Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY);
default:
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown type for trie field: " + field.name);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/31eab319/solr/core/src/java/org/apache/solr/search/Sorting.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/Sorting.java b/solr/core/src/java/org/apache/solr/search/Sorting.java
index 8bb2171..c23b558 100644
--- a/solr/core/src/java/org/apache/solr/search/Sorting.java
+++ b/solr/core/src/java/org/apache/solr/search/Sorting.java
@@ -16,15 +16,16 @@
*/
package org.apache.solr.search;
+import org.apache.solr.schema.FieldType;
import org.apache.lucene.search.*;
/**
* Extra lucene sorting utilities & convenience methods
*
*
- *
+ * @deprecated custom {@link FieldType}s should use the helper methods in the base class. Other usage should leverage th underling lucene {@link SortField} classes directly.
*/
-
+@Deprecated
public class Sorting {
@@ -37,14 +38,19 @@ public class Sorting {
* @param nullLast true if null should come last, regardless of sort order
* @param nullFirst true if null should come first, regardless of sort order
* @return SortField
+ * @deprecated custom {@link FieldType}s should use {@link FieldType#getSortField}. Other usage should leverage th underling lucene {@link SortField} classes directly.
*/
+ @Deprecated
public static SortField getStringSortField(String fieldName, boolean reverse, boolean nullLast, boolean nullFirst) {
SortField sortField = new SortField(fieldName, SortField.Type.STRING, reverse);
applyMissingFirstLast(sortField, reverse, nullLast, nullFirst);
return sortField;
}
- /** Like {@link #getStringSortField}) except safe for tokenized fields */
+ /** Like {@link #getStringSortField}) except safe for tokenized fields
+ * @deprecated custom {@link FieldType}s should use {@link FieldType#getSortedSetSortField}. Other usage should leverage th underling lucene {@link SortedSetSortField} classes directly.
+ */
+ @Deprecated
public static SortField getTextSortField(String fieldName, boolean reverse, boolean nullLast, boolean nullFirst) {
SortField sortField = new SortedSetSortField(fieldName, reverse);
applyMissingFirstLast(sortField, reverse, nullLast, nullFirst);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/31eab319/solr/core/src/test-files/solr/collection1/conf/schema-enums.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-enums.xml b/solr/core/src/test-files/solr/collection1/conf/schema-enums.xml
index 29bede8..2443da4 100644
--- a/solr/core/src/test-files/solr/collection1/conf/schema-enums.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/schema-enums.xml
@@ -18,12 +18,22 @@
<schema name="tiny" version="1.1">
<field name="id" type="string" indexed="true" stored="true" required="true"/>
<field name="_version_" type="long" indexed="true" stored="true" multiValued="false"/>
+
<!-- Test EnumField and EnumFieldType -->
<field name="severity" type="severityType" indexed="${solr.tests.EnumFieldTest.indexed}" stored="true" multiValued="false" docValues="${solr.tests.numeric.dv}"/>
+ <!-- NOTE: because these test sortMissingLast/sortMissingFirst, we force indexed="true" so we don't get
+ random errors on schema init about inconsistent properties -->
+ <field name="severity_missingLast" type="severityType" indexed="true" stored="true" multiValued="false" docValues="${solr.tests.numeric.dv}" sortMissingLast="true"/>
+ <field name="severity_missingFirst" type="severityType" indexed="true" stored="true" multiValued="false" docValues="${solr.tests.numeric.dv}" sortMissingFirst="true"/>
+
<field name="severity_mv" type="severityType" indexed="${solr.tests.EnumFieldTest.indexed}" stored="true" multiValued="true" docValues="${solr.tests.numeric.dv}"/>
<field name="text" type="text" indexed="true" stored="true" multiValued="true"/>
+
<uniqueKey>id</uniqueKey>
+ <copyField source="severity" dest="severity_missingLast" />
+ <copyField source="severity" dest="severity_missingFirst" />
+
<fieldType name="text" class="solr.TextField">
<analyzer>
<tokenizer class="solr.WhitespaceTokenizerFactory"/>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/31eab319/solr/core/src/test/org/apache/solr/schema/EnumFieldTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/schema/EnumFieldTest.java b/solr/core/src/test/org/apache/solr/schema/EnumFieldTest.java
index d01c57f..98e55fa 100644
--- a/solr/core/src/test/org/apache/solr/schema/EnumFieldTest.java
+++ b/solr/core/src/test/org/apache/solr/schema/EnumFieldTest.java
@@ -16,6 +16,7 @@
*/
package org.apache.solr.schema;
+import java.util.Arrays;
import java.util.Iterator;
import java.util.Set;
import java.util.regex.Matcher;
@@ -385,6 +386,21 @@ public class EnumFieldTest extends SolrTestCaseJ4 {
"//doc[4]/str[@name='" + FIELD_NAME + "']/text()='High'",
"//doc[5]/str[@name='" + FIELD_NAME + "']/text()='Critical'"
);
+
+ // missing first....
+ for (String dir : Arrays.asList("asc", "desc")) {
+ assertQ(req("fl", "id", "q", "*:*", "sort", FIELD_NAME + "_missingFirst " + dir + ", id desc")
+ , "//doc[1]/str[@name='id']/text()='9'"
+ , "//doc[2]/str[@name='id']/text()='8'"
+ );
+ }
+ // missing last...
+ for (String dir : Arrays.asList("asc", "desc")) {
+ assertQ(req("fl", "id", "q", "*:*", "sort", FIELD_NAME + "_missingLast " + dir + ", id desc")
+ , "//doc[6]/str[@name='id']/text()='9'"
+ , "//doc[7]/str[@name='id']/text()='8'"
+ );
+ }
}
@Test
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/31eab319/solr/core/src/test/org/apache/solr/search/TestSort.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/search/TestSort.java b/solr/core/src/test/org/apache/solr/search/TestSort.java
index 18597dc..eb3fd00 100644
--- a/solr/core/src/test/org/apache/solr/search/TestSort.java
+++ b/solr/core/src/test/org/apache/solr/search/TestSort.java
@@ -271,9 +271,9 @@ public class TestSort extends SolrTestCaseJ4 {
if (r.nextBoolean()) sfields.add( new SortField(null, SortField.Type.SCORE));
// hit both use-cases of sort-missing-last
- sfields.add( Sorting.getStringSortField("f", reverse, sortMissingLast, sortMissingFirst) );
+ sfields.add( getStringSortField("f", reverse, sortMissingLast, sortMissingFirst) );
if (secondary) {
- sfields.add( Sorting.getStringSortField("f2", reverse2, sortMissingLast2, sortMissingFirst2) );
+ sfields.add( getStringSortField("f2", reverse2, sortMissingLast2, sortMissingFirst2) );
}
if (r.nextBoolean()) sfields.add( new SortField(null, SortField.Type.SCORE));
@@ -355,5 +355,20 @@ public class TestSort extends SolrTestCaseJ4 {
return new BitDocIdSet(obs);
}
-
+ private static SortField getStringSortField(String fieldName, boolean reverse, boolean nullLast, boolean nullFirst) {
+ SortField sortField = new SortField(fieldName, SortField.Type.STRING, reverse);
+
+ // 4 cases:
+ // missingFirst / forward: default lucene behavior
+ // missingFirst / reverse: set sortMissingLast
+ // missingLast / forward: set sortMissingLast
+ // missingLast / reverse: default lucene behavior
+
+ if (nullFirst && reverse) {
+ sortField.setMissingValue(SortField.STRING_LAST);
+ } else if (nullLast && !reverse) {
+ sortField.setMissingValue(SortField.STRING_LAST);
+ }
+ return sortField;
+ }
}
[08/50] lucene-solr:feature/autoscaling: SOLR-10783: add (partial)
package-info.java to fix precommit
Posted by sh...@apache.org.
SOLR-10783: add (partial) package-info.java to fix precommit
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/a864c628
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/a864c628
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/a864c628
Branch: refs/heads/feature/autoscaling
Commit: a864c6289a8132988fc51cc711db79238ed2ce04
Parents: 2a8d9b0
Author: Christine Poerschke <cp...@apache.org>
Authored: Thu Sep 7 11:56:45 2017 +0100
Committer: Christine Poerschke <cp...@apache.org>
Committed: Thu Sep 7 12:18:03 2017 +0100
----------------------------------------------------------------------
.../configuration/providers/package-info.java | 23 ++++++++++++++++++++
1 file changed, 23 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a864c628/solr/core/src/java/org/apache/solr/util/configuration/providers/package-info.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/util/configuration/providers/package-info.java b/solr/core/src/java/org/apache/solr/util/configuration/providers/package-info.java
new file mode 100644
index 0000000..7b5e8f8
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/util/configuration/providers/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * TODO
+ */
+package org.apache.solr.util.configuration.providers;
+
+
[10/50] lucene-solr:feature/autoscaling: SOLR-11215: Support
retrieval of any property of a regular metric when using 'key' parameter.
Posted by sh...@apache.org.
SOLR-11215: Support retrieval of any property of a regular metric when
using 'key' parameter.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/26ed40bc
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/26ed40bc
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/26ed40bc
Branch: refs/heads/feature/autoscaling
Commit: 26ed40bcdbd42fc94e18d9e491536da5515ac9fe
Parents: 5324c74
Author: Andrzej Bialecki <ab...@apache.org>
Authored: Thu Sep 7 18:21:55 2017 +0200
Committer: Andrzej Bialecki <ab...@apache.org>
Committed: Thu Sep 7 18:21:55 2017 +0200
----------------------------------------------------------------------
.../org/apache/solr/handler/admin/MetricsHandler.java | 11 ++++++++---
.../apache/solr/handler/admin/MetricsHandlerTest.java | 9 +++++++++
2 files changed, 17 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/26ed40bc/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java
index a9a2d1f..ed1e474 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java
@@ -22,6 +22,7 @@ import java.util.Collections;
import java.util.EnumSet;
import java.util.HashSet;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
@@ -139,14 +140,18 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName
continue;
}
MetricUtils.PropertyFilter propertyFilter = MetricUtils.PropertyFilter.ALL;
- boolean simple = false;
if (propertyName != null) {
propertyFilter = (name) -> name.equals(propertyName);
- simple = true;
// use escaped versions
key = parts[0] + ":" + parts[1];
}
- MetricUtils.convertMetric(key, m, propertyFilter, false, true, true, simple, ":", (k, v) -> result.add(k, v));
+ MetricUtils.convertMetric(key, m, propertyFilter, false, true, true, false, ":", (k, v) -> {
+ if ((v instanceof Map) && propertyName != null) {
+ ((Map)v).forEach((k1, v1) -> result.add(k + ":" + k1, v1));
+ } else {
+ result.add(k, v);
+ }
+ });
}
rsp.getValues().add("metrics", result);
if (errors.size() > 0) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/26ed40bc/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java
index 7eb8f12..0fe5ad7 100644
--- a/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java
@@ -277,6 +277,15 @@ public class MetricsHandlerTest extends SolrTestCaseJ4 {
val = values.findRecursive("metrics", key3);
assertNotNull(val);
+ String key4 = "solr.core.collection1:QUERY./select.requestTimes:1minRate";
+ resp = new SolrQueryResponse();
+ handler.handleRequestBody(req(CommonParams.QT, "/admin/metrics", CommonParams.WT, "json",
+ MetricsHandler.KEY_PARAM, key4), resp);
+ values = resp.getValues();
+ val = values.findRecursive("metrics", key4);
+ assertNotNull(val);
+ assertTrue(val instanceof Number);
+
// test errors
// invalid keys
[11/50] lucene-solr:feature/autoscaling: SOLR-11241: Fix sha1 file
Posted by sh...@apache.org.
SOLR-11241: Fix sha1 file
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/f828edf3
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/f828edf3
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/f828edf3
Branch: refs/heads/feature/autoscaling
Commit: f828edf332000ff83b228bf35f75dd17f9c6ceb9
Parents: 26ed40b
Author: Joel Bernstein <jb...@apache.org>
Authored: Thu Sep 7 13:45:14 2017 -0400
Committer: Joel Bernstein <jb...@apache.org>
Committed: Thu Sep 7 13:45:23 2017 -0400
----------------------------------------------------------------------
solr/licenses/commons-math3-3.6.1.jar.sha1 | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f828edf3/solr/licenses/commons-math3-3.6.1.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/commons-math3-3.6.1.jar.sha1 b/solr/licenses/commons-math3-3.6.1.jar.sha1
index 72975be..ed9a549 100644
--- a/solr/licenses/commons-math3-3.6.1.jar.sha1
+++ b/solr/licenses/commons-math3-3.6.1.jar.sha1
@@ -1 +1 @@
-e4ba98f1d4b3c80ec46392f25e094a6a2e58fcbf
\ No newline at end of file
+e4ba98f1d4b3c80ec46392f25e094a6a2e58fcbf
[02/50] lucene-solr:feature/autoscaling: SOLR-11241: Fix precommit
Posted by sh...@apache.org.
SOLR-11241: Fix precommit
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/26201c8d
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/26201c8d
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/26201c8d
Branch: refs/heads/feature/autoscaling
Commit: 26201c8dad6465ea029da60fe5a492b2b6f0bf3c
Parents: 4496612
Author: Joel Bernstein <jb...@apache.org>
Authored: Wed Sep 6 11:31:42 2017 -0400
Committer: Joel Bernstein <jb...@apache.org>
Committed: Wed Sep 6 11:32:04 2017 -0400
----------------------------------------------------------------------
solr/core/src/java/org/apache/solr/handler/StreamHandler.java | 1 -
solr/licenses/commons-math3-3.4.1.jar.sha1 | 1 -
solr/licenses/commons-math3-3.6.1.jar.sha1 | 1 +
.../apache/solr/client/solrj/io/eval/FrequencyTableEvaluator.java | 2 --
4 files changed, 1 insertion(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/26201c8d/solr/core/src/java/org/apache/solr/handler/StreamHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/StreamHandler.java b/solr/core/src/java/org/apache/solr/handler/StreamHandler.java
index 5396acd..e1203aa 100644
--- a/solr/core/src/java/org/apache/solr/handler/StreamHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/StreamHandler.java
@@ -29,7 +29,6 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
-import org.apache.commons.math3.distribution.PoissonDistribution;
import org.apache.solr.client.solrj.io.ModelCache;
import org.apache.solr.client.solrj.io.SolrClientCache;
import org.apache.solr.client.solrj.io.Tuple;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/26201c8d/solr/licenses/commons-math3-3.4.1.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/commons-math3-3.4.1.jar.sha1 b/solr/licenses/commons-math3-3.4.1.jar.sha1
deleted file mode 100644
index a26859d..0000000
--- a/solr/licenses/commons-math3-3.4.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-3ac44a8664228384bc68437264cf7c4cf112f579
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/26201c8d/solr/licenses/commons-math3-3.6.1.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/commons-math3-3.6.1.jar.sha1 b/solr/licenses/commons-math3-3.6.1.jar.sha1
new file mode 100644
index 0000000..72975be
--- /dev/null
+++ b/solr/licenses/commons-math3-3.6.1.jar.sha1
@@ -0,0 +1 @@
+e4ba98f1d4b3c80ec46392f25e094a6a2e58fcbf
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/26201c8d/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/FrequencyTableEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/FrequencyTableEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/FrequencyTableEvaluator.java
index ae65dc1..b648e3f 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/FrequencyTableEvaluator.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/FrequencyTableEvaluator.java
@@ -25,10 +25,8 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
-import org.apache.commons.math3.random.EmpiricalDistribution;
import org.apache.commons.math3.stat.Frequency;
-import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
import org.apache.solr.client.solrj.io.Tuple;
import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
[34/50] lucene-solr:feature/autoscaling: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/lucene-solr
Posted by sh...@apache.org.
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/lucene-solr
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/fe34e141
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/fe34e141
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/fe34e141
Branch: refs/heads/feature/autoscaling
Commit: fe34e141ca1914ea811878919590ef574d2c77fa
Parents: e44ee16 5f68eb9
Author: Karl Wright <Da...@gmail.com>
Authored: Mon Sep 11 10:48:17 2017 -0400
Committer: Karl Wright <Da...@gmail.com>
Committed: Mon Sep 11 10:48:17 2017 -0400
----------------------------------------------------------------------
.../src/test/org/apache/solr/schema/TestPointFields.java | 9 ++++++++-
1 file changed, 8 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
[43/50] lucene-solr:feature/autoscaling: SOLR-11348: fix the DIH
database example
Posted by sh...@apache.org.
SOLR-11348: fix the DIH database example
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/d45ff7b6
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/d45ff7b6
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/d45ff7b6
Branch: refs/heads/feature/autoscaling
Commit: d45ff7b601c2c13618486575def4544836aca7e5
Parents: b125107
Author: jdyer1 <jd...@apache.org>
Authored: Tue Sep 12 09:02:25 2017 -0500
Committer: jdyer1 <jd...@apache.org>
Committed: Tue Sep 12 09:03:26 2017 -0500
----------------------------------------------------------------------
solr/CHANGES.txt | 2 +
solr/example/example-DIH/.gitignore | 1 +
solr/example/example-DIH/hsqldb/.gitignore | 5 +
solr/example/example-DIH/hsqldb/ex.backup | Bin 11354 -> 0 bytes
solr/example/example-DIH/hsqldb/ex.data | Bin 1048576 -> 0 bytes
solr/example/example-DIH/hsqldb/ex.script | 173 ++++++++++++++++++++++--
6 files changed, 171 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d45ff7b6/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 5d243c5..5d035c3 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -131,6 +131,8 @@ Bug Fixes
* SOLR-11332: Fix sorting on 'enum' fieldTypes that use sortMissingFirst or sortMissingLast (hossman)
+* SOLR-11348: Fix the DIH database example (James Dyer)
+
Optimizations
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d45ff7b6/solr/example/example-DIH/.gitignore
----------------------------------------------------------------------
diff --git a/solr/example/example-DIH/.gitignore b/solr/example/example-DIH/.gitignore
new file mode 100644
index 0000000..6d9594a
--- /dev/null
+++ b/solr/example/example-DIH/.gitignore
@@ -0,0 +1 @@
+/logs/
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d45ff7b6/solr/example/example-DIH/hsqldb/.gitignore
----------------------------------------------------------------------
diff --git a/solr/example/example-DIH/hsqldb/.gitignore b/solr/example/example-DIH/hsqldb/.gitignore
new file mode 100644
index 0000000..e75d109
--- /dev/null
+++ b/solr/example/example-DIH/hsqldb/.gitignore
@@ -0,0 +1,5 @@
+/ex.tmp/
+ex.log
+ex.lck
+ex.properties
+
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d45ff7b6/solr/example/example-DIH/hsqldb/ex.backup
----------------------------------------------------------------------
diff --git a/solr/example/example-DIH/hsqldb/ex.backup b/solr/example/example-DIH/hsqldb/ex.backup
deleted file mode 100644
index 7471d1f..0000000
Binary files a/solr/example/example-DIH/hsqldb/ex.backup and /dev/null differ
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d45ff7b6/solr/example/example-DIH/hsqldb/ex.data
----------------------------------------------------------------------
diff --git a/solr/example/example-DIH/hsqldb/ex.data b/solr/example/example-DIH/hsqldb/ex.data
deleted file mode 100644
index dac9b13..0000000
Binary files a/solr/example/example-DIH/hsqldb/ex.data and /dev/null differ
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d45ff7b6/solr/example/example-DIH/hsqldb/ex.script
----------------------------------------------------------------------
diff --git a/solr/example/example-DIH/hsqldb/ex.script b/solr/example/example-DIH/hsqldb/ex.script
index 6b151f4..b78f6cf 100644
--- a/solr/example/example-DIH/hsqldb/ex.script
+++ b/solr/example/example-DIH/hsqldb/ex.script
@@ -1,12 +1,165 @@
+SET DATABASE UNIQUE NAME HSQLDB5E727295B6
+SET DATABASE GC 0
+SET DATABASE DEFAULT RESULT MEMORY ROWS 0
+SET DATABASE EVENT LOG LEVEL 0
+SET DATABASE TRANSACTION CONTROL LOCKS
+SET DATABASE DEFAULT ISOLATION LEVEL READ COMMITTED
+SET DATABASE TRANSACTION ROLLBACK ON CONFLICT TRUE
+SET DATABASE TEXT TABLE DEFAULTS ''
+SET DATABASE SQL NAMES FALSE
+SET DATABASE SQL REFERENCES FALSE
+SET DATABASE SQL SIZE TRUE
+SET DATABASE SQL TYPES FALSE
+SET DATABASE SQL TDC DELETE TRUE
+SET DATABASE SQL TDC UPDATE TRUE
+SET DATABASE SQL CONCAT NULLS TRUE
+SET DATABASE SQL UNIQUE NULLS TRUE
+SET DATABASE SQL CONVERT TRUNCATE TRUE
+SET DATABASE SQL AVG SCALE 0
+SET DATABASE SQL DOUBLE NAN TRUE
+SET FILES WRITE DELAY 500 MILLIS
+SET FILES BACKUP INCREMENT TRUE
+SET FILES CACHE SIZE 10000
+SET FILES CACHE ROWS 50000
+SET FILES SCALE 32
+SET FILES LOB SCALE 32
+SET FILES DEFRAG 0
+SET FILES NIO TRUE
+SET FILES NIO SIZE 256
+SET FILES LOG TRUE
+SET FILES LOG SIZE 50
+CREATE USER SA PASSWORD DIGEST 'd41d8cd98f00b204e9800998ecf8427e'
+ALTER USER SA SET LOCAL TRUE
CREATE SCHEMA PUBLIC AUTHORIZATION DBA
-CREATE CACHED TABLE ITEM(ID CHAR(8),NAME VARCHAR(100),MANU VARCHAR(50),WEIGHT REAL,PRICE REAL,POPULARITY INTEGER,INCLUDES VARCHAR(200),LAST_MODIFIED TIMESTAMP DEFAULT NOW)
-CREATE CACHED TABLE FEATURE(ITEM_ID CHAR(8),DESCRIPTION VARCHAR(200),LAST_MODIFIED TIMESTAMP DEFAULT NOW)
-CREATE CACHED TABLE CATEGORY(ID INTEGER,DESCRIPTION VARCHAR(30),LAST_MODIFIED TIMESTAMP DEFAULT NOW)
-CREATE CACHED TABLE ITEM_CATEGORY(ITEM_ID CHAR(8),CATEGORY_ID INTEGER,LAST_MODIFIED TIMESTAMP DEFAULT NOW)
-SET TABLE ITEM INDEX'15056 0'
-SET TABLE FEATURE INDEX'22424 0'
-SET TABLE CATEGORY INDEX'18856 0'
-SET TABLE ITEM_CATEGORY INDEX'17328 0'
-CREATE USER SA PASSWORD ""
+SET SCHEMA PUBLIC
+CREATE MEMORY TABLE PUBLIC.ITEM(ID VARCHAR(100),NAME VARCHAR(1024),MANU VARCHAR(50),WEIGHT DOUBLE,PRICE DOUBLE,POPULARITY INTEGER,INCLUDES VARCHAR(200),LAST_MODIFIED TIMESTAMP)
+CREATE MEMORY TABLE PUBLIC.FEATURE(ITEM_ID VARCHAR(100),DESCRIPTION VARCHAR(1024),LAST_MODIFIED TIMESTAMP)
+CREATE MEMORY TABLE PUBLIC.CATEGORY(ID INTEGER,DESCRIPTION VARCHAR(30),LAST_MODIFIED TIMESTAMP)
+CREATE MEMORY TABLE PUBLIC.ITEM_CATEGORY(ITEM_ID VARCHAR(100),CATEGORY_ID INTEGER,LAST_MODIFIED TIMESTAMP)
+ALTER SEQUENCE SYSTEM_LOBS.LOB_ID RESTART WITH 1
+SET DATABASE DEFAULT INITIAL SCHEMA PUBLIC
+GRANT USAGE ON DOMAIN INFORMATION_SCHEMA.SQL_IDENTIFIER TO PUBLIC
+GRANT USAGE ON DOMAIN INFORMATION_SCHEMA.YES_OR_NO TO PUBLIC
+GRANT USAGE ON DOMAIN INFORMATION_SCHEMA.TIME_STAMP TO PUBLIC
+GRANT USAGE ON DOMAIN INFORMATION_SCHEMA.CARDINAL_NUMBER TO PUBLIC
+GRANT USAGE ON DOMAIN INFORMATION_SCHEMA.CHARACTER_DATA TO PUBLIC
GRANT DBA TO SA
-SET WRITE_DELAY 20
+SET SCHEMA SYSTEM_LOBS
+INSERT INTO BLOCKS VALUES(0,2147483647,0)
+SET SCHEMA PUBLIC
+INSERT INTO ITEM VALUES('6H500F0','Maxtor DiamondMax 11 - hard drive - 500 GB - SATA-300','Maxtor Corp.',0.0E0,350.0E0,6,'null','2017-09-01 12:34:56.000000')
+INSERT INTO ITEM VALUES('F8V7067-APL-KIT','Belkin Mobile Power Cord for iPod w/ Dock','Belkin',4.0E0,19.95E0,1,'null','2017-09-01 12:34:56.000000')
+INSERT INTO ITEM VALUES('IW-02','iPod & iPod Mini USB 2.0 Cable','Belkin',2.0E0,11.5E0,1,'null','2017-09-01 12:34:56.000000')
+INSERT INTO ITEM VALUES('MA147LL/A','Apple 60 GB iPod with Video Playback Black','Apple Computer Inc.',5.5E0,399.0E0,10,'earbud headphones, USB cable','2017-09-01 12:34:56.000000')
+INSERT INTO ITEM VALUES('TWINX2048-3200PRO','CORSAIR XMS 2GB (2 x 1GB) 184-Pin DDR SDRAM Unbuffered DDR 400 (PC 3200) Dual Channel Kit System Memory - Retail','Corsair Microsystems Inc.',0.0E0,185.0E0,5,'null','2017-09-01 12:34:56.000000')
+INSERT INTO ITEM VALUES('VS1GB400C3','CORSAIR ValueSelect 1GB 184-Pin DDR SDRAM Unbuffered DDR 400 (PC 3200) System Memory - Retail','Corsair Microsystems Inc.',0.0E0,74.99E0,7,'null','2017-09-01 12:34:56.000000')
+INSERT INTO ITEM VALUES('VDBDB1A16','A-DATA V-Series 1GB 184-Pin DDR SDRAM Unbuffered DDR 400 (PC 3200) System Memory - OEM','A-DATA Technology Inc.',0.0E0,0.0E0,5,'null','2017-09-01 12:34:56.000000')
+INSERT INTO ITEM VALUES('3007WFP','Dell Widescreen UltraSharp 3007WFP','Dell, Inc.',401.6E0,2199.0E0,6,'USB cable','2017-09-01 12:34:56.000000')
+INSERT INTO ITEM VALUES('VA902B','ViewSonic VA902B - flat panel display - TFT - 19"','ViewSonic Corp.',190.4E0,279.95E0,6,'null','2017-09-01 12:34:56.000000')
+INSERT INTO ITEM VALUES('0579B002','Canon PIXMA MP500 All-In-One Photo Printer','Canon Inc.',352.0E0,179.99E0,6,'null','2017-09-01 12:34:56.000000')
+INSERT INTO ITEM VALUES('9885A004','Canon PowerShot SD500','Canon Inc.',6.4E0,329.95E0,7,'32MB SD card, USB cable, AV cable, battery','2017-09-01 12:34:56.000000')
+INSERT INTO ITEM VALUES('SOLR1000','Solr, the Enterprise Search Server','Apache Software Foundation',0.0E0,0.0E0,10,'null','2017-09-01 12:34:56.000000')
+INSERT INTO ITEM VALUES('UTF8TEST','Test with some UTF-8 encoded characters','Apache Software Foundation',0.0E0,0.0E0,0,'null','2017-09-01 12:34:56.000000')
+INSERT INTO ITEM VALUES('EN7800GTX/2DHTV/256M','ASUS Extreme N7800GTX/2DHTV (256 MB)','ASUS Computer Inc.',16.0E0,479.95E0,7,'null','2017-09-01 12:34:56.000000')
+INSERT INTO ITEM VALUES('100-435805','ATI Radeon X1900 XTX 512 MB PCIE Video Card','ATI Technologies',48.0E0,649.99E0,7,'null','2017-09-01 12:34:56.000000')
+INSERT INTO ITEM VALUES('SP2514N','Samsung SpinPoint P120 SP2514N - hard drive - 250 GB - ATA-133','Samsung Electronics Co. Ltd.',0.0E0,92.0E0,6,'null','2008-03-12 13:30:00.000000')
+INSERT INTO FEATURE VALUES('SP2514N','7200RPM, 8MB cache, IDE Ultra ATA-133','2017-09-01 12:34:56.000000')
+INSERT INTO FEATURE VALUES('SP2514N','NoiseGuard, SilentSeek technology, Fluid Dynamic Bearing (FDB) motor','2017-09-01 12:34:56.000000')
+INSERT INTO FEATURE VALUES('6H500F0','SATA 3.0Gb/s, NCQ','2017-09-01 12:34:56.000000')
+INSERT INTO FEATURE VALUES('6H500F0','8.5ms seek','2017-09-01 12:34:56.000000')
+INSERT INTO FEATURE VALUES('6H500F0','16MB cache','2017-09-01 12:34:56.000000')
+INSERT INTO FEATURE VALUES('F8V7067-APL-KIT','car power adapter, white','2017-09-01 12:34:56.000000')
+INSERT INTO FEATURE VALUES('IW-02','car power adapter for iPod, white','2017-09-01 12:34:56.000000')
+INSERT INTO FEATURE VALUES('MA147LL/A','iTunes, Podcasts, Audiobooks','2017-09-01 12:34:56.000000')
+INSERT INTO FEATURE VALUES('MA147LL/A','Stores up to 15,000 songs, 25,000 photos, or 150 hours of video','2017-09-01 12:34:56.000000')
+INSERT INTO FEATURE VALUES('MA147LL/A','2.5-inch, 320x240 color TFT LCD display with LED backlight','2017-09-01 12:34:56.000000')
+INSERT INTO FEATURE VALUES('MA147LL/A','Up to 20 hours of battery life','2017-09-01 12:34:56.000000')
+INSERT INTO FEATURE VALUES('MA147LL/A','Plays AAC, MP3, WAV, AIFF, Audible, Apple Lossless, H.264 video','2017-09-01 12:34:56.000000')
+INSERT INTO FEATURE VALUES('MA147LL/A','Notes, Calendar, Phone book, Hold button, Date display, Photo wallet, Built-in games, JPEG photo playback, Upgradeable firmware, USB 2.0 compatibility, Playback speed control, Rechargeable capability, Battery level indication','2017-09-01 12:34:56.000000')
+INSERT INTO FEATURE VALUES('TWINX2048-3200PRO','CAS latency 2,\u00092-3-3-6 timing, 2.75v, unbuffered, heat-spreader','2017-09-01 12:34:56.000000')
+INSERT INTO FEATURE VALUES('VDBDB1A16','CAS latency 3,\u0009 2.7v','2017-09-01 12:34:56.000000')
+INSERT INTO FEATURE VALUES('3007WFP','30" TFT active matrix LCD, 2560 x 1600, .25mm dot pitch, 700:1 contrast','2017-09-01 12:34:56.000000')
+INSERT INTO FEATURE VALUES('VA902B','19" TFT active matrix LCD, 8ms response time, 1280 x 1024 native resolution','2017-09-01 12:34:56.000000')
+INSERT INTO FEATURE VALUES('0579B002','Multifunction ink-jet color photo printer','2017-09-01 12:34:56.000000')
+INSERT INTO FEATURE VALUES('0579B002','Flatbed scanner, optical scan resolution of 1,200 x 2,400 dpi','2017-09-01 12:34:56.000000')
+INSERT INTO FEATURE VALUES('0579B002','2.5" color LCD preview screen','2017-09-01 12:34:56.000000')
+INSERT INTO FEATURE VALUES('0579B002','Duplex Copying','2017-09-01 12:34:56.000000')
+INSERT INTO FEATURE VALUES('0579B002','Printing speed up to 29ppm black, 19ppm color','2017-09-01 12:34:56.000000')
+INSERT INTO FEATURE VALUES('0579B002','Hi-Speed USB','2017-09-01 12:34:56.000000')
+INSERT INTO FEATURE VALUES('0579B002','memory card: CompactFlash, Micro Drive, SmartMedia, Memory Stick, Memory Stick Pro, SD Card, and MultiMediaCard','2017-09-01 12:34:56.000000')
+INSERT INTO FEATURE VALUES('9885A004','3x zoop, 7.1 megapixel Digital ELPH','2017-09-01 12:34:56.000000')
+INSERT INTO FEATURE VALUES('9885A004','movie clips up to 640x480 @30 fps','2017-09-01 12:34:56.000000')
+INSERT INTO FEATURE VALUES('9885A004','2.0" TFT LCD, 118,000 pixels','2017-09-01 12:34:56.000000')
+INSERT INTO FEATURE VALUES('9885A004','built in flash, red-eye reduction','2017-09-01 12:34:56.000000')
+INSERT INTO FEATURE VALUES('SOLR1000','Advanced Full-Text Search Capabilities using Lucene','2017-09-01 12:34:56.000000')
+INSERT INTO FEATURE VALUES('SOLR1000','Optimizied for High Volume Web Traffic','2017-09-01 12:34:56.000000')
+INSERT INTO FEATURE VALUES('SOLR1000','Standards Based Open Interfaces - XML and HTTP','2017-09-01 12:34:56.000000')
+INSERT INTO FEATURE VALUES('SOLR1000','Comprehensive HTML Administration Interfaces','2017-09-01 12:34:56.000000')
+INSERT INTO FEATURE VALUES('SOLR1000','Scalability - Efficient Replication to other Solr Search Servers','2017-09-01 12:34:56.000000')
+INSERT INTO FEATURE VALUES('SOLR1000','Flexible and Adaptable with XML configuration and Schema','2017-09-01 12:34:56.000000')
+INSERT INTO FEATURE VALUES('SOLR1000','Good unicode support: h\u00e9llo (hello with an accent over the e)','2017-09-01 12:34:56.000000')
+INSERT INTO FEATURE VALUES('UTF8TEST','No accents here','2017-09-01 12:34:56.000000')
+INSERT INTO FEATURE VALUES('UTF8TEST','This is an e acute: \u00e9','2017-09-01 12:34:56.000000')
+INSERT INTO FEATURE VALUES('UTF8TEST','eaiou with circumflexes: \u00ea\u00e2\u00ee\u00f4\u00fb','2017-09-01 12:34:56.000000')
+INSERT INTO FEATURE VALUES('UTF8TEST','This is in Turkish: bu T\u00fcrk\u00e7e','2017-09-01 12:34:56.000000')
+INSERT INTO FEATURE VALUES('UTF8TEST','This is in Korean: \uc774\uac83\uc740 \ud55c\uad6d\uc5b4\uc774\ub2e4.','2017-09-01 12:34:56.000000')
+INSERT INTO FEATURE VALUES('UTF8TEST','This is in Greek: \u0391\u03c5\u03c4\u03cc \u03b5\u03af\u03bd\u03b1\u03b9 \u03c3\u03c4\u03b1 \u03b5\u03bb\u03bb\u03b7\u03bd\u03b9\u03ba\u03ac','2017-09-01 12:34:56.000000')
+INSERT INTO FEATURE VALUES('EN7800GTX/2DHTV/256M','NVIDIA GeForce 7800 GTX GPU/VPU clocked at 486MHz','2017-09-01 12:34:56.000000')
+INSERT INTO FEATURE VALUES('EN7800GTX/2DHTV/256M','256MB GDDR3 Memory clocked at 1.35GHz','2017-09-01 12:34:56.000000')
+INSERT INTO FEATURE VALUES('EN7800GTX/2DHTV/256M','PCI Express x16','2017-09-01 12:34:56.000000')
+INSERT INTO FEATURE VALUES('EN7800GTX/2DHTV/256M','Dual DVI connectors, HDTV out, video input','2017-09-01 12:34:56.000000')
+INSERT INTO FEATURE VALUES('EN7800GTX/2DHTV/256M','OpenGL 2.0, DirectX 9.0','2017-09-01 12:34:56.000000')
+INSERT INTO FEATURE VALUES('100-435805','ATI RADEON X1900 GPU/VPU clocked at 650MHz','2017-09-01 12:34:56.000000')
+INSERT INTO FEATURE VALUES('100-435805','512MB GDDR3 SDRAM clocked at 1.55GHz','2017-09-01 12:34:56.000000')
+INSERT INTO FEATURE VALUES('100-435805','PCI Express x16','2017-09-01 12:34:56.000000')
+INSERT INTO FEATURE VALUES('100-435805','dual DVI, HDTV, svideo, composite out','2017-09-01 12:34:56.000000')
+INSERT INTO FEATURE VALUES('100-435805','OpenGL 2.0, DirectX 9.0','2017-09-01 12:34:56.000000')
+INSERT INTO CATEGORY VALUES(1,'electronics','2017-09-01 12:34:56.000000')
+INSERT INTO CATEGORY VALUES(2,'hard drive','2017-09-01 12:34:56.000000')
+INSERT INTO CATEGORY VALUES(3,'connector','2017-09-01 12:34:56.000000')
+INSERT INTO CATEGORY VALUES(4,'music','2017-09-01 12:34:56.000000')
+INSERT INTO CATEGORY VALUES(5,'memory','2017-09-01 12:34:56.000000')
+INSERT INTO CATEGORY VALUES(6,'monitor','2017-09-01 12:34:56.000000')
+INSERT INTO CATEGORY VALUES(7,'multifunction printer','2017-09-01 12:34:56.000000')
+INSERT INTO CATEGORY VALUES(8,'printer','2017-09-01 12:34:56.000000')
+INSERT INTO CATEGORY VALUES(9,'scanner','2017-09-01 12:34:56.000000')
+INSERT INTO CATEGORY VALUES(10,'copier','2017-09-01 12:34:56.000000')
+INSERT INTO CATEGORY VALUES(11,'camera','2017-09-01 12:34:56.000000')
+INSERT INTO CATEGORY VALUES(12,'software','2017-09-01 12:34:56.000000')
+INSERT INTO CATEGORY VALUES(13,'search','2017-09-01 12:34:56.000000')
+INSERT INTO CATEGORY VALUES(14,'graphics card','2017-09-01 12:34:56.000000')
+INSERT INTO ITEM_CATEGORY VALUES('SP2514N',1,'2017-09-01 12:34:56.000000')
+INSERT INTO ITEM_CATEGORY VALUES('SP2514N',2,'2017-09-01 12:34:56.000000')
+INSERT INTO ITEM_CATEGORY VALUES('6H500F0',1,'2017-09-01 12:34:56.000000')
+INSERT INTO ITEM_CATEGORY VALUES('6H500F0',2,'2017-09-01 12:34:56.000000')
+INSERT INTO ITEM_CATEGORY VALUES('F8V7067-APL-KIT',1,'2017-09-01 12:34:56.000000')
+INSERT INTO ITEM_CATEGORY VALUES('F8V7067-APL-KIT',3,'2017-09-01 12:34:56.000000')
+INSERT INTO ITEM_CATEGORY VALUES('IW-02',1,'2017-09-01 12:34:56.000000')
+INSERT INTO ITEM_CATEGORY VALUES('IW-02',3,'2017-09-01 12:34:56.000000')
+INSERT INTO ITEM_CATEGORY VALUES('MA147LL/A',1,'2017-09-01 12:34:56.000000')
+INSERT INTO ITEM_CATEGORY VALUES('MA147LL/A',4,'2017-09-01 12:34:56.000000')
+INSERT INTO ITEM_CATEGORY VALUES('TWINX2048-3200PRO',1,'2017-09-01 12:34:56.000000')
+INSERT INTO ITEM_CATEGORY VALUES('TWINX2048-3200PRO',5,'2017-09-01 12:34:56.000000')
+INSERT INTO ITEM_CATEGORY VALUES('VS1GB400C3',1,'2017-09-01 12:34:56.000000')
+INSERT INTO ITEM_CATEGORY VALUES('VS1GB400C3',5,'2017-09-01 12:34:56.000000')
+INSERT INTO ITEM_CATEGORY VALUES('VDBDB1A16',1,'2017-09-01 12:34:56.000000')
+INSERT INTO ITEM_CATEGORY VALUES('VDBDB1A16',5,'2017-09-01 12:34:56.000000')
+INSERT INTO ITEM_CATEGORY VALUES('3007WFP',1,'2017-09-01 12:34:56.000000')
+INSERT INTO ITEM_CATEGORY VALUES('3007WFP',6,'2017-09-01 12:34:56.000000')
+INSERT INTO ITEM_CATEGORY VALUES('VA902B',1,'2017-09-01 12:34:56.000000')
+INSERT INTO ITEM_CATEGORY VALUES('VA902B',6,'2017-09-01 12:34:56.000000')
+INSERT INTO ITEM_CATEGORY VALUES('0579B002',1,'2017-09-01 12:34:56.000000')
+INSERT INTO ITEM_CATEGORY VALUES('0579B002',7,'2017-09-01 12:34:56.000000')
+INSERT INTO ITEM_CATEGORY VALUES('0579B002',8,'2017-09-01 12:34:56.000000')
+INSERT INTO ITEM_CATEGORY VALUES('0579B002',9,'2017-09-01 12:34:56.000000')
+INSERT INTO ITEM_CATEGORY VALUES('0579B002',10,'2017-09-01 12:34:56.000000')
+INSERT INTO ITEM_CATEGORY VALUES('9885A004',1,'2017-09-01 12:34:56.000000')
+INSERT INTO ITEM_CATEGORY VALUES('9885A004',11,'2017-09-01 12:34:56.000000')
+INSERT INTO ITEM_CATEGORY VALUES('SOLR1000',12,'2017-09-01 12:34:56.000000')
+INSERT INTO ITEM_CATEGORY VALUES('SOLR1000',13,'2017-09-01 12:34:56.000000')
+INSERT INTO ITEM_CATEGORY VALUES('UTF8TEST',12,'2017-09-01 12:34:56.000000')
+INSERT INTO ITEM_CATEGORY VALUES('UTF8TEST',13,'2017-09-01 12:34:56.000000')
+INSERT INTO ITEM_CATEGORY VALUES('EN7800GTX/2DHTV/256M',1,'2017-09-01 12:34:56.000000')
+INSERT INTO ITEM_CATEGORY VALUES('EN7800GTX/2DHTV/256M',14,'2017-09-01 12:34:56.000000')
+INSERT INTO ITEM_CATEGORY VALUES('100-435805',1,'2017-09-01 12:34:56.000000')
+INSERT INTO ITEM_CATEGORY VALUES('100-435805',14,'2017-09-01 12:34:56.000000')
[30/50] lucene-solr:feature/autoscaling: LUCENE-7955: Add the new
shape to the standard objects class
Posted by sh...@apache.org.
LUCENE-7955: Add the new shape to the standard objects class
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/64d14285
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/64d14285
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/64d14285
Branch: refs/heads/feature/autoscaling
Commit: 64d142858df5066524dd777dd7c5d506fc9fb874
Parents: 80ae269
Author: Karl Wright <Da...@gmail.com>
Authored: Sun Sep 10 07:33:29 2017 -0400
Committer: Karl Wright <Da...@gmail.com>
Committed: Sun Sep 10 07:33:29 2017 -0400
----------------------------------------------------------------------
.../src/java/org/apache/lucene/spatial3d/geom/StandardObjects.java | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/64d14285/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/StandardObjects.java
----------------------------------------------------------------------
diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/StandardObjects.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/StandardObjects.java
index 4d4dcd8..bad4651 100644
--- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/StandardObjects.java
+++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/StandardObjects.java
@@ -73,6 +73,7 @@ class StandardObjects {
classRegsitry.put(XYdZSolid.class, 33);
classRegsitry.put(StandardXYZSolid.class, 34);
classRegsitry.put(PlanetModel.class, 35);
+ classRegsitry.put(GeoDegeneratePath.class, 36);
for (Class<?> clazz : classRegsitry.keySet()){
codeRegsitry.put(classRegsitry.get(clazz), clazz);
[40/50] lucene-solr:feature/autoscaling:
TestRerankBase.getModelInJson to permit empty features array.
Posted by sh...@apache.org.
TestRerankBase.getModelInJson to permit empty features array.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/708ed8fb
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/708ed8fb
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/708ed8fb
Branch: refs/heads/feature/autoscaling
Commit: 708ed8fb4880bf1a7dcb9211f4eb1923bdd325d8
Parents: 0c831b9
Author: Christine Poerschke <cp...@apache.org>
Authored: Mon Sep 11 12:41:19 2017 -0700
Committer: Christine Poerschke <cp...@apache.org>
Committed: Mon Sep 11 12:47:44 2017 -0700
----------------------------------------------------------------------
.../src/test/org/apache/solr/ltr/TestRerankBase.java | 12 +++++++-----
1 file changed, 7 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/708ed8fb/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestRerankBase.java
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestRerankBase.java b/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestRerankBase.java
index 6f5e0ce..796fe39 100644
--- a/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestRerankBase.java
+++ b/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestRerankBase.java
@@ -239,12 +239,14 @@ public class TestRerankBase extends RestTestBase {
.append(",\n");
sb.append("\"class\":").append('"').append(type).append('"').append(",\n");
sb.append("\"features\":").append('[');
- for (final String feature : features) {
- sb.append("\n\t{ ");
- sb.append("\"name\":").append('"').append(feature).append('"')
- .append("},");
+ if (features.length > 0) {
+ for (final String feature : features) {
+ sb.append("\n\t{ ");
+ sb.append("\"name\":").append('"').append(feature).append('"')
+ .append("},");
+ }
+ sb.deleteCharAt(sb.length() - 1);
}
- sb.deleteCharAt(sb.length() - 1);
sb.append("\n]\n");
if (params != null) {
sb.append(",\n");
[49/50] lucene-solr:feature/autoscaling: LUCENE-7968: Analyzing
Suggester's comparator compares incorrectly
Posted by sh...@apache.org.
LUCENE-7968: Analyzing Suggester's comparator compares incorrectly
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/2e5f9a43
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/2e5f9a43
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/2e5f9a43
Branch: refs/heads/feature/autoscaling
Commit: 2e5f9a4369e0f4f664868718ce3ee8fbea43a98b
Parents: ab965d5
Author: Robert Muir <rm...@apache.org>
Authored: Sun Sep 17 11:37:24 2017 -0400
Committer: Robert Muir <rm...@apache.org>
Committed: Sun Sep 17 11:37:24 2017 -0400
----------------------------------------------------------------------
lucene/CHANGES.txt | 4 ++
.../suggest/analyzing/AnalyzingSuggester.java | 6 +-
.../analyzing/AnalyzingSuggesterTest.java | 60 ++++++++++++++++++++
3 files changed, 68 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2e5f9a43/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 555c97d..5a95d6c 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -61,6 +61,10 @@ Bug Fixes
write.lock for an empty index due to bad permissions/read-only filesystem/etc.
(Erick Erickson, Shawn Heisey, Robert Muir)
+* LUCENE-7968: AnalyzingSuggester would sometimes order suggestions incorrectly,
+ it did not properly break ties on the surface forms when both the weights and
+ the analyzed forms were equal. (Robert Muir)
+
Build
* SOLR-11181: Switch order of maven artifact publishing procedure: deploy first
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2e5f9a43/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java
----------------------------------------------------------------------
diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java
index 9c6a624..cc11fe1 100644
--- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java
+++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java
@@ -390,9 +390,11 @@ public class AnalyzingSuggester extends Lookup implements Accountable {
} else {
scratchA.offset = readerA.getPosition();
scratchB.offset = readerB.getPosition();
- scratchA.length = a.length - scratchA.offset;
- scratchB.length = b.length - scratchB.offset;
+ scratchA.length = readerA.length() - readerA.getPosition();
+ scratchB.length = readerB.length() - readerB.getPosition();
}
+ assert scratchA.isValid();
+ assert scratchB.isValid();
return scratchA.compareTo(scratchB);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2e5f9a43/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java
----------------------------------------------------------------------
diff --git a/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java b/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java
index 67ff056..7302d9e 100644
--- a/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java
+++ b/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java
@@ -1111,6 +1111,66 @@ public class AnalyzingSuggesterTest extends LuceneTestCase {
IOUtils.close(a, tempDir);
}
+ /**
+ * Adds 50 random keys, that all analyze to the same thing (dog), with the same cost,
+ * and checks that they come back in surface-form order.
+ */
+ public void testTieBreakOnSurfaceForm() throws Exception {
+ Analyzer a = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.SIMPLE, true);
+
+ return new TokenStreamComponents(tokenizer) {
+ @Override
+ public TokenStream getTokenStream() {
+ return new CannedTokenStream(new Token[] {
+ token("dog", 1, 1)
+ });
+ }
+ @Override
+ protected void setReader(final Reader reader) {
+ }
+ };
+ }
+ };
+
+ Directory tempDir = getDirectory();
+ AnalyzingSuggester suggester = new AnalyzingSuggester(tempDir, "suggest", a, a, 0, 256, -1, true);
+
+ // make 50 inputs all with the same cost of 1, random strings
+ Input[] inputs = new Input[100];
+ for (int i = 0; i < inputs.length; i++) {
+ inputs[i] = new Input(TestUtil.randomSimpleString(random()), 1);
+ }
+
+ suggester.build(new InputArrayIterator(inputs));
+
+ // Try to save/load:
+ Path tmpDir = createTempDir("AnalyzingSuggesterTest");
+ Path path = tmpDir.resolve("suggester");
+
+ OutputStream os = Files.newOutputStream(path);
+ suggester.store(os);
+ os.close();
+
+ InputStream is = Files.newInputStream(path);
+ suggester.load(is);
+ is.close();
+
+ // now suggest everything, and check that stuff comes back in order
+ List<LookupResult> results = suggester.lookup("", false, 50);
+ assertEquals(50, results.size());
+ for (int i = 1; i < 50; i++) {
+ String previous = results.get(i-1).toString();
+ String current = results.get(i).toString();
+ assertTrue("surface forms out of order: previous=" + previous + ",current=" + current,
+ current.compareTo(previous) >= 0);
+ }
+
+ IOUtils.close(a, tempDir);
+ }
+
public void test0ByteKeys() throws Exception {
final Analyzer a = new Analyzer() {
@Override
[29/50] lucene-solr:feature/autoscaling: LUCENE-7955: Adjust
degenerate path logic to not consider endpoints unless needed.
Posted by sh...@apache.org.
LUCENE-7955: Adjust degenerate path logic to not consider endpoints unless needed.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/80ae2699
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/80ae2699
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/80ae2699
Branch: refs/heads/feature/autoscaling
Commit: 80ae2699cf2e8f87587fa5670edb9a5bad0dbedd
Parents: a4fc14b
Author: Karl Wright <Da...@gmail.com>
Authored: Sun Sep 10 07:00:20 2017 -0400
Committer: Karl Wright <Da...@gmail.com>
Committed: Sun Sep 10 07:03:30 2017 -0400
----------------------------------------------------------------------
.../spatial3d/geom/GeoDegeneratePath.java | 21 ++++++++++----------
.../geom/SimpleGeoPolygonRelationshipsTest.java | 15 ++++++++++++++
2 files changed, 26 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/80ae2699/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoDegeneratePath.java
----------------------------------------------------------------------
diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoDegeneratePath.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoDegeneratePath.java
index 175ca33..eaceb26 100644
--- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoDegeneratePath.java
+++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoDegeneratePath.java
@@ -259,10 +259,11 @@ class GeoDegeneratePath extends GeoBasePath {
// Well, sort of. We can detect intersections also due to overlap of segments with each other.
// But that's an edge case and we won't be optimizing for it.
//System.err.println(" Looking for intersection of plane "+plane+" with path "+this);
- for (final SegmentEndpoint pathPoint : endPoints) {
- if (pathPoint.intersects(planetModel, plane, notablePoints, bounds)) {
- return true;
- }
+
+ // Since the endpoints are included in the path segments, we only need to do this if there are
+ // no path segments
+ if (endPoints.size() == 1) {
+ return endPoints.get(0).intersects(planetModel, plane, notablePoints, bounds);
}
for (final PathSegment pathSegment : segments) {
@@ -276,10 +277,10 @@ class GeoDegeneratePath extends GeoBasePath {
@Override
public boolean intersects(GeoShape geoShape) {
- for (final SegmentEndpoint pathPoint : endPoints) {
- if (pathPoint.intersects(geoShape)) {
- return true;
- }
+ // Since the endpoints are included in the path segments, we only need to do this if there are
+ // no path segments
+ if (endPoints.size() == 1) {
+ return endPoints.get(0).intersects(geoShape);
}
for (final PathSegment pathSegment : segments) {
@@ -300,8 +301,8 @@ class GeoDegeneratePath extends GeoBasePath {
for (PathSegment pathSegment : segments) {
pathSegment.getBounds(planetModel, bounds);
}
- for (SegmentEndpoint pathPoint : endPoints) {
- pathPoint.getBounds(planetModel, bounds);
+ if (endPoints.size() == 1) {
+ endPoints.get(0).getBounds(planetModel, bounds);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/80ae2699/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/SimpleGeoPolygonRelationshipsTest.java
----------------------------------------------------------------------
diff --git a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/SimpleGeoPolygonRelationshipsTest.java b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/SimpleGeoPolygonRelationshipsTest.java
index 06aa8b9..2589a19 100644
--- a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/SimpleGeoPolygonRelationshipsTest.java
+++ b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/SimpleGeoPolygonRelationshipsTest.java
@@ -736,6 +736,21 @@ public class SimpleGeoPolygonRelationshipsTest {
assertEquals(GeoArea.CONTAINS, rel);
}
+ @Test
+ public void testDegeneratePathShape(){
+ GeoPoint point1 = new GeoPoint(PlanetModel.SPHERE, 0, 0);
+ GeoPoint point2 = new GeoPoint(PlanetModel.SPHERE, 0, 1);
+ GeoPoint[] pointPath1 = new GeoPoint[] {point1, point2};
+ GeoPath path1 = GeoPathFactory.makeGeoPath(PlanetModel.SPHERE, 0, pointPath1);
+ GeoPath path2 = GeoPathFactory.makeGeoPath(PlanetModel.SPHERE, 1, pointPath1);
+ int rel = path1.getRelationship(path2);
+ //if an end point is inside the shape it will always return intersects
+ assertEquals(GeoArea.CONTAINS, rel); //should be contains?
+ rel = path2.getRelationship(path1);
+ assertEquals(GeoArea.WITHIN, rel);
+ }
+
+
private GeoPolygon buildConvexGeoPolygon(double lon1, double lat1,
double lon2, double lat2,
double lon3, double lat3,
[21/50] lucene-solr:feature/autoscaling: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/lucene-solr
Posted by sh...@apache.org.
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/lucene-solr
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/2b6c82ed
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/2b6c82ed
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/2b6c82ed
Branch: refs/heads/feature/autoscaling
Commit: 2b6c82ed329ca82249ebad2a46b344f95c968ca1
Parents: c09678e 244ebd1
Author: Karl Wright <Da...@gmail.com>
Authored: Fri Sep 8 12:18:59 2017 -0400
Committer: Karl Wright <Da...@gmail.com>
Committed: Fri Sep 8 12:18:59 2017 -0400
----------------------------------------------------------------------
lucene/CHANGES.txt | 4 +
.../lucene/index/DefaultIndexingChain.java | 2 -
solr/CHANGES.txt | 1 +
.../solr/handler/component/QueryComponent.java | 467 +++++++++++--------
4 files changed, 277 insertions(+), 197 deletions(-)
----------------------------------------------------------------------
[19/50] lucene-solr:feature/autoscaling: SOLR-10990: Breakup
QueryComponent.process method for readability. (Christine Poerschke)
Posted by sh...@apache.org.
SOLR-10990: Breakup QueryComponent.process method for readability. (Christine Poerschke)
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/244ebd1a
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/244ebd1a
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/244ebd1a
Branch: refs/heads/feature/autoscaling
Commit: 244ebd1aa4c2311dcce4a52d8fd425e47acaf9ad
Parents: a5402f6
Author: Christine Poerschke <cp...@apache.org>
Authored: Fri Sep 8 16:26:56 2017 +0100
Committer: Christine Poerschke <cp...@apache.org>
Committed: Fri Sep 8 16:42:46 2017 +0100
----------------------------------------------------------------------
solr/CHANGES.txt | 1 +
.../solr/handler/component/QueryComponent.java | 467 +++++++++++--------
2 files changed, 273 insertions(+), 195 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/244ebd1a/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index f652fb2..d278821 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -197,6 +197,7 @@ Other Changes
* SOLR-11322: JSON Facet API: instead of returning NaN, min & max aggregations omit
the value for any bucket with no values in the numeric field. (yonik)
+* SOLR-10990: Breakup QueryComponent.process method for readability. (Christine Poerschke)
================== 7.0.0 ==================
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/244ebd1a/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java b/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java
index 1d88adc..417d583 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java
@@ -306,12 +306,12 @@ public class QueryComponent extends SearchComponent
if (!params.getBool(COMPONENT_NAME, true)) {
return;
}
- SolrIndexSearcher searcher = req.getSearcher();
StatsCache statsCache = req.getCore().getStatsCache();
int purpose = params.getInt(ShardParams.SHARDS_PURPOSE, ShardRequest.PURPOSE_GET_TOP_IDS);
if ((purpose & ShardRequest.PURPOSE_GET_TERM_STATS) != 0) {
+ SolrIndexSearcher searcher = req.getSearcher();
statsCache.returnLocalStats(rb, searcher);
return;
}
@@ -321,50 +321,11 @@ public class QueryComponent extends SearchComponent
statsCache.receiveGlobalStats(req);
}
- SolrQueryResponse rsp = rb.rsp;
- IndexSchema schema = searcher.getSchema();
-
// Optional: This could also be implemented by the top-level searcher sending
// a filter that lists the ids... that would be transparent to
// the request handler, but would be more expensive (and would preserve score
// too if desired).
- String ids = params.get(ShardParams.IDS);
- if (ids != null) {
- SchemaField idField = schema.getUniqueKeyField();
- List<String> idArr = StrUtils.splitSmart(ids, ",", true);
- int[] luceneIds = new int[idArr.size()];
- int docs = 0;
- if (idField.getType().isPointField()) {
- for (int i=0; i<idArr.size(); i++) {
- int id = searcher.search(
- idField.getType().getFieldQuery(null, idField, idArr.get(i)), 1).scoreDocs[0].doc;
- if (id >= 0) {
- luceneIds[docs++] = id;
- }
- }
- } else {
- for (int i=0; i<idArr.size(); i++) {
- int id = searcher.getFirstMatch(
- new Term(idField.getName(), idField.getType().toInternal(idArr.get(i))));
- if (id >= 0)
- luceneIds[docs++] = id;
- }
- }
-
- DocListAndSet res = new DocListAndSet();
- res.docList = new DocSlice(0, docs, luceneIds, null, docs, 0);
- if (rb.isNeedDocSet()) {
- // TODO: create a cache for this!
- List<Query> queries = new ArrayList<>();
- queries.add(rb.getQuery());
- List<Query> filters = rb.getFilters();
- if (filters != null) queries.addAll(filters);
- res.docSet = searcher.getDocSet(queries);
- }
- rb.setResults(res);
-
- ResultContext ctx = new BasicResultContext(rb);
- rsp.addResponse(ctx);
+ if (doProcessSearchByIds(rb)) {
return;
}
@@ -395,145 +356,15 @@ public class QueryComponent extends SearchComponent
if (groupingSpec != null) {
cmd.setSegmentTerminateEarly(false); // not supported, silently ignore any segmentTerminateEarly flag
try {
- boolean needScores = (cmd.getFlags() & SolrIndexSearcher.GET_SCORES) != 0;
if (params.getBool(GroupParams.GROUP_DISTRIBUTED_FIRST, false)) {
- CommandHandler.Builder topsGroupsActionBuilder = new CommandHandler.Builder()
- .setQueryCommand(cmd)
- .setNeedDocSet(false) // Order matters here
- .setIncludeHitCount(true)
- .setSearcher(searcher);
-
- for (String field : groupingSpec.getFields()) {
- topsGroupsActionBuilder.addCommandField(new SearchGroupsFieldCommand.Builder()
- .setField(schema.getField(field))
- .setGroupSort(groupingSpec.getGroupSort())
- .setTopNGroups(cmd.getOffset() + cmd.getLen())
- .setIncludeGroupCount(groupingSpec.isIncludeGroupCount())
- .build()
- );
- }
-
- CommandHandler commandHandler = topsGroupsActionBuilder.build();
- commandHandler.execute();
- SearchGroupsResultTransformer serializer = new SearchGroupsResultTransformer(searcher);
- rsp.add("firstPhase", commandHandler.processResult(result, serializer));
- rsp.add("totalHitCount", commandHandler.getTotalHitCount());
- rb.setResult(result);
+ doProcessGroupedDistributedSearchFirstPhase(rb, result);
return;
} else if (params.getBool(GroupParams.GROUP_DISTRIBUTED_SECOND, false)) {
- CommandHandler.Builder secondPhaseBuilder = new CommandHandler.Builder()
- .setQueryCommand(cmd)
- .setTruncateGroups(groupingSpec.isTruncateGroups() && groupingSpec.getFields().length > 0)
- .setSearcher(searcher);
-
- int docsToCollect = Grouping.getMax(groupingSpec.getWithinGroupOffset(), groupingSpec.getWithinGroupLimit(), searcher.maxDoc());
- docsToCollect = Math.max(docsToCollect, 1);
-
- for (String field : groupingSpec.getFields()) {
- SchemaField schemaField = schema.getField(field);
- String[] topGroupsParam = params.getParams(GroupParams.GROUP_DISTRIBUTED_TOPGROUPS_PREFIX + field);
- if (topGroupsParam == null) {
- topGroupsParam = new String[0];
- }
-
- List<SearchGroup<BytesRef>> topGroups = new ArrayList<>(topGroupsParam.length);
- for (String topGroup : topGroupsParam) {
- SearchGroup<BytesRef> searchGroup = new SearchGroup<>();
- if (!topGroup.equals(TopGroupsShardRequestFactory.GROUP_NULL_VALUE)) {
- BytesRefBuilder builder = new BytesRefBuilder();
- schemaField.getType().readableToIndexed(topGroup, builder);
- searchGroup.groupValue = builder.get();
- }
- topGroups.add(searchGroup);
- }
-
- secondPhaseBuilder.addCommandField(
- new TopGroupsFieldCommand.Builder()
- .setField(schemaField)
- .setGroupSort(groupingSpec.getGroupSort())
- .setSortWithinGroup(groupingSpec.getSortWithinGroup())
- .setFirstPhaseGroups(topGroups)
- .setMaxDocPerGroup(docsToCollect)
- .setNeedScores(needScores)
- .setNeedMaxScore(needScores)
- .build()
- );
- }
-
- for (String query : groupingSpec.getQueries()) {
- secondPhaseBuilder.addCommandField(new Builder()
- .setDocsToCollect(docsToCollect)
- .setSort(groupingSpec.getGroupSort())
- .setQuery(query, rb.req)
- .setDocSet(searcher)
- .build()
- );
- }
-
- CommandHandler commandHandler = secondPhaseBuilder.build();
- commandHandler.execute();
- TopGroupsResultTransformer serializer = new TopGroupsResultTransformer(rb);
- rsp.add("secondPhase", commandHandler.processResult(result, serializer));
- rb.setResult(result);
+ doProcessGroupedDistributedSearchSecondPhase(rb, result);
return;
}
- int maxDocsPercentageToCache = params.getInt(GroupParams.GROUP_CACHE_PERCENTAGE, 0);
- boolean cacheSecondPassSearch = maxDocsPercentageToCache >= 1 && maxDocsPercentageToCache <= 100;
- Grouping.TotalCount defaultTotalCount = groupingSpec.isIncludeGroupCount() ?
- Grouping.TotalCount.grouped : Grouping.TotalCount.ungrouped;
- int limitDefault = cmd.getLen(); // this is normally from "rows"
- Grouping grouping =
- new Grouping(searcher, result, cmd, cacheSecondPassSearch, maxDocsPercentageToCache, groupingSpec.isMain());
- grouping.setGroupSort(groupingSpec.getGroupSort())
- .setWithinGroupSort(groupingSpec.getSortWithinGroup())
- .setDefaultFormat(groupingSpec.getResponseFormat())
- .setLimitDefault(limitDefault)
- .setDefaultTotalCount(defaultTotalCount)
- .setDocsPerGroupDefault(groupingSpec.getWithinGroupLimit())
- .setGroupOffsetDefault(groupingSpec.getWithinGroupOffset())
- .setGetGroupedDocSet(groupingSpec.isTruncateGroups());
-
- if (groupingSpec.getFields() != null) {
- for (String field : groupingSpec.getFields()) {
- grouping.addFieldCommand(field, rb.req);
- }
- }
-
- if (groupingSpec.getFunctions() != null) {
- for (String groupByStr : groupingSpec.getFunctions()) {
- grouping.addFunctionCommand(groupByStr, rb.req);
- }
- }
-
- if (groupingSpec.getQueries() != null) {
- for (String groupByStr : groupingSpec.getQueries()) {
- grouping.addQueryCommand(groupByStr, rb.req);
- }
- }
-
- if( rb.isNeedDocList() || rb.isDebug() ){
- // we need a single list of the returned docs
- cmd.setFlags(SolrIndexSearcher.GET_DOCLIST);
- }
-
- grouping.execute();
- if (grouping.isSignalCacheWarning()) {
- rsp.add(
- "cacheWarning",
- String.format(Locale.ROOT, "Cache limit of %d percent relative to maxdoc has exceeded. Please increase cache size or disable caching.", maxDocsPercentageToCache)
- );
- }
- rb.setResult(result);
-
- if (grouping.mainResult != null) {
- ResultContext ctx = new BasicResultContext(rb, grouping.mainResult);
- rsp.addResponse(ctx);
- rsp.getToLog().add("hits", grouping.mainResult.matches());
- } else if (!grouping.getCommands().isEmpty()) { // Can never be empty since grouping.execute() checks for this.
- rsp.add("grouped", result.groupedResults);
- rsp.getToLog().add("hits", grouping.getCommands().get(0).getMatches());
- }
+ doProcessGroupedSearch(rb, result);
return;
} catch (SyntaxError e) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
@@ -541,27 +372,7 @@ public class QueryComponent extends SearchComponent
}
// normal search result
- searcher.search(result, cmd);
- rb.setResult(result);
-
- ResultContext ctx = new BasicResultContext(rb);
- rsp.addResponse(ctx);
- rsp.getToLog().add("hits", rb.getResults().docList.matches());
-
- if ( ! rb.req.getParams().getBool(ShardParams.IS_SHARD,false) ) {
- if (null != rb.getNextCursorMark()) {
- rb.rsp.add(CursorMarkParams.CURSOR_MARK_NEXT,
- rb.getNextCursorMark().getSerializedTotem());
- }
- }
-
- if(rb.mergeFieldHandler != null) {
- rb.mergeFieldHandler.handleMergeFields(rb, searcher);
- } else {
- doFieldSortValues(rb, searcher);
- }
-
- doPrefetch(rb);
+ doProcessUngroupedSearch(rb, result);
}
protected void doFieldSortValues(ResponseBuilder rb, SolrIndexSearcher searcher) throws IOException
@@ -1385,6 +1196,272 @@ public class QueryComponent extends SearchComponent
return Category.QUERY;
}
+ private boolean doProcessSearchByIds(ResponseBuilder rb) throws IOException {
+
+ SolrQueryRequest req = rb.req;
+ SolrQueryResponse rsp = rb.rsp;
+
+ SolrParams params = req.getParams();
+
+ String ids = params.get(ShardParams.IDS);
+ if (ids == null) {
+ return false;
+ }
+
+ SolrIndexSearcher searcher = req.getSearcher();
+ IndexSchema schema = searcher.getSchema();
+ SchemaField idField = schema.getUniqueKeyField();
+ List<String> idArr = StrUtils.splitSmart(ids, ",", true);
+ int[] luceneIds = new int[idArr.size()];
+ int docs = 0;
+ if (idField.getType().isPointField()) {
+ for (int i=0; i<idArr.size(); i++) {
+ int id = searcher.search(
+ idField.getType().getFieldQuery(null, idField, idArr.get(i)), 1).scoreDocs[0].doc;
+ if (id >= 0) {
+ luceneIds[docs++] = id;
+ }
+ }
+ } else {
+ for (int i=0; i<idArr.size(); i++) {
+ int id = searcher.getFirstMatch(
+ new Term(idField.getName(), idField.getType().toInternal(idArr.get(i))));
+ if (id >= 0)
+ luceneIds[docs++] = id;
+ }
+ }
+
+ DocListAndSet res = new DocListAndSet();
+ res.docList = new DocSlice(0, docs, luceneIds, null, docs, 0);
+ if (rb.isNeedDocSet()) {
+ // TODO: create a cache for this!
+ List<Query> queries = new ArrayList<>();
+ queries.add(rb.getQuery());
+ List<Query> filters = rb.getFilters();
+ if (filters != null) queries.addAll(filters);
+ res.docSet = searcher.getDocSet(queries);
+ }
+ rb.setResults(res);
+
+ ResultContext ctx = new BasicResultContext(rb);
+ rsp.addResponse(ctx);
+ return true;
+ }
+
+ private void doProcessGroupedDistributedSearchFirstPhase(ResponseBuilder rb, QueryResult result) throws IOException {
+
+ GroupingSpecification groupingSpec = rb.getGroupingSpec();
+ assert null != groupingSpec : "GroupingSpecification is null";
+
+ SolrQueryRequest req = rb.req;
+ SolrQueryResponse rsp = rb.rsp;
+
+ SolrIndexSearcher searcher = req.getSearcher();
+ IndexSchema schema = searcher.getSchema();
+
+ QueryCommand cmd = rb.getQueryCommand();
+
+ CommandHandler.Builder topsGroupsActionBuilder = new CommandHandler.Builder()
+ .setQueryCommand(cmd)
+ .setNeedDocSet(false) // Order matters here
+ .setIncludeHitCount(true)
+ .setSearcher(searcher);
+
+ for (String field : groupingSpec.getFields()) {
+ topsGroupsActionBuilder.addCommandField(new SearchGroupsFieldCommand.Builder()
+ .setField(schema.getField(field))
+ .setGroupSort(groupingSpec.getGroupSort())
+ .setTopNGroups(cmd.getOffset() + cmd.getLen())
+ .setIncludeGroupCount(groupingSpec.isIncludeGroupCount())
+ .build()
+ );
+ }
+
+ CommandHandler commandHandler = topsGroupsActionBuilder.build();
+ commandHandler.execute();
+ SearchGroupsResultTransformer serializer = new SearchGroupsResultTransformer(searcher);
+
+ rsp.add("firstPhase", commandHandler.processResult(result, serializer));
+ rsp.add("totalHitCount", commandHandler.getTotalHitCount());
+ rb.setResult(result);
+ }
+
+ private void doProcessGroupedDistributedSearchSecondPhase(ResponseBuilder rb, QueryResult result) throws IOException, SyntaxError {
+
+ GroupingSpecification groupingSpec = rb.getGroupingSpec();
+ assert null != groupingSpec : "GroupingSpecification is null";
+
+ SolrQueryRequest req = rb.req;
+ SolrQueryResponse rsp = rb.rsp;
+
+ SolrParams params = req.getParams();
+
+ SolrIndexSearcher searcher = req.getSearcher();
+ IndexSchema schema = searcher.getSchema();
+
+ QueryCommand cmd = rb.getQueryCommand();
+ boolean needScores = (cmd.getFlags() & SolrIndexSearcher.GET_SCORES) != 0;
+
+ CommandHandler.Builder secondPhaseBuilder = new CommandHandler.Builder()
+ .setQueryCommand(cmd)
+ .setTruncateGroups(groupingSpec.isTruncateGroups() && groupingSpec.getFields().length > 0)
+ .setSearcher(searcher);
+
+ int docsToCollect = Grouping.getMax(groupingSpec.getWithinGroupOffset(), groupingSpec.getWithinGroupLimit(), searcher.maxDoc());
+ docsToCollect = Math.max(docsToCollect, 1);
+
+ for (String field : groupingSpec.getFields()) {
+ SchemaField schemaField = schema.getField(field);
+ String[] topGroupsParam = params.getParams(GroupParams.GROUP_DISTRIBUTED_TOPGROUPS_PREFIX + field);
+ if (topGroupsParam == null) {
+ topGroupsParam = new String[0];
+ }
+
+ List<SearchGroup<BytesRef>> topGroups = new ArrayList<>(topGroupsParam.length);
+ for (String topGroup : topGroupsParam) {
+ SearchGroup<BytesRef> searchGroup = new SearchGroup<>();
+ if (!topGroup.equals(TopGroupsShardRequestFactory.GROUP_NULL_VALUE)) {
+ BytesRefBuilder builder = new BytesRefBuilder();
+ schemaField.getType().readableToIndexed(topGroup, builder);
+ searchGroup.groupValue = builder.get();
+ }
+ topGroups.add(searchGroup);
+ }
+
+ secondPhaseBuilder.addCommandField(
+ new TopGroupsFieldCommand.Builder()
+ .setField(schemaField)
+ .setGroupSort(groupingSpec.getGroupSort())
+ .setSortWithinGroup(groupingSpec.getSortWithinGroup())
+ .setFirstPhaseGroups(topGroups)
+ .setMaxDocPerGroup(docsToCollect)
+ .setNeedScores(needScores)
+ .setNeedMaxScore(needScores)
+ .build()
+ );
+ }
+
+ for (String query : groupingSpec.getQueries()) {
+ secondPhaseBuilder.addCommandField(new Builder()
+ .setDocsToCollect(docsToCollect)
+ .setSort(groupingSpec.getGroupSort())
+ .setQuery(query, rb.req)
+ .setDocSet(searcher)
+ .build()
+ );
+ }
+
+ CommandHandler commandHandler = secondPhaseBuilder.build();
+ commandHandler.execute();
+ TopGroupsResultTransformer serializer = new TopGroupsResultTransformer(rb);
+ rsp.add("secondPhase", commandHandler.processResult(result, serializer));
+ rb.setResult(result);
+ }
+
+ private void doProcessGroupedSearch(ResponseBuilder rb, QueryResult result) throws IOException, SyntaxError {
+
+ GroupingSpecification groupingSpec = rb.getGroupingSpec();
+ assert null != groupingSpec : "GroupingSpecification is null";
+
+ SolrQueryRequest req = rb.req;
+ SolrQueryResponse rsp = rb.rsp;
+
+ SolrParams params = req.getParams();
+
+ SolrIndexSearcher searcher = req.getSearcher();
+
+ QueryCommand cmd = rb.getQueryCommand();
+
+ int maxDocsPercentageToCache = params.getInt(GroupParams.GROUP_CACHE_PERCENTAGE, 0);
+ boolean cacheSecondPassSearch = maxDocsPercentageToCache >= 1 && maxDocsPercentageToCache <= 100;
+ Grouping.TotalCount defaultTotalCount = groupingSpec.isIncludeGroupCount() ?
+ Grouping.TotalCount.grouped : Grouping.TotalCount.ungrouped;
+ int limitDefault = cmd.getLen(); // this is normally from "rows"
+ Grouping grouping =
+ new Grouping(searcher, result, cmd, cacheSecondPassSearch, maxDocsPercentageToCache, groupingSpec.isMain());
+ grouping.setGroupSort(groupingSpec.getGroupSort())
+ .setWithinGroupSort(groupingSpec.getSortWithinGroup())
+ .setDefaultFormat(groupingSpec.getResponseFormat())
+ .setLimitDefault(limitDefault)
+ .setDefaultTotalCount(defaultTotalCount)
+ .setDocsPerGroupDefault(groupingSpec.getWithinGroupLimit())
+ .setGroupOffsetDefault(groupingSpec.getWithinGroupOffset())
+ .setGetGroupedDocSet(groupingSpec.isTruncateGroups());
+
+ if (groupingSpec.getFields() != null) {
+ for (String field : groupingSpec.getFields()) {
+ grouping.addFieldCommand(field, rb.req);
+ }
+ }
+
+ if (groupingSpec.getFunctions() != null) {
+ for (String groupByStr : groupingSpec.getFunctions()) {
+ grouping.addFunctionCommand(groupByStr, rb.req);
+ }
+ }
+
+ if (groupingSpec.getQueries() != null) {
+ for (String groupByStr : groupingSpec.getQueries()) {
+ grouping.addQueryCommand(groupByStr, rb.req);
+ }
+ }
+
+ if( rb.isNeedDocList() || rb.isDebug() ){
+ // we need a single list of the returned docs
+ cmd.setFlags(SolrIndexSearcher.GET_DOCLIST);
+ }
+
+ grouping.execute();
+ if (grouping.isSignalCacheWarning()) {
+ rsp.add(
+ "cacheWarning",
+ String.format(Locale.ROOT, "Cache limit of %d percent relative to maxdoc has exceeded. Please increase cache size or disable caching.", maxDocsPercentageToCache)
+ );
+ }
+ rb.setResult(result);
+
+ if (grouping.mainResult != null) {
+ ResultContext ctx = new BasicResultContext(rb, grouping.mainResult);
+ rsp.addResponse(ctx);
+ rsp.getToLog().add("hits", grouping.mainResult.matches());
+ } else if (!grouping.getCommands().isEmpty()) { // Can never be empty since grouping.execute() checks for this.
+ rsp.add("grouped", result.groupedResults);
+ rsp.getToLog().add("hits", grouping.getCommands().get(0).getMatches());
+ }
+ }
+
+ private void doProcessUngroupedSearch(ResponseBuilder rb, QueryResult result) throws IOException {
+
+ SolrQueryRequest req = rb.req;
+ SolrQueryResponse rsp = rb.rsp;
+
+ SolrIndexSearcher searcher = req.getSearcher();
+
+ QueryCommand cmd = rb.getQueryCommand();
+
+ searcher.search(result, cmd);
+ rb.setResult(result);
+
+ ResultContext ctx = new BasicResultContext(rb);
+ rsp.addResponse(ctx);
+ rsp.getToLog().add("hits", rb.getResults().docList.matches());
+
+ if ( ! rb.req.getParams().getBool(ShardParams.IS_SHARD,false) ) {
+ if (null != rb.getNextCursorMark()) {
+ rb.rsp.add(CursorMarkParams.CURSOR_MARK_NEXT,
+ rb.getNextCursorMark().getSerializedTotem());
+ }
+ }
+
+ if(rb.mergeFieldHandler != null) {
+ rb.mergeFieldHandler.handleMergeFields(rb, searcher);
+ } else {
+ doFieldSortValues(rb, searcher);
+ }
+
+ doPrefetch(rb);
+ }
+
/**
* Fake scorer for a single document
*
[33/50] lucene-solr:feature/autoscaling: LUCENE-7906: Include missing
file
Posted by sh...@apache.org.
LUCENE-7906: Include missing file
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/e44ee16c
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/e44ee16c
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/e44ee16c
Branch: refs/heads/feature/autoscaling
Commit: e44ee16c74832031556e0c2c2e0b9fd0588e968b
Parents: cd425d6
Author: Karl Wright <Da...@gmail.com>
Authored: Mon Sep 11 10:48:03 2017 -0400
Committer: Karl Wright <Da...@gmail.com>
Committed: Mon Sep 11 10:48:03 2017 -0400
----------------------------------------------------------------------
.../geom/RandomGeo3dShapeGenerator.java | 1019 ++++++++++++++++++
1 file changed, 1019 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e44ee16c/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/RandomGeo3dShapeGenerator.java
----------------------------------------------------------------------
diff --git a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/RandomGeo3dShapeGenerator.java b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/RandomGeo3dShapeGenerator.java
new file mode 100644
index 0000000..72b7109
--- /dev/null
+++ b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/RandomGeo3dShapeGenerator.java
@@ -0,0 +1,1019 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.lucene.spatial3d.geom;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+
+import org.apache.lucene.util.LuceneTestCase;
+
+import static com.carrotsearch.randomizedtesting.RandomizedTest.randomDouble;
+
+/**
+ * Class for generating random Geo3dShapes. They can be generated under
+ * given constraints which are expressed as a shape and a relationship.
+ *
+ * note that convexity for polygons is defined as polygons that contains
+ * antipodal points, otherwise they are convex. Internally they can be
+ * created using GeoConvexPolygons and GeoConcavePolygons.
+ *
+ */
+public class RandomGeo3dShapeGenerator extends LuceneTestCase {
+
+ /* Max num of iterations to find right shape under given constrains */
+ final private static int MAX_SHAPE_ITERATIONS = 50;
+ /* Max num of iterations to find right point under given constrains */
+ final private static int MAX_POINT_ITERATIONS = 1000;
+
+ /* Supported shapes */
+ final protected static int CONVEX_POLYGON = 0;
+ final protected static int CONVEX_POLYGON_WITH_HOLES = 1;
+ final protected static int CONCAVE_POLYGON = 2;
+ final protected static int CONCAVE_POLYGON_WITH_HOLES = 3;
+ final protected static int COMPLEX_POLYGON = 4;
+ final protected static int CIRCLE = 5;
+ final protected static int RECTANGLE = 6;
+ final protected static int PATH = 7;
+ final protected static int COLLECTION = 8;
+ final protected static int POINT = 9;
+ final protected static int LINE = 10;
+
+ /* Helper shapes for generating constraints whch are just three sided polygons */
+ final protected static int CONVEX_SIMPLE_POLYGON = 500;
+ final protected static int CONCAVE_SIMPLE_POLYGON = 501;
+
+ /**
+ * Method that returns a random generated Planet model from the supported
+ * Planet models. currently SPHERE and WGS84
+ *
+ * @return a random generated Planet model
+ */
+ public PlanetModel randomPlanetModel() {
+ final int shapeType = random().nextInt(2);
+ switch (shapeType) {
+ case 0: {
+ return PlanetModel.SPHERE;
+ }
+ case 1: {
+ return PlanetModel.WGS84;
+ }
+ default:
+ throw new IllegalStateException("Unexpected planet model");
+ }
+ }
+
+ /**
+ * Method that returns a random generated a random Shape code from all
+ * supported shapes.
+ *
+ * @return a random generated shape code
+ */
+ public int randomShapeType(){
+ return random().nextInt(11);
+ }
+
+ /**
+ * Method that returns a random generated GeoAreaShape code from all
+ * supported GeoAreaShapes.
+ *
+ * We are removing Collections because it is difficult to create shapes
+ * with properties in some cases.
+ *
+ * @return a random generated polygon code
+ */
+ public int randomGeoAreaShapeType(){
+ return random().nextInt(11);
+ }
+
+ /**
+ * Method that returns a random generated a random Shape code from all
+ * convex supported shapes.
+ *
+ * @return a random generated convex shape code
+ */
+ public int randomConvexShapeType(){
+ int shapeType = randomShapeType();
+ while (isConcave(shapeType)){
+ shapeType = randomShapeType();
+ }
+ return shapeType;
+ }
+
+ /**
+ * Method that returns a random generated a random Shape code from all
+ * concave supported shapes.
+ *
+ * @return a random generated concave shape code
+ */
+ public int randomConcaveShapeType(){
+ int shapeType = randomShapeType();
+ while (!isConcave(shapeType)){
+ shapeType = randomShapeType();
+ }
+ return shapeType;
+ }
+
+ /**
+ * Check if a shape code represents a concave shape
+ *
+ * @return true if the shape represented by the code is concave
+ */
+ public boolean isConcave(int shapeType){
+ return (shapeType == CONCAVE_POLYGON);
+ }
+
+ /**
+ * Method that returns empty Constraints object..
+ *
+ * @return an empty Constraints object
+ */
+ public Constraints getEmptyConstraint(){
+ return new Constraints();
+ }
+
+ /**
+ * Method that returns a random generated GeoPoint.
+ *
+ * @param planetModel The planet model.
+ * @return The random generated GeoPoint.
+ */
+ public GeoPoint randomGeoPoint(PlanetModel planetModel) {
+ GeoPoint point = null;
+ while (point == null) {
+ point = randomGeoPoint(planetModel, getEmptyConstraint());
+ }
+ return point;
+ }
+
+ /**
+ * Method that returns a random generated GeoPoint under given constraints. Returns
+ * NULL if it cannot find a point under the given constraints.
+ *
+ * @param planetModel The planet model.
+ * @param constraints The given constraints.
+ * @return The random generated GeoPoint.
+ */
+ public GeoPoint randomGeoPoint(PlanetModel planetModel, Constraints constraints) {
+ int iterations = 0;
+ while (iterations < MAX_POINT_ITERATIONS) {
+ double lat = randomDouble();
+ if (Math.PI/2 - Math.abs(lat) <0){
+ continue;
+ }
+ double lon = randomDouble();
+ if (Math.PI - Math.abs(lat) <0){
+ continue;
+ }
+ iterations++;
+ GeoPoint point = new GeoPoint(planetModel, lat, lon);
+ if (constraints.isWithin(point)) {
+ return point;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Method that returns a random generated GeoAreaShape.
+ *
+ * @param shapeType The GeoAreaShape code.
+ * @param planetModel The planet model.
+ * @return The random generated GeoAreaShape.
+ */
+ public GeoAreaShape randomGeoAreaShape(int shapeType, PlanetModel planetModel){
+ GeoAreaShape geoAreaShape = null;
+ while (geoAreaShape == null){
+ geoAreaShape = randomGeoAreaShape(shapeType,planetModel,new Constraints());
+ }
+ return geoAreaShape;
+ }
+
+ /**
+ * Method that returns a random generated GeoAreaShape under given constraints. Returns
+ * NULL if it cannot build the GeoAreaShape under the given constraints.
+ *
+ * @param shapeType The GeoAreaShape code.
+ * @param planetModel The planet model.
+ * @param constraints The given constraints.
+ * @return The random generated GeoAreaShape.
+ */
+ public GeoAreaShape randomGeoAreaShape(int shapeType, PlanetModel planetModel, Constraints constraints){
+ return (GeoAreaShape)randomGeoShape(shapeType, planetModel, constraints);
+ }
+
+ /**
+ * Method that returns a random generated GeoShape.
+ *
+ * @param shapeType The shape code.
+ * @param planetModel The planet model.
+ * @return The random generated GeoShape.
+ */
+ public GeoShape randomGeoShape(int shapeType, PlanetModel planetModel){
+ GeoShape geoShape = null;
+ while (geoShape == null){
+ geoShape = randomGeoShape(shapeType,planetModel,new Constraints());
+ }
+ return geoShape;
+ }
+
+ /**
+ * Method that returns a random generated GeoShape under given constraints. Returns
+ * NULL if it cannot build the GeoShape under the given constraints.
+ *
+ * @param shapeType The polygon code.
+ * @param planetModel The planet model.
+ * @param constraints The given constraints.
+ * @return The random generated GeoShape.
+ */
+ public GeoShape randomGeoShape(int shapeType, PlanetModel planetModel, Constraints constraints){
+ switch (shapeType) {
+ case CONVEX_POLYGON: {
+ return convexPolygon(planetModel, constraints);
+ }
+ case CONVEX_POLYGON_WITH_HOLES: {
+ return convexPolygonWithHoles(planetModel, constraints);
+ }
+ case CONCAVE_POLYGON: {
+ return concavePolygon(planetModel, constraints);
+ }
+ case CONCAVE_POLYGON_WITH_HOLES: {
+ return concavePolygonWithHoles(planetModel, constraints);
+ }
+ case COMPLEX_POLYGON: {
+ return complexPolygon(planetModel, constraints);
+ }
+ case CIRCLE: {
+ return circle(planetModel, constraints);
+ }
+ case RECTANGLE: {
+ return rectangle(planetModel, constraints);
+ }
+ case PATH: {
+ return path(planetModel, constraints);
+ }
+ case COLLECTION: {
+ return collection(planetModel, constraints);
+ }
+ case POINT: {
+ return point(planetModel, constraints);
+ }
+ case LINE: {
+ return line(planetModel, constraints);
+ }
+ case CONVEX_SIMPLE_POLYGON: {
+ return simpleConvexPolygon(planetModel, constraints);
+ }
+ case CONCAVE_SIMPLE_POLYGON: {
+ return concaveSimplePolygon(planetModel, constraints);
+ }
+ default:
+ throw new IllegalStateException("Unexpected shape type");
+ }
+ }
+
+ /**
+ * Method that returns a random generated a GeoPointShape under given constraints. Returns
+ * NULL if it cannot build the GeoCircle under the given constraints.
+ *
+ * @param planetModel The planet model.
+ * @param constraints The given constraints.
+ * @return The random generated GeoPointShape.
+ */
+ private GeoPointShape point(PlanetModel planetModel , Constraints constraints) {
+ int iterations=0;
+ while (iterations < MAX_SHAPE_ITERATIONS) {
+ iterations++;
+ final GeoPoint point = randomGeoPoint(planetModel, constraints);
+ if (point == null){
+ continue;
+ }
+ try {
+
+ GeoPointShape pointShape = GeoPointShapeFactory.makeGeoPointShape(planetModel, point.getLatitude(), point.getLongitude());
+ if (!constraints.valid(pointShape)) {
+ continue;
+ }
+ return pointShape;
+ } catch (IllegalArgumentException e) {
+ continue;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Method that returns a random generated a GeoCircle under given constraints. Returns
+ * NULL if it cannot build the GeoCircle under the given constraints.
+ *
+ * @param planetModel The planet model.
+ * @param constraints The given constraints.
+ * @return The random generated GeoCircle.
+ */
+ private GeoCircle circle(PlanetModel planetModel , Constraints constraints) {
+ int iterations=0;
+ while (iterations < MAX_SHAPE_ITERATIONS) {
+ iterations++;
+ final GeoPoint center = randomGeoPoint(planetModel, constraints);
+ if (center == null){
+ continue;
+ }
+ final double radius = randomCutoffAngle();
+ try {
+
+ GeoCircle circle = GeoCircleFactory.makeGeoCircle(planetModel, center.getLatitude(), center.getLongitude(), radius);
+ if (!constraints.valid(circle)) {
+ continue;
+ }
+ return circle;
+ } catch (IllegalArgumentException e) {
+ continue;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Method that returns a random generated a GeoBBox under given constraints. Returns
+ * NULL if it cannot build the GeoBBox under the given constraints.
+ *
+ * @param planetModel The planet model.
+ * @param constraints The given constraints.
+ * @return The random generated GeoBBox.
+ */
+ private GeoBBox rectangle(PlanetModel planetModel, Constraints constraints) {
+
+ int iterations = 0;
+ while (iterations < MAX_SHAPE_ITERATIONS) {
+ iterations++;
+ final GeoPoint point1 = randomGeoPoint(planetModel, constraints);
+ if (point1 == null){
+ continue;
+ }
+ final GeoPoint point2 = randomGeoPoint(planetModel, constraints);
+ if (point2 == null){
+ continue;
+ }
+
+ double minLat = Math.min(point1.getLatitude(), point2.getLatitude());
+ double maxLat = Math.max(point1.getLatitude(), point2.getLatitude());
+ double minLon = Math.min(point1.getLongitude(), point2.getLongitude());
+ double maxLon = Math.max(point1.getLongitude(), point2.getLongitude());
+
+ try {
+ GeoBBox bbox = GeoBBoxFactory.makeGeoBBox(planetModel, maxLat, minLat, minLon, maxLon);
+ if (!constraints.valid(bbox)) {
+ continue;
+ }
+ return bbox;
+ } catch (IllegalArgumentException e) {
+ continue;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Method that returns a random generated degenerate GeoPath under given constraints. Returns
+ * NULL if it cannot build the degenerate GeoPath under the given constraints.
+ *
+ * @param planetModel The planet model.
+ * @param constraints The given constraints.
+ * @return The random generated degenerated GeoPath.
+ */
+ private GeoPath line(PlanetModel planetModel, Constraints constraints) {
+ int iterations = 0;
+ while (iterations < MAX_SHAPE_ITERATIONS) {
+ iterations++;
+ int vertexCount = random().nextInt(2) + 2;
+ List<GeoPoint> geoPoints = points(vertexCount, planetModel, constraints);
+ try {
+ GeoPath path = GeoPathFactory.makeGeoPath(planetModel, 0, geoPoints.toArray(new GeoPoint[geoPoints.size()]));
+ if (!constraints.valid(path)) {
+ continue;
+ }
+ return path;
+ } catch (IllegalArgumentException e) {
+ continue;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Method that returns a random generated a GeoPath under given constraints. Returns
+ * NULL if it cannot build the GeoPath under the given constraints.
+ *
+ * @param planetModel The planet model.
+ * @param constraints The given constraints.
+ * @return The random generated GeoPath.
+ */
+ private GeoPath path(PlanetModel planetModel, Constraints constraints) {
+ int iterations = 0;
+ while (iterations < MAX_SHAPE_ITERATIONS) {
+ iterations++;
+ int vertexCount = random().nextInt(2) + 2;
+ List<GeoPoint> geoPoints = points(vertexCount, planetModel, constraints);
+ double width =randomCutoffAngle();
+ try {
+ GeoPath path = GeoPathFactory.makeGeoPath(planetModel, width, geoPoints.toArray(new GeoPoint[geoPoints.size()]));
+ if (!constraints.valid(path)) {
+ continue;
+ }
+ return path;
+ } catch (IllegalArgumentException e) {
+ continue;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Method that returns a random generated a GeoCompositeMembershipShape under given constraints. Returns
+ * NULL if it cannot build the GGeoCompositeMembershipShape under the given constraints.
+ *
+ * @param planetModel The planet model.
+ * @param constraints The given constraints.
+ * @return The random generated GeoCompositeMembershipShape.
+ */
+ private GeoCompositeAreaShape collection(PlanetModel planetModel, Constraints constraints) {
+ int iterations = 0;
+ while (iterations < MAX_SHAPE_ITERATIONS) {
+ iterations++;
+ int numberShapes = random().nextInt(3) + 2;
+ GeoCompositeAreaShape collection = new GeoCompositeAreaShape(planetModel);
+ for(int i=0; i<numberShapes;i++){
+ GeoPolygon member = convexPolygon(planetModel, constraints);
+ if (member != null){
+ collection.addShape(member);
+ }
+ }
+ if (collection.shapes.size() ==0){
+ continue;
+ }
+ return collection;
+ }
+ return null;
+ }
+
+ /**
+ * Method that returns a random generated a convex GeoPolygon under given constraints. Returns
+ * NULL if it cannot build the GePolygon under the given constraints.
+ *
+ * @param planetModel The planet model.
+ * @param constraints The given constraints.
+ * @return The random generated GeoPolygon.
+ */
+ private GeoPolygon convexPolygon(PlanetModel planetModel, Constraints constraints) {
+ int vertexCount = random().nextInt(4) + 3;
+ int iterations = 0;
+ while (iterations < MAX_SHAPE_ITERATIONS) {
+ iterations++;
+ List<GeoPoint> geoPoints = points(vertexCount,planetModel, constraints);
+ List<GeoPoint> orderedGeoPoints = orderPoints(geoPoints);
+ try {
+ GeoPolygon polygon = GeoPolygonFactory.makeGeoPolygon(planetModel, orderedGeoPoints);
+ if (!constraints.valid(polygon) || isConcave(planetModel, polygon)) {
+ continue;
+ }
+ return polygon;
+ } catch (IllegalArgumentException e) {
+ continue;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Method that returns a random generated a convex GeoPolygon with holes under given constraints. Returns
+ * NULL if it cannot build the GeoPolygon with holes under the given constraints.
+ *
+ * @param planetModel The planet model.
+ * @param constraints The given constraints.
+ * @return The random generated GeoPolygon.
+ */
+ private GeoPolygon convexPolygonWithHoles(PlanetModel planetModel, Constraints constraints) {
+ int vertexCount = random().nextInt(4) + 3;
+ int iterations = 0;
+ while (iterations < MAX_SHAPE_ITERATIONS) {
+ iterations++;
+ List<GeoPoint> geoPoints = points(vertexCount,planetModel, constraints);
+ List<GeoPoint> orderedGeoPoints = orderPoints(geoPoints);
+ try {
+ GeoPolygon polygon = GeoPolygonFactory.makeGeoPolygon(planetModel, orderedGeoPoints);
+ //polygon should comply with all constraints except disjoint as we have holes
+ Constraints polygonConstraints = new Constraints();
+ polygonConstraints.putAll(constraints.getContains());
+ polygonConstraints.putAll(constraints.getWithin());
+ polygonConstraints.putAll(constraints.getDisjoint());
+ if (!polygonConstraints.valid(polygon) || isConcave(planetModel, polygon)){
+ continue;
+ }
+ //hole must overlap with polygon and comply with any CONTAINS constraint.
+ Constraints holeConstraints = new Constraints();
+ holeConstraints.putAll(constraints.getContains());
+ holeConstraints.put(polygon,GeoArea.OVERLAPS);
+ //Points must be with in the polygon and must comply
+ // CONTAINS and DISJOINT constraints
+ Constraints pointsConstraints = new Constraints();
+ pointsConstraints.put(polygon,GeoArea.WITHIN);
+ pointsConstraints.putAll(constraints.getContains());
+ pointsConstraints.putAll(constraints.getDisjoint());
+ List<GeoPolygon> holes = concavePolygonHoles(planetModel, holeConstraints, pointsConstraints);
+ //we should have at least one hole
+ if (holes.size() == 0){
+ continue;
+ }
+ polygon = GeoPolygonFactory.makeGeoPolygon(planetModel,orderedGeoPoints,holes);
+ if (!constraints.valid(polygon) || isConcave(planetModel, polygon)){
+ continue;
+ }
+ return polygon;
+ } catch (IllegalArgumentException e) {
+ continue;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Method that returns a random list if concave GeoPolygons under given constraints. Method
+ * use to generate convex holes. Note that constraints for points and holes are different,
+ *
+ * @param planetModel The planet model.
+ * @param holeConstraints The given constraints that a hole must comply.
+ * @param pointConstraints The given constraints that a point must comply.
+ * @return The random generated GeoPolygon.
+ */
+ private List<GeoPolygon> concavePolygonHoles(PlanetModel planetModel,
+ Constraints holeConstraints,
+ Constraints pointConstraints) {
+ int iterations =0;
+ int holesCount = random().nextInt(3) + 1;
+ List<GeoPolygon> holes = new ArrayList<>();
+ while (iterations < MAX_SHAPE_ITERATIONS) {
+ iterations++;
+ int vertexCount = random().nextInt(3) + 3;
+ List<GeoPoint> geoPoints = points(vertexCount, planetModel, pointConstraints);
+ geoPoints = orderPoints(geoPoints);
+ Collections.reverse(geoPoints);
+ try {
+ GeoPolygon hole = GeoPolygonFactory.makeGeoPolygon(planetModel, geoPoints);
+ if (!holeConstraints.valid(hole) || isConvex(planetModel, hole)) {
+ continue;
+ }
+ holes.add(hole);
+ if (holes.size() == holesCount){
+ return holes;
+ }
+ pointConstraints.put(hole, GeoArea.DISJOINT);
+ } catch (IllegalArgumentException e) {
+ continue;
+ }
+ }
+ return holes;
+ }
+
+ /**
+ * Method that returns a random generated a concave GeoPolygon under given constraints. Returns
+ * NULL if it cannot build the concave GeoPolygon under the given constraints.
+ *
+ * @param planetModel The planet model.
+ * @param constraints The given constraints.
+ * @return The random generated GeoPolygon.
+ */
+ private GeoPolygon concavePolygon(PlanetModel planetModel, Constraints constraints) {
+
+ int vertexCount = random().nextInt(4) + 3;
+ int iterations = 0;
+ while (iterations < MAX_SHAPE_ITERATIONS) {
+ iterations++;
+ List<GeoPoint> geoPoints = points(vertexCount,planetModel, constraints);
+ List<GeoPoint> orderedGeoPoints = orderPoints(geoPoints);
+ Collections.reverse(orderedGeoPoints);
+ try {
+ GeoPolygon polygon = GeoPolygonFactory.makeGeoPolygon(planetModel, orderedGeoPoints);
+ if (!constraints.valid(polygon) || isConvex(planetModel, polygon)) {
+ continue;
+ }
+ return polygon;
+ } catch (IllegalArgumentException e) {
+ continue;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Method that returns a random generated a concave GeoPolygon with holes under given constraints. Returns
+ * NULL if it cannot build the GeoPolygon under the given constraints. Note that the final GeoPolygon is
+ * convex as the hole wraps the convex GeoPolygon.
+ *
+ * @param planetModel The planet model.
+ * @param constraints The given constraints.
+ * @return The random generated GeoPolygon.
+ */
+ private GeoPolygon concavePolygonWithHoles(PlanetModel planetModel, Constraints constraints) {
+ int vertexCount = random().nextInt(4) + 3;
+ int iterations = 0;
+ while (iterations < MAX_SHAPE_ITERATIONS) {
+ iterations++;
+ //we first build the hole. We consider all constraints except
+ // disjoint as we have a hole
+ Constraints holeConstraints = new Constraints();
+ holeConstraints.putAll(constraints.getContains());
+ holeConstraints.putAll(constraints.getWithin());
+ holeConstraints.putAll(constraints.getOverlaps());
+ GeoPolygon hole = convexPolygon(planetModel, holeConstraints);
+ if (hole == null){
+ continue;
+ }
+ // Now we get points for polygon. Must we with in the hole
+ // and we add contain constraints
+ Constraints pointConstraints = new Constraints();
+ pointConstraints.put(hole, GeoArea.WITHIN);
+ pointConstraints.putAll(constraints.getContains());
+ List<GeoPoint> geoPoints = points(vertexCount,planetModel, pointConstraints);
+ List<GeoPoint> orderedGeoPoints = orderPoints(geoPoints);
+ Collections.reverse(orderedGeoPoints);
+ try {
+ GeoPolygon polygon = GeoPolygonFactory.makeGeoPolygon(planetModel, orderedGeoPoints, Collections.singletonList(hole));
+ //final polygon must be convex
+ if (!constraints.valid(polygon) || isConcave(planetModel,polygon)) {
+ continue;
+ }
+ return polygon;
+ } catch (IllegalArgumentException e) {
+ continue;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Method that returns a random generated complex GeoPolygon under given constraints. Returns
+ * NULL if it cannot build the complex GeoPolygon under the given constraints.
+ *
+ * @param planetModel The planet model.
+ * @param constraints The given constraints.
+ * @return The random generated GeoPolygon.
+ */
+ private GeoPolygon complexPolygon(PlanetModel planetModel, Constraints constraints) {
+ int polygonsCount =random().nextInt(2) + 1;
+ int iterations = 0;
+ while (iterations < MAX_SHAPE_ITERATIONS) {
+ iterations++;
+ List<GeoPolygonFactory.PolygonDescription> polDescription = new ArrayList<>();
+ while(polDescription.size() < polygonsCount){
+ int vertexCount = random().nextInt(14) + 3;
+ List<GeoPoint> geoPoints = points(vertexCount,planetModel, constraints);
+ orderPoints(geoPoints);
+ polDescription.add(new GeoPolygonFactory.PolygonDescription(geoPoints));
+ }
+ try {
+ GeoPolygon polygon = GeoPolygonFactory.makeLargeGeoPolygon(planetModel,polDescription);
+ if (!constraints.valid(polygon)) {
+ continue;
+ }
+ return polygon;
+ } catch (IllegalArgumentException e) {
+ continue;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Method that returns a random generated a concave square GeoPolygon under given constraints. Returns
+ * NULL if it cannot build the concave GeoPolygon under the given constraints. This shape is an utility
+ * to build constraints.
+ *
+ * @param planetModel The planet model.
+ * @param constraints The given constraints.
+ * @return The random generated GeoPolygon.
+ */
+ private GeoPolygon simpleConvexPolygon(PlanetModel planetModel, Constraints constraints) {
+ int iterations = 0;
+ while (iterations < MAX_SHAPE_ITERATIONS) {
+ iterations++;
+ List<GeoPoint> points = points(3,planetModel,constraints);
+ points = orderPoints(points);
+ try {
+ GeoPolygon polygon = GeoPolygonFactory.makeGeoConvexPolygon(planetModel, points);
+ if(!constraints.valid(polygon) || isConcave(planetModel,polygon)){
+ continue;
+ }
+ return polygon;
+ } catch (IllegalArgumentException e) {
+ continue;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Method that returns a random generated a convex square GeoPolygon under given constraints. Returns
+ * NULL if it cannot build the convex GeoPolygon under the given constraints. This shape is an utility
+ * to build constraints.
+ *
+ * @param planetModel The planet model.
+ * @param constraints The given constraints.
+ * @return The random generated GeoPolygon.
+ */
+ private GeoPolygon concaveSimplePolygon(PlanetModel planetModel, Constraints constraints) {
+ int iterations = 0;
+ while (iterations < MAX_SHAPE_ITERATIONS) {
+ iterations++;
+ List<GeoPoint> points = points(3, planetModel, constraints);
+ points = orderPoints(points);
+ Collections.reverse(points);
+ try {
+ GeoPolygon polygon = GeoPolygonFactory.makeGeoConcavePolygon(planetModel, points);
+ if(!constraints.valid(polygon) || isConvex(planetModel, polygon)){
+ continue;
+ }
+ return polygon;
+ } catch (IllegalArgumentException e) {
+ continue;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Method that returns a random list of generated GeoPoints under given constraints. If it cannot
+ * find a point it will add a point that might not comply with the constraints.
+ *
+ * @param count The number of points
+ * @param planetModel The planet model.
+ * @param constraints The given constraints.
+ * @return The random generated List of GeoPoints.
+ */
+ private List<GeoPoint> points(int count, PlanetModel planetModel, Constraints constraints){
+ List<GeoPoint> geoPoints = new ArrayList<>(count);
+ for(int i= 0; i< count; i++) {
+ GeoPoint point = randomGeoPoint(planetModel, constraints);
+ if (point == null){
+ point = randomGeoPoint(planetModel, new Constraints());
+ }
+ geoPoints.add(point);
+ }
+ return geoPoints;
+ }
+
+ /**
+ * Check if a GeoPolygon is pure concave. Note that our definition for concavity is that the polygon
+ * contains antipodal points.
+ *
+ * @param planetModel The planet model.
+ * @param shape The polygon to check.
+ * @return True if the polygon contains antipodal points.
+ */
+ private boolean isConcave(PlanetModel planetModel, GeoPolygon shape){
+ return (shape.isWithin(planetModel.NORTH_POLE) && shape.isWithin(planetModel.SOUTH_POLE))||
+ (shape.isWithin(planetModel.MAX_X_POLE) && shape.isWithin(planetModel.MIN_X_POLE)) ||
+ (shape.isWithin(planetModel.MAX_Y_POLE) && shape.isWithin(planetModel.MIN_Y_POLE));
+ }
+
+ /**
+ * Check if a GeoPolygon is pure convex. Note that our definition for convexity is that the polygon
+ * does not contain antipodal points.
+ *
+ * @param planetModel The planet model.
+ * @param shape The polygon to check.
+ * @return True if the polygon dies not contains antipodal points.
+ */
+ private boolean isConvex(PlanetModel planetModel, GeoPolygon shape){
+ return !isConcave(planetModel,shape);
+ }
+
+ /**
+ * Generates a random number between 0 and PI.
+ *
+ * @return the cutoff angle.
+ */
+ private double randomCutoffAngle() {
+ while(true) {
+ double radius = randomDouble();
+ if (radius <0 || radius > Math.PI){
+ continue;
+ }
+ return radius;
+ }
+ }
+
+ /**
+ * Method that orders a lit of points anti-clock-wise to prevent crossing edges.
+ *
+ * @param originalPoints The points to order.
+ * @return The list of ordered points anti-clockwise.
+ */
+ private List<GeoPoint> orderPoints(List<GeoPoint> originalPoints){
+ List<GeoPoint> points = new ArrayList<>(originalPoints.size());
+ points.addAll(originalPoints); //make a copy
+ GeoPoint lPoint = getPointLefLon(points);
+ points.remove(lPoint);
+ GeoPoint rPoint = getPointRigthLon(points);
+ points.remove(rPoint);
+ List<GeoPoint> APoints = getPointsBelowAndSort(points, lPoint);
+ List<GeoPoint> BPoints = getPointsAboveAndsort(points, lPoint);
+ List<GeoPoint> result = new ArrayList<>();
+ result.add(lPoint);
+ result.addAll(APoints);
+ result.add(rPoint);
+ result.addAll(BPoints);
+ return result;
+ }
+
+ private List<GeoPoint> getPointsAboveAndsort(List<GeoPoint> points,GeoPoint lPoint) {
+ List<GeoPoint> BPoints = new ArrayList<>();
+ for (GeoPoint point : points){
+ if(point.getLatitude() > lPoint.getLatitude()){
+ BPoints.add(point);
+ }
+ }
+ Collections.sort(BPoints, new Comparator<GeoPoint>() {
+ public int compare(GeoPoint idx1, GeoPoint idx2) {
+ return Double.compare(idx1.getLongitude(), idx2.getLongitude());
+ }
+ });
+ return BPoints;
+ }
+
+ private List<GeoPoint> getPointsBelowAndSort(List<GeoPoint> points,GeoPoint lPoint) {
+ List<GeoPoint> APoints = new ArrayList<>();
+ for (GeoPoint point : points){
+ if(point.getLatitude() < lPoint.getLatitude()){
+ APoints.add(point);
+ }
+ }
+ Collections.sort(APoints, new Comparator<GeoPoint>() {
+ public int compare(GeoPoint idx1, GeoPoint idx2) {
+ return Double.compare(idx1.getLongitude(), idx2.getLongitude());
+ }
+ });
+ return APoints;
+ }
+
+ private GeoPoint getPointLefLon(List<GeoPoint> points) {
+ GeoPoint lPoint = null;
+ for (GeoPoint point : points){
+ if(lPoint == null ){
+ lPoint = point;
+ }
+ else{
+ if (lPoint.getLongitude() > point.getLongitude()){
+ lPoint = point;
+ }
+ }
+ }
+ return lPoint;
+ }
+
+ private GeoPoint getPointRigthLon(List<GeoPoint> points) {
+ GeoPoint rPoint = null;
+ for (GeoPoint point : points){
+ if(rPoint == null ){
+ rPoint = point;
+ }
+ else{
+ if (rPoint.getLongitude() < point.getLongitude()){
+ rPoint = point;
+ }
+ }
+ }
+ return rPoint;
+ }
+
+ /**
+ * Class that holds the constraints that are given to
+ * build shapes. It consists in a list of GeoAreaShapes
+ * and relationships the new shape needs to satisfy.
+ */
+ class Constraints extends HashMap<GeoAreaShape, Integer>{
+
+ /**
+ * Check if the shape is valid under the constraints.
+ *
+ * @param shape The shape to check
+ * @return true if the shape satisfy the constraints, else false.
+ */
+ public boolean valid(GeoShape shape) {
+ if (shape == null){
+ return false;
+ }
+ for (GeoAreaShape constraint : keySet()) {
+ if (constraint.getRelationship(shape) != get(constraint)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /**
+ * Check if a point is Within the constraints.
+ *
+ * @param point The point to check
+ * @return true if the point satisfy the constraints, else false.
+ */
+ public boolean isWithin(GeoPoint point) {
+ for (GeoShape constraint : keySet()) {
+ if (!(validPoint(point, constraint, get(constraint)))) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /**
+ * Check if a point is Within one constraint given by a shape and a relationship.
+ *
+ * @param point The point to check
+ * @param shape The shape of the constraint
+ * @param relationship The relationship of the constraint.
+ * @return true if the point satisfy the constraint, else false.
+ */
+ private boolean validPoint(GeoPoint point, GeoShape shape, int relationship) {
+ //For GeoCompositeMembershipShape we only consider the first shape to help
+ // converging
+ if (relationship == GeoArea.WITHIN && shape instanceof GeoCompositeMembershipShape) {
+ shape = (((GeoCompositeMembershipShape) shape).shapes.get(0));
+ }
+ switch (relationship) {
+ case GeoArea.DISJOINT:
+ return !shape.isWithin(point);
+ case GeoArea.OVERLAPS:
+ return true;
+ case GeoArea.CONTAINS:
+ return !shape.isWithin(point);
+ case GeoArea.WITHIN:
+ return shape.isWithin(point);
+ default:
+ return true;
+ }
+ }
+
+ /**
+ * Collect the CONTAINS constraints in the object
+ *
+ * @return the CONTAINS constraints.
+ */
+ public Constraints getContains(){
+ return getConstraintsOfType(GeoArea.CONTAINS);
+ }
+
+ /**
+ * Collect the WITHIN constraints in the object
+ *
+ * @return the WITHIN constraints.
+ */
+ public Constraints getWithin(){
+ return getConstraintsOfType(GeoArea.WITHIN);
+ }
+
+ /**
+ * Collect the OVERLAPS constraints in the object
+ *
+ * @return the OVERLAPS constraints.
+ */
+ public Constraints getOverlaps(){
+ return getConstraintsOfType(GeoArea.OVERLAPS);
+ }
+
+ /**
+ * Collect the DISJOINT constraints in the object
+ *
+ * @return the DISJOINT constraints.
+ */
+ public Constraints getDisjoint(){
+ return getConstraintsOfType(GeoArea.DISJOINT);
+ }
+
+ private Constraints getConstraintsOfType(int type){
+ Constraints constraints = new Constraints();
+ for (GeoAreaShape constraint : keySet()) {
+ if (type == get(constraint)) {
+ constraints.put(constraint, type);
+ }
+ }
+ return constraints;
+ }
+ }
+}
[36/50] lucene-solr:feature/autoscaling: SOLR-11343: fix
TestSegmentSorting.testSegmentTerminateEarly test failure caused by
SOLR-10990
Posted by sh...@apache.org.
SOLR-11343: fix TestSegmentSorting.testSegmentTerminateEarly test failure caused by SOLR-10990
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/37b8ac6f
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/37b8ac6f
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/37b8ac6f
Branch: refs/heads/feature/autoscaling
Commit: 37b8ac6ff5efc68a4637ddb65c7c15faef21227a
Parents: 0359db9
Author: Christine Poerschke <cp...@apache.org>
Authored: Mon Sep 11 09:24:00 2017 -0700
Committer: Christine Poerschke <cp...@apache.org>
Committed: Mon Sep 11 10:32:15 2017 -0700
----------------------------------------------------------------------
.../solr/handler/component/QueryComponent.java | 23 +++++++-------------
1 file changed, 8 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/37b8ac6f/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java b/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java
index 417d583..3d2a4ed 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java
@@ -357,14 +357,14 @@ public class QueryComponent extends SearchComponent
cmd.setSegmentTerminateEarly(false); // not supported, silently ignore any segmentTerminateEarly flag
try {
if (params.getBool(GroupParams.GROUP_DISTRIBUTED_FIRST, false)) {
- doProcessGroupedDistributedSearchFirstPhase(rb, result);
+ doProcessGroupedDistributedSearchFirstPhase(rb, cmd, result);
return;
} else if (params.getBool(GroupParams.GROUP_DISTRIBUTED_SECOND, false)) {
- doProcessGroupedDistributedSearchSecondPhase(rb, result);
+ doProcessGroupedDistributedSearchSecondPhase(rb, cmd, result);
return;
}
- doProcessGroupedSearch(rb, result);
+ doProcessGroupedSearch(rb, cmd, result);
return;
} catch (SyntaxError e) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
@@ -372,7 +372,7 @@ public class QueryComponent extends SearchComponent
}
// normal search result
- doProcessUngroupedSearch(rb, result);
+ doProcessUngroupedSearch(rb, cmd, result);
}
protected void doFieldSortValues(ResponseBuilder rb, SolrIndexSearcher searcher) throws IOException
@@ -1248,7 +1248,7 @@ public class QueryComponent extends SearchComponent
return true;
}
- private void doProcessGroupedDistributedSearchFirstPhase(ResponseBuilder rb, QueryResult result) throws IOException {
+ private void doProcessGroupedDistributedSearchFirstPhase(ResponseBuilder rb, QueryCommand cmd, QueryResult result) throws IOException {
GroupingSpecification groupingSpec = rb.getGroupingSpec();
assert null != groupingSpec : "GroupingSpecification is null";
@@ -1259,8 +1259,6 @@ public class QueryComponent extends SearchComponent
SolrIndexSearcher searcher = req.getSearcher();
IndexSchema schema = searcher.getSchema();
- QueryCommand cmd = rb.getQueryCommand();
-
CommandHandler.Builder topsGroupsActionBuilder = new CommandHandler.Builder()
.setQueryCommand(cmd)
.setNeedDocSet(false) // Order matters here
@@ -1286,7 +1284,7 @@ public class QueryComponent extends SearchComponent
rb.setResult(result);
}
- private void doProcessGroupedDistributedSearchSecondPhase(ResponseBuilder rb, QueryResult result) throws IOException, SyntaxError {
+ private void doProcessGroupedDistributedSearchSecondPhase(ResponseBuilder rb, QueryCommand cmd, QueryResult result) throws IOException, SyntaxError {
GroupingSpecification groupingSpec = rb.getGroupingSpec();
assert null != groupingSpec : "GroupingSpecification is null";
@@ -1299,7 +1297,6 @@ public class QueryComponent extends SearchComponent
SolrIndexSearcher searcher = req.getSearcher();
IndexSchema schema = searcher.getSchema();
- QueryCommand cmd = rb.getQueryCommand();
boolean needScores = (cmd.getFlags() & SolrIndexSearcher.GET_SCORES) != 0;
CommandHandler.Builder secondPhaseBuilder = new CommandHandler.Builder()
@@ -1358,7 +1355,7 @@ public class QueryComponent extends SearchComponent
rb.setResult(result);
}
- private void doProcessGroupedSearch(ResponseBuilder rb, QueryResult result) throws IOException, SyntaxError {
+ private void doProcessGroupedSearch(ResponseBuilder rb, QueryCommand cmd, QueryResult result) throws IOException, SyntaxError {
GroupingSpecification groupingSpec = rb.getGroupingSpec();
assert null != groupingSpec : "GroupingSpecification is null";
@@ -1370,8 +1367,6 @@ public class QueryComponent extends SearchComponent
SolrIndexSearcher searcher = req.getSearcher();
- QueryCommand cmd = rb.getQueryCommand();
-
int maxDocsPercentageToCache = params.getInt(GroupParams.GROUP_CACHE_PERCENTAGE, 0);
boolean cacheSecondPassSearch = maxDocsPercentageToCache >= 1 && maxDocsPercentageToCache <= 100;
Grouping.TotalCount defaultTotalCount = groupingSpec.isIncludeGroupCount() ?
@@ -1430,15 +1425,13 @@ public class QueryComponent extends SearchComponent
}
}
- private void doProcessUngroupedSearch(ResponseBuilder rb, QueryResult result) throws IOException {
+ private void doProcessUngroupedSearch(ResponseBuilder rb, QueryCommand cmd, QueryResult result) throws IOException {
SolrQueryRequest req = rb.req;
SolrQueryResponse rsp = rb.rsp;
SolrIndexSearcher searcher = req.getSearcher();
- QueryCommand cmd = rb.getQueryCommand();
-
searcher.search(result, cmd);
rb.setResult(result);
[22/50] lucene-solr:feature/autoscaling: SOLR-11338: Add Kendall's
Tau-b rank and Spearmans rank correlation Stream Evaluators
Posted by sh...@apache.org.
SOLR-11338: Add Kendall's Tau-b rank and Spearmans rank correlation Stream Evaluators
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/a6142b3b
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/a6142b3b
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/a6142b3b
Branch: refs/heads/feature/autoscaling
Commit: a6142b3b61b7265129dffee48b7e11a3924cafd1
Parents: 2b6c82e
Author: Joel Bernstein <jb...@apache.org>
Authored: Fri Sep 8 13:00:59 2017 -0400
Committer: Joel Bernstein <jb...@apache.org>
Committed: Fri Sep 8 13:01:16 2017 -0400
----------------------------------------------------------------------
.../org/apache/solr/handler/StreamHandler.java | 2 +
.../io/eval/KendallsCorrelationEvaluator.java | 57 ++++++++++++++++++++
.../io/eval/SpearmansCorrelationEvaluator.java | 57 ++++++++++++++++++++
.../solrj/io/stream/StreamExpressionTest.java | 5 +-
4 files changed, 120 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a6142b3b/solr/core/src/java/org/apache/solr/handler/StreamHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/StreamHandler.java b/solr/core/src/java/org/apache/solr/handler/StreamHandler.java
index e1203aa..bfbf4fb 100644
--- a/solr/core/src/java/org/apache/solr/handler/StreamHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/StreamHandler.java
@@ -250,6 +250,8 @@ public class StreamHandler extends RequestHandlerBase implements SolrCoreAware,
.withFunctionName("copyOf", CopyOfEvaluator.class)
.withFunctionName("cov", CovarianceEvaluator.class)
.withFunctionName("corr", CorrelationEvaluator.class)
+ .withFunctionName("kendallsCorr", KendallsCorrelationEvaluator.class)
+ .withFunctionName("spearmansCorr", SpearmansCorrelationEvaluator.class)
.withFunctionName("describe", DescribeEvaluator.class)
.withFunctionName("distance", EuclideanDistanceEvaluator.class)
.withFunctionName("empiricalDistribution", EmpiricalDistributionEvaluator.class)
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a6142b3b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/KendallsCorrelationEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/KendallsCorrelationEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/KendallsCorrelationEvaluator.java
new file mode 100644
index 0000000..0bf42ce
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/KendallsCorrelationEvaluator.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.client.solrj.io.eval;
+
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.util.List;
+import java.util.Locale;
+
+import org.apache.commons.math3.stat.correlation.KendallsCorrelation;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+
+public class KendallsCorrelationEvaluator extends RecursiveNumericEvaluator implements TwoValueWorker {
+ protected static final long serialVersionUID = 1L;
+
+ public KendallsCorrelationEvaluator(StreamExpression expression, StreamFactory factory) throws IOException{
+ super(expression, factory);
+ }
+
+ @Override
+ public Object doWork(Object first, Object second) throws IOException{
+ if(null == first){
+ throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - null found for the first value",toExpression(constructingFactory)));
+ }
+ if(null == second){
+ throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - null found for the second value",toExpression(constructingFactory)));
+ }
+ if(!(first instanceof List<?>)){
+ throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - found type %s for the first value, expecting a list of numbers",toExpression(constructingFactory), first.getClass().getSimpleName()));
+ }
+ if(!(second instanceof List<?>)){
+ throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - found type %s for the second value, expecting a list of numbers",toExpression(constructingFactory), first.getClass().getSimpleName()));
+ }
+
+ KendallsCorrelation kendallsCorrelation = new KendallsCorrelation();
+
+ return kendallsCorrelation.correlation(
+ ((List)first).stream().mapToDouble(value -> ((BigDecimal)value).doubleValue()).toArray(),
+ ((List)second).stream().mapToDouble(value -> ((BigDecimal)value).doubleValue()).toArray()
+ );
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a6142b3b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/SpearmansCorrelationEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/SpearmansCorrelationEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/SpearmansCorrelationEvaluator.java
new file mode 100644
index 0000000..2f30f08
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/SpearmansCorrelationEvaluator.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.client.solrj.io.eval;
+
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.util.List;
+import java.util.Locale;
+
+import org.apache.commons.math3.stat.correlation.SpearmansCorrelation;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+
+public class SpearmansCorrelationEvaluator extends RecursiveNumericEvaluator implements TwoValueWorker {
+ protected static final long serialVersionUID = 1L;
+
+ public SpearmansCorrelationEvaluator(StreamExpression expression, StreamFactory factory) throws IOException{
+ super(expression, factory);
+ }
+
+ @Override
+ public Object doWork(Object first, Object second) throws IOException{
+ if(null == first){
+ throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - null found for the first value",toExpression(constructingFactory)));
+ }
+ if(null == second){
+ throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - null found for the second value",toExpression(constructingFactory)));
+ }
+ if(!(first instanceof List<?>)){
+ throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - found type %s for the first value, expecting a list of numbers",toExpression(constructingFactory), first.getClass().getSimpleName()));
+ }
+ if(!(second instanceof List<?>)){
+ throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - found type %s for the second value, expecting a list of numbers",toExpression(constructingFactory), first.getClass().getSimpleName()));
+ }
+
+ SpearmansCorrelation spearmansCorrelation = new SpearmansCorrelation();
+
+ return spearmansCorrelation.correlation(
+ ((List)first).stream().mapToDouble(value -> ((BigDecimal)value).doubleValue()).toArray(),
+ ((List)second).stream().mapToDouble(value -> ((BigDecimal)value).doubleValue()).toArray()
+ );
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a6142b3b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
index fafac6f..47ccd86 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
@@ -5514,7 +5514,8 @@ public class StreamExpressionTest extends SolrCloudTestCase {
"field=\"test_dt\", " +
"count(*), sum(price_f), max(price_f), min(price_f))";
- String cexpr = "let(a="+expr+", b=select("+expr+",mult(-1, count(*)) as nvalue), c=col(a, count(*)), d=col(b, nvalue), tuple(corr=corr(c,d)))";
+ String cexpr = "let(a="+expr+", b=select("+expr+",mult(-1, count(*)) as nvalue), c=col(a, count(*)), d=col(b, nvalue), " +
+ "tuple(corr=corr(c,d), scorr=spearmansCorr(array(500, 50, 50, 50),d), kcorr=kendallsCorr(array(500, 50, 50, 50),d), d=d))";
ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
paramsLoc.set("expr", cexpr);
@@ -5528,6 +5529,8 @@ public class StreamExpressionTest extends SolrCloudTestCase {
List<Tuple> tuples = getTuples(solrStream);
assertTrue(tuples.size() == 1);
assertTrue(tuples.get(0).getDouble("corr").equals(-1.0D));
+ assertTrue(tuples.get(0).getDouble("scorr").equals(-1.0D));
+ assertTrue(tuples.get(0).getDouble("kcorr").equals(-1.0D));
}
[07/50] lucene-solr:feature/autoscaling: Update DOAP files with 6.6.1
release
Posted by sh...@apache.org.
Update DOAP files with 6.6.1 release
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/2a8d9b0e
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/2a8d9b0e
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/2a8d9b0e
Branch: refs/heads/feature/autoscaling
Commit: 2a8d9b0e6c19b0ed46075f27fd8dd0398b7f01b2
Parents: b4d280f
Author: Varun Thacker <va...@apache.org>
Authored: Wed Sep 6 22:15:13 2017 -0700
Committer: Varun Thacker <va...@apache.org>
Committed: Wed Sep 6 22:15:44 2017 -0700
----------------------------------------------------------------------
dev-tools/doap/lucene.rdf | 9 ++++++++-
dev-tools/doap/solr.rdf | 7 +++++++
2 files changed, 15 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2a8d9b0e/dev-tools/doap/lucene.rdf
----------------------------------------------------------------------
diff --git a/dev-tools/doap/lucene.rdf b/dev-tools/doap/lucene.rdf
index 1666886..7d6f60c 100644
--- a/dev-tools/doap/lucene.rdf
+++ b/dev-tools/doap/lucene.rdf
@@ -68,9 +68,16 @@
<release>
<Version>
+ <name>lucene-6.6.1</name>
+ <created>2017-09-07</created>
+ <revision>6.6.1</revision>
+ </Version>
+ </release>
+ <release>
+ <Version>
<name>lucene-6.6.0</name>
<created>2017-06-06</created>
- <revision>6.6.0</revision>
+ <revision>6.6.0</revision>
</Version>
</release>
<release>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2a8d9b0e/dev-tools/doap/solr.rdf
----------------------------------------------------------------------
diff --git a/dev-tools/doap/solr.rdf b/dev-tools/doap/solr.rdf
index a6000fa..8881fc3 100644
--- a/dev-tools/doap/solr.rdf
+++ b/dev-tools/doap/solr.rdf
@@ -68,6 +68,13 @@
<release>
<Version>
+ <name>solr-6.6.1</name>
+ <created>2017-09-07</created>
+ <revision>6.6.1</revision>
+ </Version>
+ </release>
+ <release>
+ <Version>
<name>solr-6.6.0</name>
<created>2017-06-06</created>
<revision>6.6.0</revision>
[32/50] lucene-solr:feature/autoscaling: SOLR-11345: restrict double
field values used for function queries to float range
Posted by sh...@apache.org.
SOLR-11345: restrict double field values used for function queries to float range
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/5f68eb94
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/5f68eb94
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/5f68eb94
Branch: refs/heads/feature/autoscaling
Commit: 5f68eb945626d6fb25d7fb23ee16f255aa23a9f2
Parents: cd425d6
Author: Steve Rowe <sa...@apache.org>
Authored: Mon Sep 11 10:08:01 2017 -0400
Committer: Steve Rowe <sa...@apache.org>
Committed: Mon Sep 11 10:08:01 2017 -0400
----------------------------------------------------------------------
.../src/test/org/apache/solr/schema/TestPointFields.java | 9 ++++++++-
1 file changed, 8 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5f68eb94/solr/core/src/test/org/apache/solr/schema/TestPointFields.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/schema/TestPointFields.java b/solr/core/src/test/org/apache/solr/schema/TestPointFields.java
index 5d2adb4..703ddd8 100644
--- a/solr/core/src/test/org/apache/solr/schema/TestPointFields.java
+++ b/solr/core/src/test/org/apache/solr/schema/TestPointFields.java
@@ -3330,7 +3330,14 @@ public class TestPointFields extends SolrTestCaseJ4 {
private void doTestDoublePointFunctionQuery(String field) throws Exception {
assertTrue(h.getCore().getLatestSchema().getField(field).getType() instanceof PointField);
int numVals = 10 * RANDOM_MULTIPLIER;
- List<Double> values = getRandomDoubles(numVals, false);
+ // Restrict values to float range; otherwise conversion to float will cause truncation -> undefined results
+ List<Double> values = getRandomList(10, false, () -> {
+ Float f = Float.NaN;
+ while (f.isNaN()) {
+ f = Float.intBitsToFloat(random().nextInt());
+ }
+ return f.doubleValue();
+ });
String assertNumFound = "//*[@numFound='" + numVals + "']";
String[] idAscXpathChecks = new String[numVals + 1];
String[] idAscNegXpathChecks = new String[numVals + 1];
[04/50] lucene-solr:feature/autoscaling: ref guide: fix dangling link
Posted by sh...@apache.org.
ref guide: fix dangling link
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/5e83475f
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/5e83475f
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/5e83475f
Branch: refs/heads/feature/autoscaling
Commit: 5e83475f9d268f3b20df0f5a72a3de8bfcfcb722
Parents: 26201c8
Author: Steve Rowe <sa...@gmail.com>
Authored: Wed Sep 6 13:40:09 2017 -0400
Committer: Steve Rowe <sa...@gmail.com>
Committed: Wed Sep 6 13:40:09 2017 -0400
----------------------------------------------------------------------
solr/solr-ref-guide/src/solrcloud-autoscaling-overview.adoc | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5e83475f/solr/solr-ref-guide/src/solrcloud-autoscaling-overview.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/solrcloud-autoscaling-overview.adoc b/solr/solr-ref-guide/src/solrcloud-autoscaling-overview.adoc
index 3facb65..d1c9822 100644
--- a/solr/solr-ref-guide/src/solrcloud-autoscaling-overview.adoc
+++ b/solr/solr-ref-guide/src/solrcloud-autoscaling-overview.adoc
@@ -50,7 +50,7 @@ The above create collection command will associate a policy named `policy1` with
Note that the collection-specific policy is applied *in addition to* the cluster policy, i.e., it is not an override but an augmentation. Therefore the collection will follow all conditions laid out in the cluster preferences, cluster policy, and the policy named `policy1`.
-You can learn more about collection-specific policies in the section <<solrcloud-autoscaling-policy-preferences.adoc#defining-collection-specific-policies,Defining Collection-Specific Policies>>.
+You can learn more about collection-specific policies in the section <<solrcloud-autoscaling-policy-preferences.adoc#collection-specific-policy,Defining Collection-Specific Policies>>.
== Autoscaling APIs
[14/50] lucene-solr:feature/autoscaling: SOLR-11314: FastCharStream:
re-use the READ_PAST_EOF exception
Posted by sh...@apache.org.
SOLR-11314: FastCharStream: re-use the READ_PAST_EOF exception
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/89feb150
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/89feb150
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/89feb150
Branch: refs/heads/feature/autoscaling
Commit: 89feb1500848d8d566d63be21d351d27a1bdcf6f
Parents: ce29124
Author: David Smiley <ds...@apache.org>
Authored: Thu Sep 7 21:12:45 2017 -0400
Committer: David Smiley <ds...@apache.org>
Committed: Thu Sep 7 21:12:45 2017 -0400
----------------------------------------------------------------------
solr/CHANGES.txt | 2 ++
solr/core/src/java/org/apache/solr/parser/FastCharStream.java | 7 ++++++-
2 files changed, 8 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/89feb150/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index be89fb1..f652fb2 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -139,6 +139,8 @@ Optimizations
* SOLR-11242: QueryParser: re-use the LookaheadSuccess exception. (Michael Braun via David Smiley)
+* SOLR-11314: FastCharStream: re-use the READ_PAST_EOF exception. (Michael Braun via David Smiley)
+
Other Changes
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/89feb150/solr/core/src/java/org/apache/solr/parser/FastCharStream.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/parser/FastCharStream.java b/solr/core/src/java/org/apache/solr/parser/FastCharStream.java
index 7039c0a..b9cb27a 100644
--- a/solr/core/src/java/org/apache/solr/parser/FastCharStream.java
+++ b/solr/core/src/java/org/apache/solr/parser/FastCharStream.java
@@ -69,7 +69,7 @@ public final class FastCharStream implements CharStream {
int charsRead = // fill space in buffer
input.read(buffer, newPosition, buffer.length-newPosition);
if (charsRead == -1)
- throw new IOException("read past eof");
+ throw READ_PAST_EOF;
else
bufferLength += charsRead;
}
@@ -80,6 +80,11 @@ public final class FastCharStream implements CharStream {
return readChar();
}
+ /**
+ * This Exception is used as a signal rather than an exceptional state.
+ */
+ private static final IOException READ_PAST_EOF = new IOException("read past eof");
+
@Override
public final void backup(int amount) {
bufferPosition -= amount;
[23/50] lucene-solr:feature/autoscaling: SOLR-11339: Add Canberra,
Chebyshev, Earth Movers and Manhattan Distance Stream Evaluators
Posted by sh...@apache.org.
SOLR-11339: Add Canberra, Chebyshev, Earth Movers and Manhattan Distance Stream Evaluators
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/e8dee89c
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/e8dee89c
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/e8dee89c
Branch: refs/heads/feature/autoscaling
Commit: e8dee89c878fd1d053531dcc3a83d264d8fa30ce
Parents: a6142b3
Author: Joel Bernstein <jb...@apache.org>
Authored: Fri Sep 8 15:44:28 2017 -0400
Committer: Joel Bernstein <jb...@apache.org>
Committed: Fri Sep 8 15:44:28 2017 -0400
----------------------------------------------------------------------
.../org/apache/solr/handler/StreamHandler.java | 4 ++
.../io/eval/CanberraDistanceEvaluator.java | 56 ++++++++++++++++++++
.../io/eval/ChebyshevDistanceEvaluator.java | 56 ++++++++++++++++++++
.../io/eval/EarthMoversDistanceEvaluator.java | 56 ++++++++++++++++++++
.../io/eval/ManhattanDistanceEvaluator.java | 56 ++++++++++++++++++++
.../solrj/io/stream/StreamExpressionTest.java | 10 +++-
6 files changed, 236 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e8dee89c/solr/core/src/java/org/apache/solr/handler/StreamHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/StreamHandler.java b/solr/core/src/java/org/apache/solr/handler/StreamHandler.java
index bfbf4fb..252def2 100644
--- a/solr/core/src/java/org/apache/solr/handler/StreamHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/StreamHandler.java
@@ -254,6 +254,10 @@ public class StreamHandler extends RequestHandlerBase implements SolrCoreAware,
.withFunctionName("spearmansCorr", SpearmansCorrelationEvaluator.class)
.withFunctionName("describe", DescribeEvaluator.class)
.withFunctionName("distance", EuclideanDistanceEvaluator.class)
+ .withFunctionName("manhattanDistance", ManhattanDistanceEvaluator.class)
+ .withFunctionName("earthMoversDistance", EarthMoversDistanceEvaluator.class)
+ .withFunctionName("canberraDistance", CanberraDistanceEvaluator.class)
+ .withFunctionName("chebyshevDistance", ChebyshevDistanceEvaluator.class)
.withFunctionName("empiricalDistribution", EmpiricalDistributionEvaluator.class)
.withFunctionName("finddelay", FindDelayEvaluator.class)
.withFunctionName("hist", HistogramEvaluator.class)
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e8dee89c/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/CanberraDistanceEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/CanberraDistanceEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/CanberraDistanceEvaluator.java
new file mode 100644
index 0000000..2271a34
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/CanberraDistanceEvaluator.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.client.solrj.io.eval;
+
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.util.List;
+import java.util.Locale;
+
+import org.apache.commons.math3.ml.distance.CanberraDistance;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+
+public class CanberraDistanceEvaluator extends RecursiveNumericEvaluator implements TwoValueWorker {
+ protected static final long serialVersionUID = 1L;
+
+ public CanberraDistanceEvaluator(StreamExpression expression, StreamFactory factory) throws IOException{
+ super(expression, factory);
+ }
+
+ @Override
+ public Object doWork(Object first, Object second) throws IOException{
+ if(null == first){
+ throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - null found for the first value",toExpression(constructingFactory)));
+ }
+ if(null == second){
+ throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - null found for the second value",toExpression(constructingFactory)));
+ }
+ if(!(first instanceof List<?>)){
+ throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - found type %s for the first value, expecting a list of numbers",toExpression(constructingFactory), first.getClass().getSimpleName()));
+ }
+ if(!(second instanceof List<?>)){
+ throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - found type %s for the second value, expecting a list of numbers",toExpression(constructingFactory), first.getClass().getSimpleName()));
+ }
+
+ CanberraDistance distance = new CanberraDistance();
+ return distance.compute(
+ ((List)first).stream().mapToDouble(value -> ((BigDecimal)value).doubleValue()).toArray(),
+ ((List)second).stream().mapToDouble(value -> ((BigDecimal)value).doubleValue()).toArray()
+ );
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e8dee89c/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/ChebyshevDistanceEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/ChebyshevDistanceEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/ChebyshevDistanceEvaluator.java
new file mode 100644
index 0000000..3a9294e
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/ChebyshevDistanceEvaluator.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.client.solrj.io.eval;
+
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.util.List;
+import java.util.Locale;
+
+import org.apache.commons.math3.ml.distance.ChebyshevDistance;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+
+public class ChebyshevDistanceEvaluator extends RecursiveNumericEvaluator implements TwoValueWorker {
+ protected static final long serialVersionUID = 1L;
+
+ public ChebyshevDistanceEvaluator(StreamExpression expression, StreamFactory factory) throws IOException{
+ super(expression, factory);
+ }
+
+ @Override
+ public Object doWork(Object first, Object second) throws IOException{
+ if(null == first){
+ throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - null found for the first value",toExpression(constructingFactory)));
+ }
+ if(null == second){
+ throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - null found for the second value",toExpression(constructingFactory)));
+ }
+ if(!(first instanceof List<?>)){
+ throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - found type %s for the first value, expecting a list of numbers",toExpression(constructingFactory), first.getClass().getSimpleName()));
+ }
+ if(!(second instanceof List<?>)){
+ throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - found type %s for the second value, expecting a list of numbers",toExpression(constructingFactory), first.getClass().getSimpleName()));
+ }
+
+ ChebyshevDistance distance = new ChebyshevDistance();
+ return distance.compute(
+ ((List)first).stream().mapToDouble(value -> ((BigDecimal)value).doubleValue()).toArray(),
+ ((List)second).stream().mapToDouble(value -> ((BigDecimal)value).doubleValue()).toArray()
+ );
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e8dee89c/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/EarthMoversDistanceEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/EarthMoversDistanceEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/EarthMoversDistanceEvaluator.java
new file mode 100644
index 0000000..0769d20
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/EarthMoversDistanceEvaluator.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.client.solrj.io.eval;
+
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.util.List;
+import java.util.Locale;
+
+import org.apache.commons.math3.ml.distance.EarthMoversDistance;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+
+public class EarthMoversDistanceEvaluator extends RecursiveNumericEvaluator implements TwoValueWorker {
+ protected static final long serialVersionUID = 1L;
+
+ public EarthMoversDistanceEvaluator(StreamExpression expression, StreamFactory factory) throws IOException{
+ super(expression, factory);
+ }
+
+ @Override
+ public Object doWork(Object first, Object second) throws IOException{
+ if(null == first){
+ throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - null found for the first value",toExpression(constructingFactory)));
+ }
+ if(null == second){
+ throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - null found for the second value",toExpression(constructingFactory)));
+ }
+ if(!(first instanceof List<?>)){
+ throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - found type %s for the first value, expecting a list of numbers",toExpression(constructingFactory), first.getClass().getSimpleName()));
+ }
+ if(!(second instanceof List<?>)){
+ throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - found type %s for the second value, expecting a list of numbers",toExpression(constructingFactory), first.getClass().getSimpleName()));
+ }
+
+ EarthMoversDistance distance = new EarthMoversDistance();
+ return distance.compute(
+ ((List)first).stream().mapToDouble(value -> ((BigDecimal)value).doubleValue()).toArray(),
+ ((List)second).stream().mapToDouble(value -> ((BigDecimal)value).doubleValue()).toArray()
+ );
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e8dee89c/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/ManhattanDistanceEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/ManhattanDistanceEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/ManhattanDistanceEvaluator.java
new file mode 100644
index 0000000..c0a8ed1
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/ManhattanDistanceEvaluator.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.client.solrj.io.eval;
+
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.util.List;
+import java.util.Locale;
+
+import org.apache.commons.math3.ml.distance.ManhattanDistance;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+
+public class ManhattanDistanceEvaluator extends RecursiveNumericEvaluator implements TwoValueWorker {
+ protected static final long serialVersionUID = 1L;
+
+ public ManhattanDistanceEvaluator(StreamExpression expression, StreamFactory factory) throws IOException{
+ super(expression, factory);
+ }
+
+ @Override
+ public Object doWork(Object first, Object second) throws IOException{
+ if(null == first){
+ throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - null found for the first value",toExpression(constructingFactory)));
+ }
+ if(null == second){
+ throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - null found for the second value",toExpression(constructingFactory)));
+ }
+ if(!(first instanceof List<?>)){
+ throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - found type %s for the first value, expecting a list of numbers",toExpression(constructingFactory), first.getClass().getSimpleName()));
+ }
+ if(!(second instanceof List<?>)){
+ throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - found type %s for the second value, expecting a list of numbers",toExpression(constructingFactory), first.getClass().getSimpleName()));
+ }
+
+ ManhattanDistance distance = new ManhattanDistance();
+ return distance.compute(
+ ((List)first).stream().mapToDouble(value -> ((BigDecimal)value).doubleValue()).toArray(),
+ ((List)second).stream().mapToDouble(value -> ((BigDecimal)value).doubleValue()).toArray()
+ );
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e8dee89c/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
index 47ccd86..01d6577 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
@@ -5608,7 +5608,10 @@ public class StreamExpressionTest extends SolrCloudTestCase {
"field=\"test_dt\", " +
"count(*), sum(price_f), max(price_f), min(price_f))";
- String cexpr = "let(a="+expr+", b=select("+expr+",mult(-1, count(*)) as nvalue), c=col(a, count(*)), d=col(b, nvalue), tuple(colc=c, cold=d, cov=cov(c,d), dist=distance(c,d)))";
+ String cexpr = "let(a="+expr+", b=select("+expr+",mult(-1, count(*)) as nvalue), c=col(a, count(*)), d=col(b, nvalue), " +
+ "tuple(colc=c, cold=d, cov=cov(c,d), dist=distance(c,d), " +
+ "mdist=manhattanDistance(c,d), edist=earthMoversDistance(c, d), cdist=canberraDistance(c,d)," +
+ "chdist=chebyshevDistance(c,d)))";
ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
paramsLoc.set("expr", cexpr);
@@ -5623,7 +5626,10 @@ public class StreamExpressionTest extends SolrCloudTestCase {
assertTrue(tuples.size() == 1);
assertTrue(tuples.get(0).getDouble("cov").equals(-625.0D));
assertTrue(tuples.get(0).getDouble("dist").equals(264.5751311064591D));
-
+ assertTrue(tuples.get(0).getDouble("mdist").equals(500.0D));
+ assertTrue(tuples.get(0).getDouble("cdist").equals(4.0D));
+ assertTrue(tuples.get(0).getDouble("chdist").equals(200.0D));
+ assertTrue(tuples.get(0).getDouble("edist").equals(1400.0D));
}
[12/50] lucene-solr:feature/autoscaling: SOLR-10783: Fix constructor.
Posted by sh...@apache.org.
SOLR-10783: Fix constructor.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/93882086
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/93882086
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/93882086
Branch: refs/heads/feature/autoscaling
Commit: 938820861334c6ad1de5efc520d78ac3fec71981
Parents: f828edf
Author: Mark Miller <ma...@apache.org>
Authored: Thu Sep 7 14:46:51 2017 -0500
Committer: Mark Miller <ma...@apache.org>
Committed: Thu Sep 7 14:46:57 2017 -0500
----------------------------------------------------------------------
.../providers/EnvSSLCredentialProvider.java | 4 ++++
.../providers/EnvSSLCredentialProviderTest.java | 9 +++++++++
.../providers/SysPropSSLCredentialProviderTest.java | 11 +++++++++++
3 files changed, 24 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/93882086/solr/core/src/java/org/apache/solr/util/configuration/providers/EnvSSLCredentialProvider.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/util/configuration/providers/EnvSSLCredentialProvider.java b/solr/core/src/java/org/apache/solr/util/configuration/providers/EnvSSLCredentialProvider.java
index ccfd807..6641cfe 100644
--- a/solr/core/src/java/org/apache/solr/util/configuration/providers/EnvSSLCredentialProvider.java
+++ b/solr/core/src/java/org/apache/solr/util/configuration/providers/EnvSSLCredentialProvider.java
@@ -44,6 +44,10 @@ public class EnvSSLCredentialProvider extends AbstractSSLCredentialProvider {
private Map<String, String> envVars;
+ public EnvSSLCredentialProvider() {
+ this.envVars = System.getenv();
+ }
+
protected EnumMap<CredentialType, String> getCredentialKeyMap() {
return Maps.newEnumMap(ImmutableMap.of(
SSL_KEY_STORE_PASSWORD, EnvVars.SOLR_SSL_KEY_STORE_PASSWORD,
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/93882086/solr/core/src/test/org/apache/solr/util/configuration/providers/EnvSSLCredentialProviderTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/util/configuration/providers/EnvSSLCredentialProviderTest.java b/solr/core/src/test/org/apache/solr/util/configuration/providers/EnvSSLCredentialProviderTest.java
index c355dce..2b85e7d 100644
--- a/solr/core/src/test/org/apache/solr/util/configuration/providers/EnvSSLCredentialProviderTest.java
+++ b/solr/core/src/test/org/apache/solr/util/configuration/providers/EnvSSLCredentialProviderTest.java
@@ -49,4 +49,13 @@ public class EnvSSLCredentialProviderTest {
}
}
+ @Test
+ public void testGetCredentialsWithEnvVars() throws Exception {
+ EnvSSLCredentialProvider sut = new EnvSSLCredentialProvider();
+ // assuming not to fail
+ sut.getCredential(SSLCredentialProvider.CredentialType.SSL_KEY_STORE_PASSWORD);
+ sut.getCredential(SSLCredentialProvider.CredentialType.SSL_CLIENT_KEY_STORE_PASSWORD);
+ sut.getCredential(SSLCredentialProvider.CredentialType.SSL_TRUST_STORE_PASSWORD);
+ sut.getCredential(SSLCredentialProvider.CredentialType.SSL_CLIENT_TRUST_STORE_PASSWORD);
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/93882086/solr/core/src/test/org/apache/solr/util/configuration/providers/SysPropSSLCredentialProviderTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/util/configuration/providers/SysPropSSLCredentialProviderTest.java b/solr/core/src/test/org/apache/solr/util/configuration/providers/SysPropSSLCredentialProviderTest.java
index e278cb5..4a5894d 100644
--- a/solr/core/src/test/org/apache/solr/util/configuration/providers/SysPropSSLCredentialProviderTest.java
+++ b/solr/core/src/test/org/apache/solr/util/configuration/providers/SysPropSSLCredentialProviderTest.java
@@ -52,4 +52,15 @@ public class SysPropSSLCredentialProviderTest {
assertThat(sut.getCredential(set.getKey()), is(pw));
}
}
+
+
+ @Test
+ public void testGetCredentialsWithoutSetup() throws Exception {
+ SysPropSSLCredentialProvider sut = new SysPropSSLCredentialProvider();
+ // assuming not to fail
+ sut.getCredential(SSLCredentialProvider.CredentialType.SSL_KEY_STORE_PASSWORD);
+ sut.getCredential(SSLCredentialProvider.CredentialType.SSL_CLIENT_KEY_STORE_PASSWORD);
+ sut.getCredential(SSLCredentialProvider.CredentialType.SSL_TRUST_STORE_PASSWORD);
+ sut.getCredential(SSLCredentialProvider.CredentialType.SSL_CLIENT_TRUST_STORE_PASSWORD);
+ }
}
[28/50] lucene-solr:feature/autoscaling: SOLR-11344: Fix copyOfRange
Stream Evaluator so the end index parameter can be equal to the length array
Posted by sh...@apache.org.
SOLR-11344: Fix copyOfRange Stream Evaluator so the end index parameter can be equal to the length array
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/a4fc14ba
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/a4fc14ba
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/a4fc14ba
Branch: refs/heads/feature/autoscaling
Commit: a4fc14baed323c1e52dfe85a3eff5aa7c7b6c5ec
Parents: 0283d3e
Author: Joel Bernstein <jb...@apache.org>
Authored: Sat Sep 9 20:23:07 2017 -0400
Committer: Joel Bernstein <jb...@apache.org>
Committed: Sat Sep 9 20:24:05 2017 -0400
----------------------------------------------------------------------
.../solr/client/solrj/io/eval/CopyOfRangeEvaluator.java | 4 ++--
.../client/solrj/io/stream/StreamExpressionTest.java | 11 ++++++++++-
2 files changed, 12 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a4fc14ba/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/CopyOfRangeEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/CopyOfRangeEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/CopyOfRangeEvaluator.java
index 685ef30..6688571 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/CopyOfRangeEvaluator.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/CopyOfRangeEvaluator.java
@@ -79,8 +79,8 @@ public class CopyOfRangeEvaluator extends RecursiveNumericEvaluator implements M
throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - startIdx (%d) must be less than endIdx (%d)", toExpression(constructingFactory), startIdx, endIdx));
}
- if(endIdx >= sourceValues.size()){
- throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - endIdx (%d) must be less than size of source array (%d)", toExpression(constructingFactory), endIdx, sourceValues.size()));
+ if(endIdx > sourceValues.size()){
+ throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - endIdx (%d) must not be greater then size of source array (%d)", toExpression(constructingFactory), endIdx, sourceValues.size()));
}
return Arrays.stream(Arrays.copyOfRange(sourceValues.toArray(), startIdx, endIdx)).collect(Collectors.toList());
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a4fc14ba/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
index 2ee881e..9b2b186 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
@@ -5776,7 +5776,7 @@ public class StreamExpressionTest extends SolrCloudTestCase {
"field=\"test_dt\", " +
"count(*), sum(price_f), max(price_f), min(price_f))";
- String cexpr = "let(a="+expr+", c=col(a, max(price_f)), tuple(copy=copyOfRange(c, 1, 3)))";
+ String cexpr = "let(a="+expr+", c=col(a, max(price_f)), tuple(copy=copyOfRange(c, 1, 3), copy2=copyOfRange(c, 2, 4), l=length(c)))";
ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
paramsLoc.set("expr", cexpr);
@@ -5793,6 +5793,15 @@ public class StreamExpressionTest extends SolrCloudTestCase {
assertTrue(copy1.size() == 2);
assertTrue(copy1.get(0).doubleValue() == 500D);
assertTrue(copy1.get(1).doubleValue() == 300D);
+
+ List<Number> copy2 = (List<Number>)tuples.get(0).get("copy2");
+ assertTrue(copy2.size() == 2);
+ assertTrue(copy2.get(0).doubleValue() == 300D);
+ assertTrue(copy2.get(1).doubleValue() == 400D);
+
+ long l = tuples.get(0).getLong("l");
+ assertTrue(l == 4);
+
}
[39/50] lucene-solr:feature/autoscaling: SOLR-11351: make
LTRScoringModel model more extensible
Posted by sh...@apache.org.
SOLR-11351: make LTRScoringModel model more extensible
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/0c831b99
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/0c831b99
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/0c831b99
Branch: refs/heads/feature/autoscaling
Commit: 0c831b9970550716e800478fe77ec05eecc05d57
Parents: 1544839
Author: Christine Poerschke <cp...@apache.org>
Authored: Mon Sep 11 12:35:27 2017 -0700
Committer: Christine Poerschke <cp...@apache.org>
Committed: Mon Sep 11 12:47:35 2017 -0700
----------------------------------------------------------------------
solr/CHANGES.txt | 2 ++
.../ltr/src/java/org/apache/solr/ltr/model/LTRScoringModel.java | 4 +++-
2 files changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0c831b99/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 519852e..5d243c5 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -203,6 +203,8 @@ Other Changes
* SOLR-11132: Refactor common getSortField logic in various FieldTypes (Jason Gerlowski, hossman)
+* SOLR-11351: Make LTRScoringModel model more extensible. (Christine Poerschke)
+
================== 7.0.0 ==================
Versions of Major Components
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0c831b99/solr/contrib/ltr/src/java/org/apache/solr/ltr/model/LTRScoringModel.java
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/java/org/apache/solr/ltr/model/LTRScoringModel.java b/solr/contrib/ltr/src/java/org/apache/solr/ltr/model/LTRScoringModel.java
index 9edcfe5..2b4f575 100644
--- a/solr/contrib/ltr/src/java/org/apache/solr/ltr/model/LTRScoringModel.java
+++ b/solr/contrib/ltr/src/java/org/apache/solr/ltr/model/LTRScoringModel.java
@@ -80,7 +80,7 @@ public abstract class LTRScoringModel {
protected final List<Feature> features;
private final List<Feature> allFeatures;
private final Map<String,Object> params;
- private final List<Normalizer> norms;
+ protected final List<Normalizer> norms;
public static LTRScoringModel getInstance(SolrResourceLoader solrResourceLoader,
String className, String name, List<Feature> features,
@@ -123,6 +123,8 @@ public abstract class LTRScoringModel {
* {@link ModelException} if they do not make sense.
*/
protected void validate() throws ModelException {
+ final List<Feature> features = getFeatures();
+ final List<Normalizer> norms = getNorms();
if (features.isEmpty()) {
throw new ModelException("no features declared for model "+name);
}
[09/50] lucene-solr:feature/autoscaling: SOLR-11181: fix
validate-maven-dependencies
Posted by sh...@apache.org.
SOLR-11181: fix validate-maven-dependencies
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/5324c743
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/5324c743
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/5324c743
Branch: refs/heads/feature/autoscaling
Commit: 5324c7436c8ba578fa38d8e7fc8219abac154558
Parents: a864c62
Author: Steve Rowe <sa...@gmail.com>
Authored: Thu Sep 7 09:22:56 2017 -0400
Committer: Steve Rowe <sa...@gmail.com>
Committed: Thu Sep 7 09:24:02 2017 -0400
----------------------------------------------------------------------
build.xml | 18 +++++++++++++-
lucene/analysis/build.xml | 4 ++++
lucene/build.xml | 13 +++++++++++
lucene/codecs/build.xml | 2 ++
lucene/common-build.xml | 53 +++++++++++++++++++++++++++++++++++++++++-
lucene/core/build.xml | 2 ++
solr/build.xml | 16 ++++++++++++-
solr/core/build.xml | 2 ++
solr/solrj/build.xml | 2 ++
solr/webapp/build.xml | 3 +++
10 files changed, 112 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5324c743/build.xml
----------------------------------------------------------------------
diff --git a/build.xml b/build.xml
index f2043e2..5335957 100755
--- a/build.xml
+++ b/build.xml
@@ -374,7 +374,23 @@
</subant>
</target>
- <target name="validate-maven-dependencies" depends="generate-maven-artifacts"
+ <target name="-install-maven-artifacts" depends="resolve,resolve-groovy,resolve-markdown,install-maven-tasks">
+ <ant dir="lucene" inheritall="false">
+ <target name="-unpack-lucene-tgz"/>
+ <target name="-filter-pom-templates"/>
+ <propertyset refid="uptodate.and.compiled.properties"/>
+ </ant>
+ <ant dir="solr" target="-unpack-solr-tgz" inheritall="false">
+ <propertyset refid="uptodate.and.compiled.properties"/>
+ </ant>
+ <subant target="-install-to-maven-local-repo" inheritall="false" failonerror="true">
+ <fileset dir="lucene" includes="build.xml" />
+ <fileset dir="solr" includes="build.xml" />
+ <propertyset refid="uptodate.and.compiled.properties"/>
+ </subant>
+ </target>
+
+ <target name="validate-maven-dependencies" depends="-install-maven-artifacts"
description="Validates maven dependencies, licenses, etc.">
<subant target="-validate-maven-dependencies" inheritall="false" failonerror="true">
<fileset dir="lucene" includes="build.xml"/>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5324c743/lucene/analysis/build.xml
----------------------------------------------------------------------
diff --git a/lucene/analysis/build.xml b/lucene/analysis/build.xml
index 5212cd6..844f5f3 100644
--- a/lucene/analysis/build.xml
+++ b/lucene/analysis/build.xml
@@ -125,6 +125,10 @@
<forall-analyzers target="-dist-maven"/>
</target>
+ <target name="-install-to-maven-local-repo">
+ <forall-analyzers target="-install-to-maven-local-repo"/>
+ </target>
+
<target name="-validate-maven-dependencies">
<forall-analyzers target="-validate-maven-dependencies"/>
</target>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5324c743/lucene/build.xml
----------------------------------------------------------------------
diff --git a/lucene/build.xml b/lucene/build.xml
index 9397000..3aa4e42 100644
--- a/lucene/build.xml
+++ b/lucene/build.xml
@@ -428,6 +428,19 @@
</sequential>
</target>
+ <target name="-install-to-maven-local-repo" depends="install-maven-tasks">
+ <sequential>
+ <m2-install pom.xml="${filtered.pom.templates.dir}/pom.xml"/> <!-- Lucene/Solr grandparent POM -->
+ <m2-install pom.xml="${filtered.pom.templates.dir}/lucene/pom.xml"/> <!-- Lucene parent POM -->
+ <subant target="-install-to-maven-local-repo" failonerror="true" inheritall="false">
+ <propertyset refid="uptodate.and.compiled.properties"/>
+ <fileset dir="${common.dir}/core" includes="build.xml"/>
+ <fileset dir="${common.dir}/test-framework" includes="build.xml"/>
+ </subant>
+ <modules-crawl target="-install-to-maven-local-repo"/>
+ </sequential>
+ </target>
+
<target name="generate-maven-artifacts" depends="-unpack-lucene-tgz">
<ant dir=".." target="resolve" inheritall="false"/>
<antcall target="-filter-pom-templates" inheritall="false"/>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5324c743/lucene/codecs/build.xml
----------------------------------------------------------------------
diff --git a/lucene/codecs/build.xml b/lucene/codecs/build.xml
index b1bf5ac..c50af6c 100644
--- a/lucene/codecs/build.xml
+++ b/lucene/codecs/build.xml
@@ -23,4 +23,6 @@
<import file="../module-build.xml"/>
<target name="-dist-maven" depends="-dist-maven-src-java"/>
+
+ <target name="-install-to-maven-local-repo" depends="-install-src-java-to-maven-local-repo"/>
</project>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5324c743/lucene/common-build.xml
----------------------------------------------------------------------
diff --git a/lucene/common-build.xml b/lucene/common-build.xml
index 763fe8f..cc0b356 100644
--- a/lucene/common-build.xml
+++ b/lucene/common-build.xml
@@ -595,7 +595,20 @@
</artifact:install>
</sequential>
</macrodef>
-
+
+ <macrodef name="m2-install" description="Installs a Maven artifact into the local repository">
+ <element name="parent-poms" optional="yes"/>
+ <attribute name="pom.xml"/>
+ <attribute name="jar.file" default="${dist.jar.dir.prefix}-${version}/${dist.jar.dir.suffix}/${final.name}.jar"/>
+ <sequential>
+ <parent-poms/>
+ <artifact:pom id="maven.project" file="@{pom.xml}"/>
+ <artifact:install file="@{jar.file}">
+ <pom refid="maven.project"/>
+ </artifact:install>
+ </sequential>
+ </macrodef>
+
<!-- validate maven dependencies -->
<macrodef name="m2-validate-dependencies">
<attribute name="pom.xml"/>
@@ -1713,6 +1726,44 @@ ${tests-output}/junit4-*.suites - per-JVM executed suites
</sequential>
</target>
+ <target name="-install-to-maven-local-repo" depends="install-maven-tasks">
+ <sequential>
+ <property name="top.level.dir" location="${common.dir}/.."/>
+ <pathconvert property="pom.xml">
+ <mapper>
+ <chainedmapper>
+ <globmapper from="${top.level.dir}*" to="${filtered.pom.templates.dir}*"/>
+ <globmapper from="*build.xml" to="*pom.xml"/>
+ </chainedmapper>
+ </mapper>
+ <path location="${ant.file}"/>
+ </pathconvert>
+ <artifact:pom id="maven.project" file="${pom.xml}"/>
+ <artifact:install file="${dist.jar.dir.prefix}-${version}/${dist.jar.dir.suffix}/${final.name}.jar">
+ <pom refid="maven.project"/>
+ </artifact:install>
+ </sequential>
+ </target>
+
+ <target name="-install-src-java-to-maven-local-repo" depends="install-maven-tasks">
+ <sequential>
+ <property name="top.level.dir" location="${common.dir}/.."/>
+ <pathconvert property="pom.xml">
+ <mapper>
+ <chainedmapper>
+ <globmapper from="${top.level.dir}*" to="${filtered.pom.templates.dir}*"/>
+ <globmapper from="*build.xml" to="*/src/java/pom.xml"/>
+ </chainedmapper>
+ </mapper>
+ <path location="${ant.file}"/>
+ </pathconvert>
+ <artifact:pom id="maven.project" file="${pom.xml}"/>
+ <artifact:install file="${dist.jar.dir.prefix}-${version}/${dist.jar.dir.suffix}/${final.name}.jar">
+ <pom refid="maven.project"/>
+ </artifact:install>
+ </sequential>
+ </target>
+
<target name="-dist-maven-src-java" depends="install-maven-tasks, jar-src, javadocs">
<sequential>
<property name="top.level.dir" location="${common.dir}/.."/>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5324c743/lucene/core/build.xml
----------------------------------------------------------------------
diff --git a/lucene/core/build.xml b/lucene/core/build.xml
index 4e62e1c..d968798 100644
--- a/lucene/core/build.xml
+++ b/lucene/core/build.xml
@@ -64,6 +64,8 @@
<target name="-dist-maven" depends="-dist-maven-src-java"/>
+ <target name="-install-to-maven-local-repo" depends="-install-src-java-to-maven-local-repo"/>
+
<macrodef name="createLevAutomaton">
<attribute name="n"/>
<sequential>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5324c743/solr/build.xml
----------------------------------------------------------------------
diff --git a/solr/build.xml b/solr/build.xml
index 5048f1d..16d4201 100644
--- a/solr/build.xml
+++ b/solr/build.xml
@@ -709,7 +709,21 @@
<contrib-crawl target="-dist-maven"/>
</sequential>
</target>
-
+
+ <target name="-install-to-maven-local-repo" depends="install-maven-tasks">
+ <sequential>
+ <m2-install pom.xml="${filtered.pom.templates.dir}/solr/pom.xml"/> <!-- Solr parent POM -->
+ <subant target="-install-to-maven-local-repo" inheritall="false">
+ <propertyset refid="uptodate.and.compiled.properties"/>
+ <fileset dir="core" includes="build.xml"/>
+ <fileset dir="solrj" includes="build.xml"/>
+ <fileset dir="test-framework" includes="build.xml"/>
+ <fileset dir="webapp" includes="build.xml"/>
+ </subant>
+ <contrib-crawl target="-install-to-maven-local-repo"/>
+ </sequential>
+ </target>
+
<target name="generate-maven-artifacts" depends="-unpack-solr-tgz">
<ant dir=".." target="resolve" inheritall="false"/>
<antcall target="-filter-pom-templates" inheritall="false"/>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5324c743/solr/core/build.xml
----------------------------------------------------------------------
diff --git a/solr/core/build.xml b/solr/core/build.xml
index af4c262..f7a2a59 100644
--- a/solr/core/build.xml
+++ b/solr/core/build.xml
@@ -60,6 +60,8 @@
<target name="-dist-maven" depends="-dist-maven-src-java"/>
+ <target name="-install-to-maven-local-repo" depends="-install-src-java-to-maven-local-repo"/>
+
<target name="resolve" depends="ivy-availability-check,ivy-fail,ivy-configure">
<sequential>
<ivy:retrieve conf="compile,compile.hadoop" type="jar,bundle" sync="${ivy.sync}" log="download-only" symlink="${ivy.symlink}"/>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5324c743/solr/solrj/build.xml
----------------------------------------------------------------------
diff --git a/solr/solrj/build.xml b/solr/solrj/build.xml
index 097171b..3393438 100644
--- a/solr/solrj/build.xml
+++ b/solr/solrj/build.xml
@@ -83,4 +83,6 @@
</target>
<target name="-dist-maven" depends="-dist-maven-src-java"/>
+
+ <target name="-install-to-maven-local-repo" depends="-install-src-java-to-maven-local-repo"/>
</project>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5324c743/solr/webapp/build.xml
----------------------------------------------------------------------
diff --git a/solr/webapp/build.xml b/solr/webapp/build.xml
index 2ebe084..99f8d17 100644
--- a/solr/webapp/build.xml
+++ b/solr/webapp/build.xml
@@ -62,5 +62,8 @@
<target name="-dist-maven"/>
<!-- nothing to do -->
+ <target name="-install-to-maven-local-repo"/>
+
+ <!-- nothing to do -->
<target name="-validate-maven-dependencies"/>
</project>
[41/50] lucene-solr:feature/autoscaling: LUCENE-7967: Within check
for GeoDegeneratePoint did not allow precision allowance.
Posted by sh...@apache.org.
LUCENE-7967: Within check for GeoDegeneratePoint did not allow precision allowance.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/b8d48fa7
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/b8d48fa7
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/b8d48fa7
Branch: refs/heads/feature/autoscaling
Commit: b8d48fa7a623cc4edb11998d44760555055f65eb
Parents: fe34e14
Author: Karl Wright <Da...@gmail.com>
Authored: Tue Sep 12 08:34:39 2017 -0400
Committer: Karl Wright <Da...@gmail.com>
Committed: Tue Sep 12 08:34:39 2017 -0400
----------------------------------------------------------------------
.../apache/lucene/spatial3d/geom/GeoDegeneratePoint.java | 2 +-
.../spatial3d/geom/SimpleGeoPolygonRelationshipsTest.java | 9 +++++++++
2 files changed, 10 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b8d48fa7/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoDegeneratePoint.java
----------------------------------------------------------------------
diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoDegeneratePoint.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoDegeneratePoint.java
index 572d00a..fe9ac79 100644
--- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoDegeneratePoint.java
+++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoDegeneratePoint.java
@@ -125,7 +125,7 @@ class GeoDegeneratePoint extends GeoPoint implements GeoPointShape {
@Override
public boolean isWithin(final double x, final double y, final double z) {
- return x == this.x && y == this.y && z == this.z;
+ return this.isIdentical(x, y, z);
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b8d48fa7/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/SimpleGeoPolygonRelationshipsTest.java
----------------------------------------------------------------------
diff --git a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/SimpleGeoPolygonRelationshipsTest.java b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/SimpleGeoPolygonRelationshipsTest.java
index 2589a19..cab8e06 100644
--- a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/SimpleGeoPolygonRelationshipsTest.java
+++ b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/SimpleGeoPolygonRelationshipsTest.java
@@ -24,6 +24,7 @@ import java.util.List;
import org.junit.Test;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
/**
* Check relationship between polygon and GeoShapes of basic polygons. Normally we construct
@@ -737,6 +738,14 @@ public class SimpleGeoPolygonRelationshipsTest {
}
@Test
+ public void testDegeneratedPointInPole(){
+ GeoBBox bBox1 = GeoBBoxFactory.makeGeoBBox(PlanetModel.SPHERE, Math.PI*0.5, Math.PI*0.5, 0, 0);
+ GeoPoint point = new GeoPoint(PlanetModel.SPHERE, Math.PI*0.5, Math.PI);
+ System.out.println("bbox1 = "+bBox1+"; point = "+point);
+ assertTrue(bBox1.isWithin(point));
+ }
+
+ @Test
public void testDegeneratePathShape(){
GeoPoint point1 = new GeoPoint(PlanetModel.SPHERE, 0, 0);
GeoPoint point2 = new GeoPoint(PlanetModel.SPHERE, 0, 1);
[03/50] lucene-solr:feature/autoscaling: SOLR-11241: Add discrete
counting and probability Stream Evaluators
Posted by sh...@apache.org.
SOLR-11241: Add discrete counting and probability Stream Evaluators
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/44966121
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/44966121
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/44966121
Branch: refs/heads/feature/autoscaling
Commit: 4496612120e361fc9cf2df27115297ceb35a81cc
Parents: 1fe611d
Author: Joel Bernstein <jb...@apache.org>
Authored: Wed Sep 6 11:02:24 2017 -0400
Committer: Joel Bernstein <jb...@apache.org>
Committed: Wed Sep 6 11:32:04 2017 -0400
----------------------------------------------------------------------
lucene/ivy-versions.properties | 2 +-
.../org/apache/solr/handler/StreamHandler.java | 14 +-
.../io/eval/BinomialDistributionEvaluator.java | 48 +++++++
.../io/eval/CumulativeProbabilityEvaluator.java | 21 +--
.../eval/EnumeratedDistributionEvaluator.java | 45 ++++++
.../solrj/io/eval/FrequencyTableEvaluator.java | 86 +++++++++++
.../solrj/io/eval/HistogramEvaluator.java | 7 +-
.../io/eval/PoissonDistributionEvaluator.java | 44 ++++++
.../solrj/io/eval/ProbabilityEvaluator.java | 52 +++++++
.../client/solrj/io/eval/SampleEvaluator.java | 18 ++-
.../UniformIntegerDistributionEvaluator.java | 48 +++++++
.../solrj/io/stream/StreamExpressionTest.java | 144 +++++++++++++++++++
12 files changed, 509 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/44966121/lucene/ivy-versions.properties
----------------------------------------------------------------------
diff --git a/lucene/ivy-versions.properties b/lucene/ivy-versions.properties
index 66b7c89..8c21e99 100644
--- a/lucene/ivy-versions.properties
+++ b/lucene/ivy-versions.properties
@@ -94,7 +94,7 @@ org.apache.calcite.version = 1.13.0
/org.apache.commons/commons-compress = 1.11
/org.apache.commons/commons-exec = 1.3
/org.apache.commons/commons-lang3 = 3.6
-/org.apache.commons/commons-math3 = 3.4.1
+/org.apache.commons/commons-math3 = 3.6.1
org.apache.curator.version = 2.8.0
/org.apache.curator/curator-client = ${org.apache.curator.version}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/44966121/solr/core/src/java/org/apache/solr/handler/StreamHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/StreamHandler.java b/solr/core/src/java/org/apache/solr/handler/StreamHandler.java
index 9613ec3..5396acd 100644
--- a/solr/core/src/java/org/apache/solr/handler/StreamHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/StreamHandler.java
@@ -29,6 +29,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import org.apache.commons.math3.distribution.PoissonDistribution;
import org.apache.solr.client.solrj.io.ModelCache;
import org.apache.solr.client.solrj.io.SolrClientCache;
import org.apache.solr.client.solrj.io.Tuple;
@@ -249,6 +250,7 @@ public class StreamHandler extends RequestHandlerBase implements SolrCoreAware,
.withFunctionName("copyOfRange", CopyOfRangeEvaluator.class)
.withFunctionName("copyOf", CopyOfEvaluator.class)
.withFunctionName("cov", CovarianceEvaluator.class)
+ .withFunctionName("corr", CorrelationEvaluator.class)
.withFunctionName("describe", DescribeEvaluator.class)
.withFunctionName("distance", EuclideanDistanceEvaluator.class)
.withFunctionName("empiricalDistribution", EmpiricalDistributionEvaluator.class)
@@ -281,9 +283,18 @@ public class StreamHandler extends RequestHandlerBase implements SolrCoreAware,
.withFunctionName("ebeDivide", EBEDivideEvaluator.class)
.withFunctionName("dotProduct", DotProductEvaluator.class)
.withFunctionName("cosineSimilarity", CosineSimilarityEvaluator.class)
+ .withFunctionName("freqTable", FrequencyTableEvaluator.class)
+ .withFunctionName("uniformIntegerDistribution", UniformIntegerDistributionEvaluator.class)
+ .withFunctionName("binomialDistribution", BinomialDistributionEvaluator.class)
+ .withFunctionName("poissonDistribution", PoissonDistributionEvaluator.class)
+ .withFunctionName("enumeratedDistribution", EnumeratedDistributionEvaluator.class)
+ .withFunctionName("probability", ProbabilityEvaluator.class)
// Boolean Stream Evaluators
- .withFunctionName("and", AndEvaluator.class)
+
+
+
+ .withFunctionName("and", AndEvaluator.class)
.withFunctionName("eor", ExclusiveOrEvaluator.class)
.withFunctionName("eq", EqualToEvaluator.class)
.withFunctionName("gt", GreaterThanEvaluator.class)
@@ -331,7 +342,6 @@ public class StreamHandler extends RequestHandlerBase implements SolrCoreAware,
.withFunctionName("cbrt", CubedRootEvaluator.class)
.withFunctionName("coalesce", CoalesceEvaluator.class)
.withFunctionName("uuid", UuidEvaluator.class)
- .withFunctionName("corr", CorrelationEvaluator.class)
// Conditional Stream Evaluators
.withFunctionName("if", IfThenElseEvaluator.class)
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/44966121/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/BinomialDistributionEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/BinomialDistributionEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/BinomialDistributionEvaluator.java
new file mode 100644
index 0000000..a0a6fb7
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/BinomialDistributionEvaluator.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.client.solrj.io.eval;
+
+import java.io.IOException;
+import java.util.Locale;
+
+import org.apache.commons.math3.distribution.BinomialDistribution;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+
+public class BinomialDistributionEvaluator extends RecursiveNumericEvaluator implements TwoValueWorker {
+
+ private static final long serialVersionUID = 1;
+
+ public BinomialDistributionEvaluator(StreamExpression expression, StreamFactory factory) throws IOException {
+ super(expression, factory);
+ }
+
+ @Override
+ public Object doWork(Object first, Object second) throws IOException{
+ if(null == first){
+ throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - null found for the first value",toExpression(constructingFactory)));
+ }
+ if(null == second){
+ throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - null found for the second value",toExpression(constructingFactory)));
+ }
+
+ Number numberOfTrials = (Number)first;
+ Number successProb = (Number)second;
+
+ return new BinomialDistribution(numberOfTrials.intValue(), successProb.doubleValue());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/44966121/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/CumulativeProbabilityEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/CumulativeProbabilityEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/CumulativeProbabilityEvaluator.java
index 8ecf35d..a7e6d5a 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/CumulativeProbabilityEvaluator.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/CumulativeProbabilityEvaluator.java
@@ -20,6 +20,7 @@ import java.io.IOException;
import java.util.Locale;
import org.apache.commons.math3.distribution.RealDistribution;
+import org.apache.commons.math3.distribution.IntegerDistribution;
import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
@@ -38,17 +39,21 @@ public class CumulativeProbabilityEvaluator extends RecursiveObjectEvaluator imp
if(null == second){
throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - null found for the second value",toExpression(constructingFactory)));
}
- if(!(first instanceof RealDistribution)){
- throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - found type %s for the first value, expecting a RealDistribution",toExpression(constructingFactory), first.getClass().getSimpleName()));
+ if(!(first instanceof RealDistribution) && !(first instanceof IntegerDistribution)){
+ throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - found type %s for the first value, expecting a real or integer Distribution",toExpression(constructingFactory), first.getClass().getSimpleName()));
}
if(!(second instanceof Number)){
throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - found type %s for the second value, expecting a Number",toExpression(constructingFactory), first.getClass().getSimpleName()));
}
-
- RealDistribution rd = (RealDistribution)first;
- Number predictOver = (Number)second;
-
- return rd.cumulativeProbability(predictOver.doubleValue());
+
+ if(first instanceof RealDistribution) {
+ RealDistribution rd = (RealDistribution) first;
+ Number predictOver = (Number) second;
+ return rd.cumulativeProbability(predictOver.doubleValue());
+ } else {
+ IntegerDistribution id = (IntegerDistribution) first;
+ Number predictOver = (Number) second;
+ return id.cumulativeProbability(predictOver.intValue());
+ }
}
-
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/44966121/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/EnumeratedDistributionEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/EnumeratedDistributionEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/EnumeratedDistributionEvaluator.java
new file mode 100644
index 0000000..a14e54b
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/EnumeratedDistributionEvaluator.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.client.solrj.io.eval;
+
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.util.List;
+import java.util.Locale;
+
+import org.apache.commons.math3.distribution.EnumeratedIntegerDistribution;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+
+public class EnumeratedDistributionEvaluator extends RecursiveNumericEvaluator implements OneValueWorker {
+
+ private static final long serialVersionUID = 1;
+
+ public EnumeratedDistributionEvaluator(StreamExpression expression, StreamFactory factory) throws IOException {
+ super(expression, factory);
+ }
+
+ @Override
+ public Object doWork(Object first) throws IOException{
+ if(null == first){
+ throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - null found for the first value",toExpression(constructingFactory)));
+ }
+
+ int[] samples = ((List)first).stream().mapToInt(value -> ((BigDecimal) value).intValue()).toArray();
+ return new EnumeratedIntegerDistribution(samples);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/44966121/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/FrequencyTableEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/FrequencyTableEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/FrequencyTableEvaluator.java
new file mode 100644
index 0000000..ae65dc1
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/FrequencyTableEvaluator.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.client.solrj.io.eval;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+
+import org.apache.commons.math3.random.EmpiricalDistribution;
+import org.apache.commons.math3.stat.Frequency;
+
+import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
+import org.apache.solr.client.solrj.io.Tuple;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+
+public class FrequencyTableEvaluator extends RecursiveNumericEvaluator implements ManyValueWorker {
+ protected static final long serialVersionUID = 1L;
+
+ public FrequencyTableEvaluator(StreamExpression expression, StreamFactory factory) throws IOException{
+ super(expression, factory);
+
+ if(containedEvaluators.size() < 1){
+ throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - expecting at least one value but found %d",expression,containedEvaluators.size()));
+ }
+ }
+
+ @Override
+ public Object doWork(Object... values) throws IOException {
+ if(Arrays.stream(values).anyMatch(item -> null == item)){
+ return null;
+ }
+
+ List<?> sourceValues;
+
+ if(values.length == 1){
+ sourceValues = values[0] instanceof List<?> ? (List<?>)values[0] : Arrays.asList(values[0]);
+ }
+ else
+ {
+ throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - expecting at least one value but found %d",toExpression(constructingFactory),containedEvaluators.size()));
+ }
+
+ Frequency frequency = new Frequency();
+
+ for(Object o : sourceValues) {
+ Number number = (Number)o;
+ frequency.addValue(number.longValue());
+ }
+
+ List<Tuple> histogramBins = new ArrayList<>();
+
+ Iterator iterator = frequency.valuesIterator();
+
+ while(iterator.hasNext()){
+ Long value = (Long)iterator.next();
+ Map<String,Number> map = new HashMap<>();
+ map.put("value", value.longValue());
+ map.put("count", frequency.getCount(value));
+ map.put("cumFreq", frequency.getCumFreq(value));
+ map.put("cumPct", frequency.getCumPct(value));
+ map.put("pct", frequency.getPct(value));
+ histogramBins.add(new Tuple(map));
+ }
+ return histogramBins;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/44966121/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/HistogramEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/HistogramEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/HistogramEvaluator.java
index f58f319..8d27614 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/HistogramEvaluator.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/HistogramEvaluator.java
@@ -26,6 +26,7 @@ import java.util.Map;
import org.apache.commons.math3.random.EmpiricalDistribution;
import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
+import org.apache.solr.client.solrj.io.Tuple;
import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
@@ -68,7 +69,7 @@ public class HistogramEvaluator extends RecursiveNumericEvaluator implements Man
EmpiricalDistribution distribution = new EmpiricalDistribution(bins);
distribution.load(((List<?>)sourceValues).stream().mapToDouble(value -> ((Number)value).doubleValue()).toArray());;
- List<Map<String,Number>> histogramBins = new ArrayList<>();
+ List<Tuple> histogramBins = new ArrayList<>();
for(SummaryStatistics binSummary : distribution.getBinStats()) {
Map<String,Number> map = new HashMap<>();
map.put("max", binSummary.getMax());
@@ -78,7 +79,9 @@ public class HistogramEvaluator extends RecursiveNumericEvaluator implements Man
map.put("sum", binSummary.getSum());
map.put("N", binSummary.getN());
map.put("var", binSummary.getVariance());
- histogramBins.add(map);
+ map.put("cumProb", distribution.cumulativeProbability(binSummary.getMean()));
+ map.put("prob", distribution.probability(binSummary.getMin(), binSummary.getMax()));
+ histogramBins.add(new Tuple(map));
}
return histogramBins;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/44966121/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/PoissonDistributionEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/PoissonDistributionEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/PoissonDistributionEvaluator.java
new file mode 100644
index 0000000..864a039
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/PoissonDistributionEvaluator.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.client.solrj.io.eval;
+
+import java.io.IOException;
+import java.util.Locale;
+
+import org.apache.commons.math3.distribution.PoissonDistribution;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+
+public class PoissonDistributionEvaluator extends RecursiveNumericEvaluator implements OneValueWorker {
+
+ private static final long serialVersionUID = 1;
+
+ public PoissonDistributionEvaluator(StreamExpression expression, StreamFactory factory) throws IOException {
+ super(expression, factory);
+ }
+
+ @Override
+ public Object doWork(Object first) throws IOException{
+ if(null == first){
+ throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - null found for the first value",toExpression(constructingFactory)));
+ }
+
+ Number mean = (Number)first;
+
+ return new PoissonDistribution(mean.intValue());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/44966121/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/ProbabilityEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/ProbabilityEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/ProbabilityEvaluator.java
new file mode 100644
index 0000000..f0c25cb
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/ProbabilityEvaluator.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.client.solrj.io.eval;
+
+import java.io.IOException;
+import java.util.Locale;
+
+import org.apache.commons.math3.distribution.IntegerDistribution;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+
+public class ProbabilityEvaluator extends RecursiveObjectEvaluator implements TwoValueWorker {
+ protected static final long serialVersionUID = 1L;
+
+ public ProbabilityEvaluator(StreamExpression expression, StreamFactory factory) throws IOException{
+ super(expression, factory);
+ }
+
+ @Override
+ public Object doWork(Object first, Object second) throws IOException{
+ if(null == first){
+ throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - null found for the first value",toExpression(constructingFactory)));
+ }
+ if(null == second){
+ throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - null found for the second value",toExpression(constructingFactory)));
+ }
+ if(!(first instanceof IntegerDistribution)){
+ throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - found type %s for the first value, expecting a IntegerDistribution",toExpression(constructingFactory), first.getClass().getSimpleName()));
+ }
+ if(!(second instanceof Number)){
+ throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - found type %s for the second value, expecting a Number",toExpression(constructingFactory), first.getClass().getSimpleName()));
+ }
+
+ IntegerDistribution d = (IntegerDistribution) first;
+ Number predictOver = (Number) second;
+ return d.probability(predictOver.intValue());
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/44966121/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/SampleEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/SampleEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/SampleEvaluator.java
index 5732f85..be98305 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/SampleEvaluator.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/SampleEvaluator.java
@@ -22,6 +22,7 @@ import java.util.Arrays;
import java.util.Locale;
import java.util.stream.Collectors;
+import org.apache.commons.math3.distribution.IntegerDistribution;
import org.apache.commons.math3.distribution.RealDistribution;
import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
@@ -42,16 +43,19 @@ public class SampleEvaluator extends RecursiveObjectEvaluator implements TwoValu
if(null == second){
throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - null found for the second value",toExpression(constructingFactory)));
}
- if(!(first instanceof RealDistribution)){
- throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - found type %s for the first value, expecting a RealDistribution",toExpression(constructingFactory), first.getClass().getSimpleName()));
+ if(!(first instanceof RealDistribution) && !(first instanceof IntegerDistribution)){
+ throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - found type %s for the first value, expecting a Real or Integer Distribution",toExpression(constructingFactory), first.getClass().getSimpleName()));
}
if(!(second instanceof Number)){
throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - found type %s for the second value, expecting a Number",toExpression(constructingFactory), first.getClass().getSimpleName()));
}
-
- RealDistribution realDistribution = (RealDistribution)first;
-
- return Arrays.stream(realDistribution.sample(((Number)second).intValue())).mapToObj(item -> item).collect(Collectors.toList());
- }
+ if(first instanceof RealDistribution) {
+ RealDistribution realDistribution = (RealDistribution) first;
+ return Arrays.stream(realDistribution.sample(((Number) second).intValue())).mapToObj(item -> item).collect(Collectors.toList());
+ } else {
+ IntegerDistribution integerDistribution = (IntegerDistribution) first;
+ return Arrays.stream(integerDistribution.sample(((Number) second).intValue())).mapToObj(item -> item).collect(Collectors.toList());
+ }
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/44966121/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/UniformIntegerDistributionEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/UniformIntegerDistributionEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/UniformIntegerDistributionEvaluator.java
new file mode 100644
index 0000000..adf1aff
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/UniformIntegerDistributionEvaluator.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.client.solrj.io.eval;
+
+import java.io.IOException;
+import java.util.Locale;
+
+import org.apache.commons.math3.distribution.UniformIntegerDistribution;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+
+public class UniformIntegerDistributionEvaluator extends RecursiveNumericEvaluator implements TwoValueWorker {
+
+ private static final long serialVersionUID = 1;
+
+ public UniformIntegerDistributionEvaluator(StreamExpression expression, StreamFactory factory) throws IOException {
+ super(expression, factory);
+ }
+
+ @Override
+ public Object doWork(Object first, Object second) throws IOException{
+ if(null == first){
+ throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - null found for the first value",toExpression(constructingFactory)));
+ }
+ if(null == second){
+ throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - null found for the second value",toExpression(constructingFactory)));
+ }
+
+ Number lower = (Number)first;
+ Number upper = (Number)second;
+
+ return new UniformIntegerDistribution(lower.intValue(), upper.intValue());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/44966121/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
index f831dac..fafac6f 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
@@ -6155,6 +6155,48 @@ public class StreamExpressionTest extends SolrCloudTestCase {
assertTrue(out.get(5).intValue() == 2);
}
+
+ @Test
+ public void testFreqTable() throws Exception {
+ String cexpr = "freqTable(array(2,4,6,8,10,12,12,4,8,8,8,2))";
+ ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", cexpr);
+ paramsLoc.set("qt", "/stream");
+ String url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString()+"/"+COLLECTIONORALIAS;
+ TupleStream solrStream = new SolrStream(url, paramsLoc);
+ StreamContext context = new StreamContext();
+ solrStream.setStreamContext(context);
+ List<Tuple> tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+ List<Map<String,Number>> out = (List<Map<String, Number>>)tuples.get(0).get("return-value");
+ assertTrue(out.size() == 6);
+ Map<String, Number> bucket = out.get(0);
+ assertEquals(bucket.get("value").longValue(), 2);
+ assertEquals(bucket.get("count").longValue(), 2);
+
+ bucket = out.get(1);
+ assertEquals(bucket.get("value").longValue(), 4);
+ assertEquals(bucket.get("count").longValue(), 2);
+
+ bucket = out.get(2);
+ assertEquals(bucket.get("value").longValue(), 6);
+ assertEquals(bucket.get("count").longValue(), 1);
+
+ bucket = out.get(3);
+ assertEquals(bucket.get("value").longValue(), 8);
+ assertEquals(bucket.get("count").longValue(), 4);
+
+ bucket = out.get(4);
+ assertEquals(bucket.get("value").longValue(), 10);
+ assertEquals(bucket.get("count").longValue(), 1);
+
+ bucket = out.get(5);
+ assertEquals(bucket.get("value").longValue(), 12);
+ assertEquals(bucket.get("count").longValue(), 2);
+ }
+
+
+
@Test
public void testCosineSimilarity() throws Exception {
String cexpr = "cosineSimilarity(array(2,4,6,8),array(1,1,3,4))";
@@ -6171,7 +6213,109 @@ public class StreamExpressionTest extends SolrCloudTestCase {
assertTrue(cs.doubleValue() == 0.9838197164968291);
}
+ @Test
+ public void testPoissonDistribution() throws Exception {
+ String cexpr = "let(a=poissonDistribution(100)," +
+ " b=sample(a, 10000)," +
+ " tuple(d=describe(b), " +
+ " p=probability(a, 100), " +
+ " c=cumulativeProbability(a, 100)))";
+
+ ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", cexpr);
+ paramsLoc.set("qt", "/stream");
+ String url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString()+"/"+COLLECTIONORALIAS;
+ TupleStream solrStream = new SolrStream(url, paramsLoc);
+ StreamContext context = new StreamContext();
+ solrStream.setStreamContext(context);
+ List<Tuple> tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+ Map map = (Map)tuples.get(0).get("d");
+ Number mean = (Number)map.get("mean");
+ Number var = (Number)map.get("var");
+ //The mean and variance should be almost the same for poisson distribution
+ assertEquals(mean.doubleValue(), var.doubleValue(), 3.0);
+ Number prob = (Number)tuples.get(0).get("p");
+ assertEquals(prob.doubleValue(), 0.03986099680914713, 0.0);
+ Number cprob = (Number)tuples.get(0).get("c");
+ assertEquals(cprob.doubleValue(), 0.5265621985303708, 0.0);
+ }
+
+ @Test
+ public void testBinomialDistribution() throws Exception {
+ String cexpr = "let(a=binomialDistribution(100, .50)," +
+ " b=sample(a, 10000)," +
+ " tuple(d=describe(b), " +
+ " p=probability(a, 50), " +
+ " c=cumulativeProbability(a, 50)))";
+ ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", cexpr);
+ paramsLoc.set("qt", "/stream");
+ String url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString()+"/"+COLLECTIONORALIAS;
+ TupleStream solrStream = new SolrStream(url, paramsLoc);
+ StreamContext context = new StreamContext();
+ solrStream.setStreamContext(context);
+ List<Tuple> tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+ Number prob = (Number)tuples.get(0).get("p");
+ assertEquals(prob.doubleValue(),0.07958923738717877, 0.0);
+ Number cprob = (Number)tuples.get(0).get("c");
+ assertEquals(cprob.doubleValue(), 0.5397946186935851, 0.0);
+ }
+
+ @Test
+ public void testUniformIntegerDistribution() throws Exception {
+ String cexpr = "let(a=uniformIntegerDistribution(1, 10)," +
+ " b=sample(a, 10000)," +
+ " tuple(d=describe(b), " +
+ " p=probability(a, 5), " +
+ " c=cumulativeProbability(a, 5)))";
+
+ ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", cexpr);
+ paramsLoc.set("qt", "/stream");
+ String url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString()+"/"+COLLECTIONORALIAS;
+ TupleStream solrStream = new SolrStream(url, paramsLoc);
+ StreamContext context = new StreamContext();
+ solrStream.setStreamContext(context);
+ List<Tuple> tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+ Map map = (Map)tuples.get(0).get("d");
+ Number N = (Number)map.get("N");
+ assertEquals(N.intValue(), 10000);
+ Number prob = (Number)tuples.get(0).get("p");
+ assertEquals(prob.doubleValue(), 0.1, 0.0);
+ Number cprob = (Number)tuples.get(0).get("c");
+ assertEquals(cprob.doubleValue(), 0.5, 0.0);
+ }
+
+ @Test
+ public void testEnumeratedDistribution() throws Exception {
+ String cexpr = "let(a=uniformIntegerDistribution(1, 10)," +
+ " b=sample(a, 10000)," +
+ " c=enumeratedDistribution(b),"+
+ " tuple(d=describe(b), " +
+ " p=probability(c, 5), " +
+ " c=cumulativeProbability(c, 5)))";
+
+ ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", cexpr);
+ paramsLoc.set("qt", "/stream");
+ String url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString()+"/"+COLLECTIONORALIAS;
+ TupleStream solrStream = new SolrStream(url, paramsLoc);
+ StreamContext context = new StreamContext();
+ solrStream.setStreamContext(context);
+ List<Tuple> tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+ Map map = (Map)tuples.get(0).get("d");
+ Number N = (Number)map.get("N");
+ assertEquals(N.intValue(), 10000);
+ Number prob = (Number)tuples.get(0).get("p");
+ assertEquals(prob.doubleValue(), 0.1, 0.07);
+ Number cprob = (Number)tuples.get(0).get("c");
+ assertEquals(cprob.doubleValue(), 0.5, 0.07);
+ }
@Test
public void testDotProduct() throws Exception {
[45/50] lucene-solr:feature/autoscaling: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/lucene-solr
Posted by sh...@apache.org.
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/lucene-solr
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/277bc3a9
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/277bc3a9
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/277bc3a9
Branch: refs/heads/feature/autoscaling
Commit: 277bc3a94be284d41c001291dc0ff5d03dae11e1
Parents: 12b0acd d45ff7b
Author: Karl Wright <Da...@gmail.com>
Authored: Thu Sep 14 03:52:24 2017 -0400
Committer: Karl Wright <Da...@gmail.com>
Committed: Thu Sep 14 03:52:24 2017 -0400
----------------------------------------------------------------------
solr/CHANGES.txt | 2 +
solr/example/example-DIH/.gitignore | 1 +
solr/example/example-DIH/hsqldb/.gitignore | 5 +
solr/example/example-DIH/hsqldb/ex.backup | Bin 11354 -> 0 bytes
solr/example/example-DIH/hsqldb/ex.data | Bin 1048576 -> 0 bytes
solr/example/example-DIH/hsqldb/ex.script | 173 ++++++++++++++++++++++--
6 files changed, 171 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
[05/50] lucene-solr:feature/autoscaling: SOLR-10792: remove
OldLuceneQParser (FINALLY deprecated in 7.0) from master
Posted by sh...@apache.org.
SOLR-10792: remove OldLuceneQParser (FINALLY deprecated in 7.0) from master
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/ba9504d4
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/ba9504d4
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/ba9504d4
Branch: refs/heads/feature/autoscaling
Commit: ba9504d49dea4465afec75cb0c5e784117d4bf8a
Parents: 5e83475
Author: Chris Hostetter <ho...@apache.org>
Authored: Wed Sep 6 16:24:11 2017 -0700
Committer: Chris Hostetter <ho...@apache.org>
Committed: Wed Sep 6 16:24:11 2017 -0700
----------------------------------------------------------------------
.../apache/solr/search/LuceneQParserPlugin.java | 54 ---------------
.../solr/search/OldLuceneQParserPlugin.java | 37 -----------
.../solr/collection1/conf/solrconfig.xml | 3 -
.../org/apache/solr/ConvertedLegacyTest.java | 70 ++++++++++----------
.../apache/solr/handler/SearchHandlerTest.java | 4 +-
.../apache/solr/search/QueryEqualityTest.java | 8 ---
.../apache/solr/search/QueryParsingTest.java | 1 -
7 files changed, 36 insertions(+), 141 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ba9504d4/solr/core/src/java/org/apache/solr/search/LuceneQParserPlugin.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/LuceneQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/LuceneQParserPlugin.java
index 912b57f..91c824a 100644
--- a/solr/core/src/java/org/apache/solr/search/LuceneQParserPlugin.java
+++ b/solr/core/src/java/org/apache/solr/search/LuceneQParserPlugin.java
@@ -16,13 +16,9 @@
*/
package org.apache.solr.search;
-import org.apache.lucene.search.Query;
-import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.SolrParams;
-import org.apache.solr.common.util.StrUtils;
import org.apache.solr.request.SolrQueryRequest;
-import java.util.List;
/**
* Parse Solr's variant on the Lucene QueryParser syntax.
* <br>Other parameters:<ul>
@@ -42,53 +38,3 @@ public class LuceneQParserPlugin extends QParserPlugin {
}
}
-@Deprecated
-class OldLuceneQParser extends LuceneQParser {
- String sortStr;
-
- public OldLuceneQParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) {
- super(qstr, localParams, params, req);
- }
-
- @Override
- public Query parse() throws SyntaxError {
- // handle legacy "query;sort" syntax
- if (getLocalParams() == null) {
- String qstr = getString();
- if (qstr == null || qstr.length() == 0)
- return null;
- sortStr = getParams().get(CommonParams.SORT);
- if (sortStr == null) {
- // sort may be legacy form, included in the query string
- List<String> commands = StrUtils.splitSmart(qstr,';');
- if (commands.size() == 2) {
- qstr = commands.get(0);
- sortStr = commands.get(1);
- } else if (commands.size() == 1) {
- // This is need to support the case where someone sends: "q=query;"
- qstr = commands.get(0);
- }
- else if (commands.size() > 2) {
- throw new SyntaxError("If you want to use multiple ';' in the query, use the 'sort' param.");
- }
- }
- setString(qstr);
- }
-
- return super.parse();
- }
-
- @Override
- public SortSpec getSortSpec(boolean useGlobal) throws SyntaxError {
- SortSpec sort = super.getSortSpec(useGlobal);
- if (sortStr != null && sortStr.length()>0 && sort.getSort()==null) {
- SortSpec oldSort = SortSpecParsing.parseSortSpec(sortStr, getReq());
- if( oldSort.getSort() != null ) {
- sort.setSortAndFields(oldSort.getSort(), oldSort.getSchemaFields());
- }
- }
- return sort;
- }
-
-}
-
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ba9504d4/solr/core/src/java/org/apache/solr/search/OldLuceneQParserPlugin.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/OldLuceneQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/OldLuceneQParserPlugin.java
deleted file mode 100644
index d4cbfcb..0000000
--- a/solr/core/src/java/org/apache/solr/search/OldLuceneQParserPlugin.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.search;
-
-import org.apache.solr.common.params.SolrParams;
-import org.apache.solr.request.SolrQueryRequest;
-
-/**
- * Parse Solr's variant of Lucene QueryParser syntax, including the
- * deprecated sort specification after the query.
- * <br>Example: <code>{!lucenePlusSort}myfield:foo +bar -baz;price asc</code>
- *
- * @deprecated This class should have been removed a long time ago, it will be removed in Solr 8.0
- */
-@Deprecated
-public class OldLuceneQParserPlugin extends QParserPlugin {
- public static final String NAME = "lucenePlusSort";
-
- @Override
- public QParser createParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) {
- return new OldLuceneQParser(qstr, localParams, params, req);
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ba9504d4/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml
index 9422a4d..6528aba 100644
--- a/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml
@@ -489,9 +489,6 @@
<queryParser name="foo" class="FooQParserPlugin"/>
- <!-- deprecated parser, delete once class is deleted in Solr 8.0 -->
- <queryParser name="lucenePlusSort" class="solr.OldLuceneQParserPlugin"/>
-
<updateRequestProcessorChain name="dedupe">
<processor class="org.apache.solr.update.processor.SignatureUpdateProcessorFactory">
<bool name="enabled">false</bool>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ba9504d4/solr/core/src/test/org/apache/solr/ConvertedLegacyTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/ConvertedLegacyTest.java b/solr/core/src/test/org/apache/solr/ConvertedLegacyTest.java
index 35bd107..603dd03 100644
--- a/solr/core/src/test/org/apache/solr/ConvertedLegacyTest.java
+++ b/solr/core/src/test/org/apache/solr/ConvertedLegacyTest.java
@@ -43,7 +43,6 @@ public class ConvertedLegacyTest extends SolrTestCaseJ4 {
SolrQueryRequest req = null;
Map<String,String> args = new HashMap<>();
lrf.args.put(CommonParams.VERSION,"2.2");
- lrf.args.put("defType","lucenePlusSort");
// compact the index, keep things from getting out of hand
@@ -215,16 +214,16 @@ public class ConvertedLegacyTest extends SolrTestCaseJ4 {
,"*[count(//doc)=0]"
);
args = new HashMap<>();
- args.put("defType","lucenePlusSort");
- req = new LocalSolrQueryRequest(h.getCore(), "val_s:[a TO z];val_s1 asc",
+ args.put("sort","val_s1 asc");
+ req = new LocalSolrQueryRequest(h.getCore(), "val_s:[a TO z]",
"/select", 0, 0 , args);
assertQ(req
,"//*[@numFound='3'] "
,"*[count(//doc)=0]"
);
args = new HashMap<>();
- args.put("defType","lucenePlusSort");
- req = new LocalSolrQueryRequest(h.getCore(), "val_s:[a TO z];val_s1 desc",
+ args.put("sort","val_s1 desc");
+ req = new LocalSolrQueryRequest(h.getCore(), "val_s:[a TO z]",
"/select", 0, 0 , args);
assertQ(req
,"//*[@numFound='3'] "
@@ -518,11 +517,11 @@ public class ConvertedLegacyTest extends SolrTestCaseJ4 {
,"//@numFound[.='1'] "
,"//int[.='-2147483648']"
);
- assertQ(req("id:44;num_i1 asc;")
+ assertQ(req("q", "id:44", "sort","num_i1 asc")
,"//doc[1]/int[.='-2147483648'] "
,"//doc[last()]/int[.='2147483647']"
);
- assertQ(req("id:44;num_i1 desc;")
+ assertQ(req("q","id:44","sort","num_i1 desc")
,"//doc[1]/int[.='2147483647'] "
,"//doc[last()]/int[.='-2147483648']"
);
@@ -561,11 +560,11 @@ public class ConvertedLegacyTest extends SolrTestCaseJ4 {
,"//@numFound[.='1'] "
,"//long[.='-9223372036854775808']"
);
- assertQ(req("id:44;num_l1 asc;")
+ assertQ(req("q","id:44","sort","num_l1 asc")
,"//doc[1]/long[.='-9223372036854775808'] "
,"//doc[last()]/long[.='9223372036854775807']"
);
- assertQ(req("id:44;num_l1 desc;")
+ assertQ(req("q","id:44", "sort", "num_l1 desc")
,"//doc[1]/long[.='9223372036854775807'] "
,"//doc[last()]/long[.='-9223372036854775808']"
);
@@ -611,11 +610,11 @@ public class ConvertedLegacyTest extends SolrTestCaseJ4 {
assertQ(req("num_f1:\"-1e20\"")
,"//@numFound[.='1']"
);
- assertQ(req("id:44;num_f1 asc;")
+ assertQ(req("q", "id:44", "sort", "num_f1 asc")
,"//doc[1]/float[.='-Infinity'] "
,"//doc[last()]/float[.='NaN']"
);
- assertQ(req("id:44;num_f1 desc;")
+ assertQ(req("q", "id:44", "sort","num_f1 desc")
,"//doc[1]/float[.='NaN'] "
,"//doc[last()]/float[.='-Infinity']"
);
@@ -663,11 +662,11 @@ public class ConvertedLegacyTest extends SolrTestCaseJ4 {
assertQ(req("num_d1:\"1e-100\"")
,"//@numFound[.='1']"
);
- assertQ(req("id:44;num_d1 asc;")
+ assertQ(req("q", "id:44", "sort", "num_d1 asc")
,"//doc[1]/double[.='-Infinity'] "
,"//doc[last()]/double[.='NaN']"
);
- assertQ(req("id:44;num_d1 desc;")
+ assertQ(req("q","id:44","sort","num_d1 desc")
,"//doc[1]/double[.='NaN'] "
,"//doc[last()]/double[.='-Infinity']"
);
@@ -693,27 +692,27 @@ public class ConvertedLegacyTest extends SolrTestCaseJ4 {
,"*[count(//doc)=6]"
);
- assertQ(req("id:44; a_i1 asc,b_i1 desc")
+ assertQ(req("q","id:44", "sort", "a_i1 asc,b_i1 desc")
,"*[count(//doc)=6] "
,"//doc[3]/int[.='100'] "
,"//doc[4]/int[.='50']"
);
- assertQ(req("id:44;a_i1 asc , b_i1 asc;")
+ assertQ(req("q","id:44", "sort", "a_i1 asc , b_i1 asc")
,"*[count(//doc)=6] "
,"//doc[3]/int[.='50'] "
,"//doc[4]/int[.='100']"
);
- assertQ(req("id:44;a_i1 asc;")
+ assertQ(req("q", "id:44", "sort", "a_i1 asc")
,"*[count(//doc)=6] "
,"//doc[1]/int[.='-1'] "
,"//doc[last()]/int[.='15']"
);
- assertQ(req("id:44;a_i1 asc , score top;")
+ assertQ(req("q","id:44","sort","a_i1 asc , score top")
,"*[count(//doc)=6] "
,"//doc[1]/int[.='-1'] "
,"//doc[last()]/int[.='15']"
);
- assertQ(req("id:44; score top , a_i1 top, b_i1 bottom ;")
+ assertQ(req("q","id:44","sort","score top , a_i1 top, b_i1 bottom ")
,"*[count(//doc)=6] "
,"//doc[last()]/int[.='-1'] "
,"//doc[1]/int[.='15'] "
@@ -736,36 +735,36 @@ public class ConvertedLegacyTest extends SolrTestCaseJ4 {
assertQ(req("id_i:[1000 TO 1010]")
,"*[count(//doc)=7]"
);
- assertQ(req("id_i:[1000 TO 1010]; b_i1 asc")
+ assertQ(req("q","id_i:[1000 TO 1010]","sort","b_i1 asc")
,"*[count(//doc)=7] "
,"//doc[1]/int[.='50'] "
,"//doc[2]/int[.='100']"
);
- assertQ(req("id_i:[1000 TO 1010]; b_i1 desc")
+ assertQ(req("q","id_i:[1000 TO 1010]","sort"," b_i1 desc")
,"*[count(//doc)=7] "
,"//doc[1]/int[.='100'] "
,"//doc[2]/int[.='50']"
);
- assertQ(req("id_i:[1000 TO 1010]; a_i1 asc,b_i1 desc")
+ assertQ(req("q","id_i:[1000 TO 1010]","sort"," a_i1 asc,b_i1 desc")
,"*[count(//doc)=7] "
,"//doc[3]/int[@name='b_i1' and .='100'] "
,"//doc[4]/int[@name='b_i1' and .='50'] "
,"//doc[5]/arr[@name='id_i' and .='1000']"
);
- assertQ(req("id_i:[1000 TO 1010]; a_i1 asc,b_i1 asc")
+ assertQ(req("q","id_i:[1000 TO 1010]","sort"," a_i1 asc,b_i1 asc")
,"*[count(//doc)=7] "
,"//doc[3]/int[@name='b_i1' and .='50'] "
,"//doc[4]/int[@name='b_i1' and .='100'] "
,"//doc[5]/arr[@name='id_i' and .='1000']"
);
// nullfirst tests
- assertQ(req("id_i:[1000 TO 1002]; nullfirst asc")
+ assertQ(req("q","id_i:[1000 TO 1002]","sort"," nullfirst asc")
,"*[count(//doc)=3] "
,"//doc[1]/arr[@name='id_i' and .='1002']"
,"//doc[2]/arr[@name='id_i' and .='1001'] "
,"//doc[3]/arr[@name='id_i' and .='1000']"
);
- assertQ(req("id_i:[1000 TO 1002]; nullfirst desc")
+ assertQ(req("q","id_i:[1000 TO 1002]","sort"," nullfirst desc")
,"*[count(//doc)=3] "
,"//doc[1]/arr[@name='id_i' and .='1002']"
,"//doc[2]/arr[@name='id_i' and .='1000'] "
@@ -779,16 +778,16 @@ public class ConvertedLegacyTest extends SolrTestCaseJ4 {
// Sort parsing exception tests. (SOLR-6, SOLR-99)
assertQEx( "can not sort unindexed fields",
- req( "id_i:1000; shouldbeunindexed asc" ), 400 );
+ req( "q","id_i:1000", "sort", "shouldbeunindexed asc" ), 400 );
assertQEx( "invalid query format",
- req( "id_i:1000; nullfirst" ), 400 );
+ req( "q","id_i:1000", "sort", "nullfirst" ), 400 );
assertQEx( "unknown sort field",
- req( "id_i:1000; abcde12345 asc" ), 400 );
+ req( "q","id_i:1000", "sort", "abcde12345 asc" ), 400 );
assertQEx( "unknown sort order",
- req( "id_i:1000; nullfirst aaa" ), 400 );
+ req( "q","id_i:1000", "sort", "nullfirst aaa" ), 400 );
resetExceptionIgnores();
@@ -1166,32 +1165,31 @@ public class ConvertedLegacyTest extends SolrTestCaseJ4 {
);
args = new HashMap<>();
args.put("fl","score ");
- args.put("defType","lucenePlusSort");
- req = new LocalSolrQueryRequest(h.getCore(), "id:44;id desc;",
+ args.put("sort","id desc");
+ req = new LocalSolrQueryRequest(h.getCore(), "id:44",
"/select", 0, 10, args);
assertQ(req
,"//result[@maxScore>0]"
);
args = new HashMap<>();
args.put("fl","score ");
- args.put("defType","lucenePlusSort");
- req = new LocalSolrQueryRequest(h.getCore(), "id:44;",
+ req = new LocalSolrQueryRequest(h.getCore(), "id:44",
"/select", 0, 10, args);
assertQ(req
,"//@maxScore = //doc/float[@name='score']"
);
args = new HashMap<>();
args.put("fl","score ");
- args.put("defType","lucenePlusSort");
- req = new LocalSolrQueryRequest(h.getCore(), "id:44;id desc;",
+ args.put("sort","id desc");
+ req = new LocalSolrQueryRequest(h.getCore(), "id:44",
"/select", 0, 10, args);
assertQ(req
,"//@maxScore = //doc/float[@name='score']"
);
args = new HashMap<>();
args.put("fl","*,score");
- args.put("defType","lucenePlusSort");
- req = new LocalSolrQueryRequest(h.getCore(), "id:44;id desc;",
+ args.put("sort","id desc");
+ req = new LocalSolrQueryRequest(h.getCore(), "id:44",
"/select", 0, 0 , args);
assertQ(req
,"//result[@maxScore>0]"
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ba9504d4/solr/core/src/test/org/apache/solr/handler/SearchHandlerTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/SearchHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/SearchHandlerTest.java
index b35103b..4fa5f10 100644
--- a/solr/core/src/test/org/apache/solr/handler/SearchHandlerTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/SearchHandlerTest.java
@@ -63,14 +63,14 @@ public class SearchHandlerTest extends AbstractSolrTestCase {
);
// Using legacy ';' param
- assertQ(req("q", "title:test; val_s1 desc", "defType","lucenePlusSort")
+ assertQ(req("q", "title:test", "sort","val_s1 desc")
,"//*[@numFound='3']"
,"//result/doc[1]/str[@name='id'][.='12']"
,"//result/doc[2]/str[@name='id'][.='11']"
,"//result/doc[3]/str[@name='id'][.='10']"
);
- assertQ(req("q", "title:test; val_s1 asc", "defType","lucenePlusSort")
+ assertQ(req("q", "title:test", "sort", "val_s1 asc")
,"//*[@numFound='3']"
,"//result/doc[1]/str[@name='id'][.='10']"
,"//result/doc[2]/str[@name='id'][.='11']"
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ba9504d4/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java b/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java
index eab254f..4177d81 100644
--- a/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java
+++ b/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java
@@ -94,14 +94,6 @@ public class QueryEqualityTest extends SolrTestCaseJ4 {
" +apache +solr");
}
- @Deprecated
- public void testQueryLucenePlusSort() throws Exception {
- assertQueryEquals("lucenePlusSort",
- "apache solr", "apache solr", "apache solr ; score desc");
- assertQueryEquals("lucenePlusSort",
- "+apache +solr", "apache AND solr", " +apache +solr; score desc");
- }
-
public void testQueryPrefix() throws Exception {
SolrQueryRequest req = req("myField","foo_s");
try {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ba9504d4/solr/core/src/test/org/apache/solr/search/QueryParsingTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/search/QueryParsingTest.java b/solr/core/src/test/org/apache/solr/search/QueryParsingTest.java
index 925f71b..20ea941 100644
--- a/solr/core/src/test/org/apache/solr/search/QueryParsingTest.java
+++ b/solr/core/src/test/org/apache/solr/search/QueryParsingTest.java
@@ -43,7 +43,6 @@ public class QueryParsingTest extends SolrTestCaseJ4 {
SolrQueryRequest req = req("df", "text");
final String[] parsersTested = new String[] {
- OldLuceneQParserPlugin.NAME,
LuceneQParserPlugin.NAME,
DisMaxQParserPlugin.NAME,
ExtendedDismaxQParserPlugin.NAME
[18/50] lucene-solr:feature/autoscaling: LUCENE-7963: Remove useless
getAttribute() in DefaultIndexingChain that causes performance drop,
introduced by LUCENE-7626
Posted by sh...@apache.org.
LUCENE-7963: Remove useless getAttribute() in DefaultIndexingChain that causes performance drop, introduced by LUCENE-7626
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/a5402f68
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/a5402f68
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/a5402f68
Branch: refs/heads/feature/autoscaling
Commit: a5402f68631768bae57d923613211128de077982
Parents: 5fee937
Author: Uwe Schindler <us...@apache.org>
Authored: Fri Sep 8 17:39:12 2017 +0200
Committer: Uwe Schindler <us...@apache.org>
Committed: Fri Sep 8 17:39:12 2017 +0200
----------------------------------------------------------------------
lucene/CHANGES.txt | 4 ++++
.../src/java/org/apache/lucene/index/DefaultIndexingChain.java | 2 --
2 files changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a5402f68/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 0dced28..a2fd223 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -200,6 +200,10 @@ Bug Fixes
* LUCENE-7956: Fixed potential stack overflow error in ICUNormalizer2CharFilter.
(Adrien Grand)
+* LUCENE-7963: Remove useless getAttribute() in DefaultIndexingChain that
+ causes performance drop, introduced by LUCENE-7626. (Daniel Mitterdorfer
+ via Uwe Schindler)
+
Improvements
* LUCENE-7489: Better storage of sparse doc-values fields with the default
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a5402f68/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java b/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java
index f2c3de1..fd24105 100644
--- a/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java
+++ b/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java
@@ -27,7 +27,6 @@ import java.util.Map;
import java.util.Set;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.codecs.DocValuesConsumer;
import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.codecs.NormsConsumer;
@@ -733,7 +732,6 @@ final class DefaultIndexingChain extends DocConsumer {
stream.reset();
invertState.setAttributeSource(stream);
termsHashPerField.start(field, first);
- CharTermAttribute termAtt = tokenStream.getAttribute(CharTermAttribute.class);
while (stream.incrementToken()) {
[47/50] lucene-solr:feature/autoscaling: LUCENE-7962: Revamp how
computeNearestDistance is done to make it return the distance where the point
is truly nearest to the given path.
Posted by sh...@apache.org.
LUCENE-7962: Revamp how computeNearestDistance is done to make it return the distance where the point is truly nearest to the given path.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/8cb64ed9
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/8cb64ed9
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/8cb64ed9
Branch: refs/heads/feature/autoscaling
Commit: 8cb64ed926de59776f87e8151a72c3ce6f09caae
Parents: c317ad8
Author: Karl Wright <Da...@gmail.com>
Authored: Thu Sep 14 13:50:41 2017 -0400
Committer: Karl Wright <Da...@gmail.com>
Committed: Thu Sep 14 13:50:41 2017 -0400
----------------------------------------------------------------------
.../spatial3d/geom/GeoDegeneratePath.java | 113 ++++++++++++++++---
.../apache/lucene/spatial3d/geom/GeoPath.java | 28 +++++
.../lucene/spatial3d/geom/GeoStandardPath.java | 113 ++++++++++++++++---
3 files changed, 222 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8cb64ed9/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoDegeneratePath.java
----------------------------------------------------------------------
diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoDegeneratePath.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoDegeneratePath.java
index 861a2d4..6f2f81e 100644
--- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoDegeneratePath.java
+++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoDegeneratePath.java
@@ -152,31 +152,52 @@ class GeoDegeneratePath extends GeoBasePath {
}
@Override
- public double computeNearestDistance(final DistanceStyle distanceStyle, final double x, final double y, final double z) {
- // Algorithm:
- // (1) If the point is within any of the segments along the path, return that value.
- // (2) If the point is within any of the segment end circles along the path, return that value.
- double currentDistance = 0.0;
+ public double computePathCenterDistance(final DistanceStyle distanceStyle, final double x, final double y, final double z) {
+ // Walk along path and keep track of the closest distance we find
+ double closestDistance = Double.POSITIVE_INFINITY;
+ // Segments first
for (PathSegment segment : segments) {
- double distance = segment.nearestPathDistance(planetModel, distanceStyle, x,y,z);
- if (distance != Double.POSITIVE_INFINITY)
- return distanceStyle.fromAggregationForm(distanceStyle.aggregateDistances(currentDistance, distance));
- currentDistance = distanceStyle.aggregateDistances(currentDistance, segment.fullPathDistance(distanceStyle));
+ final double segmentDistance = segment.pathCenterDistance(planetModel, distanceStyle, x, y, z);
+ if (segmentDistance < closestDistance) {
+ closestDistance = segmentDistance;
+ }
}
+ // Now, endpoints
+ for (SegmentEndpoint endpoint : endPoints) {
+ final double endpointDistance = endpoint.pathCenterDistance(distanceStyle, x, y, z);
+ if (endpointDistance < closestDistance) {
+ closestDistance = endpointDistance;
+ }
+ }
+ return closestDistance;
+ }
+ @Override
+ public double computeNearestDistance(final DistanceStyle distanceStyle, final double x, final double y, final double z) {
+ double currentDistance = 0.0;
+ double minPathCenterDistance = Double.POSITIVE_INFINITY;
+ double bestDistance = Double.POSITIVE_INFINITY;
int segmentIndex = 0;
- currentDistance = 0.0;
+
for (SegmentEndpoint endpoint : endPoints) {
- double distance = endpoint.nearestPathDistance(distanceStyle, x, y, z);
- if (distance != Double.POSITIVE_INFINITY) {
- return distanceStyle.fromAggregationForm(distanceStyle.aggregateDistances(currentDistance, distance));
+ final double endpointPathCenterDistance = endpoint.pathCenterDistance(distanceStyle, x, y, z);
+ if (endpointPathCenterDistance < minPathCenterDistance) {
+ // Use this endpoint
+ minPathCenterDistance = endpointPathCenterDistance;
+ bestDistance = currentDistance;
}
+ // Look at the following segment, if any
if (segmentIndex < segments.size()) {
- currentDistance = distanceStyle.aggregateDistances(currentDistance, segments.get(segmentIndex++).fullPathDistance(distanceStyle));
+ final PathSegment segment = segments.get(segmentIndex++);
+ final double segmentPathCenterDistance = segment.pathCenterDistance(planetModel, distanceStyle, x, y, z);
+ if (segmentPathCenterDistance < minPathCenterDistance) {
+ minPathCenterDistance = segmentPathCenterDistance;
+ bestDistance = distanceStyle.aggregateDistances(currentDistance, segment.nearestPathDistance(planetModel, distanceStyle, x, y, z));
+ }
+ currentDistance = distanceStyle.aggregateDistances(currentDistance, segment.fullPathDistance(distanceStyle));
}
}
-
- return Double.POSITIVE_INFINITY;
+ return bestDistance;
}
@Override
@@ -428,6 +449,23 @@ class GeoDegeneratePath extends GeoBasePath {
return distanceStyle.toAggregationForm(0.0);
}
+ /** Compute path center distance.
+ *@param distanceStyle is the distance style.
+ *@param x is the point x.
+ *@param y is the point y.
+ *@param z is the point z.
+ *@return the distance metric, or POSITIVE_INFINITY
+ * if the point is not within the bounds of the endpoint.
+ */
+ public double pathCenterDistance(final DistanceStyle distanceStyle, final double x, final double y, final double z) {
+ for (final Membership m : cutoffPlanes) {
+ if (!m.isWithin(x,y,z)) {
+ return Double.POSITIVE_INFINITY;
+ }
+ }
+ return distanceStyle.computeDistance(this.point, x, y, z);
+ }
+
/** Compute external distance.
*@param distanceStyle is the distance style.
*@param x is the point x.
@@ -567,6 +605,49 @@ class GeoDegeneratePath extends GeoBasePath {
normalizedConnectingPlane.evaluateIsZero(x, y, z);
}
+ /** Compute path center distance.
+ *@param planetModel is the planet model.
+ *@param distanceStyle is the distance style.
+ *@param x is the point x.
+ *@param y is the point y.
+ *@param z is the point z.
+ *@return the distance metric, or Double.POSITIVE_INFINITY if outside this segment
+ */
+ public double pathCenterDistance(final PlanetModel planetModel, final DistanceStyle distanceStyle, final double x, final double y, final double z) {
+ // First, if this point is outside the endplanes of the segment, return POSITIVE_INFINITY.
+ if (!startCutoffPlane.isWithin(x, y, z) || !endCutoffPlane.isWithin(x, y, z)) {
+ return Double.POSITIVE_INFINITY;
+ }
+ // (1) Compute normalizedPerpPlane. If degenerate, then there is no such plane, which means that the point given
+ // is insufficient to distinguish between a family of such planes. This can happen only if the point is one of the
+ // "poles", imagining the normalized plane to be the "equator". In that case, the distance returned should be zero.
+ // Want no allocations or expensive operations! so we do this the hard way
+ final double perpX = normalizedConnectingPlane.y * z - normalizedConnectingPlane.z * y;
+ final double perpY = normalizedConnectingPlane.z * x - normalizedConnectingPlane.x * z;
+ final double perpZ = normalizedConnectingPlane.x * y - normalizedConnectingPlane.y * x;
+ final double magnitude = Math.sqrt(perpX * perpX + perpY * perpY + perpZ * perpZ);
+ if (Math.abs(magnitude) < Vector.MINIMUM_RESOLUTION)
+ return distanceStyle.computeDistance(start, x, y, z);
+ final double normFactor = 1.0/magnitude;
+ final Plane normalizedPerpPlane = new Plane(perpX * normFactor, perpY * normFactor, perpZ * normFactor, 0.0);
+
+ final GeoPoint[] intersectionPoints = normalizedConnectingPlane.findIntersections(planetModel, normalizedPerpPlane);
+ GeoPoint thePoint;
+ if (intersectionPoints.length == 0)
+ throw new RuntimeException("Can't find world intersection for point x="+x+" y="+y+" z="+z);
+ else if (intersectionPoints.length == 1)
+ thePoint = intersectionPoints[0];
+ else {
+ if (startCutoffPlane.isWithin(intersectionPoints[0]) && endCutoffPlane.isWithin(intersectionPoints[0]))
+ thePoint = intersectionPoints[0];
+ else if (startCutoffPlane.isWithin(intersectionPoints[1]) && endCutoffPlane.isWithin(intersectionPoints[1]))
+ thePoint = intersectionPoints[1];
+ else
+ throw new RuntimeException("Can't find world intersection for point x="+x+" y="+y+" z="+z);
+ }
+ return distanceStyle.computeDistance(thePoint, x, y, z);
+ }
+
/** Compute nearest path distance.
*@param planetModel is the planet model.
*@param distanceStyle is the distance style.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8cb64ed9/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPath.java
----------------------------------------------------------------------
diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPath.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPath.java
index 891c896..8fe5632 100644
--- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPath.java
+++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoPath.java
@@ -52,4 +52,32 @@ public interface GeoPath extends GeoDistanceShape {
*/
public double computeNearestDistance(final DistanceStyle distanceStyle, final double x, final double y, final double z);
+ // The following methods compute the best distance from the path center to the point.
+
+ /**
+ * Compute the shortest distance from the path center to the GeoPoint.
+ * The distance is meant to allow comparisons between different
+ * paths to find the one that goes closest to a point.
+ *
+ * @param distanceStyle is the distance style.
+ * @param point is the point to compute the distance to.
+ * @return the shortest distance from the path center to the point.
+ */
+ public default double computePathCenterDistance(final DistanceStyle distanceStyle, final GeoPoint point) {
+ return computePathCenterDistance(distanceStyle, point.x, point.y, point.z);
+ }
+
+ /**
+ * Compute the shortest distance from the path center to the GeoPoint.
+ * The distance is meant to allow comparisons between different
+ * paths to find the one that goes closest to a point.
+ *
+ * @param distanceStyle is the distance style.
+ * @param x is the point's unit x coordinate (using U.S. convention).
+ * @param y is the point's unit y coordinate (using U.S. convention).
+ * @param z is the point's unit z coordinate (using U.S. convention).
+ * @return the shortest distance from the path center to the point.
+ */
+ public double computePathCenterDistance(final DistanceStyle distanceStyle, final double x, final double y, final double z);
+
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8cb64ed9/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoStandardPath.java
----------------------------------------------------------------------
diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoStandardPath.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoStandardPath.java
index 841b981..195d596 100755
--- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoStandardPath.java
+++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoStandardPath.java
@@ -215,31 +215,52 @@ class GeoStandardPath extends GeoBasePath {
}
@Override
- public double computeNearestDistance(final DistanceStyle distanceStyle, final double x, final double y, final double z) {
- // Algorithm:
- // (1) If the point is within any of the segments along the path, return that value.
- // (2) If the point is within any of the segment end circles along the path, return that value.
- double currentDistance = 0.0;
+ public double computePathCenterDistance(final DistanceStyle distanceStyle, final double x, final double y, final double z) {
+ // Walk along path and keep track of the closest distance we find
+ double closestDistance = Double.POSITIVE_INFINITY;
+ // Segments first
for (PathSegment segment : segments) {
- double distance = segment.nearestPathDistance(planetModel, distanceStyle, x,y,z);
- if (distance != Double.POSITIVE_INFINITY)
- return distanceStyle.fromAggregationForm(distanceStyle.aggregateDistances(currentDistance, distance));
- currentDistance = distanceStyle.aggregateDistances(currentDistance, segment.fullPathDistance(distanceStyle));
+ final double segmentDistance = segment.pathCenterDistance(planetModel, distanceStyle, x, y, z);
+ if (segmentDistance < closestDistance) {
+ closestDistance = segmentDistance;
+ }
}
+ // Now, endpoints
+ for (SegmentEndpoint endpoint : endPoints) {
+ final double endpointDistance = endpoint.pathCenterDistance(distanceStyle, x, y, z);
+ if (endpointDistance < closestDistance) {
+ closestDistance = endpointDistance;
+ }
+ }
+ return closestDistance;
+ }
+ @Override
+ public double computeNearestDistance(final DistanceStyle distanceStyle, final double x, final double y, final double z) {
+ double currentDistance = 0.0;
+ double minPathCenterDistance = Double.POSITIVE_INFINITY;
+ double bestDistance = Double.POSITIVE_INFINITY;
int segmentIndex = 0;
- currentDistance = 0.0;
+
for (SegmentEndpoint endpoint : endPoints) {
- double distance = endpoint.nearestPathDistance(distanceStyle, x, y, z);
- if (distance != Double.POSITIVE_INFINITY) {
- return distanceStyle.fromAggregationForm(distanceStyle.aggregateDistances(currentDistance, distance));
+ final double endpointPathCenterDistance = endpoint.pathCenterDistance(distanceStyle, x, y, z);
+ if (endpointPathCenterDistance < minPathCenterDistance) {
+ // Use this endpoint
+ minPathCenterDistance = endpointPathCenterDistance;
+ bestDistance = currentDistance;
}
+ // Look at the following segment, if any
if (segmentIndex < segments.size()) {
- currentDistance = distanceStyle.aggregateDistances(currentDistance, segments.get(segmentIndex++).fullPathDistance(distanceStyle));
+ final PathSegment segment = segments.get(segmentIndex++);
+ final double segmentPathCenterDistance = segment.pathCenterDistance(planetModel, distanceStyle, x, y, z);
+ if (segmentPathCenterDistance < minPathCenterDistance) {
+ minPathCenterDistance = segmentPathCenterDistance;
+ bestDistance = distanceStyle.aggregateDistances(currentDistance, segment.nearestPathDistance(planetModel, distanceStyle, x, y, z));
+ }
+ currentDistance = distanceStyle.aggregateDistances(currentDistance, segment.fullPathDistance(distanceStyle));
}
}
-
- return Double.POSITIVE_INFINITY;
+ return bestDistance;
}
@Override
@@ -607,6 +628,23 @@ class GeoStandardPath extends GeoBasePath {
return distanceStyle.toAggregationForm(0.0);
}
+ /** Compute path center distance.
+ *@param distanceStyle is the distance style.
+ *@param x is the point x.
+ *@param y is the point y.
+ *@param z is the point z.
+ *@return the distance metric, or POSITIVE_INFINITY
+ * if the point is not within the bounds of the endpoint.
+ */
+ public double pathCenterDistance(final DistanceStyle distanceStyle, final double x, final double y, final double z) {
+ for (final Membership m : cutoffPlanes) {
+ if (!m.isWithin(x,y,z)) {
+ return Double.POSITIVE_INFINITY;
+ }
+ }
+ return distanceStyle.computeDistance(this.point, x, y, z);
+ }
+
/** Compute external distance.
*@param distanceStyle is the distance style.
*@param x is the point x.
@@ -811,6 +849,49 @@ class GeoStandardPath extends GeoBasePath {
lowerConnectingPlane.isWithin(x, y, z);
}
+ /** Compute path center distance.
+ *@param planetModel is the planet model.
+ *@param distanceStyle is the distance style.
+ *@param x is the point x.
+ *@param y is the point y.
+ *@param z is the point z.
+ *@return the distance metric, or Double.POSITIVE_INFINITY if outside this segment
+ */
+ public double pathCenterDistance(final PlanetModel planetModel, final DistanceStyle distanceStyle, final double x, final double y, final double z) {
+ // First, if this point is outside the endplanes of the segment, return POSITIVE_INFINITY.
+ if (!startCutoffPlane.isWithin(x, y, z) || !endCutoffPlane.isWithin(x, y, z)) {
+ return Double.POSITIVE_INFINITY;
+ }
+ // (1) Compute normalizedPerpPlane. If degenerate, then there is no such plane, which means that the point given
+ // is insufficient to distinguish between a family of such planes. This can happen only if the point is one of the
+ // "poles", imagining the normalized plane to be the "equator". In that case, the distance returned should be zero.
+ // Want no allocations or expensive operations! so we do this the hard way
+ final double perpX = normalizedConnectingPlane.y * z - normalizedConnectingPlane.z * y;
+ final double perpY = normalizedConnectingPlane.z * x - normalizedConnectingPlane.x * z;
+ final double perpZ = normalizedConnectingPlane.x * y - normalizedConnectingPlane.y * x;
+ final double magnitude = Math.sqrt(perpX * perpX + perpY * perpY + perpZ * perpZ);
+ if (Math.abs(magnitude) < Vector.MINIMUM_RESOLUTION)
+ return distanceStyle.computeDistance(start, x, y, z);
+ final double normFactor = 1.0/magnitude;
+ final Plane normalizedPerpPlane = new Plane(perpX * normFactor, perpY * normFactor, perpZ * normFactor, 0.0);
+
+ final GeoPoint[] intersectionPoints = normalizedConnectingPlane.findIntersections(planetModel, normalizedPerpPlane);
+ GeoPoint thePoint;
+ if (intersectionPoints.length == 0)
+ throw new RuntimeException("Can't find world intersection for point x="+x+" y="+y+" z="+z);
+ else if (intersectionPoints.length == 1)
+ thePoint = intersectionPoints[0];
+ else {
+ if (startCutoffPlane.isWithin(intersectionPoints[0]) && endCutoffPlane.isWithin(intersectionPoints[0]))
+ thePoint = intersectionPoints[0];
+ else if (startCutoffPlane.isWithin(intersectionPoints[1]) && endCutoffPlane.isWithin(intersectionPoints[1]))
+ thePoint = intersectionPoints[1];
+ else
+ throw new RuntimeException("Can't find world intersection for point x="+x+" y="+y+" z="+z);
+ }
+ return distanceStyle.computeDistance(thePoint, x, y, z);
+ }
+
/** Compute nearest path distance.
*@param planetModel is the planet model.
*@param distanceStyle is the distance style.
[26/50] lucene-solr:feature/autoscaling: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/lucene-solr
Posted by sh...@apache.org.
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/lucene-solr
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/8452edb1
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/8452edb1
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/8452edb1
Branch: refs/heads/feature/autoscaling
Commit: 8452edb1306a3c146e6d132ad97c5882f7d9d023
Parents: fe7582e c229dc3
Author: Karl Wright <Da...@gmail.com>
Authored: Sat Sep 9 01:02:51 2017 -0400
Committer: Karl Wright <Da...@gmail.com>
Committed: Sat Sep 9 01:02:51 2017 -0400
----------------------------------------------------------------------
.../org/apache/solr/handler/StreamHandler.java | 8 +++
.../io/eval/CanberraDistanceEvaluator.java | 56 +++++++++++++++++++
.../io/eval/ChebyshevDistanceEvaluator.java | 56 +++++++++++++++++++
.../io/eval/EarthMoversDistanceEvaluator.java | 56 +++++++++++++++++++
.../io/eval/KendallsCorrelationEvaluator.java | 57 ++++++++++++++++++++
.../io/eval/ManhattanDistanceEvaluator.java | 56 +++++++++++++++++++
.../solrj/io/eval/MeanDifferenceEvaluator.java | 55 +++++++++++++++++++
.../io/eval/SpearmansCorrelationEvaluator.java | 57 ++++++++++++++++++++
.../solrj/io/eval/SumDifferenceEvaluator.java | 55 +++++++++++++++++++
.../solrj/io/stream/StreamExpressionTest.java | 49 +++++++++++++++--
10 files changed, 502 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
[31/50] lucene-solr:feature/autoscaling: LUCENE-7906: Add new shapes
to testing paradigm. Committed on behalf of Ignacio Vera.
Posted by sh...@apache.org.
LUCENE-7906: Add new shapes to testing paradigm. Committed on behalf of Ignacio Vera.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/cd425d60
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/cd425d60
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/cd425d60
Branch: refs/heads/feature/autoscaling
Commit: cd425d609cee8bcea6dbfeab8b3d42b1ce48eb40
Parents: 64d1428
Author: Karl Wright <Da...@gmail.com>
Authored: Mon Sep 11 09:00:47 2017 -0400
Committer: Karl Wright <Da...@gmail.com>
Committed: Mon Sep 11 09:00:47 2017 -0400
----------------------------------------------------------------------
.../spatial3d/geom/RandomBinaryCodecTest.java | 10 +-
.../spatial3d/geom/RandomGeoShapeGenerator.java | 944 -------------------
.../geom/RandomGeoShapeRelationshipTest.java | 65 +-
3 files changed, 53 insertions(+), 966 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cd425d60/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/RandomBinaryCodecTest.java
----------------------------------------------------------------------
diff --git a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/RandomBinaryCodecTest.java b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/RandomBinaryCodecTest.java
index ba9ee6e..250b652 100644
--- a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/RandomBinaryCodecTest.java
+++ b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/RandomBinaryCodecTest.java
@@ -27,18 +27,18 @@ import org.junit.Test;
/**
* Test to check Serialization
*/
-public class RandomBinaryCodecTest extends RandomGeoShapeGenerator{
+public class RandomBinaryCodecTest extends RandomGeo3dShapeGenerator {
@Test
@Repeat(iterations = 10)
public void testRandomPointCodec() throws IOException{
PlanetModel planetModel = randomPlanetModel();
- GeoPoint shape = randomGeoPoint(planetModel, getEmptyConstraint());
+ GeoPoint shape = randomGeoPoint(planetModel);
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
SerializableObject.writeObject(outputStream, shape);
ByteArrayInputStream inputStream = new ByteArrayInputStream(outputStream.toByteArray());
SerializableObject shapeCopy = SerializableObject.readObject(planetModel, inputStream);
- assertEquals(shape, shapeCopy);
+ assertEquals(shape.toString(), shape, shapeCopy);
}
@Test
@@ -51,7 +51,7 @@ public class RandomBinaryCodecTest extends RandomGeoShapeGenerator{
SerializableObject.writePlanetObject(outputStream, shape);
ByteArrayInputStream inputStream = new ByteArrayInputStream(outputStream.toByteArray());
SerializableObject shapeCopy = SerializableObject.readPlanetObject(inputStream);
- assertEquals(shape, shapeCopy);
+ assertEquals(shape.toString(), shape, shapeCopy);
}
@Test
@@ -64,6 +64,6 @@ public class RandomBinaryCodecTest extends RandomGeoShapeGenerator{
SerializableObject.writeObject(outputStream, shape);
ByteArrayInputStream inputStream = new ByteArrayInputStream(outputStream.toByteArray());
SerializableObject shapeCopy = SerializableObject.readObject(planetModel, inputStream);
- assertEquals(shape, shapeCopy);
+ assertEquals(shape.toString(), shape, shapeCopy);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cd425d60/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/RandomGeoShapeGenerator.java
----------------------------------------------------------------------
diff --git a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/RandomGeoShapeGenerator.java b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/RandomGeoShapeGenerator.java
deleted file mode 100644
index ca7aecd..0000000
--- a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/RandomGeoShapeGenerator.java
+++ /dev/null
@@ -1,944 +0,0 @@
-/*
-* Licensed to the Apache Software Foundation (ASF) under one or more
-* contributor license agreements. See the NOTICE file distributed with
-* this work for additional information regarding copyright ownership.
-* The ASF licenses this file to You under the Apache License, Version 2.0
-* (the "License"); you may not use this file except in compliance with
-* the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.lucene.spatial3d.geom;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.List;
-
-import org.apache.lucene.util.LuceneTestCase;
-
-import static com.carrotsearch.randomizedtesting.RandomizedTest.randomDouble;
-
-/**
- * Class for generating random Geo3dShapes. They can be generated under
- * given constraints which are expressed as a shape and a relationship.
- *
- * note that convexity for polygons is defined as polygons that contains
- * antipodal points, otherwise they are convex. Internally they can be
- * created using GeoConvexPolygons and GeoConcavePolygons.
- *
- */
-public class RandomGeoShapeGenerator extends LuceneTestCase {
-
- /* Max num of iterations to find right shape under given constrains */
- final private static int MAX_SHAPE_ITERATIONS = 50;
- /* Max num of iterations to find right point under given constrains */
- final private static int MAX_POINT_ITERATIONS = 1000;
-
- /* Supported shapes */
- final protected static int CONVEX_POLYGON = 0;
- final protected static int CONVEX_POLYGON_WITH_HOLES = 1;
- final protected static int CONCAVE_POLYGON = 2;
- final protected static int CONCAVE_POLYGON_WITH_HOLES = 3;
- final protected static int COMPLEX_POLYGON = 4;
- final protected static int CIRCLE = 5;
- final protected static int RECTANGLE = 6;
- final protected static int PATH = 7;
- final protected static int COLLECTION = 8;
-
- /* Helper shapes for generating constraints whch are just three sided polygons */
- final protected static int CONVEX_SIMPLE_POLYGON = 500;
- final protected static int CONCAVE_SIMPLE_POLYGON = 501;
-
-
- /**
- * Method that returns empty Constraints object..
- *
- * @return an empty Constraints object
- */
- public Constraints getEmptyConstraint(){
- return new Constraints();
- }
-
- /**
- * Method that returns a random generated a random Shape code from all
- * supported shapes.
- *
- * @return a random generated shape code
- */
- public int randomShapeType(){
- return random().nextInt(9);
- }
-
- /**
- * Method that returns a random generated a random Shape code from all
- * convex supported shapes.
- *
- * @return a random generated convex shape code
- */
- public int randomConvexShapeType(){
- int shapeType = randomShapeType();
- while (isConcave(shapeType)){
- shapeType = randomShapeType();
- }
- return shapeType;
- }
-
- /**
- * Method that returns a random generated a random Shape code from all
- * concave supported shapes.
- *
- * @return a random generated concave shape code
- */
- public int randomConcaveShapeType(){
- int shapeType = randomShapeType();
- while (!isConcave(shapeType)){
- shapeType = randomShapeType();
- }
- return shapeType;
- }
-
- /**
- * Method that returns a random generated GeoAreaShape code from all
- * supported GeoAreaShapes.
- *
- * We are removing Collections because it is difficult to create shapes
- * with properties in some cases.
- *
- * @return a random generated polygon code
- */
- public int randomGeoAreaShapeType(){
- return random().nextInt(8);
- }
-
- /**
- * Check if a shape code represents a concave shape
- *
- * @return true if the shape represented by the code is concave
- */
- public boolean isConcave(int shapeType){
- return (shapeType == CONCAVE_POLYGON);
- }
-
- /**
- * Method that returns a random generated Planet model from the supported
- * Planet models. currently SPHERE and WGS84
- *
- * @return a random generated Planet model
- */
- public PlanetModel randomPlanetModel() {
- final int shapeType = random().nextInt(2);
- switch (shapeType) {
- case 0: {
- return PlanetModel.SPHERE;
- }
- case 1: {
- return PlanetModel.WGS84;
- }
- default:
- throw new IllegalStateException("Unexpected planet model");
- }
- }
-
- /**
- * Method that returns a random generated GeoPoint under given constraints. Returns
- * NULL if it cannot find a point under the given constraints.
- *
- * @param planetModel The planet model.
- * @param constraints The given constraints.
- * @return The random generated GeoPoint.
- */
- public GeoPoint randomGeoPoint(PlanetModel planetModel, Constraints constraints) {
- int iterations = 0;
- while (iterations < MAX_POINT_ITERATIONS) {
- double lat = randomDouble();
- if (Math.PI/2 - Math.abs(lat) <0){
- continue;
- }
- double lon = randomDouble();
- if (Math.PI - Math.abs(lat) <0){
- continue;
- }
- iterations++;
- GeoPoint point = new GeoPoint(planetModel, lat, lon);
- if (constraints.isWithin(point)) {
- return point;
- }
- }
- return null;
- }
-
- /**
- * Method that returns a random generated GeoAreaShape.
- *
- * @param shapeType The GeoAreaShape code.
- * @param planetModel The planet model.
- * @return The random generated GeoAreaShape.
- */
- public GeoAreaShape randomGeoAreaShape(int shapeType, PlanetModel planetModel){
- GeoAreaShape geoAreaShape = null;
- while (geoAreaShape == null){
- geoAreaShape = randomGeoAreaShape(shapeType,planetModel,new Constraints());
- }
- return geoAreaShape;
- }
-
- /**
- * Method that returns a random generated GeoAreaShape under given constraints. Returns
- * NULL if it cannot build the GeoAreaShape under the given constraints.
- *
- * @param shapeType The GeoAreaShape code.
- * @param planetModel The planet model.
- * @param constraints The given constraints.
- * @return The random generated GeoAreaShape.
- */
- public GeoAreaShape randomGeoAreaShape(int shapeType, PlanetModel planetModel, Constraints constraints){
- return (GeoAreaShape)randomGeoShape(shapeType, planetModel, constraints);
- }
-
- /**
- * Method that returns a random generated GeoShape.
- *
- * @param shapeType The shape code.
- * @param planetModel The planet model.
- * @return The random generated GeoShape.
- */
- public GeoShape randomGeoShape(int shapeType, PlanetModel planetModel){
- GeoShape geoShape = null;
- while (geoShape == null){
- geoShape = randomGeoShape(shapeType,planetModel,new Constraints());
- }
- return geoShape;
- }
-
- /**
- * Method that returns a random generated GeoShape under given constraints. Returns
- * NULL if it cannot build the GeoShape under the given constraints.
- *
- * @param shapeType The polygon code.
- * @param planetModel The planet model.
- * @param constraints The given constraints.
- * @return The random generated GeoShape.
- */
- public GeoShape randomGeoShape(int shapeType, PlanetModel planetModel, Constraints constraints){
- switch (shapeType) {
- case CONVEX_POLYGON: {
- return convexPolygon(planetModel, constraints);
- }
- case CONVEX_POLYGON_WITH_HOLES: {
- return convexPolygonWithHoles(planetModel, constraints);
- }
- case CONCAVE_POLYGON: {
- return concavePolygon(planetModel, constraints);
- }
- case CONCAVE_POLYGON_WITH_HOLES: {
- return concavePolygonWithHoles(planetModel, constraints);
- }
- case COMPLEX_POLYGON: {
- return complexPolygon(planetModel, constraints);
- }
- case CIRCLE: {
- return circle(planetModel, constraints);
- }
- case RECTANGLE: {
- return rectangle(planetModel, constraints);
- }
- case PATH: {
- return path(planetModel, constraints);
- }
- case COLLECTION: {
- return collection(planetModel, constraints);
- }
- case CONVEX_SIMPLE_POLYGON: {
- return simpleConvexPolygon(planetModel, constraints);
- }
- case CONCAVE_SIMPLE_POLYGON: {
- return concaveSimplePolygon(planetModel, constraints);
- }
- default:
- throw new IllegalStateException("Unexpected shape type");
- }
- }
-
- /**
- * Method that returns a random generated a GeoCircle under given constraints. Returns
- * NULL if it cannot build the GeoCircle under the given constraints.
- *
- * @param planetModel The planet model.
- * @param constraints The given constraints.
- * @return The random generated GeoCircle.
- */
- private GeoCircle circle(PlanetModel planetModel , Constraints constraints) {
- int iterations=0;
- while (iterations < MAX_SHAPE_ITERATIONS) {
- iterations++;
- final GeoPoint center = randomGeoPoint(planetModel, constraints);
- if (center == null){
- continue;
- }
- final double radius = randomCutoffAngle();
- try {
-
- GeoCircle circle = GeoCircleFactory.makeGeoCircle(planetModel, center.getLatitude(), center.getLongitude(), radius);
- if (!constraints.valid(circle)) {
- continue;
- }
- return circle;
- } catch (IllegalArgumentException e) {
- continue;
- }
- }
- return null;
- }
-
- /**
- * Method that returns a random generated a GeoBBox under given constraints. Returns
- * NULL if it cannot build the GeoBBox under the given constraints.
- *
- * @param planetModel The planet model.
- * @param constraints The given constraints.
- * @return The random generated GeoBBox.
- */
- private GeoBBox rectangle(PlanetModel planetModel, Constraints constraints) {
-
- int iterations = 0;
- while (iterations < MAX_SHAPE_ITERATIONS) {
- iterations++;
- final GeoPoint point1 = randomGeoPoint(planetModel, constraints);
- if (point1 == null){
- continue;
- }
- final GeoPoint point2 = randomGeoPoint(planetModel, constraints);
- if (point2 == null){
- continue;
- }
-
- double minLat = Math.min(point1.getLatitude(), point2.getLatitude());
- double maxLat = Math.max(point1.getLatitude(), point2.getLatitude());
- double minLon = Math.min(point1.getLongitude(), point2.getLongitude());
- double maxLon = Math.max(point1.getLongitude(), point2.getLongitude());
-
- try {
- GeoBBox bbox = GeoBBoxFactory.makeGeoBBox(planetModel, maxLat, minLat, minLon, maxLon);
- if (!constraints.valid(bbox)) {
- continue;
- }
- return bbox;
- } catch (IllegalArgumentException e) {
- continue;
- }
- }
- return null;
- }
-
- /**
- * Method that returns a random generated a GeoPath under given constraints. Returns
- * NULL if it cannot build the GeoPath under the given constraints.
- *
- * @param planetModel The planet model.
- * @param constraints The given constraints.
- * @return The random generated GeoPath.
- */
- private GeoPath path(PlanetModel planetModel, Constraints constraints) {
- int iterations = 0;
- while (iterations < MAX_SHAPE_ITERATIONS) {
- iterations++;
- int vertexCount = random().nextInt(2) + 2;
- List<GeoPoint> geoPoints = points(vertexCount, planetModel, constraints);
- double width =randomCutoffAngle();
- try {
- GeoPath path = GeoPathFactory.makeGeoPath(planetModel, width, geoPoints.toArray(new GeoPoint[geoPoints.size()]));
- if (!constraints.valid(path)) {
- continue;
- }
- return path;
- } catch (IllegalArgumentException e) {
- continue;
- }
- }
- return null;
- }
-
- /**
- * Method that returns a random generated a GeoCompositeMembershipShape under given constraints. Returns
- * NULL if it cannot build the GGeoCompositeMembershipShape under the given constraints.
- *
- * @param planetModel The planet model.
- * @param constraints The given constraints.
- * @return The random generated GeoCompositeMembershipShape.
- */
- private GeoCompositeAreaShape collection(PlanetModel planetModel, Constraints constraints) {
- int iterations = 0;
- while (iterations < MAX_SHAPE_ITERATIONS) {
- iterations++;
- int numberShapes = random().nextInt(3) + 2;
- GeoCompositeAreaShape collection = new GeoCompositeAreaShape(planetModel);
- for(int i=0; i<numberShapes;i++){
- GeoPolygon member = convexPolygon(planetModel, constraints);
- if (member != null){
- collection.addShape(member);
- }
- }
- if (collection.shapes.size() ==0){
- continue;
- }
- return collection;
- }
- return null;
- }
-
- /**
- * Method that returns a random generated a convex GeoPolygon under given constraints. Returns
- * NULL if it cannot build the GePolygon under the given constraints.
- *
- * @param planetModel The planet model.
- * @param constraints The given constraints.
- * @return The random generated GeoPolygon.
- */
- private GeoPolygon convexPolygon(PlanetModel planetModel, Constraints constraints) {
- int vertexCount = random().nextInt(4) + 3;
- int iterations = 0;
- while (iterations < MAX_SHAPE_ITERATIONS) {
- iterations++;
- List<GeoPoint> geoPoints = points(vertexCount,planetModel, constraints);
- List<GeoPoint> orderedGeoPoints = orderPoints(geoPoints);
- try {
- GeoPolygon polygon = GeoPolygonFactory.makeGeoPolygon(planetModel, orderedGeoPoints);
- if (!constraints.valid(polygon) || isConcave(planetModel, polygon)) {
- continue;
- }
- return polygon;
- } catch (IllegalArgumentException e) {
- continue;
- }
- }
- return null;
- }
-
- /**
- * Method that returns a random generated a convex GeoPolygon with holes under given constraints. Returns
- * NULL if it cannot build the GeoPolygon with holes under the given constraints.
- *
- * @param planetModel The planet model.
- * @param constraints The given constraints.
- * @return The random generated GeoPolygon.
- */
- private GeoPolygon convexPolygonWithHoles(PlanetModel planetModel, Constraints constraints) {
- int vertexCount = random().nextInt(4) + 3;
- int iterations = 0;
- while (iterations < MAX_SHAPE_ITERATIONS) {
- iterations++;
- List<GeoPoint> geoPoints = points(vertexCount,planetModel, constraints);
- List<GeoPoint> orderedGeoPoints = orderPoints(geoPoints);
- try {
- GeoPolygon polygon = GeoPolygonFactory.makeGeoPolygon(planetModel, orderedGeoPoints);
- //polygon should comply with all constraints except disjoint as we have holes
- Constraints polygonConstraints = new Constraints();
- polygonConstraints.putAll(constraints.getContains());
- polygonConstraints.putAll(constraints.getWithin());
- polygonConstraints.putAll(constraints.getDisjoint());
- if (!polygonConstraints.valid(polygon) || isConcave(planetModel, polygon)){
- continue;
- }
- //hole must overlap with polygon and comply with any CONTAINS constraint.
- Constraints holeConstraints = new Constraints();
- holeConstraints.putAll(constraints.getContains());
- holeConstraints.put(polygon,GeoArea.OVERLAPS);
- //Points must be with in the polygon and must comply
- // CONTAINS and DISJOINT constraints
- Constraints pointsConstraints = new Constraints();
- pointsConstraints.put(polygon,GeoArea.WITHIN);
- pointsConstraints.putAll(constraints.getContains());
- pointsConstraints.putAll(constraints.getDisjoint());
- List<GeoPolygon> holes = concavePolygonHoles(planetModel, holeConstraints, pointsConstraints);
- //we should have at least one hole
- if (holes.size() == 0){
- continue;
- }
- polygon = GeoPolygonFactory.makeGeoPolygon(planetModel,orderedGeoPoints,holes);
- if (!constraints.valid(polygon) || isConcave(planetModel, polygon)){
- continue;
- }
- return polygon;
- } catch (IllegalArgumentException e) {
- continue;
- }
- }
- return null;
- }
-
- /**
- * Method that returns a random list if concave GeoPolygons under given constraints. Method
- * use to generate convex holes. Note that constraints for points and holes are different,
- *
- * @param planetModel The planet model.
- * @param holeConstraints The given constraints that a hole must comply.
- * @param pointConstraints The given constraints that a point must comply.
- * @return The random generated GeoPolygon.
- */
- private List<GeoPolygon> concavePolygonHoles(PlanetModel planetModel,
- Constraints holeConstraints,
- Constraints pointConstraints) {
- int iterations =0;
- int holesCount = random().nextInt(3) + 1;
- List<GeoPolygon> holes = new ArrayList<>();
- while (iterations < MAX_SHAPE_ITERATIONS) {
- iterations++;
- int vertexCount = random().nextInt(3) + 3;
- List<GeoPoint> geoPoints = points(vertexCount, planetModel, pointConstraints);
- geoPoints = orderPoints(geoPoints);
- Collections.reverse(geoPoints);
- try {
- GeoPolygon hole = GeoPolygonFactory.makeGeoPolygon(planetModel, geoPoints);
- if (!holeConstraints.valid(hole) || isConvex(planetModel, hole)) {
- continue;
- }
- holes.add(hole);
- if (holes.size() == holesCount){
- return holes;
- }
- pointConstraints.put(hole, GeoArea.DISJOINT);
- } catch (IllegalArgumentException e) {
- continue;
- }
- }
- return holes;
- }
-
- /**
- * Method that returns a random generated a concave GeoPolygon under given constraints. Returns
- * NULL if it cannot build the concave GeoPolygon under the given constraints.
- *
- * @param planetModel The planet model.
- * @param constraints The given constraints.
- * @return The random generated GeoPolygon.
- */
- private GeoPolygon concavePolygon(PlanetModel planetModel, Constraints constraints) {
-
- int vertexCount = random().nextInt(4) + 3;
- int iterations = 0;
- while (iterations < MAX_SHAPE_ITERATIONS) {
- iterations++;
- List<GeoPoint> geoPoints = points(vertexCount,planetModel, constraints);
- List<GeoPoint> orderedGeoPoints = orderPoints(geoPoints);
- Collections.reverse(orderedGeoPoints);
- try {
- GeoPolygon polygon = GeoPolygonFactory.makeGeoPolygon(planetModel, orderedGeoPoints);
- if (!constraints.valid(polygon) || isConvex(planetModel, polygon)) {
- continue;
- }
- return polygon;
- } catch (IllegalArgumentException e) {
- continue;
- }
- }
- return null;
- }
-
- /**
- * Method that returns a random generated a concave GeoPolygon with holes under given constraints. Returns
- * NULL if it cannot build the GeoPolygon under the given constraints. Note that the final GeoPolygon is
- * convex as the hole wraps the convex GeoPolygon.
- *
- * @param planetModel The planet model.
- * @param constraints The given constraints.
- * @return The random generated GeoPolygon.
- */
- private GeoPolygon concavePolygonWithHoles(PlanetModel planetModel, Constraints constraints) {
- int vertexCount = random().nextInt(4) + 3;
- int iterations = 0;
- while (iterations < MAX_SHAPE_ITERATIONS) {
- iterations++;
- //we first build the hole. We consider all constraints except
- // disjoint as we have a hole
- Constraints holeConstraints = new Constraints();
- holeConstraints.putAll(constraints.getContains());
- holeConstraints.putAll(constraints.getWithin());
- holeConstraints.putAll(constraints.getOverlaps());
- GeoPolygon hole = convexPolygon(planetModel, holeConstraints);
- if (hole == null){
- continue;
- }
- // Now we get points for polygon. Must we with in the hole
- // and we add contain constraints
- Constraints pointConstraints = new Constraints();
- pointConstraints.put(hole, GeoArea.WITHIN);
- pointConstraints.putAll(constraints.getContains());
- List<GeoPoint> geoPoints = points(vertexCount,planetModel, pointConstraints);
- List<GeoPoint> orderedGeoPoints = orderPoints(geoPoints);
- Collections.reverse(orderedGeoPoints);
- try {
- GeoPolygon polygon = GeoPolygonFactory.makeGeoPolygon(planetModel, orderedGeoPoints, Collections.singletonList(hole));
- //final polygon must be convex
- if (!constraints.valid(polygon) || isConcave(planetModel,polygon)) {
- continue;
- }
- return polygon;
- } catch (IllegalArgumentException e) {
- continue;
- }
- }
- return null;
- }
-
- /**
- * Method that returns a random generated complex GeoPolygon under given constraints. Returns
- * NULL if it cannot build the complex GeoPolygon under the given constraints.
- *
- * @param planetModel The planet model.
- * @param constraints The given constraints.
- * @return The random generated GeoPolygon.
- */
- private GeoPolygon complexPolygon(PlanetModel planetModel, Constraints constraints) {
- int polygonsCount =random().nextInt(2) + 1;
- int iterations = 0;
- while (iterations < MAX_SHAPE_ITERATIONS) {
- iterations++;
- List<GeoPolygonFactory.PolygonDescription> polDescription = new ArrayList<>();
- while(polDescription.size() < polygonsCount){
- int vertexCount = random().nextInt(14) + 3;
- List<GeoPoint> geoPoints = points(vertexCount,planetModel, constraints);
- orderPoints(geoPoints);
- polDescription.add(new GeoPolygonFactory.PolygonDescription(geoPoints));
- }
- try {
- GeoPolygon polygon = GeoPolygonFactory.makeLargeGeoPolygon(planetModel,polDescription);
- if (!constraints.valid(polygon)) {
- continue;
- }
- return polygon;
- } catch (IllegalArgumentException e) {
- continue;
- }
- }
- return null;
- }
-
- /**
- * Method that returns a random generated a concave square GeoPolygon under given constraints. Returns
- * NULL if it cannot build the concave GeoPolygon under the given constraints. This shape is an utility
- * to build constraints.
- *
- * @param planetModel The planet model.
- * @param constraints The given constraints.
- * @return The random generated GeoPolygon.
- */
- private GeoPolygon simpleConvexPolygon(PlanetModel planetModel, Constraints constraints) {
- int iterations = 0;
- while (iterations < MAX_SHAPE_ITERATIONS) {
- iterations++;
- List<GeoPoint> points = points(3,planetModel,constraints);
- points = orderPoints(points);
- try {
- GeoPolygon polygon = GeoPolygonFactory.makeGeoConvexPolygon(planetModel, points);
- if(!constraints.valid(polygon) || isConcave(planetModel,polygon)){
- continue;
- }
- return polygon;
- } catch (IllegalArgumentException e) {
- continue;
- }
- }
- return null;
- }
-
- /**
- * Method that returns a random generated a convex square GeoPolygon under given constraints. Returns
- * NULL if it cannot build the convex GeoPolygon under the given constraints. This shape is an utility
- * to build constraints.
- *
- * @param planetModel The planet model.
- * @param constraints The given constraints.
- * @return The random generated GeoPolygon.
- */
- private GeoPolygon concaveSimplePolygon(PlanetModel planetModel, Constraints constraints) {
- int iterations = 0;
- while (iterations < MAX_SHAPE_ITERATIONS) {
- iterations++;
- List<GeoPoint> points = points(3, planetModel, constraints);
- points = orderPoints(points);
- Collections.reverse(points);
- try {
- GeoPolygon polygon = GeoPolygonFactory.makeGeoConcavePolygon(planetModel, points);
- if(!constraints.valid(polygon) || isConvex(planetModel, polygon)){
- continue;
- }
- return polygon;
- } catch (IllegalArgumentException e) {
- continue;
- }
- }
- return null;
- }
-
- /**
- * Method that returns a random list of generated GeoPoints under given constraints. If it cannot
- * find a point it will add a point that might not comply with the constraints.
- *
- * @param count The number of points
- * @param planetModel The planet model.
- * @param constraints The given constraints.
- * @return The random generated List of GeoPoints.
- */
- private List<GeoPoint> points(int count, PlanetModel planetModel, Constraints constraints){
- List<GeoPoint> geoPoints = new ArrayList<>(count);
- for(int i= 0; i< count; i++) {
- GeoPoint point = randomGeoPoint(planetModel, constraints);
- if (point == null){
- point = randomGeoPoint(planetModel, new Constraints());
- }
- geoPoints.add(point);
- }
- return geoPoints;
- }
-
- /**
- * Check if a GeoPolygon is pure concave. Note that our definition for concavity is that the polygon
- * contains antipodal points.
- *
- * @param planetModel The planet model.
- * @param shape The polygon to check.
- * @return True if the polygon contains antipodal points.
- */
- private boolean isConcave(PlanetModel planetModel, GeoPolygon shape){
- return (shape.isWithin(planetModel.NORTH_POLE) && shape.isWithin(planetModel.SOUTH_POLE))||
- (shape.isWithin(planetModel.MAX_X_POLE) && shape.isWithin(planetModel.MIN_X_POLE)) ||
- (shape.isWithin(planetModel.MAX_Y_POLE) && shape.isWithin(planetModel.MIN_Y_POLE));
- }
-
- /**
- * Check if a GeoPolygon is pure convex. Note that our definition for convexity is that the polygon
- * does not contain antipodal points.
- *
- * @param planetModel The planet model.
- * @param shape The polygon to check.
- * @return True if the polygon dies not contains antipodal points.
- */
- private boolean isConvex(PlanetModel planetModel, GeoPolygon shape){
- return !isConcave(planetModel,shape);
- }
-
- /**
- * Generates a random number between 0 and PI.
- *
- * @return the cutoff angle.
- */
- private double randomCutoffAngle() {
- while(true) {
- double radius = randomDouble();
- if (radius <0 || radius > Math.PI){
- continue;
- }
- return radius;
- }
- }
-
- /**
- * Method that orders a lit of points anti-clock-wise to prevent crossing edges.
- *
- * @param originalPoints The points to order.
- * @return The list of ordered points anti-clockwise.
- */
- private List<GeoPoint> orderPoints(List<GeoPoint> originalPoints){
- List<GeoPoint> points = new ArrayList<>(originalPoints.size());
- points.addAll(originalPoints); //make a copy
- GeoPoint lPoint = getPointLefLon(points);
- points.remove(lPoint);
- GeoPoint rPoint = getPointRigthLon(points);
- points.remove(rPoint);
- List<GeoPoint> APoints = getPointsBelowAndSort(points, lPoint);
- List<GeoPoint> BPoints = getPointsAboveAndsort(points, lPoint);
- List<GeoPoint> result = new ArrayList<>();
- result.add(lPoint);
- result.addAll(APoints);
- result.add(rPoint);
- result.addAll(BPoints);
- return result;
- }
-
- private List<GeoPoint> getPointsAboveAndsort(List<GeoPoint> points,GeoPoint lPoint) {
- List<GeoPoint> BPoints = new ArrayList<>();
- for (GeoPoint point : points){
- if(point.getLatitude() > lPoint.getLatitude()){
- BPoints.add(point);
- }
- }
- Collections.sort(BPoints, new Comparator<GeoPoint>() {
- public int compare(GeoPoint idx1, GeoPoint idx2) {
- return Double.compare(idx1.getLongitude(), idx2.getLongitude());
- }
- });
- return BPoints;
- }
-
- private List<GeoPoint> getPointsBelowAndSort(List<GeoPoint> points,GeoPoint lPoint) {
- List<GeoPoint> APoints = new ArrayList<>();
- for (GeoPoint point : points){
- if(point.getLatitude() < lPoint.getLatitude()){
- APoints.add(point);
- }
- }
- Collections.sort(APoints, new Comparator<GeoPoint>() {
- public int compare(GeoPoint idx1, GeoPoint idx2) {
- return Double.compare(idx1.getLongitude(), idx2.getLongitude());
- }
- });
- return APoints;
- }
-
- private GeoPoint getPointLefLon(List<GeoPoint> points) {
- GeoPoint lPoint = null;
- for (GeoPoint point : points){
- if(lPoint == null ){
- lPoint = point;
- }
- else{
- if (lPoint.getLongitude() > point.getLongitude()){
- lPoint = point;
- }
- }
- }
- return lPoint;
- }
-
- private GeoPoint getPointRigthLon(List<GeoPoint> points) {
- GeoPoint rPoint = null;
- for (GeoPoint point : points){
- if(rPoint == null ){
- rPoint = point;
- }
- else{
- if (rPoint.getLongitude() < point.getLongitude()){
- rPoint = point;
- }
- }
- }
- return rPoint;
- }
-
- /**
- * Class that holds the constraints that are given to
- * build shapes. It consists in a list of GeoAreaShapes
- * and relationships the new shape needs to satisfy.
- */
- class Constraints extends HashMap<GeoAreaShape, Integer>{
-
- /**
- * Check if the shape is valid under the constraints.
- *
- * @param shape The shape to check
- * @return true if the shape satisfy the constraints, else false.
- */
- public boolean valid(GeoShape shape) {
- if (shape == null){
- return false;
- }
- for (GeoAreaShape constraint : keySet()) {
- if (constraint.getRelationship(shape) != get(constraint)) {
- return false;
- }
- }
- return true;
- }
-
- /**
- * Check if a point is Within the constraints.
- *
- * @param point The point to check
- * @return true if the point satisfy the constraints, else false.
- */
- public boolean isWithin(GeoPoint point) {
- for (GeoShape constraint : keySet()) {
- if (!(validPoint(point, constraint, get(constraint)))) {
- return false;
- }
- }
- return true;
- }
-
- /**
- * Check if a point is Within one constraint given by a shape and a relationship.
- *
- * @param point The point to check
- * @param shape The shape of the constraint
- * @param relationship The relationship of the constraint.
- * @return true if the point satisfy the constraint, else false.
- */
- private boolean validPoint(GeoPoint point, GeoShape shape, int relationship) {
- //For GeoCompositeMembershipShape we only consider the first shape to help
- // converging
- if (relationship == GeoArea.WITHIN && shape instanceof GeoCompositeMembershipShape) {
- shape = (((GeoCompositeMembershipShape) shape).shapes.get(0));
- }
- switch (relationship) {
- case GeoArea.DISJOINT:
- return !shape.isWithin(point);
- case GeoArea.OVERLAPS:
- return true;
- case GeoArea.CONTAINS:
- return !shape.isWithin(point);
- case GeoArea.WITHIN:
- return shape.isWithin(point);
- default:
- return true;
- }
- }
-
- /**
- * Collect the CONTAINS constraints in the object
- *
- * @return the CONTAINS constraints.
- */
- public Constraints getContains(){
- return getConstraintsOfType(GeoArea.CONTAINS);
- }
-
- /**
- * Collect the WITHIN constraints in the object
- *
- * @return the WITHIN constraints.
- */
- public Constraints getWithin(){
- return getConstraintsOfType(GeoArea.WITHIN);
- }
-
- /**
- * Collect the OVERLAPS constraints in the object
- *
- * @return the OVERLAPS constraints.
- */
- public Constraints getOverlaps(){
- return getConstraintsOfType(GeoArea.OVERLAPS);
- }
-
- /**
- * Collect the DISJOINT constraints in the object
- *
- * @return the DISJOINT constraints.
- */
- public Constraints getDisjoint(){
- return getConstraintsOfType(GeoArea.DISJOINT);
- }
-
- private Constraints getConstraintsOfType(int type){
- Constraints constraints = new Constraints();
- for (GeoAreaShape constraint : keySet()) {
- if (type == get(constraint)) {
- constraints.put(constraint, type);
- }
- }
- return constraints;
- }
- }
-}
-
-
-
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cd425d60/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/RandomGeoShapeRelationshipTest.java
----------------------------------------------------------------------
diff --git a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/RandomGeoShapeRelationshipTest.java b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/RandomGeoShapeRelationshipTest.java
index 75a8c28..9c791b7 100644
--- a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/RandomGeoShapeRelationshipTest.java
+++ b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/RandomGeoShapeRelationshipTest.java
@@ -23,8 +23,7 @@ import org.junit.Test;
/**
* Random test to check relationship between GeoAreaShapes and GeoShapes.
*/
-public class RandomGeoShapeRelationshipTest extends RandomGeoShapeGenerator {
-
+public class RandomGeoShapeRelationshipTest extends RandomGeo3dShapeGenerator {
/**
* Test for WITHIN points. We build a WITHIN shape with respect the geoAreaShape
@@ -38,6 +37,9 @@ public class RandomGeoShapeRelationshipTest extends RandomGeoShapeGenerator {
int referenceShapeType = CONVEX_POLYGON;
PlanetModel planetModel = randomPlanetModel();
int shapeType = randomShapeType();
+ while (shapeType == POINT || shapeType == LINE) {
+ shapeType = randomShapeType();
+ }
GeoAreaShape shape = null;
GeoPoint point = null;
while (point == null) {
@@ -51,7 +53,10 @@ public class RandomGeoShapeRelationshipTest extends RandomGeoShapeGenerator {
point = randomGeoPoint(planetModel, constraints);
}
}
- assertTrue(shape.isWithin(point));
+ StringBuilder b = new StringBuilder();
+ b.append("shape: " + shape + "\n");
+ b.append("point: " + point);
+ assertTrue(b.toString(), shape.isWithin(point));
}
/**
@@ -78,7 +83,10 @@ public class RandomGeoShapeRelationshipTest extends RandomGeoShapeGenerator {
point = randomGeoPoint(planetModel, constraints);
}
}
- assertFalse(shape.isWithin(point));
+ StringBuilder b = new StringBuilder();
+ b.append("shape: " + shape + "\n");
+ b.append("point: " + point);
+ assertFalse(b.toString(), shape.isWithin(point));
}
/**
@@ -94,7 +102,7 @@ public class RandomGeoShapeRelationshipTest extends RandomGeoShapeGenerator {
int referenceShapeType = CONVEX_SIMPLE_POLYGON;
PlanetModel planetModel = randomPlanetModel();
int geoAreaShapeType = randomGeoAreaShapeType();
- int shapeType =randomConvexShapeType();
+ int shapeType = randomConvexShapeType();
GeoShape shape = null;
GeoAreaShape geoAreaShape = null;
@@ -109,11 +117,14 @@ public class RandomGeoShapeRelationshipTest extends RandomGeoShapeGenerator {
shape = randomGeoShape(shapeType, planetModel, constraints);
}
}
+ StringBuilder b = new StringBuilder();
+ b.append("geoAreaShape: " + geoAreaShape + "\n");
+ b.append("shape: " + shape);
int rel = geoAreaShape.getRelationship(shape);
- assertEquals(GeoArea.DISJOINT, rel);
+ assertEquals(b.toString(), GeoArea.DISJOINT, rel);
if (shape instanceof GeoArea) {
rel = ((GeoArea)shape).getRelationship(geoAreaShape);
- assertEquals(GeoArea.DISJOINT, rel);
+ assertEquals(b.toString(), GeoArea.DISJOINT, rel);
}
}
@@ -129,7 +140,11 @@ public class RandomGeoShapeRelationshipTest extends RandomGeoShapeGenerator {
public void testRandomWithIn() {
PlanetModel planetModel = randomPlanetModel();
int geoAreaShapeType = randomGeoAreaShapeType();
- int shapeType =randomShapeType();
+ //shapes cannot be point or line -- no area!
+ while(geoAreaShapeType == POINT || geoAreaShapeType == LINE) {
+ geoAreaShapeType = randomGeoAreaShapeType();
+ }
+ int shapeType = LINE;//randomShapeType();
int referenceShapeType = CONVEX_SIMPLE_POLYGON;
if (!isConcave(geoAreaShapeType)){
shapeType =randomConvexShapeType();
@@ -150,11 +165,14 @@ public class RandomGeoShapeRelationshipTest extends RandomGeoShapeGenerator {
shape = randomGeoShape(shapeType, planetModel, constraints);
}
}
+ StringBuilder b = new StringBuilder();
+ b.append("geoAreaShape: " + geoAreaShape + "\n");
+ b.append("shape: " + shape);
int rel = geoAreaShape.getRelationship(shape);
- assertEquals(GeoArea.WITHIN, rel);
+ assertEquals(b.toString(), GeoArea.WITHIN, rel);
if (shape instanceof GeoArea) {
rel = ((GeoArea)shape).getRelationship(geoAreaShape);
- assertEquals(GeoArea.CONTAINS, rel);
+ assertEquals(b.toString(), GeoArea.CONTAINS, rel);
}
}
@@ -174,10 +192,13 @@ public class RandomGeoShapeRelationshipTest extends RandomGeoShapeGenerator {
int referenceShapeType = CONVEX_SIMPLE_POLYGON;
PlanetModel planetModel = randomPlanetModel();
int geoAreaShapeType = randomGeoAreaShapeType();
- while (geoAreaShapeType == COLLECTION){
+ while (geoAreaShapeType == COLLECTION ){
geoAreaShapeType = randomGeoAreaShapeType();
}
int shapeType = randomShapeType();
+ while (shapeType == POINT || shapeType == LINE) {
+ shapeType = randomShapeType();
+ }
if (isConcave(geoAreaShapeType)){
shapeType = randomConcaveShapeType();
}
@@ -197,11 +218,14 @@ public class RandomGeoShapeRelationshipTest extends RandomGeoShapeGenerator {
shape = randomGeoShape(shapeType, planetModel, constraints);
}
}
+ StringBuilder b = new StringBuilder();
+ b.append("geoAreaShape: " + geoAreaShape + "\n");
+ b.append("shape: " + shape);
int rel = geoAreaShape.getRelationship(shape);
- assertEquals(GeoArea.CONTAINS, rel);
+ assertEquals(b.toString(), GeoArea.CONTAINS, rel);
if (shape instanceof GeoArea) {
rel = ((GeoArea)shape).getRelationship(geoAreaShape);
- assertEquals(GeoArea.WITHIN, rel);
+ assertEquals(b.toString(), GeoArea.WITHIN, rel);
}
}
@@ -216,8 +240,13 @@ public class RandomGeoShapeRelationshipTest extends RandomGeoShapeGenerator {
public void testRandomOverlaps() {
PlanetModel planetModel = randomPlanetModel();
int geoAreaShapeType = randomGeoAreaShapeType();
+ while (geoAreaShapeType == POINT || geoAreaShapeType == LINE) {
+ geoAreaShapeType = randomGeoAreaShapeType();
+ }
int shapeType = randomShapeType();
-
+ while (shapeType == POINT || shapeType == LINE) {
+ shapeType = randomShapeType();
+ }
GeoShape shape = null;
GeoAreaShape geoAreaShape = null;
while (shape == null) {
@@ -246,12 +275,14 @@ public class RandomGeoShapeRelationshipTest extends RandomGeoShapeGenerator {
shape = randomGeoShape(shapeType, planetModel, constraints);
}
}
+ StringBuilder b = new StringBuilder();
+ b.append("geoAreaShape: " + geoAreaShape + "\n");
+ b.append("shape: " + shape);
int rel = geoAreaShape.getRelationship(shape);
- assertEquals(GeoArea.OVERLAPS, rel);
+ assertEquals(b.toString(), GeoArea.OVERLAPS, rel);
if (shape instanceof GeoArea) {
rel = ((GeoArea)shape).getRelationship(geoAreaShape);
- assertEquals(GeoArea.OVERLAPS, rel);
+ assertEquals(b.toString(), GeoArea.OVERLAPS, rel);
}
}
}
-
[25/50] lucene-solr:feature/autoscaling: LUCENE-7965: GeoBBoxFactory
was constructing the wrong shape at the poles,
if the longitude range provided was greater than 180 degrees.
Posted by sh...@apache.org.
LUCENE-7965: GeoBBoxFactory was constructing the wrong shape at the poles, if the longitude range provided was greater than 180 degrees.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/fe7582e9
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/fe7582e9
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/fe7582e9
Branch: refs/heads/feature/autoscaling
Commit: fe7582e9d725b0fbeeac03a901203dfd45813266
Parents: 2b6c82e
Author: Karl Wright <Da...@gmail.com>
Authored: Sat Sep 9 01:01:33 2017 -0400
Committer: Karl Wright <Da...@gmail.com>
Committed: Sat Sep 9 01:01:33 2017 -0400
----------------------------------------------------------------------
.../org/apache/lucene/spatial3d/geom/GeoBBoxFactory.java | 9 ++++++++-
1 file changed, 8 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/fe7582e9/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoBBoxFactory.java
----------------------------------------------------------------------
diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoBBoxFactory.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoBBoxFactory.java
index d0427bc..12d479b 100755
--- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoBBoxFactory.java
+++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoBBoxFactory.java
@@ -82,6 +82,11 @@ public class GeoBBoxFactory {
//System.err.println(" not vertical line");
if (extent >= Math.PI) {
if (Math.abs(topLat - bottomLat) < Vector.MINIMUM_ANGULAR_RESOLUTION) {
+ if (Math.abs(topLat - Math.PI * 0.5) < Vector.MINIMUM_ANGULAR_RESOLUTION) {
+ return new GeoDegeneratePoint(planetModel, topLat, 0.0);
+ } else if (Math.abs(bottomLat + Math.PI * 0.5) < Vector.MINIMUM_ANGULAR_RESOLUTION) {
+ return new GeoDegeneratePoint(planetModel, bottomLat, 0.0);
+ }
//System.err.println(" wide degenerate line");
return new GeoWideDegenerateHorizontalLine(planetModel, topLat, leftLon, rightLon);
}
@@ -94,8 +99,10 @@ public class GeoBBoxFactory {
return new GeoWideRectangle(planetModel, topLat, bottomLat, leftLon, rightLon);
}
if (Math.abs(topLat - bottomLat) < Vector.MINIMUM_ANGULAR_RESOLUTION) {
- if (Math.abs(topLat - Math.PI * 0.5) < Vector.MINIMUM_ANGULAR_RESOLUTION || Math.abs(topLat + Math.PI * 0.5) < Vector.MINIMUM_ANGULAR_RESOLUTION) {
+ if (Math.abs(topLat - Math.PI * 0.5) < Vector.MINIMUM_ANGULAR_RESOLUTION) {
return new GeoDegeneratePoint(planetModel, topLat, 0.0);
+ } else if (Math.abs(bottomLat + Math.PI * 0.5) < Vector.MINIMUM_ANGULAR_RESOLUTION) {
+ return new GeoDegeneratePoint(planetModel, bottomLat, 0.0);
}
//System.err.println(" horizontal line");
return new GeoDegenerateHorizontalLine(planetModel, topLat, leftLon, rightLon);
[46/50] lucene-solr:feature/autoscaling: LUCENE-7955:
GeoDegeneratePath did not compute endpoint bounding planes properly.
Posted by sh...@apache.org.
LUCENE-7955: GeoDegeneratePath did not compute endpoint bounding planes properly.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/c317ad8c
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/c317ad8c
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/c317ad8c
Branch: refs/heads/feature/autoscaling
Commit: c317ad8c1fb4a2eb56fa22f8e9b9fad79ce661ba
Parents: 277bc3a
Author: Karl Wright <Da...@gmail.com>
Authored: Thu Sep 14 10:50:44 2017 -0400
Committer: Karl Wright <Da...@gmail.com>
Committed: Thu Sep 14 10:50:44 2017 -0400
----------------------------------------------------------------------
.../org/apache/lucene/spatial3d/geom/GeoDegeneratePath.java | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c317ad8c/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoDegeneratePath.java
----------------------------------------------------------------------
diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoDegeneratePath.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoDegeneratePath.java
index eaceb26..861a2d4 100644
--- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoDegeneratePath.java
+++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoDegeneratePath.java
@@ -119,20 +119,20 @@ class GeoDegeneratePath extends GeoBasePath {
if (i == 0) {
// Starting endpoint
final SegmentEndpoint startEndpoint = new SegmentEndpoint(currentSegment.start,
- new SidedPlane(currentSegment.startCutoffPlane));
+ currentSegment.startCutoffPlane);
endPoints.add(startEndpoint);
this.edgePoints = new GeoPoint[]{currentSegment.start};
continue;
}
endPoints.add(new SegmentEndpoint(currentSegment.start,
- new SidedPlane(segments.get(i-1).endCutoffPlane),
- new SidedPlane(currentSegment.startCutoffPlane)));
+ segments.get(i-1).endCutoffPlane,
+ currentSegment.startCutoffPlane));
}
// Do final endpoint
final PathSegment lastSegment = segments.get(segments.size()-1);
endPoints.add(new SegmentEndpoint(lastSegment.end,
- new SidedPlane(lastSegment.endCutoffPlane)));
+ lastSegment.endCutoffPlane));
}
[42/50] lucene-solr:feature/autoscaling: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/lucene-solr
Posted by sh...@apache.org.
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/lucene-solr
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/b125107b
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/b125107b
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/b125107b
Branch: refs/heads/feature/autoscaling
Commit: b125107b7c8f093ffb21e2acaa1c8fc68a11d883
Parents: b8d48fa 708ed8f
Author: Karl Wright <Da...@gmail.com>
Authored: Tue Sep 12 08:34:52 2017 -0400
Committer: Karl Wright <Da...@gmail.com>
Committed: Tue Sep 12 08:34:52 2017 -0400
----------------------------------------------------------------------
solr/CHANGES.txt | 6 ++
.../apache/solr/ltr/model/LTRScoringModel.java | 4 +-
.../org/apache/solr/ltr/TestRerankBase.java | 12 ++--
.../org/apache/solr/handler/StreamHandler.java | 1 +
.../solr/handler/component/QueryComponent.java | 23 +++----
.../apache/solr/schema/AbstractEnumField.java | 11 ++--
.../org/apache/solr/schema/DatePointField.java | 15 +----
.../apache/solr/schema/DoublePointField.java | 15 +----
.../java/org/apache/solr/schema/FieldType.java | 65 +++++++++++++++++++-
.../org/apache/solr/schema/FloatPointField.java | 15 +----
.../org/apache/solr/schema/IntPointField.java | 15 +----
.../org/apache/solr/schema/LongPointField.java | 15 +----
.../apache/solr/schema/PreAnalyzedField.java | 6 +-
.../java/org/apache/solr/schema/TextField.java | 5 +-
.../java/org/apache/solr/schema/TrieField.java | 44 ++-----------
.../java/org/apache/solr/search/Sorting.java | 12 +++-
.../solr/collection1/conf/schema-enums.xml | 10 +++
.../org/apache/solr/schema/EnumFieldTest.java | 16 +++++
.../org/apache/solr/schema/TestPointFields.java | 2 +-
.../test/org/apache/solr/search/TestSort.java | 21 ++++++-
.../client/solrj/io/eval/PrimesEvaluator.java | 58 +++++++++++++++++
.../solrj/io/stream/StreamExpressionTest.java | 29 +++++++++
22 files changed, 248 insertions(+), 152 deletions(-)
----------------------------------------------------------------------
[48/50] lucene-solr:feature/autoscaling: LUCENE-7955: Add tests for
nearest path distance, degenerate path.
Posted by sh...@apache.org.
LUCENE-7955: Add tests for nearest path distance, degenerate path.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/ab965d50
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/ab965d50
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/ab965d50
Branch: refs/heads/feature/autoscaling
Commit: ab965d506cdc49b7c038b3c6b99623006409bf89
Parents: 8cb64ed
Author: Karl Wright <Da...@gmail.com>
Authored: Fri Sep 15 02:20:11 2017 -0400
Committer: Karl Wright <Da...@gmail.com>
Committed: Fri Sep 15 02:20:11 2017 -0400
----------------------------------------------------------------------
.../lucene/spatial3d/geom/GeoPathTest.java | 73 ++++++++++++++++++++
1 file changed, 73 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ab965d50/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPathTest.java
----------------------------------------------------------------------
diff --git a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPathTest.java b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPathTest.java
index 3aada9a..922617d 100755
--- a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPathTest.java
+++ b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPathTest.java
@@ -304,4 +304,77 @@ public class GeoPathTest {
assertTrue(solid.isWithin(point));
}
+ @Test
+ public void testInterpolation() {
+ final double lat = 52.51607;
+ final double lon = 13.37698;
+ final double[] pathLats = new double[] {52.5355,52.54,52.5626,52.5665,52.6007,52.6135,52.6303,52.6651,52.7074};
+ final double[] pathLons = new double[] {13.3634,13.3704,13.3307,13.3076,13.2806,13.2484,13.2406,13.241,13.1926};
+
+ // Set up a point in the right way
+ final GeoPoint carPoint = new GeoPoint(PlanetModel.SPHERE, toRadians(lat), toRadians(lon));
+ // Create the path, but use a tiny width (e.g. zero)
+ final GeoPoint[] pathPoints = new GeoPoint[pathLats.length];
+ for (int i = 0; i < pathPoints.length; i++) {
+ pathPoints[i] = new GeoPoint(PlanetModel.SPHERE, toRadians(pathLats[i]), toRadians(pathLons[i]));
+ }
+ // Construct a path with no width
+ final GeoPath thisPath = GeoPathFactory.makeGeoPath(PlanetModel.SPHERE, 0.0, pathPoints);
+ // Construct a path with a width
+ final GeoPath legacyPath = GeoPathFactory.makeGeoPath(PlanetModel.SPHERE, 1e-6, pathPoints);
+ // Compute the inside distance to the atPoint using zero-width path
+ final double distance = thisPath.computeNearestDistance(DistanceStyle.ARC, carPoint);
+ // Compute the inside distance using legacy path
+ final double legacyDistance = legacyPath.computeNearestDistance(DistanceStyle.ARC, carPoint);
+ // Compute the inside distance using the legacy formula
+ final double oldFormulaDistance = thisPath.computeDistance(DistanceStyle.ARC, carPoint);
+ // Compute the inside distance using the legacy formula with the legacy shape
+ final double oldFormulaLegacyDistance = legacyPath.computeDistance(DistanceStyle.ARC, carPoint);
+
+ // These should be about the same
+ assertEquals(legacyDistance, distance, 1e-12);
+ assertEquals(oldFormulaLegacyDistance, oldFormulaDistance, 1e-12);
+ // This isn't true because example search center is off of the path.
+ //assertEquals(oldFormulaDistance, distance, 1e-12);
+
+ }
+
+ @Test
+ public void testInterpolation2() {
+ final double lat = 52.5665;
+ final double lon = 13.3076;
+ final double[] pathLats = new double[] {52.5355,52.54,52.5626,52.5665,52.6007,52.6135,52.6303,52.6651,52.7074};
+ final double[] pathLons = new double[] {13.3634,13.3704,13.3307,13.3076,13.2806,13.2484,13.2406,13.241,13.1926};
+
+ final GeoPoint carPoint = new GeoPoint(PlanetModel.SPHERE, toRadians(lat), toRadians(lon));
+ final GeoPoint[] pathPoints = new GeoPoint[pathLats.length];
+ for (int i = 0; i < pathPoints.length; i++) {
+ pathPoints[i] = new GeoPoint(PlanetModel.SPHERE, toRadians(pathLats[i]), toRadians(pathLons[i]));
+ }
+
+ // Construct a path with no width
+ final GeoPath thisPath = GeoPathFactory.makeGeoPath(PlanetModel.SPHERE, 0.0, pathPoints);
+ // Construct a path with a width
+ final GeoPath legacyPath = GeoPathFactory.makeGeoPath(PlanetModel.SPHERE, 1e-6, pathPoints);
+
+ // Compute the inside distance to the atPoint using zero-width path
+ final double distance = thisPath.computeNearestDistance(DistanceStyle.ARC, carPoint);
+ // Compute the inside distance using legacy path
+ final double legacyDistance = legacyPath.computeNearestDistance(DistanceStyle.ARC, carPoint);
+
+ // Compute the inside distance using the legacy formula
+ final double oldFormulaDistance = thisPath.computeDistance(DistanceStyle.ARC, carPoint);
+ // Compute the inside distance using the legacy formula with the legacy shape
+ final double oldFormulaLegacyDistance = legacyPath.computeDistance(DistanceStyle.ARC, carPoint);
+
+ // These should be about the same
+ assertEquals(legacyDistance, distance, 1e-12);
+
+ assertEquals(oldFormulaLegacyDistance, oldFormulaDistance, 1e-12);
+
+ // Since the point we picked is actually on the path, this should also be true
+ assertEquals(oldFormulaDistance, distance, 1e-12);
+
+ }
+
}