You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2014/02/06 12:38:32 UTC
svn commit: r1565184 [1/3] - in /lucene/dev/branches/lucene5376: ./ lucene/
lucene/core/ lucene/core/src/java/org/apache/lucene/codecs/lucene45/
lucene/core/src/java/org/apache/lucene/search/
lucene/core/src/java/org/apache/lucene/store/ lucene/core/sr...
Author: mikemccand
Date: Thu Feb 6 11:38:30 2014
New Revision: 1565184
URL: http://svn.apache.org/r1565184
Log:
LUCENE-5376: merge trunk
Added:
lucene/dev/branches/lucene5376/lucene/core/src/test/org/apache/lucene/index/index.45.cfs.zip
- copied unchanged from r1565169, lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/index.45.cfs.zip
lucene/dev/branches/lucene5376/lucene/core/src/test/org/apache/lucene/index/index.45.nocfs.zip
- copied unchanged from r1565169, lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/index.45.nocfs.zip
lucene/dev/branches/lucene5376/lucene/core/src/test/org/apache/lucene/store/TestFilterDirectory.java
- copied unchanged from r1565169, lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/store/TestFilterDirectory.java
lucene/dev/branches/lucene5376/lucene/facet/src/java/org/apache/lucene/facet/sortedset/DefaultSortedSetDocValuesReaderState.java
- copied unchanged from r1565169, lucene/dev/trunk/lucene/facet/src/java/org/apache/lucene/facet/sortedset/DefaultSortedSetDocValuesReaderState.java
lucene/dev/branches/lucene5376/solr/core/src/test-files/solr/analysisconfs/
- copied from r1565169, lucene/dev/trunk/solr/core/src/test-files/solr/analysisconfs/
lucene/dev/branches/lucene5376/solr/core/src/test-files/solr/collection1/conf/solrconfig-query-parser-init.xml
- copied unchanged from r1565169, lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/solrconfig-query-parser-init.xml
lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/analysis/ThrowingMockTokenFilterFactory.java
- copied unchanged from r1565169, lucene/dev/trunk/solr/core/src/test/org/apache/solr/analysis/ThrowingMockTokenFilterFactory.java
lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/search/TestInitQParser.java
- copied unchanged from r1565169, lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/TestInitQParser.java
lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/search/TestStandardQParsers.java
- copied unchanged from r1565169, lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/TestStandardQParsers.java
lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/update/AnalysisErrorHandlingTest.java
- copied unchanged from r1565169, lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/AnalysisErrorHandlingTest.java
lucene/dev/branches/lucene5376/solr/solrj/src/java/org/apache/solr/client/solrj/impl/NoOpResponseParser.java
- copied unchanged from r1565169, lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/client/solrj/impl/NoOpResponseParser.java
lucene/dev/branches/lucene5376/solr/solrj/src/test-files/solrj/sampleSimpleGroupResponse.xml
- copied unchanged from r1565169, lucene/dev/trunk/solr/solrj/src/test-files/solrj/sampleSimpleGroupResponse.xml
lucene/dev/branches/lucene5376/solr/solrj/src/test/org/apache/solr/client/solrj/response/NoOpResponseParserTest.java
- copied unchanged from r1565169, lucene/dev/trunk/solr/solrj/src/test/org/apache/solr/client/solrj/response/NoOpResponseParserTest.java
Modified:
lucene/dev/branches/lucene5376/ (props changed)
lucene/dev/branches/lucene5376/.gitignore
lucene/dev/branches/lucene5376/lucene/ (props changed)
lucene/dev/branches/lucene5376/lucene/CHANGES.txt (contents, props changed)
lucene/dev/branches/lucene5376/lucene/core/ (props changed)
lucene/dev/branches/lucene5376/lucene/core/src/java/org/apache/lucene/codecs/lucene45/Lucene45DocValuesProducer.java
lucene/dev/branches/lucene5376/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java
lucene/dev/branches/lucene5376/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java
lucene/dev/branches/lucene5376/lucene/core/src/java/org/apache/lucene/store/Directory.java
lucene/dev/branches/lucene5376/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java
lucene/dev/branches/lucene5376/lucene/core/src/java/org/apache/lucene/store/FSLockFactory.java
lucene/dev/branches/lucene5376/lucene/core/src/java/org/apache/lucene/store/FilterDirectory.java
lucene/dev/branches/lucene5376/lucene/core/src/java/org/apache/lucene/store/RAMDirectory.java
lucene/dev/branches/lucene5376/lucene/core/src/java/org/apache/lucene/util/FixedBitSet.java
lucene/dev/branches/lucene5376/lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
lucene/dev/branches/lucene5376/lucene/core/src/test/org/apache/lucene/index/TestCrashCausesCorruptIndex.java
lucene/dev/branches/lucene5376/lucene/core/src/test/org/apache/lucene/search/TestMultiPhraseQuery.java
lucene/dev/branches/lucene5376/lucene/core/src/test/org/apache/lucene/search/TestPhraseQuery.java
lucene/dev/branches/lucene5376/lucene/demo/ (props changed)
lucene/dev/branches/lucene5376/lucene/demo/src/java/org/apache/lucene/demo/facet/SimpleSortedSetFacetsExample.java
lucene/dev/branches/lucene5376/lucene/facet/ (props changed)
lucene/dev/branches/lucene5376/lucene/facet/src/java/org/apache/lucene/facet/FacetsCollector.java
lucene/dev/branches/lucene5376/lucene/facet/src/java/org/apache/lucene/facet/range/DoubleRangeFacetCounts.java
lucene/dev/branches/lucene5376/lucene/facet/src/java/org/apache/lucene/facet/range/LongRangeFacetCounts.java
lucene/dev/branches/lucene5376/lucene/facet/src/java/org/apache/lucene/facet/sortedset/SortedSetDocValuesFacetCounts.java
lucene/dev/branches/lucene5376/lucene/facet/src/java/org/apache/lucene/facet/sortedset/SortedSetDocValuesReaderState.java
lucene/dev/branches/lucene5376/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/FastTaxonomyFacetCounts.java
lucene/dev/branches/lucene5376/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetCounts.java
lucene/dev/branches/lucene5376/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetSumFloatAssociations.java
lucene/dev/branches/lucene5376/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetSumIntAssociations.java
lucene/dev/branches/lucene5376/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetSumValueSource.java
lucene/dev/branches/lucene5376/lucene/facet/src/test/org/apache/lucene/facet/TestDrillSideways.java
lucene/dev/branches/lucene5376/lucene/facet/src/test/org/apache/lucene/facet/range/TestRangeFacetCounts.java
lucene/dev/branches/lucene5376/lucene/facet/src/test/org/apache/lucene/facet/sortedset/TestSortedSetDocValuesFacets.java
lucene/dev/branches/lucene5376/lucene/queryparser/ (props changed)
lucene/dev/branches/lucene5376/lucene/queryparser/src/java/org/apache/lucene/queryparser/simple/SimpleQueryParser.java
lucene/dev/branches/lucene5376/lucene/queryparser/src/test/org/apache/lucene/queryparser/simple/TestSimpleQueryParser.java
lucene/dev/branches/lucene5376/lucene/test-framework/ (props changed)
lucene/dev/branches/lucene5376/lucene/test-framework/src/java/org/apache/lucene/search/AssertingScorer.java
lucene/dev/branches/lucene5376/solr/ (props changed)
lucene/dev/branches/lucene5376/solr/CHANGES.txt (contents, props changed)
lucene/dev/branches/lucene5376/solr/contrib/ (props changed)
lucene/dev/branches/lucene5376/solr/contrib/langid/src/java/org/apache/solr/update/processor/LanguageIdentifierUpdateProcessor.java
lucene/dev/branches/lucene5376/solr/contrib/langid/src/test/org/apache/solr/update/processor/LanguageIdentifierUpdateProcessorFactoryTestCase.java
lucene/dev/branches/lucene5376/solr/core/ (props changed)
lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java
lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java
lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/core/SolrConfig.java
lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/core/SolrCore.java
lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/handler/admin/InfoHandler.java
lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java
lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java
lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java
lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/search/BoostQParserPlugin.java
lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/search/DisMaxQParserPlugin.java
lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/search/FieldQParserPlugin.java
lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/search/FunctionQParserPlugin.java
lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/search/FunctionRangeQParserPlugin.java
lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java
lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/search/LuceneQParserPlugin.java
lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/search/MaxScoreQParserPlugin.java
lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/search/NestedQParserPlugin.java
lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/search/OldLuceneQParserPlugin.java
lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/search/PrefixQParserPlugin.java
lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/search/QParserPlugin.java
lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/search/RawQParserPlugin.java
lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/search/SimpleQParserPlugin.java
lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/search/SpatialBoxQParserPlugin.java
lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/search/SpatialFilterQParserPlugin.java
lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/search/SurroundQParserPlugin.java
lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/search/SwitchQParserPlugin.java
lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/search/TermQParserPlugin.java
lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/search/grouping/distributed/responseprocessor/SearchGroupShardResponseProcessor.java
lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/search/grouping/distributed/responseprocessor/TopGroupsShardResponseProcessor.java
lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/search/join/BlockJoinChildQParserPlugin.java
lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/search/join/BlockJoinParentQParserPlugin.java
lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java
lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/store/blockcache/CustomBufferedIndexInput.java
lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java
lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/TestDistributedSearch.java
lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java
lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java
lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/CollectionsAPIDistributedZkTest.java
lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/ZkCLITest.java
lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/handler/admin/InfoHandlerTest.java
lucene/dev/branches/lucene5376/solr/solrj/ (props changed)
lucene/dev/branches/lucene5376/solr/solrj/src/java/org/apache/solr/client/solrj/request/RequestWriter.java
lucene/dev/branches/lucene5376/solr/solrj/src/java/org/apache/solr/client/solrj/response/QueryResponse.java
lucene/dev/branches/lucene5376/solr/solrj/src/java/org/apache/solr/common/cloud/ConnectionManager.java
lucene/dev/branches/lucene5376/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java
lucene/dev/branches/lucene5376/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
lucene/dev/branches/lucene5376/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java
lucene/dev/branches/lucene5376/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java
lucene/dev/branches/lucene5376/solr/solrj/src/java/org/apache/solr/common/params/SimpleParams.java
lucene/dev/branches/lucene5376/solr/solrj/src/java/org/apache/solr/common/params/SolrParams.java
lucene/dev/branches/lucene5376/solr/solrj/src/test/org/apache/solr/client/solrj/response/QueryResponseTest.java
Modified: lucene/dev/branches/lucene5376/.gitignore
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/.gitignore?rev=1565184&r1=1565183&r2=1565184&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/.gitignore (original)
+++ lucene/dev/branches/lucene5376/.gitignore Thu Feb 6 11:38:30 2014
@@ -26,6 +26,7 @@
/solr/package
# can this be minimized?
+/solr/example/start.jar
/solr/example/webapps/*
/solr/example/logs/*.log
/solr/example/**/data
@@ -38,3 +39,8 @@
/solr/example/example-DIH/**/data
/solr/example/example-DIH/**/dataimport.properties
/solr/example/example-DIH/solr/mail/lib/*.jar
+
+solr/contrib/dataimporthandler/test-lib/
+solr/contrib/morphlines-core/test-lib/
+
+solr/core/test-lib/
Modified: lucene/dev/branches/lucene5376/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/CHANGES.txt?rev=1565184&r1=1565183&r2=1565184&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/lucene5376/lucene/CHANGES.txt Thu Feb 6 11:38:30 2014
@@ -48,11 +48,11 @@ API Changes
this term index, pass it directly in your codec, where it can also be configured
per-field. (Robert Muir)
-* LUCENE-5388: Remove Reader from Tokenizer's constructor.
+* LUCENE-5388: Remove Reader from Tokenizer's constructor and from
+ Analyzer's createComponents. TokenStreams now always get their input
+ via setReader.
(Benson Margulies via Robert Muir - pull request #16)
-* LUCENE-5405: Make ShingleAnalzyerWrapper.getWrappedAnalyzer() public final (gsingers)
-
Documentation
* LUCENE-5392: Add/improve analysis package documentation to reflect
@@ -110,6 +110,13 @@ New Features
fixes too. More info:
https://github.com/spatial4j/spatial4j/blob/master/CHANGES.md (David Smiley)
+* LUCENE-5405: If an analysis component throws an exception, Lucene
+ logs the field name to the info stream to assist in
+ diagnosis. (Benson Margulies)
+
+* LUCENE-5406: Make ShingleAnalzyerWrapper.getWrappedAnalyzer() public
+ final (gsingers)
+
* LUCENE-5415: Add multitermquery (wildcards,prefix,etc) to PostingsHighlighter.
(Mike McCandless, Robert Muir)
@@ -125,6 +132,13 @@ New Features
* LUCENE-5320: Add SearcherTaxonomyManager over search and taxonomy index
directories (i.e. not only NRT). (Shai Erera)
+* LUCENE-5410: Add fuzzy and near support via '~' operator to SimpleQueryParser.
+ (Lee Hinman via Robert Muir)
+
+* LUCENE-5426: Make SortedSetDocValuesReaderState abstract to allow
+ custom implementations for Lucene doc values faceting (John Wang via
+ Mike McCandless)
+
Build
* LUCENE-5217,LUCENE-5420: Maven config: get dependencies from Ant+Ivy config;
@@ -243,6 +257,10 @@ API Changes
responsible for encoding/decoding a block of terms. Instead, they
should encode/decode each term to/from a long[] and byte[]. (Han
Jiang, Mike McCandless)
+
+* LUCENE-5425: FacetsCollector and MatchingDocs use a general DocIdSet,
+ allowing for custom implementations to be used when faceting.
+ (John Wang, Lei Wang, Shai Erera)
Optimizations
Modified: lucene/dev/branches/lucene5376/lucene/core/src/java/org/apache/lucene/codecs/lucene45/Lucene45DocValuesProducer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/core/src/java/org/apache/lucene/codecs/lucene45/Lucene45DocValuesProducer.java?rev=1565184&r1=1565183&r2=1565184&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/core/src/java/org/apache/lucene/codecs/lucene45/Lucene45DocValuesProducer.java (original)
+++ lucene/dev/branches/lucene5376/lucene/core/src/java/org/apache/lucene/codecs/lucene45/Lucene45DocValuesProducer.java Thu Feb 6 11:38:30 2014
@@ -104,7 +104,7 @@ public class Lucene45DocValuesProducer e
String dataName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, dataExtension);
data = state.directory.openInput(dataName, state.context);
final int version2 = CodecUtil.checkHeader(data, dataCodec,
- Lucene45DocValuesFormat.VERSION_CURRENT,
+ Lucene45DocValuesFormat.VERSION_START,
Lucene45DocValuesFormat.VERSION_CURRENT);
if (version != version2) {
throw new CorruptIndexException("Format versions mismatch");
Modified: lucene/dev/branches/lucene5376/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java?rev=1565184&r1=1565183&r2=1565184&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java (original)
+++ lucene/dev/branches/lucene5376/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java Thu Feb 6 11:38:30 2014
@@ -58,7 +58,12 @@ public class MultiPhraseQuery extends Qu
/** Sets the phrase slop for this query.
* @see PhraseQuery#setSlop(int)
*/
- public void setSlop(int s) { slop = s; }
+ public void setSlop(int s) {
+ if (s < 0) {
+ throw new IllegalArgumentException("slop value cannot be negative");
+ }
+ slop = s;
+ }
/** Sets the phrase slop for this query.
* @see PhraseQuery#getSlop()
Modified: lucene/dev/branches/lucene5376/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java?rev=1565184&r1=1565183&r2=1565184&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java (original)
+++ lucene/dev/branches/lucene5376/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java Thu Feb 6 11:38:30 2014
@@ -68,7 +68,12 @@ public class PhraseQuery extends Query {
results are sorted by exactness.
<p>The slop is zero by default, requiring exact matches.*/
- public void setSlop(int s) { slop = s; }
+ public void setSlop(int s) {
+ if (s < 0) {
+ throw new IllegalArgumentException("slop value cannot be negative");
+ }
+ slop = s;
+ }
/** Returns the slop. See setSlop(). */
public int getSlop() { return slop; }
Modified: lucene/dev/branches/lucene5376/lucene/core/src/java/org/apache/lucene/store/Directory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/core/src/java/org/apache/lucene/store/Directory.java?rev=1565184&r1=1565183&r2=1565184&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/core/src/java/org/apache/lucene/store/Directory.java (original)
+++ lucene/dev/branches/lucene5376/lucene/core/src/java/org/apache/lucene/store/Directory.java Thu Feb 6 11:38:30 2014
@@ -152,12 +152,12 @@ public abstract class Directory implemen
* "scopes" to the right index.
*/
public String getLockID() {
- return this.toString();
+ return this.toString();
}
@Override
public String toString() {
- return super.toString() + " lockFactory=" + getLockFactory();
+ return getClass().getSimpleName() + '@' + Integer.toHexString(hashCode()) + " lockFactory=" + getLockFactory();
}
/**
Modified: lucene/dev/branches/lucene5376/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java?rev=1565184&r1=1565183&r2=1565184&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java (original)
+++ lucene/dev/branches/lucene5376/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java Thu Feb 6 11:38:30 2014
@@ -337,7 +337,7 @@ public abstract class FSDirectory extend
/** For debug output. */
@Override
public String toString() {
- return this.getClass().getName() + "@" + directory + " lockFactory=" + getLockFactory();
+ return this.getClass().getSimpleName() + "@" + directory + " lockFactory=" + getLockFactory();
}
/**
Modified: lucene/dev/branches/lucene5376/lucene/core/src/java/org/apache/lucene/store/FSLockFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/core/src/java/org/apache/lucene/store/FSLockFactory.java?rev=1565184&r1=1565183&r2=1565184&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/core/src/java/org/apache/lucene/store/FSLockFactory.java (original)
+++ lucene/dev/branches/lucene5376/lucene/core/src/java/org/apache/lucene/store/FSLockFactory.java Thu Feb 6 11:38:30 2014
@@ -50,4 +50,9 @@ public abstract class FSLockFactory exte
return lockDir;
}
+ @Override
+ public String toString() {
+ return this.getClass().getSimpleName() + "@" + lockDir;
+ }
+
}
Modified: lucene/dev/branches/lucene5376/lucene/core/src/java/org/apache/lucene/store/FilterDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/core/src/java/org/apache/lucene/store/FilterDirectory.java?rev=1565184&r1=1565183&r2=1565184&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/core/src/java/org/apache/lucene/store/FilterDirectory.java (original)
+++ lucene/dev/branches/lucene5376/lucene/core/src/java/org/apache/lucene/store/FilterDirectory.java Thu Feb 6 11:38:30 2014
@@ -101,6 +101,11 @@ public class FilterDirectory extends Dir
}
@Override
+ public String getLockID() {
+ return in.getLockID();
+ }
+
+ @Override
public LockFactory getLockFactory() {
return in.getLockFactory();
}
Modified: lucene/dev/branches/lucene5376/lucene/core/src/java/org/apache/lucene/store/RAMDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/core/src/java/org/apache/lucene/store/RAMDirectory.java?rev=1565184&r1=1565183&r2=1565184&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/core/src/java/org/apache/lucene/store/RAMDirectory.java (original)
+++ lucene/dev/branches/lucene5376/lucene/core/src/java/org/apache/lucene/store/RAMDirectory.java Thu Feb 6 11:38:30 2014
@@ -103,6 +103,11 @@ public class RAMDirectory extends BaseDi
}
@Override
+ public String getLockID() {
+ return "lucene-" + Integer.toHexString(hashCode());
+ }
+
+ @Override
public final String[] listAll() {
ensureOpen();
// NOTE: fileMap.keySet().toArray(new String[0]) is broken in non Sun JDKs,
@@ -201,4 +206,5 @@ public class RAMDirectory extends BaseDi
isOpen = false;
fileMap.clear();
}
+
}
Modified: lucene/dev/branches/lucene5376/lucene/core/src/java/org/apache/lucene/util/FixedBitSet.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/core/src/java/org/apache/lucene/util/FixedBitSet.java?rev=1565184&r1=1565183&r2=1565184&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/core/src/java/org/apache/lucene/util/FixedBitSet.java (original)
+++ lucene/dev/branches/lucene5376/lucene/core/src/java/org/apache/lucene/util/FixedBitSet.java Thu Feb 6 11:38:30 2014
@@ -78,7 +78,68 @@ public final class FixedBitSet extends D
@Override
public DocIdSetIterator iterator() {
- return new OpenBitSetIterator(bits, wordLength);
+ // define locally so we don't have "enclosing acces" issue
+ final long[] bits = this.bits;
+ final int wordLength = this.wordLength;
+ final int numBits = this.numBits;
+ return new DocIdSetIterator() {
+ int doc = -1;
+ @Override
+ public int nextDoc() throws IOException {
+ if (doc == NO_MORE_DOCS || ++doc >= numBits) {
+ return doc = NO_MORE_DOCS;
+ }
+ int i = doc >> 6;
+ final int subIndex = doc & 0x3f; // index within the word
+ long word = bits[i] >> subIndex; // skip all the bits to the right of index
+
+ if (word != 0) {
+ return doc = doc + Long.numberOfTrailingZeros(word);
+ }
+
+ while (++i < wordLength) {
+ word = bits[i];
+ if (word != 0) {
+ return doc = (i << 6) + Long.numberOfTrailingZeros(word);
+ }
+ }
+
+ return doc = NO_MORE_DOCS;
+ }
+
+ @Override
+ public int docID() {
+ return doc;
+ }
+
+ @Override
+ public long cost() {
+ return bits.length;
+ }
+
+ @Override
+ public int advance(int target) throws IOException {
+ if (doc == NO_MORE_DOCS || target >= numBits) {
+ return doc = NO_MORE_DOCS;
+ }
+ int i = target >> 6;
+ final int subIndex = target & 0x3f; // index within the word
+ long word = bits[i] >> subIndex; // skip all the bits to the right of index
+
+ if (word != 0) {
+ return doc = target + Long.numberOfTrailingZeros(word);
+ }
+
+ while (++i < wordLength) {
+ word = bits[i];
+ if (word != 0) {
+ return doc = (i << 6) + Long.numberOfTrailingZeros(word);
+ }
+ }
+
+ return doc = NO_MORE_DOCS;
+ }
+ };
}
@Override
@@ -166,7 +227,7 @@ public final class FixedBitSet extends D
long word = bits[i] >> subIndex; // skip all the bits to the right of index
if (word!=0) {
- return (i<<6) + subIndex + Long.numberOfTrailingZeros(word);
+ return index + Long.numberOfTrailingZeros(word);
}
while(++i < wordLength) {
Modified: lucene/dev/branches/lucene5376/lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java?rev=1565184&r1=1565183&r2=1565184&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (original)
+++ lucene/dev/branches/lucene5376/lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java Thu Feb 6 11:38:30 2014
@@ -164,8 +164,8 @@ public class TestBackwardsCompatibility
"42.cfs",
"42.nocfs",
// TODO: these are on 4x, but something is wrong (they seem to be a too old DV format):
- //"45.cfs",
- //"45.nocfs",
+ "45.cfs",
+ "45.nocfs",
"461.cfs",
"461.nocfs",
};
Modified: lucene/dev/branches/lucene5376/lucene/core/src/test/org/apache/lucene/index/TestCrashCausesCorruptIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/core/src/test/org/apache/lucene/index/TestCrashCausesCorruptIndex.java?rev=1565184&r1=1565183&r2=1565184&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/core/src/test/org/apache/lucene/index/TestCrashCausesCorruptIndex.java (original)
+++ lucene/dev/branches/lucene5376/lucene/core/src/test/org/apache/lucene/index/TestCrashCausesCorruptIndex.java Thu Feb 6 11:38:30 2014
@@ -19,7 +19,6 @@ package org.apache.lucene.index;
import java.io.File;
import java.io.IOException;
-import java.util.Collection;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
@@ -27,11 +26,10 @@ import org.apache.lucene.document.Field;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.store.BaseDirectory;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.store.FilterDirectory;
import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
@@ -147,28 +145,22 @@ public class TestCrashCausesCorruptIndex
* This test class provides direct access to "simulating" a crash right after
* realDirectory.createOutput(..) has been called on a certain specified name.
*/
- private static class CrashAfterCreateOutput extends BaseDirectory {
+ private static class CrashAfterCreateOutput extends FilterDirectory {
- private Directory realDirectory;
private String crashAfterCreateOutput;
public CrashAfterCreateOutput(Directory realDirectory) throws IOException {
- this.realDirectory = realDirectory;
+ super(realDirectory);
setLockFactory(realDirectory.getLockFactory());
}
public void setCrashAfterCreateOutput(String name) {
this.crashAfterCreateOutput = name;
}
-
- @Override
- public void close() throws IOException {
- realDirectory.close();
- }
-
+
@Override
public IndexOutput createOutput(String name, IOContext cxt) throws IOException {
- IndexOutput indexOutput = realDirectory.createOutput(name, cxt);
+ IndexOutput indexOutput = in.createOutput(name, cxt);
if (null != crashAfterCreateOutput && name.equals(crashAfterCreateOutput)) {
// CRASH!
indexOutput.close();
@@ -181,34 +173,6 @@ public class TestCrashCausesCorruptIndex
return indexOutput;
}
- @Override
- public void deleteFile(String name) throws IOException {
- realDirectory.deleteFile(name);
- }
-
- @Override
- public boolean fileExists(String name) throws IOException {
- return realDirectory.fileExists(name);
- }
-
- @Override
- public long fileLength(String name) throws IOException {
- return realDirectory.fileLength(name);
- }
-
- @Override
- public String[] listAll() throws IOException {
- return realDirectory.listAll();
- }
-
- @Override
- public IndexInput openInput(String name, IOContext cxt) throws IOException {
- return realDirectory.openInput(name, cxt);
- }
-
- @Override
- public void sync(Collection<String> names) throws IOException {
- realDirectory.sync(names);
- }
}
+
}
Modified: lucene/dev/branches/lucene5376/lucene/core/src/test/org/apache/lucene/search/TestMultiPhraseQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/core/src/test/org/apache/lucene/search/TestMultiPhraseQuery.java?rev=1565184&r1=1565183&r2=1565184&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/core/src/test/org/apache/lucene/search/TestMultiPhraseQuery.java (original)
+++ lucene/dev/branches/lucene5376/lucene/core/src/test/org/apache/lucene/search/TestMultiPhraseQuery.java Thu Feb 6 11:38:30 2014
@@ -568,4 +568,16 @@ public class TestMultiPhraseQuery extend
return terms;
}
+ public void testNegativeSlop() throws Exception {
+ MultiPhraseQuery query = new MultiPhraseQuery();
+ query.add(new Term("field", "two"));
+ query.add(new Term("field", "one"));
+ try {
+ query.setSlop(-2);
+ fail("didn't get expected exception");
+ } catch (IllegalArgumentException expected) {
+ // expected exception
+ }
+ }
+
}
Modified: lucene/dev/branches/lucene5376/lucene/core/src/test/org/apache/lucene/search/TestPhraseQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/core/src/test/org/apache/lucene/search/TestPhraseQuery.java?rev=1565184&r1=1565183&r2=1565184&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/core/src/test/org/apache/lucene/search/TestPhraseQuery.java (original)
+++ lucene/dev/branches/lucene5376/lucene/core/src/test/org/apache/lucene/search/TestPhraseQuery.java Thu Feb 6 11:38:30 2014
@@ -678,4 +678,16 @@ public class TestPhraseQuery extends Luc
reader.close();
dir.close();
}
+
+ public void testNegativeSlop() throws Exception {
+ PhraseQuery query = new PhraseQuery();
+ query.add(new Term("field", "two"));
+ query.add(new Term("field", "one"));
+ try {
+ query.setSlop(-2);
+ fail("didn't get expected exception");
+ } catch (IllegalArgumentException expected) {
+ // expected exception
+ }
+ }
}
Modified: lucene/dev/branches/lucene5376/lucene/demo/src/java/org/apache/lucene/demo/facet/SimpleSortedSetFacetsExample.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/demo/src/java/org/apache/lucene/demo/facet/SimpleSortedSetFacetsExample.java?rev=1565184&r1=1565183&r2=1565184&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/demo/src/java/org/apache/lucene/demo/facet/SimpleSortedSetFacetsExample.java (original)
+++ lucene/dev/branches/lucene5376/lucene/demo/src/java/org/apache/lucene/demo/facet/SimpleSortedSetFacetsExample.java Thu Feb 6 11:38:30 2014
@@ -28,6 +28,7 @@ import org.apache.lucene.facet.FacetResu
import org.apache.lucene.facet.Facets;
import org.apache.lucene.facet.FacetsCollector;
import org.apache.lucene.facet.FacetsConfig;
+import org.apache.lucene.facet.sortedset.DefaultSortedSetDocValuesReaderState;
import org.apache.lucene.facet.sortedset.SortedSetDocValuesFacetCounts;
import org.apache.lucene.facet.sortedset.SortedSetDocValuesFacetField;
import org.apache.lucene.facet.sortedset.SortedSetDocValuesReaderState;
@@ -88,7 +89,7 @@ public class SimpleSortedSetFacetsExampl
private List<FacetResult> search() throws IOException {
DirectoryReader indexReader = DirectoryReader.open(indexDir);
IndexSearcher searcher = new IndexSearcher(indexReader);
- SortedSetDocValuesReaderState state = new SortedSetDocValuesReaderState(indexReader);
+ SortedSetDocValuesReaderState state = new DefaultSortedSetDocValuesReaderState(indexReader);
// Aggregatses the facet counts
FacetsCollector fc = new FacetsCollector();
@@ -113,7 +114,7 @@ public class SimpleSortedSetFacetsExampl
private FacetResult drillDown() throws IOException {
DirectoryReader indexReader = DirectoryReader.open(indexDir);
IndexSearcher searcher = new IndexSearcher(indexReader);
- SortedSetDocValuesReaderState state = new SortedSetDocValuesReaderState(indexReader);
+ SortedSetDocValuesReaderState state = new DefaultSortedSetDocValuesReaderState(indexReader);
// Now user drills down on Publish Year/2010:
DrillDownQuery q = new DrillDownQuery(config);
Modified: lucene/dev/branches/lucene5376/lucene/facet/src/java/org/apache/lucene/facet/FacetsCollector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/facet/src/java/org/apache/lucene/facet/FacetsCollector.java?rev=1565184&r1=1565183&r2=1565184&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/facet/src/java/org/apache/lucene/facet/FacetsCollector.java (original)
+++ lucene/dev/branches/lucene5376/lucene/facet/src/java/org/apache/lucene/facet/FacetsCollector.java Thu Feb 6 11:38:30 2014
@@ -23,6 +23,7 @@ import java.util.List;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.search.Collector;
+import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.FieldDoc;
import org.apache.lucene.search.Filter;
import org.apache.lucene.search.FilteredQuery;
@@ -46,17 +47,33 @@ import org.apache.lucene.util.FixedBitSe
* counting. Use the {@code search} utility methods to
* perform an "ordinary" search but also collect into a
* {@link Collector}. */
-public final class FacetsCollector extends Collector {
+public class FacetsCollector extends Collector {
private AtomicReaderContext context;
private Scorer scorer;
- private FixedBitSet bits;
private int totalHits;
private float[] scores;
private final boolean keepScores;
private final List<MatchingDocs> matchingDocs = new ArrayList<MatchingDocs>();
+ private Docs docs;
/**
+ * Used during collection to record matching docs and then return a
+ * {@link DocIdSet} that contains them.
+ */
+ protected static abstract class Docs {
+
+ /** Solr constructor. */
+ public Docs() {}
+
+ /** Record the given document. */
+ public abstract void addDoc(int docId) throws IOException;
+
+ /** Return the {@link DocIdSet} which contains all the recorded docs. */
+ public abstract DocIdSet getDocIdSet();
+ }
+
+ /**
* Holds the documents that were matched in the {@link AtomicReaderContext}.
* If scores were required, then {@code scores} is not null.
*/
@@ -66,7 +83,7 @@ public final class FacetsCollector exten
public final AtomicReaderContext context;
/** Which documents were seen. */
- public final FixedBitSet bits;
+ public final DocIdSet bits;
/** Non-sparse scores array. */
public final float[] scores;
@@ -75,7 +92,7 @@ public final class FacetsCollector exten
public final int totalHits;
/** Sole constructor. */
- public MatchingDocs(AtomicReaderContext context, FixedBitSet bits, int totalHits, float[] scores) {
+ public MatchingDocs(AtomicReaderContext context, DocIdSet bits, int totalHits, float[] scores) {
this.context = context;
this.bits = bits;
this.scores = scores;
@@ -93,9 +110,30 @@ public final class FacetsCollector exten
public FacetsCollector(boolean keepScores) {
this.keepScores = keepScores;
}
+
+ /**
+ * Creates a {@link Docs} to record hits. The default uses {@link FixedBitSet}
+ * to record hits and you can override to e.g. record the docs in your own
+ * {@link DocIdSet}.
+ */
+ protected Docs createDocs(final int maxDoc) {
+ return new Docs() {
+ private final FixedBitSet bits = new FixedBitSet(maxDoc);
+
+ @Override
+ public void addDoc(int docId) throws IOException {
+ bits.set(docId);
+ }
+
+ @Override
+ public DocIdSet getDocIdSet() {
+ return bits;
+ }
+ };
+ }
/** True if scores were saved. */
- public boolean getKeepScores() {
+ public final boolean getKeepScores() {
return keepScores;
}
@@ -104,9 +142,9 @@ public final class FacetsCollector exten
* visited segment.
*/
public List<MatchingDocs> getMatchingDocs() {
- if (bits != null) {
- matchingDocs.add(new MatchingDocs(this.context, bits, totalHits, scores));
- bits = null;
+ if (docs != null) {
+ matchingDocs.add(new MatchingDocs(this.context, docs.getDocIdSet(), totalHits, scores));
+ docs = null;
scores = null;
context = null;
}
@@ -124,7 +162,7 @@ public final class FacetsCollector exten
@Override
public final void collect(int doc) throws IOException {
- bits.set(doc);
+ docs.addDoc(doc);
if (keepScores) {
if (totalHits >= scores.length) {
float[] newScores = new float[ArrayUtil.oversize(totalHits + 1, 4)];
@@ -143,10 +181,10 @@ public final class FacetsCollector exten
@Override
public final void setNextReader(AtomicReaderContext context) throws IOException {
- if (bits != null) {
- matchingDocs.add(new MatchingDocs(this.context, bits, totalHits, scores));
+ if (docs != null) {
+ matchingDocs.add(new MatchingDocs(this.context, docs.getDocIdSet(), totalHits, scores));
}
- bits = new FixedBitSet(context.reader().maxDoc());
+ docs = createDocs(context.reader().maxDoc());
totalHits = 0;
if (keepScores) {
scores = new float[64]; // some initial size
Modified: lucene/dev/branches/lucene5376/lucene/facet/src/java/org/apache/lucene/facet/range/DoubleRangeFacetCounts.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/facet/src/java/org/apache/lucene/facet/range/DoubleRangeFacetCounts.java?rev=1565184&r1=1565183&r2=1565184&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/facet/src/java/org/apache/lucene/facet/range/DoubleRangeFacetCounts.java (original)
+++ lucene/dev/branches/lucene5376/lucene/facet/src/java/org/apache/lucene/facet/range/DoubleRangeFacetCounts.java Thu Feb 6 11:38:30 2014
@@ -33,6 +33,7 @@ import org.apache.lucene.queries.functio
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.Filter;
import org.apache.lucene.util.Bits;
+import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.util.NumericUtils;
/** {@link Facets} implementation that computes counts for
@@ -93,8 +94,7 @@ public class DoubleRangeFacetCounts exte
int missingCount = 0;
for (MatchingDocs hits : matchingDocs) {
FunctionValues fv = valueSource.getValues(Collections.emptyMap(), hits.context);
- final int length = hits.bits.length();
- int doc = 0;
+
totCount += hits.totalHits;
Bits bits;
if (fastMatchFilter != null) {
@@ -111,7 +111,10 @@ public class DoubleRangeFacetCounts exte
bits = null;
}
- while (doc < length && (doc = hits.bits.nextSetBit(doc)) != -1) {
+ DocIdSetIterator docs = hits.bits.iterator();
+
+ int doc;
+ while ((doc = docs.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
if (bits != null && bits.get(doc) == false) {
doc++;
continue;
@@ -122,7 +125,6 @@ public class DoubleRangeFacetCounts exte
} else {
missingCount++;
}
- doc++;
}
}
Modified: lucene/dev/branches/lucene5376/lucene/facet/src/java/org/apache/lucene/facet/range/LongRangeFacetCounts.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/facet/src/java/org/apache/lucene/facet/range/LongRangeFacetCounts.java?rev=1565184&r1=1565183&r2=1565184&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/facet/src/java/org/apache/lucene/facet/range/LongRangeFacetCounts.java (original)
+++ lucene/dev/branches/lucene5376/lucene/facet/src/java/org/apache/lucene/facet/range/LongRangeFacetCounts.java Thu Feb 6 11:38:30 2014
@@ -30,6 +30,7 @@ import org.apache.lucene.queries.functio
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.Filter;
import org.apache.lucene.util.Bits;
+import org.apache.lucene.search.DocIdSetIterator;
/** {@link Facets} implementation that computes counts for
* dynamic long ranges from a provided {@link ValueSource},
@@ -74,8 +75,7 @@ public class LongRangeFacetCounts extend
int missingCount = 0;
for (MatchingDocs hits : matchingDocs) {
FunctionValues fv = valueSource.getValues(Collections.emptyMap(), hits.context);
- final int length = hits.bits.length();
- int doc = 0;
+
totCount += hits.totalHits;
Bits bits;
if (fastMatchFilter != null) {
@@ -92,7 +92,9 @@ public class LongRangeFacetCounts extend
bits = null;
}
- while (doc < length && (doc = hits.bits.nextSetBit(doc)) != -1) {
+ DocIdSetIterator docs = hits.bits.iterator();
+ int doc;
+ while ((doc = docs.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
if (bits != null && bits.get(doc) == false) {
doc++;
continue;
@@ -103,8 +105,6 @@ public class LongRangeFacetCounts extend
} else {
missingCount++;
}
-
- doc++;
}
}
Modified: lucene/dev/branches/lucene5376/lucene/facet/src/java/org/apache/lucene/facet/sortedset/SortedSetDocValuesFacetCounts.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/facet/src/java/org/apache/lucene/facet/sortedset/SortedSetDocValuesFacetCounts.java?rev=1565184&r1=1565183&r2=1565184&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/facet/src/java/org/apache/lucene/facet/sortedset/SortedSetDocValuesFacetCounts.java (original)
+++ lucene/dev/branches/lucene5376/lucene/facet/src/java/org/apache/lucene/facet/sortedset/SortedSetDocValuesFacetCounts.java Thu Feb 6 11:38:30 2014
@@ -38,6 +38,7 @@ import org.apache.lucene.index.MultiDocV
import org.apache.lucene.index.MultiDocValues.MultiSortedSetDocValues;
import org.apache.lucene.index.ReaderUtil;
import org.apache.lucene.index.SortedSetDocValues;
+import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.util.BytesRef;
/** Compute facets counts from previously
@@ -70,8 +71,8 @@ public class SortedSetDocValuesFacetCoun
throws IOException {
this.state = state;
this.field = state.getField();
+ dv = state.getDocValues();
counts = new int[state.getSize()];
- dv = state.getDocValues();
//System.out.println("field=" + field);
count(hits.getMatchingDocs());
}
@@ -157,6 +158,8 @@ public class SortedSetDocValuesFacetCoun
} else {
ordinalMap = null;
}
+
+ IndexReader origReader = state.getOrigReader();
for(MatchingDocs hits : matchingDocs) {
@@ -166,7 +169,7 @@ public class SortedSetDocValuesFacetCoun
// the top-level reader passed to the
// SortedSetDocValuesReaderState, else cryptic
// AIOOBE can happen:
- if (ReaderUtil.getTopLevelContext(hits.context).reader() != state.origReader) {
+ if (ReaderUtil.getTopLevelContext(hits.context).reader() != origReader) {
throw new IllegalStateException("the SortedSetDocValuesReaderState provided to this class does not match the reader being searched; you must create a new SortedSetDocValuesReaderState every time you open a new IndexReader");
}
@@ -175,9 +178,7 @@ public class SortedSetDocValuesFacetCoun
continue;
}
- final int maxDoc = reader.maxDoc();
- assert maxDoc == hits.bits.length();
- //System.out.println(" dv=" + dv);
+ DocIdSetIterator docs = hits.bits.iterator();
// TODO: yet another option is to count all segs
// first, only in seg-ord space, and then do a
@@ -196,8 +197,8 @@ public class SortedSetDocValuesFacetCoun
if (hits.totalHits < numSegOrds/10) {
//System.out.println(" remap as-we-go");
// Remap every ord to global ord as we iterate:
- int doc = 0;
- while (doc < maxDoc && (doc = hits.bits.nextSetBit(doc)) != -1) {
+ int doc;
+ while ((doc = docs.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
//System.out.println(" doc=" + doc);
segValues.setDocument(doc);
int term = (int) segValues.nextOrd();
@@ -206,15 +207,14 @@ public class SortedSetDocValuesFacetCoun
counts[(int) ordinalMap.getGlobalOrd(segOrd, term)]++;
term = (int) segValues.nextOrd();
}
- ++doc;
}
} else {
//System.out.println(" count in seg ord first");
// First count in seg-ord space:
final int[] segCounts = new int[numSegOrds];
- int doc = 0;
- while (doc < maxDoc && (doc = hits.bits.nextSetBit(doc)) != -1) {
+ int doc;
+ while ((doc = docs.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
//System.out.println(" doc=" + doc);
segValues.setDocument(doc);
int term = (int) segValues.nextOrd();
@@ -223,7 +223,6 @@ public class SortedSetDocValuesFacetCoun
segCounts[term]++;
term = (int) segValues.nextOrd();
}
- ++doc;
}
// Then, migrate to global ords:
@@ -238,16 +237,14 @@ public class SortedSetDocValuesFacetCoun
} else {
// No ord mapping (e.g., single segment index):
// just aggregate directly into counts:
-
- int doc = 0;
- while (doc < maxDoc && (doc = hits.bits.nextSetBit(doc)) != -1) {
+ int doc;
+ while ((doc = docs.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
segValues.setDocument(doc);
int term = (int) segValues.nextOrd();
while (term != SortedSetDocValues.NO_MORE_ORDS) {
counts[term]++;
term = (int) segValues.nextOrd();
}
- ++doc;
}
}
}
Modified: lucene/dev/branches/lucene5376/lucene/facet/src/java/org/apache/lucene/facet/sortedset/SortedSetDocValuesReaderState.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/facet/src/java/org/apache/lucene/facet/sortedset/SortedSetDocValuesReaderState.java?rev=1565184&r1=1565183&r2=1565184&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/facet/src/java/org/apache/lucene/facet/sortedset/SortedSetDocValuesReaderState.java (original)
+++ lucene/dev/branches/lucene5376/lucene/facet/src/java/org/apache/lucene/facet/sortedset/SortedSetDocValuesReaderState.java Thu Feb 6 11:38:30 2014
@@ -18,16 +18,10 @@ package org.apache.lucene.facet.sortedse
*/
import java.io.IOException;
-import java.util.Arrays;
-import java.util.HashMap;
import java.util.Map;
-import org.apache.lucene.facet.FacetsConfig;
-import org.apache.lucene.index.AtomicReader;
import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.SlowCompositeReaderWrapper;
import org.apache.lucene.index.SortedSetDocValues;
-import org.apache.lucene.util.BytesRef;
/** Wraps a {@link IndexReader} and resolves ords
* using existing {@link SortedSetDocValues} APIs without a
@@ -45,14 +39,7 @@ import org.apache.lucene.util.BytesRef;
* so you should create it once and re-use that one instance
* for a given {@link IndexReader}. */
-public final class SortedSetDocValuesReaderState {
-
- private final String field;
- private final AtomicReader topReader;
- private final int valueCount;
-
- /** {@link IndexReader} passed to the constructor. */
- public final IndexReader origReader;
+public abstract class SortedSetDocValuesReaderState {
/** Holds start/end range of ords, which maps to one
* dimension (someday we may generalize it to map to
@@ -70,86 +57,25 @@ public final class SortedSetDocValuesRea
}
}
- private final Map<String,OrdRange> prefixToOrdRange = new HashMap<String,OrdRange>();
-
- /** Creates this, pulling doc values from the default {@link
- * FacetsConfig#DEFAULT_INDEX_FIELD_NAME}. */
- public SortedSetDocValuesReaderState(IndexReader reader) throws IOException {
- this(reader, FacetsConfig.DEFAULT_INDEX_FIELD_NAME);
+ /** Sole constructor. */
+ protected SortedSetDocValuesReaderState() {
}
-
- /** Creates this, pulling doc values from the specified
- * field. */
- public SortedSetDocValuesReaderState(IndexReader reader, String field) throws IOException {
-
- this.field = field;
- this.origReader = reader;
-
- // We need this to create thread-safe MultiSortedSetDV
- // per collector:
- topReader = SlowCompositeReaderWrapper.wrap(reader);
- SortedSetDocValues dv = topReader.getSortedSetDocValues(field);
- if (dv == null) {
- throw new IllegalArgumentException("field \"" + field + "\" was not indexed with SortedSetDocValues");
- }
- if (dv.getValueCount() > Integer.MAX_VALUE) {
- throw new IllegalArgumentException("can only handle valueCount < Integer.MAX_VALUE; got " + dv.getValueCount());
- }
- valueCount = (int) dv.getValueCount();
-
- // TODO: we can make this more efficient if eg we can be
- // "involved" when OrdinalMap is being created? Ie see
- // each term/ord it's assigning as it goes...
- String lastDim = null;
- int startOrd = -1;
- BytesRef spare = new BytesRef();
-
- // TODO: this approach can work for full hierarchy?;
- // TaxoReader can't do this since ords are not in
- // "sorted order" ... but we should generalize this to
- // support arbitrary hierarchy:
- for(int ord=0;ord<valueCount;ord++) {
- dv.lookupOrd(ord, spare);
- String[] components = FacetsConfig.stringToPath(spare.utf8ToString());
- if (components.length != 2) {
- throw new IllegalArgumentException("this class can only handle 2 level hierarchy (dim/value); got: " + Arrays.toString(components) + " " + spare.utf8ToString());
- }
- if (!components[0].equals(lastDim)) {
- if (lastDim != null) {
- prefixToOrdRange.put(lastDim, new OrdRange(startOrd, ord-1));
- }
- startOrd = ord;
- lastDim = components[0];
- }
- }
-
- if (lastDim != null) {
- prefixToOrdRange.put(lastDim, new OrdRange(startOrd, valueCount-1));
- }
- }
-
+
/** Return top-level doc values. */
- public SortedSetDocValues getDocValues() throws IOException {
- return topReader.getSortedSetDocValues(field);
- }
-
- /** Returns mapping from prefix to {@link OrdRange}. */
- public Map<String,OrdRange> getPrefixToOrdRange() {
- return prefixToOrdRange;
- }
-
- /** Returns the {@link OrdRange} for this dimension. */
- public OrdRange getOrdRange(String dim) {
- return prefixToOrdRange.get(dim);
- }
-
+ public abstract SortedSetDocValues getDocValues() throws IOException;
+
/** Indexed field we are reading. */
- public String getField() {
- return field;
- }
-
+ public abstract String getField();
+
+ /** Returns the {@link OrdRange} for this dimension. */
+ public abstract OrdRange getOrdRange(String dim);
+
+ /** Returns mapping from prefix to {@link OrdRange}. */
+ public abstract Map<String,OrdRange> getPrefixToOrdRange();
+
+ /** Returns top-level index reader. */
+ public abstract IndexReader getOrigReader();
+
/** Number of unique labels. */
- public int getSize() {
- return valueCount;
- }
+ public abstract int getSize();
}
Modified: lucene/dev/branches/lucene5376/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/FastTaxonomyFacetCounts.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/FastTaxonomyFacetCounts.java?rev=1565184&r1=1565183&r2=1565184&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/FastTaxonomyFacetCounts.java (original)
+++ lucene/dev/branches/lucene5376/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/FastTaxonomyFacetCounts.java Thu Feb 6 11:38:30 2014
@@ -21,11 +21,11 @@ import java.io.IOException;
import java.util.List;
import org.apache.lucene.facet.FacetsCollector;
-import org.apache.lucene.facet.FacetsConfig;
import org.apache.lucene.facet.FacetsCollector.MatchingDocs;
+import org.apache.lucene.facet.FacetsConfig;
import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.FixedBitSet;
/** Computes facets counts, assuming the default encoding
* into DocValues was used.
@@ -55,12 +55,12 @@ public class FastTaxonomyFacetCounts ext
if (dv == null) { // this reader does not have DocValues for the requested category list
continue;
}
- FixedBitSet bits = hits.bits;
-
- final int length = hits.bits.length();
- int doc = 0;
+
BytesRef scratch = new BytesRef();
- while (doc < length && (doc = bits.nextSetBit(doc)) != -1) {
+ DocIdSetIterator docs = hits.bits.iterator();
+
+ int doc;
+ while ((doc = docs.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
dv.get(doc, scratch);
byte[] bytes = scratch.bytes;
int end = scratch.offset + scratch.length;
@@ -77,7 +77,6 @@ public class FastTaxonomyFacetCounts ext
ord = (ord << 7) | (b & 0x7F);
}
}
- ++doc;
}
}
Modified: lucene/dev/branches/lucene5376/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetCounts.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetCounts.java?rev=1565184&r1=1565183&r2=1565184&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetCounts.java (original)
+++ lucene/dev/branches/lucene5376/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetCounts.java Thu Feb 6 11:38:30 2014
@@ -24,7 +24,7 @@ import org.apache.lucene.facet.FacetsCol
import org.apache.lucene.facet.FacetsCollector.MatchingDocs;
import org.apache.lucene.facet.FacetsConfig;
import org.apache.lucene.index.BinaryDocValues;
-import org.apache.lucene.util.FixedBitSet;
+import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.util.IntsRef;
/** Reads from any {@link OrdinalsReader}; use {@link
@@ -49,16 +49,14 @@ public class TaxonomyFacetCounts extends
IntsRef scratch = new IntsRef();
for(MatchingDocs hits : matchingDocs) {
OrdinalsReader.OrdinalsSegmentReader ords = ordinalsReader.getReader(hits.context);
- FixedBitSet bits = hits.bits;
-
- final int length = hits.bits.length();
- int doc = 0;
- while (doc < length && (doc = bits.nextSetBit(doc)) != -1) {
+ DocIdSetIterator docs = hits.bits.iterator();
+
+ int doc;
+ while ((doc = docs.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
ords.get(doc, scratch);
for(int i=0;i<scratch.length;i++) {
values[scratch.ints[scratch.offset+i]]++;
}
- ++doc;
}
}
Modified: lucene/dev/branches/lucene5376/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetSumFloatAssociations.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetSumFloatAssociations.java?rev=1565184&r1=1565183&r2=1565184&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetSumFloatAssociations.java (original)
+++ lucene/dev/branches/lucene5376/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetSumFloatAssociations.java Thu Feb 6 11:38:30 2014
@@ -21,11 +21,11 @@ import java.io.IOException;
import java.util.List;
import org.apache.lucene.facet.FacetsCollector;
-import org.apache.lucene.facet.FacetsConfig;
import org.apache.lucene.facet.FacetsCollector.MatchingDocs;
+import org.apache.lucene.facet.FacetsConfig;
import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.FixedBitSet;
/** Aggregates sum of int values previously indexed with
* {@link FloatAssociationFacetField}, assuming the default
@@ -54,13 +54,12 @@ public class TaxonomyFacetSumFloatAssoci
if (dv == null) { // this reader does not have DocValues for the requested category list
continue;
}
- FixedBitSet bits = hits.bits;
- final int length = hits.bits.length();
- int doc = 0;
BytesRef scratch = new BytesRef();
- //System.out.println("count seg=" + hits.context.reader());
- while (doc < length && (doc = bits.nextSetBit(doc)) != -1) {
+ DocIdSetIterator docs = hits.bits.iterator();
+
+ int doc;
+ while ((doc = docs.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
//System.out.println(" doc=" + doc);
// TODO: use OrdinalsReader? we'd need to add a
// BytesRef getAssociation()?
@@ -81,7 +80,6 @@ public class TaxonomyFacetSumFloatAssoci
offset += 4;
values[ord] += Float.intBitsToFloat(value);
}
- ++doc;
}
}
Modified: lucene/dev/branches/lucene5376/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetSumIntAssociations.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetSumIntAssociations.java?rev=1565184&r1=1565183&r2=1565184&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetSumIntAssociations.java (original)
+++ lucene/dev/branches/lucene5376/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetSumIntAssociations.java Thu Feb 6 11:38:30 2014
@@ -21,11 +21,11 @@ import java.io.IOException;
import java.util.List;
import org.apache.lucene.facet.FacetsCollector;
-import org.apache.lucene.facet.FacetsConfig;
import org.apache.lucene.facet.FacetsCollector.MatchingDocs;
+import org.apache.lucene.facet.FacetsConfig;
import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.FixedBitSet;
/** Aggregates sum of int values previously indexed with
* {@link IntAssociationFacetField}, assuming the default
@@ -54,13 +54,12 @@ public class TaxonomyFacetSumIntAssociat
if (dv == null) { // this reader does not have DocValues for the requested category list
continue;
}
- FixedBitSet bits = hits.bits;
-
- final int length = hits.bits.length();
- int doc = 0;
+
BytesRef scratch = new BytesRef();
- //System.out.println("count seg=" + hits.context.reader());
- while (doc < length && (doc = bits.nextSetBit(doc)) != -1) {
+ DocIdSetIterator docs = hits.bits.iterator();
+
+ int doc;
+ while ((doc = docs.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
//System.out.println(" doc=" + doc);
// TODO: use OrdinalsReader? we'd need to add a
// BytesRef getAssociation()?
@@ -81,7 +80,6 @@ public class TaxonomyFacetSumIntAssociat
offset += 4;
values[ord] += value;
}
- ++doc;
}
}
Modified: lucene/dev/branches/lucene5376/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetSumValueSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetSumValueSource.java?rev=1565184&r1=1565183&r2=1565184&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetSumValueSource.java (original)
+++ lucene/dev/branches/lucene5376/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetSumValueSource.java Thu Feb 6 11:38:30 2014
@@ -29,8 +29,8 @@ import org.apache.lucene.index.AtomicRea
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.docvalues.DoubleDocValues;
+import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.Scorer;
-import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.IntsRef;
/** Aggregates sum of values from {@link
@@ -81,15 +81,15 @@ public class TaxonomyFacetSumValueSource
IntsRef scratch = new IntsRef();
for(MatchingDocs hits : matchingDocs) {
OrdinalsReader.OrdinalsSegmentReader ords = ordinalsReader.getReader(hits.context);
- FixedBitSet bits = hits.bits;
-
- final int length = hits.bits.length();
- int doc = 0;
+
int scoresIdx = 0;
float[] scores = hits.scores;
FunctionValues functionValues = valueSource.getValues(context, hits.context);
- while (doc < length && (doc = bits.nextSetBit(doc)) != -1) {
+ DocIdSetIterator docs = hits.bits.iterator();
+
+ int doc;
+ while ((doc = docs.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
ords.get(doc, scratch);
if (keepScores) {
scorer.docID = doc;
@@ -99,7 +99,6 @@ public class TaxonomyFacetSumValueSource
for(int i=0;i<scratch.length;i++) {
values[scratch.ints[i]] += value;
}
- ++doc;
}
}
Modified: lucene/dev/branches/lucene5376/lucene/facet/src/test/org/apache/lucene/facet/TestDrillSideways.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/facet/src/test/org/apache/lucene/facet/TestDrillSideways.java?rev=1565184&r1=1565183&r2=1565184&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/facet/src/test/org/apache/lucene/facet/TestDrillSideways.java (original)
+++ lucene/dev/branches/lucene5376/lucene/facet/src/test/org/apache/lucene/facet/TestDrillSideways.java Thu Feb 6 11:38:30 2014
@@ -32,6 +32,7 @@ import org.apache.lucene.document.Docume
import org.apache.lucene.document.Field;
import org.apache.lucene.document.StringField;
import org.apache.lucene.facet.DrillSideways.DrillSidewaysResult;
+import org.apache.lucene.facet.sortedset.DefaultSortedSetDocValuesReaderState;
import org.apache.lucene.facet.sortedset.SortedSetDocValuesFacetField;
import org.apache.lucene.facet.sortedset.SortedSetDocValuesReaderState;
import org.apache.lucene.facet.taxonomy.TaxonomyReader;
@@ -552,7 +553,7 @@ public class TestDrillSideways extends F
IndexSearcher s = newSearcher(r);
if (doUseDV) {
- sortedSetDVState = new SortedSetDocValuesReaderState(s.getIndexReader());
+ sortedSetDVState = new DefaultSortedSetDocValuesReaderState(s.getIndexReader());
} else {
sortedSetDVState = null;
}
Modified: lucene/dev/branches/lucene5376/lucene/facet/src/test/org/apache/lucene/facet/range/TestRangeFacetCounts.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/facet/src/test/org/apache/lucene/facet/range/TestRangeFacetCounts.java?rev=1565184&r1=1565183&r2=1565184&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/facet/src/test/org/apache/lucene/facet/range/TestRangeFacetCounts.java (original)
+++ lucene/dev/branches/lucene5376/lucene/facet/src/test/org/apache/lucene/facet/range/TestRangeFacetCounts.java Thu Feb 6 11:38:30 2014
@@ -20,6 +20,7 @@ package org.apache.lucene.facet.range;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.DoubleDocValuesField;
@@ -911,6 +912,7 @@ public class TestRangeFacetCounts extend
new DoubleRange("< 50", 0.0, true, 50.0, false)};
Filter fastMatchFilter;
+ final AtomicBoolean filterWasUsed = new AtomicBoolean();
if (random().nextBoolean()) {
// Sort of silly:
fastMatchFilter = new CachingWrapperFilter(new QueryWrapperFilter(new MatchAllDocsQuery())) {
@@ -918,6 +920,7 @@ public class TestRangeFacetCounts extend
protected DocIdSet cacheImpl(DocIdSetIterator iterator, AtomicReader reader)
throws IOException {
final FixedBitSet cached = new FixedBitSet(reader.maxDoc());
+ filterWasUsed.set(true);
cached.or(iterator);
return cached;
}
@@ -935,6 +938,8 @@ public class TestRangeFacetCounts extend
// Test simple drill-down:
assertEquals(1, s.search(ddq, 10).totalHits);
+ assertTrue(fastMatchFilter == null || filterWasUsed.get());
+ filterWasUsed.set(false);
// Test drill-sideways after drill-down
DrillSideways ds = new DrillSideways(s, config, (TaxonomyReader) null) {
@@ -956,6 +961,7 @@ public class TestRangeFacetCounts extend
assertEquals(1, dsr.hits.totalHits);
assertEquals("dim=field path=[] value=3 childCount=6\n < 1 (0)\n < 2 (1)\n < 5 (3)\n < 10 (3)\n < 20 (3)\n < 50 (3)\n",
dsr.facets.getTopChildren(10, "field").toString());
+ assertTrue(fastMatchFilter == null || filterWasUsed.get());
IOUtils.close(r, writer, dir);
}
Modified: lucene/dev/branches/lucene5376/lucene/facet/src/test/org/apache/lucene/facet/sortedset/TestSortedSetDocValuesFacets.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/facet/src/test/org/apache/lucene/facet/sortedset/TestSortedSetDocValuesFacets.java?rev=1565184&r1=1565183&r2=1565184&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/facet/src/test/org/apache/lucene/facet/sortedset/TestSortedSetDocValuesFacets.java (original)
+++ lucene/dev/branches/lucene5376/lucene/facet/src/test/org/apache/lucene/facet/sortedset/TestSortedSetDocValuesFacets.java Thu Feb 6 11:38:30 2014
@@ -74,7 +74,7 @@ public class TestSortedSetDocValuesFacet
IndexSearcher searcher = newSearcher(writer.getReader());
// Per-top-reader state:
- SortedSetDocValuesReaderState state = new SortedSetDocValuesReaderState(searcher.getIndexReader());
+ SortedSetDocValuesReaderState state = new DefaultSortedSetDocValuesReaderState(searcher.getIndexReader());
FacetsCollector c = new FacetsCollector();
@@ -110,7 +110,7 @@ public class TestSortedSetDocValuesFacet
writer.addDocument(config.build(doc));
IndexReader r = writer.getReader();
- SortedSetDocValuesReaderState state = new SortedSetDocValuesReaderState(r);
+ SortedSetDocValuesReaderState state = new DefaultSortedSetDocValuesReaderState(r);
doc = new Document();
doc.add(new SortedSetDocValuesFacetField("a", "bar"));
@@ -176,7 +176,7 @@ public class TestSortedSetDocValuesFacet
writer.close();
// Per-top-reader state:
- SortedSetDocValuesReaderState state = new SortedSetDocValuesReaderState(searcher.getIndexReader());
+ SortedSetDocValuesReaderState state = new DefaultSortedSetDocValuesReaderState(searcher.getIndexReader());
FacetsCollector c = new FacetsCollector();
searcher.search(new MatchAllDocsQuery(), c);
@@ -221,7 +221,7 @@ public class TestSortedSetDocValuesFacet
writer.close();
// Per-top-reader state:
- SortedSetDocValuesReaderState state = new SortedSetDocValuesReaderState(searcher.getIndexReader());
+ SortedSetDocValuesReaderState state = new DefaultSortedSetDocValuesReaderState(searcher.getIndexReader());
FacetsCollector c = new FacetsCollector();
searcher.search(new MatchAllDocsQuery(), c);
@@ -256,7 +256,7 @@ public class TestSortedSetDocValuesFacet
IndexSearcher searcher = new IndexSearcher(SlowCompositeReaderWrapper.wrap(writer.getReader()));
// Per-top-reader state:
- SortedSetDocValuesReaderState state = new SortedSetDocValuesReaderState(searcher.getIndexReader());
+ SortedSetDocValuesReaderState state = new DefaultSortedSetDocValuesReaderState(searcher.getIndexReader());
FacetsCollector c = new FacetsCollector();
searcher.search(new MatchAllDocsQuery(), c);
@@ -295,7 +295,7 @@ public class TestSortedSetDocValuesFacet
IndexSearcher searcher = newSearcher(w.getReader());
// Per-top-reader state:
- SortedSetDocValuesReaderState state = new SortedSetDocValuesReaderState(searcher.getIndexReader());
+ SortedSetDocValuesReaderState state = new DefaultSortedSetDocValuesReaderState(searcher.getIndexReader());
int iters = atLeast(100);
for(int iter=0;iter<iters;iter++) {
Modified: lucene/dev/branches/lucene5376/lucene/queryparser/src/java/org/apache/lucene/queryparser/simple/SimpleQueryParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/queryparser/src/java/org/apache/lucene/queryparser/simple/SimpleQueryParser.java?rev=1565184&r1=1565183&r2=1565184&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/queryparser/src/java/org/apache/lucene/queryparser/simple/SimpleQueryParser.java (original)
+++ lucene/dev/branches/lucene5376/lucene/queryparser/src/java/org/apache/lucene/queryparser/simple/SimpleQueryParser.java Thu Feb 6 11:38:30 2014
@@ -21,10 +21,12 @@ import org.apache.lucene.analysis.Analyz
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.FuzzyQuery;
import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.PrefixQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.util.QueryBuilder;
+import org.apache.lucene.util.automaton.LevenshteinAutomata;
import java.util.Collections;
import java.util.Map;
@@ -48,6 +50,8 @@ import java.util.Map;
* <li>'{@code -}' negates a single token: <tt>-token0</tt>
* <li>'{@code "}' creates phrases of terms: <tt>"term1 term2 ..."</tt>
* <li>'{@code *}' at the end of terms specifies prefix query: <tt>term*</tt>
+ * <li>'{@code ~}N' at the end of terms specifies fuzzy query: <tt>term~1</tt>
+ * <li>'{@code ~}N' at the end of phrases specifies near query: <tt>"term1 term2"~5</tt>
* <li>'{@code (}' and '{@code )}' specifies precedence: <tt>token1 + (token2 | token3)</tt>
* </ul>
* <p>
@@ -111,6 +115,11 @@ public class SimpleQueryParser extends Q
public static final int ESCAPE_OPERATOR = 1<<6;
/** Enables {@code WHITESPACE} operators: ' ' '\n' '\r' '\t' */
public static final int WHITESPACE_OPERATOR = 1<<7;
+ /** Enables {@code FUZZY} operators: (~) on single terms */
+ public static final int FUZZY_OPERATOR = 1<<8;
+ /** Enables {@code NEAR} operators: (~) on phrases */
+ public static final int NEAR_OPERATOR = 1<<9;
+
private BooleanClause.Occur defaultOperator = BooleanClause.Occur.SHOULD;
@@ -266,6 +275,7 @@ public class SimpleQueryParser extends Q
int start = ++state.index;
int copied = 0;
boolean escaped = false;
+ boolean hasSlop = false;
while (state.index < state.length) {
if (!escaped) {
@@ -279,10 +289,23 @@ public class SimpleQueryParser extends Q
continue;
} else if (state.data[state.index] == '"') {
- // this should be the end of the phrase
- // all characters found will used for
- // creating the phrase query
- break;
+ // if there are still characters after the closing ", check for a
+ // tilde
+ if (state.length > (state.index + 1) &&
+ state.data[state.index+1] == '~' &&
+ (flags & NEAR_OPERATOR) != 0) {
+ state.index++;
+ // check for characters after the tilde
+ if (state.length > (state.index + 1)) {
+ hasSlop = true;
+ }
+ break;
+ } else {
+ // this should be the end of the phrase
+ // all characters found will used for
+ // creating the phrase query
+ break;
+ }
}
}
@@ -305,7 +328,12 @@ public class SimpleQueryParser extends Q
// a complete phrase has been found and is parsed through
// through the analyzer from the given field
String phrase = new String(state.buffer, 0, copied);
- Query branch = newPhraseQuery(phrase);
+ Query branch;
+ if (hasSlop) {
+ branch = newPhraseQuery(phrase, parseFuzziness(state));
+ } else {
+ branch = newPhraseQuery(phrase, 0);
+ }
buildQueryTree(state, branch);
++state.index;
@@ -316,6 +344,7 @@ public class SimpleQueryParser extends Q
int copied = 0;
boolean escaped = false;
boolean prefix = false;
+ boolean fuzzy = false;
while (state.index < state.length) {
if (!escaped) {
@@ -329,19 +358,14 @@ public class SimpleQueryParser extends Q
++state.index;
continue;
- } else if ((state.data[state.index] == '"' && (flags & PHRASE_OPERATOR) != 0)
- || (state.data[state.index] == '|' && (flags & OR_OPERATOR) != 0)
- || (state.data[state.index] == '+' && (flags & AND_OPERATOR) != 0)
- || (state.data[state.index] == '(' && (flags & PRECEDENCE_OPERATORS) != 0)
- || (state.data[state.index] == ')' && (flags & PRECEDENCE_OPERATORS) != 0)
- || ((state.data[state.index] == ' '
- || state.data[state.index] == '\t'
- || state.data[state.index] == '\n'
- || state.data[state.index] == '\r') && (flags & WHITESPACE_OPERATOR) != 0)) {
+ } else if (tokenFinished(state)) {
// this should be the end of the term
// all characters found will used for
// creating the term query
break;
+ } else if (copied > 0 && state.data[state.index] == '~' && (flags & FUZZY_OPERATOR) != 0) {
+ fuzzy = true;
+ break;
}
// wildcard tracks whether or not the last character
@@ -358,7 +382,17 @@ public class SimpleQueryParser extends Q
if (copied > 0) {
final Query branch;
- if (prefix) {
+ if (fuzzy && (flags & FUZZY_OPERATOR) != 0) {
+ String token = new String(state.buffer, 0, copied);
+ int fuzziness = parseFuzziness(state);
+ // edit distance has a maximum, limit to the maximum supported
+ fuzziness = Math.min(fuzziness, LevenshteinAutomata.MAXIMUM_SUPPORTED_DISTANCE);
+ if (fuzziness == 0) {
+ branch = newDefaultQuery(token);
+ } else {
+ branch = newFuzzyQuery(token, fuzziness);
+ }
+ } else if (prefix) {
// if a term is found with a closing '*' it is considered to be a prefix query
// and will have prefix added as an option
String token = new String(state.buffer, 0, copied - 1);
@@ -421,6 +455,60 @@ public class SimpleQueryParser extends Q
}
/**
+ * Helper parsing fuzziness from parsing state
+ * @return slop/edit distance, 0 in the case of non-parsing slop/edit string
+ */
+ private int parseFuzziness(State state) {
+ char slopText[] = new char[state.length];
+ int slopLength = 0;
+
+ if (state.data[state.index] == '~') {
+ while (state.index < state.length) {
+ state.index++;
+ // it's possible that the ~ was at the end, so check after incrementing
+ // to make sure we don't go out of bounds
+ if (state.index < state.length) {
+ if (tokenFinished(state)) {
+ break;
+ }
+ slopText[slopLength] = state.data[state.index];
+ slopLength++;
+ }
+ }
+ int fuzziness = 0;
+ try {
+ fuzziness = Integer.parseInt(new String(slopText, 0, slopLength));
+ } catch (NumberFormatException e) {
+ // swallow number format exceptions parsing fuzziness
+ }
+ // negative -> 0
+ if (fuzziness < 0) {
+ fuzziness = 0;
+ }
+ return fuzziness;
+ }
+ return 0;
+ }
+
+ /**
+ * Helper returning true if the state has reached the end of token.
+ */
+ private boolean tokenFinished(State state) {
+ if ((state.data[state.index] == '"' && (flags & PHRASE_OPERATOR) != 0)
+ || (state.data[state.index] == '|' && (flags & OR_OPERATOR) != 0)
+ || (state.data[state.index] == '+' && (flags & AND_OPERATOR) != 0)
+ || (state.data[state.index] == '(' && (flags & PRECEDENCE_OPERATORS) != 0)
+ || (state.data[state.index] == ')' && (flags & PRECEDENCE_OPERATORS) != 0)
+ || ((state.data[state.index] == ' '
+ || state.data[state.index] == '\t'
+ || state.data[state.index] == '\n'
+ || state.data[state.index] == '\r') && (flags & WHITESPACE_OPERATOR) != 0)) {
+ return true;
+ }
+ return false;
+ }
+
+ /**
* Factory method to generate a standard query (no phrase or prefix operators).
*/
protected Query newDefaultQuery(String text) {
@@ -436,12 +524,27 @@ public class SimpleQueryParser extends Q
}
/**
- * Factory method to generate a phrase query.
+ * Factory method to generate a fuzzy query.
+ */
+ protected Query newFuzzyQuery(String text, int fuzziness) {
+ BooleanQuery bq = new BooleanQuery(true);
+ for (Map.Entry<String,Float> entry : weights.entrySet()) {
+ Query q = new FuzzyQuery(new Term(entry.getKey(), text), fuzziness);
+ if (q != null) {
+ q.setBoost(entry.getValue());
+ bq.add(q, BooleanClause.Occur.SHOULD);
+ }
+ }
+ return simplify(bq);
+ }
+
+ /**
+ * Factory method to generate a phrase query with slop.
*/
- protected Query newPhraseQuery(String text) {
+ protected Query newPhraseQuery(String text, int slop) {
BooleanQuery bq = new BooleanQuery(true);
for (Map.Entry<String,Float> entry : weights.entrySet()) {
- Query q = createPhraseQuery(entry.getKey(), text);
+ Query q = createPhraseQuery(entry.getKey(), text, slop);
if (q != null) {
q.setBoost(entry.getValue());
bq.add(q, BooleanClause.Occur.SHOULD);