You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2011/11/15 13:25:49 UTC
svn commit: r1202154 - in /lucene/dev/branches/lucene2621: ./
dev-tools/idea/lucene/contrib/ lucene/
lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/
lucene/contrib/sandbox/src/test/org/apache/lucene/sandbox/queries/regex/
lucene...
Author: rmuir
Date: Tue Nov 15 12:25:46 2011
New Revision: 1202154
URL: http://svn.apache.org/viewvc?rev=1202154&view=rev
Log:
merge trunk (1201712:1202151)
Added:
lucene/dev/branches/lucene2621/solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java
- copied unchanged from r1202151, lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java
Removed:
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/CachingSpanFilter.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/SpanFilter.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/SpanFilterResult.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/SpanQueryFilter.java
lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/search/TestCachingSpanFilter.java
lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/search/TestSpanQueryFilter.java
Modified:
lucene/dev/branches/lucene2621/ (props changed)
lucene/dev/branches/lucene2621/dev-tools/idea/lucene/contrib/ (props changed)
lucene/dev/branches/lucene2621/lucene/ (props changed)
lucene/dev/branches/lucene2621/lucene/CHANGES.txt
lucene/dev/branches/lucene2621/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java
lucene/dev/branches/lucene2621/lucene/contrib/sandbox/src/test/org/apache/lucene/sandbox/queries/regex/TestSpanRegexQuery.java (props changed)
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/TermInfosReaderIndex.java (props changed)
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/values/MultiIndexDocValues.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/payloads/PayloadNearQuery.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/payloads/PayloadSpanUtil.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/spans/FieldMaskingSpanQuery.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/spans/NearSpansOrdered.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/spans/NearSpansUnordered.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/spans/SpanMultiTermQueryWrapper.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/spans/SpanNearQuery.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/spans/SpanNotQuery.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/spans/SpanOrQuery.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/spans/SpanPositionCheckQuery.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/spans/SpanQuery.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/spans/SpanTermQuery.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/spans/SpanWeight.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/util/Constants.java
lucene/dev/branches/lucene2621/lucene/src/test-framework/java/org/apache/lucene/index/RandomIndexWriter.java
lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java
lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/search/JustCompileSearch.java
lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/search/spans/JustCompileSearchSpans.java
lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/search/spans/MultiSpansWrapper.java
lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/util/TestIOUtils.java
lucene/dev/branches/lucene2621/modules/queryparser/src/test/org/apache/lucene/queryparser/xml/builders/TestNumericRangeFilterBuilder.java (props changed)
lucene/dev/branches/lucene2621/solr/ (props changed)
lucene/dev/branches/lucene2621/solr/CHANGES.txt (props changed)
lucene/dev/branches/lucene2621/solr/LICENSE.txt (props changed)
lucene/dev/branches/lucene2621/solr/NOTICE.txt (props changed)
lucene/dev/branches/lucene2621/solr/README.txt (props changed)
lucene/dev/branches/lucene2621/solr/build.xml (props changed)
lucene/dev/branches/lucene2621/solr/client/ (props changed)
lucene/dev/branches/lucene2621/solr/common-build.xml (props changed)
lucene/dev/branches/lucene2621/solr/contrib/ (props changed)
lucene/dev/branches/lucene2621/solr/contrib/clustering/src/test-files/ (props changed)
lucene/dev/branches/lucene2621/solr/contrib/dataimporthandler-extras/src/java/ (props changed)
lucene/dev/branches/lucene2621/solr/contrib/dataimporthandler/src/java/ (props changed)
lucene/dev/branches/lucene2621/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHCache.java
lucene/dev/branches/lucene2621/solr/contrib/dataimporthandler/src/test-files/ (props changed)
lucene/dev/branches/lucene2621/solr/contrib/dataimporthandler/src/test/org/ (props changed)
lucene/dev/branches/lucene2621/solr/contrib/uima/src/java/ (props changed)
lucene/dev/branches/lucene2621/solr/contrib/uima/src/test-files/ (props changed)
lucene/dev/branches/lucene2621/solr/core/ (props changed)
lucene/dev/branches/lucene2621/solr/core/src/java/ (props changed)
lucene/dev/branches/lucene2621/solr/core/src/java/org/apache/solr/handler/component/FieldFacetStats.java
lucene/dev/branches/lucene2621/solr/core/src/java/org/apache/solr/handler/component/StatsComponent.java
lucene/dev/branches/lucene2621/solr/core/src/java/org/apache/solr/handler/component/StatsValues.java
lucene/dev/branches/lucene2621/solr/core/src/java/org/apache/solr/request/UnInvertedField.java
lucene/dev/branches/lucene2621/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java
lucene/dev/branches/lucene2621/solr/core/src/test/ (props changed)
lucene/dev/branches/lucene2621/solr/core/src/test/org/apache/solr/handler/component/StatsComponentTest.java
lucene/dev/branches/lucene2621/solr/dev-tools/ (props changed)
lucene/dev/branches/lucene2621/solr/example/ (props changed)
lucene/dev/branches/lucene2621/solr/lib/ (props changed)
lucene/dev/branches/lucene2621/solr/scripts/ (props changed)
lucene/dev/branches/lucene2621/solr/site/ (props changed)
lucene/dev/branches/lucene2621/solr/site-src/ (props changed)
lucene/dev/branches/lucene2621/solr/solrj/ (props changed)
lucene/dev/branches/lucene2621/solr/solrj/src/java/ (props changed)
lucene/dev/branches/lucene2621/solr/solrj/src/java/org/apache/solr/common/SolrInputDocument.java
lucene/dev/branches/lucene2621/solr/solrj/src/test/org/apache/solr/client/ (props changed)
lucene/dev/branches/lucene2621/solr/solrj/src/test/org/apache/solr/client/solrj/ (props changed)
lucene/dev/branches/lucene2621/solr/solrj/src/test/org/apache/solr/common/ (props changed)
lucene/dev/branches/lucene2621/solr/test-framework/ (props changed)
lucene/dev/branches/lucene2621/solr/testlogging.properties (props changed)
lucene/dev/branches/lucene2621/solr/webapp/ (props changed)
Modified: lucene/dev/branches/lucene2621/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/CHANGES.txt?rev=1202154&r1=1202153&r2=1202154&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/lucene2621/lucene/CHANGES.txt Tue Nov 15 12:25:46 2011
@@ -199,6 +199,9 @@ Changes in backwards compatibility polic
as these are no longer used by the scoring system. See MIGRATE.txt for more
details. (Robert Muir)
+* LUCENE-3533: Removed SpanFilters, they created large lists of objects and
+ did not scale. (Robert Muir)
+
Changes in Runtime Behavior
* LUCENE-2846: omitNorms now behaves like omitTermFrequencyAndPositions, if you
@@ -735,6 +738,9 @@ API Changes
for multi-segment indexes. They were only needed for tests of
NumericRangeQuery. (Mike McCandless, Uwe Schindler)
+* LUCENE-3574: Deprecate outdated constants in org.apache.lucene.util.Constants
+ and add new ones for Java 6 and Java 7. (Uwe Schindler)
+
New Features
* LUCENE-3448: Added FixedBitSet.and(other/DISI), andNot(other/DISI).
Modified: lucene/dev/branches/lucene2621/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java?rev=1202154&r1=1202153&r2=1202154&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java (original)
+++ lucene/dev/branches/lucene2621/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java Tue Nov 15 12:25:46 2011
@@ -25,6 +25,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.TreeSet;
import org.apache.lucene.analysis.CachingTokenFilter;
import org.apache.lucene.analysis.TokenStream;
@@ -42,6 +43,7 @@ import org.apache.lucene.search.spans.Sp
import org.apache.lucene.search.spans.SpanTermQuery;
import org.apache.lucene.search.spans.Spans;
import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.TermContext;
/**
* Class used to extract {@link WeightedSpanTerm}s from a {@link Query} based on whether
@@ -247,16 +249,21 @@ public class WeightedSpanTermExtractor {
List<PositionSpan> spanPositions = new ArrayList<PositionSpan>();
for (final String field : fieldNames) {
-
- AtomicReaderContext context = getLeafContextForField(field);
- Bits acceptDocs = context.reader.getLiveDocs();
- final Spans spans;
+ final SpanQuery q;
if (mustRewriteQuery) {
- spans = queries.get(field).getSpans(context, acceptDocs);
+ q = queries.get(field);
} else {
- spans = spanQuery.getSpans(context, acceptDocs);
+ q = spanQuery;
}
-
+ AtomicReaderContext context = getLeafContextForField(field);
+ Map<Term,TermContext> termContexts = new HashMap<Term,TermContext>();
+ TreeSet<Term> extractedTerms = new TreeSet<Term>();
+ q.extractTerms(extractedTerms);
+ for (Term term : extractedTerms) {
+ termContexts.put(term, TermContext.build(context, term, true));
+ }
+ Bits acceptDocs = context.reader.getLiveDocs();
+ final Spans spans = q.getSpans(context, acceptDocs, termContexts);
// collect span positions
while (spans.next()) {
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/values/MultiIndexDocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/values/MultiIndexDocValues.java?rev=1202154&r1=1202153&r2=1202154&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/values/MultiIndexDocValues.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/values/MultiIndexDocValues.java Tue Nov 15 12:25:46 2011
@@ -169,6 +169,7 @@ public class MultiIndexDocValues extends
}
}
+ // TODO: this is dup of IndexDocValues.getDefaultSource()?
private static class EmptySource extends Source {
public EmptySource(ValueType type) {
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/payloads/PayloadNearQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/payloads/PayloadNearQuery.java?rev=1202154&r1=1202153&r2=1202154&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/payloads/PayloadNearQuery.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/payloads/PayloadNearQuery.java Tue Nov 15 12:25:46 2011
@@ -150,7 +150,7 @@ public class PayloadNearQuery extends Sp
@Override
public Scorer scorer(AtomicReaderContext context, boolean scoreDocsInOrder,
boolean topScorer, Bits acceptDocs) throws IOException {
- return new PayloadNearSpanScorer(query.getSpans(context, acceptDocs), this,
+ return new PayloadNearSpanScorer(query.getSpans(context, acceptDocs, termContexts), this,
similarity, similarity.sloppyDocScorer(stats, query.getField(), context));
}
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/payloads/PayloadSpanUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/payloads/PayloadSpanUtil.java?rev=1202154&r1=1202153&r2=1202154&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/payloads/PayloadSpanUtil.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/payloads/PayloadSpanUtil.java Tue Nov 15 12:25:46 2011
@@ -20,8 +20,11 @@ package org.apache.lucene.search.payload
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
+import java.util.Map;
+import java.util.TreeSet;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
@@ -41,6 +44,7 @@ import org.apache.lucene.search.spans.Sp
import org.apache.lucene.search.spans.SpanTermQuery;
import org.apache.lucene.search.spans.Spans;
import org.apache.lucene.util.ReaderUtil;
+import org.apache.lucene.util.TermContext;
/**
* Experimental class to get set of payloads for most standard Lucene queries.
@@ -174,9 +178,15 @@ public class PayloadSpanUtil {
private void getPayloads(Collection<byte []> payloads, SpanQuery query)
throws IOException {
+ Map<Term,TermContext> termContexts = new HashMap<Term,TermContext>();
+ TreeSet<Term> terms = new TreeSet<Term>();
+ query.extractTerms(terms);
+ for (Term term : terms) {
+ termContexts.put(term, TermContext.build(context, term, true));
+ }
final AtomicReaderContext[] leaves = ReaderUtil.leaves(context);
for (AtomicReaderContext atomicReaderContext : leaves) {
- final Spans spans = query.getSpans(atomicReaderContext, atomicReaderContext.reader.getLiveDocs());
+ final Spans spans = query.getSpans(atomicReaderContext, atomicReaderContext.reader.getLiveDocs(), termContexts);
while (spans.next() == true) {
if (spans.isPayloadAvailable()) {
Collection<byte[]> payload = spans.getPayload();
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java?rev=1202154&r1=1202153&r2=1202154&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java Tue Nov 15 12:25:46 2011
@@ -81,7 +81,7 @@ public class PayloadTermQuery extends Sp
@Override
public Scorer scorer(AtomicReaderContext context, boolean scoreDocsInOrder,
boolean topScorer, Bits acceptDocs) throws IOException {
- return new PayloadTermSpanScorer((TermSpans) query.getSpans(context, acceptDocs),
+ return new PayloadTermSpanScorer((TermSpans) query.getSpans(context, acceptDocs, termContexts),
this, similarity.sloppyDocScorer(stats, query.getField(), context));
}
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/spans/FieldMaskingSpanQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/spans/FieldMaskingSpanQuery.java?rev=1202154&r1=1202153&r2=1202154&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/spans/FieldMaskingSpanQuery.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/spans/FieldMaskingSpanQuery.java Tue Nov 15 12:25:46 2011
@@ -18,6 +18,7 @@ package org.apache.lucene.search.spans;
*/
import java.io.IOException;
+import java.util.Map;
import java.util.Set;
import org.apache.lucene.index.IndexReader;
@@ -27,6 +28,7 @@ import org.apache.lucene.search.Query;
import org.apache.lucene.search.Weight;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.TermContext;
import org.apache.lucene.util.ToStringUtils;
/**
@@ -93,8 +95,8 @@ public class FieldMaskingSpanQuery exten
// ...this is done to be more consistent with things like SpanFirstQuery
@Override
- public Spans getSpans(AtomicReaderContext context, Bits acceptDocs) throws IOException {
- return maskedQuery.getSpans(context, acceptDocs);
+ public Spans getSpans(AtomicReaderContext context, Bits acceptDocs, Map<Term,TermContext> termContexts) throws IOException {
+ return maskedQuery.getSpans(context, acceptDocs, termContexts);
}
@Override
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/spans/NearSpansOrdered.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/spans/NearSpansOrdered.java?rev=1202154&r1=1202153&r2=1202154&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/spans/NearSpansOrdered.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/spans/NearSpansOrdered.java Tue Nov 15 12:25:46 2011
@@ -17,9 +17,11 @@ package org.apache.lucene.search.spans;
* limitations under the License.
*/
+import org.apache.lucene.index.Term;
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.TermContext;
import java.io.IOException;
import java.util.ArrayList;
@@ -28,6 +30,7 @@ import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Collection;
+import java.util.Map;
import java.util.Set;
/** A Spans that is formed from the ordered subspans of a SpanNearQuery
@@ -78,11 +81,11 @@ public class NearSpansOrdered extends Sp
private SpanNearQuery query;
private boolean collectPayloads = true;
- public NearSpansOrdered(SpanNearQuery spanNearQuery, AtomicReaderContext context, Bits acceptDocs) throws IOException {
- this(spanNearQuery, context, acceptDocs, true);
+ public NearSpansOrdered(SpanNearQuery spanNearQuery, AtomicReaderContext context, Bits acceptDocs, Map<Term,TermContext> termContexts) throws IOException {
+ this(spanNearQuery, context, acceptDocs, termContexts, true);
}
- public NearSpansOrdered(SpanNearQuery spanNearQuery, AtomicReaderContext context, Bits acceptDocs, boolean collectPayloads)
+ public NearSpansOrdered(SpanNearQuery spanNearQuery, AtomicReaderContext context, Bits acceptDocs, Map<Term,TermContext> termContexts, boolean collectPayloads)
throws IOException {
if (spanNearQuery.getClauses().length < 2) {
throw new IllegalArgumentException("Less than 2 clauses: "
@@ -95,7 +98,7 @@ public class NearSpansOrdered extends Sp
matchPayload = new LinkedList<byte[]>();
subSpansByDoc = new Spans[clauses.length];
for (int i = 0; i < clauses.length; i++) {
- subSpans[i] = clauses[i].getSpans(context, acceptDocs);
+ subSpans[i] = clauses[i].getSpans(context, acceptDocs, termContexts);
subSpansByDoc[i] = subSpans[i]; // used in toSameDoc()
}
query = spanNearQuery; // kept for toString() only.
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/spans/NearSpansUnordered.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/spans/NearSpansUnordered.java?rev=1202154&r1=1202153&r2=1202154&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/spans/NearSpansUnordered.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/spans/NearSpansUnordered.java Tue Nov 15 12:25:46 2011
@@ -17,14 +17,17 @@ package org.apache.lucene.search.spans;
* limitations under the License.
*/
+import org.apache.lucene.index.Term;
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.PriorityQueue;
+import org.apache.lucene.util.TermContext;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import java.util.HashSet;
@@ -132,7 +135,7 @@ public class NearSpansUnordered extends
}
- public NearSpansUnordered(SpanNearQuery query, AtomicReaderContext context, Bits acceptDocs)
+ public NearSpansUnordered(SpanNearQuery query, AtomicReaderContext context, Bits acceptDocs, Map<Term,TermContext> termContexts)
throws IOException {
this.query = query;
this.slop = query.getSlop();
@@ -142,7 +145,7 @@ public class NearSpansUnordered extends
subSpans = new Spans[clauses.length];
for (int i = 0; i < clauses.length; i++) {
SpansCell cell =
- new SpansCell(clauses[i].getSpans(context, acceptDocs), i);
+ new SpansCell(clauses[i].getSpans(context, acceptDocs, termContexts), i);
ordered.add(cell);
subSpans[i] = cell.spans;
}
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/spans/SpanMultiTermQueryWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/spans/SpanMultiTermQueryWrapper.java?rev=1202154&r1=1202153&r2=1202154&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/spans/SpanMultiTermQueryWrapper.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/spans/SpanMultiTermQueryWrapper.java Tue Nov 15 12:25:46 2011
@@ -18,6 +18,7 @@ package org.apache.lucene.search.spans;
*/
import java.io.IOException;
+import java.util.Map;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
@@ -90,7 +91,7 @@ public class SpanMultiTermQueryWrapper<Q
}
@Override
- public Spans getSpans(AtomicReaderContext context, Bits acceptDocs) throws IOException {
+ public Spans getSpans(AtomicReaderContext context, Bits acceptDocs, Map<Term,TermContext> termContexts) throws IOException {
throw new UnsupportedOperationException("Query should have been rewritten");
}
@@ -157,6 +158,9 @@ public class SpanMultiTermQueryWrapper<Q
@Override
protected void addClause(SpanOrQuery topLevel, Term term, int docCount, float boost, TermContext states) {
+ // TODO: would be nice to not lose term-state here.
+ // we could add a hack option to SpanOrQuery, but the hack would only work if this is the top-level Span
+ // (if you put this thing in another span query, it would extractTerms/double-seek anyway)
final SpanTermQuery q = new SpanTermQuery(term);
q.setBoost(boost);
topLevel.addClause(q);
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/spans/SpanNearQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/spans/SpanNearQuery.java?rev=1202154&r1=1202153&r2=1202154&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/spans/SpanNearQuery.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/spans/SpanNearQuery.java Tue Nov 15 12:25:46 2011
@@ -23,6 +23,7 @@ import java.io.IOException;
import java.util.List;
import java.util.ArrayList;
import java.util.Iterator;
+import java.util.Map;
import java.util.Set;
@@ -31,6 +32,7 @@ import org.apache.lucene.index.IndexRead
import org.apache.lucene.index.Term;
import org.apache.lucene.search.Query;
import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.TermContext;
import org.apache.lucene.util.ToStringUtils;
/** Matches spans which are near one another. One can specify <i>slop</i>, the
@@ -118,16 +120,16 @@ public class SpanNearQuery extends SpanQ
}
@Override
- public Spans getSpans(final AtomicReaderContext context, Bits acceptDocs) throws IOException {
+ public Spans getSpans(final AtomicReaderContext context, Bits acceptDocs, Map<Term,TermContext> termContexts) throws IOException {
if (clauses.size() == 0) // optimize 0-clause case
- return new SpanOrQuery(getClauses()).getSpans(context, acceptDocs);
+ return new SpanOrQuery(getClauses()).getSpans(context, acceptDocs, termContexts);
if (clauses.size() == 1) // optimize 1-clause case
- return clauses.get(0).getSpans(context, acceptDocs);
+ return clauses.get(0).getSpans(context, acceptDocs, termContexts);
return inOrder
- ? (Spans) new NearSpansOrdered(this, context, acceptDocs, collectPayloads)
- : (Spans) new NearSpansUnordered(this, context, acceptDocs);
+ ? (Spans) new NearSpansOrdered(this, context, acceptDocs, termContexts, collectPayloads)
+ : (Spans) new NearSpansUnordered(this, context, acceptDocs, termContexts);
}
@Override
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/spans/SpanNotQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/spans/SpanNotQuery.java?rev=1202154&r1=1202153&r2=1202154&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/spans/SpanNotQuery.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/spans/SpanNotQuery.java Tue Nov 15 12:25:46 2011
@@ -22,11 +22,13 @@ import org.apache.lucene.index.IndexRead
import org.apache.lucene.index.Term;
import org.apache.lucene.search.Query;
import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.TermContext;
import org.apache.lucene.util.ToStringUtils;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Map;
import java.util.Set;
/** Removes matches which overlap with another SpanQuery. */
@@ -76,12 +78,12 @@ public class SpanNotQuery extends SpanQu
}
@Override
- public Spans getSpans(final AtomicReaderContext context, final Bits acceptDocs) throws IOException {
+ public Spans getSpans(final AtomicReaderContext context, final Bits acceptDocs, final Map<Term,TermContext> termContexts) throws IOException {
return new Spans() {
- private Spans includeSpans = include.getSpans(context, acceptDocs);
+ private Spans includeSpans = include.getSpans(context, acceptDocs, termContexts);
private boolean moreInclude = true;
- private Spans excludeSpans = exclude.getSpans(context, acceptDocs);
+ private Spans excludeSpans = exclude.getSpans(context, acceptDocs, termContexts);
private boolean moreExclude = excludeSpans.next();
@Override
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/spans/SpanOrQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/spans/SpanOrQuery.java?rev=1202154&r1=1202153&r2=1202154&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/spans/SpanOrQuery.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/spans/SpanOrQuery.java Tue Nov 15 12:25:46 2011
@@ -23,6 +23,7 @@ import java.util.List;
import java.util.Collection;
import java.util.ArrayList;
import java.util.Iterator;
+import java.util.Map;
import java.util.Set;
import org.apache.lucene.index.IndexReader;
@@ -30,6 +31,7 @@ import org.apache.lucene.index.IndexRead
import org.apache.lucene.index.Term;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.PriorityQueue;
+import org.apache.lucene.util.TermContext;
import org.apache.lucene.util.ToStringUtils;
import org.apache.lucene.search.Query;
@@ -164,9 +166,9 @@ public class SpanOrQuery extends SpanQue
}
@Override
- public Spans getSpans(final AtomicReaderContext context, final Bits acceptDocs) throws IOException {
+ public Spans getSpans(final AtomicReaderContext context, final Bits acceptDocs, final Map<Term,TermContext> termContexts) throws IOException {
if (clauses.size() == 1) // optimize 1-clause case
- return (clauses.get(0)).getSpans(context, acceptDocs);
+ return (clauses.get(0)).getSpans(context, acceptDocs, termContexts);
return new Spans() {
private SpanQueue queue = null;
@@ -175,7 +177,7 @@ public class SpanOrQuery extends SpanQue
queue = new SpanQueue(clauses.size());
Iterator<SpanQuery> i = clauses.iterator();
while (i.hasNext()) {
- Spans spans = i.next().getSpans(context, acceptDocs);
+ Spans spans = i.next().getSpans(context, acceptDocs, termContexts);
if ( ((target == -1) && spans.next())
|| ((target != -1) && spans.skipTo(target))) {
queue.add(spans);
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/spans/SpanPositionCheckQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/spans/SpanPositionCheckQuery.java?rev=1202154&r1=1202153&r2=1202154&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/spans/SpanPositionCheckQuery.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/spans/SpanPositionCheckQuery.java Tue Nov 15 12:25:46 2011
@@ -22,10 +22,12 @@ import org.apache.lucene.index.IndexRead
import org.apache.lucene.index.Term;
import org.apache.lucene.search.Query;
import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.TermContext;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Map;
import java.util.Set;
@@ -82,8 +84,8 @@ public abstract class SpanPositionCheckQ
protected abstract AcceptStatus acceptPosition(Spans spans) throws IOException;
@Override
- public Spans getSpans(final AtomicReaderContext context, Bits acceptDocs) throws IOException {
- return new PositionCheckSpan(context, acceptDocs);
+ public Spans getSpans(final AtomicReaderContext context, Bits acceptDocs, Map<Term,TermContext> termContexts) throws IOException {
+ return new PositionCheckSpan(context, acceptDocs, termContexts);
}
@@ -107,8 +109,8 @@ public abstract class SpanPositionCheckQ
protected class PositionCheckSpan extends Spans {
private Spans spans;
- public PositionCheckSpan(AtomicReaderContext context, Bits acceptDocs) throws IOException {
- spans = match.getSpans(context, acceptDocs);
+ public PositionCheckSpan(AtomicReaderContext context, Bits acceptDocs, Map<Term,TermContext> termContexts) throws IOException {
+ spans = match.getSpans(context, acceptDocs, termContexts);
}
@Override
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/spans/SpanQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/spans/SpanQuery.java?rev=1202154&r1=1202153&r2=1202154&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/spans/SpanQuery.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/spans/SpanQuery.java Tue Nov 15 12:25:46 2011
@@ -18,18 +18,21 @@ package org.apache.lucene.search.spans;
*/
import java.io.IOException;
+import java.util.Map;
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
+import org.apache.lucene.index.Term;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Weight;
import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.TermContext;
/** Base class for span-based queries. */
public abstract class SpanQuery extends Query {
/** Expert: Returns the matches for this query in an index. Used internally
* to search for spans. */
- public abstract Spans getSpans(AtomicReaderContext context, Bits acceptDocs) throws IOException;
+ public abstract Spans getSpans(AtomicReaderContext context, Bits acceptDocs, Map<Term,TermContext> termContexts) throws IOException;
/** Returns the name of the field matched by this query.*/
public abstract String getField();
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/spans/SpanTermQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/spans/SpanTermQuery.java?rev=1202154&r1=1202153&r2=1202154&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/spans/SpanTermQuery.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/spans/SpanTermQuery.java Tue Nov 15 12:25:46 2011
@@ -19,12 +19,19 @@ package org.apache.lucene.search.spans;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
+import org.apache.lucene.index.Fields;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.DocsAndPositionsEnum;
+import org.apache.lucene.index.TermState;
+import org.apache.lucene.index.Terms;
+import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.ReaderUtil;
+import org.apache.lucene.util.TermContext;
import org.apache.lucene.util.ToStringUtils;
import java.io.IOException;
+import java.util.Map;
import java.util.Set;
/** Matches spans containing a term. */
@@ -82,22 +89,46 @@ public class SpanTermQuery extends SpanQ
}
@Override
- public Spans getSpans(final AtomicReaderContext context, Bits acceptDocs) throws IOException {
- final IndexReader reader = context.reader;
- final DocsAndPositionsEnum postings = reader.termPositionsEnum(acceptDocs,
- term.field(),
- term.bytes());
+ public Spans getSpans(final AtomicReaderContext context, Bits acceptDocs, Map<Term,TermContext> termContexts) throws IOException {
+ TermContext termContext = termContexts.get(term);
+ final TermState state;
+ if (termContext == null) {
+ // this happens with span-not query, as it doesn't include the NOT side in extractTerms()
+ // so we seek to the term now in this segment..., this sucks because its ugly mostly!
+ final Fields fields = context.reader.fields();
+ if (fields != null) {
+ final Terms terms = fields.terms(term.field());
+ if (terms != null) {
+ final TermsEnum termsEnum = terms.getThreadTermsEnum(); // thread-private don't share!
+ if (termsEnum.seekExact(term.bytes(), true)) {
+ state = termsEnum.termState();
+ } else {
+ state = null;
+ }
+ } else {
+ state = null;
+ }
+ } else {
+ state = null;
+ }
+ } else {
+ state = termContext.get(context.ord);
+ }
+
+ if (state == null) { // term is not present in that reader
+ return TermSpans.EMPTY_TERM_SPANS;
+ }
+
+ final TermsEnum termsEnum = context.reader.terms(term.field()).getThreadTermsEnum();
+ termsEnum.seekExact(term.bytes(), state);
+
+ final DocsAndPositionsEnum postings = termsEnum.docsAndPositions(acceptDocs, null);
if (postings != null) {
return new TermSpans(postings, term);
} else {
- if (reader.termDocsEnum(reader.getLiveDocs(), term.field(), term.bytes()) != null) {
- // term does exist, but has no positions
- throw new IllegalStateException("field \"" + term.field() + "\" was indexed without position data; cannot run SpanTermQuery (term=" + term.text() + ")");
- } else {
- // term does not exist
- return TermSpans.EMPTY_TERM_SPANS;
- }
+ // term does exist, but has no positions
+ throw new IllegalStateException("field \"" + term.field() + "\" was indexed without position data; cannot run SpanTermQuery (term=" + term.text() + ")");
}
}
}
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/spans/SpanWeight.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/spans/SpanWeight.java?rev=1202154&r1=1202153&r2=1202154&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/spans/SpanWeight.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/spans/SpanWeight.java Tue Nov 15 12:25:46 2011
@@ -27,7 +27,8 @@ import org.apache.lucene.util.Bits;
import org.apache.lucene.util.TermContext;
import java.io.IOException;
-import java.util.Set;
+import java.util.HashMap;
+import java.util.Map;
import java.util.TreeSet;
/**
@@ -35,7 +36,7 @@ import java.util.TreeSet;
*/
public class SpanWeight extends Weight {
protected Similarity similarity;
- protected Set<Term> terms;
+ protected Map<Term,TermContext> termContexts;
protected SpanQuery query;
protected Similarity.Stats stats;
@@ -44,15 +45,16 @@ public class SpanWeight extends Weight {
this.similarity = searcher.getSimilarityProvider().get(query.getField());
this.query = query;
- terms=new TreeSet<Term>();
+ termContexts = new HashMap<Term,TermContext>();
+ TreeSet<Term> terms = new TreeSet<Term>();
query.extractTerms(terms);
final ReaderContext context = searcher.getTopReaderContext();
- final TermContext states[] = new TermContext[terms.size()];
final TermStatistics termStats[] = new TermStatistics[terms.size()];
int i = 0;
for (Term term : terms) {
- states[i] = TermContext.build(context, term, true);
- termStats[i] = searcher.termStatistics(term, states[i]);
+ TermContext state = TermContext.build(context, term, true);
+ termStats[i] = searcher.termStatistics(term, state);
+ termContexts.put(term, state);
i++;
}
stats = similarity.computeStats(
@@ -77,7 +79,7 @@ public class SpanWeight extends Weight {
@Override
public Scorer scorer(AtomicReaderContext context, boolean scoreDocsInOrder,
boolean topScorer, Bits acceptDocs) throws IOException {
- return new SpanScorer(query.getSpans(context, acceptDocs), this, similarity.sloppyDocScorer(stats, query.getField(), context));
+ return new SpanScorer(query.getSpans(context, acceptDocs, termContexts), this, similarity.sloppyDocScorer(stats, query.getField(), context));
}
@Override
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/util/Constants.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/util/Constants.java?rev=1202154&r1=1202153&r2=1202154&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/util/Constants.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/util/Constants.java Tue Nov 15 12:25:46 2011
@@ -28,12 +28,6 @@ public final class Constants {
/** The value of <tt>System.getProperty("java.version")<tt>. **/
public static final String JAVA_VERSION = System.getProperty("java.version");
- /** True iff this is Java version 1.1. */
- public static final boolean JAVA_1_1 = JAVA_VERSION.startsWith("1.1.");
- /** True iff this is Java version 1.2. */
- public static final boolean JAVA_1_2 = JAVA_VERSION.startsWith("1.2.");
- /** True iff this is Java version 1.3. */
- public static final boolean JAVA_1_3 = JAVA_VERSION.startsWith("1.3.");
/** The value of <tt>System.getProperty("os.name")<tt>. **/
public static final String OS_NAME = System.getProperty("os.name");
@@ -50,11 +44,17 @@ public final class Constants {
public static final String OS_VERSION = System.getProperty("os.version");
public static final String JAVA_VENDOR = System.getProperty("java.vendor");
- // NOTE: this logic may not be correct; if you know of a
- // more reliable approach please raise it on java-dev!
- public static final boolean JRE_IS_64BIT;
+ /** @deprecated With Lucene 4.0, we are always on Java 6 */
+ @Deprecated
+ public static final boolean JRE_IS_MINIMUM_JAVA6 =
+ new Boolean(true).booleanValue(); // prevent inlining in foreign class files
+
+ public static final boolean JRE_IS_64BIT;
+ public static final boolean JRE_IS_MINIMUM_JAVA7;
static {
- String x = System.getProperty("sun.arch.data.model");
+ // NOTE: this logic may not be correct; if you know of a
+ // more reliable approach please raise it on java-dev!
+ final String x = System.getProperty("sun.arch.data.model");
if (x != null) {
JRE_IS_64BIT = x.indexOf("64") != -1;
} else {
@@ -64,6 +64,15 @@ public final class Constants {
JRE_IS_64BIT = false;
}
}
+
+ // this method only exists in Java 7:
+ boolean v7 = true;
+ try {
+ Throwable.class.getMethod("getSuppressed");
+ } catch (NoSuchMethodException nsme) {
+ v7 = false;
+ }
+ JRE_IS_MINIMUM_JAVA7 = v7;
}
// this method prevents inlining the final version constant in compiled classes,
Modified: lucene/dev/branches/lucene2621/lucene/src/test-framework/java/org/apache/lucene/index/RandomIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/test-framework/java/org/apache/lucene/index/RandomIndexWriter.java?rev=1202154&r1=1202153&r2=1202154&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/test-framework/java/org/apache/lucene/index/RandomIndexWriter.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/test-framework/java/org/apache/lucene/index/RandomIndexWriter.java Tue Nov 15 12:25:46 2011
@@ -293,6 +293,10 @@ public class RandomIndexWriter implement
public void addIndexes(Directory... dirs) throws CorruptIndexException, IOException {
w.addIndexes(dirs);
}
+
+ public void addIndexes(IndexReader... readers) throws CorruptIndexException, IOException {
+ w.addIndexes(readers);
+ }
public void deleteDocuments(Term term) throws CorruptIndexException, IOException {
w.deleteDocuments(term);
Modified: lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java?rev=1202154&r1=1202153&r2=1202154&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java Tue Nov 15 12:25:46 2011
@@ -1230,4 +1230,51 @@ public class TestAddIndexes extends Luce
toAdd.close();
}
+ // LUCENE-3575
+ public void testFieldNamesChanged() throws IOException {
+ Directory d1 = newDirectory();
+ RandomIndexWriter w = new RandomIndexWriter(random, d1);
+ Document doc = new Document();
+ doc.add(newField("f1", "doc1 field1", StringField.TYPE_STORED));
+ doc.add(newField("id", "1", StringField.TYPE_STORED));
+ //doc.add(newField("f7", "doc1 field7", StringField.TYPE_STORED));
+ w.addDocument(doc);
+ IndexReader r1 = w.getReader();
+ w.close();
+
+ Directory d2 = newDirectory();
+ w = new RandomIndexWriter(random, d2);
+ doc = new Document();
+ doc.add(newField("f2", "doc2 field2", StringField.TYPE_STORED));
+ doc.add(newField("id", "2", StringField.TYPE_STORED));
+ //doc.add(newField("f7", "doc2 field7", StringField.TYPE_STORED));
+ w.addDocument(doc);
+ IndexReader r2 = w.getReader();
+ w.close();
+
+ Directory d3 = newDirectory();
+ w = new RandomIndexWriter(random, d3);
+ w.addIndexes(r1, r2);
+ r1.close();
+ d1.close();
+ r2.close();
+ d2.close();
+
+ IndexReader r3 = w.getReader();
+ w.close();
+ System.out.println("r1=" + r1);
+ System.out.println("r2=" + r2);
+ System.out.println("r3=" + r3);
+ assertEquals(2, r3.numDocs());
+ for(int docID=0;docID<2;docID++) {
+ Document d = r3.document(docID);
+ if (d.get("id").equals("1")) {
+ assertEquals("doc1 field1", d.get("f1"));
+ } else {
+ assertEquals("doc2 field2", d.get("f2"));
+ }
+ }
+ r3.close();
+ d3.close();
+ }
}
Modified: lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/search/JustCompileSearch.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/search/JustCompileSearch.java?rev=1202154&r1=1202153&r2=1202154&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/search/JustCompileSearch.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/search/JustCompileSearch.java Tue Nov 15 12:25:46 2011
@@ -281,19 +281,6 @@ final class JustCompileSearch {
}
}
- static final class JustCompileSpanFilter extends SpanFilter {
-
- @Override
- public SpanFilterResult bitSpans(AtomicReaderContext context, Bits acceptDocs) throws IOException {
- throw new UnsupportedOperationException(UNSUPPORTED_MSG);
- }
-
- @Override
- public DocIdSet getDocIdSet(AtomicReaderContext context, Bits acceptDocs) throws IOException {
- return null;
- }
- }
-
static final class JustCompileTopDocsCollector extends TopDocsCollector<ScoreDoc> {
protected JustCompileTopDocsCollector(PriorityQueue<ScoreDoc> pq) {
Modified: lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/search/spans/JustCompileSearchSpans.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/search/spans/JustCompileSearchSpans.java?rev=1202154&r1=1202153&r2=1202154&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/search/spans/JustCompileSearchSpans.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/search/spans/JustCompileSearchSpans.java Tue Nov 15 12:25:46 2011
@@ -19,11 +19,14 @@ package org.apache.lucene.search.spans;
import java.io.IOException;
import java.util.Collection;
+import java.util.Map;
+import org.apache.lucene.index.Term;
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
import org.apache.lucene.search.Weight;
import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.TermContext;
/**
* Holds all implementations of classes in the o.a.l.s.spans package as a
@@ -83,7 +86,7 @@ final class JustCompileSearchSpans {
}
@Override
- public Spans getSpans(AtomicReaderContext context, Bits acceptDocs) throws IOException {
+ public Spans getSpans(AtomicReaderContext context, Bits acceptDocs, Map<Term,TermContext> termContexts) throws IOException {
throw new UnsupportedOperationException(UNSUPPORTED_MSG);
}
Modified: lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/search/spans/MultiSpansWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/search/spans/MultiSpansWrapper.java?rev=1202154&r1=1202153&r2=1202154&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/search/spans/MultiSpansWrapper.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/search/spans/MultiSpansWrapper.java Tue Nov 15 12:25:46 2011
@@ -20,11 +20,16 @@ package org.apache.lucene.search.spans;
import java.io.IOException;
import java.util.Collection;
import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.TreeSet;
import org.apache.lucene.index.DocsEnum;
+import org.apache.lucene.index.Term;
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
import org.apache.lucene.index.IndexReader.ReaderContext;
import org.apache.lucene.util.ReaderUtil;
+import org.apache.lucene.util.TermContext;
/**
*
@@ -39,19 +44,27 @@ public class MultiSpansWrapper extends S
private AtomicReaderContext[] leaves;
private int leafOrd = 0;
private Spans current;
+ private Map<Term,TermContext> termContexts;
- private MultiSpansWrapper(AtomicReaderContext[] leaves, SpanQuery query) {
+ private MultiSpansWrapper(AtomicReaderContext[] leaves, SpanQuery query, Map<Term,TermContext> termContexts) {
this.query = query;
this.leaves = leaves;
+ this.termContexts = termContexts;
}
public static Spans wrap(ReaderContext topLevelReaderContext, SpanQuery query) throws IOException {
+ Map<Term,TermContext> termContexts = new HashMap<Term,TermContext>();
+ TreeSet<Term> terms = new TreeSet<Term>();
+ query.extractTerms(terms);
+ for (Term term : terms) {
+ termContexts.put(term, TermContext.build(topLevelReaderContext, term, true));
+ }
AtomicReaderContext[] leaves = ReaderUtil.leaves(topLevelReaderContext);
if(leaves.length == 1) {
- return query.getSpans(leaves[0], leaves[0].reader.getLiveDocs());
+ return query.getSpans(leaves[0], leaves[0].reader.getLiveDocs(), termContexts);
}
- return new MultiSpansWrapper(leaves, query);
+ return new MultiSpansWrapper(leaves, query, termContexts);
}
@Override
@@ -60,14 +73,14 @@ public class MultiSpansWrapper extends S
return false;
}
if (current == null) {
- current = query.getSpans(leaves[leafOrd], leaves[leafOrd].reader.getLiveDocs());
+ current = query.getSpans(leaves[leafOrd], leaves[leafOrd].reader.getLiveDocs(), termContexts);
}
while(true) {
if (current.next()) {
return true;
}
if (++leafOrd < leaves.length) {
- current = query.getSpans(leaves[leafOrd], leaves[leafOrd].reader.getLiveDocs());
+ current = query.getSpans(leaves[leafOrd], leaves[leafOrd].reader.getLiveDocs(), termContexts);
} else {
current = null;
break;
@@ -85,17 +98,17 @@ public class MultiSpansWrapper extends S
int subIndex = ReaderUtil.subIndex(target, leaves);
assert subIndex >= leafOrd;
if (subIndex != leafOrd) {
- current = query.getSpans(leaves[subIndex], leaves[subIndex].reader.getLiveDocs());
+ current = query.getSpans(leaves[subIndex], leaves[subIndex].reader.getLiveDocs(), termContexts);
leafOrd = subIndex;
} else if (current == null) {
- current = query.getSpans(leaves[leafOrd], leaves[leafOrd].reader.getLiveDocs());
+ current = query.getSpans(leaves[leafOrd], leaves[leafOrd].reader.getLiveDocs(), termContexts);
}
while (true) {
if (current.skipTo(target - leaves[leafOrd].docBase)) {
return true;
}
if (++leafOrd < leaves.length) {
- current = query.getSpans(leaves[leafOrd], leaves[leafOrd].reader.getLiveDocs());
+ current = query.getSpans(leaves[leafOrd], leaves[leafOrd].reader.getLiveDocs(), termContexts);
} else {
current = null;
break;
Modified: lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/util/TestIOUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/util/TestIOUtils.java?rev=1202154&r1=1202153&r2=1202154&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/util/TestIOUtils.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/util/TestIOUtils.java Tue Nov 15 12:25:46 2011
@@ -44,15 +44,7 @@ public class TestIOUtils extends LuceneT
}
public void testSuppressedExceptions() {
- boolean isJava7 = true;
- try {
- // this class only exists in Java 7:
- Class.forName("java.lang.AutoCloseable");
- } catch (ClassNotFoundException cnfe) {
- isJava7 = false;
- }
-
- if (!isJava7) {
+ if (!Constants.JRE_IS_MINIMUM_JAVA7) {
System.err.println("WARNING: TestIOUtils.testSuppressedExceptions: Full test coverage only with Java 7, as suppressed exception recording is not supported before.");
}
@@ -71,7 +63,7 @@ public class TestIOUtils extends LuceneT
System.out.println("TestIOUtils.testSuppressedExceptions: Thrown Exception stack trace:");
System.out.println(trace);
}
- if (isJava7) {
+ if (Constants.JRE_IS_MINIMUM_JAVA7) {
assertTrue("Stack trace does not contain first suppressed Exception: " + trace,
trace.contains("java.io.IOException: TEST-IO-EXCEPTION-1"));
assertTrue("Stack trace does not contain second suppressed Exception: " + trace,
@@ -97,7 +89,7 @@ public class TestIOUtils extends LuceneT
System.out.println("TestIOUtils.testSuppressedExceptions: Thrown Exception stack trace:");
System.out.println(trace);
}
- if (isJava7) {
+ if (Constants.JRE_IS_MINIMUM_JAVA7) {
assertTrue("Stack trace does not contain suppressed Exception: " + trace,
trace.contains("java.io.IOException: TEST-IO-EXCEPTION-2"));
}
Modified: lucene/dev/branches/lucene2621/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHCache.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHCache.java?rev=1202154&r1=1202153&r2=1202154&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHCache.java (original)
+++ lucene/dev/branches/lucene2621/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHCache.java Tue Nov 15 12:25:46 2011
@@ -1,5 +1,22 @@
package org.apache.solr.handler.dataimport;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
import java.util.Iterator;
import java.util.Map;
@@ -9,7 +26,7 @@ import java.util.Map;
* to other data and/or indexed.
* </p>
*
- * @solr.experimental
+ * @lucene.experimental
*/
public interface DIHCache extends Iterable<Map<String,Object>> {
Modified: lucene/dev/branches/lucene2621/solr/core/src/java/org/apache/solr/handler/component/FieldFacetStats.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/solr/core/src/java/org/apache/solr/handler/component/FieldFacetStats.java?rev=1202154&r1=1202153&r2=1202154&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/solr/core/src/java/org/apache/solr/handler/component/FieldFacetStats.java (original)
+++ lucene/dev/branches/lucene2621/solr/core/src/java/org/apache/solr/handler/component/FieldFacetStats.java Tue Nov 15 12:25:46 2011
@@ -19,6 +19,7 @@ package org.apache.solr.handler.componen
import org.apache.lucene.search.FieldCache;
import org.apache.lucene.util.BytesRef;
import org.apache.solr.schema.FieldType;
+import org.apache.solr.schema.SchemaField;
import java.util.ArrayList;
import java.util.HashMap;
@@ -38,7 +39,8 @@ import java.util.Map;
public class FieldFacetStats {
public final String name;
final FieldCache.DocTermsIndex si;
- final FieldType ft;
+ final SchemaField facet_sf;
+ final SchemaField field_sf;
final int startTermIndex;
final int endTermIndex;
@@ -52,10 +54,11 @@ public class FieldFacetStats {
private final BytesRef tempBR = new BytesRef();
- public FieldFacetStats(String name, FieldCache.DocTermsIndex si, FieldType ft, int numStatsTerms) {
+ public FieldFacetStats(String name, FieldCache.DocTermsIndex si, SchemaField field_sf, SchemaField facet_sf, int numStatsTerms) {
this.name = name;
this.si = si;
- this.ft = ft;
+ this.field_sf = field_sf;
+ this.facet_sf = facet_sf;
this.numStatsTerms = numStatsTerms;
startTermIndex = 1;
@@ -82,22 +85,22 @@ public class FieldFacetStats {
}
}
- public boolean facet(int docID, Double v) {
+ public boolean facet(int docID, BytesRef v) {
int term = si.getOrd(docID);
int arrIdx = term - startTermIndex;
if (arrIdx >= 0 && arrIdx < nTerms) {
final BytesRef br = si.lookup(term, tempBR);
- String key = ft.indexedToReadable(br == null ? null : br.utf8ToString());
+ String key = (br == null)?null:facet_sf.getType().indexedToReadable(br.utf8ToString());
StatsValues stats = facetStatsValues.get(key);
if (stats == null) {
- stats = new StatsValues();
+ stats = StatsValuesFactory.createStatsValues(field_sf);
facetStatsValues.put(key, stats);
}
- if (v != null) {
+ if (v != null && v.length>0) {
stats.accumulate(v);
} else {
- stats.missing++;
+ stats.missing();
return false;
}
return true;
@@ -129,14 +132,14 @@ public class FieldFacetStats {
//function to accumulate counts for statsTermNum to specified value
- public boolean accumulateTermNum(int statsTermNum, Double value) {
+ public boolean accumulateTermNum(int statsTermNum, BytesRef value) {
if (value == null) return false;
for (Map.Entry<String, Integer> stringIntegerEntry : facetStatsTerms.get(statsTermNum).entrySet()) {
Map.Entry pairs = (Map.Entry) stringIntegerEntry;
String key = (String) pairs.getKey();
StatsValues facetStats = facetStatsValues.get(key);
if (facetStats == null) {
- facetStats = new StatsValues();
+ facetStats = StatsValuesFactory.createStatsValues(field_sf);
facetStatsValues.put(key, facetStats);
}
Integer count = (Integer) pairs.getValue();
Modified: lucene/dev/branches/lucene2621/solr/core/src/java/org/apache/solr/handler/component/StatsComponent.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/solr/core/src/java/org/apache/solr/handler/component/StatsComponent.java?rev=1202154&r1=1202153&r2=1202154&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/solr/core/src/java/org/apache/solr/handler/component/StatsComponent.java (original)
+++ lucene/dev/branches/lucene2621/solr/core/src/java/org/apache/solr/handler/component/StatsComponent.java Tue Nov 15 12:25:46 2011
@@ -18,12 +18,14 @@
package org.apache.solr.handler.component;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import org.apache.lucene.search.FieldCache;
import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.CharsRef;
+import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.params.StatsParams;
import org.apache.solr.common.params.ShardParams;
@@ -177,7 +179,8 @@ class StatsInfo {
String[] statsFs = params.getParams(StatsParams.STATS_FIELD);
if (statsFs != null) {
for (String field : statsFs) {
- statsFields.put(field,new StatsValues());
+ SchemaField sf = rb.req.getSchema().getField(field);
+ statsFields.put(field, StatsValuesFactory.createStatsValues(sf));
}
}
}
@@ -244,58 +247,60 @@ class SimpleStats {
}
public NamedList<?> getFieldCacheStats(String fieldName, String[] facet ) {
- FieldType ft = searcher.getSchema().getFieldType(fieldName);
-
- FieldCache.DocTermsIndex si = null;
+ SchemaField sf = searcher.getSchema().getField(fieldName);
+
+ FieldCache.DocTermsIndex si;
try {
si = FieldCache.DEFAULT.getTermsIndex(searcher.getIndexReader(), fieldName);
}
catch (IOException e) {
throw new RuntimeException( "failed to open field cache for: "+fieldName, e );
}
- FieldFacetStats all = new FieldFacetStats( "all", si, ft, 0 );
- StatsValues allstats = new StatsValues();
- if ( all.nTerms <= 0 || docs.size() <= 0 ) return allstats.getStatsValues();
-
- // don't worry about faceting if the no documents match...
- int i=0;
- final FieldFacetStats[] finfo = new FieldFacetStats[facet.length];
- for( String f : facet ) {
- ft = searcher.getSchema().getFieldType(f);
+ StatsValues allstats = StatsValuesFactory.createStatsValues(sf);
+ final int nTerms = si.numOrd();
+ if ( nTerms <= 0 || docs.size() <= 0 ) return allstats.getStatsValues();
+
+ // don't worry about faceting if no documents match...
+ List<FieldFacetStats> facetStats = new ArrayList<FieldFacetStats>();
+ FieldCache.DocTermsIndex facetTermsIndex;
+ for( String facetField : facet ) {
+ SchemaField fsf = searcher.getSchema().getField(facetField);
+ FieldType facetFieldType = fsf.getType();
+
+ if (facetFieldType.isTokenized() || facetFieldType.isMultiValued()) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ "Stats can only facet on single-valued fields, not: " + facetField
+ + "[" + facetFieldType + "]");
+ }
try {
- si = FieldCache.DEFAULT.getTermsIndex(searcher.getIndexReader(), f);
- }
+ facetTermsIndex = FieldCache.DEFAULT.getTermsIndex(searcher.getIndexReader(), facetField);
+ }
catch (IOException e) {
- throw new RuntimeException( "failed to open field cache for: "+f, e );
+ throw new RuntimeException( "failed to open field cache for: "
+ + facetField, e );
}
- finfo[i++] = new FieldFacetStats( f, si, ft, 0 );
+ facetStats.add(new FieldFacetStats(facetField, facetTermsIndex, sf, fsf, nTerms));
}
- final CharsRef spare = new CharsRef();
+
final BytesRef tempBR = new BytesRef();
DocIterator iter = docs.iterator();
while (iter.hasNext()) {
int docID = iter.nextDoc();
- BytesRef raw = all.getTermText(docID, tempBR);
- Double v = null;
- if( raw != null ) {
- v = Double.parseDouble(all.ft.indexedToReadable(raw, spare).toString());
- allstats.accumulate(v);
- }
- else {
- allstats.missing++;
+ BytesRef raw = si.lookup(si.getOrd(docID), tempBR);
+ if( raw.length > 0 ) {
+ allstats.accumulate(raw);
+ } else {
+ allstats.missing();
}
-
- // now check the facets
- for( FieldFacetStats f : finfo ) {
- f.facet(docID, v);
+
+ // now update the facets
+ for (FieldFacetStats f : facetStats) {
+ f.facet(docID, raw);
}
}
-
- if( finfo.length > 0 ) {
- allstats.facets = new HashMap<String, Map<String,StatsValues>>();
- for( FieldFacetStats f : finfo ) {
- allstats.facets.put( f.name, f.facetStatsValues );
- }
+
+ for (FieldFacetStats f : facetStats) {
+ allstats.addFacet(f.name, f.facetStatsValues);
}
return allstats.getStatsValues();
}
Modified: lucene/dev/branches/lucene2621/solr/core/src/java/org/apache/solr/handler/component/StatsValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/solr/core/src/java/org/apache/solr/handler/component/StatsValues.java?rev=1202154&r1=1202153&r2=1202154&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/solr/core/src/java/org/apache/solr/handler/component/StatsValues.java (original)
+++ lucene/dev/branches/lucene2621/solr/core/src/java/org/apache/solr/handler/component/StatsValues.java Tue Nov 15 12:25:46 2011
@@ -19,141 +19,62 @@
package org.apache.solr.handler.component;
-import java.util.HashMap;
import java.util.Map;
+import org.apache.lucene.util.BytesRef;
import org.apache.solr.common.util.NamedList;
-import org.apache.solr.common.util.SimpleOrderedMap;
-
-/** 2/11/2009 - Moved out of StatsComponent to allow open access to UnInvertedField
- * StatsValues is a utility to accumulate statistics on a set of values
- *
- * </p>
- * @see org.apache.solr.handler.component.StatsComponent
- *
-*/
-
-public class StatsValues {
- private static final String FACETS = "facets";
- double min;
- double max;
- double sum;
- double sumOfSquares;
- long count;
- long missing;
-
- // facetField facetValue
- public Map<String, Map<String,StatsValues>> facets;
-
- public StatsValues() {
- reset();
- }
-
- public void accumulate(NamedList stv){
- min = Math.min(min, (Double)stv.get("min"));
- max = Math.max(max, (Double)stv.get("max"));
- sum += (Double)stv.get("sum");
- count += (Long)stv.get("count");
- missing += (Long)stv.get("missing");
- sumOfSquares += (Double)stv.get("sumOfSquares");
-
- NamedList f = (NamedList)stv.get( FACETS );
- if( f != null ) {
- if( facets == null ) {
- facets = new HashMap<String, Map<String,StatsValues>>();
- }
-
- for( int i=0; i< f.size(); i++ ) {
- String field = f.getName(i);
- NamedList vals = (NamedList)f.getVal( i );
- Map<String,StatsValues> addTo = facets.get( field );
- if( addTo == null ) {
- addTo = new HashMap<String,StatsValues>();
- facets.put( field, addTo );
- }
- for( int j=0; j< vals.size(); j++ ) {
- String val = vals.getName(j);
- StatsValues vvals = addTo.get( val );
- if( vvals == null ) {
- vvals = new StatsValues();
- addTo.put( val, vvals );
- }
- vvals.accumulate( (NamedList)vals.getVal( j ) );
- }
- }
- }
- }
-
- public void accumulate(double v){
- sumOfSquares += (v*v); // for std deviation
- min = Math.min(min, v);
- max = Math.max(max, v);
- sum += v;
- count++;
- }
-
- public void accumulate(double v, int c){
- sumOfSquares += (v*v*c); // for std deviation
- min = Math.min(min, v);
- max = Math.max(max, v);
- sum += v*c;
- count+= c;
- }
-
- public void addMissing(int c){
- missing += c;
- }
-
- public double getAverage(){
- return sum / count;
- }
-
- public double getStandardDeviation()
- {
- if( count <= 1.0D )
- return 0.0D;
-
- return Math.sqrt( ( ( count * sumOfSquares ) - ( sum * sum ) )
- / ( count * ( count - 1.0D ) ) );
- }
-
- public long getCount()
- {
- return count;
- }
-
- public void reset(){
- min = Double.MAX_VALUE;
- max = -1.0*Double.MAX_VALUE;
- sum = count = missing = 0;
- sumOfSquares = 0;
- facets = null;
- }
-
- public NamedList<?> getStatsValues(){
- NamedList<Object> res = new SimpleOrderedMap<Object>();
- res.add("min", min);
- res.add("max", max);
- res.add("sum", sum);
- res.add("count", count);
- res.add("missing", missing);
- res.add("sumOfSquares", sumOfSquares );
- res.add("mean", getAverage());
- res.add( "stddev", getStandardDeviation() );
-
- // add the facet stats
- if( facets != null && facets.size() > 0 ) {
- NamedList<NamedList<?>> nl = new SimpleOrderedMap<NamedList<?>>();
- for( Map.Entry<String, Map<String,StatsValues>> entry : facets.entrySet() ) {
- NamedList<NamedList<?>> nl2 = new SimpleOrderedMap<NamedList<?>>();
- nl.add( entry.getKey(), nl2 );
- for( Map.Entry<String, StatsValues> e2 : entry.getValue().entrySet() ) {
- nl2.add( e2.getKey(), e2.getValue().getStatsValues() );
- }
- }
- res.add( FACETS, nl );
- }
- return res;
- }
+/**
+ * StatsValue defines the interface for the collection of statistical values about fields and facets.
+ */
+public interface StatsValues {
+
+ /**
+ * Accumulate the values based on those in the given NamedList
+ *
+ * @param stv NamedList whose values will be used to accumulate the current values
+ */
+ void accumulate(NamedList stv);
+
+ /**
+ * Accumulate the values based on the given value
+ *
+ * @param value Value to use to accumulate the current values
+ */
+ void accumulate(BytesRef value);
+
+ /**
+ * Accumulate the values based on the given value
+ *
+ * @param value Value to use to accumulate the current values
+ * @param count number of times to accumulate this value
+ */
+ void accumulate(BytesRef value, int count);
+
+ /**
+ * Updates the statistics when a document is missing a value
+ */
+ void missing();
+
+ /**
+ * Updates the statistics when multiple documents are missing a value
+ *
+ * @param count number of times to count a missing value
+ */
+ void addMissing(int count);
+
+ /**
+ * Adds the facet statistics for the facet with the given name
+ *
+ * @param facetName Name of the facet
+ * @param facetValues Facet statistics on a per facet value basis
+ */
+ void addFacet(String facetName, Map<String, StatsValues> facetValues);
+
+ /**
+ * Translates the values into a NamedList representation
+ *
+ * @return NamedList representation of the current values
+ */
+ NamedList<?> getStatsValues();
}
Modified: lucene/dev/branches/lucene2621/solr/core/src/java/org/apache/solr/request/UnInvertedField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/solr/core/src/java/org/apache/solr/request/UnInvertedField.java?rev=1202154&r1=1202153&r2=1202154&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/solr/core/src/java/org/apache/solr/request/UnInvertedField.java (original)
+++ lucene/dev/branches/lucene2621/solr/core/src/java/org/apache/solr/request/UnInvertedField.java Tue Nov 15 12:25:46 2011
@@ -29,11 +29,13 @@ import org.apache.solr.common.SolrExcept
import org.apache.solr.core.SolrCore;
import org.apache.solr.schema.FieldType;
+import org.apache.solr.schema.SchemaField;
import org.apache.solr.schema.TrieField;
import org.apache.solr.search.*;
import org.apache.solr.util.LongPriorityQueue;
import org.apache.solr.util.PrimUtils;
import org.apache.solr.handler.component.StatsValues;
+import org.apache.solr.handler.component.StatsValuesFactory;
import org.apache.solr.handler.component.FieldFacetStats;
import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.OpenBitSet;
@@ -461,7 +463,10 @@ public class UnInvertedField extends Doc
//functionality between the two and refactor code somewhat
use.incrementAndGet();
- StatsValues allstats = new StatsValues();
+ SchemaField sf = searcher.getSchema().getField(field);
+ // FieldType ft = sf.getType();
+
+ StatsValues allstats = StatsValuesFactory.createStatsValues(sf);
DocSet docs = baseDocs;
@@ -470,8 +475,6 @@ public class UnInvertedField extends Doc
if (baseSize <= 0) return allstats;
- FieldType ft = searcher.getSchema().getFieldType(field);
-
DocSet missing = docs.andNot( searcher.getDocSet(new TermRangeQuery(field, null, null, false, false)) );
int i = 0;
@@ -479,14 +482,14 @@ public class UnInvertedField extends Doc
//Initialize facetstats, if facets have been passed in
FieldCache.DocTermsIndex si;
for (String f : facet) {
- FieldType facet_ft = searcher.getSchema().getFieldType(f);
+ SchemaField facet_sf = searcher.getSchema().getField(f);
try {
si = FieldCache.DEFAULT.getTermsIndex(searcher.getIndexReader(), f);
}
catch (IOException e) {
throw new RuntimeException("failed to open field cache for: " + f, e);
}
- finfo[i] = new FieldFacetStats(f, si, facet_ft, numTermsInField);
+ finfo[i] = new FieldFacetStats(f, si, sf, facet_sf, numTermsInField);
i++;
}
@@ -580,14 +583,12 @@ public class UnInvertedField extends Doc
}
}
}
- final CharsRef charsRef = new CharsRef();
+
// add results in index order
for (i = 0; i < numTermsInField; i++) {
int c = doNegative ? maxTermCounts[i] - counts[i] : counts[i];
if (c == 0) continue;
- String label = getReadableValue(getTermValue(te, i), ft, charsRef);
- // TODO: we should avoid this re-parse
- Double value = Double.parseDouble(label);
+ BytesRef value = getTermValue(te, i);
allstats.accumulate(value, c);
//as we've parsed the termnum into a value, lets also accumulate fieldfacet statistics
@@ -600,7 +601,6 @@ public class UnInvertedField extends Doc
allstats.addMissing(c);
if (finfo.length > 0) {
- allstats.facets = new HashMap<String, Map<String, StatsValues>>();
for (FieldFacetStats f : finfo) {
Map<String, StatsValues> facetStatsValues = f.facetStatsValues;
FieldType facetType = searcher.getSchema().getFieldType(f.name);
@@ -609,7 +609,7 @@ public class UnInvertedField extends Doc
int missingCount = searcher.numDocs(new TermQuery(new Term(f.name, facetType.toInternal(termLabel))), missing);
entry.getValue().addMissing(missingCount);
}
- allstats.facets.put(f.name, facetStatsValues);
+ allstats.addFacet(f.name, facetStatsValues);
}
}
Modified: lucene/dev/branches/lucene2621/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java?rev=1202154&r1=1202153&r2=1202154&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java (original)
+++ lucene/dev/branches/lucene2621/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java Tue Nov 15 12:25:46 2011
@@ -143,7 +143,6 @@ public class DirectUpdateHandler2 extend
try {
- commitTracker.addedDocument( cmd.commitWithin );
softCommitTracker.addedDocument( -1 ); // TODO: support commitWithin with soft update
if (cmd.overwrite) {
@@ -172,7 +171,7 @@ public class DirectUpdateHandler2 extend
// allow duplicates
writer.addDocument(cmd.getLuceneDocument());
}
-
+ commitTracker.addedDocument( cmd.commitWithin );
// Add to the transaction log *after* successfully adding to the index, if there was no error.
// This ordering ensures that if we log it, it's definitely been added to the the index.
// This also ensures that if a commit sneaks in-between, that we know everything in a particular
Modified: lucene/dev/branches/lucene2621/solr/core/src/test/org/apache/solr/handler/component/StatsComponentTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/solr/core/src/test/org/apache/solr/handler/component/StatsComponentTest.java?rev=1202154&r1=1202153&r2=1202154&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/solr/core/src/test/org/apache/solr/handler/component/StatsComponentTest.java (original)
+++ lucene/dev/branches/lucene2621/solr/core/src/test/org/apache/solr/handler/component/StatsComponentTest.java Tue Nov 15 12:25:46 2011
@@ -16,8 +16,23 @@ package org.apache.solr.handler.componen
* limitations under the License.
*/
+import java.util.Date;
+import java.util.Locale;
+import java.util.Map;
+import java.util.HashMap;
+import java.util.TimeZone;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.MapSolrParams;
+import org.apache.solr.common.params.StatsParams;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.request.LocalSolrQueryRequest;
+import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.util.AbstractSolrTestCase;
+
/**
* Statistics Component Test
*/
@@ -134,6 +149,59 @@ public class StatsComponentTest extends
}
+ public void testFieldStatisticsResultsStringField() throws Exception {
+ SolrCore core = h.getCore();
+ assertU(adoc("id", "1", "active_s", "string1"));
+ assertU(adoc("id", "2", "active_s", "string2"));
+ assertU(adoc("id", "3", "active_s", "string3"));
+ assertU(adoc("id", "4"));
+ assertU(commit());
+
+ Map<String, String> args = new HashMap<String, String>();
+ args.put(CommonParams.Q, "*:*");
+ args.put(StatsParams.STATS, "true");
+ args.put(StatsParams.STATS_FIELD, "active_s");
+ args.put("indent", "true");
+ SolrQueryRequest req = new LocalSolrQueryRequest(core, new MapSolrParams(args));
+
+ assertQ("test string statistics values", req,
+ "//str[@name='min'][.='string1']",
+ "//str[@name='max'][.='string3']",
+ "//long[@name='count'][.='3']",
+ "//long[@name='missing'][.='1']");
+ }
+
+ public void testFieldStatisticsResultsDateField() throws Exception {
+ SolrCore core = h.getCore();
+
+ DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.US);
+ dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+
+ String date1 = dateFormat.format(new Date(123456789)) + "Z";
+ String date2 = dateFormat.format(new Date(987654321)) + "Z";
+
+ assertU(adoc("id", "1", "active_dt", date1));
+ assertU(adoc("id", "2", "active_dt", date2));
+ assertU(adoc("id", "3"));
+ assertU(commit());
+
+ Map<String, String> args = new HashMap<String, String>();
+ args.put(CommonParams.Q, "*:*");
+ args.put(StatsParams.STATS, "true");
+ args.put(StatsParams.STATS_FIELD, "active_dt");
+ args.put("indent", "true");
+ SolrQueryRequest req = new LocalSolrQueryRequest(core, new MapSolrParams(args));
+
+ assertQ("test date statistics values", req,
+ "//long[@name='count'][.='2']",
+ "//long[@name='missing'][.='1']",
+ "//date[@name='min'][.='1970-01-02T10:17:36Z']",
+ "//date[@name='max'][.='1970-01-12T10:20:54Z']",
+ "//date[@name='sum'][.='1970-01-13T20:38:30Z']",
+ "//date[@name='mean'][.='1970-01-07T10:19:15Z']");
+ }
+
+
public void doTestFieldStatisticsMissingResult(String f) throws Exception {
assertU(adoc("id", "1", f, "-10"));
@@ -160,8 +228,8 @@ public class StatsComponentTest extends
assertU(adoc("id", "3", f, "30", "active_s", "false"));
assertU(adoc("id", "4", f, "40", "active_s", "false"));
assertU(commit());
-
- assertQ("test value for active_s=true", req("q","*:*", "stats","true", "stats.field",f, "stats.facet","active_s")
+
+ assertQ("test value for active_s=true", req("q","*:*", "stats","true", "stats.field",f, "stats.facet","active_s","indent","true")
, "//lst[@name='true']/double[@name='min'][.='10.0']"
, "//lst[@name='true']/double[@name='max'][.='20.0']"
, "//lst[@name='true']/double[@name='sum'][.='30.0']"
Modified: lucene/dev/branches/lucene2621/solr/solrj/src/java/org/apache/solr/common/SolrInputDocument.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/solr/solrj/src/java/org/apache/solr/common/SolrInputDocument.java?rev=1202154&r1=1202153&r2=1202154&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/solr/solrj/src/java/org/apache/solr/common/SolrInputDocument.java (original)
+++ lucene/dev/branches/lucene2621/solr/solrj/src/java/org/apache/solr/common/SolrInputDocument.java Tue Nov 15 12:25:46 2011
@@ -41,6 +41,10 @@ public class SolrInputDocument implement
_fields = new LinkedHashMap<String,SolrInputField>();
}
+ public SolrInputDocument(Map<String,SolrInputField> fields) {
+ _fields = fields;
+ }
+
/**
* Remove all fields and boosts from the document
*/