You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rj...@apache.org on 2015/03/31 07:22:50 UTC
svn commit: r1670257 [12/39] - in /lucene/dev/branches/lucene6271: ./
dev-tools/ dev-tools/idea/.idea/libraries/ dev-tools/scripts/ lucene/
lucene/analysis/ lucene/analysis/common/
lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneou...
Modified: lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/search/TestUsageTrackingFilterCachingPolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/search/TestUsageTrackingFilterCachingPolicy.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/search/TestUsageTrackingFilterCachingPolicy.java (original)
+++ lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/search/TestUsageTrackingFilterCachingPolicy.java Tue Mar 31 05:22:40 2015
@@ -19,26 +19,26 @@ package org.apache.lucene.search;
import org.apache.lucene.index.Term;
import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util.RoaringDocIdSet;
public class TestUsageTrackingFilterCachingPolicy extends LuceneTestCase {
- public void testCheapToCache() {
- assertTrue(UsageTrackingFilterCachingPolicy.isCheapToCache(null));
- assertTrue(UsageTrackingFilterCachingPolicy.isCheapToCache(DocIdSet.EMPTY));
- assertTrue(UsageTrackingFilterCachingPolicy.isCheapToCache(new RoaringDocIdSet.Builder(5).add(3).build()));
- assertFalse(UsageTrackingFilterCachingPolicy.isCheapToCache(new DocValuesDocIdSet(5, null) {
- @Override
- protected boolean matchDoc(int doc) {
- return false;
- }
- }));
+ public void testCostlyFilter() {
+ assertTrue(UsageTrackingQueryCachingPolicy.isCostly(new PrefixQuery(new Term("field", "prefix"))));
+ assertTrue(UsageTrackingQueryCachingPolicy.isCostly(NumericRangeQuery.newIntRange("intField", 8, 1, 1000, true, true)));
+ assertFalse(UsageTrackingQueryCachingPolicy.isCostly(new TermQuery(new Term("field", "value"))));
}
- public void testCostlyFilter() {
- assertTrue(UsageTrackingFilterCachingPolicy.isCostly(new PrefixFilter(new Term("field", "prefix"))));
- assertTrue(UsageTrackingFilterCachingPolicy.isCostly(NumericRangeFilter.newIntRange("intField", 8, 1, 1000, true, true)));
- assertFalse(UsageTrackingFilterCachingPolicy.isCostly(new QueryWrapperFilter(new TermQuery(new Term("field", "value")))));
+ public void testBoostIgnored() {
+ Query q1 = new TermQuery(new Term("foo", "bar"));
+ q1.setBoost(2);
+ Query q2 = q1.clone();
+ q2.setBoost(3);
+ Query q3 = q1.clone();
+ q3.setBoost(4);
+ UsageTrackingQueryCachingPolicy policy = new UsageTrackingQueryCachingPolicy();
+ policy.onUse(q1);
+ policy.onUse(q2);
+ assertEquals(2, policy.frequency(q3));
}
}
Modified: lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/search/TestWildcard.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/search/TestWildcard.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/search/TestWildcard.java (original)
+++ lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/search/TestWildcard.java Tue Mar 31 05:22:40 2015
@@ -75,19 +75,19 @@ public class TestWildcard
MultiTermQuery wq = new WildcardQuery(new Term("field", "nowildcard"));
assertMatches(searcher, wq, 1);
- wq.setRewriteMethod(MultiTermQuery.SCORING_BOOLEAN_QUERY_REWRITE);
+ wq.setRewriteMethod(MultiTermQuery.SCORING_BOOLEAN_REWRITE);
wq.setBoost(0.1F);
Query q = searcher.rewrite(wq);
assertTrue(q instanceof TermQuery);
assertEquals(q.getBoost(), wq.getBoost(), 0);
- wq.setRewriteMethod(MultiTermQuery.CONSTANT_SCORE_FILTER_REWRITE);
+ wq.setRewriteMethod(MultiTermQuery.CONSTANT_SCORE_REWRITE);
wq.setBoost(0.2F);
q = searcher.rewrite(wq);
- assertTrue(q instanceof ConstantScoreQuery);
+ assertTrue(q instanceof MultiTermQueryConstantScoreWrapper);
assertEquals(q.getBoost(), wq.getBoost(), 0.1);
- wq.setRewriteMethod(MultiTermQuery.CONSTANT_SCORE_BOOLEAN_QUERY_REWRITE);
+ wq.setRewriteMethod(MultiTermQuery.CONSTANT_SCORE_BOOLEAN_REWRITE);
wq.setBoost(0.4F);
q = searcher.rewrite(wq);
assertTrue(q instanceof ConstantScoreQuery);
@@ -105,7 +105,7 @@ public class TestWildcard
IndexSearcher searcher = newSearcher(reader);
MultiTermQuery wq = new WildcardQuery(new Term("field", ""));
- wq.setRewriteMethod(MultiTermQuery.SCORING_BOOLEAN_QUERY_REWRITE);
+ wq.setRewriteMethod(MultiTermQuery.SCORING_BOOLEAN_REWRITE);
assertMatches(searcher, wq, 0);
Query q = searcher.rewrite(wq);
assertTrue(q instanceof BooleanQuery);
@@ -127,11 +127,9 @@ public class TestWildcard
MultiTermQuery wq = new WildcardQuery(new Term("field", "prefix*"));
assertMatches(searcher, wq, 2);
Terms terms = MultiFields.getTerms(searcher.getIndexReader(), "field");
- assertTrue(wq.getTermsEnum(terms) instanceof PrefixTermsEnum);
wq = new WildcardQuery(new Term("field", "*"));
assertMatches(searcher, wq, 2);
- assertFalse(wq.getTermsEnum(terms) instanceof PrefixTermsEnum);
assertFalse(wq.getTermsEnum(terms).getClass().getSimpleName().contains("AutomatonTermsEnum"));
reader.close();
indexStore.close();
Modified: lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/store/TestMockDirectoryWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/store/TestMockDirectoryWrapper.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/store/TestMockDirectoryWrapper.java (original)
+++ lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/store/TestMockDirectoryWrapper.java Tue Mar 31 05:22:40 2015
@@ -19,8 +19,10 @@ package org.apache.lucene.store;
import java.io.IOException;
+import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LuceneTestCase;
@@ -91,4 +93,16 @@ public class TestMockDirectoryWrapper ex
dir.close();
}
+ public void testMDWinsideOfMDW() throws Exception {
+ // add MDW inside another MDW
+ Directory dir = new MockDirectoryWrapper(random(), newMockDirectory());
+ RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
+ for (int i = 0; i < 20; i++) {
+ iw.addDocument(new Document());
+ }
+ iw.commit();
+ iw.close();
+ dir.close();
+ }
+
}
Modified: lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/util/TestIOUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/util/TestIOUtils.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/util/TestIOUtils.java (original)
+++ lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/util/TestIOUtils.java Tue Mar 31 05:22:40 2015
@@ -215,7 +215,7 @@ public class TestIOUtils extends LuceneT
@Override
public void checkAccess(Path path, AccessMode... modes) throws IOException {
// TODO: kinda screwed up how we do this, but it's easy to get lost. just unravel completely.
- delegate.checkAccess(FilterPath.unwrap(path), modes);
+ delegate.checkAccess(maybeChroot(FilterPath.unwrap(path)), modes);
}
@Override
Modified: lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/util/TestMaxFailuresRule.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/util/TestMaxFailuresRule.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/util/TestMaxFailuresRule.java (original)
+++ lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/util/TestMaxFailuresRule.java Tue Mar 31 05:22:40 2015
@@ -22,7 +22,6 @@ import java.util.concurrent.CountDownLat
import org.apache.lucene.util.junitcompat.WithNestedTests;
import org.junit.Assert;
import org.junit.BeforeClass;
-import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.Description;
import org.junit.runner.JUnitCore;
@@ -37,17 +36,11 @@ import com.carrotsearch.randomizedtestin
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope.Scope;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakZombies;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakZombies.Consequence;
-import com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule;
-import com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule;
/**
* @see TestRuleIgnoreAfterMaxFailures
- * @see SystemPropertiesInvariantRule
*/
public class TestMaxFailuresRule extends WithNestedTests {
- @Rule
- public SystemPropertiesRestoreRule restoreSysProps = new SystemPropertiesRestoreRule();
-
public TestMaxFailuresRule() {
super(true);
}
Modified: lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/util/TestNotDocIdSet.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/util/TestNotDocIdSet.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/util/TestNotDocIdSet.java (original)
+++ lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/util/TestNotDocIdSet.java Tue Mar 31 05:22:40 2015
@@ -38,7 +38,6 @@ public class TestNotDocIdSet extends Bas
throws IOException {
super.assertEquals(numBits, ds1, ds2);
final Bits bits2 = ds2.bits();
- assertTrue(ds2.isCacheable()); // since we wrapped a FixedBitSet
assertNotNull(bits2); // since we wrapped a FixedBitSet
assertEquals(numBits, bits2.length());
for (int i = 0; i < numBits; ++i) {
Modified: lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/util/TestStringHelper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/util/TestStringHelper.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/util/TestStringHelper.java (original)
+++ lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/util/TestStringHelper.java Tue Mar 31 05:22:40 2015
@@ -18,6 +18,73 @@ package org.apache.lucene.util;
*/
public class TestStringHelper extends LuceneTestCase {
+
+ public void testBytesDifference() {
+ BytesRef left = new BytesRef("foobar");
+ BytesRef right = new BytesRef("foozo");
+ assertEquals(3, StringHelper.bytesDifference(left, right));
+ }
+
+ public void testEquals() {
+ assertTrue(StringHelper.equals("foo", "foo"));
+ assertFalse(StringHelper.equals("foo", null));
+ assertFalse(StringHelper.equals(null, "foo"));
+ assertTrue(StringHelper.equals(null, null));
+ }
+
+ public void testStartsWith() {
+ BytesRef ref = new BytesRef("foobar");
+ BytesRef slice = new BytesRef("foo");
+ assertTrue(StringHelper.startsWith(ref, slice));
+ }
+
+ public void testEndsWith() {
+ BytesRef ref = new BytesRef("foobar");
+ BytesRef slice = new BytesRef("bar");
+ assertTrue(StringHelper.endsWith(ref, slice));
+ }
+
+ public void testStartsWithWhole() {
+ BytesRef ref = new BytesRef("foobar");
+ BytesRef slice = new BytesRef("foobar");
+ assertTrue(StringHelper.startsWith(ref, slice));
+ }
+
+ public void testEndsWithWhole() {
+ BytesRef ref = new BytesRef("foobar");
+ BytesRef slice = new BytesRef("foobar");
+ assertTrue(StringHelper.endsWith(ref, slice));
+ }
+
+ public void testContainsAtStart() {
+ BytesRef ref = new BytesRef("foobar");
+ BytesRef slice = new BytesRef("foo");
+ assertTrue(StringHelper.contains(ref, slice, false));
+ }
+
+ public void testContains() {
+ BytesRef ref = new BytesRef("foobar");
+ BytesRef slice = new BytesRef("ooba");
+ assertTrue(StringHelper.contains(ref, slice, false));
+ }
+
+ public void testContainsAtEnd() {
+ BytesRef ref = new BytesRef("foobar");
+ BytesRef slice = new BytesRef("bar");
+ assertTrue(StringHelper.contains(ref, slice, false));
+ }
+
+ public void testContainsWhole() {
+ BytesRef ref = new BytesRef("foobar");
+ BytesRef slice = new BytesRef("foobar");
+ assertTrue(StringHelper.contains(ref, slice, false));
+ }
+
+ public void testContainsIgnoreCase() {
+ BytesRef ref = new BytesRef("FooBar");
+ BytesRef slice = new BytesRef("bar");
+ assertTrue(StringHelper.contains(ref, slice, true));
+ }
public void testMurmurHash3() throws Exception {
// Hashes computed using murmur3_32 from https://code.google.com/p/pyfasthash
Modified: lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/util/TestTimSorter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/util/TestTimSorter.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/util/TestTimSorter.java (original)
+++ lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/util/TestTimSorter.java Tue Mar 31 05:22:40 2015
@@ -17,6 +17,13 @@ package org.apache.lucene.util;
* limitations under the License.
*/
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.lucene.util.packed.PackedInts;
+
+import com.carrotsearch.randomizedtesting.generators.RandomInts;
+
public class TestTimSorter extends BaseSortTestCase {
public TestTimSorter() {
@@ -28,4 +35,140 @@ public class TestTimSorter extends BaseS
return new ArrayTimSorter<>(arr, ArrayUtil.<Entry>naturalComparator(), TestUtil.nextInt(random(), 0, arr.length));
}
+ public void testWorstCaseStackSize() {
+ // we need large arrays to be able to reproduce this bug
+ final int length;
+ if (TEST_NIGHTLY) {
+ length = RandomInts.randomIntBetween(random(), 140000000, Integer.MAX_VALUE);
+ } else {
+ length = RandomInts.randomIntBetween(random(), 140000000, 200000000);
+ }
+ final PackedInts.Mutable arr = generateWorstCaseArray(length);
+ new TimSorter(0) {
+
+ @Override
+ protected void swap(int i, int j) {
+ final long tmp = arr.get(i);
+ arr.set(i, arr.get(j));
+ arr.set(j, tmp);
+ }
+
+ @Override
+ protected int compare(int i, int j) {
+ return Long.compare(arr.get(i), arr.get(j));
+ }
+
+ @Override
+ protected void save(int i, int len) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ protected void restore(int i, int j) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ protected void copy(int src, int dest) {
+ arr.set(dest, arr.get(src));
+ }
+
+ @Override
+ protected int compareSaved(int i, int j) {
+ throw new UnsupportedOperationException();
+ }
+ }.sort(0, length);
+ }
+
+ /** Create an array for the given list of runs. */
+ private static PackedInts.Mutable createArray(int length, List<Integer> runs) {
+ PackedInts.Mutable array = PackedInts.getMutable(length, 1, 0);
+ int endRun = -1;
+ for (long len : runs) {
+ array.set(endRun += len, 1);
+ }
+ array.set(length - 1, 0);
+ return array;
+ }
+
+ /** Create an array that triggers a worst-case sequence of run lens. */
+ public static PackedInts.Mutable generateWorstCaseArray(int length) {
+ final int minRun = TimSorter.minRun(length);
+ final List<Integer> runs = runsWorstCase(length, minRun);
+ return createArray(length, runs);
+ }
+
+ //
+ // Code below is borrowed from https://github.com/abstools/java-timsort-bug/blob/master/TestTimSort.java
+ //
+
+ /**
+ * Fills <code>runs</code> with a sequence of run lengths of the form<br>
+ * Y_n x_{n,1} x_{n,2} ... x_{n,l_n} <br>
+ * Y_{n-1} x_{n-1,1} x_{n-1,2} ... x_{n-1,l_{n-1}} <br>
+ * ... <br>
+ * Y_1 x_{1,1} x_{1,2} ... x_{1,l_1}<br>
+ * The Y_i's are chosen to satisfy the invariant throughout execution,
+ * but the x_{i,j}'s are merged (by <code>TimSort.mergeCollapse</code>)
+ * into an X_i that violates the invariant.
+ */
+ private static List<Integer> runsWorstCase(int length, int minRun) {
+ List<Integer> runs = new LinkedList<>();
+
+ int runningTotal = 0, Y=minRun+4, X=minRun;
+
+ while((long) runningTotal+Y+X <= length) {
+ runningTotal += X + Y;
+ generateWrongElem(X, minRun, runs);
+ runs.add(0,Y);
+
+ // X_{i+1} = Y_i + x_{i,1} + 1, since runs.get(1) = x_{i,1}
+ X = Y + runs.get(1) + 1;
+
+ // Y_{i+1} = X_{i+1} + Y_i + 1
+ Y += X + 1;
+ }
+
+ if((long) runningTotal + X <= length) {
+ runningTotal += X;
+ generateWrongElem(X, minRun, runs);
+ }
+
+ runs.add(length-runningTotal);
+ return runs;
+ }
+
+ /**
+ * Adds a sequence x_1, ..., x_n of run lengths to <code>runs</code> such that:<br>
+ * 1. X = x_1 + ... + x_n <br>
+ * 2. x_j >= minRun for all j <br>
+ * 3. x_1 + ... + x_{j-2} < x_j < x_1 + ... + x_{j-1} for all j <br>
+ * These conditions guarantee that TimSort merges all x_j's one by one
+ * (resulting in X) using only merges on the second-to-last element.
+ * @param X The sum of the sequence that should be added to runs.
+ */
+ private static void generateWrongElem(int X, int minRun, List<Integer> runs) {
+ for(int newTotal; X >= 2*minRun+1; X = newTotal) {
+ //Default strategy
+ newTotal = X/2 + 1;
+
+ //Specialized strategies
+ if(3*minRun+3 <= X && X <= 4*minRun+1) {
+ // add x_1=MIN+1, x_2=MIN, x_3=X-newTotal to runs
+ newTotal = 2*minRun+1;
+ } else if(5*minRun+5 <= X && X <= 6*minRun+5) {
+ // add x_1=MIN+1, x_2=MIN, x_3=MIN+2, x_4=X-newTotal to runs
+ newTotal = 3*minRun+3;
+ } else if(8*minRun+9 <= X && X <= 10*minRun+9) {
+ // add x_1=MIN+1, x_2=MIN, x_3=MIN+2, x_4=2MIN+2, x_5=X-newTotal to runs
+ newTotal = 5*minRun+5;
+ } else if(13*minRun+15 <= X && X <= 16*minRun+17) {
+ // add x_1=MIN+1, x_2=MIN, x_3=MIN+2, x_4=2MIN+2, x_5=3MIN+4, x_6=X-newTotal to runs
+ newTotal = 8*minRun+9;
+ }
+ runs.add(0, X-newTotal);
+ }
+ runs.add(0, X);
+ }
+
}
Modified: lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/util/automaton/TestAutomaton.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/util/automaton/TestAutomaton.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/util/automaton/TestAutomaton.java (original)
+++ lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/util/automaton/TestAutomaton.java Tue Mar 31 05:22:40 2015
@@ -1104,4 +1104,51 @@ public class TestAutomaton extends Lucen
throw ae;
}
}
+
+ private static IntsRef toIntsRef(String s) {
+ IntsRefBuilder b = new IntsRefBuilder();
+ for (int i = 0, cp = 0; i < s.length(); i += Character.charCount(cp)) {
+ cp = s.codePointAt(i);
+ b.append(cp);
+ }
+
+ return b.get();
+ }
+
+ public void testGetSingleton() {
+ int iters = atLeast(10000);
+ for(int iter=0;iter<iters;iter++) {
+ String s = TestUtil.randomRealisticUnicodeString(random());
+ Automaton a = Automata.makeString(s);
+ assertEquals(toIntsRef(s), Operations.getSingleton(a));
+ }
+ }
+
+ public void testGetSingletonEmptyString() {
+ Automaton a = new Automaton();
+ int s = a.createState();
+ a.setAccept(s, true);
+ a.finishState();
+ assertEquals(new IntsRef(), Operations.getSingleton(a));
+ }
+
+ public void testGetSingletonNothing() {
+ Automaton a = new Automaton();
+ a.createState();
+ a.finishState();
+ assertNull(Operations.getSingleton(a));
+ }
+
+ public void testGetSingletonTwo() {
+ Automaton a = new Automaton();
+ int s = a.createState();
+ int x = a.createState();
+ a.setAccept(x, true);
+ a.addTransition(s, x, 55);
+ int y = a.createState();
+ a.setAccept(y, true);
+ a.addTransition(s, y, 58);
+ a.finishState();
+ assertNull(Operations.getSingleton(a));
+ }
}
Modified: lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/util/automaton/TestCompiledAutomaton.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/util/automaton/TestCompiledAutomaton.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/util/automaton/TestCompiledAutomaton.java (original)
+++ lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/util/automaton/TestCompiledAutomaton.java Tue Mar 31 05:22:40 2015
@@ -38,7 +38,7 @@ public class TestCompiledAutomaton exten
}
Collections.sort(terms);
final Automaton a = DaciukMihovAutomatonBuilder.build(terms);
- return new CompiledAutomaton(a, true, false, maxDeterminizedStates);
+ return new CompiledAutomaton(a, true, false, maxDeterminizedStates, false);
}
private void testFloor(CompiledAutomaton c, String input, String expected) {
@@ -121,4 +121,43 @@ public class TestCompiledAutomaton exten
testFloor(c, "aa", null);
testFloor(c, "zzz", "goo");
}
+
+ // LUCENE-6367
+ public void testBinaryAll() throws Exception {
+ Automaton a = new Automaton();
+ int state = a.createState();
+ a.setAccept(state, true);
+ a.addTransition(state, state, 0, 0xff);
+ a.finishState();
+
+ CompiledAutomaton ca = new CompiledAutomaton(a, null, true, Integer.MAX_VALUE, true);
+ assertEquals(CompiledAutomaton.AUTOMATON_TYPE.ALL, ca.type);
+ }
+
+ // LUCENE-6367
+ public void testUnicodeAll() throws Exception {
+ Automaton a = new Automaton();
+ int state = a.createState();
+ a.setAccept(state, true);
+ a.addTransition(state, state, 0, Character.MAX_CODE_POINT);
+ a.finishState();
+
+ CompiledAutomaton ca = new CompiledAutomaton(a, null, true, Integer.MAX_VALUE, false);
+ assertEquals(CompiledAutomaton.AUTOMATON_TYPE.ALL, ca.type);
+ }
+
+ // LUCENE-6367
+ public void testBinarySingleton() throws Exception {
+ // This is just ascii so we can pretend it's binary:
+ Automaton a = Automata.makeString("foobar");
+ CompiledAutomaton ca = new CompiledAutomaton(a, null, true, Integer.MAX_VALUE, true);
+ assertEquals(CompiledAutomaton.AUTOMATON_TYPE.SINGLE, ca.type);
+ }
+
+ // LUCENE-6367
+ public void testUnicodeSingleton() throws Exception {
+ Automaton a = Automata.makeString(TestUtil.randomRealisticUnicodeString(random()));
+ CompiledAutomaton ca = new CompiledAutomaton(a, null, true, Integer.MAX_VALUE, false);
+ assertEquals(CompiledAutomaton.AUTOMATON_TYPE.SINGLE, ca.type);
+ }
}
Modified: lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestSeedFromUncaught.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestSeedFromUncaught.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestSeedFromUncaught.java (original)
+++ lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestSeedFromUncaught.java Tue Mar 31 05:22:40 2015
@@ -18,7 +18,6 @@ package org.apache.lucene.util.junitcomp
*/
import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util.LuceneTestCase.SuppressSysoutChecks;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.JUnitCore;
Modified: lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/util/junitcompat/WithNestedTests.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/util/junitcompat/WithNestedTests.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/util/junitcompat/WithNestedTests.java (original)
+++ lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/util/junitcompat/WithNestedTests.java Tue Mar 31 05:22:40 2015
@@ -29,6 +29,7 @@ import org.apache.lucene.util.LuceneTest
import org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures;
import org.apache.lucene.util.TestRuleIgnoreTestSuites;
import org.apache.lucene.util.TestRuleMarkFailure;
+import org.apache.lucene.util.TestRuleRestoreSystemProperties;
import org.apache.lucene.util.LuceneTestCase.SuppressSysoutChecks;
import org.junit.After;
import org.junit.Assert;
@@ -44,7 +45,6 @@ import org.junit.runner.notification.Fai
import com.carrotsearch.randomizedtesting.RandomizedRunner;
import com.carrotsearch.randomizedtesting.RandomizedTest;
import com.carrotsearch.randomizedtesting.SysGlobals;
-import com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule;
import com.carrotsearch.randomizedtesting.rules.TestRuleAdapter;
/**
@@ -124,7 +124,7 @@ public abstract class WithNestedTests {
{
final TestRuleMarkFailure marker = new TestRuleMarkFailure();
rules = RuleChain
- .outerRule(new SystemPropertiesRestoreRule())
+ .outerRule(new TestRuleRestoreSystemProperties(TestRuleIgnoreTestSuites.PROPERTY_RUN_NESTED))
.around(new TestRuleAdapter() {
@Override
protected void afterAlways(List<Throwable> errors) throws Throwable {
Modified: lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/util/makeEuroparlLineFile.py
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/util/makeEuroparlLineFile.py?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/util/makeEuroparlLineFile.py (original)
+++ lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/util/makeEuroparlLineFile.py Tue Mar 31 05:22:40 2015
@@ -35,16 +35,16 @@ reChapterOnly = re.compile('^<CHAPTER ID
reTagOnly = re.compile('^<.*?>$')
reNumberOnly = re.compile(r'^\d+\.?$')
-docCount = 0
+maxDoc = 0
didEnglish = False
def write(date, title, pending, fOut):
- global docCount
+ global maxDoc
body = ' '.join(pending).replace('\t', ' ').strip()
if len(body) > 0:
line = '%s\t%s\t%s\n' % (title, date, body)
fOut.write(line)
- docCount += 1
+ maxDoc += 1
del pending[:]
if VERBOSE:
print len(body)
@@ -109,10 +109,10 @@ fOut = open(fileOut, 'wb')
for fileName in glob.glob('%s/??-??.tgz' % dirIn):
if fileName.endswith('.tgz'):
- print 'process %s; %d docs so far...' % (fileName, docCount)
+ print 'process %s; %d docs so far...' % (fileName, maxDoc)
processTar(fileName, fOut)
-print 'TOTAL: %s' % docCount
+print 'TOTAL: %s' % maxDoc
#run something like this:
"""
Modified: lucene/dev/branches/lucene6271/lucene/demo/src/java/org/apache/lucene/demo/facet/DistanceFacetsExample.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/demo/src/java/org/apache/lucene/demo/facet/DistanceFacetsExample.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/demo/src/java/org/apache/lucene/demo/facet/DistanceFacetsExample.java (original)
+++ lucene/dev/branches/lucene6271/lucene/demo/src/java/org/apache/lucene/demo/facet/DistanceFacetsExample.java Tue Mar 31 05:22:40 2015
@@ -48,7 +48,6 @@ import org.apache.lucene.search.BooleanQ
import org.apache.lucene.search.Filter;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.MatchAllDocsQuery;
-import org.apache.lucene.search.NumericRangeFilter;
import org.apache.lucene.search.NumericRangeQuery;
import org.apache.lucene.search.QueryWrapperFilter;
import org.apache.lucene.search.SortField;
@@ -184,7 +183,7 @@ public class DistanceFacetsExample imple
BooleanQuery f = new BooleanQuery();
// Add latitude range filter:
- f.add(NumericRangeFilter.newDoubleRange("latitude", Math.toDegrees(minLat), Math.toDegrees(maxLat), true, true),
+ f.add(NumericRangeQuery.newDoubleRange("latitude", Math.toDegrees(minLat), Math.toDegrees(maxLat), true, true),
BooleanClause.Occur.FILTER);
// Add longitude range filter:
@@ -198,7 +197,7 @@ public class DistanceFacetsExample imple
BooleanClause.Occur.SHOULD);
f.add(lonF, BooleanClause.Occur.MUST);
} else {
- f.add(NumericRangeFilter.newDoubleRange("longitude", Math.toDegrees(minLng), Math.toDegrees(maxLng), true, true),
+ f.add(NumericRangeQuery.newDoubleRange("longitude", Math.toDegrees(minLng), Math.toDegrees(maxLng), true, true),
BooleanClause.Occur.FILTER);
}
Modified: lucene/dev/branches/lucene6271/lucene/facet/src/java/org/apache/lucene/facet/DrillSideways.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/facet/src/java/org/apache/lucene/facet/DrillSideways.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/facet/src/java/org/apache/lucene/facet/DrillSideways.java (original)
+++ lucene/dev/branches/lucene6271/lucene/facet/src/java/org/apache/lucene/facet/DrillSideways.java Tue Mar 31 05:22:40 2015
@@ -26,6 +26,7 @@ import org.apache.lucene.facet.sortedset
import org.apache.lucene.facet.sortedset.SortedSetDocValuesReaderState;
import org.apache.lucene.facet.taxonomy.FastTaxonomyFacetCounts;
import org.apache.lucene.facet.taxonomy.TaxonomyReader;
+import org.apache.lucene.search.FilterCollector;
import org.apache.lucene.search.LeafCollector;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
@@ -173,6 +174,16 @@ public class DrillSideways {
drillDownQueries[i-startClause] = clauses[i].getQuery();
}
DrillSidewaysQuery dsq = new DrillSidewaysQuery(baseQuery, drillDownCollector, drillSidewaysCollectors, drillDownQueries, scoreSubDocsAtOnce());
+ if (hitCollector.needsScores() == false) {
+ // this is a borrible hack in order to make sure IndexSearcher will not
+ // attempt to cache the DrillSidewaysQuery
+ hitCollector = new FilterCollector(hitCollector) {
+ @Override
+ public boolean needsScores() {
+ return true;
+ }
+ };
+ }
searcher.search(dsq, hitCollector);
return new DrillSidewaysResult(buildFacetsResult(drillDownCollector, drillSidewaysCollectors, drillDownDims.keySet().toArray(new String[drillDownDims.size()])), null);
Modified: lucene/dev/branches/lucene6271/lucene/facet/src/java/org/apache/lucene/facet/DrillSidewaysQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/facet/src/java/org/apache/lucene/facet/DrillSidewaysQuery.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/facet/src/java/org/apache/lucene/facet/DrillSidewaysQuery.java (original)
+++ lucene/dev/branches/lucene6271/lucene/facet/src/java/org/apache/lucene/facet/DrillSidewaysQuery.java Tue Mar 31 05:22:40 2015
@@ -33,8 +33,10 @@ import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.Weight;
import org.apache.lucene.util.Bits;
/** Only purpose is to punch through and return a
- * DrillSidewaysScorer */
+ * DrillSidewaysScorer*/
+// TODO change the way DrillSidewaysScorer is used, this query does not work
+// with filter caching
class DrillSidewaysQuery extends Query {
final Query baseQuery;
final Collector drillDownCollector;
Modified: lucene/dev/branches/lucene6271/lucene/facet/src/java/org/apache/lucene/facet/DrillSidewaysScorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/facet/src/java/org/apache/lucene/facet/DrillSidewaysScorer.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/facet/src/java/org/apache/lucene/facet/DrillSidewaysScorer.java (original)
+++ lucene/dev/branches/lucene6271/lucene/facet/src/java/org/apache/lucene/facet/DrillSidewaysScorer.java Tue Mar 31 05:22:40 2015
@@ -98,6 +98,24 @@ class DrillSidewaysScorer extends BulkSc
// change up the order of the conjuntions below
assert baseScorer != null;
+ // some scorers, eg ReqExlScorer, can hit NPE if cost is called after nextDoc
+ long baseQueryCost = baseScorer.cost();
+
+ final int numDims = dims.length;
+
+ long drillDownCost = 0;
+ for (int dim=0;dim<numDims;dim++) {
+ DocIdSetIterator disi = dims[dim].disi;
+ if (dims[dim].bits == null && disi != null) {
+ drillDownCost += disi.cost();
+ }
+ }
+
+ long drillDownAdvancedCost = 0;
+ if (numDims > 1 && dims[1].disi != null) {
+ drillDownAdvancedCost = dims[1].disi.cost();
+ }
+
// Position all scorers to their first matching doc:
baseScorer.nextDoc();
int numBits = 0;
@@ -109,14 +127,11 @@ class DrillSidewaysScorer extends BulkSc
}
}
- final int numDims = dims.length;
-
Bits[] bits = new Bits[numBits];
LeafCollector[] bitsSidewaysCollectors = new LeafCollector[numBits];
DocIdSetIterator[] disis = new DocIdSetIterator[numDims-numBits];
LeafCollector[] sidewaysCollectors = new LeafCollector[numDims-numBits];
- long drillDownCost = 0;
int disiUpto = 0;
int bitsUpto = 0;
for (int dim=0;dim<numDims;dim++) {
@@ -125,9 +140,6 @@ class DrillSidewaysScorer extends BulkSc
disis[disiUpto] = disi;
sidewaysCollectors[disiUpto] = dims[dim].sidewaysLeafCollector;
disiUpto++;
- if (disi != null) {
- drillDownCost += disi.cost();
- }
} else {
bits[bitsUpto] = dims[dim].bits;
bitsSidewaysCollectors[bitsUpto] = dims[dim].sidewaysLeafCollector;
@@ -135,8 +147,6 @@ class DrillSidewaysScorer extends BulkSc
}
}
- long baseQueryCost = baseScorer.cost();
-
/*
System.out.println("\nbaseDocID=" + baseScorer.docID() + " est=" + estBaseHitCount);
System.out.println(" maxDoc=" + context.reader().maxDoc());
@@ -150,7 +160,7 @@ class DrillSidewaysScorer extends BulkSc
if (bitsUpto > 0 || scoreSubDocsAtOnce || baseQueryCost < drillDownCost/10) {
//System.out.println("queryFirst: baseScorer=" + baseScorer + " disis.length=" + disis.length + " bits.length=" + bits.length);
doQueryFirstScoring(collector, disis, sidewaysCollectors, bits, bitsSidewaysCollectors);
- } else if (numDims > 1 && (dims[1].disi == null || dims[1].disi.cost() < baseQueryCost/10)) {
+ } else if (numDims > 1 && (dims[1].disi == null || drillDownAdvancedCost < baseQueryCost/10)) {
//System.out.println("drillDownAdvance");
doDrillDownAdvanceScoring(collector, disis, sidewaysCollectors);
} else {
Modified: lucene/dev/branches/lucene6271/lucene/facet/src/java/org/apache/lucene/facet/range/LongRange.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/facet/src/java/org/apache/lucene/facet/range/LongRange.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/facet/src/java/org/apache/lucene/facet/range/LongRange.java (original)
+++ lucene/dev/branches/lucene6271/lucene/facet/src/java/org/apache/lucene/facet/range/LongRange.java Tue Mar 31 05:22:40 2015
@@ -26,6 +26,7 @@ import org.apache.lucene.queries.functio
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.Filter;
+import org.apache.lucene.search.FilteredDocIdSet;
import org.apache.lucene.util.Bits;
/** Represents a range over long values.
@@ -113,52 +114,33 @@ public final class LongRange extends Ran
final int maxDoc = context.reader().maxDoc();
- final Bits fastMatchBits;
+ final DocIdSet fastMatchDocs;
if (fastMatchFilter != null) {
- DocIdSet dis = fastMatchFilter.getDocIdSet(context, null);
- if (dis == null) {
+ fastMatchDocs = fastMatchFilter.getDocIdSet(context, null);
+ if (fastMatchDocs == null) {
// No documents match
return null;
}
- fastMatchBits = dis.bits();
- if (fastMatchBits == null) {
- throw new IllegalArgumentException("fastMatchFilter does not implement DocIdSet.bits");
- }
} else {
- fastMatchBits = null;
+ fastMatchDocs = new DocIdSet() {
+ @Override
+ public long ramBytesUsed() {
+ return 0;
+ }
+ @Override
+ public DocIdSetIterator iterator() throws IOException {
+ return DocIdSetIterator.all(maxDoc);
+ }
+ };
}
- return new DocIdSet() {
-
- @Override
- public Bits bits() {
- return new Bits() {
- @Override
- public boolean get(int docID) {
- if (acceptDocs != null && acceptDocs.get(docID) == false) {
- return false;
- }
- if (fastMatchBits != null && fastMatchBits.get(docID) == false) {
- return false;
- }
- return accept(values.longVal(docID));
- }
-
- @Override
- public int length() {
- return maxDoc;
- }
- };
- }
-
- @Override
- public DocIdSetIterator iterator() {
- throw new UnsupportedOperationException("this filter can only be accessed via bits()");
- }
-
+ return new FilteredDocIdSet(fastMatchDocs) {
@Override
- public long ramBytesUsed() {
- return 0L;
+ protected boolean match(int docID) {
+ if (acceptDocs != null && acceptDocs.get(docID) == false) {
+ return false;
+ }
+ return accept(values.longVal(docID));
}
};
}
Modified: lucene/dev/branches/lucene6271/lucene/facet/src/java/org/apache/lucene/facet/range/LongRangeFacetCounts.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/facet/src/java/org/apache/lucene/facet/range/LongRangeFacetCounts.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/facet/src/java/org/apache/lucene/facet/range/LongRangeFacetCounts.java (original)
+++ lucene/dev/branches/lucene6271/lucene/facet/src/java/org/apache/lucene/facet/range/LongRangeFacetCounts.java Tue Mar 31 05:22:40 2015
@@ -77,27 +77,30 @@ public class LongRangeFacetCounts extend
FunctionValues fv = valueSource.getValues(Collections.emptyMap(), hits.context);
totCount += hits.totalHits;
- Bits bits;
+ final DocIdSetIterator fastMatchDocs;
if (fastMatchFilter != null) {
DocIdSet dis = fastMatchFilter.getDocIdSet(hits.context, null);
if (dis == null) {
// No documents match
continue;
}
- bits = dis.bits();
- if (bits == null) {
- throw new IllegalArgumentException("fastMatchFilter does not implement DocIdSet.bits");
- }
+ fastMatchDocs = dis.iterator();
} else {
- bits = null;
+ fastMatchDocs = null;
}
DocIdSetIterator docs = hits.bits.iterator();
- int doc;
- while ((doc = docs.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
- if (bits != null && bits.get(doc) == false) {
- doc++;
- continue;
+ for (int doc = docs.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; ) {
+ if (fastMatchDocs != null) {
+ int fastMatchDoc = fastMatchDocs.docID();
+ if (fastMatchDoc < doc) {
+ fastMatchDoc = fastMatchDocs.advance(doc);
+ }
+
+ if (doc != fastMatchDoc) {
+ doc = docs.advance(fastMatchDoc);
+ continue;
+ }
}
// Skip missing docs:
if (fv.exists(doc)) {
@@ -105,6 +108,8 @@ public class LongRangeFacetCounts extend
} else {
missingCount++;
}
+
+ doc = docs.nextDoc();
}
}
Modified: lucene/dev/branches/lucene6271/lucene/facet/src/java/org/apache/lucene/facet/range/Range.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/facet/src/java/org/apache/lucene/facet/range/Range.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/facet/src/java/org/apache/lucene/facet/range/Range.java (original)
+++ lucene/dev/branches/lucene6271/lucene/facet/src/java/org/apache/lucene/facet/range/Range.java Tue Mar 31 05:22:40 2015
@@ -22,7 +22,7 @@ import org.apache.lucene.facet.DrillSide
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.search.Filter;
import org.apache.lucene.search.FilteredQuery; // javadocs
-import org.apache.lucene.search.NumericRangeFilter; // javadocs
+import org.apache.lucene.search.NumericRangeQuery;
/** Base class for a single labeled range.
*
@@ -48,7 +48,7 @@ public abstract class Range {
* FilteredQuery#QUERY_FIRST_FILTER_STRATEGY}. If the
* {@link ValueSource} is static, e.g. an indexed numeric
* field, then it may be more efficient to use {@link
- * NumericRangeFilter}. The provided fastMatchFilter,
+ * NumericRangeQuery}. The provided fastMatchFilter,
* if non-null, will first be consulted, and only if
* that is set for each document will the range then be
* checked. */
@@ -61,7 +61,7 @@ public abstract class Range {
* {@link FilteredQuery} using its {@link
* FilteredQuery#QUERY_FIRST_FILTER_STRATEGY}. If the
* {@link ValueSource} is static, e.g. an indexed numeric
- * field, then it may be more efficient to use {@link NumericRangeFilter}. */
+ * field, then it may be more efficient to use {@link NumericRangeQuery}. */
public Filter getFilter(ValueSource valueSource) {
return getFilter(null, valueSource);
}
Modified: lucene/dev/branches/lucene6271/lucene/facet/src/test/org/apache/lucene/facet/TestDrillSideways.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/facet/src/test/org/apache/lucene/facet/TestDrillSideways.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/facet/src/test/org/apache/lucene/facet/TestDrillSideways.java (original)
+++ lucene/dev/branches/lucene6271/lucene/facet/src/test/org/apache/lucene/facet/TestDrillSideways.java Tue Mar 31 05:22:40 2015
@@ -44,6 +44,9 @@ import org.apache.lucene.index.IndexRead
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.Collector;
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.Filter;
import org.apache.lucene.search.FilteredQuery;
@@ -1076,5 +1079,46 @@ public class TestDrillSideways extends F
writer.close();
IOUtils.close(taxoWriter, searcher.getIndexReader(), taxoReader, dir, taxoDir);
}
+
+ public void testScorer() throws Exception {
+ // LUCENE-6001 some scorers, eg ReqExlScorer, can hit NPE if cost is called after nextDoc
+ Directory dir = newDirectory();
+ Directory taxoDir = newDirectory();
+
+ // Writes facet ords to a separate directory from the
+ // main index:
+ DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir, IndexWriterConfig.OpenMode.CREATE);
+
+ FacetsConfig config = new FacetsConfig();
+
+ RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
+
+ Document doc = new Document();
+ doc.add(newTextField("field", "foo bar", Field.Store.NO));
+ doc.add(new FacetField("Author", "Bob"));
+ doc.add(new FacetField("dim", "a"));
+ writer.addDocument(config.build(taxoWriter, doc));
+
+ // NRT open
+ IndexSearcher searcher = newSearcher(writer.getReader());
+
+ // NRT open
+ TaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoWriter);
+
+ DrillSideways ds = new DrillSideways(searcher, config, taxoReader);
+
+ BooleanQuery bq = new BooleanQuery(true);
+ bq.add(new TermQuery(new Term("field", "foo")), BooleanClause.Occur.MUST);
+ bq.add(new TermQuery(new Term("field", "bar")), BooleanClause.Occur.MUST_NOT);
+ DrillDownQuery ddq = new DrillDownQuery(config, bq);
+ ddq.add("field", "foo");
+ ddq.add("author", bq);
+ ddq.add("dim", bq);
+ DrillSidewaysResult r = ds.search(null, ddq, 10);
+ assertEquals(0, r.hits.totalHits);
+
+ writer.close();
+ IOUtils.close(searcher.getIndexReader(), taxoReader, taxoWriter, dir, taxoDir);
+ }
}
Modified: lucene/dev/branches/lucene6271/lucene/facet/src/test/org/apache/lucene/facet/range/TestRangeFacetCounts.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/facet/src/test/org/apache/lucene/facet/range/TestRangeFacetCounts.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/facet/src/test/org/apache/lucene/facet/range/TestRangeFacetCounts.java (original)
+++ lucene/dev/branches/lucene6271/lucene/facet/src/test/org/apache/lucene/facet/range/TestRangeFacetCounts.java Tue Mar 31 05:22:40 2015
@@ -31,8 +31,8 @@ import org.apache.lucene.document.FloatF
import org.apache.lucene.document.LongField;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.facet.DrillDownQuery;
-import org.apache.lucene.facet.DrillSideways.DrillSidewaysResult;
import org.apache.lucene.facet.DrillSideways;
+import org.apache.lucene.facet.DrillSideways.DrillSidewaysResult;
import org.apache.lucene.facet.FacetField;
import org.apache.lucene.facet.FacetResult;
import org.apache.lucene.facet.FacetTestCase;
@@ -44,10 +44,10 @@ import org.apache.lucene.facet.MultiFace
import org.apache.lucene.facet.taxonomy.TaxonomyReader;
import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader;
import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter;
-import org.apache.lucene.index.LeafReader;
-import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
@@ -55,18 +55,18 @@ import org.apache.lucene.queries.functio
import org.apache.lucene.queries.function.valuesource.DoubleFieldSource;
import org.apache.lucene.queries.function.valuesource.FloatFieldSource;
import org.apache.lucene.queries.function.valuesource.LongFieldSource;
-import org.apache.lucene.search.CachingWrapperFilter;
+import org.apache.lucene.search.CachingWrapperQuery;
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.Filter;
-import org.apache.lucene.search.FilterCachingPolicy;
+import org.apache.lucene.search.QueryCachingPolicy;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.MatchAllDocsQuery;
-import org.apache.lucene.search.NumericRangeFilter;
import org.apache.lucene.search.NumericRangeQuery;
import org.apache.lucene.search.QueryWrapperFilter;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BitDocIdSet;
+import org.apache.lucene.util.Bits;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.TestUtil;
@@ -481,9 +481,9 @@ public class TestRangeFacetCounts extend
Filter fastMatchFilter;
if (random().nextBoolean()) {
if (random().nextBoolean()) {
- fastMatchFilter = NumericRangeFilter.newLongRange("field", minValue, maxValue, true, true);
+ fastMatchFilter = new QueryWrapperFilter(NumericRangeQuery.newLongRange("field", minValue, maxValue, true, true));
} else {
- fastMatchFilter = NumericRangeFilter.newLongRange("field", minAcceptedValue, maxAcceptedValue, true, true);
+ fastMatchFilter = new QueryWrapperFilter(NumericRangeQuery.newLongRange("field", minAcceptedValue, maxAcceptedValue, true, true));
}
} else {
fastMatchFilter = null;
@@ -505,11 +505,7 @@ public class TestRangeFacetCounts extend
// Test drill-down:
DrillDownQuery ddq = new DrillDownQuery(config);
if (random().nextBoolean()) {
- if (random().nextBoolean()) {
- ddq.add("field", NumericRangeFilter.newLongRange("field", range.min, range.max, range.minInclusive, range.maxInclusive));
- } else {
- ddq.add("field", NumericRangeQuery.newLongRange("field", range.min, range.max, range.minInclusive, range.maxInclusive));
- }
+ ddq.add("field", NumericRangeQuery.newLongRange("field", range.min, range.max, range.minInclusive, range.maxInclusive));
} else {
ddq.add("field", range.getFilter(fastMatchFilter, vs));
}
@@ -640,9 +636,9 @@ public class TestRangeFacetCounts extend
Filter fastMatchFilter;
if (random().nextBoolean()) {
if (random().nextBoolean()) {
- fastMatchFilter = NumericRangeFilter.newFloatRange("field", minValue, maxValue, true, true);
+ fastMatchFilter = new QueryWrapperFilter(NumericRangeQuery.newFloatRange("field", minValue, maxValue, true, true));
} else {
- fastMatchFilter = NumericRangeFilter.newFloatRange("field", minAcceptedValue, maxAcceptedValue, true, true);
+ fastMatchFilter = new QueryWrapperFilter(NumericRangeQuery.newFloatRange("field", minAcceptedValue, maxAcceptedValue, true, true));
}
} else {
fastMatchFilter = null;
@@ -664,11 +660,7 @@ public class TestRangeFacetCounts extend
// Test drill-down:
DrillDownQuery ddq = new DrillDownQuery(config);
if (random().nextBoolean()) {
- if (random().nextBoolean()) {
- ddq.add("field", NumericRangeFilter.newFloatRange("field", (float) range.min, (float) range.max, range.minInclusive, range.maxInclusive));
- } else {
- ddq.add("field", NumericRangeQuery.newFloatRange("field", (float) range.min, (float) range.max, range.minInclusive, range.maxInclusive));
- }
+ ddq.add("field", NumericRangeQuery.newFloatRange("field", (float) range.min, (float) range.max, range.minInclusive, range.maxInclusive));
} else {
ddq.add("field", range.getFilter(fastMatchFilter, vs));
}
@@ -783,9 +775,9 @@ public class TestRangeFacetCounts extend
Filter fastMatchFilter;
if (random().nextBoolean()) {
if (random().nextBoolean()) {
- fastMatchFilter = NumericRangeFilter.newDoubleRange("field", minValue, maxValue, true, true);
+ fastMatchFilter = new QueryWrapperFilter(NumericRangeQuery.newDoubleRange("field", minValue, maxValue, true, true));
} else {
- fastMatchFilter = NumericRangeFilter.newDoubleRange("field", minAcceptedValue, maxAcceptedValue, true, true);
+ fastMatchFilter = new QueryWrapperFilter(NumericRangeQuery.newDoubleRange("field", minAcceptedValue, maxAcceptedValue, true, true));
}
} else {
fastMatchFilter = null;
@@ -807,11 +799,7 @@ public class TestRangeFacetCounts extend
// Test drill-down:
DrillDownQuery ddq = new DrillDownQuery(config);
if (random().nextBoolean()) {
- if (random().nextBoolean()) {
- ddq.add("field", NumericRangeFilter.newDoubleRange("field", range.min, range.max, range.minInclusive, range.maxInclusive));
- } else {
- ddq.add("field", NumericRangeQuery.newDoubleRange("field", range.min, range.max, range.minInclusive, range.maxInclusive));
- }
+ ddq.add("field", NumericRangeQuery.newDoubleRange("field", range.min, range.max, range.minInclusive, range.maxInclusive));
} else {
ddq.add("field", range.getFilter(fastMatchFilter, vs));
}
@@ -921,16 +909,19 @@ public class TestRangeFacetCounts extend
final AtomicBoolean filterWasUsed = new AtomicBoolean();
if (random().nextBoolean()) {
// Sort of silly:
- fastMatchFilter = new CachingWrapperFilter(new QueryWrapperFilter(new MatchAllDocsQuery()), FilterCachingPolicy.ALWAYS_CACHE) {
- @Override
- protected DocIdSet cacheImpl(DocIdSetIterator iterator, LeafReader reader)
+ final Filter in = new QueryWrapperFilter(new MatchAllDocsQuery());
+ fastMatchFilter = new Filter() {
+ @Override
+ public DocIdSet getDocIdSet(LeafReaderContext context, Bits acceptDocs)
throws IOException {
- final FixedBitSet cached = new FixedBitSet(reader.maxDoc());
- filterWasUsed.set(true);
- cached.or(iterator);
- return new BitDocIdSet(cached);
- }
- };
+ filterWasUsed.set(true);
+ return in.getDocIdSet(context, acceptDocs);
+ }
+ @Override
+ public String toString(String field) {
+ return in.toString(field);
+ }
+ };
} else {
fastMatchFilter = null;
}
Modified: lucene/dev/branches/lucene6271/lucene/grouping/src/test/org/apache/lucene/search/grouping/GroupingSearchTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/grouping/src/test/org/apache/lucene/search/grouping/GroupingSearchTest.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/grouping/src/test/org/apache/lucene/search/grouping/GroupingSearchTest.java (original)
+++ lucene/dev/branches/lucene6271/lucene/grouping/src/test/org/apache/lucene/search/grouping/GroupingSearchTest.java Tue Mar 31 05:22:40 2015
@@ -28,7 +28,7 @@ import org.apache.lucene.index.RandomInd
import org.apache.lucene.index.Term;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.valuesource.BytesRefFieldSource;
-import org.apache.lucene.search.CachingWrapperFilter;
+import org.apache.lucene.search.CachingWrapperQuery;
import org.apache.lucene.search.Filter;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.QueryWrapperFilter;
@@ -159,7 +159,7 @@ public class GroupingSearchTest extends
assertEquals(1, group.scoreDocs.length);
assertEquals(6, group.scoreDocs[0].doc);
- Filter lastDocInBlock = new CachingWrapperFilter(new QueryWrapperFilter(new TermQuery(new Term("groupend", "x"))));
+ Filter lastDocInBlock = new QueryWrapperFilter(new TermQuery(new Term("groupend", "x")));
groupingSearch = new GroupingSearch(lastDocInBlock);
groups = groupingSearch.search(indexSearcher, new TermQuery(new Term("content", "random")), 0, 10);
Modified: lucene/dev/branches/lucene6271/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestGrouping.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestGrouping.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestGrouping.java (original)
+++ lucene/dev/branches/lucene6271/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestGrouping.java Tue Mar 31 05:22:40 2015
@@ -51,7 +51,7 @@ import org.apache.lucene.index.Term;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.valuesource.BytesRefFieldSource;
import org.apache.lucene.search.CachingCollector;
-import org.apache.lucene.search.CachingWrapperFilter;
+import org.apache.lucene.search.CachingWrapperQuery;
import org.apache.lucene.search.Collector;
import org.apache.lucene.search.FieldDoc;
import org.apache.lucene.search.Filter;
@@ -798,7 +798,7 @@ public class TestGrouping extends Lucene
// group, so we can use single pass collector
dirBlocks = newDirectory();
rBlocks = getDocBlockReader(dirBlocks, groupDocs);
- final Filter lastDocInBlock = new CachingWrapperFilter(new QueryWrapperFilter(new TermQuery(new Term("groupend", "x"))));
+ final Filter lastDocInBlock = new QueryWrapperFilter(new TermQuery(new Term("groupend", "x")));
final NumericDocValues docIDToIDBlocks = MultiDocValues.getNumericValues(rBlocks, "id");
assertNotNull(docIDToIDBlocks);
Modified: lucene/dev/branches/lucene6271/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TermVectorLeafReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TermVectorLeafReader.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TermVectorLeafReader.java (original)
+++ lucene/dev/branches/lucene6271/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TermVectorLeafReader.java Tue Mar 31 05:22:40 2015
@@ -79,7 +79,7 @@ public class TermVectorLeafReader extend
}
FieldInfo fieldInfo = new FieldInfo(field, 0,
true, true, terms.hasPayloads(),
- indexOptions, DocValuesType.NONE, -1, null);
+ indexOptions, DocValuesType.NONE, -1, Collections.emptyMap());
fieldInfos = new FieldInfos(new FieldInfo[]{fieldInfo});
}
Modified: lucene/dev/branches/lucene6271/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java (original)
+++ lucene/dev/branches/lucene6271/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java Tue Mar 31 05:22:40 2015
@@ -230,7 +230,7 @@ public class WeightedSpanTermExtractor {
return;
}
MultiTermQuery copy = (MultiTermQuery) query.clone();
- copy.setRewriteMethod(MultiTermQuery.SCORING_BOOLEAN_QUERY_REWRITE);
+ copy.setRewriteMethod(MultiTermQuery.SCORING_BOOLEAN_REWRITE);
origQuery = copy;
}
final IndexReader reader = getLeafContext().reader();
Modified: lucene/dev/branches/lucene6271/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/PostingsHighlighter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/PostingsHighlighter.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/PostingsHighlighter.java (original)
+++ lucene/dev/branches/lucene6271/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/PostingsHighlighter.java Tue Mar 31 05:22:40 2015
@@ -31,7 +31,6 @@ import java.util.SortedSet;
import java.util.TreeSet;
import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.IndexReader;
@@ -39,6 +38,7 @@ import org.apache.lucene.index.IndexRead
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.MultiReader;
+import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.ReaderUtil;
import org.apache.lucene.index.StoredFieldVisitor;
import org.apache.lucene.index.Term;
@@ -100,7 +100,14 @@ public class PostingsHighlighter {
// unnecessary.
/** for rewriting: we don't want slow processing from MTQs */
- private static final IndexReader EMPTY_INDEXREADER = new MultiReader();
+ private static final IndexReader EMPTY_INDEXREADER;
+ static {
+ try {
+ EMPTY_INDEXREADER = new MultiReader();
+ } catch (IOException bogus) {
+ throw new RuntimeException(bogus);
+ }
+ }
/** Default maximum content size to process. Typically snippets
* closer to the beginning of the document better summarize its content */
Modified: lucene/dev/branches/lucene6271/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java (original)
+++ lucene/dev/branches/lucene6271/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java Tue Mar 31 05:22:40 2015
@@ -17,8 +17,6 @@ package org.apache.lucene.search.highlig
* limitations under the License.
*/
-import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
@@ -31,6 +29,9 @@ import java.util.List;
import java.util.Map;
import java.util.StringTokenizer;
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.CachingTokenFilter;
@@ -61,7 +62,6 @@ import org.apache.lucene.queries.CommonT
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.ConstantScoreQuery;
-import org.apache.lucene.search.FilteredQuery;
import org.apache.lucene.search.FuzzyQuery;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.MultiPhraseQuery;
@@ -73,7 +73,6 @@ import org.apache.lucene.search.Query;
import org.apache.lucene.search.QueryWrapperFilter;
import org.apache.lucene.search.RegexpQuery;
import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.search.TermRangeFilter;
import org.apache.lucene.search.TermRangeQuery;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.WildcardQuery;
@@ -926,7 +925,7 @@ public class HighlighterTest extends Bas
public void run() throws Exception {
numHighlights = 0;
FuzzyQuery fuzzyQuery = new FuzzyQuery(new Term(FIELD_NAME, "kinnedy"), 2);
- fuzzyQuery.setRewriteMethod(MultiTermQuery.SCORING_BOOLEAN_QUERY_REWRITE);
+ fuzzyQuery.setRewriteMethod(MultiTermQuery.SCORING_BOOLEAN_REWRITE);
doSearching(fuzzyQuery);
doStandardHighlights(analyzer, searcher, hits, query, HighlighterTest.this, true);
assertTrue("Failed to find correct number of highlights " + numHighlights + " found",
@@ -944,7 +943,7 @@ public class HighlighterTest extends Bas
public void run() throws Exception {
numHighlights = 0;
WildcardQuery wildcardQuery = new WildcardQuery(new Term(FIELD_NAME, "k?nnedy"));
- wildcardQuery.setRewriteMethod(MultiTermQuery.SCORING_BOOLEAN_QUERY_REWRITE);
+ wildcardQuery.setRewriteMethod(MultiTermQuery.SCORING_BOOLEAN_REWRITE);
doSearching(wildcardQuery);
doStandardHighlights(analyzer, searcher, hits, query, HighlighterTest.this);
assertTrue("Failed to find correct number of highlights " + numHighlights + " found",
@@ -962,7 +961,7 @@ public class HighlighterTest extends Bas
public void run() throws Exception {
numHighlights = 0;
WildcardQuery wildcardQuery = new WildcardQuery(new Term(FIELD_NAME, "k*dy"));
- wildcardQuery.setRewriteMethod(MultiTermQuery.SCORING_BOOLEAN_QUERY_REWRITE);
+ wildcardQuery.setRewriteMethod(MultiTermQuery.SCORING_BOOLEAN_REWRITE);
doSearching(wildcardQuery);
doStandardHighlights(analyzer, searcher, hits, query, HighlighterTest.this);
assertTrue("Failed to find correct number of highlights " + numHighlights + " found",
@@ -989,7 +988,7 @@ public class HighlighterTest extends Bas
new BytesRef("kannedy"),
new BytesRef("kznnedy"),
true, true);
- rangeQuery.setRewriteMethod(MultiTermQuery.SCORING_BOOLEAN_QUERY_REWRITE);
+ rangeQuery.setRewriteMethod(MultiTermQuery.SCORING_BOOLEAN_REWRITE);
query = rangeQuery;
doSearching(query);
@@ -1008,7 +1007,7 @@ public class HighlighterTest extends Bas
numHighlights = 0;
query = new WildcardQuery(new Term(FIELD_NAME, "ken*"));
- ((WildcardQuery)query).setRewriteMethod(MultiTermQuery.CONSTANT_SCORE_FILTER_REWRITE);
+ ((WildcardQuery)query).setRewriteMethod(MultiTermQuery.CONSTANT_SCORE_REWRITE);
searcher = newSearcher(reader);
// can't rewrite ConstantScore if you want to highlight it -
// it rewrites to ConstantScoreQuery which cannot be highlighted
@@ -1149,13 +1148,14 @@ public class HighlighterTest extends Bas
@Override
public void run() throws Exception {
numHighlights = 0;
- TermRangeFilter rf = TermRangeFilter.newStringRange("contents", "john", "john", true, true);
SpanQuery clauses[] = { new SpanTermQuery(new Term("contents", "john")),
new SpanTermQuery(new Term("contents", "kennedy")), };
SpanNearQuery snq = new SpanNearQuery(clauses, 1, true);
- FilteredQuery fq = new FilteredQuery(snq, rf);
+ BooleanQuery bq = new BooleanQuery();
+ bq.add(snq, Occur.MUST);
+ bq.add(TermRangeQuery.newStringRange("contents", "john", "john", true, true), Occur.FILTER);
- doSearching(fq);
+ doSearching(bq);
doStandardHighlights(analyzer, searcher, hits, query, HighlighterTest.this);
// Currently highlights "John" and "Kennedy" separately
assertTrue("Failed to find correct number of highlights " + numHighlights + " found",
@@ -1172,13 +1172,14 @@ public class HighlighterTest extends Bas
@Override
public void run() throws Exception {
numHighlights = 0;
- TermRangeFilter rf = TermRangeFilter.newStringRange("contents", "john", "john", true, true);
PhraseQuery pq = new PhraseQuery();
pq.add(new Term("contents", "john"));
pq.add(new Term("contents", "kennedy"));
- FilteredQuery fq = new FilteredQuery(pq, rf);
+ BooleanQuery bq = new BooleanQuery();
+ bq.add(pq, Occur.MUST);
+ bq.add(TermRangeQuery.newStringRange("contents", "john", "john", true, true), Occur.FILTER);
- doSearching(fq);
+ doSearching(bq);
doStandardHighlights(analyzer, searcher, hits, query, HighlighterTest.this);
// Currently highlights "John" and "Kennedy" separately
assertTrue("Failed to find correct number of highlights " + numHighlights + " found",
@@ -1198,7 +1199,7 @@ public class HighlighterTest extends Bas
BooleanQuery booleanQuery = new BooleanQuery();
booleanQuery.add(new TermQuery(new Term(FIELD_NAME, "john")), Occur.SHOULD);
PrefixQuery prefixQuery = new PrefixQuery(new Term(FIELD_NAME, "kenn"));
- prefixQuery.setRewriteMethod(MultiTermQuery.SCORING_BOOLEAN_QUERY_REWRITE);
+ prefixQuery.setRewriteMethod(MultiTermQuery.SCORING_BOOLEAN_REWRITE);
booleanQuery.add(prefixQuery, Occur.SHOULD);
doSearching(booleanQuery);
Modified: lucene/dev/branches/lucene6271/lucene/ivy-settings.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/ivy-settings.xml?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/ivy-settings.xml (original)
+++ lucene/dev/branches/lucene6271/lucene/ivy-settings.xml Tue Mar 31 05:22:40 2015
@@ -40,7 +40,9 @@
<!-- you might need to tweak this from china so it works -->
<ibiblio name="working-chinese-mirror" root="http://uk.maven.org/maven2" m2compatible="true" />
-
+
+ <ibiblio name="rr-snapshot" root="https://oss.sonatype.org/content/repositories/comcarrotsearch-1039" m2compatible="true" />
+
<!--
<filesystem name="local-maven-2" m2compatible="true" local="true">
<artifact
@@ -57,6 +59,7 @@
<resolver ref="maven.restlet.org" />
<resolver ref="sonatype-releases" />
<resolver ref="releases.cloudera.com"/>
+ <resolver ref="rr-snapshot"/>
<!-- <resolver ref="svnkit-releases" /> -->
<resolver ref="working-chinese-mirror" />
</chain>
Modified: lucene/dev/branches/lucene6271/lucene/ivy-versions.properties
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/ivy-versions.properties?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/ivy-versions.properties (original)
+++ lucene/dev/branches/lucene6271/lucene/ivy-versions.properties Tue Mar 31 05:22:40 2015
@@ -7,7 +7,7 @@
/cglib/cglib-nodep = 2.2
/com.adobe.xmp/xmpcore = 5.1.2
-com.carrotsearch.randomizedtesting.version = 2.1.9
+com.carrotsearch.randomizedtesting.version = 2.1.12
/com.carrotsearch.randomizedtesting/junit4-ant = ${com.carrotsearch.randomizedtesting.version}
/com.carrotsearch.randomizedtesting/randomizedtesting-runner = ${com.carrotsearch.randomizedtesting.version}
@@ -50,9 +50,10 @@ com.sun.jersey.version = 1.9
/com.sun.mail/javax.mail = 1.5.1
/com.sun.xml.bind/jaxb-impl = 2.2.3-1
+
+/com.tdunning/t-digest = 3.0
/com.thoughtworks.paranamer/paranamer = 2.3
/com.typesafe/config = 1.0.2
-/com.uwyn/jhighlight = 1.0
/commons-beanutils/commons-beanutils = 1.8.3
/commons-cli/commons-cli = 1.2
/commons-codec/commons-codec = 1.10
@@ -87,10 +88,11 @@ com.sun.jersey.version = 1.9
/org.apache.ant/ant = 1.8.2
/org.apache.avro/avro = 1.7.5
/org.apache.commons/commons-compress = 1.8.1
+/org.apache.commons/commons-math3 = 3.4.1
/org.apache.derby/derby = 10.9.1.0
/org.apache.directory.server/apacheds-all = 2.0.0-M15
-org.apache.hadoop.version = 2.3.0
+org.apache.hadoop.version = 2.6.0
/org.apache.hadoop/hadoop-annotations = ${org.apache.hadoop.version}
/org.apache.hadoop/hadoop-auth = ${org.apache.hadoop.version}
/org.apache.hadoop/hadoop-common = ${org.apache.hadoop.version}
@@ -181,12 +183,13 @@ org.codehaus.jackson.version = 1.9.13
/org.codehaus.woodstox/woodstox-core-asl = 4.4.1
/org.easymock/easymock = 3.0
-org.eclipse.jetty.version = 9.2.6.v20141205
+org.eclipse.jetty.version = 9.2.9.v20150224
/org.eclipse.jetty/jetty-continuation = ${org.eclipse.jetty.version}
/org.eclipse.jetty/jetty-deploy = ${org.eclipse.jetty.version}
/org.eclipse.jetty/jetty-http = ${org.eclipse.jetty.version}
/org.eclipse.jetty/jetty-io = ${org.eclipse.jetty.version}
/org.eclipse.jetty/jetty-jmx = ${org.eclipse.jetty.version}
+/org.eclipse.jetty/jetty-rewrite = ${org.eclipse.jetty.version}
/org.eclipse.jetty/jetty-security = ${org.eclipse.jetty.version}
/org.eclipse.jetty/jetty-server = ${org.eclipse.jetty.version}
/org.eclipse.jetty/jetty-servlet = ${org.eclipse.jetty.version}
@@ -200,6 +203,9 @@ org.gagravarr.vorbis.java.version = 0.6
/org.gagravarr/vorbis-java-core = ${org.gagravarr.vorbis.java.version}
/org.gagravarr/vorbis-java-tika = ${org.gagravarr.vorbis.java.version}
+org.htrace.version = 3.0.4
+/org.htrace/htrace-core = ${org.htrace.version}
+
org.kitesdk.kite-morphlines.version = 0.12.1
/org.kitesdk/kite-morphlines-avro = ${org.kitesdk.kite-morphlines.version}
/org.kitesdk/kite-morphlines-core = ${org.kitesdk.kite-morphlines.version}
@@ -239,3 +245,4 @@ org.slf4j.version = 1.7.7
/org.xerial.snappy/snappy-java = 1.0.5
/rome/rome = 1.0
/xerces/xercesImpl = 2.9.1
+
Modified: lucene/dev/branches/lucene6271/lucene/join/src/java/org/apache/lucene/search/join/BitDocIdSetCachingWrapperFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/join/src/java/org/apache/lucene/search/join/BitDocIdSetCachingWrapperFilter.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/join/src/java/org/apache/lucene/search/join/BitDocIdSetCachingWrapperFilter.java (original)
+++ lucene/dev/branches/lucene6271/lucene/join/src/java/org/apache/lucene/search/join/BitDocIdSetCachingWrapperFilter.java Tue Mar 31 05:22:40 2015
@@ -18,81 +18,82 @@ package org.apache.lucene.search.join;
*/
import java.io.IOException;
-import java.util.Collection;
+import java.util.Collections;
+import java.util.Map;
+import java.util.WeakHashMap;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.search.CachingWrapperFilter;
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.Filter;
-import org.apache.lucene.search.FilterCachingPolicy;
-import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.BitDocIdSet;
/**
- * A filter wrapper that transforms the produces doc id sets into
- * {@link BitDocIdSet}s if necessary and caches them.
+ * {@link Filter} wrapper that implements {@link BitDocIdSetFilter}.
*/
-public class BitDocIdSetCachingWrapperFilter extends BitDocIdSetFilter implements Accountable {
-
- private final CachingWrapperFilter filter;
-
- /** Sole constructor. */
+public class BitDocIdSetCachingWrapperFilter extends BitDocIdSetFilter {
+ private final Filter filter;
+ private final Map<Object,DocIdSet> cache = Collections.synchronizedMap(new WeakHashMap<>());
+
+ /** Wraps another filter's result and caches it into bitsets.
+ * @param filter Filter to cache results of
+ */
public BitDocIdSetCachingWrapperFilter(Filter filter) {
- super();
- this.filter = new CachingWrapperFilter(filter, FilterCachingPolicy.ALWAYS_CACHE) {
- @Override
- protected BitDocIdSet docIdSetToCache(DocIdSet docIdSet, LeafReader reader) throws IOException {
- if (docIdSet == null || docIdSet instanceof BitDocIdSet) {
- // this is different from CachingWrapperFilter: even when the DocIdSet is
- // cacheable, we convert it to a BitSet since we require all the
- // cached filters to be BitSets
- return (BitDocIdSet) docIdSet;
- }
-
- final DocIdSetIterator it = docIdSet.iterator();
- if (it == null) {
- return null;
- }
- BitDocIdSet.Builder builder = new BitDocIdSet.Builder(reader.maxDoc());
- builder.or(it);
- return builder.build();
- }
- };
+ this.filter = filter;
}
- @Override
- public BitDocIdSet getDocIdSet(LeafReaderContext context) throws IOException {
- return (BitDocIdSet) filter.getDocIdSet(context, null);
+ /**
+ * Gets the contained filter.
+ * @return the contained filter.
+ */
+ public Filter getFilter() {
+ return filter;
+ }
+
+ private BitDocIdSet docIdSetToCache(DocIdSet docIdSet, LeafReader reader) throws IOException {
+ final DocIdSetIterator it = docIdSet.iterator();
+ if (it == null) {
+ return null;
+ } else {
+ BitDocIdSet.Builder builder = new BitDocIdSet.Builder(reader.maxDoc());
+ builder.or(it);
+ return builder.build();
+ }
}
-
+
@Override
- public int hashCode() {
- return getClass().hashCode() ^ filter.hashCode();
- }
+ public BitDocIdSet getDocIdSet(LeafReaderContext context) throws IOException {
+ final LeafReader reader = context.reader();
+ final Object key = reader.getCoreCacheKey();
- @Override
- public boolean equals(Object obj) {
- if (obj instanceof BitDocIdSetCachingWrapperFilter == false) {
- return false;
+ DocIdSet docIdSet = cache.get(key);
+ if (docIdSet == null) {
+ docIdSet = filter.getDocIdSet(context, null);
+ docIdSet = docIdSetToCache(docIdSet, reader);
+ if (docIdSet == null) {
+ // We use EMPTY as a sentinel for the empty set, which is cacheable
+ docIdSet = DocIdSet.EMPTY;
+ }
+ cache.put(key, docIdSet);
}
- return filter.equals(((BitDocIdSetCachingWrapperFilter) obj).filter);
+ return docIdSet == DocIdSet.EMPTY ? null : (BitDocIdSet) docIdSet;
}
-
+
@Override
public String toString(String field) {
- return filter.toString();
+ return getClass().getSimpleName() + "("+filter.toString(field)+")";
}
@Override
- public long ramBytesUsed() {
- return filter.ramBytesUsed();
+ public boolean equals(Object o) {
+ if (o == null || !getClass().equals(o.getClass())) return false;
+ final BitDocIdSetCachingWrapperFilter other = (BitDocIdSetCachingWrapperFilter) o;
+ return this.filter.equals(other.filter);
}
@Override
- public Collection<Accountable> getChildResources() {
- return filter.getChildResources();
+ public int hashCode() {
+ return (filter.hashCode() ^ getClass().hashCode());
}
-
}
Modified: lucene/dev/branches/lucene6271/lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinCollector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinCollector.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinCollector.java (original)
+++ lucene/dev/branches/lucene6271/lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinCollector.java Tue Mar 31 05:22:40 2015
@@ -501,6 +501,8 @@ public class ToParentBlockJoinCollector
@Override
public boolean needsScores() {
- return trackScores || trackMaxScore || sort.needsScores();
+ // needed so that eg. BooleanQuery does not rewrite its MUST clauses to
+ // FILTER since the filter scorers are hidden in Scorer.getChildren().
+ return true;
}
}
Modified: lucene/dev/branches/lucene6271/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java (original)
+++ lucene/dev/branches/lucene6271/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java Tue Mar 31 05:22:40 2015
@@ -276,7 +276,7 @@ public class TestBlockJoin extends Lucen
MultiTermQuery qc = NumericRangeQuery.newIntRange("year", 2007, 2007, true, true);
// Hacky: this causes the query to need 2 rewrite
// iterations:
- qc.setRewriteMethod(MultiTermQuery.CONSTANT_SCORE_BOOLEAN_QUERY_REWRITE);
+ qc.setRewriteMethod(MultiTermQuery.CONSTANT_SCORE_BOOLEAN_REWRITE);
BitDocIdSetFilter parentsFilter = new BitDocIdSetCachingWrapperFilter(new QueryWrapperFilter(new TermQuery(new Term("docType", "resume"))));
Modified: lucene/dev/branches/lucene6271/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoinValidation.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoinValidation.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoinValidation.java (original)
+++ lucene/dev/branches/lucene6271/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoinValidation.java Tue Mar 31 05:22:40 2015
@@ -30,8 +30,6 @@ import org.apache.lucene.index.IndexWrit
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.Filter;
-import org.apache.lucene.search.FilteredQuery;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.QueryWrapperFilter;
@@ -39,11 +37,6 @@ import org.apache.lucene.search.TermQuer
import org.apache.lucene.search.WildcardQuery;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
public class TestBlockJoinValidation extends LuceneTestCase {
@@ -57,11 +50,9 @@ public class TestBlockJoinValidation ext
private IndexSearcher indexSearcher;
private BitDocIdSetFilter parentsFilter;
- @Rule
- public ExpectedException thrown = ExpectedException.none();
-
- @Before
- public void before() throws Exception {
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
directory = newDirectory();
final IndexWriterConfig config = new IndexWriterConfig(new MockAnalyzer(random()));
final IndexWriter indexWriter = new IndexWriter(directory, config);
@@ -76,16 +67,24 @@ public class TestBlockJoinValidation ext
parentsFilter = new BitDocIdSetCachingWrapperFilter(new QueryWrapperFilter(new WildcardQuery(new Term("parent", "*"))));
}
- @Test
+ @Override
+ public void tearDown() throws Exception {
+ indexReader.close();
+ directory.close();
+ super.tearDown();
+ }
+
public void testNextDocValidationForToParentBjq() throws Exception {
Query parentQueryWithRandomChild = createChildrenQueryWithOneParent(getRandomChildNumber(0));
ToParentBlockJoinQuery blockJoinQuery = new ToParentBlockJoinQuery(parentQueryWithRandomChild, parentsFilter, ScoreMode.None);
- thrown.expect(IllegalStateException.class);
- thrown.expectMessage("child query must only match non-parent docs");
- indexSearcher.search(blockJoinQuery, 1);
+ try {
+ indexSearcher.search(blockJoinQuery, 1);
+ fail("didn't get expected exception");
+ } catch (IllegalStateException expected) {
+ assertTrue(expected.getMessage() != null && expected.getMessage().contains("child query must only match non-parent docs"));
+ }
}
- @Test
public void testAdvanceValidationForToParentBjq() throws Exception {
int randomChildNumber = getRandomChildNumber(0);
// we need to make advance method meet wrong document, so random child number
@@ -98,34 +97,28 @@ public class TestBlockJoinValidation ext
WildcardQuery childQuery = new WildcardQuery(new Term("child", createFieldValue(randomChildNumber)));
conjunctionQuery.add(new BooleanClause(childQuery, BooleanClause.Occur.MUST));
conjunctionQuery.add(new BooleanClause(blockJoinQuery, BooleanClause.Occur.MUST));
-
- thrown.expect(IllegalStateException.class);
- thrown.expectMessage("child query must only match non-parent docs");
- indexSearcher.search(conjunctionQuery, 1);
+
+ try {
+ indexSearcher.search(conjunctionQuery, 1);
+ fail("didn't get expected exception");
+ } catch (IllegalStateException expected) {
+ assertTrue(expected.getMessage() != null && expected.getMessage().contains("child query must only match non-parent docs"));
+ }
}
- @Test
public void testNextDocValidationForToChildBjq() throws Exception {
Query parentQueryWithRandomChild = createParentsQueryWithOneChild(getRandomChildNumber(0));
ToChildBlockJoinQuery blockJoinQuery = new ToChildBlockJoinQuery(parentQueryWithRandomChild, parentsFilter);
- thrown.expect(IllegalStateException.class);
- thrown.expectMessage(ToChildBlockJoinQuery.INVALID_QUERY_MESSAGE);
- indexSearcher.search(blockJoinQuery, 1);
- }
-
- @Test
- public void testValidationForToChildBjqWithChildFilterQuery() throws Exception {
- Query parentQueryWithRandomChild = createParentQuery();
-
- ToChildBlockJoinQuery blockJoinQuery = new ToChildBlockJoinQuery(parentQueryWithRandomChild, parentsFilter);
- Filter childFilter = new QueryWrapperFilter(new TermQuery(new Term("common_field", "1")));
- thrown.expect(IllegalStateException.class);
- thrown.expectMessage(ToChildBlockJoinQuery.ILLEGAL_ADVANCE_ON_PARENT);
- indexSearcher.search(new FilteredQuery(blockJoinQuery, childFilter), 1);
+
+ try {
+ indexSearcher.search(blockJoinQuery, 1);
+ fail("didn't get expected exception");
+ } catch (IllegalStateException expected) {
+ assertTrue(expected.getMessage() != null && expected.getMessage().contains(ToChildBlockJoinQuery.INVALID_QUERY_MESSAGE));
+ }
}
- @Test
public void testAdvanceValidationForToChildBjq() throws Exception {
int randomChildNumber = getRandomChildNumber(0);
// we need to make advance method meet wrong document, so random child number
@@ -136,19 +129,15 @@ public class TestBlockJoinValidation ext
// advance() method is used by ConjunctionScorer, so we need to create Boolean conjunction query
BooleanQuery conjunctionQuery = new BooleanQuery();
WildcardQuery childQuery = new WildcardQuery(new Term("child", createFieldValue(randomChildNumber)));
- conjunctionQuery.add(new BooleanClause(childQuery, BooleanClause.Occur.MUST));
- conjunctionQuery.add(new BooleanClause(blockJoinQuery, BooleanClause.Occur.MUST));
-
- thrown.expect(IllegalStateException.class);
- thrown.expectMessage(ToChildBlockJoinQuery.INVALID_QUERY_MESSAGE);
- indexSearcher.search(conjunctionQuery, 1);
- }
-
-
- @After
- public void after() throws Exception {
- indexReader.close();
- directory.close();
+ conjunctionQuery.add(childQuery, BooleanClause.Occur.MUST);
+ conjunctionQuery.add(blockJoinQuery, BooleanClause.Occur.MUST);
+
+ try {
+ indexSearcher.search(conjunctionQuery, 1);
+ fail("didn't get expected exception");
+ } catch (IllegalStateException expected) {
+ assertTrue(expected.getMessage() != null && expected.getMessage().contains(ToChildBlockJoinQuery.INVALID_QUERY_MESSAGE));
+ }
}
private static List<Document> createDocsForSegment(int segmentNumber) {