You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sh...@apache.org on 2013/07/02 09:12:03 UTC
svn commit: r1498804 [8/8] - in /lucene/dev/branches/lucene4258: lucene/
lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/
lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/
lucene/analysis/common/src/java/org/apache/l...
Modified: lucene/dev/branches/lucene4258/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/TestSlowFuzzyQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/TestSlowFuzzyQuery.java?rev=1498804&r1=1498803&r2=1498804&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/TestSlowFuzzyQuery.java (original)
+++ lucene/dev/branches/lucene4258/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/TestSlowFuzzyQuery.java Tue Jul 2 07:12:00 2013
@@ -43,6 +43,9 @@ import org.apache.lucene.util.LuceneTest
public class TestSlowFuzzyQuery extends LuceneTestCase {
public void testFuzziness() throws Exception {
+ //every test with SlowFuzzyQuery.defaultMinSimilarity
+ //is exercising the Automaton, not the brute force linear method
+
Directory directory = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), directory);
addDoc("aaaaa", writer);
@@ -194,6 +197,30 @@ public class TestSlowFuzzyQuery extends
directory.close();
}
+ public void testFuzzinessLong2() throws Exception {
+ //Lucene-5033
+ Directory directory = newDirectory();
+ RandomIndexWriter writer = new RandomIndexWriter(random(), directory);
+ addDoc("abcdef", writer);
+ addDoc("segment", writer);
+
+ IndexReader reader = writer.getReader();
+ IndexSearcher searcher = newSearcher(reader);
+ writer.close();
+
+ SlowFuzzyQuery query;
+
+ query = new SlowFuzzyQuery(new Term("field", "abcxxxx"), 3f, 0);
+ ScoreDoc[] hits = searcher.search(query, null, 1000).scoreDocs;
+ assertEquals(0, hits.length);
+
+ query = new SlowFuzzyQuery(new Term("field", "abcxxxx"), 4f, 0);
+ hits = searcher.search(query, null, 1000).scoreDocs;
+ assertEquals(1, hits.length);
+ reader.close();
+ directory.close();
+ }
+
public void testFuzzinessLong() throws Exception {
Directory directory = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), directory);
@@ -385,7 +412,6 @@ public class TestSlowFuzzyQuery extends
public void testGiga() throws Exception {
- MockAnalyzer analyzer = new MockAnalyzer(random());
Directory index = newDirectory();
RandomIndexWriter w = new RandomIndexWriter(random(), index);
@@ -440,25 +466,21 @@ public class TestSlowFuzzyQuery extends
assertEquals(1, hits.length);
assertEquals("foobar", searcher.doc(hits[0].doc).get("field"));
- // TODO: cannot really be supported given the legacy scoring
- // system which scores negative, if the distance > min term len,
- // so such matches were always impossible with lucene 3.x, etc
- //
- //q = new SlowFuzzyQuery(new Term("field", "t"), 3);
- //hits = searcher.search(q, 10).scoreDocs;
- //assertEquals(1, hits.length);
- //assertEquals("test", searcher.doc(hits[0].doc).get("field"));
-
- // q = new SlowFuzzyQuery(new Term("field", "a"), 4f, 0, 50);
- // hits = searcher.search(q, 10).scoreDocs;
- // assertEquals(1, hits.length);
- // assertEquals("test", searcher.doc(hits[0].doc).get("field"));
-
- // q = new SlowFuzzyQuery(new Term("field", "a"), 6f, 0, 50);
- // hits = searcher.search(q, 10).scoreDocs;
- // assertEquals(2, hits.length);
- // assertEquals("test", searcher.doc(hits[0].doc).get("field"));
- // assertEquals("foobar", searcher.doc(hits[1].doc).get("field"));
+ q = new SlowFuzzyQuery(new Term("field", "t"), 3);
+ hits = searcher.search(q, 10).scoreDocs;
+ assertEquals(1, hits.length);
+ assertEquals("test", searcher.doc(hits[0].doc).get("field"));
+
+ q = new SlowFuzzyQuery(new Term("field", "a"), 4f, 0, 50);
+ hits = searcher.search(q, 10).scoreDocs;
+ assertEquals(1, hits.length);
+ assertEquals("test", searcher.doc(hits[0].doc).get("field"));
+
+ q = new SlowFuzzyQuery(new Term("field", "a"), 6f, 0, 50);
+ hits = searcher.search(q, 10).scoreDocs;
+ assertEquals(2, hits.length);
+ assertEquals("test", searcher.doc(hits[0].doc).get("field"));
+ assertEquals("foobar", searcher.doc(hits[1].doc).get("field"));
reader.close();
index.close();
Modified: lucene/dev/branches/lucene4258/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/SpatialOpRecursivePrefixTreeTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/SpatialOpRecursivePrefixTreeTest.java?rev=1498804&r1=1498803&r2=1498804&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/SpatialOpRecursivePrefixTreeTest.java (original)
+++ lucene/dev/branches/lucene4258/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/SpatialOpRecursivePrefixTreeTest.java Tue Jul 2 07:12:00 2013
@@ -58,6 +58,8 @@ import static com.spatial4j.core.shape.S
public class SpatialOpRecursivePrefixTreeTest extends StrategyTestCase {
+ static final int ITERATIONS = 10;//Test Iterations
+
private SpatialPrefixTree grid;
@Before
@@ -81,28 +83,28 @@ public class SpatialOpRecursivePrefixTre
}
@Test
- @Repeat(iterations = 10)
+ @Repeat(iterations = ITERATIONS)
public void testIntersects() throws IOException {
mySetup(-1);
doTest(SpatialOperation.Intersects);
}
@Test
- @Repeat(iterations = 10)
+ @Repeat(iterations = ITERATIONS)
public void testWithin() throws IOException {
mySetup(-1);
doTest(SpatialOperation.IsWithin);
}
@Test
- @Repeat(iterations = 10)
+ @Repeat(iterations = ITERATIONS)
public void testContains() throws IOException {
mySetup(-1);
doTest(SpatialOperation.Contains);
}
@Test
- @Repeat(iterations = 10)
+ @Repeat(iterations = ITERATIONS)
public void testDisjoint() throws IOException {
mySetup(-1);
doTest(SpatialOperation.IsDisjointTo);
@@ -334,9 +336,10 @@ public class SpatialOpRecursivePrefixTre
@Override
public SpatialRelation relate(Shape other) {
SpatialRelation r = relateApprox(other);
- if (r != INTERSECTS)
+ if (r != INTERSECTS && !(r == WITHIN && biasContainsThenWithin))
return r;
- //See if the correct answer is actually Contains
+ //See if the correct answer is actually Contains, when the indexed shapes are adjacent,
+ // creating a larger shape that contains the input shape.
Rectangle oRect = (Rectangle)other;
boolean pairTouches = shape1.relate(shape2).intersects();
if (!pairTouches)
Modified: lucene/dev/branches/lucene4258/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java?rev=1498804&r1=1498803&r2=1498804&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java (original)
+++ lucene/dev/branches/lucene4258/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java Tue Jul 2 07:12:00 2013
@@ -512,7 +512,7 @@ public class AnalyzingSuggester extends
reader = new Sort.ByteSequencesReader(tempSorted);
- PairOutputs<Long,BytesRef> outputs = new PairOutputs<Long,BytesRef>(PositiveIntOutputs.getSingleton(true), ByteSequenceOutputs.getSingleton());
+ PairOutputs<Long,BytesRef> outputs = new PairOutputs<Long,BytesRef>(PositiveIntOutputs.getSingleton(), ByteSequenceOutputs.getSingleton());
Builder<Pair<Long,BytesRef>> builder = new Builder<Pair<Long,BytesRef>>(FST.INPUT_TYPE.BYTE1, outputs);
// Build FST:
@@ -634,7 +634,7 @@ public class AnalyzingSuggester extends
public boolean load(InputStream input) throws IOException {
DataInput dataIn = new InputStreamDataInput(input);
try {
- this.fst = new FST<Pair<Long,BytesRef>>(dataIn, new PairOutputs<Long,BytesRef>(PositiveIntOutputs.getSingleton(true), ByteSequenceOutputs.getSingleton()));
+ this.fst = new FST<Pair<Long,BytesRef>>(dataIn, new PairOutputs<Long,BytesRef>(PositiveIntOutputs.getSingleton(), ByteSequenceOutputs.getSingleton()));
maxAnalyzedPathsForOneInput = dataIn.readVInt();
hasPayloads = dataIn.readByte() == 1;
} finally {
Modified: lucene/dev/branches/lucene4258/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java?rev=1498804&r1=1498803&r2=1498804&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java (original)
+++ lucene/dev/branches/lucene4258/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java Tue Jul 2 07:12:00 2013
@@ -101,7 +101,7 @@ public class WFSTCompletionLookup extend
TermFreqIterator iter = new WFSTTermFreqIteratorWrapper(iterator);
IntsRef scratchInts = new IntsRef();
BytesRef previous = null;
- PositiveIntOutputs outputs = PositiveIntOutputs.getSingleton(true);
+ PositiveIntOutputs outputs = PositiveIntOutputs.getSingleton();
Builder<Long> builder = new Builder<Long>(FST.INPUT_TYPE.BYTE1, outputs);
while ((scratch = iter.next()) != null) {
long cost = iter.weight();
@@ -136,7 +136,7 @@ public class WFSTCompletionLookup extend
@Override
public boolean load(InputStream input) throws IOException {
try {
- this.fst = new FST<Long>(new InputStreamDataInput(input), PositiveIntOutputs.getSingleton(true));
+ this.fst = new FST<Long>(new InputStreamDataInput(input), PositiveIntOutputs.getSingleton());
} finally {
IOUtils.close(input);
}
Modified: lucene/dev/branches/lucene4258/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingStoredFieldsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingStoredFieldsFormat.java?rev=1498804&r1=1498803&r2=1498804&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingStoredFieldsFormat.java (original)
+++ lucene/dev/branches/lucene4258/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingStoredFieldsFormat.java Tue Jul 2 07:12:00 2013
@@ -63,7 +63,8 @@ public class AssertingStoredFieldsFormat
}
@Override
- public void visitDocument(int n, StoredFieldVisitor visitor, Set<String> ignoreFields) throws IOException {
+ public void visitDocument(int n, StoredFieldVisitor visitor,
+ Set<String> ignoreFields) throws IOException {
assert n >= 0 && n < maxDoc;
in.visitDocument(n, visitor, ignoreFields);
}
Modified: lucene/dev/branches/lucene4258/lucene/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java?rev=1498804&r1=1498803&r2=1498804&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java (original)
+++ lucene/dev/branches/lucene4258/lucene/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java Tue Jul 2 07:12:00 2013
@@ -53,6 +53,7 @@ import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MMapDirectory;
import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.store.MockDirectoryWrapper.Throttling;
import org.apache.lucene.util.BytesRef;
@@ -594,7 +595,9 @@ public abstract class BaseStoredFieldsFo
public void testBigDocuments() throws IOException {
// "big" as "much bigger than the chunk size"
// for this test we force a FS dir
- Directory dir = newFSDirectory(_TestUtil.getTempDir(getClass().getSimpleName()));
+ // we can't just use newFSDirectory, because this test doesn't really index anything.
+ // so if we get NRTCachingDir+SimpleText, we make massive stored fields and OOM (LUCENE-4484)
+ Directory dir = new MockDirectoryWrapper(random(), new MMapDirectory(_TestUtil.getTempDir("testBigDocuments")));
IndexWriterConfig iwConf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
iwConf.setMaxBufferedDocs(RandomInts.randomIntBetween(random(), 2, 30));
RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwConf);
Modified: lucene/dev/branches/lucene4258/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java?rev=1498804&r1=1498803&r2=1498804&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java (original)
+++ lucene/dev/branches/lucene4258/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java Tue Jul 2 07:12:00 2013
@@ -25,17 +25,12 @@ import java.util.Random;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.document.BinaryDocValuesField;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.NumericDocValuesField;
-import org.apache.lucene.document.SortedDocValuesField;
-import org.apache.lucene.index.FieldInfo.DocValuesType;
import org.apache.lucene.index.IndexWriter; // javadoc
import org.apache.lucene.search.Query;
import org.apache.lucene.store.Directory;
-import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.InfoStream;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.NullInfoStream;
import org.apache.lucene.util.Version;
import org.apache.lucene.util._TestUtil;
@@ -55,23 +50,22 @@ public class RandomIndexWriter implement
private boolean getReaderCalled;
private final Codec codec; // sugar
- // Randomly calls Thread.yield so we mixup thread scheduling
- private static final class MockIndexWriter extends IndexWriter {
-
- private final Random r;
-
- public MockIndexWriter(Random r, Directory dir, IndexWriterConfig conf) throws IOException {
- super(dir, conf);
- // TODO: this should be solved in a different way; Random should not be shared (!).
- this.r = new Random(r.nextLong());
- }
-
- @Override
- boolean testPoint(String name) {
- if (r.nextInt(4) == 2)
- Thread.yield();
- return true;
- }
+
+ public static IndexWriter mockIndexWriter(Directory dir, IndexWriterConfig conf, Random r) throws IOException {
+ // Randomly calls Thread.yield so we mixup thread scheduling
+ final Random random = new Random(r.nextLong());
+ return mockIndexWriter(dir, conf, new TestPoint() {
+ @Override
+ public void apply(String message) {
+ if (random.nextInt(4) == 2)
+ Thread.yield();
+ }
+ });
+ }
+
+ public static IndexWriter mockIndexWriter(Directory dir, IndexWriterConfig conf, TestPoint testPoint) throws IOException {
+ conf.setInfoStream(new TestPointInfoStream(conf.getInfoStream(), testPoint));
+ return new IndexWriter(dir, conf);
}
/** create a RandomIndexWriter with a random config: Uses TEST_VERSION_CURRENT and MockAnalyzer */
@@ -93,7 +87,7 @@ public class RandomIndexWriter implement
public RandomIndexWriter(Random r, Directory dir, IndexWriterConfig c) throws IOException {
// TODO: this should be solved in a different way; Random should not be shared (!).
this.r = new Random(r.nextLong());
- w = new MockIndexWriter(r, dir, c);
+ w = mockIndexWriter(dir, c, r);
flushAt = _TestUtil.nextInt(r, 10, 1000);
codec = w.getConfig().getCodec();
if (LuceneTestCase.VERBOSE) {
@@ -345,4 +339,42 @@ public class RandomIndexWriter implement
public void forceMerge(int maxSegmentCount) throws IOException {
w.forceMerge(maxSegmentCount);
}
+
+ private static final class TestPointInfoStream extends InfoStream {
+ private final InfoStream delegate;
+ private final TestPoint testPoint;
+
+ public TestPointInfoStream(InfoStream delegate, TestPoint testPoint) {
+ this.delegate = delegate == null ? new NullInfoStream(): delegate;
+ this.testPoint = testPoint;
+ }
+
+ @Override
+ public void close() throws IOException {
+ delegate.close();
+ }
+
+ @Override
+ public void message(String component, String message) {
+ if ("TP".equals(component)) {
+ testPoint.apply(message);
+ }
+ if (delegate.isEnabled(component)) {
+ delegate.message(component, message);
+ }
+ }
+
+ @Override
+ public boolean isEnabled(String component) {
+ return "TP".equals(component) || delegate.isEnabled(component);
+ }
+ }
+
+ /**
+ * Simple interface that is executed for each <tt>TP</tt> {@link InfoStream} component
+ * message. See also {@link RandomIndexWriter#mockIndexWriter(Directory, IndexWriterConfig, TestPoint)}
+ */
+ public static interface TestPoint {
+ public abstract void apply(String message);
+ }
}
Modified: lucene/dev/branches/lucene4258/lucene/test-framework/src/java/org/apache/lucene/store/MockIndexOutputWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/test-framework/src/java/org/apache/lucene/store/MockIndexOutputWrapper.java?rev=1498804&r1=1498803&r2=1498804&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/test-framework/src/java/org/apache/lucene/store/MockIndexOutputWrapper.java (original)
+++ lucene/dev/branches/lucene4258/lucene/test-framework/src/java/org/apache/lucene/store/MockIndexOutputWrapper.java Tue Jul 2 07:12:00 2013
@@ -55,14 +55,14 @@ public class MockIndexOutputWrapper exte
long realUsage = 0;
// Enforce disk full:
- if (dir.maxSize != 0 && freeSpace < len) {
+ if (dir.maxSize != 0 && freeSpace <= len) {
// Compute the real disk free. This will greatly slow
// down our test but makes it more accurate:
realUsage = dir.getRecomputedActualSizeInBytes();
freeSpace = dir.maxSize - realUsage;
}
- if (dir.maxSize != 0 && freeSpace < len) {
+ if (dir.maxSize != 0 && freeSpace <= len) {
if (freeSpace > 0) {
realUsage += freeSpace;
if (b != null) {
Modified: lucene/dev/branches/lucene4258/lucene/tools/junit4/tests.policy
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/tools/junit4/tests.policy?rev=1498804&r1=1498803&r2=1498804&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/tools/junit4/tests.policy (original)
+++ lucene/dev/branches/lucene4258/lucene/tools/junit4/tests.policy Tue Jul 2 07:12:00 2013
@@ -54,6 +54,7 @@ grant {
// Solr needs those:
permission java.net.NetPermission "*";
+ permission java.sql.SQLPermission "*";
permission java.util.logging.LoggingPermission "control";
permission javax.management.MBeanPermission "*", "*";
permission javax.management.MBeanServerPermission "*";
Modified: lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/search/similarities/SweetSpotSimilarityFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/search/similarities/SweetSpotSimilarityFactory.java?rev=1498804&r1=1498803&r2=1498804&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/search/similarities/SweetSpotSimilarityFactory.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/search/similarities/SweetSpotSimilarityFactory.java Tue Jul 2 07:12:00 2013
@@ -180,7 +180,7 @@ public class SweetSpotSimilarityFactory
private static final class HyperbolicSweetSpotSimilarity
extends SweetSpotSimilarity {
@Override
- public float tf(int freq) {
+ public float tf(float freq) {
return hyperbolicTf(freq);
}
};