You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ja...@apache.org on 2013/07/04 01:26:45 UTC
svn commit: r1499601 [10/20] - in /lucene/dev/branches/security: ./
dev-tools/ dev-tools/eclipse/ dev-tools/idea/.idea/libraries/
dev-tools/idea/solr/core/src/test/ dev-tools/maven/ dev-tools/maven/lucene/
dev-tools/maven/lucene/analysis/stempel/ dev-t...
Modified: lucene/dev/branches/security/lucene/queries/src/test/org/apache/lucene/queries/function/FunctionTestSetup.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/queries/src/test/org/apache/lucene/queries/function/FunctionTestSetup.java?rev=1499601&r1=1499600&r2=1499601&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/queries/src/test/org/apache/lucene/queries/function/FunctionTestSetup.java (original)
+++ lucene/dev/branches/security/lucene/queries/src/test/org/apache/lucene/queries/function/FunctionTestSetup.java Wed Jul 3 23:26:32 2013
@@ -1,18 +1,25 @@
package org.apache.lucene.queries.function;
+import java.io.IOException;
+
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.FloatField;
+import org.apache.lucene.document.IntField;
import org.apache.lucene.document.TextField;
-import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.document.Field.Store;
import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.queries.function.valuesource.ByteFieldSource;
+import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.index.Terms;
+import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.queries.function.valuesource.FloatFieldSource;
import org.apache.lucene.queries.function.valuesource.IntFieldSource;
-import org.apache.lucene.queries.function.valuesource.ShortFieldSource;
+import org.apache.lucene.search.FieldCache;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
import org.junit.AfterClass;
@@ -53,10 +60,21 @@ public abstract class FunctionTestSetup
protected static final String INT_FIELD = "iii";
protected static final String FLOAT_FIELD = "fff";
- protected ValueSource BYTE_VALUESOURCE = new ByteFieldSource(INT_FIELD);
- protected ValueSource SHORT_VALUESOURCE = new ShortFieldSource(INT_FIELD);
+ private static final FieldCache.FloatParser CUSTOM_FLOAT_PARSER = new FieldCache.FloatParser() {
+
+ @Override
+ public TermsEnum termsEnum(Terms terms) throws IOException {
+ return FieldCache.NUMERIC_UTILS_INT_PARSER.termsEnum(terms);
+ }
+
+ @Override
+ public float parseFloat(BytesRef term) {
+ return (float) FieldCache.NUMERIC_UTILS_INT_PARSER.parseInt(term);
+ }
+ };
+
protected ValueSource INT_VALUESOURCE = new IntFieldSource(INT_FIELD);
- protected ValueSource INT_AS_FLOAT_VALUESOURCE = new FloatFieldSource(INT_FIELD);
+ protected ValueSource INT_AS_FLOAT_VALUESOURCE = new FloatFieldSource(INT_FIELD, CUSTOM_FLOAT_PARSER);
protected ValueSource FLOAT_VALUESOURCE = new FloatFieldSource(FLOAT_FIELD);
private static final String DOC_TEXT_LINES[] = {
@@ -140,10 +158,10 @@ public abstract class FunctionTestSetup
f = newField(TEXT_FIELD, "text of doc" + scoreAndID + textLine(i), customType2); // for regular search
d.add(f);
- f = newField(INT_FIELD, "" + scoreAndID, customType); // for function scoring
+ f = new IntField(INT_FIELD, scoreAndID, Store.YES); // for function scoring
d.add(f);
- f = newField(FLOAT_FIELD, scoreAndID + ".000", customType); // for function scoring
+ f = new FloatField(FLOAT_FIELD, scoreAndID, Store.YES); // for function scoring
d.add(f);
iw.addDocument(d);
Modified: lucene/dev/branches/security/lucene/queries/src/test/org/apache/lucene/queries/function/TestFieldScoreQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/queries/src/test/org/apache/lucene/queries/function/TestFieldScoreQuery.java?rev=1499601&r1=1499600&r2=1499601&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/queries/src/test/org/apache/lucene/queries/function/TestFieldScoreQuery.java (original)
+++ lucene/dev/branches/security/lucene/queries/src/test/org/apache/lucene/queries/function/TestFieldScoreQuery.java Wed Jul 3 23:26:32 2013
@@ -19,12 +19,6 @@ package org.apache.lucene.queries.functi
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.queries.function.FunctionQuery;
-import org.apache.lucene.queries.function.ValueSource;
-import org.apache.lucene.queries.function.valuesource.ByteFieldSource;
-import org.apache.lucene.queries.function.valuesource.FloatFieldSource;
-import org.apache.lucene.queries.function.valuesource.IntFieldSource;
-import org.apache.lucene.queries.function.valuesource.ShortFieldSource;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.QueryUtils;
import org.apache.lucene.search.ScoreDoc;
@@ -50,20 +44,6 @@ public class TestFieldScoreQuery extends
createIndex(true);
}
- /** Test that FieldScoreQuery of Type.BYTE returns docs in expected order. */
- @Test
- public void testRankByte () throws Exception {
- // INT field values are small enough to be parsed as byte
- doTestRank(BYTE_VALUESOURCE);
- }
-
- /** Test that FieldScoreQuery of Type.SHORT returns docs in expected order. */
- @Test
- public void testRankShort () throws Exception {
- // INT field values are small enough to be parsed as short
- doTestRank(SHORT_VALUESOURCE);
- }
-
/** Test that FieldScoreQuery of Type.INT returns docs in expected order. */
@Test
public void testRankInt () throws Exception {
@@ -99,20 +79,6 @@ public class TestFieldScoreQuery extends
r.close();
}
- /** Test that FieldScoreQuery of Type.BYTE returns the expected scores. */
- @Test
- public void testExactScoreByte () throws Exception {
- // INT field values are small enough to be parsed as byte
- doTestExactScore(BYTE_VALUESOURCE);
- }
-
- /** Test that FieldScoreQuery of Type.SHORT returns the expected scores. */
- @Test
- public void testExactScoreShort () throws Exception {
- // INT field values are small enough to be parsed as short
- doTestExactScore(SHORT_VALUESOURCE);
- }
-
/** Test that FieldScoreQuery of Type.INT returns the expected scores. */
@Test
public void testExactScoreInt () throws Exception {
Modified: lucene/dev/branches/security/lucene/queries/src/test/org/apache/lucene/queries/function/TestFunctionQuerySort.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/queries/src/test/org/apache/lucene/queries/function/TestFunctionQuerySort.java?rev=1499601&r1=1499600&r2=1499601&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/queries/src/test/org/apache/lucene/queries/function/TestFunctionQuerySort.java (original)
+++ lucene/dev/branches/security/lucene/queries/src/test/org/apache/lucene/queries/function/TestFunctionQuerySort.java Wed Jul 3 23:26:32 2013
@@ -18,9 +18,10 @@ package org.apache.lucene.queries.functi
*/
import java.io.IOException;
+
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
-import org.apache.lucene.document.StringField;
+import org.apache.lucene.document.IntField;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.RandomIndexWriter;
@@ -46,13 +47,13 @@ public class TestFunctionQuerySort exten
RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc);
Document doc = new Document();
- Field field = new StringField("value", "", Field.Store.YES);
+ Field field = new IntField("value", 0, Field.Store.YES);
doc.add(field);
// Save docs unsorted (decreasing value n, n-1, ...)
final int NUM_VALS = 5;
for (int val = NUM_VALS; val > 0; val--) {
- field.setStringValue(Integer.toString(val));
+ field.setIntValue(val);
writer.addDocument(doc);
}
Modified: lucene/dev/branches/security/lucene/queries/src/test/org/apache/lucene/queries/function/TestValueSources.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/queries/src/test/org/apache/lucene/queries/function/TestValueSources.java?rev=1499601&r1=1499600&r2=1499601&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/queries/src/test/org/apache/lucene/queries/function/TestValueSources.java (original)
+++ lucene/dev/branches/security/lucene/queries/src/test/org/apache/lucene/queries/function/TestValueSources.java Wed Jul 3 23:26:32 2013
@@ -22,13 +22,16 @@ import java.util.List;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
+import org.apache.lucene.document.DoubleField;
import org.apache.lucene.document.Field;
+import org.apache.lucene.document.FloatField;
+import org.apache.lucene.document.IntField;
+import org.apache.lucene.document.LongField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.RandomIndexWriter;
-import org.apache.lucene.queries.function.valuesource.ByteFieldSource;
import org.apache.lucene.queries.function.valuesource.BytesRefFieldSource;
import org.apache.lucene.queries.function.valuesource.ConstValueSource;
import org.apache.lucene.queries.function.valuesource.DivFloatFunction;
@@ -54,7 +57,6 @@ import org.apache.lucene.queries.functio
import org.apache.lucene.queries.function.valuesource.RangeMapFloatFunction;
import org.apache.lucene.queries.function.valuesource.ReciprocalFloatFunction;
import org.apache.lucene.queries.function.valuesource.ScaleFloatFunction;
-import org.apache.lucene.queries.function.valuesource.ShortFieldSource;
import org.apache.lucene.queries.function.valuesource.SumFloatFunction;
import org.apache.lucene.queries.function.valuesource.SumTotalTermFreqValueSource;
import org.apache.lucene.queries.function.valuesource.TFValueSource;
@@ -85,9 +87,9 @@ public class TestValueSources extends Lu
static IndexSearcher searcher;
static final List<String[]> documents = Arrays.asList(new String[][] {
- /* id, byte, double, float, int, long, short, string, text */
- new String[] { "0", "5", "3.63", "5.2", "35", "4343", "945", "test", "this is a test test test" },
- new String[] { "1", "12", "5.65", "9.3", "54", "1954", "123", "bar", "second test" },
+ /* id, double, float, int, long, string, text */
+ new String[] { "0", "3.63", "5.2", "35", "4343", "test", "this is a test test test" },
+ new String[] { "1", "5.65", "9.3", "54", "1954", "bar", "second test" },
});
@BeforeClass
@@ -99,18 +101,14 @@ public class TestValueSources extends Lu
Document document = new Document();
Field idField = new StringField("id", "", Field.Store.NO);
document.add(idField);
- Field byteField = new StringField("byte", "", Field.Store.NO);
- document.add(byteField);
- Field doubleField = new StringField("double", "", Field.Store.NO);
+ Field doubleField = new DoubleField("double", 0d, Field.Store.NO);
document.add(doubleField);
- Field floatField = new StringField("float", "", Field.Store.NO);
+ Field floatField = new FloatField("float", 0f, Field.Store.NO);
document.add(floatField);
- Field intField = new StringField("int", "", Field.Store.NO);
+ Field intField = new IntField("int", 0, Field.Store.NO);
document.add(intField);
- Field longField = new StringField("long", "", Field.Store.NO);
+ Field longField = new LongField("long", 0L, Field.Store.NO);
document.add(longField);
- Field shortField = new StringField("short", "", Field.Store.NO);
- document.add(shortField);
Field stringField = new StringField("string", "", Field.Store.NO);
document.add(stringField);
Field textField = new TextField("text", "", Field.Store.NO);
@@ -118,14 +116,12 @@ public class TestValueSources extends Lu
for (String [] doc : documents) {
idField.setStringValue(doc[0]);
- byteField.setStringValue(doc[1]);
- doubleField.setStringValue(doc[2]);
- floatField.setStringValue(doc[3]);
- intField.setStringValue(doc[4]);
- longField.setStringValue(doc[5]);
- shortField.setStringValue(doc[6]);
- stringField.setStringValue(doc[7]);
- textField.setStringValue(doc[8]);
+ doubleField.setDoubleValue(Double.valueOf(doc[1]));
+ floatField.setFloatValue(Float.valueOf(doc[2]));
+ intField.setIntValue(Integer.valueOf(doc[3]));
+ longField.setLongValue(Long.valueOf(doc[4]));
+ stringField.setStringValue(doc[5]);
+ textField.setStringValue(doc[6]);
iw.addDocument(document);
}
@@ -143,11 +139,6 @@ public class TestValueSources extends Lu
dir = null;
}
- public void testByte() throws Exception {
- assertHits(new FunctionQuery(new ByteFieldSource("byte")),
- new float[] { 5f, 12f });
- }
-
public void testConst() throws Exception {
assertHits(new FunctionQuery(new ConstValueSource(0.3f)),
new float[] { 0.3f, 0.3f });
@@ -298,11 +289,6 @@ public class TestValueSources extends Lu
new float[] { 0.0f, 1.0f });
}
- public void testShort() throws Exception {
- assertHits(new FunctionQuery(new ShortFieldSource("short")),
- new float[] { 945f, 123f });
- }
-
public void testSumFloat() throws Exception {
assertHits(new FunctionQuery(new SumFloatFunction(new ValueSource[] {
new ConstValueSource(1f), new ConstValueSource(2f)})),
Modified: lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/builders/QueryTreeBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/builders/QueryTreeBuilder.java?rev=1499601&r1=1499600&r2=1499601&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/builders/QueryTreeBuilder.java (original)
+++ lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/builders/QueryTreeBuilder.java Wed Jul 3 23:26:32 2013
@@ -150,7 +150,7 @@ public class QueryTreeBuilder implements
builder = getQueryBuilder(clazz);
if (builder == null) {
- Class<?>[] classes = node.getClass().getInterfaces();
+ Class<?>[] classes = clazz.getInterfaces();
for (Class<?> actualClass : classes) {
builder = getQueryBuilder(actualClass);
Modified: lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/QueryParserUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/QueryParserUtil.java?rev=1499601&r1=1499600&r2=1499601&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/QueryParserUtil.java (original)
+++ lucene/dev/branches/security/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/QueryParserUtil.java Wed Jul 3 23:26:32 2013
@@ -195,7 +195,7 @@ final public class QueryParserUtil {
if (c == '\\' || c == '+' || c == '-' || c == '!' || c == '(' || c == ')'
|| c == ':' || c == '^' || c == '[' || c == ']' || c == '\"'
|| c == '{' || c == '}' || c == '~' || c == '*' || c == '?'
- || c == '|' || c == '&') {
+ || c == '|' || c == '&' || c == '/') {
sb.append('\\');
}
sb.append(c);
Modified: lucene/dev/branches/security/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/core/builders/TestQueryTreeBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/core/builders/TestQueryTreeBuilder.java?rev=1499601&r1=1499600&r2=1499601&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/core/builders/TestQueryTreeBuilder.java (original)
+++ lucene/dev/branches/security/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/core/builders/TestQueryTreeBuilder.java Wed Jul 3 23:26:32 2013
@@ -22,6 +22,8 @@ import junit.framework.Assert;
import org.apache.lucene.queryparser.flexible.core.QueryNodeException;
import org.apache.lucene.queryparser.flexible.core.nodes.FieldQueryNode;
import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode;
+import org.apache.lucene.queryparser.flexible.core.nodes.QueryNodeImpl;
+import org.apache.lucene.queryparser.flexible.core.parser.EscapeQuerySyntax;
import org.apache.lucene.queryparser.flexible.core.util.UnescapedCharSequence;
import org.apache.lucene.util.LuceneTestCase;
import org.junit.Test;
@@ -35,6 +37,28 @@ public class TestQueryTreeBuilder extend
Object result = qtb.build(new FieldQueryNode(new UnescapedCharSequence("field"), "foo", 0, 0));
Assert.assertEquals("OK", result);
+ // LUCENE-4890
+ qtb = new QueryTreeBuilder();
+ qtb.setBuilder(DummyQueryNodeInterface.class, new DummyBuilder());
+ result = qtb.build(new DummyQueryNode());
+ Assert.assertEquals("OK", result);
+ }
+
+ private static interface DummyQueryNodeInterface extends QueryNode {
+
+ }
+
+ private static abstract class AbstractDummyQueryNode extends QueryNodeImpl implements DummyQueryNodeInterface {
+
+ }
+
+ private static class DummyQueryNode extends AbstractDummyQueryNode {
+
+ @Override
+ public CharSequence toQueryString(EscapeQuerySyntax escapeSyntaxParser) {
+ return "DummyQueryNode";
+ }
+
}
private static class DummyBuilder implements QueryBuilder {
Modified: lucene/dev/branches/security/lucene/replicator/src/test/org/apache/lucene/replicator/LocalReplicatorTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/replicator/src/test/org/apache/lucene/replicator/LocalReplicatorTest.java?rev=1499601&r1=1499600&r2=1499601&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/replicator/src/test/org/apache/lucene/replicator/LocalReplicatorTest.java (original)
+++ lucene/dev/branches/security/lucene/replicator/src/test/org/apache/lucene/replicator/LocalReplicatorTest.java Wed Jul 3 23:26:32 2013
@@ -19,6 +19,7 @@ package org.apache.lucene.replicator;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.nio.file.NoSuchFileException;
import java.util.HashMap;
import java.util.List;
import java.util.Map.Entry;
@@ -154,7 +155,7 @@ public class LocalReplicatorTest extends
try {
replicator.obtainFile(res.id, res.sourceFiles.keySet().iterator().next(), "madeUpFile");
fail("should have failed obtaining an unrecognized file");
- } catch (FileNotFoundException e) {
+ } catch (FileNotFoundException | NoSuchFileException e) {
// expected
}
}
Modified: lucene/dev/branches/security/lucene/replicator/src/test/org/apache/lucene/replicator/ReplicatorTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/replicator/src/test/org/apache/lucene/replicator/ReplicatorTestCase.java?rev=1499601&r1=1499600&r2=1499601&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/replicator/src/test/org/apache/lucene/replicator/ReplicatorTestCase.java (original)
+++ lucene/dev/branches/security/lucene/replicator/src/test/org/apache/lucene/replicator/ReplicatorTestCase.java Wed Jul 3 23:26:32 2013
@@ -17,7 +17,6 @@ package org.apache.lucene.replicator;
* limitations under the License.
*/
-import java.net.SocketException;
import java.util.Random;
import org.apache.http.conn.ClientConnectionManager;
@@ -125,12 +124,14 @@ public abstract class ReplicatorTestCase
return server;
}
- /**
- * Returns a {@link Server}'s port. This method assumes that no
- * {@link Connector}s were added to the Server besides the default one.
- */
- public static int serverPort(Server httpServer) {
- return httpServer.getConnectors()[0].getLocalPort();
+ /** Returns a {@link Server}'s port. */
+ public static int serverPort(Server server) {
+ return server.getConnectors()[0].getLocalPort();
+ }
+
+ /** Returns a {@link Server}'s host. */
+ public static String serverHost(Server server) {
+ return server.getConnectors()[0].getHost();
}
/**
Modified: lucene/dev/branches/security/lucene/replicator/src/test/org/apache/lucene/replicator/http/HttpReplicatorTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/replicator/src/test/org/apache/lucene/replicator/http/HttpReplicatorTest.java?rev=1499601&r1=1499600&r2=1499601&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/replicator/src/test/org/apache/lucene/replicator/http/HttpReplicatorTest.java (original)
+++ lucene/dev/branches/security/lucene/replicator/src/test/org/apache/lucene/replicator/http/HttpReplicatorTest.java Wed Jul 3 23:26:32 2013
@@ -50,6 +50,7 @@ public class HttpReplicatorTest extends
private DirectoryReader reader;
private Server server;
private int port;
+ private String host;
private Directory serverIndexDir, handlerIndexDir;
private void startServer() throws Exception {
@@ -59,6 +60,7 @@ public class HttpReplicatorTest extends
replicationHandler.addServletWithMapping(servlet, ReplicationService.REPLICATION_CONTEXT + "/*");
server = newHttpServer(replicationHandler);
port = serverPort(server);
+ host = serverHost(server);
}
@Before
@@ -103,7 +105,7 @@ public class HttpReplicatorTest extends
@Test
public void testBasic() throws Exception {
- Replicator replicator = new HttpReplicator("localhost", port, ReplicationService.REPLICATION_CONTEXT + "/s1",
+ Replicator replicator = new HttpReplicator(host, port, ReplicationService.REPLICATION_CONTEXT + "/s1",
getClientConnectionManager());
ReplicationClient client = new ReplicationClient(replicator, new IndexReplicationHandler(handlerIndexDir, null),
new PerSessionDirectoryFactory(clientWorkDir));
Modified: lucene/dev/branches/security/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowFuzzyTermsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowFuzzyTermsEnum.java?rev=1499601&r1=1499600&r2=1499601&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowFuzzyTermsEnum.java (original)
+++ lucene/dev/branches/security/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowFuzzyTermsEnum.java Wed Jul 3 23:26:32 2013
@@ -31,9 +31,12 @@ import org.apache.lucene.util.IntsRef;
import org.apache.lucene.util.StringHelper;
import org.apache.lucene.util.UnicodeUtil;
-/** Classic fuzzy TermsEnum for enumerating all terms that are similar
+/** Potentially slow fuzzy TermsEnum for enumerating all terms that are similar
* to the specified filter term.
- *
+ * <p> If the minSimilarity or maxEdits is greater than the Automaton's
+ * allowable range, this backs off to the classic (brute force)
+ * fuzzy terms enum method by calling FuzzyTermsEnum's getAutomatonEnum.
+ * </p>
* <p>Term enumerations are always ordered by
* {@link #getComparator}. Each term in the enumeration is
* greater than all that precede it.</p>
@@ -103,18 +106,43 @@ public final class SlowFuzzyTermsEnum ex
private final IntsRef utf32 = new IntsRef(20);
/**
- * The termCompare method in FuzzyTermEnum uses Levenshtein distance to
+ * <p>The termCompare method in FuzzyTermEnum uses Levenshtein distance to
* calculate the distance between the given term and the comparing term.
+ * </p>
+ * <p>If the minSimilarity is >= 1.0, this uses the maxEdits as the comparison.
+ * Otherwise, this method uses the following logic to calculate similarity.
+ * <pre>
+ * similarity = 1 - ((float)distance / (float) (prefixLength + Math.min(textlen, targetlen)));
+ * </pre>
+ * where distance is the Levenshtein distance for the two words.
+ * </p>
+ *
*/
@Override
protected final AcceptStatus accept(BytesRef term) {
if (StringHelper.startsWith(term, prefixBytesRef)) {
UnicodeUtil.UTF8toUTF32(term, utf32);
- final float similarity = similarity(utf32.ints, realPrefixLength, utf32.length - realPrefixLength);
- if (similarity > minSimilarity) {
+ final int distance = calcDistance(utf32.ints, realPrefixLength, utf32.length - realPrefixLength);
+
+ //Integer.MIN_VALUE is the sentinel that Levenshtein stopped early
+ if (distance == Integer.MIN_VALUE){
+ return AcceptStatus.NO;
+ }
+ //no need to calc similarity, if raw is true and distance > maxEdits
+ if (raw == true && distance > maxEdits){
+ return AcceptStatus.NO;
+ }
+ final float similarity = calcSimilarity(distance, (utf32.length - realPrefixLength), text.length);
+
+ //if raw is true, then distance must also be <= maxEdits by now
+ //given the previous if statement
+ if (raw == true ||
+ (raw == false && similarity > minSimilarity)) {
boostAtt.setBoost((similarity - minSimilarity) * scale_factor);
return AcceptStatus.YES;
- } else return AcceptStatus.NO;
+ } else {
+ return AcceptStatus.NO;
+ }
} else {
return AcceptStatus.END;
}
@@ -125,52 +153,34 @@ public final class SlowFuzzyTermsEnum ex
******************************/
/**
- * <p>Similarity returns a number that is 1.0f or less (including negative numbers)
- * based on how similar the Term is compared to a target term. It returns
- * exactly 0.0f when
- * <pre>
- * editDistance > maximumEditDistance</pre>
- * Otherwise it returns:
- * <pre>
- * 1 - (editDistance / length)</pre>
- * where length is the length of the shortest term (text or target) including a
- * prefix that are identical and editDistance is the Levenshtein distance for
- * the two words.</p>
- *
+ * <p>calcDistance returns the Levenshtein distance between the query term
+ * and the target term.</p>
+ *
* <p>Embedded within this algorithm is a fail-fast Levenshtein distance
* algorithm. The fail-fast algorithm differs from the standard Levenshtein
* distance algorithm in that it is aborted if it is discovered that the
* minimum distance between the words is greater than some threshold.
- *
- * <p>To calculate the maximum distance threshold we use the following formula:
- * <pre>
- * (1 - minimumSimilarity) * length</pre>
- * where length is the shortest term including any prefix that is not part of the
- * similarity comparison. This formula was derived by solving for what maximum value
- * of distance returns false for the following statements:
- * <pre>
- * similarity = 1 - ((float)distance / (float) (prefixLength + Math.min(textlen, targetlen)));
- * return (similarity > minimumSimilarity);</pre>
- * where distance is the Levenshtein distance for the two words.
- * </p>
+
* <p>Levenshtein distance (also known as edit distance) is a measure of similarity
* between two strings where the distance is measured as the number of character
* deletions, insertions or substitutions required to transform one string to
* the other string.
* @param target the target word or phrase
- * @return the similarity, 0.0 or less indicates that it matches less than the required
- * threshold and 1.0 indicates that the text and target are identical
+ * @param offset the offset at which to start the comparison
+ * @param length the length of what's left of the string to compare
+ * @return the number of edits or Integer.MIN_VALUE if the edit distance is
+ * greater than maxDistance.
*/
- private final float similarity(final int[] target, int offset, int length) {
+ private final int calcDistance(final int[] target, int offset, int length) {
final int m = length;
final int n = text.length;
if (n == 0) {
//we don't have anything to compare. That means if we just add
//the letters for m we get the new word
- return realPrefixLength == 0 ? 0.0f : 1.0f - ((float) m / realPrefixLength);
+ return m;
}
if (m == 0) {
- return realPrefixLength == 0 ? 0.0f : 1.0f - ((float) n / realPrefixLength);
+ return n;
}
final int maxDistance = calculateMaxDistance(m);
@@ -183,7 +193,7 @@ public final class SlowFuzzyTermsEnum ex
//which is 8-3 or more precisely Math.abs(3-8).
//if our maximum edit distance is 4, then we can discard this word
//without looking at it.
- return Float.NEGATIVE_INFINITY;
+ return Integer.MIN_VALUE;
}
// init matrix d
@@ -214,7 +224,7 @@ public final class SlowFuzzyTermsEnum ex
if (j > maxDistance && bestPossibleEditDistance > maxDistance) { //equal is okay, but not greater
//the closest the target can be to the text is just too far away.
//this target is leaving the party early.
- return Float.NEGATIVE_INFINITY;
+ return Integer.MIN_VALUE;
}
// copy current distance counts to 'previous row' distance counts: swap p and d
@@ -226,12 +236,17 @@ public final class SlowFuzzyTermsEnum ex
// our last action in the above loop was to switch d and p, so p now
// actually has the most recent cost counts
+ return p[n];
+ }
+
+ private float calcSimilarity(int edits, int m, int n){
// this will return less than 0.0 when the edit distance is
// greater than the number of characters in the shorter word.
// but this was the formula that was previously used in FuzzyTermEnum,
// so it has not been changed (even though minimumSimilarity must be
// greater than 0.0)
- return 1.0f - ((float)p[n] / (float) (realPrefixLength + Math.min(n, m)));
+
+ return 1.0f - ((float)edits / (float) (realPrefixLength + Math.min(n, m)));
}
/**
Modified: lucene/dev/branches/security/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/TestSlowFuzzyQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/TestSlowFuzzyQuery.java?rev=1499601&r1=1499600&r2=1499601&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/TestSlowFuzzyQuery.java (original)
+++ lucene/dev/branches/security/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/TestSlowFuzzyQuery.java Wed Jul 3 23:26:32 2013
@@ -43,6 +43,9 @@ import org.apache.lucene.util.LuceneTest
public class TestSlowFuzzyQuery extends LuceneTestCase {
public void testFuzziness() throws Exception {
+ //every test with SlowFuzzyQuery.defaultMinSimilarity
+ //is exercising the Automaton, not the brute force linear method
+
Directory directory = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), directory);
addDoc("aaaaa", writer);
@@ -194,6 +197,30 @@ public class TestSlowFuzzyQuery extends
directory.close();
}
+ public void testFuzzinessLong2() throws Exception {
+ //Lucene-5033
+ Directory directory = newDirectory();
+ RandomIndexWriter writer = new RandomIndexWriter(random(), directory);
+ addDoc("abcdef", writer);
+ addDoc("segment", writer);
+
+ IndexReader reader = writer.getReader();
+ IndexSearcher searcher = newSearcher(reader);
+ writer.close();
+
+ SlowFuzzyQuery query;
+
+ query = new SlowFuzzyQuery(new Term("field", "abcxxxx"), 3f, 0);
+ ScoreDoc[] hits = searcher.search(query, null, 1000).scoreDocs;
+ assertEquals(0, hits.length);
+
+ query = new SlowFuzzyQuery(new Term("field", "abcxxxx"), 4f, 0);
+ hits = searcher.search(query, null, 1000).scoreDocs;
+ assertEquals(1, hits.length);
+ reader.close();
+ directory.close();
+ }
+
public void testFuzzinessLong() throws Exception {
Directory directory = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), directory);
@@ -385,7 +412,6 @@ public class TestSlowFuzzyQuery extends
public void testGiga() throws Exception {
- MockAnalyzer analyzer = new MockAnalyzer(random());
Directory index = newDirectory();
RandomIndexWriter w = new RandomIndexWriter(random(), index);
@@ -440,25 +466,21 @@ public class TestSlowFuzzyQuery extends
assertEquals(1, hits.length);
assertEquals("foobar", searcher.doc(hits[0].doc).get("field"));
- // TODO: cannot really be supported given the legacy scoring
- // system which scores negative, if the distance > min term len,
- // so such matches were always impossible with lucene 3.x, etc
- //
- //q = new SlowFuzzyQuery(new Term("field", "t"), 3);
- //hits = searcher.search(q, 10).scoreDocs;
- //assertEquals(1, hits.length);
- //assertEquals("test", searcher.doc(hits[0].doc).get("field"));
-
- // q = new SlowFuzzyQuery(new Term("field", "a"), 4f, 0, 50);
- // hits = searcher.search(q, 10).scoreDocs;
- // assertEquals(1, hits.length);
- // assertEquals("test", searcher.doc(hits[0].doc).get("field"));
-
- // q = new SlowFuzzyQuery(new Term("field", "a"), 6f, 0, 50);
- // hits = searcher.search(q, 10).scoreDocs;
- // assertEquals(2, hits.length);
- // assertEquals("test", searcher.doc(hits[0].doc).get("field"));
- // assertEquals("foobar", searcher.doc(hits[1].doc).get("field"));
+ q = new SlowFuzzyQuery(new Term("field", "t"), 3);
+ hits = searcher.search(q, 10).scoreDocs;
+ assertEquals(1, hits.length);
+ assertEquals("test", searcher.doc(hits[0].doc).get("field"));
+
+ q = new SlowFuzzyQuery(new Term("field", "a"), 4f, 0, 50);
+ hits = searcher.search(q, 10).scoreDocs;
+ assertEquals(1, hits.length);
+ assertEquals("test", searcher.doc(hits[0].doc).get("field"));
+
+ q = new SlowFuzzyQuery(new Term("field", "a"), 6f, 0, 50);
+ hits = searcher.search(q, 10).scoreDocs;
+ assertEquals(2, hits.length);
+ assertEquals("test", searcher.doc(hits[0].doc).get("field"));
+ assertEquals("foobar", searcher.doc(hits[1].doc).get("field"));
reader.close();
index.close();
Modified: lucene/dev/branches/security/lucene/spatial/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/spatial/build.xml?rev=1499601&r1=1499600&r2=1499601&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/spatial/build.xml (original)
+++ lucene/dev/branches/security/lucene/spatial/build.xml Wed Jul 3 23:26:32 2013
@@ -1,4 +1,22 @@
<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
<project name="spatial" default="default">
<description>
Geospatial search
Modified: lucene/dev/branches/security/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/ContainsPrefixTreeFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/ContainsPrefixTreeFilter.java?rev=1499601&r1=1499600&r2=1499601&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/ContainsPrefixTreeFilter.java (original)
+++ lucene/dev/branches/security/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/ContainsPrefixTreeFilter.java Wed Jul 3 23:26:32 2013
@@ -41,8 +41,35 @@ import java.util.Collection;
*/
public class ContainsPrefixTreeFilter extends AbstractPrefixTreeFilter {
- public ContainsPrefixTreeFilter(Shape queryShape, String fieldName, SpatialPrefixTree grid, int detailLevel) {
+ /*
+ Future optimizations:
+ Instead of seekExact, use seekCeil with some leap-frogging, like Intersects does.
+ */
+
+ /**
+ * If the spatial data for a document is comprised of multiple overlapping or adjacent parts,
+ * it might fail to match a query shape when doing the CONTAINS predicate when the sum of
+ * those shapes contain the query shape but none do individually. Set this to false to
+ * increase performance if you don't care about that circumstance (such as if your indexed
+ * data doesn't even have such conditions). See LUCENE-5062.
+ */
+ protected final boolean multiOverlappingIndexedShapes;
+
+ public ContainsPrefixTreeFilter(Shape queryShape, String fieldName, SpatialPrefixTree grid, int detailLevel, boolean multiOverlappingIndexedShapes) {
super(queryShape, fieldName, grid, detailLevel);
+ this.multiOverlappingIndexedShapes = multiOverlappingIndexedShapes;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (!super.equals(o))
+ return false;
+ return multiOverlappingIndexedShapes == ((ContainsPrefixTreeFilter)o).multiOverlappingIndexedShapes;
+ }
+
+ @Override
+ public int hashCode() {
+ return super.hashCode() + (multiOverlappingIndexedShapes ? 1 : 0);
}
@Override
@@ -65,18 +92,25 @@ public class ContainsPrefixTreeFilter ex
if (termsEnum == null)//signals all done
return null;
- //Leaf docs match all query shape
+ // Leaf docs match all query shape
SmallDocSet leafDocs = getLeafDocs(cell, acceptContains);
- // Get the AND of all child results
+ // Get the AND of all child results (into combinedSubResults)
SmallDocSet combinedSubResults = null;
- Collection<Cell> subCells = cell.getSubCells(queryShape);
+ // Optimization: use null subCellsFilter when we know cell is within the query shape.
+ Shape subCellsFilter = queryShape;
+ if (cell.getLevel() != 0 && ((cell.getShapeRel() == null || cell.getShapeRel() == SpatialRelation.WITHIN))) {
+ subCellsFilter = null;
+ assert cell.getShape().relate(queryShape) == SpatialRelation.WITHIN;
+ }
+ Collection <Cell> subCells = cell.getSubCells(subCellsFilter);
for (Cell subCell : subCells) {
if (!seekExact(subCell))
combinedSubResults = null;
else if (subCell.getLevel() == detailLevel)
combinedSubResults = getDocs(subCell, acceptContains);
- else if (subCell.getShapeRel() == SpatialRelation.WITHIN)
+ else if (!multiOverlappingIndexedShapes &&
+ subCell.getShapeRel() == SpatialRelation.WITHIN)
combinedSubResults = getLeafDocs(subCell, acceptContains);
else
combinedSubResults = visit(subCell, acceptContains); //recursion
@@ -90,7 +124,7 @@ public class ContainsPrefixTreeFilter ex
if (combinedSubResults != null) {
if (leafDocs == null)
return combinedSubResults;
- return leafDocs.union(combinedSubResults);
+ return leafDocs.union(combinedSubResults);//union is 'or'
}
return leafDocs;
}
@@ -109,8 +143,12 @@ public class ContainsPrefixTreeFilter ex
return collectDocs(acceptContains);
}
+ private Cell lastLeaf = null;//just for assertion
+
private SmallDocSet getLeafDocs(Cell leafCell, Bits acceptContains) throws IOException {
assert new BytesRef(leafCell.getTokenBytes()).equals(termBytes);
+ assert ! leafCell.equals(lastLeaf);//don't call for same leaf again
+ lastLeaf = leafCell;
BytesRef nextTerm = termsEnum.next();
if (nextTerm == null) {
Modified: lucene/dev/branches/security/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/RecursivePrefixTreeStrategy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/RecursivePrefixTreeStrategy.java?rev=1499601&r1=1499600&r2=1499601&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/RecursivePrefixTreeStrategy.java (original)
+++ lucene/dev/branches/security/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/RecursivePrefixTreeStrategy.java Wed Jul 3 23:26:32 2013
@@ -38,6 +38,13 @@ public class RecursivePrefixTreeStrategy
private int prefixGridScanLevel;
+ /** True if only indexed points shall be supported. See
+ * {@link IntersectsPrefixTreeFilter#hasIndexedLeaves}. */
+ protected boolean pointsOnly = false;
+
+ /** See {@link ContainsPrefixTreeFilter#multiOverlappingIndexedShapes}. */
+ protected boolean multiOverlappingIndexedShapes = true;
+
public RecursivePrefixTreeStrategy(SpatialPrefixTree grid, String fieldName) {
super(grid, fieldName,
true);//simplify indexed cells
@@ -69,18 +76,17 @@ public class RecursivePrefixTreeStrategy
Shape shape = args.getShape();
int detailLevel = grid.getLevelForDistance(args.resolveDistErr(ctx, distErrPct));
- final boolean hasIndexedLeaves = true;
- if (op == SpatialOperation.Intersects) {
+ if (pointsOnly || op == SpatialOperation.Intersects) {
return new IntersectsPrefixTreeFilter(
- shape, getFieldName(), grid, detailLevel, prefixGridScanLevel,
- hasIndexedLeaves);
+ shape, getFieldName(), grid, detailLevel, prefixGridScanLevel, !pointsOnly);
} else if (op == SpatialOperation.IsWithin) {
return new WithinPrefixTreeFilter(
shape, getFieldName(), grid, detailLevel, prefixGridScanLevel,
-1);//-1 flag is slower but ensures correct results
} else if (op == SpatialOperation.Contains) {
- return new ContainsPrefixTreeFilter(shape, getFieldName(), grid, detailLevel);
+ return new ContainsPrefixTreeFilter(shape, getFieldName(), grid, detailLevel,
+ multiOverlappingIndexedShapes);
}
throw new UnsupportedSpatialOperation(op);
}
Modified: lucene/dev/branches/security/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/SpatialOpRecursivePrefixTreeTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/SpatialOpRecursivePrefixTreeTest.java?rev=1499601&r1=1499600&r2=1499601&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/SpatialOpRecursivePrefixTreeTest.java (original)
+++ lucene/dev/branches/security/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/SpatialOpRecursivePrefixTreeTest.java Wed Jul 3 23:26:32 2013
@@ -110,6 +110,18 @@ public class SpatialOpRecursivePrefixTre
doTest(SpatialOperation.IsDisjointTo);
}
+ /** See LUCENE-5062, {@link ContainsPrefixTreeFilter#multiOverlappingIndexedShapes}. */
+ @Test
+ public void testContainsPairOverlap() throws IOException {
+ mySetup(3);
+ adoc("0", new ShapePair(ctx.makeRectangle(0, 33, -128, 128), ctx.makeRectangle(33, 128, -128, 128), true));
+ commit();
+ Query query = strategy.makeQuery(new SpatialArgs(SpatialOperation.Contains,
+ ctx.makeRectangle(0, 128, -16, 128)));
+ SearchResults searchResults = executeQuery(query, 1);
+ assertEquals(1, searchResults.numFound);
+ }
+
@Test
public void testWithinDisjointParts() throws IOException {
mySetup(7);
@@ -184,10 +196,10 @@ public class SpatialOpRecursivePrefixTre
Shape indexedShape;
Shape indexedShapeGS; //(grid-snapped)
int R = random().nextInt(12);
- if (R == 0) {//1 in 10
+ if (R == 0) {//1 in 12
indexedShape = null; //no shape for this doc
indexedShapeGS = null;
- } else if (R % 4 == 0) {//3 in 12
+ } else if (R % 3 == 0) {//4-1 in 12
//comprised of more than one shape
Rectangle shape1 = randomRectangle();
Rectangle shape2 = randomRectangle();
Modified: lucene/dev/branches/security/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java?rev=1499601&r1=1499600&r2=1499601&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java (original)
+++ lucene/dev/branches/security/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java Wed Jul 3 23:26:32 2013
@@ -512,7 +512,7 @@ public class AnalyzingSuggester extends
reader = new Sort.ByteSequencesReader(tempSorted);
- PairOutputs<Long,BytesRef> outputs = new PairOutputs<Long,BytesRef>(PositiveIntOutputs.getSingleton(true), ByteSequenceOutputs.getSingleton());
+ PairOutputs<Long,BytesRef> outputs = new PairOutputs<Long,BytesRef>(PositiveIntOutputs.getSingleton(), ByteSequenceOutputs.getSingleton());
Builder<Pair<Long,BytesRef>> builder = new Builder<Pair<Long,BytesRef>>(FST.INPUT_TYPE.BYTE1, outputs);
// Build FST:
@@ -634,7 +634,7 @@ public class AnalyzingSuggester extends
public boolean load(InputStream input) throws IOException {
DataInput dataIn = new InputStreamDataInput(input);
try {
- this.fst = new FST<Pair<Long,BytesRef>>(dataIn, new PairOutputs<Long,BytesRef>(PositiveIntOutputs.getSingleton(true), ByteSequenceOutputs.getSingleton()));
+ this.fst = new FST<Pair<Long,BytesRef>>(dataIn, new PairOutputs<Long,BytesRef>(PositiveIntOutputs.getSingleton(), ByteSequenceOutputs.getSingleton()));
maxAnalyzedPathsForOneInput = dataIn.readVInt();
hasPayloads = dataIn.readByte() == 1;
} finally {
Modified: lucene/dev/branches/security/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java?rev=1499601&r1=1499600&r2=1499601&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java (original)
+++ lucene/dev/branches/security/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java Wed Jul 3 23:26:32 2013
@@ -101,7 +101,7 @@ public class WFSTCompletionLookup extend
TermFreqIterator iter = new WFSTTermFreqIteratorWrapper(iterator);
IntsRef scratchInts = new IntsRef();
BytesRef previous = null;
- PositiveIntOutputs outputs = PositiveIntOutputs.getSingleton(true);
+ PositiveIntOutputs outputs = PositiveIntOutputs.getSingleton();
Builder<Long> builder = new Builder<Long>(FST.INPUT_TYPE.BYTE1, outputs);
while ((scratch = iter.next()) != null) {
long cost = iter.weight();
@@ -136,7 +136,7 @@ public class WFSTCompletionLookup extend
@Override
public boolean load(InputStream input) throws IOException {
try {
- this.fst = new FST<Long>(new InputStreamDataInput(input), PositiveIntOutputs.getSingleton(true));
+ this.fst = new FST<Long>(new InputStreamDataInput(input), PositiveIntOutputs.getSingleton());
} finally {
IOUtils.close(input);
}
Modified: lucene/dev/branches/security/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java?rev=1499601&r1=1499600&r2=1499601&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java (original)
+++ lucene/dev/branches/security/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java Wed Jul 3 23:26:32 2013
@@ -1164,4 +1164,33 @@ public class AnalyzingSuggesterTest exte
assertEquals("[isla de muerta/8, i love lucy/7]", suggester.lookup("i", false, 3).toString());
assertEquals("[i love lucy/7]", suggester.lookup("i ", false, 3).toString());
}
+
+ public void testTooManyExpansions() throws Exception {
+
+ final Analyzer a = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
+
+ return new TokenStreamComponents(tokenizer) {
+ @Override
+ public TokenStream getTokenStream() {
+ Token a = new Token("a", 0, 1);
+ a.setPositionIncrement(1);
+ Token b = new Token("b", 0, 1);
+ b.setPositionIncrement(0);
+ return new CannedTokenStream(new Token[] {a, b});
+ }
+
+ @Override
+ protected void setReader(final Reader reader) throws IOException {
+ }
+ };
+ }
+ };
+
+ AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, 0, 256, 1);
+ suggester.build(new TermFreqArrayIterator(new TermFreq[] {new TermFreq("a", 1)}));
+ assertEquals("[a/1]", suggester.lookup("a", false, 1).toString());
+ }
}
Modified: lucene/dev/branches/security/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/FastCompressingCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/FastCompressingCodec.java?rev=1499601&r1=1499600&r2=1499601&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/FastCompressingCodec.java (original)
+++ lucene/dev/branches/security/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/FastCompressingCodec.java Wed Jul 3 23:26:32 2013
@@ -1,5 +1,11 @@
package org.apache.lucene.codecs.compressing;
+import org.apache.lucene.codecs.DocValuesFormat;
+import org.apache.lucene.codecs.NormsFormat;
+import org.apache.lucene.codecs.lucene42.Lucene42DocValuesFormat;
+import org.apache.lucene.codecs.lucene42.Lucene42NormsFormat;
+import org.apache.lucene.util.packed.PackedInts;
+
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -32,4 +38,13 @@ public class FastCompressingCodec extend
this(1 << 14, false);
}
+ @Override
+ public NormsFormat normsFormat() {
+ return new Lucene42NormsFormat(PackedInts.FAST);
+ }
+
+ @Override
+ public DocValuesFormat docValuesFormat() {
+ return new Lucene42DocValuesFormat(PackedInts.FAST);
+ }
}
Modified: lucene/dev/branches/security/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/FastDecompressionCompressingCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/FastDecompressionCompressingCodec.java?rev=1499601&r1=1499600&r2=1499601&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/FastDecompressionCompressingCodec.java (original)
+++ lucene/dev/branches/security/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/FastDecompressionCompressingCodec.java Wed Jul 3 23:26:32 2013
@@ -1,5 +1,11 @@
package org.apache.lucene.codecs.compressing;
+import org.apache.lucene.codecs.DocValuesFormat;
+import org.apache.lucene.codecs.NormsFormat;
+import org.apache.lucene.codecs.lucene42.Lucene42DocValuesFormat;
+import org.apache.lucene.codecs.lucene42.Lucene42NormsFormat;
+import org.apache.lucene.util.packed.PackedInts;
+
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -32,4 +38,13 @@ public class FastDecompressionCompressin
this(1 << 14, false);
}
+ @Override
+ public NormsFormat normsFormat() {
+ return new Lucene42NormsFormat(PackedInts.DEFAULT);
+ }
+
+ @Override
+ public DocValuesFormat docValuesFormat() {
+ return new Lucene42DocValuesFormat(PackedInts.DEFAULT);
+ }
}
Modified: lucene/dev/branches/security/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/HighCompressionCompressingCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/HighCompressionCompressingCodec.java?rev=1499601&r1=1499600&r2=1499601&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/HighCompressionCompressingCodec.java (original)
+++ lucene/dev/branches/security/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/HighCompressionCompressingCodec.java Wed Jul 3 23:26:32 2013
@@ -1,5 +1,9 @@
package org.apache.lucene.codecs.compressing;
+import org.apache.lucene.codecs.NormsFormat;
+import org.apache.lucene.codecs.lucene42.Lucene42NormsFormat;
+import org.apache.lucene.util.packed.PackedInts;
+
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -32,4 +36,8 @@ public class HighCompressionCompressingC
this(1 << 14, false);
}
+ @Override
+ public NormsFormat normsFormat() {
+ return new Lucene42NormsFormat(PackedInts.COMPACT);
+ }
}
Modified: lucene/dev/branches/security/lucene/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java?rev=1499601&r1=1499600&r2=1499601&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java (original)
+++ lucene/dev/branches/security/lucene/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java Wed Jul 3 23:26:32 2013
@@ -94,7 +94,7 @@ public abstract class BaseStoredFieldsFo
Directory dir = newDirectory();
Random rand = random();
RandomIndexWriter w = new RandomIndexWriter(rand, dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())).setMaxBufferedDocs(_TestUtil.nextInt(rand, 5, 20)));
- //w.w.setUseCompoundFile(false);
+ //w.w.setNoCFSRatio(0.0);
final int docCount = atLeast(200);
final int fieldCount = _TestUtil.nextInt(rand, 1, 5);
Modified: lucene/dev/branches/security/lucene/test-framework/src/java/org/apache/lucene/index/DocHelper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/test-framework/src/java/org/apache/lucene/index/DocHelper.java?rev=1499601&r1=1499600&r2=1499601&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/test-framework/src/java/org/apache/lucene/index/DocHelper.java (original)
+++ lucene/dev/branches/security/lucene/test-framework/src/java/org/apache/lucene/index/DocHelper.java Wed Jul 3 23:26:32 2013
@@ -272,7 +272,7 @@ class DocHelper {
public static SegmentInfoPerCommit writeDoc(Random random, Directory dir, Analyzer analyzer, Similarity similarity, Document doc) throws IOException {
IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig( /* LuceneTestCase.newIndexWriterConfig(random, */
TEST_VERSION_CURRENT, analyzer).setSimilarity(similarity == null ? IndexSearcher.getDefaultSimilarity() : similarity));
- //writer.setUseCompoundFile(false);
+ //writer.setNoCFSRatio(0.0);
writer.addDocument(doc);
writer.commit();
SegmentInfoPerCommit info = writer.newestSegment();
Modified: lucene/dev/branches/security/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java?rev=1499601&r1=1499600&r2=1499601&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java (original)
+++ lucene/dev/branches/security/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java Wed Jul 3 23:26:32 2013
@@ -25,17 +25,12 @@ import java.util.Random;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.document.BinaryDocValuesField;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.NumericDocValuesField;
-import org.apache.lucene.document.SortedDocValuesField;
-import org.apache.lucene.index.FieldInfo.DocValuesType;
import org.apache.lucene.index.IndexWriter; // javadoc
import org.apache.lucene.search.Query;
import org.apache.lucene.store.Directory;
-import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.InfoStream;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.NullInfoStream;
import org.apache.lucene.util.Version;
import org.apache.lucene.util._TestUtil;
@@ -55,23 +50,22 @@ public class RandomIndexWriter implement
private boolean getReaderCalled;
private final Codec codec; // sugar
- // Randomly calls Thread.yield so we mixup thread scheduling
- private static final class MockIndexWriter extends IndexWriter {
-
- private final Random r;
-
- public MockIndexWriter(Random r, Directory dir, IndexWriterConfig conf) throws IOException {
- super(dir, conf);
- // TODO: this should be solved in a different way; Random should not be shared (!).
- this.r = new Random(r.nextLong());
- }
-
- @Override
- boolean testPoint(String name) {
- if (r.nextInt(4) == 2)
- Thread.yield();
- return true;
- }
+
+ public static IndexWriter mockIndexWriter(Directory dir, IndexWriterConfig conf, Random r) throws IOException {
+ // Randomly calls Thread.yield so we mixup thread scheduling
+ final Random random = new Random(r.nextLong());
+ return mockIndexWriter(dir, conf, new TestPoint() {
+ @Override
+ public void apply(String message) {
+ if (random.nextInt(4) == 2)
+ Thread.yield();
+ }
+ });
+ }
+
+ public static IndexWriter mockIndexWriter(Directory dir, IndexWriterConfig conf, TestPoint testPoint) throws IOException {
+ conf.setInfoStream(new TestPointInfoStream(conf.getInfoStream(), testPoint));
+ return new IndexWriter(dir, conf);
}
/** create a RandomIndexWriter with a random config: Uses TEST_VERSION_CURRENT and MockAnalyzer */
@@ -93,7 +87,7 @@ public class RandomIndexWriter implement
public RandomIndexWriter(Random r, Directory dir, IndexWriterConfig c) throws IOException {
// TODO: this should be solved in a different way; Random should not be shared (!).
this.r = new Random(r.nextLong());
- w = new MockIndexWriter(r, dir, c);
+ w = mockIndexWriter(dir, c, r);
flushAt = _TestUtil.nextInt(r, 10, 1000);
codec = w.getConfig().getCodec();
if (LuceneTestCase.VERBOSE) {
@@ -345,4 +339,42 @@ public class RandomIndexWriter implement
public void forceMerge(int maxSegmentCount) throws IOException {
w.forceMerge(maxSegmentCount);
}
+
+ private static final class TestPointInfoStream extends InfoStream {
+ private final InfoStream delegate;
+ private final TestPoint testPoint;
+
+ public TestPointInfoStream(InfoStream delegate, TestPoint testPoint) {
+ this.delegate = delegate == null ? new NullInfoStream(): delegate;
+ this.testPoint = testPoint;
+ }
+
+ @Override
+ public void close() throws IOException {
+ delegate.close();
+ }
+
+ @Override
+ public void message(String component, String message) {
+ if ("TP".equals(component)) {
+ testPoint.apply(message);
+ }
+ if (delegate.isEnabled(component)) {
+ delegate.message(component, message);
+ }
+ }
+
+ @Override
+ public boolean isEnabled(String component) {
+ return "TP".equals(component) || delegate.isEnabled(component);
+ }
+ }
+
+ /**
+ * Simple interface that is executed for each <tt>TP</tt> {@link InfoStream} component
+ * message. See also {@link RandomIndexWriter#mockIndexWriter(Directory, IndexWriterConfig, TestPoint)}
+ */
+ public static interface TestPoint {
+ public abstract void apply(String message);
+ }
}
Modified: lucene/dev/branches/security/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java?rev=1499601&r1=1499600&r2=1499601&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java (original)
+++ lucene/dev/branches/security/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java Wed Jul 3 23:26:32 2013
@@ -20,6 +20,7 @@ package org.apache.lucene.store;
import java.io.Closeable;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.nio.file.NoSuchFileException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -385,7 +386,7 @@ public class MockDirectoryWrapper extend
if (randomState.nextBoolean()) {
throw new IOException("a random IOException (" + name + ")");
} else {
- throw new FileNotFoundException("a random IOException (" + name + ")");
+ throw randomState.nextBoolean() ? new FileNotFoundException("a random IOException (" + name + ")") : new NoSuchFileException("a random IOException (" + name + ")");
}
}
}
@@ -544,7 +545,7 @@ public class MockDirectoryWrapper extend
maybeThrowDeterministicException();
}
if (!delegate.fileExists(name)) {
- throw new FileNotFoundException(name + " in dir=" + delegate);
+ throw randomState.nextBoolean() ? new FileNotFoundException(name + " in dir=" + delegate) : new NoSuchFileException(name + " in dir=" + delegate);
}
// cannot open a file for input if it's still open for
@@ -920,7 +921,7 @@ public class MockDirectoryWrapper extend
throws IOException {
maybeYield();
if (!delegate.fileExists(name)) {
- throw new FileNotFoundException(name);
+ throw randomState.nextBoolean() ? new FileNotFoundException(name) : new NoSuchFileException(name);
}
// cannot open a file for input if it's still open for
// output, except for segments.gen and segments_N
Modified: lucene/dev/branches/security/lucene/test-framework/src/java/org/apache/lucene/util/LineFileDocs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/test-framework/src/java/org/apache/lucene/util/LineFileDocs.java?rev=1499601&r1=1499600&r2=1499601&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/test-framework/src/java/org/apache/lucene/util/LineFileDocs.java (original)
+++ lucene/dev/branches/security/lucene/test-framework/src/java/org/apache/lucene/util/LineFileDocs.java Wed Jul 3 23:26:32 2013
@@ -36,6 +36,7 @@ import java.util.zip.GZIPInputStream;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.IntField;
import org.apache.lucene.document.SortedDocValuesField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
@@ -158,6 +159,7 @@ public class LineFileDocs implements Clo
final Field titleDV;
final Field body;
final Field id;
+ final Field idNum;
final Field date;
public DocState(boolean useDocValues) {
@@ -180,6 +182,9 @@ public class LineFileDocs implements Clo
id = new StringField("docid", "", Field.Store.YES);
doc.add(id);
+ idNum = new IntField("docid_int", 0, Field.Store.NO);
+ doc.add(idNum);
+
date = new StringField("date", "", Field.Store.YES);
doc.add(date);
@@ -233,7 +238,9 @@ public class LineFileDocs implements Clo
}
docState.titleTokenized.setStringValue(title);
docState.date.setStringValue(line.substring(1+spot, spot2));
- docState.id.setStringValue(Integer.toString(id.getAndIncrement()));
+ final int i = id.getAndIncrement();
+ docState.id.setStringValue(Integer.toString(i));
+ docState.idNum.setIntValue(i);
return docState.doc;
}
}
Modified: lucene/dev/branches/security/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java?rev=1499601&r1=1499600&r2=1499601&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java (original)
+++ lucene/dev/branches/security/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java Wed Jul 3 23:26:32 2013
@@ -737,6 +737,12 @@ public abstract class LuceneTestCase ext
if (r.nextBoolean()) {
c.setMergeScheduler(new SerialMergeScheduler());
+ } else if (rarely(r)) {
+ int maxThreadCount = _TestUtil.nextInt(random(), 1, 4);
+ int maxMergeCount = _TestUtil.nextInt(random(), maxThreadCount, maxThreadCount+4);
+ ConcurrentMergeScheduler cms = new ConcurrentMergeScheduler();
+ cms.setMaxMergesAndThreads(maxMergeCount, maxThreadCount);
+ c.setMergeScheduler(cms);
}
if (r.nextBoolean()) {
if (rarely(r)) {
@@ -808,6 +814,7 @@ public abstract class LuceneTestCase ext
if (rarely(r)) {
c.setMergedSegmentWarmer(new SimpleMergedSegmentWarmer(c.getInfoStream()));
}
+ c.setUseCompoundFile(r.nextBoolean());
c.setReaderPooling(r.nextBoolean());
c.setReaderTermsIndexDivisor(_TestUtil.nextInt(r, 1, 4));
return c;
@@ -831,19 +838,28 @@ public abstract class LuceneTestCase ext
public static LogMergePolicy newLogMergePolicy(Random r) {
LogMergePolicy logmp = r.nextBoolean() ? new LogDocMergePolicy() : new LogByteSizeMergePolicy();
- logmp.setUseCompoundFile(r.nextBoolean());
logmp.setCalibrateSizeByDeletes(r.nextBoolean());
if (rarely(r)) {
logmp.setMergeFactor(_TestUtil.nextInt(r, 2, 9));
} else {
logmp.setMergeFactor(_TestUtil.nextInt(r, 10, 50));
}
- logmp.setUseCompoundFile(r.nextBoolean());
- logmp.setNoCFSRatio(0.1 + r.nextDouble()*0.8);
+ configureRandom(r, logmp);
+ return logmp;
+ }
+
+ private static void configureRandom(Random r, MergePolicy mergePolicy) {
+ if (r.nextBoolean()) {
+ mergePolicy.setNoCFSRatio(0.1 + r.nextDouble()*0.8);
+ } else {
+ mergePolicy.setNoCFSRatio(r.nextBoolean() ? 1.0 : 0.0);
+ }
+
if (rarely()) {
- logmp.setMaxCFSSegmentSizeMB(0.2 + r.nextDouble() * 2.0);
+ mergePolicy.setMaxCFSSegmentSizeMB(0.2 + r.nextDouble() * 2.0);
+ } else {
+ mergePolicy.setMaxCFSSegmentSizeMB(Double.POSITIVE_INFINITY);
}
- return logmp;
}
public static TieredMergePolicy newTieredMergePolicy(Random r) {
@@ -867,29 +883,25 @@ public abstract class LuceneTestCase ext
} else {
tmp.setSegmentsPerTier(_TestUtil.nextInt(r, 10, 50));
}
- tmp.setUseCompoundFile(r.nextBoolean());
- tmp.setNoCFSRatio(0.1 + r.nextDouble()*0.8);
- if (rarely()) {
- tmp.setMaxCFSSegmentSizeMB(0.2 + r.nextDouble() * 2.0);
- }
+ configureRandom(r, tmp);
tmp.setReclaimDeletesWeight(r.nextDouble()*4);
return tmp;
}
- public static LogMergePolicy newLogMergePolicy(boolean useCFS) {
- LogMergePolicy logmp = newLogMergePolicy();
- logmp.setUseCompoundFile(useCFS);
+ public static MergePolicy newLogMergePolicy(boolean useCFS) {
+ MergePolicy logmp = newLogMergePolicy();
+ logmp.setNoCFSRatio(useCFS ? 1.0 : 0.0);
return logmp;
}
- public static LogMergePolicy newLogMergePolicy(boolean useCFS, int mergeFactor) {
+ public static MergePolicy newLogMergePolicy(boolean useCFS, int mergeFactor) {
LogMergePolicy logmp = newLogMergePolicy();
- logmp.setUseCompoundFile(useCFS);
+ logmp.setNoCFSRatio(useCFS ? 1.0 : 0.0);
logmp.setMergeFactor(mergeFactor);
return logmp;
}
- public static LogMergePolicy newLogMergePolicy(int mergeFactor) {
+ public static MergePolicy newLogMergePolicy(int mergeFactor) {
LogMergePolicy logmp = newLogMergePolicy();
logmp.setMergeFactor(mergeFactor);
return logmp;
Modified: lucene/dev/branches/security/lucene/test-framework/src/java/org/apache/lucene/util/RunListenerPrintReproduceInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/test-framework/src/java/org/apache/lucene/util/RunListenerPrintReproduceInfo.java?rev=1499601&r1=1499600&r2=1499601&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/test-framework/src/java/org/apache/lucene/util/RunListenerPrintReproduceInfo.java (original)
+++ lucene/dev/branches/security/lucene/test-framework/src/java/org/apache/lucene/util/RunListenerPrintReproduceInfo.java Wed Jul 3 23:26:32 2013
@@ -88,12 +88,27 @@ public final class RunListenerPrintRepro
@Override
public void testFinished(Description description) throws Exception {
if (testFailed) {
- reportAdditionalFailureInfo(description.getMethodName());
+ reportAdditionalFailureInfo(
+ stripTestNameAugmentations(
+ description.getMethodName()));
}
scope = LifecycleScope.SUITE;
testFailed = false;
}
+ /**
+ * The {@link Description} object in JUnit does not expose the actual test method,
+ * instead it has the concept of a unique "name" of a test. To run the same method (tests)
+ * repeatedly, randomizedtesting must make those "names" unique: it appends the current iteration
+ * and seeds to the test method's name. We strip this information here.
+ */
+ private String stripTestNameAugmentations(String methodName) {
+ if (methodName != null) {
+ methodName = methodName.replaceAll("\\s*\\{.+?\\}", "");
+ }
+ return methodName;
+ }
+
@Override
public void testRunFinished(Result result) throws Exception {
if (printDiagnosticsAfterClass || LuceneTestCase.VERBOSE) {
Modified: lucene/dev/branches/security/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleSetupAndRestoreClassEnv.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleSetupAndRestoreClassEnv.java?rev=1499601&r1=1499600&r2=1499601&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleSetupAndRestoreClassEnv.java (original)
+++ lucene/dev/branches/security/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleSetupAndRestoreClassEnv.java Wed Jul 3 23:26:32 2013
@@ -84,6 +84,9 @@ final class TestRuleSetupAndRestoreClass
@Override
public void message(String component, String message) {
+ if ("TP".equals(component)) {
+ return; // ignore test points!
+ }
final String name;
if (Thread.currentThread().getName().startsWith("TEST-")) {
// The name of the main thread is way too
Modified: lucene/dev/branches/security/lucene/test-framework/src/java/org/apache/lucene/util/_TestUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/test-framework/src/java/org/apache/lucene/util/_TestUtil.java?rev=1499601&r1=1499600&r2=1499601&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/test-framework/src/java/org/apache/lucene/util/_TestUtil.java (original)
+++ lucene/dev/branches/security/lucene/test-framework/src/java/org/apache/lucene/util/_TestUtil.java Wed Jul 3 23:26:32 2013
@@ -49,7 +49,12 @@ import org.apache.lucene.codecs.lucene42
import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;
import org.apache.lucene.document.BinaryDocValuesField;
import org.apache.lucene.document.Document;
+import org.apache.lucene.document.DoubleField;
import org.apache.lucene.document.Field;
+import org.apache.lucene.document.FieldType.NumericType;
+import org.apache.lucene.document.FloatField;
+import org.apache.lucene.document.IntField;
+import org.apache.lucene.document.LongField;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.SortedDocValuesField;
import org.apache.lucene.index.AtomicReader;
@@ -78,14 +83,15 @@ import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.index.TieredMergePolicy;
import org.apache.lucene.search.FieldDoc;
-import org.apache.lucene.search.FilteredQuery.FilterStrategy;
import org.apache.lucene.search.FilteredQuery;
+import org.apache.lucene.search.FilteredQuery.FilterStrategy;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.CompoundFileDirectory;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.junit.Assert;
+
import com.carrotsearch.randomizedtesting.RandomizedContext;
import com.carrotsearch.randomizedtesting.generators.RandomInts;
import com.carrotsearch.randomizedtesting.generators.RandomPicks;
@@ -260,17 +266,6 @@ public class _TestUtil {
}
}
- // NOTE: only works for TMP and LMP!!
- public static void setUseCompoundFile(MergePolicy mp, boolean v) {
- if (mp instanceof TieredMergePolicy) {
- ((TieredMergePolicy) mp).setUseCompoundFile(v);
- } else if (mp instanceof LogMergePolicy) {
- ((LogMergePolicy) mp).setUseCompoundFile(v);
- } else {
- throw new IllegalArgumentException("cannot set compound file for MergePolicy " + mp);
- }
- }
-
/** start and end are BOTH inclusive */
public static int nextInt(Random r, int start, int end) {
return RandomInts.randomIntBetween(r, start, end);
@@ -293,7 +288,11 @@ public class _TestUtil {
}
public static String randomSimpleString(Random r, int maxLength) {
- final int end = nextInt(r, 0, maxLength);
+ return randomSimpleString(r, 0, maxLength);
+ }
+
+ public static String randomSimpleString(Random r, int minLength, int maxLength) {
+ final int end = nextInt(r, minLength, maxLength);
if (end == 0) {
// allow 0 length
return "";
@@ -319,7 +318,7 @@ public class _TestUtil {
}
public static String randomSimpleString(Random r) {
- return randomSimpleString(r, 10);
+ return randomSimpleString(r, 0, 10);
}
/** Returns random string, including full unicode range. */
@@ -762,17 +761,17 @@ public class _TestUtil {
if (mp instanceof LogMergePolicy) {
LogMergePolicy lmp = (LogMergePolicy) mp;
lmp.setMergeFactor(Math.min(5, lmp.getMergeFactor()));
- lmp.setUseCompoundFile(true);
+ lmp.setNoCFSRatio(1.0);
} else if (mp instanceof TieredMergePolicy) {
TieredMergePolicy tmp = (TieredMergePolicy) mp;
tmp.setMaxMergeAtOnce(Math.min(5, tmp.getMaxMergeAtOnce()));
tmp.setSegmentsPerTier(Math.min(5, tmp.getSegmentsPerTier()));
- tmp.setUseCompoundFile(true);
+ tmp.setNoCFSRatio(1.0);
}
MergeScheduler ms = w.getConfig().getMergeScheduler();
if (ms instanceof ConcurrentMergeScheduler) {
- ((ConcurrentMergeScheduler) ms).setMaxThreadCount(2);
- ((ConcurrentMergeScheduler) ms).setMaxMergeCount(3);
+ // wtf... shouldnt it be even lower since its 1 by default?!?!
+ ((ConcurrentMergeScheduler) ms).setMaxMergesAndThreads(3, 2);
}
}
@@ -879,6 +878,7 @@ public class _TestUtil {
final Field field1 = (Field) f;
final Field field2;
final DocValuesType dvType = field1.fieldType().docValueType();
+ final NumericType numType = field1.fieldType().numericType();
if (dvType != null) {
switch(dvType) {
case NUMERIC:
@@ -893,6 +893,23 @@ public class _TestUtil {
default:
throw new IllegalStateException("unknown Type: " + dvType);
}
+ } else if (numType != null) {
+ switch (numType) {
+ case INT:
+ field2 = new IntField(field1.name(), field1.numericValue().intValue(), field1.fieldType());
+ break;
+ case FLOAT:
+ field2 = new FloatField(field1.name(), field1.numericValue().intValue(), field1.fieldType());
+ break;
+ case LONG:
+ field2 = new LongField(field1.name(), field1.numericValue().intValue(), field1.fieldType());
+ break;
+ case DOUBLE:
+ field2 = new DoubleField(field1.name(), field1.numericValue().intValue(), field1.fieldType());
+ break;
+ default:
+ throw new IllegalStateException("unknown Type: " + numType);
+ }
} else {
field2 = new Field(field1.name(), field1.stringValue(), field1.fieldType());
}
Modified: lucene/dev/branches/security/lucene/tools/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/tools/build.xml?rev=1499601&r1=1499600&r2=1499601&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/tools/build.xml (original)
+++ lucene/dev/branches/security/lucene/tools/build.xml Wed Jul 3 23:26:32 2013
@@ -21,6 +21,8 @@
<description>Lucene Tools</description>
<property name="build.dir" location="../build/tools"/>
+
+ <property name="rat.additional-includes" value="forbiddenApis/**,prettify/**"/>
<import file="../common-build.xml"/>
Modified: lucene/dev/branches/security/lucene/tools/custom-tasks.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/tools/custom-tasks.xml?rev=1499601&r1=1499600&r2=1499601&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/tools/custom-tasks.xml (original)
+++ lucene/dev/branches/security/lucene/tools/custom-tasks.xml Wed Jul 3 23:26:32 2013
@@ -1,4 +1,21 @@
-
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
<project name="custom-tasks">
<description>
This file is designed for importing into a main build file, and not intended
@@ -26,7 +43,7 @@
<!-- Typical version patterns. -->
<replaceregex pattern="\.rc[0-9]+" replace="" flags="gi" />
- <replaceregex pattern="\-(r)?([0-9\-\_\.])+(b(eta)?([0-9\-\.])*)?$" replace="" flags="gi" />
+ <replaceregex pattern="\-(r)?([0-9\-\_\.])+((b(eta)?)|((a(lpha)?))([0-9\-\.])*)?(\-tests)?$" replace="" flags="gi" />
<!-- git hashcode pattern: its always 40 chars right? -->
<replaceregex pattern="\-[a-z0-9]{40,40}$" replace="" flags="gi" />
Modified: lucene/dev/branches/security/lucene/tools/junit4/tests.policy
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/tools/junit4/tests.policy?rev=1499601&r1=1499600&r2=1499601&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/tools/junit4/tests.policy (original)
+++ lucene/dev/branches/security/lucene/tools/junit4/tests.policy Wed Jul 3 23:26:32 2013
@@ -59,6 +59,10 @@ grant {
permission javax.management.MBeanPermission "*", "*";
permission javax.management.MBeanServerPermission "*";
permission javax.management.MBeanTrustPermission "*";
+ permission javax.security.auth.AuthPermission "*";
+ permission javax.security.auth.PrivateCredentialPermission "org.apache.hadoop.security.Credentials * \"*\"", "read";
+ permission java.security.SecurityPermission "putProviderProperty.SaslPlainServer";
+ permission java.security.SecurityPermission "insertProvider.SaslPlainServer";
// TIKA uses BouncyCastle and that registers new provider for PDF parsing + MSOffice parsing. Maybe report as bug!
permission java.security.SecurityPermission "putProviderProperty.BC";
Modified: lucene/dev/branches/security/lucene/tools/prettify/lang-apollo.js
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/tools/prettify/lang-apollo.js?rev=1499601&r1=1499600&r2=1499601&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/tools/prettify/lang-apollo.js (original)
+++ lucene/dev/branches/security/lucene/tools/prettify/lang-apollo.js Wed Jul 3 23:26:32 2013
@@ -1,2 +1,18 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
PR.registerLangHandler(PR.createSimpleLexer([["com",/^#[^\n\r]*/,null,"#"],["pln",/^[\t\n\r \xa0]+/,null,"\t\n\r Â\xa0"],["str",/^"(?:[^"\\]|\\[\S\s])*(?:"|$)/,null,'"']],[["kwd",/^(?:ADS|AD|AUG|BZF|BZMF|CAE|CAF|CA|CCS|COM|CS|DAS|DCA|DCOM|DCS|DDOUBL|DIM|DOUBLE|DTCB|DTCF|DV|DXCH|EDRUPT|EXTEND|INCR|INDEX|NDX|INHINT|LXCH|MASK|MSK|MP|MSU|NOOP|OVSK|QXCH|RAND|READ|RELINT|RESUME|RETURN|ROR|RXOR|SQUARE|SU|TCR|TCAA|OVSK|TCF|TC|TS|WAND|WOR|WRITE|XCH|XLQ|XXALQ|ZL|ZQ|ADD|ADZ|SUB|SUZ|MPY|MPR|MPZ|DVP|COM|ABS|CLA|CLZ|LDQ|STO|STQ|ALS|LLS|LRS|TRA|TSQ|TMI|TOV|AXT|TIX|DLY|INP|OUT)\s/,
null],["typ",/^(?:-?GENADR|=MINUS|2BCADR|VN|BOF|MM|-?2CADR|-?[1-6]DNADR|ADRES|BBCON|[ES]?BANK=?|BLOCK|BNKSUM|E?CADR|COUNT\*?|2?DEC\*?|-?DNCHAN|-?DNPTR|EQUALS|ERASE|MEMORY|2?OCT|REMADR|SETLOC|SUBRO|ORG|BSS|BES|SYN|EQU|DEFINE|END)\s/,null],["lit",/^'(?:-*(?:\w|\\[!-~])(?:[\w-]*|\\[!-~])[!=?]?)?/],["pln",/^-*(?:[!-z]|\\[!-~])(?:[\w-]*|\\[!-~])[!=?]?/],["pun",/^[^\w\t\n\r "'-);\\\xa0]+/]]),["apollo","agc","aea"]);
Modified: lucene/dev/branches/security/lucene/tools/prettify/lang-css.js
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/tools/prettify/lang-css.js?rev=1499601&r1=1499600&r2=1499601&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/tools/prettify/lang-css.js (original)
+++ lucene/dev/branches/security/lucene/tools/prettify/lang-css.js Wed Jul 3 23:26:32 2013
@@ -1,2 +1,18 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
PR.registerLangHandler(PR.createSimpleLexer([["pln",/^[\t\n\f\r ]+/,null," \t\r\n "]],[["str",/^"(?:[^\n\f\r"\\]|\\(?:\r\n?|\n|\f)|\\[\S\s])*"/,null],["str",/^'(?:[^\n\f\r'\\]|\\(?:\r\n?|\n|\f)|\\[\S\s])*'/,null],["lang-css-str",/^url\(([^"')]*)\)/i],["kwd",/^(?:url|rgb|!important|@import|@page|@media|@charset|inherit)(?=[^\w-]|$)/i,null],["lang-css-kw",/^(-?(?:[_a-z]|\\[\da-f]+ ?)(?:[\w-]|\\\\[\da-f]+ ?)*)\s*:/i],["com",/^\/\*[^*]*\*+(?:[^*/][^*]*\*+)*\//],["com",
/^(?:<\!--|--\>)/],["lit",/^(?:\d+|\d*\.\d+)(?:%|[a-z]+)?/i],["lit",/^#[\da-f]{3,6}/i],["pln",/^-?(?:[_a-z]|\\[\da-f]+ ?)(?:[\w-]|\\\\[\da-f]+ ?)*/i],["pun",/^[^\s\w"']+/]]),["css"]);PR.registerLangHandler(PR.createSimpleLexer([],[["kwd",/^-?(?:[_a-z]|\\[\da-f]+ ?)(?:[\w-]|\\\\[\da-f]+ ?)*/i]]),["css-kw"]);PR.registerLangHandler(PR.createSimpleLexer([],[["str",/^[^"')]+/]]),["css-str"]);