You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2015/10/19 15:38:44 UTC
svn commit: r1709414 [4/5] - in /lucene/dev/branches/lucene6835: ./ lucene/
lucene/analysis/ lucene/analysis/common/
lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/
lucene/analysis/common/src/test/org/apache/lucene/analysis/core/ l...
Modified: lucene/dev/branches/lucene6835/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java?rev=1709414&r1=1709413&r2=1709414&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java (original)
+++ lucene/dev/branches/lucene6835/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java Mon Oct 19 13:38:42 2015
@@ -34,8 +34,8 @@ import java.util.Set;
import java.util.TreeSet;
import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.CannedBinaryTokenStream.BinaryToken;
import org.apache.lucene.analysis.CannedBinaryTokenStream;
+import org.apache.lucene.analysis.CannedBinaryTokenStream.BinaryToken;
import org.apache.lucene.analysis.CannedTokenStream;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.analysis.MockTokenFilter;
@@ -48,9 +48,11 @@ import org.apache.lucene.analysis.Tokeni
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.document.Document;
-import org.apache.lucene.search.suggest.Lookup.LookupResult;
import org.apache.lucene.search.suggest.Input;
import org.apache.lucene.search.suggest.InputArrayIterator;
+import org.apache.lucene.search.suggest.Lookup.LookupResult;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LineFileDocs;
@@ -58,7 +60,7 @@ import org.apache.lucene.util.LuceneTest
import org.apache.lucene.util.TestUtil;
public class AnalyzingSuggesterTest extends LuceneTestCase {
-
+
/** this is basically the WFST test ported to KeywordAnalyzer. so it acts the same */
public void testKeyword() throws Exception {
Iterable<Input> keys = shuffle(
@@ -71,8 +73,10 @@ public class AnalyzingSuggesterTest exte
new Input("barbara", 1)
);
+ Directory tempDir = getDirectory();
+
Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.KEYWORD, false);
- AnalyzingSuggester suggester = new AnalyzingSuggester(analyzer);
+ AnalyzingSuggester suggester = new AnalyzingSuggester(tempDir, "suggest", analyzer);
suggester.build(new InputArrayIterator(keys));
// top N of 2, but only foo is available
@@ -106,7 +110,7 @@ public class AnalyzingSuggesterTest exte
assertEquals("barbara", results.get(2).key.toString());
assertEquals(6, results.get(2).value, 0.01F);
- analyzer.close();
+ IOUtils.close(analyzer, tempDir);
}
public void testKeywordWithPayloads() throws Exception {
@@ -119,7 +123,8 @@ public class AnalyzingSuggesterTest exte
new Input("barbara", 6, new BytesRef("for all the fish")));
Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.KEYWORD, false);
- AnalyzingSuggester suggester = new AnalyzingSuggester(analyzer);
+ Directory tempDir = getDirectory();
+ AnalyzingSuggester suggester = new AnalyzingSuggester(tempDir, "suggest", analyzer);
suggester.build(new InputArrayIterator(keys));
for (int i = 0; i < 2; i++) {
// top N of 2, but only foo is available
@@ -160,7 +165,7 @@ public class AnalyzingSuggesterTest exte
assertEquals(6, results.get(2).value, 0.01F);
assertEquals(new BytesRef("for all the fish"), results.get(2).payload);
}
- analyzer.close();
+ IOUtils.close(analyzer, tempDir);
}
public void testRandomRealisticKeys() throws IOException {
@@ -180,7 +185,9 @@ public class AnalyzingSuggesterTest exte
}
Analyzer indexAnalyzer = new MockAnalyzer(random());
Analyzer queryAnalyzer = new MockAnalyzer(random());
- AnalyzingSuggester analyzingSuggester = new AnalyzingSuggester(indexAnalyzer, queryAnalyzer,
+ Directory tempDir = getDirectory();
+
+ AnalyzingSuggester analyzingSuggester = new AnalyzingSuggester(tempDir, "suggest", indexAnalyzer, queryAnalyzer,
AnalyzingSuggester.EXACT_FIRST | AnalyzingSuggester.PRESERVE_SEP, 256, -1, random().nextBoolean());
boolean doPayloads = random().nextBoolean();
if (doPayloads) {
@@ -205,7 +212,7 @@ public class AnalyzingSuggesterTest exte
}
}
- IOUtils.close(lineFile, indexAnalyzer, queryAnalyzer);
+ IOUtils.close(lineFile, indexAnalyzer, queryAnalyzer, tempDir);
}
// TODO: more tests
@@ -217,8 +224,9 @@ public class AnalyzingSuggesterTest exte
new Input("the ghost of christmas past", 50),
};
+ Directory tempDir = getDirectory();
Analyzer standard = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, true, MockTokenFilter.ENGLISH_STOPSET);
- AnalyzingSuggester suggester = new AnalyzingSuggester(standard, standard,
+ AnalyzingSuggester suggester = new AnalyzingSuggester(tempDir, "suggest", standard, standard,
AnalyzingSuggester.EXACT_FIRST | AnalyzingSuggester.PRESERVE_SEP, 256, -1, false);
suggester.build(new InputArrayIterator(keys));
@@ -240,17 +248,18 @@ public class AnalyzingSuggesterTest exte
assertEquals("the ghost of christmas past", results.get(0).key.toString());
assertEquals(50, results.get(0).value, 0.01F);
- standard.close();
+ IOUtils.close(standard, tempDir);
}
public void testEmpty() throws Exception {
Analyzer standard = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, true, MockTokenFilter.ENGLISH_STOPSET);
- AnalyzingSuggester suggester = new AnalyzingSuggester(standard);
+ Directory tempDir = getDirectory();
+ AnalyzingSuggester suggester = new AnalyzingSuggester(tempDir, "suggest", standard);
suggester.build(new InputArrayIterator(new Input[0]));
List<LookupResult> result = suggester.lookup("a", false, 20);
assertTrue(result.isEmpty());
- standard.close();
+ IOUtils.close(standard, tempDir);
}
public void testNoSeps() throws Exception {
@@ -262,7 +271,8 @@ public class AnalyzingSuggesterTest exte
int options = 0;
Analyzer a = new MockAnalyzer(random());
- AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, options, 256, -1, true);
+ Directory tempDir = getDirectory();
+ AnalyzingSuggester suggester = new AnalyzingSuggester(tempDir, "suggest", a, a, options, 256, -1, true);
suggester.build(new InputArrayIterator(keys));
// TODO: would be nice if "ab " would allow the test to
// pass, and more generally if the analyzer can know
@@ -275,7 +285,7 @@ public class AnalyzingSuggesterTest exte
// complete to "abcd", which has higher weight so should
// appear first:
assertEquals("abcd", r.get(0).key.toString());
- a.close();
+ IOUtils.close(a, tempDir);
}
public void testGraphDups() throws Exception {
@@ -330,7 +340,8 @@ public class AnalyzingSuggesterTest exte
new Input("wi fi network is fast", 10),
};
//AnalyzingSuggester suggester = new AnalyzingSuggester(analyzer, AnalyzingSuggester.EXACT_FIRST, 256, -1);
- AnalyzingSuggester suggester = new AnalyzingSuggester(analyzer);
+ Directory tempDir = getDirectory();
+ AnalyzingSuggester suggester = new AnalyzingSuggester(tempDir, "suggest", analyzer);
suggester.build(new InputArrayIterator(keys));
List<LookupResult> results = suggester.lookup("wifi network", false, 10);
if (VERBOSE) {
@@ -341,7 +352,7 @@ public class AnalyzingSuggesterTest exte
assertEquals(50, results.get(0).value);
assertEquals("wi fi network is fast", results.get(1).key);
assertEquals(10, results.get(1).value);
- analyzer.close();
+ IOUtils.close(analyzer, tempDir);
}
public void testInputPathRequired() throws Exception {
@@ -396,11 +407,12 @@ public class AnalyzingSuggesterTest exte
new Input("ab xc", 50),
new Input("ba xd", 50),
};
- AnalyzingSuggester suggester = new AnalyzingSuggester(analyzer);
+ Directory tempDir = getDirectory();
+ AnalyzingSuggester suggester = new AnalyzingSuggester(tempDir, "suggest", analyzer);
suggester.build(new InputArrayIterator(keys));
List<LookupResult> results = suggester.lookup("ab x", false, 1);
assertTrue(results.size() == 1);
- analyzer.close();
+ IOUtils.close(analyzer, tempDir);
}
private static Token token(String term, int posInc, int posLength) {
@@ -471,7 +483,8 @@ public class AnalyzingSuggesterTest exte
Analyzer a = getUnusualAnalyzer();
int options = AnalyzingSuggester.EXACT_FIRST | AnalyzingSuggester.PRESERVE_SEP;
- AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, options, 256, -1, true);
+ Directory tempDir = getDirectory();
+ AnalyzingSuggester suggester = new AnalyzingSuggester(tempDir, "suggest", a, a, options, 256, -1, true);
suggester.build(new InputArrayIterator(new Input[] {
new Input("x y", 1),
new Input("x y z", 3),
@@ -505,13 +518,14 @@ public class AnalyzingSuggesterTest exte
}
}
}
- a.close();
+ IOUtils.close(a, tempDir);
}
public void testNonExactFirst() throws Exception {
Analyzer a = getUnusualAnalyzer();
- AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, AnalyzingSuggester.PRESERVE_SEP, 256, -1, true);
+ Directory tempDir = getDirectory();
+ AnalyzingSuggester suggester = new AnalyzingSuggester(tempDir, "suggest", a, a, AnalyzingSuggester.PRESERVE_SEP, 256, -1, true);
suggester.build(new InputArrayIterator(new Input[] {
new Input("x y", 1),
@@ -543,7 +557,7 @@ public class AnalyzingSuggesterTest exte
}
}
}
- a.close();
+ IOUtils.close(a, tempDir);
}
// Holds surface form separately:
@@ -764,7 +778,8 @@ public class AnalyzingSuggesterTest exte
}
Analyzer a = new MockTokenEatingAnalyzer(numStopChars, preserveHoles);
- AnalyzingSuggester suggester = new AnalyzingSuggester(a, a,
+ Directory tempDir = getDirectory();
+ AnalyzingSuggester suggester = new AnalyzingSuggester(tempDir, "suggest", a, a,
preserveSep ? AnalyzingSuggester.PRESERVE_SEP : 0, 256, -1, true);
if (doPayloads) {
suggester.build(new InputArrayIterator(shuffle(payloadKeys)));
@@ -882,12 +897,13 @@ public class AnalyzingSuggesterTest exte
}
}
}
- a.close();
+ IOUtils.close(a, tempDir);
}
public void testMaxSurfaceFormsPerAnalyzedForm() throws Exception {
Analyzer a = new MockAnalyzer(random());
- AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, 0, 2, -1, true);
+ Directory tempDir = getDirectory();
+ AnalyzingSuggester suggester = new AnalyzingSuggester(tempDir, "suggest", a, a, 0, 2, -1, true);
suggester.build(new InputArrayIterator(shuffle(new Input("a", 40),
new Input("a ", 50), new Input(" a", 60))));
@@ -897,12 +913,13 @@ public class AnalyzingSuggesterTest exte
assertEquals(60, results.get(0).value);
assertEquals("a ", results.get(1).key);
assertEquals(50, results.get(1).value);
- a.close();
+ IOUtils.close(a, tempDir);
}
public void testQueueExhaustion() throws Exception {
Analyzer a = new MockAnalyzer(random());
- AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, AnalyzingSuggester.EXACT_FIRST, 256, -1, true);
+ Directory tempDir = getDirectory();
+ AnalyzingSuggester suggester = new AnalyzingSuggester(tempDir, "suggest", a, a, AnalyzingSuggester.EXACT_FIRST, 256, -1, true);
suggester.build(new InputArrayIterator(new Input[] {
new Input("a", 2),
@@ -912,14 +929,15 @@ public class AnalyzingSuggesterTest exte
}));
suggester.lookup("a", false, 4);
- a.close();
+ IOUtils.close(a, tempDir);
}
public void testExactFirstMissingResult() throws Exception {
Analyzer a = new MockAnalyzer(random());
- AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, AnalyzingSuggester.EXACT_FIRST, 256, -1, true);
+ Directory tempDir = getDirectory();
+ AnalyzingSuggester suggester = new AnalyzingSuggester(tempDir, "suggest", a, a, AnalyzingSuggester.EXACT_FIRST, 256, -1, true);
suggester.build(new InputArrayIterator(new Input[] {
new Input("a", 5),
@@ -959,7 +977,7 @@ public class AnalyzingSuggesterTest exte
assertEquals(4, results.get(1).value);
assertEquals("a b", results.get(2).key);
assertEquals(3, results.get(2).value);
- a.close();
+ IOUtils.close(a, tempDir);
}
public void testDupSurfaceFormsMissingResults() throws Exception {
@@ -986,7 +1004,8 @@ public class AnalyzingSuggesterTest exte
}
};
- AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, 0, 256, -1, true);
+ Directory tempDir = getDirectory();
+ AnalyzingSuggester suggester = new AnalyzingSuggester(tempDir, "suggest", a, a, 0, 256, -1, true);
suggester.build(new InputArrayIterator(shuffle(
new Input("hambone", 6),
@@ -1018,7 +1037,7 @@ public class AnalyzingSuggesterTest exte
assertEquals(6, results.get(0).value);
assertEquals("nellie", results.get(1).key);
assertEquals(5, results.get(1).value);
- a.close();
+ IOUtils.close(a, tempDir);
}
public void testDupSurfaceFormsMissingResults2() throws Exception {
@@ -1055,7 +1074,8 @@ public class AnalyzingSuggesterTest exte
}
};
- AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, 0, 256, -1, true);
+ Directory tempDir = getDirectory();
+ AnalyzingSuggester suggester = new AnalyzingSuggester(tempDir, "suggest", a, a, 0, 256, -1, true);
suggester.build(new InputArrayIterator(new Input[] {
new Input("a", 6),
@@ -1088,7 +1108,7 @@ public class AnalyzingSuggesterTest exte
assertEquals(6, results.get(0).value);
assertEquals("b", results.get(1).key);
assertEquals(5, results.get(1).value);
- a.close();
+ IOUtils.close(a, tempDir);
}
public void test0ByteKeys() throws Exception {
@@ -1128,19 +1148,21 @@ public class AnalyzingSuggesterTest exte
}
};
- AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, 0, 256, -1, true);
+ Directory tempDir = getDirectory();
+ AnalyzingSuggester suggester = new AnalyzingSuggester(tempDir, "suggest", a, a, 0, 256, -1, true);
suggester.build(new InputArrayIterator(new Input[] {
new Input("a a", 50),
new Input("a b", 50),
}));
- a.close();
+ IOUtils.close(a, tempDir);
}
public void testDupSurfaceFormsMissingResults3() throws Exception {
Analyzer a = new MockAnalyzer(random());
- AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, AnalyzingSuggester.PRESERVE_SEP, 256, -1, true);
+ Directory tempDir = getDirectory();
+ AnalyzingSuggester suggester = new AnalyzingSuggester(tempDir, "suggest", a, a, AnalyzingSuggester.PRESERVE_SEP, 256, -1, true);
suggester.build(new InputArrayIterator(new Input[] {
new Input("a a", 7),
new Input("a a", 7),
@@ -1149,19 +1171,20 @@ public class AnalyzingSuggesterTest exte
new Input("a b", 5),
}));
assertEquals("[a a/7, a c/6, a b/5]", suggester.lookup("a", false, 3).toString());
- a.close();
+ IOUtils.close(tempDir, a);
}
public void testEndingSpace() throws Exception {
Analyzer a = new MockAnalyzer(random());
- AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, AnalyzingSuggester.PRESERVE_SEP, 256, -1, true);
+ Directory tempDir = getDirectory();
+ AnalyzingSuggester suggester = new AnalyzingSuggester(tempDir, "suggest", a, a, AnalyzingSuggester.PRESERVE_SEP, 256, -1, true);
suggester.build(new InputArrayIterator(new Input[] {
new Input("i love lucy", 7),
new Input("isla de muerta", 8),
}));
assertEquals("[isla de muerta/8, i love lucy/7]", suggester.lookup("i", false, 3).toString());
assertEquals("[i love lucy/7]", suggester.lookup("i ", false, 3).toString());
- a.close();
+ IOUtils.close(a, tempDir);
}
public void testTooManyExpansions() throws Exception {
@@ -1188,15 +1211,17 @@ public class AnalyzingSuggesterTest exte
}
};
- AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, 0, 256, 1, true);
+ Directory tempDir = getDirectory();
+ AnalyzingSuggester suggester = new AnalyzingSuggester(tempDir, "suggest", a, a, 0, 256, 1, true);
suggester.build(new InputArrayIterator(new Input[] {new Input("a", 1)}));
assertEquals("[a/1]", suggester.lookup("a", false, 1).toString());
- a.close();
+ IOUtils.close(a, tempDir);
}
public void testIllegalLookupArgument() throws Exception {
Analyzer a = new MockAnalyzer(random());
- AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, 0, 256, -1, true);
+ Directory tempDir = getDirectory();
+ AnalyzingSuggester suggester = new AnalyzingSuggester(tempDir, "suggest", a, a, 0, 256, -1, true);
suggester.build(new InputArrayIterator(new Input[] {
new Input("а где ÐÑÑи?", 7),
}));
@@ -1212,7 +1237,7 @@ public class AnalyzingSuggesterTest exte
} catch (IllegalArgumentException e) {
// expected
}
- a.close();
+ IOUtils.close(a, tempDir);
}
static final Iterable<Input> shuffle(Input...values) {
@@ -1227,7 +1252,8 @@ public class AnalyzingSuggesterTest exte
// TODO: we need BaseSuggesterTestCase?
public void testTooLongSuggestion() throws Exception {
Analyzer a = new MockAnalyzer(random());
- AnalyzingSuggester suggester = new AnalyzingSuggester(a);
+ Directory tempDir = getDirectory();
+ AnalyzingSuggester suggester = new AnalyzingSuggester(tempDir, "suggest", a);
String bigString = TestUtil.randomSimpleString(random(), 30000, 30000);
try {
suggester.build(new InputArrayIterator(new Input[] {
@@ -1238,6 +1264,14 @@ public class AnalyzingSuggesterTest exte
} catch (IllegalArgumentException iae) {
// expected
}
- a.close();
+ IOUtils.close(a, tempDir);
+ }
+
+ private Directory getDirectory() {
+ Directory dir = newDirectory();
+ if (dir instanceof MockDirectoryWrapper) {
+ ((MockDirectoryWrapper) dir).setEnableVirusScanner(false);
+ }
+ return dir;
}
}
Modified: lucene/dev/branches/lucene6835/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/FuzzySuggesterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/FuzzySuggesterTest.java?rev=1709414&r1=1709413&r2=1709414&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/FuzzySuggesterTest.java (original)
+++ lucene/dev/branches/lucene6835/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/FuzzySuggesterTest.java Mon Oct 19 13:38:42 2015
@@ -43,8 +43,11 @@ import org.apache.lucene.analysis.tokena
import org.apache.lucene.search.suggest.Input;
import org.apache.lucene.search.suggest.InputArrayIterator;
import org.apache.lucene.search.suggest.Lookup.LookupResult;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefBuilder;
+import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.IntsRef;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
@@ -62,7 +65,8 @@ public class FuzzySuggesterTest extends
}
keys.add(new Input("foo bar boo far", 12));
MockAnalyzer analyzer = new MockAnalyzer(random(), MockTokenizer.KEYWORD, false);
- FuzzySuggester suggester = new FuzzySuggester(analyzer, analyzer, FuzzySuggester.EXACT_FIRST | FuzzySuggester.PRESERVE_SEP, 256, -1, true, FuzzySuggester.DEFAULT_MAX_EDITS, FuzzySuggester.DEFAULT_TRANSPOSITIONS,
+ Directory tempDir = getDirectory();
+ FuzzySuggester suggester = new FuzzySuggester(tempDir, "fuzzy", analyzer, analyzer, FuzzySuggester.EXACT_FIRST | FuzzySuggester.PRESERVE_SEP, 256, -1, true, FuzzySuggester.DEFAULT_MAX_EDITS, FuzzySuggester.DEFAULT_TRANSPOSITIONS,
0, FuzzySuggester.DEFAULT_MIN_FUZZY_LENGTH, FuzzySuggester.DEFAULT_UNICODE_AWARE);
suggester.build(new InputArrayIterator(keys));
int numIters = atLeast(10);
@@ -73,7 +77,7 @@ public class FuzzySuggesterTest extends
assertEquals("foo bar boo far", results.get(0).key.toString());
assertEquals(12, results.get(0).value, 0.01F);
}
- analyzer.close();
+ IOUtils.close(analyzer, tempDir);
}
public void testNonLatinRandomEdits() throws IOException {
@@ -84,7 +88,8 @@ public class FuzzySuggesterTest extends
}
keys.add(new Input("ÑÑÑ Ð±Ð°Ñ Ð±ÑÑ ÑаÑ", 12));
MockAnalyzer analyzer = new MockAnalyzer(random(), MockTokenizer.KEYWORD, false);
- FuzzySuggester suggester = new FuzzySuggester(analyzer, analyzer, FuzzySuggester.EXACT_FIRST | FuzzySuggester.PRESERVE_SEP, 256, -1, true, FuzzySuggester.DEFAULT_MAX_EDITS, FuzzySuggester.DEFAULT_TRANSPOSITIONS,
+ Directory tempDir = getDirectory();
+ FuzzySuggester suggester = new FuzzySuggester(tempDir, "fuzzy",analyzer, analyzer, FuzzySuggester.EXACT_FIRST | FuzzySuggester.PRESERVE_SEP, 256, -1, true, FuzzySuggester.DEFAULT_MAX_EDITS, FuzzySuggester.DEFAULT_TRANSPOSITIONS,
0, FuzzySuggester.DEFAULT_MIN_FUZZY_LENGTH, true);
suggester.build(new InputArrayIterator(keys));
int numIters = atLeast(10);
@@ -95,7 +100,7 @@ public class FuzzySuggesterTest extends
assertEquals("ÑÑÑ Ð±Ð°Ñ Ð±ÑÑ ÑаÑ", results.get(0).key.toString());
assertEquals(12, results.get(0).value, 0.01F);
}
- analyzer.close();
+ IOUtils.close(analyzer, tempDir);
}
/** this is basically the WFST test ported to KeywordAnalyzer. so it acts the same */
@@ -108,7 +113,8 @@ public class FuzzySuggesterTest extends
};
Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.KEYWORD, false);
- FuzzySuggester suggester = new FuzzySuggester(analyzer);
+ Directory tempDir = getDirectory();
+ FuzzySuggester suggester = new FuzzySuggester(tempDir, "fuzzy",analyzer);
suggester.build(new InputArrayIterator(keys));
List<LookupResult> results = suggester.lookup(TestUtil.stringToCharSequence("bariar", random()), false, 2);
@@ -171,7 +177,7 @@ public class FuzzySuggesterTest extends
assertEquals("barbara", results.get(2).key.toString());
assertEquals(6, results.get(2).value, 0.01F);
- analyzer.close();
+ IOUtils.close(analyzer, tempDir);
}
/**
@@ -183,7 +189,8 @@ public class FuzzySuggesterTest extends
};
Analyzer standard = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, true, MockTokenFilter.ENGLISH_STOPSET);
- FuzzySuggester suggester = new FuzzySuggester(standard, standard, AnalyzingSuggester.EXACT_FIRST | AnalyzingSuggester.PRESERVE_SEP, 256, -1, false, FuzzySuggester.DEFAULT_MAX_EDITS, FuzzySuggester.DEFAULT_TRANSPOSITIONS,
+ Directory tempDir = getDirectory();
+ FuzzySuggester suggester = new FuzzySuggester(tempDir, "fuzzy", standard, standard, AnalyzingSuggester.EXACT_FIRST | AnalyzingSuggester.PRESERVE_SEP, 256, -1, false, FuzzySuggester.DEFAULT_MAX_EDITS, FuzzySuggester.DEFAULT_TRANSPOSITIONS,
FuzzySuggester.DEFAULT_NON_FUZZY_PREFIX, FuzzySuggester.DEFAULT_MIN_FUZZY_LENGTH, FuzzySuggester.DEFAULT_UNICODE_AWARE);
suggester.build(new InputArrayIterator(keys));
@@ -204,7 +211,7 @@ public class FuzzySuggesterTest extends
assertEquals("the ghost of christmas past", results.get(0).key.toString());
assertEquals(50, results.get(0).value, 0.01F);
- standard.close();
+ IOUtils.close(standard, tempDir);
}
public void testNoSeps() throws Exception {
@@ -216,7 +223,8 @@ public class FuzzySuggesterTest extends
int options = 0;
Analyzer a = new MockAnalyzer(random());
- FuzzySuggester suggester = new FuzzySuggester(a, a, options, 256, -1, true, 1, true, 1, 3, false);
+ Directory tempDir = getDirectory();
+ FuzzySuggester suggester = new FuzzySuggester(tempDir, "fuzzy",a, a, options, 256, -1, true, 1, true, 1, 3, false);
suggester.build(new InputArrayIterator(keys));
// TODO: would be nice if "ab " would allow the test to
// pass, and more generally if the analyzer can know
@@ -229,7 +237,7 @@ public class FuzzySuggesterTest extends
// complete to "abcd", which has higher weight so should
// appear first:
assertEquals("abcd", r.get(0).key.toString());
- a.close();
+ IOUtils.close(a, tempDir);
}
public void testGraphDups() throws Exception {
@@ -283,7 +291,8 @@ public class FuzzySuggesterTest extends
new Input("wifi network is slow", 50),
new Input("wi fi network is fast", 10),
};
- FuzzySuggester suggester = new FuzzySuggester(analyzer);
+ Directory tempDir = getDirectory();
+ FuzzySuggester suggester = new FuzzySuggester(tempDir, "fuzzy", analyzer);
suggester.build(new InputArrayIterator(keys));
List<LookupResult> results = suggester.lookup("wifi network", false, 10);
@@ -295,17 +304,18 @@ public class FuzzySuggesterTest extends
assertEquals(50, results.get(0).value);
assertEquals("wi fi network is fast", results.get(1).key);
assertEquals(10, results.get(1).value);
- analyzer.close();
+ IOUtils.close(tempDir, analyzer);
}
public void testEmpty() throws Exception {
Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.KEYWORD, false);
- FuzzySuggester suggester = new FuzzySuggester(analyzer);
+ Directory tempDir = getDirectory();
+ FuzzySuggester suggester = new FuzzySuggester(tempDir, "fuzzy", analyzer);
suggester.build(new InputArrayIterator(new Input[0]));
List<LookupResult> result = suggester.lookup("a", false, 20);
assertTrue(result.isEmpty());
- analyzer.close();
+ IOUtils.close(analyzer, tempDir);
}
public void testInputPathRequired() throws Exception {
@@ -360,11 +370,12 @@ public class FuzzySuggesterTest extends
new Input("ab xc", 50),
new Input("ba xd", 50),
};
- FuzzySuggester suggester = new FuzzySuggester(analyzer);
+ Directory tempDir = getDirectory();
+ FuzzySuggester suggester = new FuzzySuggester(tempDir, "fuzzy", analyzer);
suggester.build(new InputArrayIterator(keys));
List<LookupResult> results = suggester.lookup("ab x", false, 1);
assertTrue(results.size() == 1);
- analyzer.close();
+ IOUtils.close(analyzer, tempDir);
}
private static Token token(String term, int posInc, int posLength) {
@@ -430,7 +441,8 @@ public class FuzzySuggesterTest extends
public void testExactFirst() throws Exception {
Analyzer a = getUnusualAnalyzer();
- FuzzySuggester suggester = new FuzzySuggester(a, a, AnalyzingSuggester.EXACT_FIRST | AnalyzingSuggester.PRESERVE_SEP, 256, -1, true, 1, true, 1, 3, false);
+ Directory tempDir = getDirectory();
+ FuzzySuggester suggester = new FuzzySuggester(tempDir, "fuzzy", a, a, AnalyzingSuggester.EXACT_FIRST | AnalyzingSuggester.PRESERVE_SEP, 256, -1, true, 1, true, 1, 3, false);
suggester.build(new InputArrayIterator(new Input[] {
new Input("x y", 1),
new Input("x y z", 3),
@@ -464,13 +476,14 @@ public class FuzzySuggesterTest extends
}
}
}
- a.close();
+ IOUtils.close(a, tempDir);
}
public void testNonExactFirst() throws Exception {
Analyzer a = getUnusualAnalyzer();
- FuzzySuggester suggester = new FuzzySuggester(a, a, AnalyzingSuggester.PRESERVE_SEP, 256, -1, true, 1, true, 1, 3, false);
+ Directory tempDir = getDirectory();
+ FuzzySuggester suggester = new FuzzySuggester(tempDir, "fuzzy", a, a, AnalyzingSuggester.PRESERVE_SEP, 256, -1, true, 1, true, 1, 3, false);
suggester.build(new InputArrayIterator(new Input[] {
new Input("x y", 1),
@@ -502,7 +515,7 @@ public class FuzzySuggesterTest extends
}
}
}
- a.close();
+ IOUtils.close(a, tempDir);
}
// Holds surface form separately:
@@ -697,7 +710,8 @@ public class FuzzySuggesterTest extends
}
Analyzer a = new MockTokenEatingAnalyzer(numStopChars, preserveHoles);
- FuzzySuggester suggester = new FuzzySuggester(a, a,
+ Directory tempDir = getDirectory();
+ FuzzySuggester suggester = new FuzzySuggester(tempDir, "fuzzy",a, a,
preserveSep ? AnalyzingSuggester.PRESERVE_SEP : 0, 256, -1, true, 1, false, 1, 3, unicodeAware);
suggester.build(new InputArrayIterator(keys));
@@ -836,12 +850,13 @@ public class FuzzySuggesterTest extends
assertEquals(matches.get(hit).value, r.get(hit).value, 0f);
}
}
- a.close();
+ IOUtils.close(a, tempDir);
}
public void testMaxSurfaceFormsPerAnalyzedForm() throws Exception {
Analyzer a = new MockAnalyzer(random());
- FuzzySuggester suggester = new FuzzySuggester(a, a, 0, 2, -1, true, 1, true, 1, 3, false);
+ Directory tempDir = getDirectory();
+ FuzzySuggester suggester = new FuzzySuggester(tempDir, "fuzzy", a, a, 0, 2, -1, true, 1, true, 1, 3, false);
List<Input> keys = Arrays.asList(new Input[] {
new Input("a", 40),
@@ -858,12 +873,13 @@ public class FuzzySuggesterTest extends
assertEquals(60, results.get(0).value);
assertEquals("a ", results.get(1).key);
assertEquals(50, results.get(1).value);
- a.close();
+ IOUtils.close(a, tempDir);
}
public void testEditSeps() throws Exception {
Analyzer a = new MockAnalyzer(random());
- FuzzySuggester suggester = new FuzzySuggester(a, a, FuzzySuggester.PRESERVE_SEP, 2, -1, true, 2, true, 1, 3, false);
+ Directory tempDir = getDirectory();
+ FuzzySuggester suggester = new FuzzySuggester(tempDir, "fuzzy", a, a, FuzzySuggester.PRESERVE_SEP, 2, -1, true, 2, true, 1, 3, false);
List<Input> keys = Arrays.asList(new Input[] {
new Input("foo bar", 40),
@@ -879,7 +895,7 @@ public class FuzzySuggesterTest extends
assertEquals("[foo bar baz/50]", suggester.lookup("foobarbaz", false, 5).toString());
assertEquals("[barbaz/60, barbazfoo/10]", suggester.lookup("bar baz", false, 5).toString());
assertEquals("[barbazfoo/10]", suggester.lookup("bar baz foo", false, 5).toString());
- a.close();
+ IOUtils.close(a, tempDir);
}
@SuppressWarnings("fallthrough")
@@ -978,7 +994,8 @@ public class FuzzySuggesterTest extends
boolean transpositions = random().nextBoolean();
// TODO: test graph analyzers
// TODO: test exactFirst / preserveSep permutations
- FuzzySuggester suggest = new FuzzySuggester(a, a, 0, 256, -1, true, maxEdits, transpositions, prefixLen, prefixLen, false);
+ Directory tempDir = getDirectory();
+ FuzzySuggester suggest = new FuzzySuggester(tempDir, "fuzzy", a, a, 0, 256, -1, true, maxEdits, transpositions, prefixLen, prefixLen, false);
if (VERBOSE) {
System.out.println("TEST: maxEdits=" + maxEdits + " prefixLen=" + prefixLen + " transpositions=" + transpositions + " num=" + NUM);
@@ -1022,7 +1039,7 @@ public class FuzzySuggesterTest extends
}
assertEquals(expected.size(), actual.size());
}
- a.close();
+ IOUtils.close(a, tempDir);
}
private List<LookupResult> slowFuzzyMatch(int prefixLen, int maxEdits, boolean allowTransposition, List<Input> answers, String frag) {
@@ -1194,4 +1211,12 @@ public class FuzzySuggesterTest extends
}
return ref;
}
+
+ private Directory getDirectory() {
+ Directory dir = newDirectory();
+ if (dir instanceof MockDirectoryWrapper) {
+ ((MockDirectoryWrapper) dir).setEnableVirusScanner(false);
+ }
+ return dir;
+ }
}
Modified: lucene/dev/branches/lucene6835/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/BytesRefSortersTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/BytesRefSortersTest.java?rev=1709414&r1=1709413&r2=1709414&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/BytesRefSortersTest.java (original)
+++ lucene/dev/branches/lucene6835/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/BytesRefSortersTest.java Mon Oct 19 13:38:42 2015
@@ -18,8 +18,11 @@ package org.apache.lucene.search.suggest
*/
import org.apache.lucene.search.suggest.InMemorySorter;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefIterator;
+import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.OfflineSorter;
import org.junit.Test;
@@ -27,9 +30,13 @@ import org.junit.Test;
public class BytesRefSortersTest extends LuceneTestCase {
@Test
public void testExternalRefSorter() throws Exception {
- ExternalRefSorter s = new ExternalRefSorter(new OfflineSorter());
+ Directory tempDir = newDirectory();
+ if (tempDir instanceof MockDirectoryWrapper) {
+ ((MockDirectoryWrapper) tempDir).setEnableVirusScanner(false);
+ }
+ ExternalRefSorter s = new ExternalRefSorter(new OfflineSorter(tempDir, "temp"));
check(s);
- s.close();
+ IOUtils.close(s, tempDir);
}
@Test
Modified: lucene/dev/branches/lucene6835/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/FSTCompletionTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/FSTCompletionTest.java?rev=1709414&r1=1709413&r2=1709414&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/FSTCompletionTest.java (original)
+++ lucene/dev/branches/lucene6835/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/FSTCompletionTest.java Mon Oct 19 13:38:42 2015
@@ -20,15 +20,18 @@ package org.apache.lucene.search.suggest
import java.nio.charset.StandardCharsets;
import java.util.*;
-import org.apache.lucene.search.suggest.Lookup.LookupResult;
import org.apache.lucene.search.suggest.*;
+import org.apache.lucene.search.suggest.Lookup.LookupResult;
import org.apache.lucene.search.suggest.fst.FSTCompletion.Completion;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.util.*;
/**
* Unit tests for {@link FSTCompletion}.
*/
public class FSTCompletionTest extends LuceneTestCase {
+
public static Input tf(String t, int v) {
return new Input(t, v);
}
@@ -155,7 +158,8 @@ public class FSTCompletionTest extends L
}
public void testLargeInputConstantWeights() throws Exception {
- FSTCompletionLookup lookup = new FSTCompletionLookup(10, true);
+ Directory tempDir = getDirectory();
+ FSTCompletionLookup lookup = new FSTCompletionLookup(tempDir, "fst", 10, true);
Random r = random();
List<Input> keys = new ArrayList<>();
@@ -175,12 +179,14 @@ public class FSTCompletionTest extends L
}
previous = current;
}
+ tempDir.close();
}
public void testMultilingualInput() throws Exception {
List<Input> input = LookupBenchmarkTest.readTop50KWiki();
- FSTCompletionLookup lookup = new FSTCompletionLookup();
+ Directory tempDir = getDirectory();
+ FSTCompletionLookup lookup = new FSTCompletionLookup(tempDir, "fst");
lookup.build(new InputArrayIterator(input));
assertEquals(input.size(), lookup.getCount());
for (Input tf : input) {
@@ -192,6 +198,7 @@ public class FSTCompletionTest extends L
assertEquals(5, result.size());
assertTrue(result.get(0).key.toString().equals("wit")); // exact match.
assertTrue(result.get(1).key.toString().equals("with")); // highest count.
+ tempDir.close();
}
public void testEmptyInput() throws Exception {
@@ -207,7 +214,8 @@ public class FSTCompletionTest extends L
freqs.add(new Input("" + rnd.nextLong(), weight));
}
- FSTCompletionLookup lookup = new FSTCompletionLookup();
+ Directory tempDir = getDirectory();
+ FSTCompletionLookup lookup = new FSTCompletionLookup(tempDir, "fst");
lookup.build(new InputArrayIterator(freqs.toArray(new Input[freqs.size()])));
for (Input tf : freqs) {
@@ -219,6 +227,7 @@ public class FSTCompletionTest extends L
}
}
}
+ tempDir.close();
}
private CharSequence stringToCharSequence(String prefix) {
@@ -262,4 +271,12 @@ public class FSTCompletionTest extends L
len = Math.max(len, s.length());
return len;
}
+
+ private Directory getDirectory() {
+ Directory dir = newDirectory();
+ if (dir instanceof MockDirectoryWrapper) {
+ ((MockDirectoryWrapper) dir).setEnableVirusScanner(false);
+ }
+ return dir;
+ }
}
Modified: lucene/dev/branches/lucene6835/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/WFSTCompletionTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/WFSTCompletionTest.java?rev=1709414&r1=1709413&r2=1709414&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/WFSTCompletionTest.java (original)
+++ lucene/dev/branches/lucene6835/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/WFSTCompletionTest.java Mon Oct 19 13:38:42 2015
@@ -19,9 +19,11 @@ package org.apache.lucene.search.suggest
import java.util.*;
-import org.apache.lucene.search.suggest.Lookup.LookupResult;
import org.apache.lucene.search.suggest.Input;
import org.apache.lucene.search.suggest.InputArrayIterator;
+import org.apache.lucene.search.suggest.Lookup.LookupResult;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
@@ -37,7 +39,8 @@ public class WFSTCompletionTest extends
};
Random random = new Random(random().nextLong());
- WFSTCompletionLookup suggester = new WFSTCompletionLookup();
+ Directory tempDir = getDirectory();
+ WFSTCompletionLookup suggester = new WFSTCompletionLookup(tempDir, "wfst");
suggester.build(new InputArrayIterator(keys));
// top N of 2, but only foo is available
@@ -75,11 +78,13 @@ public class WFSTCompletionTest extends
assertEquals(10, results.get(1).value, 0.01F);
assertEquals("barbara", results.get(2).key.toString());
assertEquals(6, results.get(2).value, 0.01F);
+ tempDir.close();
}
public void testExactFirst() throws Exception {
- WFSTCompletionLookup suggester = new WFSTCompletionLookup(true);
+ Directory tempDir = getDirectory();
+ WFSTCompletionLookup suggester = new WFSTCompletionLookup(tempDir, "wfst", true);
suggester.build(new InputArrayIterator(new Input[] {
new Input("x y", 20),
@@ -99,11 +104,13 @@ public class WFSTCompletionTest extends
assertEquals(20, results.get(1).value);
}
}
+ tempDir.close();
}
public void testNonExactFirst() throws Exception {
- WFSTCompletionLookup suggester = new WFSTCompletionLookup(false);
+ Directory tempDir = getDirectory();
+ WFSTCompletionLookup suggester = new WFSTCompletionLookup(tempDir, "wfst", false);
suggester.build(new InputArrayIterator(new Input[] {
new Input("x y", 20),
@@ -123,6 +130,7 @@ public class WFSTCompletionTest extends
assertEquals(2, results.get(1).value);
}
}
+ tempDir.close();
}
public void testRandom() throws Exception {
@@ -153,7 +161,8 @@ public class WFSTCompletionTest extends
keys[i] = new Input(s, weight);
}
- WFSTCompletionLookup suggester = new WFSTCompletionLookup(false);
+ Directory tempDir = getDirectory();
+ WFSTCompletionLookup suggester = new WFSTCompletionLookup(tempDir, "wfst", false);
suggester.build(new InputArrayIterator(keys));
assertEquals(numWords, suggester.getCount());
@@ -196,6 +205,7 @@ public class WFSTCompletionTest extends
assertEquals(matches.get(hit).value, r.get(hit).value, 0f);
}
}
+ tempDir.close();
}
public void test0ByteKeys() throws Exception {
@@ -204,20 +214,32 @@ public class WFSTCompletionTest extends
BytesRef key2 = new BytesRef(3);
key1.length = 3;
- WFSTCompletionLookup suggester = new WFSTCompletionLookup(false);
+ Directory tempDir = getDirectory();
+ WFSTCompletionLookup suggester = new WFSTCompletionLookup(tempDir, "wfst", false);
suggester.build(new InputArrayIterator(new Input[] {
new Input(key1, 50),
new Input(key2, 50),
}));
+ tempDir.close();
}
public void testEmpty() throws Exception {
- WFSTCompletionLookup suggester = new WFSTCompletionLookup(false);
+ Directory tempDir = getDirectory();
+ WFSTCompletionLookup suggester = new WFSTCompletionLookup(tempDir, "wfst", false);
suggester.build(new InputArrayIterator(new Input[0]));
assertEquals(0, suggester.getCount());
List<LookupResult> result = suggester.lookup("a", false, 20);
assertTrue(result.isEmpty());
+ tempDir.close();
+ }
+
+ private Directory getDirectory() {
+ Directory dir = newDirectory();
+ if (dir instanceof MockDirectoryWrapper) {
+ ((MockDirectoryWrapper) dir).setEnableVirusScanner(false);
+ }
+ return dir;
}
}
Modified: lucene/dev/branches/lucene6835/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java?rev=1709414&r1=1709413&r2=1709414&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java (original)
+++ lucene/dev/branches/lucene6835/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java Mon Oct 19 13:38:42 2015
@@ -23,9 +23,12 @@ import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.NoSuchFileException;
import java.nio.file.Path;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.zip.CRC32;
@@ -1163,5 +1166,26 @@ public abstract class BaseDirectoryTestC
in.close(); // close again
dir.close();
}
-}
+ public void testCreateTempOutput() throws Throwable {
+ Directory dir = getDirectory(createTempDir());
+ List<String> names = new ArrayList<>();
+ int iters = atLeast(50);
+ for(int iter=0;iter<iters;iter++) {
+ IndexOutput out = dir.createTempOutput("foo", "bar", newIOContext(random()));
+ names.add(out.getName());
+ out.writeVInt(iter);
+ out.close();
+ }
+ for(int iter=0;iter<iters;iter++) {
+ IndexInput in = dir.openInput(names.get(iter), newIOContext(random()));
+ assertEquals(iter, in.readVInt());
+ in.close();
+ }
+ Set<String> files = new HashSet<String>(Arrays.asList(dir.listAll()));
+ // In case ExtraFS struck:
+ files.remove("extra0");
+ assertEquals(new HashSet<String>(names), files);
+ dir.close();
+ }
+}
Modified: lucene/dev/branches/lucene6835/lucene/test-framework/src/java/org/apache/lucene/store/MockIndexOutputWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/test-framework/src/java/org/apache/lucene/store/MockIndexOutputWrapper.java?rev=1709414&r1=1709413&r2=1709414&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/test-framework/src/java/org/apache/lucene/store/MockIndexOutputWrapper.java (original)
+++ lucene/dev/branches/lucene6835/lucene/test-framework/src/java/org/apache/lucene/store/MockIndexOutputWrapper.java Mon Oct 19 13:38:42 2015
@@ -39,7 +39,7 @@ public class MockIndexOutputWrapper exte
/** Construct an empty output buffer. */
public MockIndexOutputWrapper(MockDirectoryWrapper dir, IndexOutput delegate, String name) {
- super("MockIndexOutputWrapper(" + delegate + ")");
+ super("MockIndexOutputWrapper(" + delegate + ")", delegate.getName());
this.dir = dir;
this.name = name;
this.delegate = delegate;
Modified: lucene/dev/branches/lucene6835/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleTemporaryFilesCleanup.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleTemporaryFilesCleanup.java?rev=1709414&r1=1709413&r2=1709414&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleTemporaryFilesCleanup.java (original)
+++ lucene/dev/branches/lucene6835/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleTemporaryFilesCleanup.java Mon Oct 19 13:38:42 2015
@@ -1,5 +1,22 @@
package org.apache.lucene.util;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
import java.io.IOException;
import java.net.URI;
import java.nio.file.FileSystem;
@@ -30,23 +47,6 @@ import org.apache.lucene.util.LuceneTest
import com.carrotsearch.randomizedtesting.RandomizedContext;
import com.carrotsearch.randomizedtesting.rules.TestRuleAdapter;
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
/**
* Checks and cleans up temporary files.
*
@@ -113,9 +113,6 @@ final class TestRuleTemporaryFilesCleanu
assert tempDirBase == null;
fileSystem = initializeFileSystem();
javaTempDir = initializeJavaTempDir();
-
- // So all code using OfflineSorter (suggesters, BKD tree, NumericRangeTree) see MockFS goodness, e.g. catching leaked file handles:
- OfflineSorter.setDefaultTempDir(javaTempDir);
}
// os/config-independent limit for too many open files
@@ -236,7 +233,7 @@ final class TestRuleTemporaryFilesCleanu
}
}
- final Path getPerTestClassTempDir() {
+ Path getPerTestClassTempDir() {
if (tempDirBase == null) {
RandomizedContext ctx = RandomizedContext.current();
Class<?> clazz = ctx.getTargetClass();
Modified: lucene/dev/branches/lucene6835/lucene/test-framework/src/java/org/apache/lucene/util/ThrottledIndexOutput.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/test-framework/src/java/org/apache/lucene/util/ThrottledIndexOutput.java?rev=1709414&r1=1709413&r2=1709414&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/test-framework/src/java/org/apache/lucene/util/ThrottledIndexOutput.java (original)
+++ lucene/dev/branches/lucene6835/lucene/test-framework/src/java/org/apache/lucene/util/ThrottledIndexOutput.java Mon Oct 19 13:38:42 2015
@@ -16,6 +16,7 @@ package org.apache.lucene.util;
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
import java.io.IOException;
import org.apache.lucene.store.DataInput;
@@ -59,7 +60,7 @@ public class ThrottledIndexOutput extend
public ThrottledIndexOutput(int bytesPerSecond, long flushDelayMillis,
long closeDelayMillis, long seekDelayMillis, long minBytesWritten,
IndexOutput delegate) {
- super("ThrottledIndexOutput(" + delegate + ")");
+ super("ThrottledIndexOutput(" + delegate + ")", delegate == null ? "n/a" : delegate.getName());
assert bytesPerSecond > 0;
this.delegate = delegate;
this.bytesPerSecond = bytesPerSecond;
@@ -117,8 +118,9 @@ public class ThrottledIndexOutput extend
}
private static final void sleep(long ms) {
- if (ms <= 0)
+ if (ms <= 0) {
return;
+ }
try {
Thread.sleep(ms);
} catch (InterruptedException e) {
Modified: lucene/dev/branches/lucene6835/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/solr/CHANGES.txt?rev=1709414&r1=1709413&r2=1709414&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/solr/CHANGES.txt (original)
+++ lucene/dev/branches/lucene6835/solr/CHANGES.txt Mon Oct 19 13:38:42 2015
@@ -111,10 +111,16 @@ Other Changes
* SOLR-7554: Add checks in Streams for incoming stream order (Dennis Gove, Joel Bernstein)
* SOLR-7441: Improve overall robustness of the Streaming stack: Streaming API,
- Streaming Expressions, Parallel SQL (Joel Bernstein)
+ Streaming Expressions, Parallel SQL (Joel Bernstein)
+
+* SOLR-8153: Support upper case and mixed case column identifiers in the SQL interface
+ (Joel Bernstein)
* SOLR-8132: HDFSDirectoryFactory now defaults to using the global block cache. (Mark Miller)
+* SOLR-8114: in Grouping.java rename groupSort and sort to withinGroupSort and groupSort
+ (Christine Poerschke)
+
================== 5.4.0 ==================
Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release
@@ -260,6 +266,22 @@ Bug Fixes
* SOLR-8130: Solr's hdfs safe mode detection does not catch all cases of being in safe mode.
(Mark Miller, Mike Drob)
+
+* SOLR-8128: Set v.locale specified locale for all LocaleConfig extending VelocityResponseWriter tools.
+ (Erik Hatcher)
+
+* SOLR-8152: Overseer Task Processor/Queue can miss responses, leading to timeouts.
+ (Gregory Chanan)
+
+* SOLR-8107: bin/solr -f should use exec to start the JVM (Martijn Koster via Timothy Potter)
+
+* SOLR-8050: Partial update on document with multivalued date field fails to parse date and can
+ also fail to remove dates in some cases. (Burkhard Buelte, Luc Vanlerberghe, shalin)
+
+* SOLR-8167: Authorization framework does not work with POST params (noble)
+
+* SOLR-8162: JmxMonitoredMap#clear triggers a query on all the MBeans thus generating lots of warnings.
+ (Marius Dumitru Florea, shalin)
Optimizations
----------------------
@@ -339,6 +361,9 @@ Other Changes
* SOLR-8151: OverseerCollectionMessageHandler was logging info data as WARN
(Alan Woodward)
+* SOLR-8116: SearchGroupsResultTransformer tweaks (String literals, list/map initialCapacity)
+ (Christine Poerschke)
+
================== 5.3.1 ==================
Bug Fixes
Modified: lucene/dev/branches/lucene6835/solr/bin/solr
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/solr/bin/solr?rev=1709414&r1=1709413&r2=1709414&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/solr/bin/solr (original)
+++ lucene/dev/branches/lucene6835/solr/bin/solr Mon Oct 19 13:38:42 2015
@@ -1308,7 +1308,7 @@ function launch_solr() {
if [ "$run_in_foreground" == "true" ]; then
echo -e "\nStarting Solr$IN_CLOUD_MODE on port $SOLR_PORT from $SOLR_SERVER_DIR\n"
- "$JAVA" "${SOLR_START_OPTS[@]}" $SOLR_ADDL_ARGS -jar start.jar "${SOLR_JETTY_CONFIG[@]}"
+ exec "$JAVA" "${SOLR_START_OPTS[@]}" $SOLR_ADDL_ARGS -jar start.jar "${SOLR_JETTY_CONFIG[@]}"
else
# run Solr in the background
nohup "$JAVA" "${SOLR_START_OPTS[@]}" $SOLR_ADDL_ARGS -jar start.jar \
Modified: lucene/dev/branches/lucene6835/solr/contrib/velocity/src/java/org/apache/solr/response/VelocityResponseWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/solr/contrib/velocity/src/java/org/apache/solr/response/VelocityResponseWriter.java?rev=1709414&r1=1709413&r2=1709414&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/solr/contrib/velocity/src/java/org/apache/solr/response/VelocityResponseWriter.java (original)
+++ lucene/dev/branches/lucene6835/solr/contrib/velocity/src/java/org/apache/solr/response/VelocityResponseWriter.java Mon Oct 19 13:38:42 2015
@@ -50,6 +50,7 @@ import org.apache.velocity.tools.generic
import org.apache.velocity.tools.generic.DisplayTool;
import org.apache.velocity.tools.generic.EscapeTool;
import org.apache.velocity.tools.generic.ListTool;
+import org.apache.velocity.tools.generic.LocaleConfig;
import org.apache.velocity.tools.generic.MathTool;
import org.apache.velocity.tools.generic.NumberTool;
import org.apache.velocity.tools.generic.ResourceTool;
@@ -194,17 +195,33 @@ public class VelocityResponseWriter impl
VelocityContext context = new VelocityContext();
// Register useful Velocity "tools"
+ String locale = request.getParams().get(LOCALE);
+ Map toolConfig = new HashMap();
+ toolConfig.put("locale", locale);
+
+
context.put("log", log); // TODO: add test; TODO: should this be overridable with a custom "log" named tool?
context.put("esc", new EscapeTool());
context.put("date", new ComparisonDateTool());
context.put("list", new ListTool());
- context.put("math", new MathTool());
- context.put("number", new NumberTool());
context.put("sort", new SortTool());
- context.put("display", new DisplayTool());
- context.put("resource", new SolrVelocityResourceTool(
- request.getCore().getSolrConfig().getResourceLoader().getClassLoader(),
- request.getParams().get(LOCALE)));
+
+ MathTool mathTool = new MathTool();
+ mathTool.configure(toolConfig);
+ context.put("math", mathTool);
+
+ NumberTool numberTool = new NumberTool();
+ numberTool.configure(toolConfig);
+ context.put("number", numberTool);
+
+
+ DisplayTool displayTool = new DisplayTool();
+ displayTool.configure(toolConfig);
+ context.put("display", displayTool);
+
+ ResourceTool resourceTool = new SolrVelocityResourceTool(request.getCore().getSolrConfig().getResourceLoader().getClassLoader());
+ resourceTool.configure(toolConfig);
+ context.put("resource", resourceTool);
/*
// Custom tools, specified in config as:
@@ -213,12 +230,14 @@ public class VelocityResponseWriter impl
<str name="mytool">com.example.solr.velocity.MyTool</str>
</lst>
</queryResponseWriter>
-
-
*/
// Custom tools can override any of the built-in tools provided above, by registering one with the same name
for(String name : customTools.keySet()) {
- context.put(name, SolrCore.createInstance(customTools.get(name), Object.class, "VrW custom tool", request.getCore(), request.getCore().getResourceLoader()));
+ Object customTool = SolrCore.createInstance(customTools.get(name), Object.class, "VrW custom tool: " + name, request.getCore(), request.getCore().getResourceLoader());
+ if (customTool instanceof LocaleConfig) {
+ ((LocaleConfig)customTool).configure(toolConfig);
+ }
+ context.put(name, customTool);
}
// custom tools _cannot_ override context objects added below, like $request and $response
@@ -359,10 +378,8 @@ public class VelocityResponseWriter impl
private ClassLoader solrClassLoader;
- public SolrVelocityResourceTool(ClassLoader cl, String localeString) {
+ public SolrVelocityResourceTool(ClassLoader cl) {
this.solrClassLoader = cl;
- Locale l = toLocale(localeString);
- this.setLocale(l == null ? Locale.ROOT : l);
}
@Override
Modified: lucene/dev/branches/lucene6835/solr/contrib/velocity/src/test/org/apache/solr/velocity/MockTool.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/solr/contrib/velocity/src/test/org/apache/solr/velocity/MockTool.java?rev=1709414&r1=1709413&r2=1709414&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/solr/contrib/velocity/src/test/org/apache/solr/velocity/MockTool.java (original)
+++ lucene/dev/branches/lucene6835/solr/contrib/velocity/src/test/org/apache/solr/velocity/MockTool.java Mon Oct 19 13:38:42 2015
@@ -18,8 +18,9 @@ package org.apache.solr.velocity;
*/
import org.apache.solr.core.SolrCore;
+import org.apache.velocity.tools.generic.LocaleConfig;
-public class MockTool {
+public class MockTool extends LocaleConfig {
private final SolrCore core;
public MockTool(SolrCore core) {
Modified: lucene/dev/branches/lucene6835/solr/contrib/velocity/src/test/org/apache/solr/velocity/VelocityResponseWriterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/solr/contrib/velocity/src/test/org/apache/solr/velocity/VelocityResponseWriterTest.java?rev=1709414&r1=1709413&r2=1709414&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/solr/contrib/velocity/src/test/org/apache/solr/velocity/VelocityResponseWriterTest.java (original)
+++ lucene/dev/branches/lucene6835/solr/contrib/velocity/src/test/org/apache/solr/velocity/VelocityResponseWriterTest.java Mon Oct 19 13:38:42 2015
@@ -121,7 +121,9 @@ public class VelocityResponseWriterTest
@Test
public void testCustomTools() throws Exception {
+ // custom_tool.vm responds with $!mytool.star("foo"), but $mytool is not defined (only in velocityWithCustomTools)
assertEquals("", h.query(req("q","*:*", "wt","velocity",VelocityResponseWriter.TEMPLATE,"custom_tool")));
+
assertEquals("** LATERALUS **", h.query(req("q","*:*", "wt","velocityWithCustomTools",VelocityResponseWriter.TEMPLATE,"t",
SolrParamResourceLoader.TEMPLATE_PARAM_PREFIX+"t", "$mytool.star(\"LATERALUS\")")));
@@ -151,6 +153,16 @@ public class VelocityResponseWriterTest
// Test that $resource.get(key,baseName,locale) works with specified locale
assertEquals("Colour", h.query(req("q","*:*", "wt","velocity",VelocityResponseWriter.TEMPLATE,"resource_get")));
+
+ // Test that $number tool uses the specified locale
+ assertEquals("2,112", h.query(req("q","*:*", "wt","velocityWithCustomTools",VelocityResponseWriter.TEMPLATE,"t",
+ SolrParamResourceLoader.TEMPLATE_PARAM_PREFIX+"t","$number.format(2112)", VelocityResponseWriter.LOCALE, "en_US")));
+ assertEquals("2.112", h.query(req("q","*:*", "wt","velocityWithCustomTools",VelocityResponseWriter.TEMPLATE,"t",
+ SolrParamResourceLoader.TEMPLATE_PARAM_PREFIX+"t","$number.format(2112)", VelocityResponseWriter.LOCALE, "de_DE")));
+
+ // Test that custom tool extending LocaleConfig gets the right locale
+ assertEquals("de_DE", h.query(req("q","*:*", "wt","velocityWithCustomTools",VelocityResponseWriter.TEMPLATE,"t",
+ SolrParamResourceLoader.TEMPLATE_PARAM_PREFIX+"t","$mytool.locale", VelocityResponseWriter.LOCALE, "de_DE")));
}
@Test
Modified: lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/cloud/OverseerTaskQueue.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/cloud/OverseerTaskQueue.java?rev=1709414&r1=1709413&r2=1709414&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/cloud/OverseerTaskQueue.java (original)
+++ lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/cloud/OverseerTaskQueue.java Mon Oct 19 13:38:42 2015
@@ -29,6 +29,7 @@ import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.WatchedEvent;
import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.data.Stat;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -91,6 +92,9 @@ public class OverseerTaskQueue extends D
+ path.substring(path.lastIndexOf("-") + 1);
if (zookeeper.exists(responsePath, true)) {
zookeeper.setData(responsePath, event.getBytes(), true);
+ } else {
+ LOG.info("Response ZK path: " + responsePath + " doesn't exist."
+ + " Requestor may have disconnected from ZooKeeper");
}
byte[] data = zookeeper.getData(path, null, null, true);
zookeeper.delete(path, -1, true);
@@ -127,8 +131,8 @@ public class OverseerTaskQueue extends D
Event.EventType eventType = event.getType();
// None events are ignored
// If latchEventType is not null, only fire if the type matches
+ LOG.info("{} fired on path {} state {} latchEventType {}", eventType, event.getPath(), event.getState(), latchEventType);
if (eventType != Event.EventType.None && (latchEventType == null || eventType == latchEventType)) {
- LOG.info("{} fired on path {} state {}", eventType, event.getPath(), event.getState());
synchronized (lock) {
this.event = event;
lock.notifyAll();
@@ -176,22 +180,31 @@ public class OverseerTaskQueue extends D
InterruptedException {
TimerContext time = stats.time(dir + "_offer");
try {
- String path = createData(dir + "/" + PREFIX, data,
- CreateMode.PERSISTENT_SEQUENTIAL);
+ // Create and watch the response node before creating the request node;
+ // otherwise we may miss the response.
String watchID = createData(
- dir + "/" + response_prefix + path.substring(path.lastIndexOf("-") + 1),
- null, CreateMode.EPHEMERAL);
+ dir + "/" + response_prefix,
+ null, CreateMode.EPHEMERAL_SEQUENTIAL);
Object lock = new Object();
LatchWatcher watcher = new LatchWatcher(lock);
+ Stat stat = zookeeper.exists(watchID, watcher, true);
+
+ // create the request node
+ createData(dir + "/" + PREFIX + watchID.substring(watchID.lastIndexOf("-") + 1),
+ data, CreateMode.PERSISTENT);
+
synchronized (lock) {
- if (zookeeper.exists(watchID, watcher, true) != null) {
+ if (stat != null && watcher.getWatchedEvent() == null) {
watcher.await(timeout);
}
}
byte[] bytes = zookeeper.getData(watchID, null, null, true);
+ // create the event before deleting the node, otherwise we can get the deleted
+ // event from the watcher.
+ QueueEvent event = new QueueEvent(watchID, bytes, watcher.getWatchedEvent());
zookeeper.delete(watchID, -1, true);
- return new QueueEvent(watchID, bytes, watcher.getWatchedEvent());
+ return event;
} finally {
time.stop();
}
Modified: lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/core/JmxMonitoredMap.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/core/JmxMonitoredMap.java?rev=1709414&r1=1709413&r2=1709414&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/core/JmxMonitoredMap.java (original)
+++ lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/core/JmxMonitoredMap.java Mon Oct 19 13:38:42 2015
@@ -141,7 +141,8 @@ public class JmxMonitoredMap<K, V> exten
Set<ObjectName> objectNames = null;
try {
- objectNames = server.queryNames(null, exp);
+ ObjectName instance = new ObjectName(jmxRootName + ":*");
+ objectNames = server.queryNames(instance, exp);
} catch (Exception e) {
LOG.warn("Exception querying for mbeans", e);
}
Modified: lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/handler/SQLHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/handler/SQLHandler.java?rev=1709414&r1=1709413&r2=1709414&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/handler/SQLHandler.java (original)
+++ lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/handler/SQLHandler.java Mon Oct 19 13:38:42 2015
@@ -94,7 +94,7 @@ public class SQLHandler extends RequestH
SolrParams params = req.getParams();
params = adjustParams(params);
req.setParams(params);
- String sql = params.get("sql");
+ String sql = params.get("stmt");
int numWorkers = params.getInt("numWorkers", 1);
String workerCollection = params.get("workerCollection", defaultWorkerCollection);
String workerZkhost = params.get("workerZkhost",defaultZkhost);
@@ -370,8 +370,9 @@ public class SQLHandler extends RequestH
Set<String> sortFields = new HashSet();
for(SortItem sortItem : sorts) {
- sortFields.add(stripSingleQuotes(stripQuotes(sortItem.getSortKey().toString())));
- adjustedSorts.add(new FieldComparator(stripSingleQuotes(stripQuotes(sortItem.getSortKey().toString())),
+
+ sortFields.add(getSortField(sortItem));
+ adjustedSorts.add(new FieldComparator(getSortField(sortItem),
ascDescComp(sortItem.getOrdering().toString())));
}
@@ -380,7 +381,7 @@ public class SQLHandler extends RequestH
}
for(SortItem sortItem : sorts) {
- String sortField = stripSingleQuotes(stripQuotes(sortItem.getSortKey().toString()));
+ String sortField = getSortField(sortItem);
if(!bucketFields.contains(sortField)) {
throw new IOException("All sort fields must be in the field list.");
}
@@ -555,7 +556,7 @@ public class SQLHandler extends RequestH
if (comma) {
siBuf.append(",");
}
- siBuf.append(stripSingleQuotes(stripQuotes(sortItem.getSortKey().toString())) + " " + ascDesc(sortItem.getOrdering().toString()));
+ siBuf.append(getSortField(sortItem) + " " + ascDesc(sortItem.getOrdering().toString()));
}
} else {
if(sqlVisitor.limit < 0) {
@@ -599,7 +600,7 @@ public class SQLHandler extends RequestH
for(int i=0; i< buckets.length; i++) {
Bucket bucket = buckets[i];
SortItem sortItem = sortItems.get(i);
- if(!bucket.toString().equals(stripSingleQuotes(stripQuotes(sortItem.getSortKey().toString())))) {
+ if(!bucket.toString().equals(getSortField(sortItem))) {
return false;
}
@@ -661,7 +662,7 @@ public class SQLHandler extends RequestH
return buf.toString();
}
- public static String getSortDirection(List<SortItem> sorts) {
+ private static String getSortDirection(List<SortItem> sorts) {
if(sorts != null && sorts.size() > 0) {
for(SortItem item : sorts) {
return ascDesc(stripSingleQuotes(stripQuotes(item.getOrdering().toString())));
@@ -692,8 +693,8 @@ public class SQLHandler extends RequestH
SortItem sortItem = sortItems.get(i);
String ordering = sortItem.getOrdering().toString();
ComparatorOrder comparatorOrder = ascDescComp(ordering);
- String sortKey = sortItem.getSortKey().toString();
- comps[i] = new FieldComparator(stripSingleQuotes(stripQuotes(sortKey)), comparatorOrder);
+ String sortKey = getSortField(sortItem);
+ comps[i] = new FieldComparator(sortKey, comparatorOrder);
}
if(comps.length == 1) {
@@ -709,8 +710,8 @@ public class SQLHandler extends RequestH
SortItem sortItem = sortItems.get(i);
String ordering = sortItem.getOrdering().toString();
ComparatorOrder comparatorOrder = ascDescComp(ordering);
- String sortKey = sortItem.getSortKey().toString();
- comps[i] = new FieldComparator(stripSingleQuotes(stripQuotes(sortKey)), comparatorOrder);
+ String sortKey = getSortField(sortItem);
+ comps[i] = new FieldComparator(sortKey, comparatorOrder);
}
return comps;
@@ -855,7 +856,7 @@ public class SQLHandler extends RequestH
}
protected Void visitComparisonExpression(ComparisonExpression node, StringBuilder buf) {
- String field = node.getLeft().toString();
+ String field = getPredicateField(node.getLeft());
String value = node.getRight().toString();
value = stripSingleQuotes(value);
@@ -864,7 +865,7 @@ public class SQLHandler extends RequestH
value = '"'+value+'"';
}
- buf.append('(').append(stripQuotes(stripSingleQuotes(field)) + ":" + value).append(')');
+ buf.append('(').append(field + ":" + value).append(')');
return null;
}
}
@@ -943,7 +944,7 @@ public class SQLHandler extends RequestH
this.groupByQuery = true;
List<Expression> groups = node.getGroupBy();
for(Expression group : groups) {
- groupBy.add(stripSingleQuotes(stripQuotes(group.toString())));
+ groupBy.add(getGroupField(group));
}
}
@@ -992,7 +993,37 @@ public class SQLHandler extends RequestH
}
protected Void visitSingleColumn(SingleColumn node, Integer indent) {
- fields.add(stripSingleQuotes(stripQuotes(ExpressionFormatter.formatExpression(node.getExpression()))));
+
+ Expression ex = node.getExpression();
+ String field = null;
+
+ if(ex instanceof QualifiedNameReference) {
+
+ QualifiedNameReference ref = (QualifiedNameReference)ex;
+ List<String> parts = ref.getName().getOriginalParts();
+ field = parts.get(0);
+
+ } else if(ex instanceof FunctionCall) {
+
+ FunctionCall functionCall = (FunctionCall)ex;
+ List<String> parts = functionCall.getName().getOriginalParts();
+ List<Expression> args = functionCall.getArguments();
+ String col = null;
+
+ if(args.size() > 0 && args.get(0) instanceof QualifiedNameReference) {
+ QualifiedNameReference ref = (QualifiedNameReference) args.get(0);
+ col = ref.getName().getOriginalParts().get(0);
+ field = parts.get(0)+"("+stripSingleQuotes(col)+")";
+ } else {
+ field = stripSingleQuotes(stripQuotes(functionCall.toString()));
+ }
+
+ } else if(ex instanceof StringLiteral) {
+ StringLiteral stringLiteral = (StringLiteral)ex;
+ field = stripSingleQuotes(stringLiteral.toString());
+ }
+
+ fields.add(field);
if(node.getAlias().isPresent()) {
}
@@ -1000,6 +1031,9 @@ public class SQLHandler extends RequestH
return null;
}
+
+
+
protected Void visitAllColumns(AllColumns node, Integer context) {
return null;
}
@@ -1041,6 +1075,98 @@ public class SQLHandler extends RequestH
}
}
+ private static String getSortField(SortItem sortItem)
+ {
+ String field;
+ Expression ex = sortItem.getSortKey();
+ if(ex instanceof QualifiedNameReference) {
+ QualifiedNameReference ref = (QualifiedNameReference)ex;
+ List<String> parts = ref.getName().getOriginalParts();
+ field = parts.get(0);
+ } else if(ex instanceof FunctionCall) {
+ FunctionCall functionCall = (FunctionCall)ex;
+ List<String> parts = functionCall.getName().getOriginalParts();
+ List<Expression> args = functionCall.getArguments();
+ String col = null;
+
+ if(args.size() > 0 && args.get(0) instanceof QualifiedNameReference) {
+ QualifiedNameReference ref = (QualifiedNameReference) args.get(0);
+ col = ref.getName().getOriginalParts().get(0);
+ field = parts.get(0)+"("+stripSingleQuotes(col)+")";
+ } else {
+ field = stripSingleQuotes(stripQuotes(functionCall.toString()));
+ }
+
+ } else {
+ StringLiteral stringLiteral = (StringLiteral)ex;
+ field = stripSingleQuotes(stringLiteral.toString());
+ }
+
+ return field;
+ }
+
+
+ private static String getHavingField(Expression ex)
+ {
+ String field;
+ if(ex instanceof QualifiedNameReference) {
+ QualifiedNameReference ref = (QualifiedNameReference)ex;
+ List<String> parts = ref.getName().getOriginalParts();
+ field = parts.get(0);
+ } else if(ex instanceof FunctionCall) {
+ FunctionCall functionCall = (FunctionCall)ex;
+ List<String> parts = functionCall.getName().getOriginalParts();
+ List<Expression> args = functionCall.getArguments();
+ String col = null;
+
+ if(args.size() > 0 && args.get(0) instanceof QualifiedNameReference) {
+ QualifiedNameReference ref = (QualifiedNameReference) args.get(0);
+ col = ref.getName().getOriginalParts().get(0);
+ field = parts.get(0)+"("+stripSingleQuotes(col)+")";
+ } else {
+ field = stripSingleQuotes(stripQuotes(functionCall.toString()));
+ }
+
+ } else {
+ StringLiteral stringLiteral = (StringLiteral)ex;
+ field = stripSingleQuotes(stringLiteral.toString());
+ }
+
+ return field;
+ }
+
+
+ private static String getPredicateField(Expression ex)
+ {
+ String field;
+ if(ex instanceof QualifiedNameReference) {
+ QualifiedNameReference ref = (QualifiedNameReference)ex;
+ List<String> parts = ref.getName().getOriginalParts();
+ field = parts.get(0);
+ } else {
+ StringLiteral stringLiteral = (StringLiteral)ex;
+ field = stripSingleQuotes(stringLiteral.toString());
+ }
+
+ return field;
+ }
+
+ private static String getGroupField(Expression ex)
+ {
+ String field;
+ if(ex instanceof QualifiedNameReference) {
+ QualifiedNameReference ref = (QualifiedNameReference)ex;
+ List<String> parts = ref.getName().getOriginalParts();
+ field = parts.get(0);
+ } else {
+ StringLiteral stringLiteral = (StringLiteral)ex;
+ field = stripSingleQuotes(stringLiteral.toString());
+ }
+
+ return field;
+ }
+
+
private static class LimitStream extends TupleStream {
private TupleStream stream;
@@ -1174,7 +1300,7 @@ public class SQLHandler extends RequestH
}
protected Boolean visitComparisonExpression(ComparisonExpression node, Tuple tuple) {
- String field = stripQuotes(node.getLeft().toString());
+ String field = getHavingField(node.getLeft());
double d = Double.parseDouble(node.getRight().toString());
double td = tuple.getDouble(field);
ComparisonExpression.Type t = node.getType();
Modified: lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java?rev=1709414&r1=1709413&r2=1709414&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java (original)
+++ lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java Mon Oct 19 13:38:42 2015
@@ -460,8 +460,8 @@ public class QueryComponent extends Sear
int limitDefault = cmd.getLen(); // this is normally from "rows"
Grouping grouping =
new Grouping(searcher, result, cmd, cacheSecondPassSearch, maxDocsPercentageToCache, groupingSpec.isMain());
- grouping.setSort(groupingSpec.getGroupSort())
- .setGroupSort(groupingSpec.getSortWithinGroup())
+ grouping.setGroupSort(groupingSpec.getGroupSort())
+ .setWithinGroupSort(groupingSpec.getSortWithinGroup())
.setDefaultFormat(groupingSpec.getResponseFormat())
.setLimitDefault(limitDefault)
.setDefaultTotalCount(defaultTotalCount)
Modified: lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/request/SimpleFacets.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/request/SimpleFacets.java?rev=1709414&r1=1709413&r2=1709414&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/request/SimpleFacets.java (original)
+++ lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/request/SimpleFacets.java Mon Oct 19 13:38:42 2015
@@ -268,7 +268,7 @@ public class SimpleFacets {
DocSet base = searcher.getDocSet(qlist);
if (rb.grouping() && rb.getGroupingSpec().isTruncateGroups()) {
Grouping grouping = new Grouping(searcher, null, rb.getQueryCommand(), false, 0, false);
- grouping.setGroupSort(rb.getGroupingSpec().getSortWithinGroup());
+ grouping.setWithinGroupSort(rb.getGroupingSpec().getSortWithinGroup());
if (rb.getGroupingSpec().getFields().length > 0) {
grouping.addFieldCommand(rb.getGroupingSpec().getFields()[0], req);
} else if (rb.getGroupingSpec().getFunctions().length > 0) {
Modified: lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/schema/TrieField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/schema/TrieField.java?rev=1709414&r1=1709413&r2=1709414&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/schema/TrieField.java (original)
+++ lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/schema/TrieField.java Mon Oct 19 13:38:42 2015
@@ -566,6 +566,11 @@ public class TrieField extends Primitive
@Override
public String storedToIndexed(StorableField f) {
final BytesRefBuilder bytes = new BytesRefBuilder();
+ storedToIndexed(f, bytes);
+ return bytes.get().utf8ToString();
+ }
+
+ private void storedToIndexed(StorableField f, final BytesRefBuilder bytes) {
final Number val = f.numericValue();
if (val != null) {
switch (type) {
@@ -620,7 +625,6 @@ public class TrieField extends Primitive
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown type for trie field: " + f.name());
}
}
- return bytes.get().utf8ToString();
}
@Override
@@ -712,7 +716,7 @@ public class TrieField extends Primitive
if (sf.multiValued()) {
BytesRefBuilder bytes = new BytesRefBuilder();
- readableToIndexed(value.toString(), bytes);
+ storedToIndexed(field, bytes);
fields.add(new SortedSetDocValuesField(sf.getName(), bytes.get()));
} else {
final long bits;
Modified: lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/search/Grouping.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/search/Grouping.java?rev=1709414&r1=1709413&r2=1709414&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/search/Grouping.java (original)
+++ lucene/dev/branches/lucene6835/solr/core/src/java/org/apache/solr/search/Grouping.java Mon Oct 19 13:38:42 2015
@@ -90,8 +90,8 @@ public class Grouping {
private final boolean cacheSecondPassSearch;
private final int maxDocsPercentageToCache;
- private Sort sort;
private Sort groupSort;
+ private Sort withinGroupSort;
private int limitDefault;
private int docsPerGroupDefault;
private int groupOffsetDefault;
@@ -158,14 +158,14 @@ public class Grouping {
}
Grouping.CommandField gc = new CommandField();
- gc.groupSort = groupSort;
+ gc.withinGroupSort = withinGroupSort;
gc.groupBy = field;
gc.key = field;
gc.numGroups = limitDefault;
gc.docsPerGroup = docsPerGroupDefault;
gc.groupOffset = groupOffsetDefault;
gc.offset = cmd.getOffset();
- gc.sort = sort;
+ gc.groupSort = groupSort;
gc.format = defaultFormat;
gc.totalCount = defaultTotalCount;
@@ -201,13 +201,13 @@ public class Grouping {
commandFunc.groupBy = new QueryValueSource(q, 0.0f);
gc = commandFunc;
}
- gc.groupSort = groupSort;
+ gc.withinGroupSort = withinGroupSort;
gc.key = groupByStr;
gc.numGroups = limitDefault;
gc.docsPerGroup = docsPerGroupDefault;
gc.groupOffset = groupOffsetDefault;
gc.offset = cmd.getOffset();
- gc.sort = sort;
+ gc.groupSort = groupSort;
gc.format = defaultFormat;
gc.totalCount = defaultTotalCount;
@@ -228,7 +228,7 @@ public class Grouping {
Query gq = parser.getQuery();
Grouping.CommandQuery gc = new CommandQuery();
gc.query = gq;
- gc.groupSort = groupSort;
+ gc.withinGroupSort = withinGroupSort;
gc.key = groupByStr;
gc.numGroups = limitDefault;
gc.docsPerGroup = docsPerGroupDefault;
@@ -251,13 +251,13 @@ public class Grouping {
commands.add(gc);
}
- public Grouping setSort(Sort sort) {
- this.sort = sort;
+ public Grouping setGroupSort(Sort groupSort) {
+ this.groupSort = groupSort;
return this;
}
- public Grouping setGroupSort(Sort groupSort) {
- this.groupSort = groupSort;
+ public Grouping setWithinGroupSort(Sort withinGroupSort) {
+ this.withinGroupSort = withinGroupSort;
return this;
}
@@ -309,12 +309,12 @@ public class Grouping {
needScores = (cmd.getFlags() & SolrIndexSearcher.GET_SCORES) != 0;
boolean cacheScores = false;
- // NOTE: Change this when groupSort can be specified per group
+ // NOTE: Change this when withinGroupSort can be specified per group
if (!needScores && !commands.isEmpty()) {
- if (commands.get(0).groupSort == null) {
+ if (commands.get(0).withinGroupSort == null) {
cacheScores = true;
} else {
- for (SortField field : commands.get(0).groupSort.getSort()) {
+ for (SortField field : commands.get(0).withinGroupSort.getSort()) {
if (field.getType() == SortField.Type.SCORE) {
cacheScores = true;
break;
@@ -525,8 +525,8 @@ public class Grouping {
public abstract class Command<GROUP_VALUE_TYPE> {
public String key; // the name to use for this group in the response
- public Sort groupSort; // the sort of the documents *within* a single group.
- public Sort sort; // the sort between groups
+ public Sort withinGroupSort; // the sort of the documents *within* a single group.
+ public Sort groupSort; // the sort between groups
public int docsPerGroup; // how many docs in each group - from "group.limit" param, default=1
public int groupOffset; // the offset within each group (for paging within each group)
public int numGroups; // how many groups - defaults to the "rows" parameter
@@ -730,8 +730,8 @@ public class Grouping {
return fallBackCollector;
}
- sort = sort == null ? Sort.RELEVANCE : sort;
- firstPass = new TermFirstPassGroupingCollector(groupBy, sort, actualGroupsToFind);
+ groupSort = groupSort == null ? Sort.RELEVANCE : groupSort;
+ firstPass = new TermFirstPassGroupingCollector(groupBy, groupSort, actualGroupsToFind);
return firstPass;
}
@@ -760,7 +760,7 @@ public class Grouping {
int groupedDocsToCollect = getMax(groupOffset, docsPerGroup, maxDoc);
groupedDocsToCollect = Math.max(groupedDocsToCollect, 1);
secondPass = new TermSecondPassGroupingCollector(
- groupBy, topGroups, sort, groupSort, groupedDocsToCollect, needScores, needScores, false
+ groupBy, topGroups, groupSort, withinGroupSort, groupedDocsToCollect, needScores, needScores, false
);
if (totalCount == TotalCount.grouped) {
@@ -776,7 +776,7 @@ public class Grouping {
*/
@Override
public AbstractAllGroupHeadsCollector<?> createAllGroupCollector() throws IOException {
- Sort sortWithinGroup = groupSort != null ? groupSort : new Sort();
+ Sort sortWithinGroup = withinGroupSort != null ? withinGroupSort : new Sort();
return TermAllGroupHeadsCollector.create(groupBy, sortWithinGroup);
}
@@ -875,7 +875,7 @@ public class Grouping {
@Override
protected Collector createFirstPassCollector() throws IOException {
DocSet groupFilt = searcher.getDocSet(query);
- topCollector = newCollector(groupSort, needScores);
+ topCollector = newCollector(withinGroupSort, needScores);
collector = new FilterCollector(groupFilt, topCollector);
return collector;
}
@@ -950,8 +950,8 @@ public class Grouping {
return fallBackCollector;
}
- sort = sort == null ? Sort.RELEVANCE : sort;
- firstPass = new FunctionFirstPassGroupingCollector(groupBy, context, searcher.weightSort(sort), actualGroupsToFind);
+ groupSort = groupSort == null ? Sort.RELEVANCE : groupSort;
+ firstPass = new FunctionFirstPassGroupingCollector(groupBy, context, searcher.weightSort(groupSort), actualGroupsToFind);
return firstPass;
}
@@ -980,7 +980,7 @@ public class Grouping {
int groupdDocsToCollect = getMax(groupOffset, docsPerGroup, maxDoc);
groupdDocsToCollect = Math.max(groupdDocsToCollect, 1);
secondPass = new FunctionSecondPassGroupingCollector(
- topGroups, sort, groupSort, groupdDocsToCollect, needScores, needScores, false, groupBy, context
+ topGroups, groupSort, withinGroupSort, groupdDocsToCollect, needScores, needScores, false, groupBy, context
);
if (totalCount == TotalCount.grouped) {
@@ -993,7 +993,7 @@ public class Grouping {
@Override
public AbstractAllGroupHeadsCollector<?> createAllGroupCollector() throws IOException {
- Sort sortWithinGroup = groupSort != null ? groupSort : new Sort();
+ Sort sortWithinGroup = withinGroupSort != null ? withinGroupSort : new Sort();
return new FunctionAllGroupHeadsCollector(groupBy, context, sortWithinGroup);
}