You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jp...@apache.org on 2016/07/12 16:03:17 UTC
[2/3] lucene-solr:branch_6x: LUCENE-7355: Add Analyzer#normalize()
and use it in query parsers.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7c2e7a0f/lucene/queryparser/src/test/org/apache/lucene/queryparser/analyzing/TestAnalyzingQueryParser.java
----------------------------------------------------------------------
diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/analyzing/TestAnalyzingQueryParser.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/analyzing/TestAnalyzingQueryParser.java
index bf5f69f..c1c6375 100644
--- a/lucene/queryparser/src/test/org/apache/lucene/queryparser/analyzing/TestAnalyzingQueryParser.java
+++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/analyzing/TestAnalyzingQueryParser.java
@@ -21,9 +21,8 @@ import java.util.Map;
import java.util.TreeMap;
import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.analysis.CharacterUtils;
import org.apache.lucene.analysis.MockBytesAnalyzer;
-import org.apache.lucene.analysis.MockTokenFilter;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
@@ -100,24 +99,6 @@ public class TestAnalyzingQueryParser extends LuceneTestCase {
a = new ASCIIAnalyzer();
}
-
- public void testSingleChunkExceptions() {
- String termStr = "the*tre";
-
- Analyzer stopsAnalyzer = new MockAnalyzer
- (random(), MockTokenizer.WHITESPACE, true, MockTokenFilter.ENGLISH_STOPSET);
-
- ParseException expected = expectThrows(ParseException.class, () -> {
- parseWithAnalyzingQueryParser(termStr, stopsAnalyzer, true);
- });
- assertTrue(expected.getMessage().contains("returned nothing"));
-
- AnalyzingQueryParser qp = new AnalyzingQueryParser(FIELD, a);
- expected = expectThrows(ParseException.class, () -> {
- qp.analyzeSingleChunk(FIELD, "", "not a single chunk");
- });
- assertTrue(expected.getMessage().contains("multiple terms"));
- }
public void testWildcardAlone() throws ParseException {
//seems like crazy edge case, but can be useful in concordance
@@ -221,12 +202,36 @@ public class TestAnalyzingQueryParser extends LuceneTestCase {
}
}
+ final static class LowercaseFilter extends TokenFilter {
+
+ final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+
+ LowercaseFilter(TokenStream input) {
+ super(input);
+ }
+
+ @Override
+ public final boolean incrementToken() throws IOException {
+ if (input.incrementToken()) {
+ CharacterUtils.toLowerCase(termAtt.buffer(), 0, termAtt.length());
+ return true;
+ } else
+ return false;
+ }
+
+ }
+
final static class ASCIIAnalyzer extends Analyzer {
@Override
public TokenStreamComponents createComponents(String fieldName) {
Tokenizer result = new MockTokenizer(MockTokenizer.WHITESPACE, true);
return new TokenStreamComponents(result, new FoldingFilter(result));
}
+
+ @Override
+ protected TokenStream normalize(String fieldName, TokenStream in) {
+ return new FoldingFilter(new LowercaseFilter(in));
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7c2e7a0f/lucene/queryparser/src/test/org/apache/lucene/queryparser/util/QueryParserTestBase.java
----------------------------------------------------------------------
diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/util/QueryParserTestBase.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/util/QueryParserTestBase.java
index d58f660..71c65f6 100644
--- a/lucene/queryparser/src/test/org/apache/lucene/queryparser/util/QueryParserTestBase.java
+++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/util/QueryParserTestBase.java
@@ -1169,6 +1169,10 @@ public abstract class QueryParserTestBase extends LuceneTestCase {
Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, true);
return new TokenStreamComponents(tokenizer, new MockCollationFilter(tokenizer));
}
+ @Override
+ protected TokenStream normalize(String fieldName, TokenStream in) {
+ return new MockCollationFilter(in);
+ }
}
public void testCollatedRange() throws Exception {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7c2e7a0f/lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java
index c57a8bc..0bb623f 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java
@@ -883,7 +883,10 @@ public abstract class BaseTokenStreamTestCase extends LuceneTestCase {
assertTokenStreamContents(ts,
tokens.toArray(new String[tokens.size()]));
}
-
+
+ a.normalize("dummy", text);
+ // TODO: what can we do besides testing that the above method does not throw?
+
if (field != null) {
reader = new StringReader(text);
random = new Random(seed);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7c2e7a0f/lucene/test-framework/src/java/org/apache/lucene/analysis/MockAnalyzer.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/analysis/MockAnalyzer.java b/lucene/test-framework/src/java/org/apache/lucene/analysis/MockAnalyzer.java
index e87bf45..bbeffe9 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/analysis/MockAnalyzer.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/analysis/MockAnalyzer.java
@@ -92,7 +92,16 @@ public final class MockAnalyzer extends Analyzer {
MockTokenFilter filt = new MockTokenFilter(tokenizer, filter);
return new TokenStreamComponents(tokenizer, maybePayload(filt, fieldName));
}
-
+
+ @Override
+ protected TokenStream normalize(String fieldName, TokenStream in) {
+ TokenStream result = in;
+ if (lowerCase) {
+ result = new MockLowerCaseFilter(result);
+ }
+ return result;
+ }
+
private synchronized TokenFilter maybePayload(TokenFilter stream, String fieldName) {
Integer val = previousMappings.get(fieldName);
if (val == null) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7c2e7a0f/lucene/test-framework/src/java/org/apache/lucene/analysis/MockBytesAnalyzer.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/analysis/MockBytesAnalyzer.java b/lucene/test-framework/src/java/org/apache/lucene/analysis/MockBytesAnalyzer.java
index 01f3d4d..b8cfc5b 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/analysis/MockBytesAnalyzer.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/analysis/MockBytesAnalyzer.java
@@ -16,6 +16,8 @@
*/
package org.apache.lucene.analysis;
+import org.apache.lucene.util.AttributeFactory;
+
/**
* Analyzer for testing that encodes terms as UTF-16 bytes.
*/
@@ -26,4 +28,9 @@ public final class MockBytesAnalyzer extends Analyzer {
MockTokenizer.KEYWORD, false, MockTokenizer.DEFAULT_MAX_TOKEN_LENGTH);
return new TokenStreamComponents(t);
}
+
+ @Override
+ protected AttributeFactory attributeFactory() {
+ return MockUTF16TermAttributeImpl.UTF16_TERM_ATTRIBUTE_FACTORY;
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7c2e7a0f/lucene/test-framework/src/java/org/apache/lucene/analysis/MockLowerCaseFilter.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/analysis/MockLowerCaseFilter.java b/lucene/test-framework/src/java/org/apache/lucene/analysis/MockLowerCaseFilter.java
new file mode 100644
index 0000000..b1aea3d
--- /dev/null
+++ b/lucene/test-framework/src/java/org/apache/lucene/analysis/MockLowerCaseFilter.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.analysis;
+
+import java.io.IOException;
+
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+
+/** A lowercasing {@link TokenFilter}. */
+public final class MockLowerCaseFilter extends TokenFilter {
+ private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+
+ /** Sole constructor. */
+ public MockLowerCaseFilter(TokenStream in) {
+ super(in);
+ }
+
+ @Override
+ public final boolean incrementToken() throws IOException {
+ if (input.incrementToken()) {
+ CharacterUtils.toLowerCase(termAtt.buffer(), 0, termAtt.length());
+ return true;
+ } else
+ return false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7c2e7a0f/solr/core/src/java/org/apache/solr/analysis/TokenizerChain.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/analysis/TokenizerChain.java b/solr/core/src/java/org/apache/solr/analysis/TokenizerChain.java
index c9f263d..a5afbec 100644
--- a/solr/core/src/java/org/apache/solr/analysis/TokenizerChain.java
+++ b/solr/core/src/java/org/apache/solr/analysis/TokenizerChain.java
@@ -18,6 +18,7 @@ package org.apache.solr.analysis;
import org.apache.lucene.analysis.*;
import org.apache.lucene.analysis.util.CharFilterFactory;
+import org.apache.lucene.analysis.util.MultiTermAwareComponent;
import org.apache.lucene.analysis.util.TokenFilterFactory;
import org.apache.lucene.analysis.util.TokenizerFactory;
@@ -84,8 +85,21 @@ public final class TokenizerChain extends SolrAnalyzer {
}
@Override
+ protected Reader initReaderForNormalization(String fieldName, Reader reader) {
+ if (charFilters != null && charFilters.length > 0) {
+ for (CharFilterFactory charFilter : charFilters) {
+ if (charFilter instanceof MultiTermAwareComponent) {
+ charFilter = (CharFilterFactory) ((MultiTermAwareComponent) charFilter).getMultiTermComponent();
+ reader = charFilter.create(reader);
+ }
+ }
+ }
+ return reader;
+ }
+
+ @Override
protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tk = tokenizer.create();
+ Tokenizer tk = tokenizer.create(attributeFactory());
TokenStream ts = tk;
for (TokenFilterFactory filter : filters) {
ts = filter.create(ts);
@@ -94,6 +108,18 @@ public final class TokenizerChain extends SolrAnalyzer {
}
@Override
+ protected TokenStream normalize(String fieldName, TokenStream in) {
+ TokenStream result = in;
+ for (TokenFilterFactory filter : filters) {
+ if (filter instanceof MultiTermAwareComponent) {
+ filter = (TokenFilterFactory) ((MultiTermAwareComponent) filter).getMultiTermComponent();
+ result = filter.create(in);
+ }
+ }
+ return result;
+ }
+
+ @Override
public String toString() {
StringBuilder sb = new StringBuilder("TokenizerChain(");
for (CharFilterFactory filter: charFilters) {