You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by us...@apache.org on 2017/01/16 10:17:21 UTC
[4/6] lucene-solr:master: add comment and test for ngram token filter
add comment and test for ngram token filter
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/80e28542
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/80e28542
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/80e28542
Branch: refs/heads/master
Commit: 80e2854247cce485920b45acdeffa3e68bcea385
Parents: 01f2a87
Author: Nathan Gass <ga...@search.ch>
Authored: Fri Jan 13 16:42:41 2017 +0100
Committer: Nathan Gass <ga...@search.ch>
Committed: Fri Jan 13 16:42:41 2017 +0100
----------------------------------------------------------------------
.../lucene/analysis/ngram/TestNGramFilters.java | 25 ++++++++++++++++++++
1 file changed, 25 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/80e28542/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/TestNGramFilters.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/TestNGramFilters.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/TestNGramFilters.java
index b6f4405..5de532f 100644
--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/TestNGramFilters.java
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/TestNGramFilters.java
@@ -80,6 +80,28 @@ public class TestNGramFilters extends BaseTokenStreamFactoryTestCase {
}
/**
+ * Test NGramFilterFactory on tokens with payloads
+ */
+ public void testNGramFilterPayload() throws Exception {
+ Reader reader = new StringReader("test|0.1");
+ TokenStream stream = whitespaceMockTokenizer(reader);
+ stream = tokenFilterFactory("DelimitedPayload", "encoder", "float").create(stream);
+ stream = tokenFilterFactory("NGram", "minGramSize", "1", "maxGramSize", "2").create(stream);
+
+ stream.reset();
+ while (stream.incrementToken()) {
+ PayloadAttribute payAttr = stream.getAttribute(PayloadAttribute.class);
+ assertNotNull(payAttr);
+ BytesRef payData = payAttr.getPayload();
+ assertNotNull(payData);
+ float payFloat = PayloadHelper.decodeFloat(payData.bytes);
+ assertEquals(0.1f, payFloat, 0.0f);
+ }
+ stream.end();
+ stream.close();
+ }
+
+ /**
* Test EdgeNGramTokenizerFactory
*/
public void testEdgeNGramTokenizer() throws Exception {
@@ -127,6 +149,9 @@ public class TestNGramFilters extends BaseTokenStreamFactoryTestCase {
new String[] { "t", "te" });
}
+ /**
+ * Test EdgeNGramFilterFactory on tokens with payloads
+ */
public void testEdgeNGramFilterPayload() throws Exception {
Reader reader = new StringReader("test|0.1");
TokenStream stream = whitespaceMockTokenizer(reader);