You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ro...@apache.org on 2018/03/21 17:38:57 UTC
[2/2] lucene-solr:branch_7x: LUCENE-8202: Add checks for shingle size
LUCENE-8202: Add checks for shingle size
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/bd6cf168
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/bd6cf168
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/bd6cf168
Branch: refs/heads/branch_7x
Commit: bd6cf168e0e129fa22545a7f614b2b146bd5f202
Parents: 092c8d8
Author: Alan Woodward <ro...@apache.org>
Authored: Wed Mar 21 17:37:04 2018 +0000
Committer: Alan Woodward <ro...@apache.org>
Committed: Wed Mar 21 17:38:32 2018 +0000
----------------------------------------------------------------------
.../analysis/shingle/FixedShingleFilter.java | 17 +++++++++++++++--
1 file changed, 15 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/bd6cf168/lucene/analysis/common/src/java/org/apache/lucene/analysis/shingle/FixedShingleFilter.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/shingle/FixedShingleFilter.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/shingle/FixedShingleFilter.java
index a223cd8..93a20ff 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/shingle/FixedShingleFilter.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/shingle/FixedShingleFilter.java
@@ -66,17 +66,30 @@ public final class FixedShingleFilter extends TokenFilter {
private Token[] currentShingleTokens;
private boolean inputStreamExhausted = false;
+ /**
+ * Creates a FixedShingleFilter over an input token stream
+ * @param input the input stream
+ * @param shingleSize the shingle size
+ */
public FixedShingleFilter(TokenStream input, int shingleSize) {
this(input, shingleSize, " ", "_");
}
+ /**
+ * Creates a FixedShingleFilter over an input token stream
+ * @param input the input tokenstream
+ * @param shingleSize the shingle size
+ * @param tokenSeparator a String to use as a token separator
+ * @param fillerToken a String to use to represent gaps in the input stream (due to eg stopwords)
+ */
public FixedShingleFilter(TokenStream input, int shingleSize, String tokenSeparator, String fillerToken) {
super(input);
+ if (shingleSize <= 1) {
+ throw new IllegalArgumentException("shingleSize must be two or greater");
+ }
this.shingleSize = shingleSize;
this.tokenSeparator = tokenSeparator;
-
this.gapToken.termAtt.setEmpty().append(fillerToken);
-
this.currentShingleTokens = new Token[shingleSize];
}