You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ro...@apache.org on 2018/03/21 17:38:56 UTC
[1/2] lucene-solr:master: LUCENE-8202: Add checks for shingle size
Repository: lucene-solr
Updated Branches:
refs/heads/branch_7x 092c8d854 -> bd6cf168e
refs/heads/master 8e87a3e33 -> 2c4b78c43
LUCENE-8202: Add checks for shingle size
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/2c4b78c4
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/2c4b78c4
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/2c4b78c4
Branch: refs/heads/master
Commit: 2c4b78c43fe2e30ef748af34a1daa174d66e29cc
Parents: 8e87a3e
Author: Alan Woodward <ro...@apache.org>
Authored: Wed Mar 21 17:37:04 2018 +0000
Committer: Alan Woodward <ro...@apache.org>
Committed: Wed Mar 21 17:38:05 2018 +0000
----------------------------------------------------------------------
.../analysis/shingle/FixedShingleFilter.java | 17 +++++++++++++++--
1 file changed, 15 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2c4b78c4/lucene/analysis/common/src/java/org/apache/lucene/analysis/shingle/FixedShingleFilter.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/shingle/FixedShingleFilter.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/shingle/FixedShingleFilter.java
index a223cd8..93a20ff 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/shingle/FixedShingleFilter.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/shingle/FixedShingleFilter.java
@@ -66,17 +66,30 @@ public final class FixedShingleFilter extends TokenFilter {
private Token[] currentShingleTokens;
private boolean inputStreamExhausted = false;
+ /**
+ * Creates a FixedShingleFilter over an input token stream
+ * @param input the input stream
+ * @param shingleSize the shingle size
+ */
public FixedShingleFilter(TokenStream input, int shingleSize) {
this(input, shingleSize, " ", "_");
}
+ /**
+ * Creates a FixedShingleFilter over an input token stream
+ * @param input the input tokenstream
+ * @param shingleSize the shingle size
+ * @param tokenSeparator a String to use as a token separator
+ * @param fillerToken a String to use to represent gaps in the input stream (due to eg stopwords)
+ */
public FixedShingleFilter(TokenStream input, int shingleSize, String tokenSeparator, String fillerToken) {
super(input);
+ if (shingleSize <= 1) {
+ throw new IllegalArgumentException("shingleSize must be two or greater");
+ }
this.shingleSize = shingleSize;
this.tokenSeparator = tokenSeparator;
-
this.gapToken.termAtt.setEmpty().append(fillerToken);
-
this.currentShingleTokens = new Token[shingleSize];
}
[2/2] lucene-solr:branch_7x: LUCENE-8202: Add checks for shingle size
Posted by ro...@apache.org.
LUCENE-8202: Add checks for shingle size
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/bd6cf168
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/bd6cf168
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/bd6cf168
Branch: refs/heads/branch_7x
Commit: bd6cf168e0e129fa22545a7f614b2b146bd5f202
Parents: 092c8d8
Author: Alan Woodward <ro...@apache.org>
Authored: Wed Mar 21 17:37:04 2018 +0000
Committer: Alan Woodward <ro...@apache.org>
Committed: Wed Mar 21 17:38:32 2018 +0000
----------------------------------------------------------------------
.../analysis/shingle/FixedShingleFilter.java | 17 +++++++++++++++--
1 file changed, 15 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/bd6cf168/lucene/analysis/common/src/java/org/apache/lucene/analysis/shingle/FixedShingleFilter.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/shingle/FixedShingleFilter.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/shingle/FixedShingleFilter.java
index a223cd8..93a20ff 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/shingle/FixedShingleFilter.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/shingle/FixedShingleFilter.java
@@ -66,17 +66,30 @@ public final class FixedShingleFilter extends TokenFilter {
private Token[] currentShingleTokens;
private boolean inputStreamExhausted = false;
+ /**
+ * Creates a FixedShingleFilter over an input token stream
+ * @param input the input stream
+ * @param shingleSize the shingle size
+ */
public FixedShingleFilter(TokenStream input, int shingleSize) {
this(input, shingleSize, " ", "_");
}
+ /**
+ * Creates a FixedShingleFilter over an input token stream
+ * @param input the input tokenstream
+ * @param shingleSize the shingle size
+ * @param tokenSeparator a String to use as a token separator
+ * @param fillerToken a String to use to represent gaps in the input stream (due to eg stopwords)
+ */
public FixedShingleFilter(TokenStream input, int shingleSize, String tokenSeparator, String fillerToken) {
super(input);
+ if (shingleSize <= 1) {
+ throw new IllegalArgumentException("shingleSize must be two or greater");
+ }
this.shingleSize = shingleSize;
this.tokenSeparator = tokenSeparator;
-
this.gapToken.termAtt.setEmpty().append(fillerToken);
-
this.currentShingleTokens = new Token[shingleSize];
}