You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rj...@apache.org on 2014/11/03 15:58:26 UTC
svn commit: r1636368 - in /lucene/dev/branches/branch_5x/lucene: ./
analysis/common/src/java/org/apache/lucene/analysis/core/
analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/
analysis/common/src/java/org/apache/lucene/analysis/util/ a...
Author: rjernst
Date: Mon Nov 3 14:58:25 2014
New Revision: 1636368
URL: http://svn.apache.org/r1636368
Log:
LUCENE-6044: Fixed backcompat support for token filters with enablePositionIncrements=false
Added:
lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/Lucene43StopFilter.java (with props)
lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/Lucene43TypeTokenFilter.java (with props)
lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/Lucene43KeepWordFilter.java (with props)
lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/Lucene43LengthFilter.java (with props)
lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/Lucene43TrimFilter.java (with props)
lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/Lucene43FilteringTokenFilter.java (with props)
Modified:
lucene/dev/branches/branch_5x/lucene/CHANGES.txt
lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/StopFilterFactory.java
lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/TypeTokenFilterFactory.java
lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/KeepWordFilterFactory.java
lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LengthFilterFactory.java
lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/TrimFilterFactory.java
lucene/dev/branches/branch_5x/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestStopFilterFactory.java
lucene/dev/branches/branch_5x/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestTypeTokenFilterFactory.java
lucene/dev/branches/branch_5x/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestKeepFilterFactory.java
lucene/dev/branches/branch_5x/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLengthFilterFactory.java
lucene/dev/branches/branch_5x/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestTrimFilterFactory.java
Modified: lucene/dev/branches/branch_5x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/CHANGES.txt?rev=1636368&r1=1636367&r2=1636368&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_5x/lucene/CHANGES.txt Mon Nov 3 14:58:25 2014
@@ -193,6 +193,11 @@ Bug Fixes
* LUCENE-6041: Remove sugar methods FieldInfo.isIndexed and
FieldInfo.hasDocValues. (Robert Muir, Mike McCandless)
+* LUCENE-6044: Fix backcompat support for token filters with enablePositionIncrements=false.
+ Also fixed backcompat for TrimFilter with updateOffsets=true. These options
+ are supported with a match version before 4.4, and no longer valid at all with 5.0.
+ (Ryan Ernst)
+
Documentation
* LUCENE-5392: Add/improve analysis package documentation to reflect
Added: lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/Lucene43StopFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/Lucene43StopFilter.java?rev=1636368&view=auto
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/Lucene43StopFilter.java (added)
+++ lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/Lucene43StopFilter.java Mon Nov 3 14:58:25 2014
@@ -0,0 +1,54 @@
+package org.apache.lucene.analysis.core;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.lucene.analysis.util.FilteringTokenFilter;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.util.CharArraySet;
+import org.apache.lucene.analysis.util.Lucene43FilteringTokenFilter;
+import org.apache.lucene.util.Version;
+
+/**
+ * Backcompat StopFilter for versions 4.3 and before.
+ * @deprecated Use {@link org.apache.lucene.analysis.core.StopFilter}
+ */
+@Deprecated
+public final class Lucene43StopFilter extends Lucene43FilteringTokenFilter {
+
+ private final CharArraySet stopWords;
+ private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+
+ public Lucene43StopFilter(boolean enablePositionIncrements, TokenStream in, CharArraySet stopWords) {
+ super(enablePositionIncrements, in);
+ this.stopWords = stopWords;
+ }
+
+ /**
+ * Returns the next input Token whose term() is not a stop word.
+ */
+ @Override
+ protected boolean accept() throws IOException {
+ return !stopWords.contains(termAtt.buffer(), 0, termAtt.length());
+ }
+
+}
Added: lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/Lucene43TypeTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/Lucene43TypeTokenFilter.java?rev=1636368&view=auto
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/Lucene43TypeTokenFilter.java (added)
+++ lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/Lucene43TypeTokenFilter.java Mon Nov 3 14:58:25 2014
@@ -0,0 +1,52 @@
+package org.apache.lucene.analysis.core;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
+import org.apache.lucene.analysis.util.Lucene43FilteringTokenFilter;
+
+import java.io.IOException;
+import java.util.Set;
+
+/**
+ * Backcompat TypeTokenFilter for versions 4.3 and before.
+ * @deprecated Use {@link org.apache.lucene.analysis.core.TypeTokenFilter}
+ */
+@Deprecated
+public final class Lucene43TypeTokenFilter extends Lucene43FilteringTokenFilter {
+
+ private final Set<String> stopTypes;
+ private final TypeAttribute typeAttribute = addAttribute(TypeAttribute.class);
+ private final boolean useWhiteList;
+
+ public Lucene43TypeTokenFilter(boolean enablePositionIncrements, TokenStream input, Set<String> stopTypes, boolean useWhiteList) {
+ super(enablePositionIncrements, input);
+ this.stopTypes = stopTypes;
+ this.useWhiteList = useWhiteList;
+ }
+
+ /**
+ * By default accept the token if its type is not a stop type.
+ * When the useWhiteList parameter is set to true then accept the token if its type is contained in the stopTypes
+ */
+ @Override
+ protected boolean accept() throws IOException {
+ return useWhiteList == stopTypes.contains(typeAttribute.type());
+ }
+}
Modified: lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/StopFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/StopFilterFactory.java?rev=1636368&r1=1636367&r2=1636368&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/StopFilterFactory.java (original)
+++ lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/StopFilterFactory.java Mon Nov 3 14:58:25 2014
@@ -23,6 +23,7 @@ import org.apache.lucene.analysis.util.R
import org.apache.lucene.analysis.util.ResourceLoaderAware;
import org.apache.lucene.analysis.util.TokenFilterFactory;
import org.apache.lucene.analysis.util.WordlistLoader; // jdocs
+import org.apache.lucene.util.Version;
import java.util.Map;
import java.io.IOException;
@@ -77,6 +78,7 @@ public class StopFilterFactory extends T
private final String stopWordFiles;
private final String format;
private final boolean ignoreCase;
+ private boolean enablePositionIncrements;
/** Creates a new StopFilterFactory */
public StopFilterFactory(Map<String,String> args) {
@@ -84,6 +86,17 @@ public class StopFilterFactory extends T
stopWordFiles = get(args, "words");
format = get(args, "format", (null == stopWordFiles ? null : FORMAT_WORDSET));
ignoreCase = getBoolean(args, "ignoreCase", false);
+
+ if (luceneMatchVersion.onOrAfter(Version.LUCENE_5_0_0) == false) {
+ boolean defaultValue = luceneMatchVersion.onOrAfter(Version.LUCENE_4_4_0);
+ enablePositionIncrements = getBoolean(args, "enablePositionIncrements", defaultValue);
+ if (enablePositionIncrements == false && luceneMatchVersion.onOrAfter(Version.LUCENE_4_4_0)) {
+ throw new IllegalArgumentException("enablePositionIncrements=false is not supported anymore as of Lucene 4.4");
+ }
+ } else if (args.containsKey("enablePositionIncrements")) {
+ throw new IllegalArgumentException("enablePositionIncrements is not a valid option as of Lucene 5.0");
+ }
+
if (!args.isEmpty()) {
throw new IllegalArgumentException("Unknown parameters: " + args);
}
@@ -117,7 +130,12 @@ public class StopFilterFactory extends T
@Override
public TokenStream create(TokenStream input) {
- StopFilter stopFilter = new StopFilter(input,stopWords);
- return stopFilter;
+ if (luceneMatchVersion.onOrAfter(Version.LUCENE_4_4_0)) {
+ return new StopFilter(input, stopWords);
+ } else {
+ @SuppressWarnings("deprecation")
+ final TokenStream filter = new Lucene43StopFilter(enablePositionIncrements, input, stopWords);
+ return filter;
+ }
}
}
Modified: lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/TypeTokenFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/TypeTokenFilterFactory.java?rev=1636368&r1=1636367&r2=1636368&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/TypeTokenFilterFactory.java (original)
+++ lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/TypeTokenFilterFactory.java Mon Nov 3 14:58:25 2014
@@ -21,6 +21,7 @@ import org.apache.lucene.analysis.TokenS
import org.apache.lucene.analysis.util.ResourceLoader;
import org.apache.lucene.analysis.util.ResourceLoaderAware;
import org.apache.lucene.analysis.util.TokenFilterFactory;
+import org.apache.lucene.util.Version;
import java.io.IOException;
import java.util.HashSet;
@@ -43,12 +44,24 @@ public class TypeTokenFilterFactory exte
private final boolean useWhitelist;
private final String stopTypesFiles;
private Set<String> stopTypes;
+ private boolean enablePositionIncrements;
/** Creates a new TypeTokenFilterFactory */
public TypeTokenFilterFactory(Map<String,String> args) {
super(args);
stopTypesFiles = require(args, "types");
useWhitelist = getBoolean(args, "useWhitelist", false);
+
+ if (luceneMatchVersion.onOrAfter(Version.LUCENE_5_0_0) == false) {
+ boolean defaultValue = luceneMatchVersion.onOrAfter(Version.LUCENE_4_4_0);
+ enablePositionIncrements = getBoolean(args, "enablePositionIncrements", defaultValue);
+ if (enablePositionIncrements == false && luceneMatchVersion.onOrAfter(Version.LUCENE_4_4_0)) {
+ throw new IllegalArgumentException("enablePositionIncrements=false is not supported anymore as of Lucene 4.4");
+ }
+ } else if (args.containsKey("enablePositionIncrements")) {
+ throw new IllegalArgumentException("enablePositionIncrements is not a valid option as of Lucene 5.0");
+ }
+
if (!args.isEmpty()) {
throw new IllegalArgumentException("Unknown parameters: " + args);
}
@@ -72,7 +85,12 @@ public class TypeTokenFilterFactory exte
@Override
public TokenStream create(TokenStream input) {
- final TokenStream filter = new TypeTokenFilter(input, stopTypes, useWhitelist);
- return filter;
+ if (luceneMatchVersion.onOrAfter(Version.LUCENE_4_4_0)) {
+ return new TypeTokenFilter(input, stopTypes, useWhitelist);
+ } else {
+ @SuppressWarnings("deprecation")
+ final TokenStream filter = new Lucene43TypeTokenFilter(enablePositionIncrements, input, stopTypes, useWhitelist);
+ return filter;
+ }
}
}
Modified: lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/KeepWordFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/KeepWordFilterFactory.java?rev=1636368&r1=1636367&r2=1636368&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/KeepWordFilterFactory.java (original)
+++ lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/KeepWordFilterFactory.java Mon Nov 3 14:58:25 2014
@@ -22,6 +22,7 @@ import org.apache.lucene.analysis.util.C
import org.apache.lucene.analysis.util.ResourceLoader;
import org.apache.lucene.analysis.util.ResourceLoaderAware;
import org.apache.lucene.analysis.util.TokenFilterFactory;
+import org.apache.lucene.util.Version;
import java.util.Map;
import java.io.IOException;
@@ -40,12 +41,24 @@ public class KeepWordFilterFactory exten
private final boolean ignoreCase;
private final String wordFiles;
private CharArraySet words;
+ private boolean enablePositionIncrements;
/** Creates a new KeepWordFilterFactory */
public KeepWordFilterFactory(Map<String,String> args) {
super(args);
wordFiles = get(args, "words");
ignoreCase = getBoolean(args, "ignoreCase", false);
+
+ if (luceneMatchVersion.onOrAfter(Version.LUCENE_5_0_0) == false) {
+ boolean defaultValue = luceneMatchVersion.onOrAfter(Version.LUCENE_4_4_0);
+ enablePositionIncrements = getBoolean(args, "enablePositionIncrements", defaultValue);
+ if (enablePositionIncrements == false && luceneMatchVersion.onOrAfter(Version.LUCENE_4_4_0)) {
+ throw new IllegalArgumentException("enablePositionIncrements=false is not supported anymore as of Lucene 4.4");
+ }
+ } else if (args.containsKey("enablePositionIncrements")) {
+ throw new IllegalArgumentException("enablePositionIncrements is not a valid option as of Lucene 5.0");
+ }
+
if (!args.isEmpty()) {
throw new IllegalArgumentException("Unknown parameters: " + args);
}
@@ -71,8 +84,11 @@ public class KeepWordFilterFactory exten
// if the set is null, it means it was empty
if (words == null) {
return input;
+ } else if (luceneMatchVersion.onOrAfter(Version.LUCENE_4_4_0)) {
+ return new KeepWordFilter(input, words);
} else {
- final TokenStream filter = new KeepWordFilter(input, words);
+ @SuppressWarnings("deprecation")
+ final TokenStream filter = new Lucene43KeepWordFilter(enablePositionIncrements, input, words);
return filter;
}
}
Modified: lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LengthFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LengthFilterFactory.java?rev=1636368&r1=1636367&r2=1636368&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LengthFilterFactory.java (original)
+++ lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LengthFilterFactory.java Mon Nov 3 14:58:25 2014
@@ -19,8 +19,10 @@ package org.apache.lucene.analysis.misce
import java.util.Map;
+import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.util.TokenFilterFactory;
+import org.apache.lucene.util.Version;
/**
* Factory for {@link LengthFilter}.
@@ -37,20 +39,37 @@ public class LengthFilterFactory extends
final int max;
public static final String MIN_KEY = "min";
public static final String MAX_KEY = "max";
+ private boolean enablePositionIncrements;
/** Creates a new LengthFilterFactory */
public LengthFilterFactory(Map<String, String> args) {
super(args);
min = requireInt(args, MIN_KEY);
max = requireInt(args, MAX_KEY);
+
+ if (luceneMatchVersion.onOrAfter(Version.LUCENE_5_0_0) == false) {
+ boolean defaultValue = luceneMatchVersion.onOrAfter(Version.LUCENE_4_4_0);
+ enablePositionIncrements = getBoolean(args, "enablePositionIncrements", defaultValue);
+ if (enablePositionIncrements == false && luceneMatchVersion.onOrAfter(Version.LUCENE_4_4_0)) {
+ throw new IllegalArgumentException("enablePositionIncrements=false is not supported anymore as of Lucene 4.4");
+ }
+ } else if (args.containsKey("enablePositionIncrements")) {
+ throw new IllegalArgumentException("enablePositionIncrements is not a valid option as of Lucene 5.0");
+ }
+
if (!args.isEmpty()) {
throw new IllegalArgumentException("Unknown parameters: " + args);
}
}
@Override
- public LengthFilter create(TokenStream input) {
- final LengthFilter filter = new LengthFilter(input,min,max);
- return filter;
+ public TokenFilter create(TokenStream input) {
+ if (luceneMatchVersion.onOrAfter(Version.LUCENE_4_4_0)) {
+ return new LengthFilter(input, min, max);
+ } else {
+ @SuppressWarnings("deprecation")
+ final TokenFilter filter = new Lucene43LengthFilter(enablePositionIncrements, input, min, max);
+ return filter;
+ }
}
}
Added: lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/Lucene43KeepWordFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/Lucene43KeepWordFilter.java?rev=1636368&view=auto
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/Lucene43KeepWordFilter.java (added)
+++ lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/Lucene43KeepWordFilter.java Mon Nov 3 14:58:25 2014
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.analysis.miscellaneous;
+
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.util.CharArraySet;
+import org.apache.lucene.analysis.util.Lucene43FilteringTokenFilter;
+
+/**
+ * Backcompat for KeepWordFilter for versions 4.3 and before.
+ * @deprecated Use {@link org.apache.lucene.analysis.miscellaneous.KeepWordFilter}
+ */
+@Deprecated
+public final class Lucene43KeepWordFilter extends Lucene43FilteringTokenFilter {
+ private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+ private final CharArraySet words;
+
+ /** The words set passed to this constructor will be directly used by this filter
+ * and should not be modified, */
+ public Lucene43KeepWordFilter(boolean enablePositionIncrements, TokenStream in, CharArraySet words) {
+ super(enablePositionIncrements, in);
+ this.words = words;
+ }
+
+ public boolean accept() {
+ return words.contains(termAtt.buffer(), 0, termAtt.length());
+ }
+}
Added: lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/Lucene43LengthFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/Lucene43LengthFilter.java?rev=1636368&view=auto
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/Lucene43LengthFilter.java (added)
+++ lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/Lucene43LengthFilter.java Mon Nov 3 14:58:25 2014
@@ -0,0 +1,53 @@
+package org.apache.lucene.analysis.miscellaneous;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.util.Lucene43FilteringTokenFilter;
+
+/**
+ * Backcompat LengthFilter for versions 4.3 and before.
+ * @deprecated Use {@link org.apache.lucene.analysis.miscellaneous.LengthFilter}
+ */
+@Deprecated
+public final class Lucene43LengthFilter extends Lucene43FilteringTokenFilter {
+
+ private final int min;
+ private final int max;
+
+ private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+
+ /**
+ * Build a filter that removes words that are too long or too
+ * short from the text.
+ */
+ public Lucene43LengthFilter(boolean enablePositionIncrements, TokenStream in, int min, int max) {
+ super(enablePositionIncrements, in);
+ this.min = min;
+ this.max = max;
+ }
+
+ @Override
+ public boolean accept() throws IOException {
+ final int len = termAtt.length();
+ return (len >= min && len <= max);
+ }
+}
Added: lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/Lucene43TrimFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/Lucene43TrimFilter.java?rev=1636368&view=auto
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/Lucene43TrimFilter.java (added)
+++ lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/Lucene43TrimFilter.java Mon Nov 3 14:58:25 2014
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.analysis.miscellaneous;
+
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+
+import java.io.IOException;
+
+/**
+ * Backcompat TrimFilter for versions 4.3 and before.
+ * @deprecated Use {@link org.apache.lucene.analysis.miscellaneous.TrimFilter}
+ */
+@Deprecated
+public final class Lucene43TrimFilter extends TokenFilter {
+
+ final boolean updateOffsets;
+ private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+ private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
+
+
+ public Lucene43TrimFilter(TokenStream in, boolean updateOffsets) {
+ super(in);
+ this.updateOffsets = updateOffsets;
+ }
+
+ @Override
+ public boolean incrementToken() throws IOException {
+ if (!input.incrementToken()) return false;
+
+ char[] termBuffer = termAtt.buffer();
+ int len = termAtt.length();
+ //TODO: Is this the right behavior or should we return false? Currently, " ", returns true, so I think this should
+ //also return true
+ if (len == 0){
+ return true;
+ }
+ int start = 0;
+ int end = 0;
+ int endOff = 0;
+
+ // eat the first characters
+ //QUESTION: Should we use Character.isWhitespace() instead?
+ for (start = 0; start < len && termBuffer[start] <= ' '; start++) {
+ }
+ // eat the end characters
+ for (end = len; end >= start && termBuffer[end - 1] <= ' '; end--) {
+ endOff++;
+ }
+ if (start > 0 || end < len) {
+ if (start < end) {
+ termAtt.copyBuffer(termBuffer, start, (end - start));
+ } else {
+ termAtt.setEmpty();
+ }
+ if (updateOffsets && len == offsetAtt.endOffset() - offsetAtt.startOffset()) {
+ int newStart = offsetAtt.startOffset()+start;
+ int newEnd = offsetAtt.endOffset() - (start<end ? endOff:0);
+ offsetAtt.setOffset(newStart, newEnd);
+ }
+ }
+
+ return true;
+ }
+}
Modified: lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/TrimFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/TrimFilterFactory.java?rev=1636368&r1=1636367&r2=1636368&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/TrimFilterFactory.java (original)
+++ lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/TrimFilterFactory.java Mon Nov 3 14:58:25 2014
@@ -19,9 +19,10 @@ package org.apache.lucene.analysis.misce
import java.util.Map;
+import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.miscellaneous.TrimFilter;
import org.apache.lucene.analysis.util.TokenFilterFactory;
+import org.apache.lucene.util.Version;
/**
* Factory for {@link TrimFilter}.
@@ -37,17 +38,34 @@ import org.apache.lucene.analysis.util.T
*/
public class TrimFilterFactory extends TokenFilterFactory {
+ private boolean updateOffsets;
+
/** Creates a new TrimFilterFactory */
public TrimFilterFactory(Map<String,String> args) {
super(args);
+
+ if (luceneMatchVersion.onOrAfter(Version.LUCENE_5_0_0) == false) {
+ updateOffsets = getBoolean(args, "updateOffsets", false);
+ if (updateOffsets && luceneMatchVersion.onOrAfter(Version.LUCENE_4_4_0)) {
+ throw new IllegalArgumentException("updateOffsets=true is not supported anymore as of Lucene 4.4");
+ }
+ } else if (args.containsKey("updateOffsets")) {
+ throw new IllegalArgumentException("updateOffsets is not a valid option as of Lucene 5.0");
+ }
+
if (!args.isEmpty()) {
throw new IllegalArgumentException("Unknown parameters: " + args);
}
}
@Override
- public TrimFilter create(TokenStream input) {
- final TrimFilter filter = new TrimFilter(input);
- return filter;
+ public TokenFilter create(TokenStream input) {
+ if (luceneMatchVersion.onOrAfter(Version.LUCENE_4_4_0)) {
+ return new TrimFilter(input);
+ } else {
+ @SuppressWarnings("deprecation")
+ final Lucene43TrimFilter filter = new Lucene43TrimFilter(input, updateOffsets);
+ return filter;
+ }
}
}
Added: lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/Lucene43FilteringTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/Lucene43FilteringTokenFilter.java?rev=1636368&view=auto
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/Lucene43FilteringTokenFilter.java (added)
+++ lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/Lucene43FilteringTokenFilter.java Mon Nov 3 14:58:25 2014
@@ -0,0 +1,81 @@
+package org.apache.lucene.analysis.util;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
+
+/**
+ * Backcompat FilteringTokenFilter for versions 4.3 and before.
+ * @deprecated Use {@link org.apache.lucene.analysis.util.FilteringTokenFilter}
+ */
+@Deprecated
+public abstract class Lucene43FilteringTokenFilter extends TokenFilter {
+
+ private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
+ private boolean enablePositionIncrements; // no init needed, as ctor enforces setting value!
+ private boolean first = true; // only used when not preserving gaps
+
+ public Lucene43FilteringTokenFilter(boolean enablePositionIncrements, TokenStream input){
+ super(input);
+ this.enablePositionIncrements = enablePositionIncrements;
+ }
+
+ /** Override this method and return if the current input token should be returned by {@link #incrementToken}. */
+ protected abstract boolean accept() throws IOException;
+
+ @Override
+ public final boolean incrementToken() throws IOException {
+ if (enablePositionIncrements) {
+ int skippedPositions = 0;
+ while (input.incrementToken()) {
+ if (accept()) {
+ if (skippedPositions != 0) {
+ posIncrAtt.setPositionIncrement(posIncrAtt.getPositionIncrement() + skippedPositions);
+ }
+ return true;
+ }
+ skippedPositions += posIncrAtt.getPositionIncrement();
+ }
+ } else {
+ while (input.incrementToken()) {
+ if (accept()) {
+ if (first) {
+ // first token having posinc=0 is illegal.
+ if (posIncrAtt.getPositionIncrement() == 0) {
+ posIncrAtt.setPositionIncrement(1);
+ }
+ first = false;
+ }
+ return true;
+ }
+ }
+ }
+ // reached EOS -- return false
+ return false;
+ }
+
+ @Override
+ public void reset() throws IOException {
+ super.reset();
+ first = true;
+ }
+}
Modified: lucene/dev/branches/branch_5x/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestStopFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestStopFilterFactory.java?rev=1636368&r1=1636367&r2=1636368&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestStopFilterFactory.java (original)
+++ lucene/dev/branches/branch_5x/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestStopFilterFactory.java Mon Nov 3 14:58:25 2014
@@ -17,10 +17,15 @@ package org.apache.lucene.analysis.core;
* limitations under the License.
*/
+import java.io.Reader;
+import java.io.StringReader;
+
+import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
import org.apache.lucene.analysis.util.CharArraySet;
import org.apache.lucene.analysis.util.ClasspathResourceLoader;
import org.apache.lucene.analysis.util.ResourceLoader;
+import org.apache.lucene.util.Version;
public class TestStopFilterFactory extends BaseTokenStreamFactoryTestCase {
@@ -99,4 +104,29 @@ public class TestStopFilterFactory exten
assertTrue(msg, msg.contains("bogus"));
}
}
+
+ public void test43Backcompat() throws Exception {
+ Reader reader = new StringReader("foo bar");
+ TokenStream stream = whitespaceMockTokenizer(reader);
+ stream = tokenFilterFactory("Stop", Version.LUCENE_4_3_1,
+ "enablePositionIncrements", "false",
+ "words", "stop-2.txt").create(stream);
+ assertTrue(stream instanceof Lucene43StopFilter);
+ assertTokenStreamContents(stream, new String[] {"foo", "bar"}, new int[] {0, 4}, new int[] {3, 7}, new int[] {1, 1});
+
+ try {
+ tokenFilterFactory("Stop", Version.LUCENE_4_4_0, "enablePositionIncrements", "false", "words", "stop-2.txt");
+ fail();
+ } catch (IllegalArgumentException expected) {
+ assertTrue(expected.getMessage().contains("enablePositionIncrements=false is not supported"));
+ }
+ tokenFilterFactory("Stop", Version.LUCENE_4_4_0, "enablePositionIncrements", "true", "words", "stop-2.txt");
+
+ try {
+ tokenFilterFactory("Stop", "enablePositionIncrements", "true", "words", "stop-2.txt");
+ fail();
+ } catch (IllegalArgumentException expected) {
+ assertTrue(expected.getMessage().contains("not a valid option"));
+ }
+ }
}
Modified: lucene/dev/branches/branch_5x/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestTypeTokenFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestTypeTokenFilterFactory.java?rev=1636368&r1=1636367&r2=1636368&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestTypeTokenFilterFactory.java (original)
+++ lucene/dev/branches/branch_5x/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestTypeTokenFilterFactory.java Mon Nov 3 14:58:25 2014
@@ -18,9 +18,13 @@ package org.apache.lucene.analysis.core;
*/
import org.apache.lucene.analysis.NumericTokenStream;
+import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
import org.apache.lucene.analysis.util.TokenFilterFactory;
+import org.apache.lucene.util.Version;
+import java.io.Reader;
+import java.io.StringReader;
import java.util.Set;
/**
@@ -80,4 +84,29 @@ public class TestTypeTokenFilterFactory
assertTrue(expected.getMessage().contains("Unknown parameters"));
}
}
+
+ public void test43Backcompat() throws Exception {
+ Reader reader = new StringReader("foo bar");
+ TokenStream stream = whitespaceMockTokenizer(reader);
+ stream = tokenFilterFactory("Type", Version.LUCENE_4_3_1,
+ "enablePositionIncrements", "false",
+ "types", "stoptypes-1.txt").create(stream);
+ assertTrue(stream instanceof Lucene43TypeTokenFilter);
+ assertTokenStreamContents(stream, new String[] {"foo", "bar"}, new int[] {0, 4}, new int[] {3, 7}, new int[] {1, 1});
+
+ try {
+ tokenFilterFactory("Type", Version.LUCENE_4_4_0, "enablePositionIncrements", "false", "types", "stoptypes-1.txt");
+ fail();
+ } catch (IllegalArgumentException expected) {
+ assertTrue(expected.getMessage().contains("enablePositionIncrements=false is not supported"));
+ }
+ tokenFilterFactory("Type", Version.LUCENE_4_4_0, "enablePositionIncrements", "true", "types", "stoptypes-1.txt");
+
+ try {
+ tokenFilterFactory("Type", "enablePositionIncrements", "true", "types", "stoptypes-1.txt");
+ fail();
+ } catch (IllegalArgumentException expected) {
+ assertTrue(expected.getMessage().contains("not a valid option"));
+ }
+ }
}
Modified: lucene/dev/branches/branch_5x/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestKeepFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestKeepFilterFactory.java?rev=1636368&r1=1636367&r2=1636368&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestKeepFilterFactory.java (original)
+++ lucene/dev/branches/branch_5x/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestKeepFilterFactory.java Mon Nov 3 14:58:25 2014
@@ -17,10 +17,15 @@ package org.apache.lucene.analysis.misce
* limitations under the License.
*/
+import java.io.Reader;
+import java.io.StringReader;
+
+import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
import org.apache.lucene.analysis.util.CharArraySet;
import org.apache.lucene.analysis.util.ClasspathResourceLoader;
import org.apache.lucene.analysis.util.ResourceLoader;
+import org.apache.lucene.util.Version;
public class TestKeepFilterFactory extends BaseTokenStreamFactoryTestCase {
@@ -51,4 +56,29 @@ public class TestKeepFilterFactory exten
assertTrue(expected.getMessage().contains("Unknown parameters"));
}
}
+
+ public void test43Backcompat() throws Exception {
+ Reader reader = new StringReader("a foo bar");
+ TokenStream stream = whitespaceMockTokenizer(reader);
+ stream = tokenFilterFactory("KeepWord", Version.LUCENE_4_3_1,
+ "enablePositionIncrements", "false",
+ "words", "keep-1.txt").create(stream);
+ assertTrue(stream instanceof Lucene43KeepWordFilter);
+ assertTokenStreamContents(stream, new String[] {"foo", "bar"}, new int[] {2, 6}, new int[] {5, 9}, new int[] {1, 1});
+
+ try {
+ tokenFilterFactory("KeepWord", Version.LUCENE_4_4_0, "enablePositionIncrements", "false");
+ fail();
+ } catch (IllegalArgumentException expected) {
+ assertTrue(expected.getMessage().contains("enablePositionIncrements=false is not supported"));
+ }
+ tokenFilterFactory("KeepWord", Version.LUCENE_4_4_0, "enablePositionIncrements", "true");
+
+ try {
+ tokenFilterFactory("KeepWord", "enablePositionIncrements", "true");
+ fail();
+ } catch (IllegalArgumentException expected) {
+ assertTrue(expected.getMessage().contains("not a valid option"));
+ }
+ }
}
\ No newline at end of file
Modified: lucene/dev/branches/branch_5x/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLengthFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLengthFilterFactory.java?rev=1636368&r1=1636367&r2=1636368&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLengthFilterFactory.java (original)
+++ lucene/dev/branches/branch_5x/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLengthFilterFactory.java Mon Nov 3 14:58:25 2014
@@ -24,6 +24,7 @@ import org.apache.lucene.analysis.MockTo
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
+import org.apache.lucene.util.Version;
public class TestLengthFilterFactory extends BaseTokenStreamFactoryTestCase {
@@ -64,4 +65,32 @@ public class TestLengthFilterFactory ext
assertTrue(expected.getMessage().contains("maximum length must not be greater than minimum length"));
}
}
+
+ public void test43Backcompat() throws Exception {
+ Reader reader = new StringReader("foo bar");
+ TokenStream stream = whitespaceMockTokenizer(reader);
+ stream = tokenFilterFactory("Length", Version.LUCENE_4_3_1,
+ "enablePositionIncrements", "false",
+ LengthFilterFactory.MIN_KEY, "2", LengthFilterFactory.MAX_KEY, "5").create(stream);
+ assertTrue(stream instanceof Lucene43LengthFilter);
+ assertTokenStreamContents(stream, new String[] {"foo", "bar"}, new int[] {0, 4}, new int[] {3, 7}, new int[] {1, 1});
+
+ try {
+ tokenFilterFactory("Length", Version.LUCENE_4_4_0, "enablePositionIncrements", "false",
+ LengthFilterFactory.MIN_KEY, "2", LengthFilterFactory.MAX_KEY, "5");
+ fail();
+ } catch (IllegalArgumentException expected) {
+ assertTrue(expected.getMessage().contains("enablePositionIncrements=false is not supported"));
+ }
+ tokenFilterFactory("Length", Version.LUCENE_4_4_0, "enablePositionIncrements", "true",
+ LengthFilterFactory.MIN_KEY, "2", LengthFilterFactory.MAX_KEY, "5");
+
+ try {
+ tokenFilterFactory("Length", "enablePositionIncrements", "true",
+ LengthFilterFactory.MIN_KEY, "2", LengthFilterFactory.MAX_KEY, "5");
+ fail();
+ } catch (IllegalArgumentException expected) {
+ assertTrue(expected.getMessage().contains("not a valid option"));
+ }
+ }
}
\ No newline at end of file
Modified: lucene/dev/branches/branch_5x/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestTrimFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestTrimFilterFactory.java?rev=1636368&r1=1636367&r2=1636368&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestTrimFilterFactory.java (original)
+++ lucene/dev/branches/branch_5x/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestTrimFilterFactory.java Mon Nov 3 14:58:25 2014
@@ -20,9 +20,9 @@ package org.apache.lucene.analysis.misce
import java.io.Reader;
import java.io.StringReader;
-import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
+import org.apache.lucene.util.Version;
/**
* Simple tests to ensure this factory is working
@@ -44,4 +44,27 @@ public class TestTrimFilterFactory exten
assertTrue(expected.getMessage().contains("Unknown parameters"));
}
}
+
+ public void test43Backcompat() throws Exception {
+ Reader reader = new StringReader(" foo ");
+ TokenStream stream = keywordMockTokenizer(reader);
+ stream = tokenFilterFactory("Trim", Version.LUCENE_4_3_1, "updateOffsets", "true").create(stream);
+ assertTrue(stream instanceof Lucene43TrimFilter);
+ assertTokenStreamContents(stream, new String[] {"foo"}, new int[] {2}, new int[] {5});
+
+ try {
+ tokenFilterFactory("Trim", Version.LUCENE_4_4_0, "updateOffsets", "true");
+ fail();
+ } catch (IllegalArgumentException expected) {
+ assertTrue(expected.getMessage().contains("updateOffsets=true is not supported"));
+ }
+ tokenFilterFactory("Trim", Version.LUCENE_4_4_0, "updateOffsets", "false");
+
+ try {
+ tokenFilterFactory("Trim", "updateOffsets", "false");
+ fail();
+ } catch (IllegalArgumentException expected) {
+ assertTrue(expected.getMessage().contains("not a valid option"));
+ }
+ }
}