You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2021/09/02 16:00:19 UTC
[lucene] branch main updated: LUCENE-10083: Analyzer and stemmer
for Telugu language (#275)
This is an automated email from the ASF dual-hosted git repository.
rmuir pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/main by this push:
new 544dbbe LUCENE-10083: Analyzer and stemmer for Telugu language (#275)
544dbbe is described below
commit 544dbbea46878b49be5c4c4e82d9b06a5679c891
Author: vinodrenu <11...@users.noreply.github.com>
AuthorDate: Thu Sep 2 09:00:13 2021 -0700
LUCENE-10083: Analyzer and stemmer for Telugu language (#275)
* initial version of Telugu analyzer
* made entries for factories and added few more terms in stemmer
* added two more terms
* added few mote terms
* added long to short vowel conversion
* added test cases
* applied code formatting rules
* fixed unclosed p tag in javadoc
* spotlessApply removed the closing p tag
---
.../apache/lucene/analysis/te/TeluguAnalyzer.java | 128 +++++++++++++++++++++
.../analysis/te/TeluguNormalizationFilter.java | 55 +++++++++
.../te/TeluguNormalizationFilterFactory.java | 64 +++++++++++
.../lucene/analysis/te/TeluguNormalizer.java | 116 +++++++++++++++++++
.../lucene/analysis/te/TeluguStemFilter.java | 50 ++++++++
.../analysis/te/TeluguStemFilterFactory.java | 58 ++++++++++
.../apache/lucene/analysis/te/TeluguStemmer.java | 64 +++++++++++
.../apache/lucene/analysis/te/package-info.java | 19 +++
.../org.apache.lucene.analysis.TokenFilterFactory | 2 +
.../org/apache/lucene/analysis/te/stopwords.txt | 51 ++++++++
.../lucene/analysis/te/TestTeluguAnalyzer.java | 57 +++++++++
.../lucene/analysis/te/TestTeluguFilters.java | 79 +++++++++++++
.../lucene/analysis/te/TestTeluguNormalizer.java | 60 ++++++++++
.../lucene/analysis/te/TestTeluguStemmer.java | 59 ++++++++++
14 files changed, 862 insertions(+)
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/te/TeluguAnalyzer.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/te/TeluguAnalyzer.java
new file mode 100644
index 0000000..f0e9b4e
--- /dev/null
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/te/TeluguAnalyzer.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.analysis.te;
+
+import java.io.IOException;
+import java.io.Reader;
+import java.io.UncheckedIOException;
+import org.apache.lucene.analysis.CharArraySet;
+import org.apache.lucene.analysis.StopFilter;
+import org.apache.lucene.analysis.StopwordAnalyzerBase;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.core.DecimalDigitFilter;
+import org.apache.lucene.analysis.in.IndicNormalizationFilter;
+import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;
+import org.apache.lucene.analysis.standard.StandardTokenizer;
+
+/**
+ * Analyzer for Telugu.
+ *
+ * @since 9.0.0
+ */
+public final class TeluguAnalyzer extends StopwordAnalyzerBase {
+ private final CharArraySet stemExclusionSet;
+
+ /** File containing default Telugu stopwords. */
+ public static final String DEFAULT_STOPWORD_FILE = "stopwords.txt";
+
+ private static final String STOPWORDS_COMMENT = "#";
+
+ /**
+ * Returns an unmodifiable instance of the default stop-words set.
+ *
+ * @return an unmodifiable instance of the default stop-words set.
+ */
+ public static CharArraySet getDefaultStopSet() {
+ return DefaultSetHolder.DEFAULT_STOP_SET;
+ }
+
+ /**
+ * Atomically loads the DEFAULT_STOP_SET in a lazy fashion once the outer class accesses the
+ * static final set the first time.;
+ */
+ private static class DefaultSetHolder {
+ static final CharArraySet DEFAULT_STOP_SET;
+
+ static {
+ try {
+ DEFAULT_STOP_SET =
+ loadStopwordSet(false, TeluguAnalyzer.class, DEFAULT_STOPWORD_FILE, STOPWORDS_COMMENT);
+ } catch (IOException ex) {
+ // default set should always be present as it is part of the
+ throw new UncheckedIOException("Unable to load default stopword set", ex);
+ }
+ }
+ }
+
+ /**
+ * Builds an analyzer with the given stop words
+ *
+ * @param stopwords a stopword set
+ * @param stemExclusionSet a stemming exclusion set
+ */
+ public TeluguAnalyzer(CharArraySet stopwords, CharArraySet stemExclusionSet) {
+ super(stopwords);
+ this.stemExclusionSet = CharArraySet.unmodifiableSet(CharArraySet.copy(stemExclusionSet));
+ }
+
+ /**
+ * Builds an analyzer with the given stop words
+ *
+ * @param stopwords a stopword set
+ */
+ public TeluguAnalyzer(CharArraySet stopwords) {
+ this(stopwords, CharArraySet.EMPTY_SET);
+ }
+
+ /** Builds an analyzer with the default stop words: {@link #DEFAULT_STOPWORD_FILE}. */
+ public TeluguAnalyzer() {
+ this(DefaultSetHolder.DEFAULT_STOP_SET);
+ }
+
+ /**
+ * Creates {@link org.apache.lucene.analysis.Analyzer.TokenStreamComponents} used to tokenize all
+ * the text in the provided {@link Reader}.
+ *
+ * @return {@link org.apache.lucene.analysis.Analyzer.TokenStreamComponents} built from a {@link
+ * StandardTokenizer} filtered with {@link DecimalDigitFilter}, {@link
+ * IndicNormalizationFilter}, {@link TeluguNormalizationFilter}, {@link
+ * SetKeywordMarkerFilter} if a stem exclusion set is provided, {@link TeluguStemFilter}, and
+ * Telugu Stop words
+ */
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName) {
+ final Tokenizer source = new StandardTokenizer();
+ TokenStream result = new DecimalDigitFilter(source);
+ if (!stemExclusionSet.isEmpty()) {
+ result = new SetKeywordMarkerFilter(result, stemExclusionSet);
+ }
+ result = new IndicNormalizationFilter(result);
+ result = new TeluguNormalizationFilter(result);
+ result = new StopFilter(result, stopwords);
+ result = new TeluguStemFilter(result);
+ return new TokenStreamComponents(source, result);
+ }
+
+ @Override
+ protected TokenStream normalize(String fieldName, TokenStream in) {
+ TokenStream result = new DecimalDigitFilter(in);
+ result = new IndicNormalizationFilter(in);
+ result = new TeluguNormalizationFilter(in);
+ return result;
+ }
+}
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/te/TeluguNormalizationFilter.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/te/TeluguNormalizationFilter.java
new file mode 100644
index 0000000..9d3e089
--- /dev/null
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/te/TeluguNormalizationFilter.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.analysis.te;
+
+import java.io.IOException;
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.KeywordAttribute;
+
+/**
+ * A {@link TokenFilter} that applies {@link TeluguNormalizer} to normalize the orthography.
+ *
+ * <p>In some cases the normalization may cause unrelated terms to conflate, so to prevent terms
+ * from being normalized use an instance of {@link SetKeywordMarkerFilter} or a custom {@link
+ * TokenFilter} that sets the {@link KeywordAttribute} before this {@link TokenStream}.
+ *
+ * @since 9.0.0
+ * @see TeluguNormalizer
+ */
+public final class TeluguNormalizationFilter extends TokenFilter {
+
+ private final TeluguNormalizer normalizer = new TeluguNormalizer();
+ private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+ private final KeywordAttribute keywordAtt = addAttribute(KeywordAttribute.class);
+
+ public TeluguNormalizationFilter(TokenStream input) {
+ super(input);
+ }
+
+ @Override
+ public boolean incrementToken() throws IOException {
+ if (input.incrementToken()) {
+ if (!keywordAtt.isKeyword())
+ termAtt.setLength(normalizer.normalize(termAtt.buffer(), termAtt.length()));
+ return true;
+ }
+ return false;
+ }
+}
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/te/TeluguNormalizationFilterFactory.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/te/TeluguNormalizationFilterFactory.java
new file mode 100644
index 0000000..8fa05ef
--- /dev/null
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/te/TeluguNormalizationFilterFactory.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.analysis.te;
+
+import java.util.Map;
+import org.apache.lucene.analysis.TokenFilterFactory;
+import org.apache.lucene.analysis.TokenStream;
+
+/**
+ * Factory for {@link TeluguNormalizationFilter}.
+ *
+ * <pre class="prettyprint">
+ * <fieldType name="text_tenormal" class="solr.TextField" positionIncrementGap="100">
+ * <analyzer>
+ * <tokenizer class="solr.StandardTokenizerFactory"/>
+ * <filter class="solr.TeluguNormalizationFilterFactory"/>
+ * </analyzer>
+ * </fieldType></pre>
+ *
+ * @since 9.0.0
+ * @lucene.spi {@value #NAME}
+ */
+public class TeluguNormalizationFilterFactory extends TokenFilterFactory {
+
+ /** SPI name */
+ public static final String NAME = "teluguNormalization";
+
+ /** Creates a new TeluguNormalizationFilterFactory */
+ public TeluguNormalizationFilterFactory(Map<String, String> args) {
+ super(args);
+ if (!args.isEmpty()) {
+ throw new IllegalArgumentException("Unknown parameters: " + args);
+ }
+ }
+
+ /** Default ctor for compatibility with SPI */
+ public TeluguNormalizationFilterFactory() {
+ throw defaultCtorException();
+ }
+
+ @Override
+ public TokenStream create(TokenStream input) {
+ return new TeluguNormalizationFilter(input);
+ }
+
+ @Override
+ public TokenStream normalize(TokenStream input) {
+ return create(input);
+ }
+}
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/te/TeluguNormalizer.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/te/TeluguNormalizer.java
new file mode 100644
index 0000000..115d44d
--- /dev/null
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/te/TeluguNormalizer.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.analysis.te;
+
+import static org.apache.lucene.analysis.util.StemmerUtil.delete;
+
+/**
+ * Normalizer for Telugu.
+ *
+ * <p>Normalizes text to remove some differences in spelling variations.
+ *
+ * @since 9.0.0
+ */
+public class TeluguNormalizer {
+
+ /**
+ * Normalize an input buffer of Telugu text
+ *
+ * @param s input buffer
+ * @param len length of input buffer
+ * @return length of input buffer after normalization
+ */
+ public int normalize(char s[], int len) {
+
+ for (int i = 0; i < len; i++) {
+ switch (s[i]) {
+ // candrabindu (ఀ and ఁ) -> bindu (ం)
+ case '\u0C00': // ఀ
+ case '\u0C01': // ఁ
+ s[i] = '\u0C02'; // ం
+ break;
+ // delete visarga (ః)
+ case '\u0C03':
+ len = delete(s, i, len);
+ i--;
+ break;
+
+ // zwj/zwnj -> delete
+ case '\u200D':
+ case '\u200C':
+ len = delete(s, i, len);
+ i--;
+ break;
+
+ // long -> short vowels
+ case '\u0C14': // ఔ
+ s[i] = '\u0C13'; // ఓ
+ break;
+ case '\u0C10': // ఐ
+ s[i] = '\u0C0F'; // ఏ
+ break;
+ case '\u0C06': // ఆ
+ s[i] = '\u0C05'; // అ
+ break;
+ case '\u0C08': // ఈ
+ s[i] = '\u0C07'; // ఇ
+ break;
+ case '\u0C0A': // ఊ
+ s[i] = '\u0C09'; // ఉ
+ break;
+
+ // long -> short vowels matras
+ case '\u0C40': // ీ
+ s[i] = '\u0C3F'; // ి
+ break;
+ case '\u0C42': // ూ
+ s[i] = '\u0C41'; // ు
+ break;
+ case '\u0C47': // ే
+ s[i] = '\u0C46'; // ె
+ break;
+ case '\u0C4B': // ో
+ s[i] = '\u0C4A'; // ొ
+ break;
+ // decomposed dipthong (ె + ౖ) -> precomposed diphthong vowel sign (ై)
+ case '\u0C46':
+ if (i + 1 < len && s[i + 1] == '\u0C56') {
+ s[i] = '\u0C48';
+ len = delete(s, i + 1, len);
+ }
+ break;
+ // composed oo or au -> oo or au
+ case '\u0C12':
+ if (i + 1 < len && s[i + 1] == '\u0C55') {
+ // (ఒ + ౕ) -> oo (ఓ)
+ s[i] = '\u0C13';
+ len = delete(s, i + 1, len);
+ } else if (i + 1 < len && s[i + 1] == '\u0C4C') {
+ // (ఒ + ౌ) -> au (ఔ)
+ s[i] = '\u0C14';
+ len = delete(s, i + 1, len);
+ }
+ break;
+
+ default:
+ break;
+ }
+ }
+
+ return len;
+ }
+}
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/te/TeluguStemFilter.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/te/TeluguStemFilter.java
new file mode 100644
index 0000000..15bc934
--- /dev/null
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/te/TeluguStemFilter.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.analysis.te;
+
+import java.io.IOException;
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.KeywordAttribute;
+
+/**
+ * A {@link TokenFilter} that applies {@link TeluguStemmer} to stem Telugu words.
+ *
+ * @since 9.0.0
+ */
+public final class TeluguStemFilter extends TokenFilter {
+ private final CharTermAttribute termAttribute = addAttribute(CharTermAttribute.class);
+ private final KeywordAttribute keywordAttribute = addAttribute(KeywordAttribute.class);
+ private final TeluguStemmer teluguStemmer = new TeluguStemmer();
+
+ public TeluguStemFilter(TokenStream input) {
+ super(input);
+ }
+
+ @Override
+ public boolean incrementToken() throws IOException {
+ if (input.incrementToken()) {
+ if (!keywordAttribute.isKeyword()) {
+ termAttribute.setLength(teluguStemmer.stem(termAttribute.buffer(), termAttribute.length()));
+ }
+ return true;
+ } else {
+ return false;
+ }
+ }
+}
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/te/TeluguStemFilterFactory.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/te/TeluguStemFilterFactory.java
new file mode 100644
index 0000000..0e26ba1
--- /dev/null
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/te/TeluguStemFilterFactory.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.analysis.te;
+
+import java.util.Map;
+import org.apache.lucene.analysis.TokenFilterFactory;
+import org.apache.lucene.analysis.TokenStream;
+
+/**
+ * Factory for {@link TeluguStemFilter}.
+ *
+ * <pre class="prettyprint">
+ * <fieldType name="text_testem" class="solr.TextField" positionIncrementGap="100">
+ * <analyzer>
+ * <tokenizer class="solr.StandardTokenizerFactory"/>
+ * <filter class="solr.TeluguStemFilterFactory"/>
+ * </analyzer>
+ * </fieldType></pre>
+ *
+ * @since 9.0.0
+ * @lucene.spi {@value #NAME}
+ */
+public class TeluguStemFilterFactory extends TokenFilterFactory {
+
+ /** SPI name */
+ public static final String NAME = "teluguStem";
+
+ public TeluguStemFilterFactory(Map<String, String> args) {
+ super(args);
+ if (!args.isEmpty()) {
+ throw new IllegalArgumentException("Unknown parameters: " + args);
+ }
+ }
+
+ /** Default ctor for compatibility with SPI */
+ public TeluguStemFilterFactory() {
+ throw defaultCtorException();
+ }
+
+ @Override
+ public TokenStream create(TokenStream input) {
+ return new TeluguStemFilter(input);
+ }
+}
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/te/TeluguStemmer.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/te/TeluguStemmer.java
new file mode 100644
index 0000000..60993c5
--- /dev/null
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/te/TeluguStemmer.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.analysis.te;
+
+import static org.apache.lucene.analysis.util.StemmerUtil.endsWith;
+
+/**
+ * Stemmer for Telugu.
+ *
+ * @since 9.0.0
+ */
+public class TeluguStemmer {
+ public int stem(char buffer[], int len) {
+ // 4
+ if ((len > 5) && (endsWith(buffer, len, "ళ్ళు") || endsWith(buffer, len, "డ్లు")))
+ return len - 4;
+
+ // 2
+ if ((len > 3)
+ && (endsWith(buffer, len, "డు")
+ || endsWith(buffer, len, "ము")
+ || endsWith(buffer, len, "వు")
+ || endsWith(buffer, len, "లు")
+ || endsWith(buffer, len, "ని")
+ || endsWith(buffer, len, "ను")
+ || endsWith(buffer, len, "చే")
+ || endsWith(buffer, len, "కై")
+ || endsWith(buffer, len, "లో")
+ || endsWith(buffer, len, "డు")
+ || endsWith(buffer, len, "ది")
+ || endsWith(buffer, len, "కి")
+ || endsWith(buffer, len, "సు")
+ || endsWith(buffer, len, "వై")
+ || endsWith(buffer, len, "పై"))) return len - 2;
+
+ // 1
+ if ((len > 2)
+ && (endsWith(buffer, len, "ి")
+ || endsWith(buffer, len, "ీ")
+ || endsWith(buffer, len, "ు")
+ || endsWith(buffer, len, "ూ")
+ || endsWith(buffer, len, "ె")
+ || endsWith(buffer, len, "ే")
+ || endsWith(buffer, len, "ొ")
+ || endsWith(buffer, len, "ో")
+ || endsWith(buffer, len, "ా"))) return len - 1;
+
+ return len;
+ }
+}
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/te/package-info.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/te/package-info.java
new file mode 100644
index 0000000..659a538
--- /dev/null
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/te/package-info.java
@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/** Analyzer for Telugu Language. */
+package org.apache.lucene.analysis.te;
diff --git a/lucene/analysis/common/src/resources/META-INF/services/org.apache.lucene.analysis.TokenFilterFactory b/lucene/analysis/common/src/resources/META-INF/services/org.apache.lucene.analysis.TokenFilterFactory
index 2899fd5..2328b87 100644
--- a/lucene/analysis/common/src/resources/META-INF/services/org.apache.lucene.analysis.TokenFilterFactory
+++ b/lucene/analysis/common/src/resources/META-INF/services/org.apache.lucene.analysis.TokenFilterFactory
@@ -117,5 +117,7 @@ org.apache.lucene.analysis.sv.SwedishMinimalStemFilterFactory
org.apache.lucene.analysis.synonym.SynonymFilterFactory
org.apache.lucene.analysis.synonym.SynonymGraphFilterFactory
org.apache.lucene.analysis.core.FlattenGraphFilterFactory
+org.apache.lucene.analysis.te.TeluguNormalizationFilterFactory
+org.apache.lucene.analysis.te.TeluguStemFilterFactory
org.apache.lucene.analysis.tr.TurkishLowerCaseFilterFactory
org.apache.lucene.analysis.util.ElisionFilterFactory
diff --git a/lucene/analysis/common/src/resources/org/apache/lucene/analysis/te/stopwords.txt b/lucene/analysis/common/src/resources/org/apache/lucene/analysis/te/stopwords.txt
new file mode 100644
index 0000000..ea175d6
--- /dev/null
+++ b/lucene/analysis/common/src/resources/org/apache/lucene/analysis/te/stopwords.txt
@@ -0,0 +1,51 @@
+# Telugu stop word list
+చేయగలిగింది
+గురించి
+పై
+ప్రకారం
+అనుగుణంగా
+అడ్డంగా
+నిజంగా
+తర్వాత
+తర్వాత
+మళ్ళీ
+వ్యతిరేకంగా
+కాదు
+అందరూ
+అనుమతించు
+అనుమతిస్తుంది
+దాదాపు
+మాత్రమే
+వెంట
+ఇప్పటికే
+కూడా
+అయితే
+ఎప్పుడు
+వద్ద
+మధ్య
+మధ్య
+ఒక
+మరియు
+మరొక
+ఏ
+ఎవరో ఒకరు
+ఏమైనప్పటికి
+ఎవరైనా
+ఏదైనా
+ఏమైనప్పటికి
+ఎక్కడైనా
+వేరుగా
+కనిపిస్తాయి
+మెచ్చుకో
+తగిన
+ఉన్నారు
+కాదు
+చుట్టూ
+గా
+ఒక ప్రక్కన
+అడగండి
+అడగడం
+సంబంధం
+వద్ద
+అందుబాటులో
+దూరంగా
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/te/TestTeluguAnalyzer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/te/TestTeluguAnalyzer.java
new file mode 100644
index 0000000..1eb47ef
--- /dev/null
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/te/TestTeluguAnalyzer.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.analysis.te;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.BaseTokenStreamTestCase;
+import org.apache.lucene.analysis.CharArraySet;
+
+public class TestTeluguAnalyzer extends BaseTokenStreamTestCase {
+
+ /** This test fails with NPE when the stopwords file is missing in classpath */
+ public void testResourcesAvailable() {
+ new TeluguAnalyzer().close();
+ }
+
+ public void testBasics() throws Exception {
+ Analyzer a = new TeluguAnalyzer();
+ // two ways to write oo letter.
+ checkOneTerm(a, "ఒౕనమాల", "ఓనమాల");
+ a.close();
+ }
+
+ public void testExclusionSet() throws Exception {
+ CharArraySet exclusionSet = new CharArraySet(asSet("వస్తువులు"), false);
+ Analyzer a = new TeluguAnalyzer(TeluguAnalyzer.getDefaultStopSet(), exclusionSet);
+ checkOneTerm(a, "వస్తువులు", "వస్తుమలు");
+ a.close();
+ }
+
+ /** test we fold digits to latin-1 */
+ public void testDigits() throws Exception {
+ TeluguAnalyzer a = new TeluguAnalyzer();
+ checkOneTerm(a, "౧౨౩౪", "1234");
+ a.close();
+ }
+
+ /** Send some random strings to the analyzer */
+ public void testRandomStrings() throws Exception {
+ TeluguAnalyzer analyzer = new TeluguAnalyzer();
+ checkRandomData(random(), analyzer, 200 * RANDOM_MULTIPLIER);
+ analyzer.close();
+ }
+}
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/te/TestTeluguFilters.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/te/TestTeluguFilters.java
new file mode 100644
index 0000000..bf08a41
--- /dev/null
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/te/TestTeluguFilters.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.analysis.te;
+
+import java.io.Reader;
+import java.io.StringReader;
+import org.apache.lucene.analysis.BaseTokenStreamFactoryTestCase;
+import org.apache.lucene.analysis.TokenStream;
+
+/** Simple tests to ensure the Telugu filter Factories are working. */
+public class TestTeluguFilters extends BaseTokenStreamFactoryTestCase {
+ /** Test IndicNormalizationFilterFactory */
+ public void testIndicNormalizer() throws Exception {
+ Reader reader = new StringReader("ప్ अाैर");
+ TokenStream stream = whitespaceMockTokenizer(reader);
+ stream = tokenFilterFactory("IndicNormalization").create(stream);
+ assertTokenStreamContents(stream, new String[] {"ప్", "और"});
+ }
+
+ /** Test TeluguNormalizationFilterFactory */
+ public void testTeluguNormalizer() throws Exception {
+ Reader reader = new StringReader("వస్తువులు");
+ TokenStream stream = whitespaceMockTokenizer(reader);
+ stream = tokenFilterFactory("IndicNormalization").create(stream);
+ stream = tokenFilterFactory("TeluguNormalization").create(stream);
+ assertTokenStreamContents(stream, new String[] {"వస్తుమలు"});
+ }
+
+ /** Test TeluguStemFilterFactory */
+ public void testStemmer() throws Exception {
+ Reader reader = new StringReader("వస్తువులు");
+ TokenStream stream = whitespaceMockTokenizer(reader);
+ stream = tokenFilterFactory("IndicNormalization").create(stream);
+ stream = tokenFilterFactory("TeluguNormalization").create(stream);
+ stream = tokenFilterFactory("TeluguStem").create(stream);
+ assertTokenStreamContents(stream, new String[] {"వస్తుమ"});
+ }
+
+ /** Test that bogus arguments result in exception */
+ public void testBogusArguments() throws Exception {
+ IllegalArgumentException expected =
+ expectThrows(
+ IllegalArgumentException.class,
+ () -> {
+ tokenFilterFactory("IndicNormalization", "bogusArg", "bogusValue");
+ });
+ assertTrue(expected.getMessage().contains("Unknown parameters"));
+
+ expected =
+ expectThrows(
+ IllegalArgumentException.class,
+ () -> {
+ tokenFilterFactory("TeluguNormalization", "bogusArg", "bogusValue");
+ });
+ assertTrue(expected.getMessage().contains("Unknown parameters"));
+
+ expected =
+ expectThrows(
+ IllegalArgumentException.class,
+ () -> {
+ tokenFilterFactory("TeluguStem", "bogusArg", "bogusValue");
+ });
+ assertTrue(expected.getMessage().contains("Unknown parameters"));
+ }
+}
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/te/TestTeluguNormalizer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/te/TestTeluguNormalizer.java
new file mode 100644
index 0000000..0571b6f
--- /dev/null
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/te/TestTeluguNormalizer.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.analysis.te;
+
+import java.io.IOException;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.BaseTokenStreamTestCase;
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.core.KeywordTokenizer;
+
+/** Test TeluguNormalizer */
+public class TestTeluguNormalizer extends BaseTokenStreamTestCase {
+
+ /** Long matras should be shortened. */
+ public void testMatra() throws IOException {
+ check("పదాలూ", "పదాలు");
+ check("అబ్బాయీ", "అబ్బాయి");
+ }
+
+ public void testVowels() throws IOException {
+ // removal of visarga (ః)
+ check("ఃౌైాిు", "ౌైాిు");
+ // vowel shortening
+ check("ఔఐఆఈఊ", "ఓఏఅఇఉ");
+ }
+
+ private void check(String input, String output) throws IOException {
+ Tokenizer tokenizer = whitespaceMockTokenizer(input);
+ TokenFilter tf = new TeluguNormalizationFilter(tokenizer);
+ assertTokenStreamContents(tf, new String[] {output});
+ }
+
+ public void testEmptyTerm() throws IOException {
+ Analyzer a =
+ new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
+ return new TokenStreamComponents(tokenizer, new TeluguNormalizationFilter(tokenizer));
+ }
+ };
+ checkOneTerm(a, "", "");
+ a.close();
+ }
+}
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/te/TestTeluguStemmer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/te/TestTeluguStemmer.java
new file mode 100644
index 0000000..bd1b271
--- /dev/null
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/te/TestTeluguStemmer.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.analysis.te;
+
+import java.io.IOException;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.BaseTokenStreamTestCase;
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.core.KeywordTokenizer;
+
+/** Test TeluguStemmer */
+public class TestTeluguStemmer extends BaseTokenStreamTestCase {
+
+ /** Test plural forms */
+ public void testPlurals() throws IOException {
+ check("వస్తువులు", "వస్తువు");
+ check("పన్నులు", "పన్ను");
+ }
+
+ /** Test some verb forms */
+ public void testVerbs() throws IOException {
+ check("చేపిస్తున్నది", "చేపిస్తున్న");
+ check("చేపిస్తున్నడు", "చేపిస్తున్న");
+ }
+
+ private void check(String input, String output) throws IOException {
+ Tokenizer tokenizer = whitespaceMockTokenizer(input);
+ TokenFilter tf = new TeluguStemFilter(tokenizer);
+ assertTokenStreamContents(tf, new String[] {output});
+ }
+
+ public void testEmptyTerm() throws IOException {
+ Analyzer a =
+ new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
+ return new TokenStreamComponents(tokenizer, new TeluguStemFilter(tokenizer));
+ }
+ };
+ checkOneTerm(a, "", "");
+ a.close();
+ }
+}