You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2012/03/26 14:59:12 UTC
svn commit: r1305339 [2/3] - in /lucene/dev/trunk:
dev-tools/idea/.idea/libraries/
lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/
lucene/contrib/highlighter/src/java/org/apache/lucene/search/vectorhighlight/
lucene/contrib/high...
Modified: lucene/dev/trunk/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/LuceneCarrot2StemmerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/LuceneCarrot2StemmerFactory.java?rev=1305339&r1=1305338&r2=1305339&view=diff
==============================================================================
--- lucene/dev/trunk/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/LuceneCarrot2StemmerFactory.java (original)
+++ lucene/dev/trunk/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/LuceneCarrot2StemmerFactory.java Mon Mar 26 12:58:58 2012
@@ -1,241 +1,241 @@
-package org.apache.solr.handler.clustering.carrot2;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.nio.CharBuffer;
-import java.util.HashMap;
-
-import org.apache.lucene.analysis.ar.ArabicNormalizer;
-import org.apache.lucene.analysis.ar.ArabicStemmer;
-import org.carrot2.core.LanguageCode;
-import org.carrot2.text.linguistic.IStemmer;
-import org.carrot2.text.linguistic.IStemmerFactory;
-import org.carrot2.util.ReflectionUtils;
-import org.slf4j.Logger;
-import org.tartarus.snowball.SnowballProgram;
-import org.tartarus.snowball.ext.DanishStemmer;
-import org.tartarus.snowball.ext.DutchStemmer;
-import org.tartarus.snowball.ext.EnglishStemmer;
-import org.tartarus.snowball.ext.FinnishStemmer;
-import org.tartarus.snowball.ext.FrenchStemmer;
-import org.tartarus.snowball.ext.GermanStemmer;
-import org.tartarus.snowball.ext.HungarianStemmer;
-import org.tartarus.snowball.ext.ItalianStemmer;
-import org.tartarus.snowball.ext.NorwegianStemmer;
-import org.tartarus.snowball.ext.PortugueseStemmer;
-import org.tartarus.snowball.ext.RomanianStemmer;
-import org.tartarus.snowball.ext.RussianStemmer;
-import org.tartarus.snowball.ext.SpanishStemmer;
-import org.tartarus.snowball.ext.SwedishStemmer;
-import org.tartarus.snowball.ext.TurkishStemmer;
-
-/**
- * An implementation of Carrot2's {@link IStemmerFactory} based on Lucene's
- * APIs. Should the relevant Lucene APIs need to change, the changes can be made
- * in this class.
- */
-public class LuceneCarrot2StemmerFactory implements IStemmerFactory {
- final static Logger logger = org.slf4j.LoggerFactory
- .getLogger(LuceneCarrot2StemmerFactory.class);
-
- @Override
- public IStemmer getStemmer(LanguageCode language) {
- switch (language) {
- case ARABIC:
- return ArabicStemmerFactory.createStemmer();
-
- case CHINESE_SIMPLIFIED:
- return IdentityStemmer.INSTANCE;
-
- default:
- /*
- * For other languages, try to use snowball's stemming.
- */
- return SnowballStemmerFactory.createStemmer(language);
- }
- }
-
- /**
- * Factory of {@link IStemmer} implementations from the <code>snowball</code>
- * project.
- */
- private final static class SnowballStemmerFactory {
- /**
- * Static hard mapping from language codes to stemmer classes in Snowball.
- * This mapping is not dynamic because we want to keep the possibility to
- * obfuscate these classes.
- */
- private static HashMap<LanguageCode, Class<? extends SnowballProgram>> snowballStemmerClasses;
- static {
- snowballStemmerClasses = new HashMap<LanguageCode, Class<? extends SnowballProgram>>();
- snowballStemmerClasses.put(LanguageCode.DANISH, DanishStemmer.class);
- snowballStemmerClasses.put(LanguageCode.DUTCH, DutchStemmer.class);
- snowballStemmerClasses.put(LanguageCode.ENGLISH, EnglishStemmer.class);
- snowballStemmerClasses.put(LanguageCode.FINNISH, FinnishStemmer.class);
- snowballStemmerClasses.put(LanguageCode.FRENCH, FrenchStemmer.class);
- snowballStemmerClasses.put(LanguageCode.GERMAN, GermanStemmer.class);
- snowballStemmerClasses
- .put(LanguageCode.HUNGARIAN, HungarianStemmer.class);
- snowballStemmerClasses.put(LanguageCode.ITALIAN, ItalianStemmer.class);
- snowballStemmerClasses
- .put(LanguageCode.NORWEGIAN, NorwegianStemmer.class);
- snowballStemmerClasses.put(LanguageCode.PORTUGUESE,
- PortugueseStemmer.class);
- snowballStemmerClasses.put(LanguageCode.ROMANIAN, RomanianStemmer.class);
- snowballStemmerClasses.put(LanguageCode.RUSSIAN, RussianStemmer.class);
- snowballStemmerClasses.put(LanguageCode.SPANISH, SpanishStemmer.class);
- snowballStemmerClasses.put(LanguageCode.SWEDISH, SwedishStemmer.class);
- snowballStemmerClasses.put(LanguageCode.TURKISH, TurkishStemmer.class);
- }
-
- /**
- * An adapter converting Snowball programs into {@link IStemmer} interface.
- */
- private static class SnowballStemmerAdapter implements IStemmer {
- private final SnowballProgram snowballStemmer;
-
- public SnowballStemmerAdapter(SnowballProgram snowballStemmer) {
- this.snowballStemmer = snowballStemmer;
- }
-
- public CharSequence stem(CharSequence word) {
- snowballStemmer.setCurrent(word.toString());
- if (snowballStemmer.stem()) {
- return snowballStemmer.getCurrent();
- } else {
- return null;
- }
- }
- }
-
- /**
- * Create and return an {@link IStemmer} adapter for a
- * {@link SnowballProgram} for a given language code. An identity stemmer is
- * returned for unknown languages.
- */
- public static IStemmer createStemmer(LanguageCode language) {
- final Class<? extends SnowballProgram> stemmerClazz = snowballStemmerClasses
- .get(language);
-
- if (stemmerClazz == null) {
- logger.warn("No Snowball stemmer class for: " + language.name()
- + ". Quality of clustering may be degraded.");
- return IdentityStemmer.INSTANCE;
- }
-
- try {
- return new SnowballStemmerAdapter(stemmerClazz.newInstance());
- } catch (Exception e) {
- logger.warn("Could not instantiate snowball stemmer"
- + " for language: " + language.name()
- + ". Quality of clustering may be degraded.", e);
-
- return IdentityStemmer.INSTANCE;
- }
- }
- }
-
- /**
- * Factory of {@link IStemmer} implementations for the
- * {@link LanguageCode#ARABIC} language. Requires <code>lucene-contrib</code>
- * to be present in classpath, otherwise an empty (identity) stemmer is
- * returned.
- */
- private static class ArabicStemmerFactory {
- static {
- try {
- ReflectionUtils.classForName(ArabicStemmer.class.getName(), false);
- ReflectionUtils.classForName(ArabicNormalizer.class.getName(), false);
- } catch (ClassNotFoundException e) {
- logger
- .warn(
- "Could not instantiate Lucene stemmer for Arabic, clustering quality "
- + "of Arabic content may be degraded. For best quality clusters, "
- + "make sure Lucene's Arabic analyzer JAR is in the classpath",
- e);
- }
- }
-
- /**
- * Adapter to lucene-contrib Arabic analyzers.
- */
- private static class LuceneStemmerAdapter implements IStemmer {
- private final org.apache.lucene.analysis.ar.ArabicStemmer delegate;
- private final org.apache.lucene.analysis.ar.ArabicNormalizer normalizer;
-
- private char[] buffer = new char[0];
-
- private LuceneStemmerAdapter() throws Exception {
- delegate = new org.apache.lucene.analysis.ar.ArabicStemmer();
- normalizer = new org.apache.lucene.analysis.ar.ArabicNormalizer();
- }
-
- public CharSequence stem(CharSequence word) {
- if (word.length() > buffer.length) {
- buffer = new char[word.length()];
- }
-
- for (int i = 0; i < word.length(); i++) {
- buffer[i] = word.charAt(i);
- }
-
- int newLen = normalizer.normalize(buffer, word.length());
- newLen = delegate.stem(buffer, newLen);
-
- if (newLen != word.length() || !equals(buffer, newLen, word)) {
- return CharBuffer.wrap(buffer, 0, newLen);
- }
-
- // Same-same.
- return null;
- }
-
- private boolean equals(char[] buffer, int len, CharSequence word) {
- assert len == word.length();
-
- for (int i = 0; i < len; i++) {
- if (buffer[i] != word.charAt(i))
- return false;
- }
-
- return true;
- }
- }
-
- public static IStemmer createStemmer() {
- try {
- return new LuceneStemmerAdapter();
- } catch (Throwable e) {
- return IdentityStemmer.INSTANCE;
- }
- }
- }
-
- /**
- * An implementation of {@link IStemmer} that always returns <code>null</code>
- * which means no stemming.
- */
- private static class IdentityStemmer implements IStemmer {
- private final static IdentityStemmer INSTANCE = new IdentityStemmer();
-
- @Override
- public CharSequence stem(CharSequence word) {
- return null;
- }
- }
-}
+package org.apache.solr.handler.clustering.carrot2;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.nio.CharBuffer;
+import java.util.HashMap;
+
+import org.apache.lucene.analysis.ar.ArabicNormalizer;
+import org.apache.lucene.analysis.ar.ArabicStemmer;
+import org.carrot2.core.LanguageCode;
+import org.carrot2.text.linguistic.IStemmer;
+import org.carrot2.text.linguistic.IStemmerFactory;
+import org.carrot2.util.ReflectionUtils;
+import org.slf4j.Logger;
+import org.tartarus.snowball.SnowballProgram;
+import org.tartarus.snowball.ext.DanishStemmer;
+import org.tartarus.snowball.ext.DutchStemmer;
+import org.tartarus.snowball.ext.EnglishStemmer;
+import org.tartarus.snowball.ext.FinnishStemmer;
+import org.tartarus.snowball.ext.FrenchStemmer;
+import org.tartarus.snowball.ext.GermanStemmer;
+import org.tartarus.snowball.ext.HungarianStemmer;
+import org.tartarus.snowball.ext.ItalianStemmer;
+import org.tartarus.snowball.ext.NorwegianStemmer;
+import org.tartarus.snowball.ext.PortugueseStemmer;
+import org.tartarus.snowball.ext.RomanianStemmer;
+import org.tartarus.snowball.ext.RussianStemmer;
+import org.tartarus.snowball.ext.SpanishStemmer;
+import org.tartarus.snowball.ext.SwedishStemmer;
+import org.tartarus.snowball.ext.TurkishStemmer;
+
+/**
+ * An implementation of Carrot2's {@link IStemmerFactory} based on Lucene's
+ * APIs. Should the relevant Lucene APIs need to change, the changes can be made
+ * in this class.
+ */
+public class LuceneCarrot2StemmerFactory implements IStemmerFactory {
+ final static Logger logger = org.slf4j.LoggerFactory
+ .getLogger(LuceneCarrot2StemmerFactory.class);
+
+ @Override
+ public IStemmer getStemmer(LanguageCode language) {
+ switch (language) {
+ case ARABIC:
+ return ArabicStemmerFactory.createStemmer();
+
+ case CHINESE_SIMPLIFIED:
+ return IdentityStemmer.INSTANCE;
+
+ default:
+ /*
+ * For other languages, try to use snowball's stemming.
+ */
+ return SnowballStemmerFactory.createStemmer(language);
+ }
+ }
+
+ /**
+ * Factory of {@link IStemmer} implementations from the <code>snowball</code>
+ * project.
+ */
+ private final static class SnowballStemmerFactory {
+ /**
+ * Static hard mapping from language codes to stemmer classes in Snowball.
+ * This mapping is not dynamic because we want to keep the possibility to
+ * obfuscate these classes.
+ */
+ private static HashMap<LanguageCode, Class<? extends SnowballProgram>> snowballStemmerClasses;
+ static {
+ snowballStemmerClasses = new HashMap<LanguageCode, Class<? extends SnowballProgram>>();
+ snowballStemmerClasses.put(LanguageCode.DANISH, DanishStemmer.class);
+ snowballStemmerClasses.put(LanguageCode.DUTCH, DutchStemmer.class);
+ snowballStemmerClasses.put(LanguageCode.ENGLISH, EnglishStemmer.class);
+ snowballStemmerClasses.put(LanguageCode.FINNISH, FinnishStemmer.class);
+ snowballStemmerClasses.put(LanguageCode.FRENCH, FrenchStemmer.class);
+ snowballStemmerClasses.put(LanguageCode.GERMAN, GermanStemmer.class);
+ snowballStemmerClasses
+ .put(LanguageCode.HUNGARIAN, HungarianStemmer.class);
+ snowballStemmerClasses.put(LanguageCode.ITALIAN, ItalianStemmer.class);
+ snowballStemmerClasses
+ .put(LanguageCode.NORWEGIAN, NorwegianStemmer.class);
+ snowballStemmerClasses.put(LanguageCode.PORTUGUESE,
+ PortugueseStemmer.class);
+ snowballStemmerClasses.put(LanguageCode.ROMANIAN, RomanianStemmer.class);
+ snowballStemmerClasses.put(LanguageCode.RUSSIAN, RussianStemmer.class);
+ snowballStemmerClasses.put(LanguageCode.SPANISH, SpanishStemmer.class);
+ snowballStemmerClasses.put(LanguageCode.SWEDISH, SwedishStemmer.class);
+ snowballStemmerClasses.put(LanguageCode.TURKISH, TurkishStemmer.class);
+ }
+
+ /**
+ * An adapter converting Snowball programs into {@link IStemmer} interface.
+ */
+ private static class SnowballStemmerAdapter implements IStemmer {
+ private final SnowballProgram snowballStemmer;
+
+ public SnowballStemmerAdapter(SnowballProgram snowballStemmer) {
+ this.snowballStemmer = snowballStemmer;
+ }
+
+ public CharSequence stem(CharSequence word) {
+ snowballStemmer.setCurrent(word.toString());
+ if (snowballStemmer.stem()) {
+ return snowballStemmer.getCurrent();
+ } else {
+ return null;
+ }
+ }
+ }
+
+ /**
+ * Create and return an {@link IStemmer} adapter for a
+ * {@link SnowballProgram} for a given language code. An identity stemmer is
+ * returned for unknown languages.
+ */
+ public static IStemmer createStemmer(LanguageCode language) {
+ final Class<? extends SnowballProgram> stemmerClazz = snowballStemmerClasses
+ .get(language);
+
+ if (stemmerClazz == null) {
+ logger.warn("No Snowball stemmer class for: " + language.name()
+ + ". Quality of clustering may be degraded.");
+ return IdentityStemmer.INSTANCE;
+ }
+
+ try {
+ return new SnowballStemmerAdapter(stemmerClazz.newInstance());
+ } catch (Exception e) {
+ logger.warn("Could not instantiate snowball stemmer"
+ + " for language: " + language.name()
+ + ". Quality of clustering may be degraded.", e);
+
+ return IdentityStemmer.INSTANCE;
+ }
+ }
+ }
+
+ /**
+ * Factory of {@link IStemmer} implementations for the
+ * {@link LanguageCode#ARABIC} language. Requires <code>lucene-contrib</code>
+ * to be present in classpath, otherwise an empty (identity) stemmer is
+ * returned.
+ */
+ private static class ArabicStemmerFactory {
+ static {
+ try {
+ ReflectionUtils.classForName(ArabicStemmer.class.getName(), false);
+ ReflectionUtils.classForName(ArabicNormalizer.class.getName(), false);
+ } catch (ClassNotFoundException e) {
+ logger
+ .warn(
+ "Could not instantiate Lucene stemmer for Arabic, clustering quality "
+ + "of Arabic content may be degraded. For best quality clusters, "
+ + "make sure Lucene's Arabic analyzer JAR is in the classpath",
+ e);
+ }
+ }
+
+ /**
+ * Adapter to lucene-contrib Arabic analyzers.
+ */
+ private static class LuceneStemmerAdapter implements IStemmer {
+ private final org.apache.lucene.analysis.ar.ArabicStemmer delegate;
+ private final org.apache.lucene.analysis.ar.ArabicNormalizer normalizer;
+
+ private char[] buffer = new char[0];
+
+ private LuceneStemmerAdapter() throws Exception {
+ delegate = new org.apache.lucene.analysis.ar.ArabicStemmer();
+ normalizer = new org.apache.lucene.analysis.ar.ArabicNormalizer();
+ }
+
+ public CharSequence stem(CharSequence word) {
+ if (word.length() > buffer.length) {
+ buffer = new char[word.length()];
+ }
+
+ for (int i = 0; i < word.length(); i++) {
+ buffer[i] = word.charAt(i);
+ }
+
+ int newLen = normalizer.normalize(buffer, word.length());
+ newLen = delegate.stem(buffer, newLen);
+
+ if (newLen != word.length() || !equals(buffer, newLen, word)) {
+ return CharBuffer.wrap(buffer, 0, newLen);
+ }
+
+ // Same-same.
+ return null;
+ }
+
+ private boolean equals(char[] buffer, int len, CharSequence word) {
+ assert len == word.length();
+
+ for (int i = 0; i < len; i++) {
+ if (buffer[i] != word.charAt(i))
+ return false;
+ }
+
+ return true;
+ }
+ }
+
+ public static IStemmer createStemmer() {
+ try {
+ return new LuceneStemmerAdapter();
+ } catch (Throwable e) {
+ return IdentityStemmer.INSTANCE;
+ }
+ }
+ }
+
+ /**
+ * An implementation of {@link IStemmer} that always returns <code>null</code>
+ * which means no stemming.
+ */
+ private static class IdentityStemmer implements IStemmer {
+ private final static IdentityStemmer INSTANCE = new IdentityStemmer();
+
+ @Override
+ public CharSequence stem(CharSequence word) {
+ return null;
+ }
+ }
+}
Modified: lucene/dev/trunk/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/LuceneCarrot2TokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/LuceneCarrot2TokenizerFactory.java?rev=1305339&r1=1305338&r2=1305339&view=diff
==============================================================================
--- lucene/dev/trunk/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/LuceneCarrot2TokenizerFactory.java (original)
+++ lucene/dev/trunk/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/LuceneCarrot2TokenizerFactory.java Mon Mar 26 12:58:58 2012
@@ -1,156 +1,156 @@
-package org.apache.solr.handler.clustering.carrot2;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.io.IOException;
-import java.io.Reader;
-import java.util.regex.Pattern;
-
-import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
-import org.carrot2.core.LanguageCode;
-import org.carrot2.text.analysis.ExtendedWhitespaceTokenizer;
-import org.carrot2.text.analysis.ITokenizer;
-import org.carrot2.text.linguistic.ITokenizerFactory;
-import org.carrot2.text.util.MutableCharArray;
-import org.carrot2.util.ExceptionUtils;
-import org.carrot2.util.ReflectionUtils;
-import org.slf4j.Logger;
-
-/**
- * An implementation of Carrot2's {@link ITokenizerFactory} based on Lucene's
- * Smart Chinese tokenizer. If Smart Chinese tokenizer is not available in
- * classpath at runtime, the default Carrot2's tokenizer is used. Should the
- * Lucene APIs need to change, the changes can be made in this class.
- */
-public class LuceneCarrot2TokenizerFactory implements ITokenizerFactory {
- final static Logger logger = org.slf4j.LoggerFactory
- .getLogger(LuceneCarrot2TokenizerFactory.class);
-
- @Override
- public ITokenizer getTokenizer(LanguageCode language) {
- switch (language) {
- case CHINESE_SIMPLIFIED:
- return ChineseTokenizerFactory.createTokenizer();
-
- /*
- * We use our own analyzer for Arabic. Lucene's version has special
- * support for Nonspacing-Mark characters (see
- * http://www.fileformat.info/info/unicode/category/Mn/index.htm), but we
- * have them included as letters in the parser.
- */
- case ARABIC:
- // Intentional fall-through.
-
- default:
- return new ExtendedWhitespaceTokenizer();
- }
- }
-
- /**
- * Creates tokenizers that adapt Lucene's Smart Chinese Tokenizer to Carrot2's
- * {@link ITokenizer}. If Smart Chinese is not available in the classpath, the
- * factory will fall back to the default white space tokenizer.
- */
- private static final class ChineseTokenizerFactory {
- static {
- try {
- ReflectionUtils.classForName(
- "org.apache.lucene.analysis.cn.smart.WordTokenFilter", false);
- ReflectionUtils.classForName(
- "org.apache.lucene.analysis.cn.smart.SentenceTokenizer", false);
- } catch (Throwable e) {
- logger
- .warn("Could not instantiate Smart Chinese Analyzer, clustering quality "
- + "of Chinese content may be degraded. For best quality clusters, "
- + "make sure Lucene's Smart Chinese Analyzer JAR is in the classpath");
- }
- }
-
- static ITokenizer createTokenizer() {
- try {
- return new ChineseTokenizer();
- } catch (Throwable e) {
- return new ExtendedWhitespaceTokenizer();
- }
- }
-
- private final static class ChineseTokenizer implements ITokenizer {
- private final static Pattern numeric = Pattern
- .compile("[\\-+'$]?\\d+([:\\-/,.]?\\d+)*[%$]?");
-
- private Tokenizer sentenceTokenizer;
- private TokenStream wordTokenFilter;
- private CharTermAttribute term = null;
-
- private final MutableCharArray tempCharSequence;
- private final Class<?> tokenFilterClass;
-
- private ChineseTokenizer() throws Exception {
- this.tempCharSequence = new MutableCharArray(new char[0]);
-
- // As Smart Chinese is not available during compile time,
- // we need to resort to reflection.
- final Class<?> tokenizerClass = ReflectionUtils.classForName(
- "org.apache.lucene.analysis.cn.smart.SentenceTokenizer", false);
- this.sentenceTokenizer = (Tokenizer) tokenizerClass.getConstructor(
- Reader.class).newInstance((Reader) null);
- this.tokenFilterClass = ReflectionUtils.classForName(
- "org.apache.lucene.analysis.cn.smart.WordTokenFilter", false);
- }
-
- public short nextToken() throws IOException {
- final boolean hasNextToken = wordTokenFilter.incrementToken();
- if (hasNextToken) {
- short flags = 0;
- final char[] image = term.buffer();
- final int length = term.length();
- tempCharSequence.reset(image, 0, length);
- if (length == 1 && image[0] == ',') {
- // ChineseTokenizer seems to convert all punctuation to ','
- // characters
- flags = ITokenizer.TT_PUNCTUATION;
- } else if (numeric.matcher(tempCharSequence).matches()) {
- flags = ITokenizer.TT_NUMERIC;
- } else {
- flags = ITokenizer.TT_TERM;
- }
- return flags;
- }
-
- return ITokenizer.TT_EOF;
- }
-
- public void setTermBuffer(MutableCharArray array) {
- array.reset(term.buffer(), 0, term.length());
- }
-
- public void reset(Reader input) throws IOException {
- try {
- sentenceTokenizer.reset(input);
- wordTokenFilter = (TokenStream) tokenFilterClass.getConstructor(
- TokenStream.class).newInstance(sentenceTokenizer);
- term = wordTokenFilter.addAttribute(CharTermAttribute.class);
- } catch (Exception e) {
- throw ExceptionUtils.wrapAsRuntimeException(e);
- }
- }
- }
- }
-}
+package org.apache.solr.handler.clustering.carrot2;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.io.Reader;
+import java.util.regex.Pattern;
+
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.carrot2.core.LanguageCode;
+import org.carrot2.text.analysis.ExtendedWhitespaceTokenizer;
+import org.carrot2.text.analysis.ITokenizer;
+import org.carrot2.text.linguistic.ITokenizerFactory;
+import org.carrot2.text.util.MutableCharArray;
+import org.carrot2.util.ExceptionUtils;
+import org.carrot2.util.ReflectionUtils;
+import org.slf4j.Logger;
+
+/**
+ * An implementation of Carrot2's {@link ITokenizerFactory} based on Lucene's
+ * Smart Chinese tokenizer. If Smart Chinese tokenizer is not available in
+ * classpath at runtime, the default Carrot2's tokenizer is used. Should the
+ * Lucene APIs need to change, the changes can be made in this class.
+ */
+public class LuceneCarrot2TokenizerFactory implements ITokenizerFactory {
+ final static Logger logger = org.slf4j.LoggerFactory
+ .getLogger(LuceneCarrot2TokenizerFactory.class);
+
+ @Override
+ public ITokenizer getTokenizer(LanguageCode language) {
+ switch (language) {
+ case CHINESE_SIMPLIFIED:
+ return ChineseTokenizerFactory.createTokenizer();
+
+ /*
+ * We use our own analyzer for Arabic. Lucene's version has special
+ * support for Nonspacing-Mark characters (see
+ * http://www.fileformat.info/info/unicode/category/Mn/index.htm), but we
+ * have them included as letters in the parser.
+ */
+ case ARABIC:
+ // Intentional fall-through.
+
+ default:
+ return new ExtendedWhitespaceTokenizer();
+ }
+ }
+
+ /**
+ * Creates tokenizers that adapt Lucene's Smart Chinese Tokenizer to Carrot2's
+ * {@link ITokenizer}. If Smart Chinese is not available in the classpath, the
+ * factory will fall back to the default white space tokenizer.
+ */
+ private static final class ChineseTokenizerFactory {
+ static {
+ try {
+ ReflectionUtils.classForName(
+ "org.apache.lucene.analysis.cn.smart.WordTokenFilter", false);
+ ReflectionUtils.classForName(
+ "org.apache.lucene.analysis.cn.smart.SentenceTokenizer", false);
+ } catch (Throwable e) {
+ logger
+ .warn("Could not instantiate Smart Chinese Analyzer, clustering quality "
+ + "of Chinese content may be degraded. For best quality clusters, "
+ + "make sure Lucene's Smart Chinese Analyzer JAR is in the classpath");
+ }
+ }
+
+ static ITokenizer createTokenizer() {
+ try {
+ return new ChineseTokenizer();
+ } catch (Throwable e) {
+ return new ExtendedWhitespaceTokenizer();
+ }
+ }
+
+ private final static class ChineseTokenizer implements ITokenizer {
+ private final static Pattern numeric = Pattern
+ .compile("[\\-+'$]?\\d+([:\\-/,.]?\\d+)*[%$]?");
+
+ private Tokenizer sentenceTokenizer;
+ private TokenStream wordTokenFilter;
+ private CharTermAttribute term = null;
+
+ private final MutableCharArray tempCharSequence;
+ private final Class<?> tokenFilterClass;
+
+ private ChineseTokenizer() throws Exception {
+ this.tempCharSequence = new MutableCharArray(new char[0]);
+
+ // As Smart Chinese is not available during compile time,
+ // we need to resort to reflection.
+ final Class<?> tokenizerClass = ReflectionUtils.classForName(
+ "org.apache.lucene.analysis.cn.smart.SentenceTokenizer", false);
+ this.sentenceTokenizer = (Tokenizer) tokenizerClass.getConstructor(
+ Reader.class).newInstance((Reader) null);
+ this.tokenFilterClass = ReflectionUtils.classForName(
+ "org.apache.lucene.analysis.cn.smart.WordTokenFilter", false);
+ }
+
+ public short nextToken() throws IOException {
+ final boolean hasNextToken = wordTokenFilter.incrementToken();
+ if (hasNextToken) {
+ short flags = 0;
+ final char[] image = term.buffer();
+ final int length = term.length();
+ tempCharSequence.reset(image, 0, length);
+ if (length == 1 && image[0] == ',') {
+ // ChineseTokenizer seems to convert all punctuation to ','
+ // characters
+ flags = ITokenizer.TT_PUNCTUATION;
+ } else if (numeric.matcher(tempCharSequence).matches()) {
+ flags = ITokenizer.TT_NUMERIC;
+ } else {
+ flags = ITokenizer.TT_TERM;
+ }
+ return flags;
+ }
+
+ return ITokenizer.TT_EOF;
+ }
+
+ public void setTermBuffer(MutableCharArray array) {
+ array.reset(term.buffer(), 0, term.length());
+ }
+
+ public void reset(Reader input) throws IOException {
+ try {
+ sentenceTokenizer.reset(input);
+ wordTokenFilter = (TokenStream) tokenFilterClass.getConstructor(
+ TokenStream.class).newInstance(sentenceTokenizer);
+ term = wordTokenFilter.addAttribute(CharTermAttribute.class);
+ } catch (Exception e) {
+ throw ExceptionUtils.wrapAsRuntimeException(e);
+ }
+ }
+ }
+ }
+}
Modified: lucene/dev/trunk/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/SolrStopwordsCarrot2LexicalDataFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/SolrStopwordsCarrot2LexicalDataFactory.java?rev=1305339&r1=1305338&r2=1305339&view=diff
==============================================================================
--- lucene/dev/trunk/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/SolrStopwordsCarrot2LexicalDataFactory.java (original)
+++ lucene/dev/trunk/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/SolrStopwordsCarrot2LexicalDataFactory.java Mon Mar 26 12:58:58 2012
@@ -1,140 +1,140 @@
-package org.apache.solr.handler.clustering.carrot2;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.util.Collection;
-import java.util.Set;
-
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.util.CharArraySet;
-import org.apache.solr.analysis.CommonGramsFilterFactory;
-import org.apache.solr.analysis.StopFilterFactory;
-import org.apache.solr.analysis.TokenFilterFactory;
-import org.apache.solr.analysis.TokenizerChain;
-import org.apache.solr.schema.IndexSchema;
-import org.carrot2.core.LanguageCode;
-import org.carrot2.core.attribute.Init;
-import org.carrot2.core.attribute.Processing;
-import org.carrot2.text.linguistic.DefaultLexicalDataFactory;
-import org.carrot2.text.linguistic.ILexicalData;
-import org.carrot2.text.linguistic.ILexicalDataFactory;
-import org.carrot2.text.util.MutableCharArray;
-import org.carrot2.util.attribute.Attribute;
-import org.carrot2.util.attribute.Bindable;
-import org.carrot2.util.attribute.Input;
-import org.slf4j.Logger;
-
-import com.google.common.collect.HashMultimap;
-import com.google.common.collect.Multimap;
-
-/**
- * An implementation of Carrot2's {@link ILexicalDataFactory} that adds stop
- * words from a field's StopFilter to the default stop words used in Carrot2,
- * for all languages Carrot2 supports. Completely replacing Carrot2 stop words
- * with Solr's wouldn't make much sense because clustering needs more aggressive
- * stop words removal. In other words, if something is a stop word during
- * indexing, then it should also be a stop word during clustering, but not the
- * other way round.
- */
-@Bindable
-public class SolrStopwordsCarrot2LexicalDataFactory implements
- ILexicalDataFactory {
- final static Logger logger = org.slf4j.LoggerFactory
- .getLogger(SolrStopwordsCarrot2LexicalDataFactory.class);
-
- @Init
- @Input
- @Attribute(key = "solrIndexSchema")
- private IndexSchema schema;
-
- @Processing
- @Input
- @Attribute(key = "solrFieldNames")
- private Set<String> fieldNames;
-
- /**
- * A lazily-built cache of stop words per field.
- */
- private Multimap<String, CharArraySet> solrStopWords = HashMultimap.create();
-
- /**
- * Carrot2's default lexical resources to use in addition to Solr's stop
- * words.
- */
- private DefaultLexicalDataFactory carrot2LexicalDataFactory = new DefaultLexicalDataFactory();
-
- /**
- * Obtains stop words for a field from the associated
- * {@link StopFilterFactory}, if any.
- */
- private Collection<CharArraySet> getSolrStopWordsForField(String fieldName) {
- // No need to synchronize here, Carrot2 ensures that instances
- // of this class are not used by multiple threads at a time.
- if (!solrStopWords.containsKey(fieldName)) {
- final Analyzer fieldAnalyzer = schema.getFieldType(fieldName)
- .getAnalyzer();
- if (fieldAnalyzer instanceof TokenizerChain) {
- final TokenFilterFactory[] filterFactories = ((TokenizerChain) fieldAnalyzer)
- .getTokenFilterFactories();
- for (TokenFilterFactory factory : filterFactories) {
- if (factory instanceof StopFilterFactory) {
- // StopFilterFactory holds the stop words in a CharArraySet
- solrStopWords.put(fieldName,
- ((StopFilterFactory) factory).getStopWords());
- }
-
- if (factory instanceof CommonGramsFilterFactory) {
- solrStopWords.put(fieldName,
- ((CommonGramsFilterFactory) factory)
- .getCommonWords());
- }
- }
- }
- }
- return solrStopWords.get(fieldName);
- }
-
- @Override
- public ILexicalData getLexicalData(LanguageCode languageCode) {
- final ILexicalData carrot2LexicalData = carrot2LexicalDataFactory
- .getLexicalData(languageCode);
-
- return new ILexicalData() {
- @Override
- public boolean isStopLabel(CharSequence word) {
- // Nothing in Solr maps to the concept of a stop label,
- // so return Carrot2's default here.
- return carrot2LexicalData.isStopLabel(word);
- }
-
- @Override
- public boolean isCommonWord(MutableCharArray word) {
- // Loop over the fields involved in clustering first
- for (String fieldName : fieldNames) {
- for (CharArraySet stopWords : getSolrStopWordsForField(fieldName)) {
- if (stopWords.contains(word)) {
- return true;
- }
- }
- }
- // Check default Carrot2 stop words too
- return carrot2LexicalData.isCommonWord(word);
- }
- };
- }
-}
+package org.apache.solr.handler.clustering.carrot2;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.Collection;
+import java.util.Set;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.util.CharArraySet;
+import org.apache.solr.analysis.CommonGramsFilterFactory;
+import org.apache.solr.analysis.StopFilterFactory;
+import org.apache.solr.analysis.TokenFilterFactory;
+import org.apache.solr.analysis.TokenizerChain;
+import org.apache.solr.schema.IndexSchema;
+import org.carrot2.core.LanguageCode;
+import org.carrot2.core.attribute.Init;
+import org.carrot2.core.attribute.Processing;
+import org.carrot2.text.linguistic.DefaultLexicalDataFactory;
+import org.carrot2.text.linguistic.ILexicalData;
+import org.carrot2.text.linguistic.ILexicalDataFactory;
+import org.carrot2.text.util.MutableCharArray;
+import org.carrot2.util.attribute.Attribute;
+import org.carrot2.util.attribute.Bindable;
+import org.carrot2.util.attribute.Input;
+import org.slf4j.Logger;
+
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Multimap;
+
+/**
+ * An implementation of Carrot2's {@link ILexicalDataFactory} that adds stop
+ * words from a field's StopFilter to the default stop words used in Carrot2,
+ * for all languages Carrot2 supports. Completely replacing Carrot2 stop words
+ * with Solr's wouldn't make much sense because clustering needs more aggressive
+ * stop words removal. In other words, if something is a stop word during
+ * indexing, then it should also be a stop word during clustering, but not the
+ * other way round.
+ */
+@Bindable
+public class SolrStopwordsCarrot2LexicalDataFactory implements
+ ILexicalDataFactory {
+ final static Logger logger = org.slf4j.LoggerFactory
+ .getLogger(SolrStopwordsCarrot2LexicalDataFactory.class);
+
+ @Init
+ @Input
+ @Attribute(key = "solrIndexSchema")
+ private IndexSchema schema;
+
+ @Processing
+ @Input
+ @Attribute(key = "solrFieldNames")
+ private Set<String> fieldNames;
+
+ /**
+ * A lazily-built cache of stop words per field.
+ */
+ private Multimap<String, CharArraySet> solrStopWords = HashMultimap.create();
+
+ /**
+ * Carrot2's default lexical resources to use in addition to Solr's stop
+ * words.
+ */
+ private DefaultLexicalDataFactory carrot2LexicalDataFactory = new DefaultLexicalDataFactory();
+
+ /**
+ * Obtains stop words for a field from the associated
+ * {@link StopFilterFactory}, if any.
+ */
+ private Collection<CharArraySet> getSolrStopWordsForField(String fieldName) {
+ // No need to synchronize here, Carrot2 ensures that instances
+ // of this class are not used by multiple threads at a time.
+ if (!solrStopWords.containsKey(fieldName)) {
+ final Analyzer fieldAnalyzer = schema.getFieldType(fieldName)
+ .getAnalyzer();
+ if (fieldAnalyzer instanceof TokenizerChain) {
+ final TokenFilterFactory[] filterFactories = ((TokenizerChain) fieldAnalyzer)
+ .getTokenFilterFactories();
+ for (TokenFilterFactory factory : filterFactories) {
+ if (factory instanceof StopFilterFactory) {
+ // StopFilterFactory holds the stop words in a CharArraySet
+ solrStopWords.put(fieldName,
+ ((StopFilterFactory) factory).getStopWords());
+ }
+
+ if (factory instanceof CommonGramsFilterFactory) {
+ solrStopWords.put(fieldName,
+ ((CommonGramsFilterFactory) factory)
+ .getCommonWords());
+ }
+ }
+ }
+ }
+ return solrStopWords.get(fieldName);
+ }
+
+ @Override
+ public ILexicalData getLexicalData(LanguageCode languageCode) {
+ final ILexicalData carrot2LexicalData = carrot2LexicalDataFactory
+ .getLexicalData(languageCode);
+
+ return new ILexicalData() {
+ @Override
+ public boolean isStopLabel(CharSequence word) {
+ // Nothing in Solr maps to the concept of a stop label,
+ // so return Carrot2's default here.
+ return carrot2LexicalData.isStopLabel(word);
+ }
+
+ @Override
+ public boolean isCommonWord(MutableCharArray word) {
+ // Loop over the fields involved in clustering first
+ for (String fieldName : fieldNames) {
+ for (CharArraySet stopWords : getSolrStopWordsForField(fieldName)) {
+ if (stopWords.contains(word)) {
+ return true;
+ }
+ }
+ }
+ // Check default Carrot2 stop words too
+ return carrot2LexicalData.isCommonWord(word);
+ }
+ };
+ }
+}
Modified: lucene/dev/trunk/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/EchoClusteringAlgorithm.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/EchoClusteringAlgorithm.java?rev=1305339&r1=1305338&r2=1305339&view=diff
==============================================================================
--- lucene/dev/trunk/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/EchoClusteringAlgorithm.java (original)
+++ lucene/dev/trunk/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/EchoClusteringAlgorithm.java Mon Mar 26 12:58:58 2012
@@ -1,78 +1,78 @@
-package org.apache.solr.handler.clustering.carrot2;
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import java.util.Collections;
-import java.util.List;
-
-import org.carrot2.core.Cluster;
-import org.carrot2.core.Document;
-import org.carrot2.core.IClusteringAlgorithm;
-import org.carrot2.core.ProcessingComponentBase;
-import org.carrot2.core.ProcessingException;
-import org.carrot2.core.attribute.AttributeNames;
-import org.carrot2.core.attribute.Processing;
-import org.carrot2.util.attribute.Attribute;
-import org.carrot2.util.attribute.Bindable;
-import org.carrot2.util.attribute.Input;
-import org.carrot2.util.attribute.Output;
-
-import com.google.common.collect.Lists;
-
-/**
- * A mock Carrot2 clustering algorithm that outputs input documents as clusters.
- * Useful only in tests.
- */
-@Bindable(prefix = "EchoClusteringAlgorithm")
-public class EchoClusteringAlgorithm extends ProcessingComponentBase implements
- IClusteringAlgorithm {
- @Input
- @Processing
- @Attribute(key = AttributeNames.DOCUMENTS)
- private List<Document> documents;
-
- @Output
- @Processing
- @Attribute(key = AttributeNames.CLUSTERS)
- private List<Cluster> clusters;
-
- @Input
- @Processing
- @Attribute(key = "custom-fields")
- private String customFields = "";
-
-
- @Override
- public void process() throws ProcessingException {
- clusters = Lists.newArrayListWithCapacity(documents.size());
-
- for (Document document : documents) {
- final Cluster cluster = new Cluster();
- cluster.addPhrases(document.getTitle(), document.getSummary());
- if (document.getLanguage() != null) {
- cluster.addPhrases(document.getLanguage().name());
- }
- for (String field : customFields.split(",")) {
- Object value = document.getField(field);
- if (value != null) {
- cluster.addPhrases(value.toString());
- }
- }
- cluster.addDocuments(document);
- clusters.add(cluster);
- }
- }
-}
+package org.apache.solr.handler.clustering.carrot2;
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.util.Collections;
+import java.util.List;
+
+import org.carrot2.core.Cluster;
+import org.carrot2.core.Document;
+import org.carrot2.core.IClusteringAlgorithm;
+import org.carrot2.core.ProcessingComponentBase;
+import org.carrot2.core.ProcessingException;
+import org.carrot2.core.attribute.AttributeNames;
+import org.carrot2.core.attribute.Processing;
+import org.carrot2.util.attribute.Attribute;
+import org.carrot2.util.attribute.Bindable;
+import org.carrot2.util.attribute.Input;
+import org.carrot2.util.attribute.Output;
+
+import com.google.common.collect.Lists;
+
+/**
+ * A mock Carrot2 clustering algorithm that outputs input documents as clusters.
+ * Useful only in tests.
+ */
+@Bindable(prefix = "EchoClusteringAlgorithm")
+public class EchoClusteringAlgorithm extends ProcessingComponentBase implements
+ IClusteringAlgorithm {
+ @Input
+ @Processing
+ @Attribute(key = AttributeNames.DOCUMENTS)
+ private List<Document> documents;
+
+ @Output
+ @Processing
+ @Attribute(key = AttributeNames.CLUSTERS)
+ private List<Cluster> clusters;
+
+ @Input
+ @Processing
+ @Attribute(key = "custom-fields")
+ private String customFields = "";
+
+
+ @Override
+ public void process() throws ProcessingException {
+ clusters = Lists.newArrayListWithCapacity(documents.size());
+
+ for (Document document : documents) {
+ final Cluster cluster = new Cluster();
+ cluster.addPhrases(document.getTitle(), document.getSummary());
+ if (document.getLanguage() != null) {
+ cluster.addPhrases(document.getLanguage().name());
+ }
+ for (String field : customFields.split(",")) {
+ Object value = document.getField(field);
+ if (value != null) {
+ cluster.addPhrases(value.toString());
+ }
+ }
+ cluster.addDocuments(document);
+ clusters.add(cluster);
+ }
+ }
+}
Modified: lucene/dev/trunk/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/LexicalResourcesCheckClusteringAlgorithm.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/LexicalResourcesCheckClusteringAlgorithm.java?rev=1305339&r1=1305338&r2=1305339&view=diff
==============================================================================
--- lucene/dev/trunk/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/LexicalResourcesCheckClusteringAlgorithm.java (original)
+++ lucene/dev/trunk/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/LexicalResourcesCheckClusteringAlgorithm.java Mon Mar 26 12:58:58 2012
@@ -1,80 +1,80 @@
-package org.apache.solr.handler.clustering.carrot2;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import java.util.List;
-
-import org.carrot2.core.Cluster;
-import org.carrot2.core.IClusteringAlgorithm;
-import org.carrot2.core.LanguageCode;
-import org.carrot2.core.ProcessingComponentBase;
-import org.carrot2.core.ProcessingException;
-import org.carrot2.core.attribute.AttributeNames;
-import org.carrot2.core.attribute.Processing;
-import org.carrot2.text.linguistic.ILexicalData;
-import org.carrot2.text.preprocessing.pipeline.BasicPreprocessingPipeline;
-import org.carrot2.text.util.MutableCharArray;
-import org.carrot2.util.attribute.Attribute;
-import org.carrot2.util.attribute.Bindable;
-import org.carrot2.util.attribute.Input;
-import org.carrot2.util.attribute.Output;
-
-import com.google.common.collect.Lists;
-
-/**
- * A mock implementation of Carrot2 clustering algorithm for testing whether the
- * customized lexical resource lookup works correctly. This algorithm ignores
- * the input documents and instead for each word from {@link #wordsToCheck}, it
- * outputs a cluster labeled with the word only if the word is neither a stop
- * word nor a stop label.
- */
-@Bindable(prefix = "LexicalResourcesCheckClusteringAlgorithm")
-public class LexicalResourcesCheckClusteringAlgorithm extends
- ProcessingComponentBase implements IClusteringAlgorithm {
-
- @Output
- @Processing
- @Attribute(key = AttributeNames.CLUSTERS)
- private List<Cluster> clusters;
-
- @Input
- @Processing
- @Attribute
- private String wordsToCheck;
-
- private BasicPreprocessingPipeline preprocessing = new BasicPreprocessingPipeline();
-
- @Override
- public void process() throws ProcessingException {
- clusters = Lists.newArrayList();
- if (wordsToCheck == null) {
- return;
- }
-
- // Test with Maltese so that the English clustering performed in other tests
- // is not affected by the test stopwords and stoplabels.
- ILexicalData lexicalData = preprocessing.lexicalDataFactory
- .getLexicalData(LanguageCode.MALTESE);
-
- for (String word : wordsToCheck.split(",")) {
- if (!lexicalData.isCommonWord(new MutableCharArray(word))
- && !lexicalData.isStopLabel(word)) {
- clusters.add(new Cluster(word));
- }
- }
- }
-}
+package org.apache.solr.handler.clustering.carrot2;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.util.List;
+
+import org.carrot2.core.Cluster;
+import org.carrot2.core.IClusteringAlgorithm;
+import org.carrot2.core.LanguageCode;
+import org.carrot2.core.ProcessingComponentBase;
+import org.carrot2.core.ProcessingException;
+import org.carrot2.core.attribute.AttributeNames;
+import org.carrot2.core.attribute.Processing;
+import org.carrot2.text.linguistic.ILexicalData;
+import org.carrot2.text.preprocessing.pipeline.BasicPreprocessingPipeline;
+import org.carrot2.text.util.MutableCharArray;
+import org.carrot2.util.attribute.Attribute;
+import org.carrot2.util.attribute.Bindable;
+import org.carrot2.util.attribute.Input;
+import org.carrot2.util.attribute.Output;
+
+import com.google.common.collect.Lists;
+
+/**
+ * A mock implementation of Carrot2 clustering algorithm for testing whether the
+ * customized lexical resource lookup works correctly. This algorithm ignores
+ * the input documents and instead for each word from {@link #wordsToCheck}, it
+ * outputs a cluster labeled with the word only if the word is neither a stop
+ * word nor a stop label.
+ */
+@Bindable(prefix = "LexicalResourcesCheckClusteringAlgorithm")
+public class LexicalResourcesCheckClusteringAlgorithm extends
+ ProcessingComponentBase implements IClusteringAlgorithm {
+
+ @Output
+ @Processing
+ @Attribute(key = AttributeNames.CLUSTERS)
+ private List<Cluster> clusters;
+
+ @Input
+ @Processing
+ @Attribute
+ private String wordsToCheck;
+
+ private BasicPreprocessingPipeline preprocessing = new BasicPreprocessingPipeline();
+
+ @Override
+ public void process() throws ProcessingException {
+ clusters = Lists.newArrayList();
+ if (wordsToCheck == null) {
+ return;
+ }
+
+ // Test with Maltese so that the English clustering performed in other tests
+ // is not affected by the test stopwords and stoplabels.
+ ILexicalData lexicalData = preprocessing.lexicalDataFactory
+ .getLexicalData(LanguageCode.MALTESE);
+
+ for (String word : wordsToCheck.split(",")) {
+ if (!lexicalData.isCommonWord(new MutableCharArray(word))
+ && !lexicalData.isStopLabel(word)) {
+ clusters.add(new Cluster(word));
+ }
+ }
+ }
+}
Modified: lucene/dev/trunk/solr/contrib/langid/src/java/overview.html
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/langid/src/java/overview.html?rev=1305339&r1=1305338&r2=1305339&view=diff
==============================================================================
--- lucene/dev/trunk/solr/contrib/langid/src/java/overview.html (original)
+++ lucene/dev/trunk/solr/contrib/langid/src/java/overview.html Mon Mar 26 12:58:58 2012
@@ -1,5 +1,5 @@
-<html>
-<body>
-Apache Solr Search Server: Solr Language Identifier contrib
-</body>
-</html>
+<html>
+<body>
+Apache Solr Search Server: Solr Language Identifier contrib
+</body>
+</html>
Modified: lucene/dev/trunk/solr/contrib/velocity/src/java/overview.html
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/velocity/src/java/overview.html?rev=1305339&r1=1305338&r2=1305339&view=diff
==============================================================================
--- lucene/dev/trunk/solr/contrib/velocity/src/java/overview.html (original)
+++ lucene/dev/trunk/solr/contrib/velocity/src/java/overview.html Mon Mar 26 12:58:58 2012
@@ -1,5 +1,5 @@
-<html>
-<body>
-Apache Solr Search Server: Velocity Response Writer contrib
-</body>
-</html>
+<html>
+<body>
+Apache Solr Search Server: Velocity Response Writer contrib
+</body>
+</html>
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java?rev=1305339&r1=1305338&r2=1305339&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java Mon Mar 26 12:58:58 2012
@@ -1,184 +1,184 @@
-package org.apache.solr.handler.component;
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import org.apache.commons.httpclient.DefaultHttpMethodRetryHandler;
-import org.apache.commons.httpclient.HttpClient;
-import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
-import org.apache.commons.httpclient.params.HttpMethodParams;
-import org.apache.solr.client.solrj.impl.LBHttpSolrServer;
-import org.apache.solr.common.SolrException;
-import org.apache.solr.common.util.NamedList;
-import org.apache.solr.core.PluginInfo;
-import org.apache.solr.util.DefaultSolrThreadFactory;
-import org.apache.solr.util.plugin.PluginInfoInitialized;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.net.MalformedURLException;
-import java.util.Random;
-import java.util.concurrent.*;
-
-public class HttpShardHandlerFactory extends ShardHandlerFactory implements PluginInfoInitialized {
- protected static Logger log = LoggerFactory.getLogger(HttpShardHandlerFactory.class);
-
- // We want an executor that doesn't take up any resources if
- // it's not used, so it could be created statically for
- // the distributed search component if desired.
- //
- // Consider CallerRuns policy and a lower max threads to throttle
- // requests at some point (or should we simply return failure?)
- ThreadPoolExecutor commExecutor = new ThreadPoolExecutor(
- 0,
- Integer.MAX_VALUE,
- 5, TimeUnit.SECONDS, // terminate idle threads after 5 sec
- new SynchronousQueue<Runnable>(), // directly hand off tasks
- new DefaultSolrThreadFactory("httpShardExecutor")
- );
-
- HttpClient client;
- Random r = new Random();
- LBHttpSolrServer loadbalancer;
- int soTimeout = 0; //current default values
- int connectionTimeout = 0; //current default values
- int maxConnectionsPerHost = 20;
- int corePoolSize = 0;
- int maximumPoolSize = 10;
- int keepAliveTime = 5;
- int queueSize = 1;
- boolean accessPolicy = true;
-
- public String scheme = "http://"; //current default values
-
- private MultiThreadedHttpConnectionManager mgr;
- // socket timeout measured in ms, closes a socket if read
- // takes longer than x ms to complete. throws
- // java.net.SocketTimeoutException: Read timed out exception
- static final String INIT_SO_TIMEOUT = "socketTimeout";
-
- // connection timeout measures in ms, closes a socket if connection
- // cannot be established within x ms. with a
- // java.net.SocketTimeoutException: Connection timed out
- static final String INIT_CONNECTION_TIMEOUT = "connTimeout";
-
- // URL scheme to be used in distributed search.
- static final String INIT_URL_SCHEME = "urlScheme";
-
- // Maximum connections allowed per host
- static final String INIT_MAX_CONNECTION_PER_HOST = "maxConnectionsPerHost";
-
- // The core size of the threadpool servicing requests
- static final String INIT_CORE_POOL_SIZE = "corePoolSize";
-
- // The maximum size of the threadpool servicing requests
- static final String INIT_MAX_POOL_SIZE = "maximumPoolSize";
-
- // The amount of time idle threads persist for in the queue, before being killed
- static final String MAX_THREAD_IDLE_TIME = "maxThreadIdleTime";
-
- // If the threadpool uses a backing queue, what is its maximum size (-1) to use direct handoff
- static final String INIT_SIZE_OF_QUEUE = "sizeOfQueue";
-
- // Configure if the threadpool favours fairness over throughput
- static final String INIT_FAIRNESS_POLICY = "fairnessPolicy";
-
- public ShardHandler getShardHandler() {
- return getShardHandler(null);
- }
-
- public ShardHandler getShardHandler(HttpClient httpClient) {
- return new HttpShardHandler(this, httpClient);
- }
-
- public void init(PluginInfo info) {
- NamedList args = info.initArgs;
- this.soTimeout = getParameter(args, INIT_SO_TIMEOUT, 0);
-
- this.scheme = getParameter(args, INIT_URL_SCHEME, "http://");
- this.scheme = (this.scheme.endsWith("://")) ? this.scheme : this.scheme + "://";
- this.connectionTimeout = getParameter(args, INIT_CONNECTION_TIMEOUT, 0);
- this.maxConnectionsPerHost = getParameter(args, INIT_MAX_CONNECTION_PER_HOST, 20);
- this.corePoolSize = getParameter(args, INIT_CORE_POOL_SIZE, 0);
- this.maximumPoolSize = getParameter(args, INIT_MAX_POOL_SIZE, Integer.MAX_VALUE);
- this.keepAliveTime = getParameter(args, MAX_THREAD_IDLE_TIME, 5);
- this.queueSize = getParameter(args, INIT_SIZE_OF_QUEUE, -1);
- this.accessPolicy = getParameter(args, INIT_FAIRNESS_POLICY, false);
-
- BlockingQueue<Runnable> blockingQueue = (this.queueSize == -1) ?
- new SynchronousQueue<Runnable>(this.accessPolicy) :
- new ArrayBlockingQueue<Runnable>(this.queueSize, this.accessPolicy);
-
- this.commExecutor = new ThreadPoolExecutor(
- this.corePoolSize,
- this.maximumPoolSize,
- this.keepAliveTime, TimeUnit.SECONDS,
- blockingQueue,
- new DefaultSolrThreadFactory("httpShardExecutor")
- );
-
- mgr = new MultiThreadedHttpConnectionManager();
- mgr.getParams().setDefaultMaxConnectionsPerHost(this.maxConnectionsPerHost);
- mgr.getParams().setMaxTotalConnections(10000);
- mgr.getParams().setConnectionTimeout(this.connectionTimeout);
- mgr.getParams().setSoTimeout(this.soTimeout);
- // mgr.getParams().setStaleCheckingEnabled(false);
-
- client = new HttpClient(mgr);
-
- // prevent retries (note: this didn't work when set on mgr.. needed to be set on client)
- DefaultHttpMethodRetryHandler retryhandler = new DefaultHttpMethodRetryHandler(0, false);
- client.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, retryhandler);
-
- try {
- loadbalancer = new LBHttpSolrServer(client);
- } catch (MalformedURLException e) {
- // should be impossible since we're not passing any URLs here
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
- }
-
- }
-
- private <T> T getParameter(NamedList initArgs, String configKey, T defaultValue) {
- T toReturn = defaultValue;
- if (initArgs != null) {
- T temp = (T) initArgs.get(configKey);
- toReturn = (temp != null) ? temp : defaultValue;
- }
- log.info("Setting {} to: {}", configKey, soTimeout);
- return toReturn;
- }
-
-
- @Override
- public void close() {
- try {
- mgr.shutdown();
- } catch (Throwable e) {
- SolrException.log(log, e);
- }
- try {
- loadbalancer.shutdown();
- } catch (Throwable e) {
- SolrException.log(log, e);
- }
- try {
- commExecutor.shutdownNow();
- } catch (Throwable e) {
- SolrException.log(log, e);
- }
- }
-}
+package org.apache.solr.handler.component;
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.commons.httpclient.DefaultHttpMethodRetryHandler;
+import org.apache.commons.httpclient.HttpClient;
+import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
+import org.apache.commons.httpclient.params.HttpMethodParams;
+import org.apache.solr.client.solrj.impl.LBHttpSolrServer;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.core.PluginInfo;
+import org.apache.solr.util.DefaultSolrThreadFactory;
+import org.apache.solr.util.plugin.PluginInfoInitialized;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.net.MalformedURLException;
+import java.util.Random;
+import java.util.concurrent.*;
+
+public class HttpShardHandlerFactory extends ShardHandlerFactory implements PluginInfoInitialized {
+ protected static Logger log = LoggerFactory.getLogger(HttpShardHandlerFactory.class);
+
+ // We want an executor that doesn't take up any resources if
+ // it's not used, so it could be created statically for
+ // the distributed search component if desired.
+ //
+ // Consider CallerRuns policy and a lower max threads to throttle
+ // requests at some point (or should we simply return failure?)
+ ThreadPoolExecutor commExecutor = new ThreadPoolExecutor(
+ 0,
+ Integer.MAX_VALUE,
+ 5, TimeUnit.SECONDS, // terminate idle threads after 5 sec
+ new SynchronousQueue<Runnable>(), // directly hand off tasks
+ new DefaultSolrThreadFactory("httpShardExecutor")
+ );
+
+ HttpClient client;
+ Random r = new Random();
+ LBHttpSolrServer loadbalancer;
+ int soTimeout = 0; //current default values
+ int connectionTimeout = 0; //current default values
+ int maxConnectionsPerHost = 20;
+ int corePoolSize = 0;
+ int maximumPoolSize = 10;
+ int keepAliveTime = 5;
+ int queueSize = 1;
+ boolean accessPolicy = true;
+
+ public String scheme = "http://"; //current default values
+
+ private MultiThreadedHttpConnectionManager mgr;
+ // socket timeout measured in ms, closes a socket if read
+ // takes longer than x ms to complete. throws
+ // java.net.SocketTimeoutException: Read timed out exception
+ static final String INIT_SO_TIMEOUT = "socketTimeout";
+
+ // connection timeout measures in ms, closes a socket if connection
+ // cannot be established within x ms. with a
+ // java.net.SocketTimeoutException: Connection timed out
+ static final String INIT_CONNECTION_TIMEOUT = "connTimeout";
+
+ // URL scheme to be used in distributed search.
+ static final String INIT_URL_SCHEME = "urlScheme";
+
+ // Maximum connections allowed per host
+ static final String INIT_MAX_CONNECTION_PER_HOST = "maxConnectionsPerHost";
+
+ // The core size of the threadpool servicing requests
+ static final String INIT_CORE_POOL_SIZE = "corePoolSize";
+
+ // The maximum size of the threadpool servicing requests
+ static final String INIT_MAX_POOL_SIZE = "maximumPoolSize";
+
+ // The amount of time idle threads persist for in the queue, before being killed
+ static final String MAX_THREAD_IDLE_TIME = "maxThreadIdleTime";
+
+ // If the threadpool uses a backing queue, what is its maximum size (-1) to use direct handoff
+ static final String INIT_SIZE_OF_QUEUE = "sizeOfQueue";
+
+ // Configure if the threadpool favours fairness over throughput
+ static final String INIT_FAIRNESS_POLICY = "fairnessPolicy";
+
+ public ShardHandler getShardHandler() {
+ return getShardHandler(null);
+ }
+
+ public ShardHandler getShardHandler(HttpClient httpClient) {
+ return new HttpShardHandler(this, httpClient);
+ }
+
+ public void init(PluginInfo info) {
+ NamedList args = info.initArgs;
+ this.soTimeout = getParameter(args, INIT_SO_TIMEOUT, 0);
+
+ this.scheme = getParameter(args, INIT_URL_SCHEME, "http://");
+ this.scheme = (this.scheme.endsWith("://")) ? this.scheme : this.scheme + "://";
+ this.connectionTimeout = getParameter(args, INIT_CONNECTION_TIMEOUT, 0);
+ this.maxConnectionsPerHost = getParameter(args, INIT_MAX_CONNECTION_PER_HOST, 20);
+ this.corePoolSize = getParameter(args, INIT_CORE_POOL_SIZE, 0);
+ this.maximumPoolSize = getParameter(args, INIT_MAX_POOL_SIZE, Integer.MAX_VALUE);
+ this.keepAliveTime = getParameter(args, MAX_THREAD_IDLE_TIME, 5);
+ this.queueSize = getParameter(args, INIT_SIZE_OF_QUEUE, -1);
+ this.accessPolicy = getParameter(args, INIT_FAIRNESS_POLICY, false);
+
+ BlockingQueue<Runnable> blockingQueue = (this.queueSize == -1) ?
+ new SynchronousQueue<Runnable>(this.accessPolicy) :
+ new ArrayBlockingQueue<Runnable>(this.queueSize, this.accessPolicy);
+
+ this.commExecutor = new ThreadPoolExecutor(
+ this.corePoolSize,
+ this.maximumPoolSize,
+ this.keepAliveTime, TimeUnit.SECONDS,
+ blockingQueue,
+ new DefaultSolrThreadFactory("httpShardExecutor")
+ );
+
+ mgr = new MultiThreadedHttpConnectionManager();
+ mgr.getParams().setDefaultMaxConnectionsPerHost(this.maxConnectionsPerHost);
+ mgr.getParams().setMaxTotalConnections(10000);
+ mgr.getParams().setConnectionTimeout(this.connectionTimeout);
+ mgr.getParams().setSoTimeout(this.soTimeout);
+ // mgr.getParams().setStaleCheckingEnabled(false);
+
+ client = new HttpClient(mgr);
+
+ // prevent retries (note: this didn't work when set on mgr.. needed to be set on client)
+ DefaultHttpMethodRetryHandler retryhandler = new DefaultHttpMethodRetryHandler(0, false);
+ client.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, retryhandler);
+
+ try {
+ loadbalancer = new LBHttpSolrServer(client);
+ } catch (MalformedURLException e) {
+ // should be impossible since we're not passing any URLs here
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
+ }
+
+ }
+
+ private <T> T getParameter(NamedList initArgs, String configKey, T defaultValue) {
+ T toReturn = defaultValue;
+ if (initArgs != null) {
+ T temp = (T) initArgs.get(configKey);
+ toReturn = (temp != null) ? temp : defaultValue;
+ }
+ log.info("Setting {} to: {}", configKey, soTimeout);
+ return toReturn;
+ }
+
+
+ @Override
+ public void close() {
+ try {
+ mgr.shutdown();
+ } catch (Throwable e) {
+ SolrException.log(log, e);
+ }
+ try {
+ loadbalancer.shutdown();
+ } catch (Throwable e) {
+ SolrException.log(log, e);
+ }
+ try {
+ commExecutor.shutdownNow();
+ } catch (Throwable e) {
+ SolrException.log(log, e);
+ }
+ }
+}
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/ShardHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/ShardHandler.java?rev=1305339&r1=1305338&r2=1305339&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/ShardHandler.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/ShardHandler.java Mon Mar 26 12:58:58 2012
@@ -1,28 +1,28 @@
-package org.apache.solr.handler.component;
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-import org.apache.solr.common.params.ModifiableSolrParams;
-
-public abstract class ShardHandler {
- public abstract void checkDistributed(ResponseBuilder rb);
- public abstract void submit(ShardRequest sreq, String shard, ModifiableSolrParams params) ;
- public abstract ShardResponse takeCompletedIncludingErrors();
- public abstract ShardResponse takeCompletedOrError();
- public abstract void cancelAll();
-}
+package org.apache.solr.handler.component;
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+import org.apache.solr.common.params.ModifiableSolrParams;
+
+public abstract class ShardHandler {
+ public abstract void checkDistributed(ResponseBuilder rb);
+ public abstract void submit(ShardRequest sreq, String shard, ModifiableSolrParams params) ;
+ public abstract ShardResponse takeCompletedIncludingErrors();
+ public abstract ShardResponse takeCompletedOrError();
+ public abstract void cancelAll();
+}
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/ShardHandlerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/ShardHandlerFactory.java?rev=1305339&r1=1305338&r2=1305339&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/ShardHandlerFactory.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/ShardHandlerFactory.java Mon Mar 26 12:58:58 2012
@@ -1,25 +1,25 @@
-package org.apache.solr.handler.component;
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-public abstract class ShardHandlerFactory {
-
- public abstract ShardHandler getShardHandler();
-
- public abstract void close();
-}
+package org.apache.solr.handler.component;
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+public abstract class ShardHandlerFactory {
+
+ public abstract ShardHandler getShardHandler();
+
+ public abstract void close();
+}
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/DocSetCollector.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/DocSetCollector.java?rev=1305339&r1=1305338&r2=1305339&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/DocSetCollector.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/DocSetCollector.java Mon Mar 26 12:58:58 2012
@@ -1,96 +1,96 @@
-package org.apache.solr.search;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.AtomicReaderContext;
-import org.apache.lucene.search.Collector;
-import org.apache.lucene.search.Scorer;
-import org.apache.lucene.util.OpenBitSet;
-
-import java.io.IOException;
-
-/**
- *
- */
-
-public class DocSetCollector extends Collector {
- int pos=0;
- OpenBitSet bits;
- final int maxDoc;
- final int smallSetSize;
- int base;
-
- // in case there aren't that many hits, we may not want a very sparse
- // bit array. Optimistically collect the first few docs in an array
- // in case there are only a few.
- final int[] scratch;
-
- public DocSetCollector(int smallSetSize, int maxDoc) {
- this.smallSetSize = smallSetSize;
- this.maxDoc = maxDoc;
- this.scratch = new int[smallSetSize];
- }
-
- @Override
- public void collect(int doc) throws IOException {
- doc += base;
- // optimistically collect the first docs in an array
- // in case the total number will be small enough to represent
- // as a small set like SortedIntDocSet instead...
- // Storing in this array will be quicker to convert
- // than scanning through a potentially huge bit vector.
- // FUTURE: when search methods all start returning docs in order, maybe
- // we could have a ListDocSet() and use the collected array directly.
- if (pos < scratch.length) {
- scratch[pos]=doc;
- } else {
- // this conditional could be removed if BitSet was preallocated, but that
- // would take up more memory, and add more GC time...
- if (bits==null) bits = new OpenBitSet(maxDoc);
- bits.fastSet(doc);
- }
-
- pos++;
- }
-
- public DocSet getDocSet() {
- if (pos<=scratch.length) {
- // assumes docs were collected in sorted order!
- return new SortedIntDocSet(scratch, pos);
- } else {
- // set the bits for ids that were collected in the array
- for (int i=0; i<scratch.length; i++) bits.fastSet(scratch[i]);
- return new BitDocSet(bits,pos);
- }
- }
-
- @Override
- public void setScorer(Scorer scorer) throws IOException {
- }
-
- @Override
- public void setNextReader(AtomicReaderContext context) throws IOException {
- this.base = context.docBase;
- }
-
- @Override
- public boolean acceptsDocsOutOfOrder() {
- return false;
- }
-}
+package org.apache.solr.search;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.AtomicReaderContext;
+import org.apache.lucene.search.Collector;
+import org.apache.lucene.search.Scorer;
+import org.apache.lucene.util.OpenBitSet;
+
+import java.io.IOException;
+
+/**
+ *
+ */
+
+public class DocSetCollector extends Collector {
+ int pos=0;
+ OpenBitSet bits;
+ final int maxDoc;
+ final int smallSetSize;
+ int base;
+
+ // in case there aren't that many hits, we may not want a very sparse
+ // bit array. Optimistically collect the first few docs in an array
+ // in case there are only a few.
+ final int[] scratch;
+
+ public DocSetCollector(int smallSetSize, int maxDoc) {
+ this.smallSetSize = smallSetSize;
+ this.maxDoc = maxDoc;
+ this.scratch = new int[smallSetSize];
+ }
+
+ @Override
+ public void collect(int doc) throws IOException {
+ doc += base;
+ // optimistically collect the first docs in an array
+ // in case the total number will be small enough to represent
+ // as a small set like SortedIntDocSet instead...
+ // Storing in this array will be quicker to convert
+ // than scanning through a potentially huge bit vector.
+ // FUTURE: when search methods all start returning docs in order, maybe
+ // we could have a ListDocSet() and use the collected array directly.
+ if (pos < scratch.length) {
+ scratch[pos]=doc;
+ } else {
+ // this conditional could be removed if BitSet was preallocated, but that
+ // would take up more memory, and add more GC time...
+ if (bits==null) bits = new OpenBitSet(maxDoc);
+ bits.fastSet(doc);
+ }
+
+ pos++;
+ }
+
+ public DocSet getDocSet() {
+ if (pos<=scratch.length) {
+ // assumes docs were collected in sorted order!
+ return new SortedIntDocSet(scratch, pos);
+ } else {
+ // set the bits for ids that were collected in the array
+ for (int i=0; i<scratch.length; i++) bits.fastSet(scratch[i]);
+ return new BitDocSet(bits,pos);
+ }
+ }
+
+ @Override
+ public void setScorer(Scorer scorer) throws IOException {
+ }
+
+ @Override
+ public void setNextReader(AtomicReaderContext context) throws IOException {
+ this.base = context.docBase;
+ }
+
+ @Override
+ public boolean acceptsDocsOutOfOrder() {
+ return false;
+ }
+}