You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@stanbol.apache.org by og...@apache.org on 2012/01/18 21:26:04 UTC
svn commit: r1233043 [1/2] - in
/incubator/stanbol/trunk/enhancer/engines/topic/src:
main/java/org/apache/stanbol/enhancer/engine/topic/
main/java/org/apache/stanbol/enhancer/topic/ main/resources/classifier/
main/resources/trainingset/ test/java/org/a...
Author: ogrisel
Date: Wed Jan 18 20:26:03 2012
New Revision: 1233043
URL: http://svn.apache.org/viewvc?rev=1233043&view=rev
Log:
STANBOL-197: work in progress on cross validation
Added:
incubator/stanbol/trunk/enhancer/engines/topic/src/main/java/org/apache/stanbol/enhancer/topic/EmbeddedSolrHelper.java
incubator/stanbol/trunk/enhancer/engines/topic/src/main/resources/classifier/
incubator/stanbol/trunk/enhancer/engines/topic/src/main/resources/classifier/schema.xml
incubator/stanbol/trunk/enhancer/engines/topic/src/main/resources/classifier/solr.xml
incubator/stanbol/trunk/enhancer/engines/topic/src/main/resources/classifier/solrconfig.xml
incubator/stanbol/trunk/enhancer/engines/topic/src/main/resources/trainingset/
incubator/stanbol/trunk/enhancer/engines/topic/src/main/resources/trainingset/schema.xml
incubator/stanbol/trunk/enhancer/engines/topic/src/main/resources/trainingset/solr.xml
incubator/stanbol/trunk/enhancer/engines/topic/src/main/resources/trainingset/solrconfig.xml
Removed:
incubator/stanbol/trunk/enhancer/engines/topic/src/test/java/org/apache/stanbol/enhancer/engine/topic/BaseTestWithSolrCore.java
incubator/stanbol/trunk/enhancer/engines/topic/src/test/resources/classifier/
incubator/stanbol/trunk/enhancer/engines/topic/src/test/resources/trainingset/
Modified:
incubator/stanbol/trunk/enhancer/engines/topic/src/main/java/org/apache/stanbol/enhancer/engine/topic/TopicClassificationEngine.java
incubator/stanbol/trunk/enhancer/engines/topic/src/main/java/org/apache/stanbol/enhancer/topic/TopicSuggestion.java
incubator/stanbol/trunk/enhancer/engines/topic/src/test/java/org/apache/stanbol/enhancer/engine/topic/TopicEngineTest.java
incubator/stanbol/trunk/enhancer/engines/topic/src/test/java/org/apache/stanbol/enhancer/engine/topic/TrainingSetTest.java
Modified: incubator/stanbol/trunk/enhancer/engines/topic/src/main/java/org/apache/stanbol/enhancer/engine/topic/TopicClassificationEngine.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/enhancer/engines/topic/src/main/java/org/apache/stanbol/enhancer/engine/topic/TopicClassificationEngine.java?rev=1233043&r1=1233042&r2=1233043&view=diff
==============================================================================
--- incubator/stanbol/trunk/enhancer/engines/topic/src/main/java/org/apache/stanbol/enhancer/engine/topic/TopicClassificationEngine.java (original)
+++ incubator/stanbol/trunk/enhancer/engines/topic/src/main/java/org/apache/stanbol/enhancer/engine/topic/TopicClassificationEngine.java Wed Jan 18 20:26:03 2012
@@ -73,6 +73,7 @@ import org.apache.stanbol.enhancer.topic
import org.apache.stanbol.enhancer.topic.ClassificationReport;
import org.apache.stanbol.enhancer.topic.ClassifierException;
import org.apache.stanbol.enhancer.topic.ConfiguredSolrCoreTracker;
+import org.apache.stanbol.enhancer.topic.EmbeddedSolrHelper;
import org.apache.stanbol.enhancer.topic.TopicClassifier;
import org.apache.stanbol.enhancer.topic.TopicSuggestion;
import org.apache.stanbol.enhancer.topic.TrainingSet;
@@ -621,12 +622,13 @@ public class TopicClassificationEngine e
public int process(List<SolrDocument> batch) throws ClassifierException, TrainingSetException {
int processed = 0;
for (SolrDocument result : batch) {
- offset++;
- if (cvFoldCount != 0 && offset % cvFoldCount == cvFoldIndex) {
+ if ((cvFoldCount != 0) && (offset % cvFoldCount == cvFoldIndex)) {
// we are performing a cross validation session and this example belong to the test
// fold hence should be skipped
+ offset++;
continue;
}
+ offset++;
String topicId = result.getFirstValue(topicUriField).toString();
List<String> impactedTopics = new ArrayList<String>();
impactedTopics.add(topicId);
@@ -758,11 +760,6 @@ public class TopicClassificationEngine e
return config;
}
- protected EmbeddedSolrServer makeTopicClassifierSolrServer(File folder) {
- // TODO
- return null;
- }
-
public boolean isEvaluationRunning() {
return evaluationFolder != null;
}
@@ -781,7 +778,7 @@ public class TopicClassificationEngine e
// TODO: make the temporary folder path configurable with a property
evaluationFolder = File.createTempFile("stanbol-classifier-evaluation-", "-solr");
for (int cvFoldIndex = 0; cvFoldIndex < cvFoldCount; cvFoldIndex++) {
- performCVFold(classifier, cvFoldIndex, cvFoldCount);
+ updatedTopics = performCVFold(classifier, cvFoldIndex, cvFoldCount, incremental);
}
} catch (ConfigurationException e) {
throw new ClassifierException(e);
@@ -794,17 +791,25 @@ public class TopicClassificationEngine e
return updatedTopics;
}
- protected void performCVFold(final TopicClassificationEngine classifier, int cvFoldIndex, int cvFoldCount) throws ConfigurationException,
- TrainingSetException,
- ClassifierException {
+ protected int performCVFold(final TopicClassificationEngine classifier,
+ int cvFoldIndex,
+ int cvFoldCount,
+ boolean incremental) throws ConfigurationException,
+ TrainingSetException,
+ ClassifierException {
log.info(String.format("Performing evaluation CV iteration %d/%d on classifier %s", cvFoldIndex + 1,
cvFoldCount, engineId));
long start = System.currentTimeMillis();
FileUtils.deleteQuietly(evaluationFolder);
evaluationFolder.mkdir();
- EmbeddedSolrServer evaluationServer = makeTopicClassifierSolrServer(evaluationFolder);
- classifier.configure(getCanonicalConfiguration(evaluationServer));
+ try {
+ EmbeddedSolrServer evaluationServer = EmbeddedSolrHelper.makeEmbeddedSolrServer(evaluationFolder,
+ "evaluationclassifierserver", "classifier", "classifier");
+ classifier.configure(getCanonicalConfiguration(evaluationServer));
+ } catch (Exception e) {
+ throw new ClassifierException(e);
+ }
// iterate over all the topics to register them in the evaluation classifier
batchOverTopics(new BatchProcessor<SolrDocument>() {
@@ -835,23 +840,21 @@ public class TopicClassificationEngine e
final int foldIndex = cvFoldIndex;
// iterate over the topics again to compute scores on the test fold
- batchOverTopics(new BatchProcessor<SolrDocument>() {
+ int updatedTopics = batchOverTopics(new BatchProcessor<SolrDocument>() {
@Override
public int process(List<SolrDocument> batch) throws TrainingSetException, ClassifierException {
int offset;
for (SolrDocument topicMetadata : batch) {
String topic = topicMetadata.getFirstValue(topicUriField).toString();
- List<String> impactedTopics = new ArrayList<String>();
-
- Batch<String> examples = Batch.emtpyBatch(String.class);
-
+ List<String> topics = Arrays.asList(topic);
List<String> falseNegativeExamples = new ArrayList<String>();
int truePositives = 0;
int falseNegatives = 0;
offset = 0;
+ Batch<String> examples = Batch.emtpyBatch(String.class);
do {
- examples = trainingSet.getPositiveExamples(impactedTopics, examples.nextOffset);
+ examples = trainingSet.getPositiveExamples(topics, examples.nextOffset);
for (String example : examples.items) {
if (!(offset % foldCount == foldIndex)) {
// this example is not part of the test fold, skip it
@@ -859,11 +862,18 @@ public class TopicClassificationEngine e
continue;
}
offset++;
- if (classifier.suggestTopics(example).contains(topic)) {
- truePositives++;
- } else {
- falseNegatives++;
- // falseNegativeExamples.add(exampleId);
+ List<TopicSuggestion> suggestedTopics = classifier.suggestTopics(example);
+ for (TopicSuggestion suggestedTopic : suggestedTopics) {
+ boolean match = false;
+ if (topic.equals(suggestedTopic.uri)) {
+ match = true;
+ truePositives++;
+ break;
+ }
+ if (!match) {
+ falseNegatives++;
+ // falseNegativeExamples.add(exampleId);
+ }
}
}
} while (examples.hasMore); // TODO: put a bound on the number of examples
@@ -872,8 +882,9 @@ public class TopicClassificationEngine e
int trueNegatives = 0;
int falsePositives = 0;
offset = 0;
+ examples = Batch.emtpyBatch(String.class);
do {
- examples = trainingSet.getNegativeExamples(impactedTopics, examples.nextOffset);
+ examples = trainingSet.getNegativeExamples(topics, examples.nextOffset);
for (String example : examples.items) {
if (!(offset % foldCount == foldIndex)) {
// TODO: change the dataset API to include exampleId
@@ -882,12 +893,18 @@ public class TopicClassificationEngine e
continue;
}
offset++;
- if (classifier.suggestTopics(example).contains(topic)) {
- falsePositives++;
- // TODO: change the dataset API to include exampleId
- // falsePositiveExamples.add(exampleId);
- } else {
- trueNegatives++;
+ List<TopicSuggestion> suggestedTopics = classifier.suggestTopics(example);
+ for (TopicSuggestion suggestedTopic : suggestedTopics) {
+ boolean match = false;
+ if (topic.equals(suggestedTopic.uri)) {
+ match = true;
+ falsePositives++;
+ // falsePositiveExamples.add(exampleId);
+ break;
+ }
+ if (!match) {
+ trueNegatives++;
+ }
}
}
} while (examples.hasMore); // TODO: put a bound on the number of examples
@@ -921,6 +938,7 @@ public class TopicClassificationEngine e
long stop = System.currentTimeMillis();
log.info(String.format("Finished CV iteration %d/%d on classifier %s in %fs. F1-score = %f",
cvFoldIndex + 1, cvFoldCount, engineId, (stop - start) / 1000.0, averageF1));
+ return updatedTopics;
}
/**
Added: incubator/stanbol/trunk/enhancer/engines/topic/src/main/java/org/apache/stanbol/enhancer/topic/EmbeddedSolrHelper.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/enhancer/engines/topic/src/main/java/org/apache/stanbol/enhancer/topic/EmbeddedSolrHelper.java?rev=1233043&view=auto
==============================================================================
--- incubator/stanbol/trunk/enhancer/engines/topic/src/main/java/org/apache/stanbol/enhancer/topic/EmbeddedSolrHelper.java (added)
+++ incubator/stanbol/trunk/enhancer/engines/topic/src/main/java/org/apache/stanbol/enhancer/topic/EmbeddedSolrHelper.java Wed Jan 18 20:26:03 2012
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.stanbol.enhancer.topic;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+
+import javax.xml.parsers.ParserConfigurationException;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer;
+import org.apache.solr.core.CoreContainer;
+import org.xml.sax.SAXException;
+
+/**
+ * Helper class to build an embedded Solr server with a configuration files looked-up from the classpath. This
+ * helper is both useful in real service usage (for Cross Validation) and in tests.
+ */
+public class EmbeddedSolrHelper {
+
+ /**
+ * Create a single core Solr server with it's own folder hierarchy.
+ */
+ public static EmbeddedSolrServer makeEmbeddedSolrServer(File rootFolder,
+ String solrServerId,
+ String coreId,
+ String configName) throws IOException,
+ ParserConfigurationException,
+ SAXException {
+ File solrFolder = new File(rootFolder, solrServerId);
+ FileUtils.deleteQuietly(solrFolder);
+ solrFolder.mkdir();
+
+ // solr conf file
+ File solrFile = new File(solrFolder, "solr.xml");
+ InputStream is = EmbeddedSolrHelper.class.getResourceAsStream("/" + configName + "/solr.xml");
+ if (is == null) {
+ throw new IllegalArgumentException("missing test solr.xml file");
+ }
+ IOUtils.copy(is, new FileOutputStream(solrFile));
+
+ // solr conf folder with schema
+ File solrCoreFolder = new File(solrFolder, coreId);
+ solrCoreFolder.mkdir();
+ File solrConfFolder = new File(solrCoreFolder, "conf");
+ solrConfFolder.mkdir();
+ File schemaFile = new File(solrConfFolder, "schema.xml");
+ is = EmbeddedSolrHelper.class.getResourceAsStream("/" + configName + "/schema.xml");
+ if (is == null) {
+ throw new IllegalArgumentException("missing test schema.xml file");
+ }
+ IOUtils.copy(is, new FileOutputStream(schemaFile));
+
+ File solrConfigFile = new File(solrConfFolder, "solrconfig.xml");
+ is = EmbeddedSolrHelper.class.getResourceAsStream("/" + configName + "/solrconfig.xml");
+ if (is == null) {
+ throw new IllegalArgumentException("missing test solrconfig.xml file");
+ }
+ IOUtils.copy(is, new FileOutputStream(solrConfigFile));
+
+ // create the embedded server
+ CoreContainer coreContainer = new CoreContainer(solrFolder.getAbsolutePath(), solrFile);
+ return new EmbeddedSolrServer(coreContainer, coreId);
+ }
+
+}
Modified: incubator/stanbol/trunk/enhancer/engines/topic/src/main/java/org/apache/stanbol/enhancer/topic/TopicSuggestion.java
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/enhancer/engines/topic/src/main/java/org/apache/stanbol/enhancer/topic/TopicSuggestion.java?rev=1233043&r1=1233042&r2=1233043&view=diff
==============================================================================
--- incubator/stanbol/trunk/enhancer/engines/topic/src/main/java/org/apache/stanbol/enhancer/topic/TopicSuggestion.java (original)
+++ incubator/stanbol/trunk/enhancer/engines/topic/src/main/java/org/apache/stanbol/enhancer/topic/TopicSuggestion.java Wed Jan 18 20:26:03 2012
@@ -19,6 +19,8 @@ package org.apache.stanbol.enhancer.topi
import java.util.ArrayList;
import java.util.List;
+import org.apache.commons.lang.StringUtils;
+
/**
* Data transfer object for the individual topic classification results.
*/
@@ -42,4 +44,9 @@ public class TopicSuggestion {
this(uri, null, score);
}
+ @Override
+ public String toString() {
+ return String.format("TopicSuggestion(\"%s\", [%s], %f)", uri, StringUtils.join(paths, "\", \""),
+ score);
+ }
}
Added: incubator/stanbol/trunk/enhancer/engines/topic/src/main/resources/classifier/schema.xml
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/enhancer/engines/topic/src/main/resources/classifier/schema.xml?rev=1233043&view=auto
==============================================================================
--- incubator/stanbol/trunk/enhancer/engines/topic/src/main/resources/classifier/schema.xml (added)
+++ incubator/stanbol/trunk/enhancer/engines/topic/src/main/resources/classifier/schema.xml Wed Jan 18 20:26:03 2012
@@ -0,0 +1,109 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
+ license agreements. See the NOTICE file distributed with this work for additional
+ information regarding copyright ownership. The ASF licenses this file to
+ You under the Apache License, Version 2.0 (the "License"); you may not use
+ this file except in compliance with the License. You may obtain a copy of
+ the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+ by applicable law or agreed to in writing, software distributed under the
+ License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+ OF ANY KIND, either express or implied. See the License for the specific
+ language governing permissions and limitations under the License. -->
+<schema name="example" version="1.3">
+ <types>
+ <fieldType name="uuid" class="solr.UUIDField" indexed="true" />
+
+ <fieldType name="string" class="solr.StrField"
+ sortMissingLast="true" omitNorms="true" />
+
+ <fieldType name="tint" class="solr.TrieIntField"
+ precisionStep="0" omitNorms="true" positionIncrementGap="0" />
+
+ <fieldType name="tfloat" class="solr.TrieFloatField"
+ precisionStep="0" omitNorms="true" positionIncrementGap="0" />
+
+ <fieldType name="tdate" class="solr.TrieDateField"
+ omitNorms="true" precisionStep="6" positionIncrementGap="0" />
+
+ <fieldType name="random" class="solr.RandomSortField"
+ indexed="true" />
+
+ <fieldType name="text" class="solr.TextField">
+ <analyzer type="index">
+ <tokenizer class="solr.StandardTokenizerFactory" />
+ <!-- <filter class="solr.StopFilterFactory" ignoreCase="true" -->
+ <!-- words="stopwords_en.txt" enablePositionIncrements="false" /> -->
+ <filter class="solr.LowerCaseFilterFactory" />
+ <!-- The use of Shingle might help improve the quality but they increase
+ the size of the index far too much. It would be better to use a collocation
+ bloom filter to mitigate this effect: http://issues.apache.org/jira/browse/MAHOUT-415
+ <filter class="solr.ShingleFilterFactory" maxShingleSize="2" outputUnigrams="true"/> -->
+ </analyzer>
+ <analyzer type="query">
+ <tokenizer class="solr.StandardTokenizerFactory" />
+ <!-- <filter class="solr.StopFilterFactory" ignoreCase="true" -->
+ <!-- words="stopwords_en.txt" enablePositionIncrements="false" /> -->
+ <!-- <filter class="solr.SynonymFilterFactory" -->
+ <!-- synonyms="synonyms.txt" ignoreCase="true" expand="true"/> -->
+ <filter class="solr.LowerCaseFilterFactory" />
+ <!-- <filter class="solr.ShingleFilterFactory" maxShingleSize="2"
+ outputUnigrams="true"/> -->
+ </analyzer>
+ </fieldType>
+
+ </types>
+
+ <fields>
+ <!-- Physical (automated) primary key. Each topic is stored into 2 Solr
+ entries to be able to handle the partial update of stored attributes such
+ as estimation of the predictive accuracy and broader topic links while preserving
+ the previous version of the statistical model -->
+ <field name="entry_id" type="string" indexed="true" stored="true"
+ required="true" />
+
+ <!-- Mandatory field for all entries: this is the logical primary key -->
+ <field name="topic" type="string" indexed="true" stored="true"
+ required="true" />
+
+ <!-- If entry_type can be model 'model' or 'metadata' -->
+ <field name="entry_type" type="string" indexed="true" stored="true"
+ required="true" />
+
+ <!-- Mandatory classifier model attribute when entry_type == 'model' -->
+ <field name="classifier_features" type="text" indexed="true"
+ stored="false" termVectors="true" termPositions="false"
+ termOffsets="false" />
+
+ <!-- Classifier model stored attributes when entry_type == 'metadata' -->
+ <field name="model_entry_id" type="string" indexed="true"
+ stored="true" />
+ <field name="broader" type="string" indexed="true" stored="true"
+ multiValued="true" />
+ <field name="last_update_dt" type="tdate" indexed="true"
+ stored="true" />
+ <!-- Accuracy evaluation of the model (accross CV folds) -->
+ <field name="precision" type="tfloat" indexed="true" stored="true"
+ multiValued="true" />
+ <field name="recall" type="tfloat" indexed="true" stored="true"
+ multiValued="true" />
+ <field name="f1" type="tfloat" indexed="true" stored="true"
+ multiValued="true" />
+ <field name="last_evaluation_dt" type="tdate" indexed="true"
+ stored="true" />
+ <field name="positive_support" type="tint" indexed="false"
+ stored="true" multiValued="true" />
+ <field name="negative_support" type="tint" indexed="false"
+ stored="true" multiValued="true" />
+ <!-- Store ids of some false positive and negative examples (accumulated
+ over several CV folds) -->
+ <field name="false_positives" type="string" indexed="false"
+ multiValued="true" stored="true" />
+ <field name="negative_positives" type="string" indexed="false"
+ multiValued="true" stored="true" />
+
+ </fields>
+
+ <uniqueKey>entry_id</uniqueKey>
+ <defaultSearchField>classifier_features</defaultSearchField>
+ <solrQueryParser defaultOperator="AND" />
+</schema>
Added: incubator/stanbol/trunk/enhancer/engines/topic/src/main/resources/classifier/solr.xml
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/enhancer/engines/topic/src/main/resources/classifier/solr.xml?rev=1233043&view=auto
==============================================================================
--- incubator/stanbol/trunk/enhancer/engines/topic/src/main/resources/classifier/solr.xml (added)
+++ incubator/stanbol/trunk/enhancer/engines/topic/src/main/resources/classifier/solr.xml Wed Jan 18 20:26:03 2012
@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<solr persistent="false">
+
+ <!--
+ adminPath: RequestHandler path to manage cores.
+ If 'null' (or absent), cores will not be manageable via request handler
+ -->
+ <cores adminPath="/admin/cores" defaultCoreName="classifier">
+ <core name="classifier" instanceDir="classifier" />
+ </cores>
+</solr>
Added: incubator/stanbol/trunk/enhancer/engines/topic/src/main/resources/classifier/solrconfig.xml
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/enhancer/engines/topic/src/main/resources/classifier/solrconfig.xml?rev=1233043&view=auto
==============================================================================
--- incubator/stanbol/trunk/enhancer/engines/topic/src/main/resources/classifier/solrconfig.xml (added)
+++ incubator/stanbol/trunk/enhancer/engines/topic/src/main/resources/classifier/solrconfig.xml Wed Jan 18 20:26:03 2012
@@ -0,0 +1,928 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!--
+ For more details about configurations options that may appear in
+ this file, see http://wiki.apache.org/solr/SolrConfigXml.
+-->
+<config>
+ <!-- In all configuration below, a prefix of "solr." for class names
+ is an alias that causes solr to search appropriate packages,
+ including org.apache.solr.(search|update|request|core|analysis)
+
+ You may also specify a fully qualified Java classname if you
+ have your own custom plugins.
+ -->
+
+ <!-- Set this to 'false' if you want solr to continue working after
+ it has encountered an severe configuration error. In a
+ production environment, you may want solr to keep working even
+ if one handler is mis-configured.
+
+ You may also set this to false using by setting the system
+ property:
+
+ -Dsolr.abortOnConfigurationError=false
+ -->
+ <abortOnConfigurationError>${solr.abortOnConfigurationError:true}</abortOnConfigurationError>
+
+ <!-- Controls what version of Lucene various components of Solr
+ adhere to. Generally, you want to use the latest version to
+ get all bug fixes and improvements. It is highly recommended
+ that you fully re-index after changing this setting as it can
+ affect both how text is indexed and queried.
+ -->
+ <luceneMatchVersion>LUCENE_32</luceneMatchVersion>
+
+ <!-- lib directives can be used to instruct Solr to load an Jars
+ identified and use them to resolve any "plugins" specified in
+ your solrconfig.xml or schema.xml (ie: Analyzers, Request
+ Handlers, etc...).
+
+ All directories and paths are resolved relative to the
+ instanceDir.
+
+ If a "./lib" directory exists in your instanceDir, all files
+ found in it are included as if you had used the following
+ syntax...
+
+ <lib dir="./lib" />
+ -->
+ <!-- A dir option by itself adds any files found in the directory to
+ the classpath, this is useful for including all jars in a
+ directory.
+ -->
+ <lib dir="../../contrib/extraction/lib" />
+ <!-- When a regex is specified in addition to a directory, only the
+ files in that directory which completely match the regex
+ (anchored on both ends) will be included.
+ -->
+ <lib dir="../../dist/" regex="apache-solr-cell-\d.*\.jar" />
+ <lib dir="../../dist/" regex="apache-solr-clustering-\d.*\.jar" />
+ <lib dir="../../dist/" regex="apache-solr-dataimporthandler-\d.*\.jar" />
+
+ <!-- If a dir option (with or without a regex) is used and nothing
+ is found that matches, it will be ignored
+ -->
+ <lib dir="../../contrib/clustering/lib/" />
+ <lib dir="/total/crap/dir/ignored" />
+ <!-- an exact path can be used to specify a specific file. This
+ will cause a serious error to be logged if it can't be loaded.
+ -->
+ <!--
+ <lib path="../a-jar-that-does-not-exist.jar" />
+ -->
+
+ <!-- Data Directory
+
+ Used to specify an alternate directory to hold all index data
+ other than the default ./data under the Solr home. If
+ replication is in use, this should match the replication
+ configuration.
+ -->
+ <dataDir>${solr.data.dir:}</dataDir>
+
+
+ <!-- The DirectoryFactory to use for indexes.
+
+ solr.StandardDirectoryFactory, the default, is filesystem
+ based. solr.RAMDirectoryFactory is memory based, not
+ persistent, and doesn't work with replication.
+ -->
+ <directoryFactory name="DirectoryFactory"
+ class="${solr.directoryFactory:solr.StandardDirectoryFactory}"/>
+
+
+ <!-- Index Defaults
+
+ Values here affect all index writers and act as a default
+ unless overridden.
+
+ WARNING: See also the <mainIndex> section below for parameters
+ that overfor Solr's main Lucene index.
+ -->
+ <indexDefaults>
+
+ <useCompoundFile>false</useCompoundFile>
+
+ <mergeFactor>10</mergeFactor>
+ <!-- Sets the amount of RAM that may be used by Lucene indexing
+ for buffering added documents and deletions before they are
+ flushed to the Directory. -->
+ <ramBufferSizeMB>32</ramBufferSizeMB>
+ <!-- If both ramBufferSizeMB and maxBufferedDocs is set, then
+ Lucene will flush based on whichever limit is hit first.
+ -->
+ <!-- <maxBufferedDocs>1000</maxBufferedDocs> -->
+
+ <maxFieldLength>100000</maxFieldLength>
+ <writeLockTimeout>1000</writeLockTimeout>
+ <commitLockTimeout>10000</commitLockTimeout>
+
+ <!-- Expert: Merge Policy
+
+ The Merge Policy in Lucene controls how merging is handled by
+ Lucene. The default in Solr 3.3 is TieredMergePolicy.
+
+ The default in 2.3 was the LogByteSizeMergePolicy,
+ previous versions used LogDocMergePolicy.
+
+ LogByteSizeMergePolicy chooses segments to merge based on
+ their size. The Lucene 2.2 default, LogDocMergePolicy chose
+ when to merge based on number of documents
+
+ Other implementations of MergePolicy must have a no-argument
+ constructor
+ -->
+ <!--
+ <mergePolicy class="org.apache.lucene.index.TieredMergePolicy"/>
+ -->
+
+ <!-- Expert: Merge Scheduler
+
+ The Merge Scheduler in Lucene controls how merges are
+ performed. The ConcurrentMergeScheduler (Lucene 2.3 default)
+ can perform merges in the background using separate threads.
+ The SerialMergeScheduler (Lucene 2.2 default) does not.
+ -->
+ <!--
+ <mergeScheduler class="org.apache.lucene.index.ConcurrentMergeScheduler"/>
+ -->
+
+ <!-- LockFactory
+
+ This option specifies which Lucene LockFactory implementation
+ to use.
+
+ single = SingleInstanceLockFactory - suggested for a
+ read-only index or when there is no possibility of
+ another process trying to modify the index.
+ native = NativeFSLockFactory - uses OS native file locking.
+ Do not use when multiple solr webapps in the same
+ JVM are attempting to share a single index.
+ simple = SimpleFSLockFactory - uses a plain file for locking
+
+ (For backwards compatibility with Solr 1.2, 'simple' is the
+ default if not specified.)
+
+ More details on the nuances of each LockFactory...
+ http://wiki.apache.org/lucene-java/AvailableLockFactories
+ -->
+ <lockType>native</lockType>
+
+ <!-- Expert: Controls how often Lucene loads terms into memory
+ Default is 128 and is likely good for most everyone.
+ -->
+ <!-- <termIndexInterval>256</termIndexInterval> -->
+ </indexDefaults>
+
+ <!-- Main Index
+
+ Values here override the values in the <indexDefaults> section
+ for the main on disk index.
+ -->
+ <mainIndex>
+
+ <useCompoundFile>false</useCompoundFile>
+ <ramBufferSizeMB>32</ramBufferSizeMB>
+ <mergeFactor>10</mergeFactor>
+
+ <!-- Unlock On Startup
+
+ If true, unlock any held write or commit locks on startup.
+ This defeats the locking mechanism that allows multiple
+ processes to safely access a lucene index, and should be used
+ with care.
+
+ This is not needed if lock type is 'none' or 'single'
+ -->
+ <unlockOnStartup>false</unlockOnStartup>
+
+ <!-- If true, IndexReaders will be reopened (often more efficient)
+ instead of closed and then opened.
+ -->
+ <reopenReaders>true</reopenReaders>
+
+ <!-- Commit Deletion Policy
+
+ Custom deletion policies can specified here. The class must
+ implement org.apache.lucene.index.IndexDeletionPolicy.
+
+ http://lucene.apache.org/java/2_9_1/api/all/org/apache/lucene/index/IndexDeletionPolicy.html
+
+ The standard Solr IndexDeletionPolicy implementation supports
+ deleting index commit points on number of commits, age of
+ commit point and optimized status.
+
+ The latest commit point should always be preserved regardless
+ of the criteria.
+ -->
+ <deletionPolicy class="solr.SolrDeletionPolicy">
+ <!-- The number of commit points to be kept -->
+ <str name="maxCommitsToKeep">1</str>
+ <!-- The number of optimized commit points to be kept -->
+ <str name="maxOptimizedCommitsToKeep">0</str>
+ <!--
+ Delete all commit points once they have reached the given age.
+ Supports DateMathParser syntax e.g.
+ -->
+ <!--
+ <str name="maxCommitAge">30MINUTES</str>
+ <str name="maxCommitAge">1DAY</str>
+ -->
+ </deletionPolicy>
+
+ <!-- Lucene Infostream
+
+ To aid in advanced debugging, Lucene provides an "InfoStream"
+ of detailed information when indexing.
+
+ Setting The value to true will instruct the underlying Lucene
+ IndexWriter to write its debugging info the specified file
+ -->
+ <infoStream file="INFOSTREAM.txt">false</infoStream>
+
+ </mainIndex>
+
+ <!-- JMX
+
+ This example enables JMX if and only if an existing MBeanServer
+ is found, use this if you want to configure JMX through JVM
+ parameters. Remove this to disable exposing Solr configuration
+ and statistics to JMX.
+
+ For more details see http://wiki.apache.org/solr/SolrJmx
+ -->
+ <jmx />
+ <!-- If you want to connect to a particular server, specify the
+ agentId
+ -->
+ <!-- <jmx agentId="myAgent" /> -->
+ <!-- If you want to start a new MBeanServer, specify the serviceUrl -->
+ <!-- <jmx serviceUrl="service:jmx:rmi:///jndi/rmi://localhost:9999/solr"/>
+ -->
+
+ <!-- The default high-performance update handler -->
+ <updateHandler class="solr.DirectUpdateHandler2">
+
+ <!-- AutoCommit
+
+ Perform a <commit/> automatically under certain conditions.
+ Instead of enabling autoCommit, consider using "commitWithin"
+ when adding documents.
+
+ http://wiki.apache.org/solr/UpdateXmlMessages
+
+ maxDocs - Maximum number of documents to add since the last
+ commit before automatically triggering a new commit.
+
+ maxTime - Maximum amount of time that is allowed to pass
+ since a document was added before automaticly
+ triggering a new commit.
+ -->
+ <!--
+ <autoCommit>
+ <maxDocs>10000</maxDocs>
+ <maxTime>1000</maxTime>
+ </autoCommit>
+ -->
+
+ <!-- Update Related Event Listeners
+
+ Various IndexWriter related events can trigger Listeners to
+ take actions.
+
+ postCommit - fired after every commit or optimize command
+ postOptimize - fired after every optimize command
+ -->
+ <!-- The RunExecutableListener executes an external command from a
+ hook such as postCommit or postOptimize.
+
+ exe - the name of the executable to run
+ dir - dir to use as the current working directory. (default=".")
+ wait - the calling thread waits until the executable returns.
+ (default="true")
+ args - the arguments to pass to the program. (default is none)
+ env - environment variables to set. (default is none)
+ -->
+ <!-- This example shows how RunExecutableListener could be used
+ with the script based replication...
+ http://wiki.apache.org/solr/CollectionDistribution
+ -->
+ <!--
+ <listener event="postCommit" class="solr.RunExecutableListener">
+ <str name="exe">solr/bin/snapshooter</str>
+ <str name="dir">.</str>
+ <bool name="wait">true</bool>
+ <arr name="args"> <str>arg1</str> <str>arg2</str> </arr>
+ <arr name="env"> <str>MYVAR=val1</str> </arr>
+ </listener>
+ -->
+ </updateHandler>
+
+ <!-- IndexReaderFactory
+
+ Use the following format to specify a custom IndexReaderFactory,
+ which allows for alternate IndexReader implementations.
+
+ ** Experimental Feature **
+
+ Please note - Using a custom IndexReaderFactory may prevent
+ certain other features from working. The API to
+ IndexReaderFactory may change without warning or may even be
+ removed from future releases if the problems cannot be
+ resolved.
+
+
+ ** Features that may not work with custom IndexReaderFactory **
+
+ The ReplicationHandler assumes a disk-resident index. Using a
+ custom IndexReader implementation may cause incompatibility
+ with ReplicationHandler and may cause replication to not work
+ correctly. See SOLR-1366 for details.
+
+ -->
+ <!--
+ <indexReaderFactory name="IndexReaderFactory" class="package.class">
+ <str name="someArg">Some Value</str>
+ </indexReaderFactory >
+ -->
+ <!-- By explicitly declaring the Factory, the termIndexDivisor can
+ be specified.
+ -->
+ <!--
+ <indexReaderFactory name="IndexReaderFactory"
+ class="solr.StandardIndexReaderFactory">
+ <int name="setTermIndexDivisor">12</int>
+ </indexReaderFactory >
+ -->
+
+
+ <query>
+ <!-- Max Boolean Clauses
+
+ Maximum number of clauses in each BooleanQuery, an exception
+ is thrown if exceeded.
+
+ ** WARNING **
+
+ This option actually modifies a global Lucene property that
+ will affect all SolrCores. If multiple solrconfig.xml files
+ disagree on this property, the value at any given moment will
+ be based on the last SolrCore to be initialized.
+
+ -->
+ <maxBooleanClauses>1024</maxBooleanClauses>
+
+
+ <!-- Solr Internal Query Caches
+
+ There are two implementations of cache available for Solr,
+ LRUCache, based on a synchronized LinkedHashMap, and
+ FastLRUCache, based on a ConcurrentHashMap.
+
+ FastLRUCache has faster gets and slower puts in single
+ threaded operation and thus is generally faster than LRUCache
+ when the hit ratio of the cache is high (> 75%), and may be
+ faster under other scenarios on multi-cpu systems.
+ -->
+
+ <!-- Filter Cache
+
+ Cache used by SolrIndexSearcher for filters (DocSets),
+ unordered sets of *all* documents that match a query. When a
+ new searcher is opened, its caches may be prepopulated or
+ "autowarmed" using data from caches in the old searcher.
+ autowarmCount is the number of items to prepopulate. For
+ LRUCache, the autowarmed items will be the most recently
+ accessed items.
+
+ Parameters:
+ class - the SolrCache implementation LRUCache or
+ (LRUCache or FastLRUCache)
+ size - the maximum number of entries in the cache
+ initialSize - the initial capacity (number of entries) of
+ the cache. (see java.util.HashMap)
+ autowarmCount - the number of entries to prepopulate from
+ and old cache.
+ -->
+ <filterCache class="solr.FastLRUCache"
+ size="512"
+ initialSize="512"
+ autowarmCount="0"/>
+
+ <!-- Query Result Cache
+
+ Caches results of searches - ordered lists of document ids
+ (DocList) based on a query, a sort, and the range of documents requested.
+ -->
+ <queryResultCache class="solr.LRUCache"
+ size="512"
+ initialSize="512"
+ autowarmCount="0"/>
+
+ <!-- Document Cache
+
+ Caches Lucene Document objects (the stored fields for each
+ document). Since Lucene internal document ids are transient,
+ this cache will not be autowarmed.
+ -->
+ <documentCache class="solr.LRUCache"
+ size="512"
+ initialSize="512"
+ autowarmCount="0"/>
+
+ <!-- Field Value Cache
+
+ Cache used to hold field values that are quickly accessible
+ by document id. The fieldValueCache is created by default
+ even if not configured here.
+ -->
+ <!--
+ <fieldValueCache class="solr.FastLRUCache"
+ size="512"
+ autowarmCount="128"
+ showItems="32" />
+ -->
+
+ <!-- Custom Cache
+
+ Example of a generic cache. These caches may be accessed by
+ name through SolrIndexSearcher.getCache(),cacheLookup(), and
+ cacheInsert(). The purpose is to enable easy caching of
+ user/application level data. The regenerator argument should
+ be specified as an implementation of solr.CacheRegenerator
+ if autowarming is desired.
+ -->
+ <!--
+ <cache name="myUserCache"
+ class="solr.LRUCache"
+ size="4096"
+ initialSize="1024"
+ autowarmCount="1024"
+ regenerator="com.mycompany.MyRegenerator"
+ />
+ -->
+
+
+ <!-- Lazy Field Loading
+
+ If true, stored fields that are not requested will be loaded
+ lazily. This can result in a significant speed improvement
+ if the usual case is to not load all stored fields,
+ especially if the skipped fields are large compressed text
+ fields.
+ -->
+ <enableLazyFieldLoading>true</enableLazyFieldLoading>
+
+ <!-- Use Filter For Sorted Query
+
+ A possible optimization that attempts to use a filter to
+ satisfy a search. If the requested sort does not include
+ score, then the filterCache will be checked for a filter
+ matching the query. If found, the filter will be used as the
+ source of document ids, and then the sort will be applied to
+ that.
+
+ For most situations, this will not be useful unless you
+ frequently get the same search repeatedly with different sort
+ options, and none of them ever use "score"
+ -->
+ <!--
+ <useFilterForSortedQuery>true</useFilterForSortedQuery>
+ -->
+
+ <!-- Result Window Size
+
+ An optimization for use with the queryResultCache. When a search
+ is requested, a superset of the requested number of document ids
+ are collected. For example, if a search for a particular query
+ requests matching documents 10 through 19, and queryWindowSize is 50,
+ then documents 0 through 49 will be collected and cached. Any further
+ requests in that range can be satisfied via the cache.
+ -->
+ <queryResultWindowSize>20</queryResultWindowSize>
+
+ <!-- Maximum number of documents to cache for any entry in the
+ queryResultCache.
+ -->
+ <queryResultMaxDocsCached>200</queryResultMaxDocsCached>
+
+ <!-- Query Related Event Listeners
+
+ Various IndexSearcher related events can trigger Listeners to
+ take actions.
+
+ newSearcher - fired whenever a new searcher is being prepared
+ and there is a current searcher handling requests (aka
+ registered). It can be used to prime certain caches to
+ prevent long request times for certain requests.
+
+ firstSearcher - fired whenever a new searcher is being
+ prepared but there is no current registered searcher to handle
+ requests or to gain autowarming data from.
+
+
+ -->
+ <!-- QuerySenderListener takes an array of NamedList and executes a
+ local query request for each NamedList in sequence.
+ -->
+ <listener event="newSearcher" class="solr.QuerySenderListener">
+ <arr name="queries">
+ <!--
+ <lst><str name="q">solr</str><str name="sort">price asc</str></lst>
+ <lst><str name="q">rocks</str><str name="sort">weight asc</str></lst>
+ -->
+ </arr>
+ </listener>
+ <listener event="firstSearcher" class="solr.QuerySenderListener">
+ <arr name="queries">
+ <lst>
+ <str name="q">static firstSearcher warming in solrconfig.xml</str>
+ </lst>
+ </arr>
+ </listener>
+
+ <!-- Use Cold Searcher
+
+ If a search request comes in and there is no current
+ registered searcher, then immediately register the still
+ warming searcher and use it. If "false" then all requests
+ will block until the first searcher is done warming.
+ -->
+ <useColdSearcher>false</useColdSearcher>
+
+ <!-- Max Warming Searchers
+
+ Maximum number of searchers that may be warming in the
+ background concurrently. An error is returned if this limit
+ is exceeded.
+
+ Recommend values of 1-2 for read-only slaves, higher for
+ masters w/o cache warming.
+ -->
+ <maxWarmingSearchers>2</maxWarmingSearchers>
+
+ </query>
+
+
+ <!-- Request Dispatcher
+
+ This section contains instructions for how the SolrDispatchFilter
+ should behave when processing requests for this SolrCore.
+
+ handleSelect affects the behavior of requests such as /select?qt=XXX
+
+ handleSelect="true" will cause the SolrDispatchFilter to process
+ the request and will result in consistent error handling and
+ formatting for all types of requests.
+
+ handleSelect="false" will cause the SolrDispatchFilter to
+ ignore "/select" requests and fallback to using the legacy
+ SolrServlet and it's Solr 1.1 style error formatting
+ -->
+ <requestDispatcher handleSelect="true" >
+ <!-- Request Parsing
+
+ These settings indicate how Solr Requests may be parsed, and
+ what restrictions may be placed on the ContentStreams from
+ those requests
+
+ enableRemoteStreaming - enables use of the stream.file
+ and stream.url parameters for specifying remote streams.
+
+ multipartUploadLimitInKB - specifies the max size of
+ Multipart File Uploads that Solr will allow in a Request.
+
+ *** WARNING ***
+ The settings below authorize Solr to fetch remote files, You
+ should make sure your system has some authentication before
+ using enableRemoteStreaming="true"
+
+ -->
+ <requestParsers enableRemoteStreaming="true"
+ multipartUploadLimitInKB="2048000" />
+
+ <!-- HTTP Caching
+
+ Set HTTP caching related parameters (for proxy caches and clients).
+
+ The options below instruct Solr not to output any HTTP Caching
+ related headers
+ -->
+ <httpCaching never304="true" />
+ <!-- If you include a <cacheControl> directive, it will be used to
+ generate a Cache-Control header (as well as an Expires header
+ if the value contains "max-age=")
+
+ By default, no Cache-Control header is generated.
+
+ You can use the <cacheControl> option even if you have set
+ never304="true"
+ -->
+ <!--
+ <httpCaching never304="true" >
+ <cacheControl>max-age=30, public</cacheControl>
+ </httpCaching>
+ -->
+ <!-- To enable Solr to respond with automatically generated HTTP
+ Caching headers, and to response to Cache Validation requests
+ correctly, set the value of never304="false"
+
+ This will cause Solr to generate Last-Modified and ETag
+ headers based on the properties of the Index.
+
+ The following options can also be specified to affect the
+ values of these headers...
+
+ lastModFrom - the default value is "openTime" which means the
+ Last-Modified value (and validation against If-Modified-Since
+ requests) will all be relative to when the current Searcher
+ was opened. You can change it to lastModFrom="dirLastMod" if
+ you want the value to exactly correspond to when the physical
+ index was last modified.
+
+ etagSeed="..." is an option you can change to force the ETag
+ header (and validation against If-None-Match requests) to be
+ different even if the index has not changed (ie: when making
+ significant changes to your config file)
+
+ (lastModifiedFrom and etagSeed are both ignored if you use
+ the never304="true" option)
+ -->
+ <!--
+ <httpCaching lastModifiedFrom="openTime"
+ etagSeed="Solr">
+ <cacheControl>max-age=30, public</cacheControl>
+ </httpCaching>
+ -->
+ </requestDispatcher>
+
+ <requestHandler name="/mlt" class="solr.MoreLikeThisHandler" startup="lazy" />
+
+ <!-- Request Handlers
+
+ http://wiki.apache.org/solr/SolrRequestHandler
+
+ incoming queries will be dispatched to the correct handler
+ based on the path or the qt (query type) param.
+
+ Names starting with a '/' are accessed with the a path equal to
+ the registered name. Names without a leading '/' are accessed
+ with: http://host/app/[core/]select?qt=name
+
+ If a /select request is processed with out a qt param
+ specified, the requestHandler that declares default="true" will
+ be used.
+
+ If a Request Handler is declared with startup="lazy", then it will
+ not be initialized until the first request that uses it.
+
+ -->
+ <!-- SearchHandler
+
+ http://wiki.apache.org/solr/SearchHandler
+
+ For processing Search Queries, the primary Request Handler
+ provided with Solr is "SearchHandler" It delegates to a sequent
+ of SearchComponents (see below) and supports distributed
+ queries across multiple shards
+ -->
+ <requestHandler name="search" class="solr.SearchHandler" default="true">
+ <!-- default values for query parameters can be specified, these
+ will be overridden by parameters in the request
+ -->
+ <lst name="defaults">
+ <str name="echoParams">explicit</str>
+ <int name="rows">10</int>
+ </lst>
+ <!-- In addition to defaults, "appends" params can be specified
+ to identify values which should be appended to the list of
+ multi-val params from the query (or the existing "defaults").
+ -->
+ <!-- In this example, the param "fq=instock:true" would be appended to
+ any query time fq params the user may specify, as a mechanism for
+ partitioning the index, independent of any user selected filtering
+ that may also be desired (perhaps as a result of faceted searching).
+
+ NOTE: there is *absolutely* nothing a client can do to prevent these
+ "appends" values from being used, so don't use this mechanism
+ unless you are sure you always want it.
+ -->
+ <!--
+ <lst name="appends">
+ <str name="fq">inStock:true</str>
+ </lst>
+ -->
+ <!-- "invariants" are a way of letting the Solr maintainer lock down
+ the options available to Solr clients. Any params values
+ specified here are used regardless of what values may be specified
+ in either the query, the "defaults", or the "appends" params.
+
+ In this example, the facet.field and facet.query params would
+ be fixed, limiting the facets clients can use. Faceting is
+ not turned on by default - but if the client does specify
+ facet=true in the request, these are the only facets they
+ will be able to see counts for; regardless of what other
+ facet.field or facet.query params they may specify.
+
+ NOTE: there is *absolutely* nothing a client can do to prevent these
+ "invariants" values from being used, so don't use this mechanism
+ unless you are sure you always want it.
+ -->
+ <!--
+ <lst name="invariants">
+ <str name="facet.field">cat</str>
+ <str name="facet.field">manu_exact</str>
+ <str name="facet.query">price:[* TO 500]</str>
+ <str name="facet.query">price:[500 TO *]</str>
+ </lst>
+ -->
+ <!-- If the default list of SearchComponents is not desired, that
+ list can either be overridden completely, or components can be
+ prepended or appended to the default list. (see below)
+ -->
+ <!--
+ <arr name="components">
+ <str>nameOfCustomComponent1</str>
+ <str>nameOfCustomComponent2</str>
+ </arr>
+ -->
+ </requestHandler>
+
+ <!-- XML Update Request Handler.
+
+ http://wiki.apache.org/solr/UpdateXmlMessages
+
+ The canonical Request Handler for Modifying the Index through
+ commands specified using XML.
+
+ Note: Since solr1.1 requestHandlers requires a valid content
+ type header if posted in the body. For example, curl now
+ requires: -H 'Content-type:text/xml; charset=utf-8'
+ -->
+ <requestHandler name="/update"
+ class="solr.XmlUpdateRequestHandler">
+ <!-- See below for information on defining
+ updateRequestProcessorChains that can be used by name
+ on each Update Request
+ -->
+ <!--
+ <lst name="defaults">
+ <str name="update.chain">dedupe</str>
+ </lst>
+ -->
+ </requestHandler>
+ <!-- Binary Update Request Handler
+ http://wiki.apache.org/solr/javabin
+ -->
+ <requestHandler name="/update/javabin"
+ class="solr.BinaryUpdateRequestHandler" />
+
+ <!-- CSV Update Request Handler
+ http://wiki.apache.org/solr/UpdateCSV
+ -->
+ <requestHandler name="/update/csv"
+ class="solr.CSVRequestHandler"
+ startup="lazy" />
+
+ <!-- JSON Update Request Handler
+ http://wiki.apache.org/solr/UpdateJSON
+ -->
+ <requestHandler name="/update/json"
+ class="solr.JsonUpdateRequestHandler"
+ startup="lazy" />
+
+ <!-- Solr Cell Update Request Handler
+
+ http://wiki.apache.org/solr/ExtractingRequestHandler
+
+ -->
+ <requestHandler name="/update/extract"
+ startup="lazy"
+ class="solr.extraction.ExtractingRequestHandler" >
+ <lst name="defaults">
+ <!-- All the main content goes into "text"... if you need to return
+ the extracted text or do highlighting, use a stored field. -->
+ <str name="fmap.content">text</str>
+ <str name="lowernames">true</str>
+ <str name="uprefix">ignored_</str>
+
+ <!-- capture link hrefs but ignore div attributes -->
+ <str name="captureAttr">true</str>
+ <str name="fmap.a">links</str>
+ <str name="fmap.div">ignored_</str>
+ </lst>
+ </requestHandler>
+
+ <!-- Field Analysis Request Handler
+
+ RequestHandler that provides much the same functionality as
+ analysis.jsp. Provides the ability to specify multiple field
+ types and field names in the same request and outputs
+ index-time and query-time analysis for each of them.
+
+ Request parameters are:
+ analysis.fieldname - field name whose analyzers are to be used
+
+ analysis.fieldtype - field type whose analyzers are to be used
+ analysis.fieldvalue - text for index-time analysis
+ q (or analysis.q) - text for query time analysis
+ analysis.showmatch (true|false) - When set to true and when
+ query analysis is performed, the produced tokens of the
+ field value analysis will be marked as "matched" for every
+ token that is produces by the query analysis
+ -->
+ <requestHandler name="/analysis/field"
+ startup="lazy"
+ class="solr.FieldAnalysisRequestHandler" />
+
+
+ <!-- Document Analysis Handler
+
+ http://wiki.apache.org/solr/AnalysisRequestHandler
+
+ An analysis handler that provides a breakdown of the analysis
+ process of provided docuemnts. This handler expects a (single)
+ content stream with the following format:
+
+ <docs>
+ <doc>
+ <field name="id">1</field>
+ <field name="name">The Name</field>
+ <field name="text">The Text Value</field>
+ </doc>
+ <doc>...</doc>
+ <doc>...</doc>
+ ...
+ </docs>
+
+ Note: Each document must contain a field which serves as the
+ unique key. This key is used in the returned response to associate
+ an analysis breakdown to the analyzed document.
+
+ Like the FieldAnalysisRequestHandler, this handler also supports
+ query analysis by sending either an "analysis.query" or "q"
+ request parameter that holds the query text to be analyzed. It
+ also supports the "analysis.showmatch" parameter which when set to
+ true, all field tokens that match the query tokens will be marked
+ as a "match".
+ -->
+ <requestHandler name="/analysis/document"
+ class="solr.DocumentAnalysisRequestHandler"
+ startup="lazy" />
+
+ <!-- Admin Handlers
+
+ Admin Handlers - This will register all the standard admin
+ RequestHandlers.
+ -->
+ <requestHandler name="/admin/"
+ class="solr.admin.AdminHandlers" />
+
+ <!-- ping/healthcheck -->
+ <requestHandler name="/admin/ping" class="solr.PingRequestHandler">
+ <lst name="defaults">
+ <str name="qt">search</str>
+ <str name="q">solrpingquery</str>
+ <str name="echoParams">all</str>
+ </lst>
+ </requestHandler>
+
+ <!-- Echo the request contents back to the client -->
+ <requestHandler name="/debug/dump" class="solr.DumpRequestHandler" >
+ <lst name="defaults">
+ <str name="echoParams">explicit</str>
+ <str name="echoHandler">true</str>
+ </lst>
+ </requestHandler>
+
+
+ <!-- Legacy config for the admin interface -->
+ <admin>
+ <defaultQuery>*:*</defaultQuery>
+
+ <!-- configure a healthcheck file for servers behind a
+ loadbalancer
+ -->
+ <!--
+ <healthcheck type="file">server-enabled</healthcheck>
+ -->
+ </admin>
+
+</config>
Added: incubator/stanbol/trunk/enhancer/engines/topic/src/main/resources/trainingset/schema.xml
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/enhancer/engines/topic/src/main/resources/trainingset/schema.xml?rev=1233043&view=auto
==============================================================================
--- incubator/stanbol/trunk/enhancer/engines/topic/src/main/resources/trainingset/schema.xml (added)
+++ incubator/stanbol/trunk/enhancer/engines/topic/src/main/resources/trainingset/schema.xml Wed Jan 18 20:26:03 2012
@@ -0,0 +1,74 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<schema name="example" version="1.3">
+ <types>
+ <fieldType name="string" class="solr.StrField" sortMissingLast="true"
+ omitNorms="true"/>
+
+ <fieldType name="int" class="solr.TrieIntField" precisionStep="0"
+ omitNorms="true" positionIncrementGap="0"/>
+
+ <fieldType name="tdate" class="solr.TrieDateField" omitNorms="true"
+ precisionStep="6" positionIncrementGap="0"/>
+
+ <fieldType name="random" class="solr.RandomSortField" indexed="true" />
+
+ <fieldType name="text" class="solr.TextField">
+ <analyzer type="index">
+ <tokenizer class="solr.StandardTokenizerFactory"/>
+<!-- <filter class="solr.StopFilterFactory" ignoreCase="true" -->
+<!-- words="stopwords_en.txt" enablePositionIncrements="false" /> -->
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <!-- The use of Shingle might help improve the quality but they increase
+ the size of the index far too much. It would be better to use a
+ collocation bloom filter to mitigate this effect:
+ http://issues.apache.org/jira/browse/MAHOUT-415
+
+ <filter class="solr.ShingleFilterFactory" maxShingleSize="2"
+ outputUnigrams="true"/>
+ -->
+ </analyzer>
+ <analyzer type="query">
+ <tokenizer class="solr.StandardTokenizerFactory"/>
+<!-- <filter class="solr.StopFilterFactory" ignoreCase="true" -->
+<!-- words="stopwords_en.txt" enablePositionIncrements="false" /> -->
+<!-- <filter class="solr.SynonymFilterFactory" -->
+<!-- synonyms="synonyms.txt" ignoreCase="true" expand="true"/> -->
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <!--
+ <filter class="solr.ShingleFilterFactory" maxShingleSize="2"
+ outputUnigrams="true"/>
+ -->
+ </analyzer>
+ </fieldType>
+
+ </types>
+
+ <fields>
+ <field name="id" type="string" indexed="true" stored="true" required="true" />
+ <field name="topics" type="string" indexed="true" stored="true" multiValued="true" />
+ <field name="text" type="text" indexed="true" stored="true"
+ termVectors="true" termPositions="false" termOffsets="false" />
+ <field name="modification_dt" type="tdate" indexed="true" stored="true" />
+ <field name="random" type="random" indexed="true" stored="false" />
+ </fields>
+
+ <uniqueKey>id</uniqueKey>
+ <defaultSearchField>text</defaultSearchField>
+ <solrQueryParser defaultOperator="AND"/>
+</schema>
Added: incubator/stanbol/trunk/enhancer/engines/topic/src/main/resources/trainingset/solr.xml
URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/enhancer/engines/topic/src/main/resources/trainingset/solr.xml?rev=1233043&view=auto
==============================================================================
--- incubator/stanbol/trunk/enhancer/engines/topic/src/main/resources/trainingset/solr.xml (added)
+++ incubator/stanbol/trunk/enhancer/engines/topic/src/main/resources/trainingset/solr.xml Wed Jan 18 20:26:03 2012
@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<solr persistent="false">
+
+ <!--
+ adminPath: RequestHandler path to manage cores.
+ If 'null' (or absent), cores will not be manageable via request handler
+ -->
+ <cores adminPath="/admin/cores" defaultCoreName="trainingset">
+ <core name="trainingset" instanceDir="trainingset" />
+ </cores>
+</solr>