You are viewing a plain text version of this content. The canonical link for it is here.
Posted to solr-commits@lucene.apache.org by gs...@apache.org on 2009/05/20 15:29:54 UTC
svn commit: r776692 [1/3] - in /lucene/solr/trunk: ./ contrib/clustering/
contrib/clustering/lib/ contrib/clustering/src/
contrib/clustering/src/main/ contrib/clustering/src/main/java/
contrib/clustering/src/main/java/org/ contrib/clustering/src/main/j...
Author: gsingers
Date: Wed May 20 13:29:52 2009
New Revision: 776692
URL: http://svn.apache.org/viewvc?rev=776692&view=rev
Log:
SOLR-769: Add clustering contrib capabilities to Solr
Added:
lucene/solr/trunk/contrib/clustering/ (with props)
lucene/solr/trunk/contrib/clustering/README.txt
lucene/solr/trunk/contrib/clustering/build.xml
lucene/solr/trunk/contrib/clustering/lib/ (with props)
lucene/solr/trunk/contrib/clustering/lib/carrot2-mini-3.0.1.jar (with props)
lucene/solr/trunk/contrib/clustering/lib/commons-lang-2.4.jar (with props)
lucene/solr/trunk/contrib/clustering/lib/ehcache-1.6.0-beta3.jar (with props)
lucene/solr/trunk/contrib/clustering/lib/google-collections.jar (with props)
lucene/solr/trunk/contrib/clustering/lib/jackson-0.9.2-patched.jar (with props)
lucene/solr/trunk/contrib/clustering/lib/log4j-1.2.14.jar (with props)
lucene/solr/trunk/contrib/clustering/solr-clustering-pom.xml.template
lucene/solr/trunk/contrib/clustering/src/
lucene/solr/trunk/contrib/clustering/src/main/
lucene/solr/trunk/contrib/clustering/src/main/java/
lucene/solr/trunk/contrib/clustering/src/main/java/org/
lucene/solr/trunk/contrib/clustering/src/main/java/org/apache/
lucene/solr/trunk/contrib/clustering/src/main/java/org/apache/solr/
lucene/solr/trunk/contrib/clustering/src/main/java/org/apache/solr/handler/
lucene/solr/trunk/contrib/clustering/src/main/java/org/apache/solr/handler/clustering/
lucene/solr/trunk/contrib/clustering/src/main/java/org/apache/solr/handler/clustering/ClusteringComponent.java
lucene/solr/trunk/contrib/clustering/src/main/java/org/apache/solr/handler/clustering/ClusteringEngine.java
lucene/solr/trunk/contrib/clustering/src/main/java/org/apache/solr/handler/clustering/ClusteringParams.java
lucene/solr/trunk/contrib/clustering/src/main/java/org/apache/solr/handler/clustering/DocumentClusteringEngine.java
lucene/solr/trunk/contrib/clustering/src/main/java/org/apache/solr/handler/clustering/SearchClusteringEngine.java
lucene/solr/trunk/contrib/clustering/src/main/java/org/apache/solr/handler/clustering/carrot2/
lucene/solr/trunk/contrib/clustering/src/main/java/org/apache/solr/handler/clustering/carrot2/CarrotClusteringEngine.java
lucene/solr/trunk/contrib/clustering/src/main/java/org/apache/solr/handler/clustering/carrot2/CarrotParams.java
lucene/solr/trunk/contrib/clustering/src/test/
lucene/solr/trunk/contrib/clustering/src/test/java/
lucene/solr/trunk/contrib/clustering/src/test/java/org/
lucene/solr/trunk/contrib/clustering/src/test/java/org/apache/
lucene/solr/trunk/contrib/clustering/src/test/java/org/apache/solr/
lucene/solr/trunk/contrib/clustering/src/test/java/org/apache/solr/handler/
lucene/solr/trunk/contrib/clustering/src/test/java/org/apache/solr/handler/clustering/
lucene/solr/trunk/contrib/clustering/src/test/java/org/apache/solr/handler/clustering/AbstractClusteringTest.java
lucene/solr/trunk/contrib/clustering/src/test/java/org/apache/solr/handler/clustering/ClusteringComponentTest.java
lucene/solr/trunk/contrib/clustering/src/test/java/org/apache/solr/handler/clustering/MockDocumentClusteringEngine.java
lucene/solr/trunk/contrib/clustering/src/test/java/org/apache/solr/handler/clustering/carrot2/
lucene/solr/trunk/contrib/clustering/src/test/java/org/apache/solr/handler/clustering/carrot2/CarrotClusteringEngineTest.java
lucene/solr/trunk/contrib/clustering/src/test/java/org/apache/solr/handler/clustering/carrot2/MockClusteringAlgorithm.java
lucene/solr/trunk/contrib/clustering/src/test/resources/
lucene/solr/trunk/contrib/clustering/src/test/resources/solr/
lucene/solr/trunk/contrib/clustering/src/test/resources/solr/conf/
lucene/solr/trunk/contrib/clustering/src/test/resources/solr/conf/mapping-ISOLatin1Accent.txt
lucene/solr/trunk/contrib/clustering/src/test/resources/solr/conf/protwords.txt
lucene/solr/trunk/contrib/clustering/src/test/resources/solr/conf/schema.xml
lucene/solr/trunk/contrib/clustering/src/test/resources/solr/conf/solrconfig.xml
lucene/solr/trunk/contrib/clustering/src/test/resources/solr/conf/spellings.txt
lucene/solr/trunk/contrib/clustering/src/test/resources/solr/conf/stopwords.txt
lucene/solr/trunk/contrib/clustering/src/test/resources/solr/conf/synonyms.txt
lucene/solr/trunk/example/clustering/
lucene/solr/trunk/example/clustering/README.txt
lucene/solr/trunk/example/clustering/solr/
lucene/solr/trunk/example/clustering/solr/conf/
lucene/solr/trunk/example/clustering/solr/conf/mapping-ISOLatin1Accent.txt
lucene/solr/trunk/example/clustering/solr/conf/protwords.txt
lucene/solr/trunk/example/clustering/solr/conf/schema.xml
lucene/solr/trunk/example/clustering/solr/conf/solrconfig.xml
lucene/solr/trunk/example/clustering/solr/conf/spellings.txt
lucene/solr/trunk/example/clustering/solr/conf/stopwords.txt
lucene/solr/trunk/example/clustering/solr/conf/synonyms.txt
lucene/solr/trunk/example/clustering/solr/lib/
Modified:
lucene/solr/trunk/LICENSE.txt
lucene/solr/trunk/NOTICE.txt
lucene/solr/trunk/build.xml
Modified: lucene/solr/trunk/LICENSE.txt
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/LICENSE.txt?rev=776692&r1=776691&r2=776692&view=diff
==============================================================================
--- lucene/solr/trunk/LICENSE.txt (original)
+++ lucene/solr/trunk/LICENSE.txt Wed May 20 13:29:52 2009
@@ -736,6 +736,101 @@
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+==========================================================================
+contrib/clustering
+==========================================================================
+Carrot2 Project
+
+Copyright (C) 2002-2008, Dawid Weiss, Stanislaw Osinski.
+Portions (C) Contributors listed in "carrot2.CONTRIBUTORS" file.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+- Redistributions of source code must retain the above copyright notice, this
+ list of conditions and the following disclaimer.
+
+- Redistributions in binary form must reproduce the above copyright notice, this
+ list of conditions and the following disclaimer in the documentation and/or
+ other materials provided with the distribution.
+
+- Neither the name of the Poznan University of Technology, Poznan, Poland nor
+ the names of its contributors may be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+- We request that you include in the end-user documentation provided with the
+ redistribution and/or in the software itself an acknowledgement equivalent to
+ the following: "This product includes software developed by the Carrot2
+ Project."
+
+- No algorithms or technical solutions in the project may be patented or claimed
+ proprietary.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+==========================================================================
+EHCache
+/**
+ * Copyright 2003-2008 Luck Consulting Pty Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+==========================================================================
+Google Collections
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+==========================================================================
+Jackson
+
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+
===========================================================================
Apache Tika Licenses - contrib/extraction
---------------------------------------------------------------------------
Modified: lucene/solr/trunk/NOTICE.txt
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/NOTICE.txt?rev=776692&r1=776691&r2=776692&view=diff
==============================================================================
--- lucene/solr/trunk/NOTICE.txt (original)
+++ lucene/solr/trunk/NOTICE.txt Wed May 20 13:29:52 2009
@@ -12,6 +12,7 @@
- Apache Tomcat (lib/servlet-api-2.4.jar)
- Apache Commons
- Apache Geronimo (stax API jar)
+ - Apache Log4j (contrib/clustering)
This product includes tests written with EasyMock Copyright 2001-2007
Tammo Freese (http://www.easymock.org/)
@@ -141,3 +142,45 @@
Copyright (c) 2000-2005 INRIA, France Telecom
+=========================================================================
+== Carrot2 Notice ==
+=========================================================================
+Copyright (C) 2002-2008, Dawid Weiss, Stanislaw Osinski.
+Portions (C) Contributors listed in "carrot2.CONTRIBUTORS" file.
+All rights reserved.
+
+This product includes software developed by the Carrot2 Project.
+
+See http://project.carrot2.org/
+=======
+=========================================================================
+== EHCache Notice ==
+=========================================================================
+Copyright 2003-2008 Luck Consulting Pty Ltd
+
+This product includes software developed by the EHCache Project
+
+See ????
+
+=========================================================================
+== Google Collections Notice ==
+
+=========================================================================
+
+Copyright ???? Google, Inc.
+
+This product includes software developed by the Google Collections project.
+
+See ????
+
+=========================================================================
+== Jackson Notice ==
+=========================================================================
+Copyright ????
+
+This product includes software developed by the Jackson project.
+
+See ????
+
+
+
Modified: lucene/solr/trunk/build.xml
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/build.xml?rev=776692&r1=776691&r2=776692&view=diff
==============================================================================
--- lucene/solr/trunk/build.xml (original)
+++ lucene/solr/trunk/build.xml Wed May 20 13:29:52 2009
@@ -188,6 +188,15 @@
<path id="javadoc.classpath">
<path refid="compile.classpath"/>
<path refid="compile.classpath.solrj"/>
+ <fileset dir="contrib/clustering/lib">
+ <include name="*.jar"/>
+ </fileset>
+ <fileset dir="contrib/clustering/lib/downloads">
+ <include name="*.jar"/>
+ </fileset>
+ <fileset dir="contrib/extraction/lib">
+ <include name="*.jar"/>
+ </fileset>
<pathelement location="${dest}/client/solrj"/>
</path>
@@ -198,11 +207,13 @@
<packageset dir="${src}/java" />
<packageset dir="${src}/webapp/src" />
<packageset dir="contrib/dataimporthandler/src/main/java" />
+ <packageset dir="contrib/clustering/src/main/java" />
<packageset dir="contrib/extraction/src/main/java" />
<group title="Core" packages="org.apache.*" />
<group title="Common" packages="org.apache.solr.common.*" />
<group title="SolrJ" packages="org.apache.solr.client.solrj*" />
<group title="contrib: DataImportHandler" packages="org.apache.solr.handler.dataimport*" />
+ <group title="contrib: Clustering" packages="org.apache.solr.handler.clustering*" />
<group title="contrib: Solr Cell" packages="org.apache.solr.handler.extraction*" />
</sources>
</invoke-javadoc>
@@ -397,6 +408,7 @@
<fileset dir="src/java"/>
<fileset dir="src/webapp/src"/>
<fileset dir="contrib/dataimporthandler/src/main/java" />
+ <fileset dir="contrib/clustering/src/main/java" />
<fileset dir="contrib/extraction/src/main/java" />
</clover-setup>
</target>
@@ -493,6 +505,8 @@
<solr-jar destfile="${dist}/apache-solr-cell-src-${version}.jar"
basedir="contrib/extraction/src" />
+ <solr-jar destfile="${dist}/apache-solr-clustering-src-${version}.jar"
+ basedir="contrib/clustering/src" />
</target>
<target name="dist-javadoc" description="Creates the Solr javadoc distribution files"
@@ -505,6 +519,8 @@
basedir="${build.javadoc}/solrj" />
<solr-jar destfile="${dist}/apache-solr-dataimporthandler-docs-${version}.jar"
basedir="${build.javadoc}/contrib-solr-dataimporthandler" />
+ <solr-jar destfile="${dist}/apache-solr-clustering-docs-${version}.jar"
+ basedir="${build.javadoc}/contrib-solr-clustering" />
<solr-jar destfile="${dist}/apache-solr-cell-docs-${version}.jar"
basedir="${build.javadoc}/contrib-solr-cell" />
</target>
@@ -686,6 +702,8 @@
<sign-maven-artifacts artifact.id="solr-core"/>
<sign-maven-artifacts artifact.id="solr-dataimporthandler"/>
<sign-maven-artifacts artifact.id="solr-dataimporthandler-extras"/>
+ <sign-maven-artifacts artifact.id="solr-clustering"/>
+
<sign-maven-artifacts artifact.id="solr-cell"/>
<sign-maven-dependency-artifacts artifact.id="solr-lucene-analyzers"/>
<sign-maven-dependency-artifacts artifact.id="solr-lucene-core"/>
@@ -761,9 +779,9 @@
<!-- ========== SOLR ARTIFACTS ========== -->
+
<m2-deploy pom.xml="contrib/dataimporthandler/solr-dataimporthandler-pom.xml.template"
jar.file="${dist}/apache-solr-dataimporthandler-${version}.jar">
-
<artifact-attachments>
<attach file="${dist}/apache-solr-dataimporthandler-src-${version}.jar" classifier="sources"/>
<attach file="${dist}/apache-solr-dataimporthandler-docs-${version}.jar" classifier="javadoc"/>
@@ -781,13 +799,23 @@
<m2-deploy pom.xml="contrib/extraction/solr-cell-pom.xml.template"
jar.file="${dist}/apache-solr-cell-${version}.jar">
-
<artifact-attachments>
<attach file="${dist}/apache-solr-cell-src-${version}.jar" classifier="sources"/>
<attach file="${dist}/apache-solr-cell-docs-${version}.jar" classifier="javadoc"/>
</artifact-attachments>
</m2-deploy>
+ <m2-deploy pom.xml="contrib/clustering/solr-clustering-pom.xml.template"
+ jar.file="${dist}/apache-solr-clustering-${version}.jar">
+
+ <artifact-attachments>
+ <attach file="${dist}/apache-solr-clustering-src-${version}.jar" classifier="sources"/>
+ <attach file="${dist}/apache-solr-clustering-docs-${version}.jar" classifier="javadoc"/>
+ </artifact-attachments>
+ </m2-deploy>
+ <!-- Clustring specific -->
+ <!-- TODO: MORE NEEDED HERE ONCE WE FINALIZE THE LIBS FOR CARROT -->
+ <!-- end clustering specific -->
<m2-deploy pom.xml="${src}/maven/solr-core-pom.xml.template"
jar.file="${dist}/apache-solr-core-${version}.jar">
@@ -834,6 +862,8 @@
</fileset>
<fileset dir="contrib/dataimporthandler/src/main/java"/>
<fileset dir="contrib/dataimporthandler/src/test/java"/>
+ <fileset dir="contrib/clustering/src/main/java"/>
+ <fileset dir="contrib/clustering/src/test/java"/>
<fileset dir="contrib/extraction/src/main/java"/>
<fileset dir="contrib/extraction/src/test/java"/>
</rat:report>
Propchange: lucene/solr/trunk/contrib/clustering/
------------------------------------------------------------------------------
--- svn:ignore (added)
+++ svn:ignore Wed May 20 13:29:52 2009
@@ -0,0 +1 @@
+build
Added: lucene/solr/trunk/contrib/clustering/README.txt
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/clustering/README.txt?rev=776692&view=auto
==============================================================================
--- lucene/solr/trunk/contrib/clustering/README.txt (added)
+++ lucene/solr/trunk/contrib/clustering/README.txt Wed May 20 13:29:52 2009
@@ -0,0 +1,8 @@
+The Clustering contrib plugin for Solr provides a generic mechanism for plugging in third party clustering implementations.
+It currently provides clustering support for search results using the Carrot2 project.
+
+See http://wiki.apache.org/solr/ClusteringComponent for how to get started.
+
+Also, note, some of the Carrot2 libraries cannot be distributed in binary form because they are LGPL. Thus, you will have
+ to download those components. See the build.xml file located in this directory for the location of the libraries.
+ The libraries you will need are: nni.jar, Colt and PNJ.
\ No newline at end of file
Added: lucene/solr/trunk/contrib/clustering/build.xml
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/clustering/build.xml?rev=776692&view=auto
==============================================================================
--- lucene/solr/trunk/contrib/clustering/build.xml (added)
+++ lucene/solr/trunk/contrib/clustering/build.xml Wed May 20 13:29:52 2009
@@ -0,0 +1,172 @@
+<?xml version="1.0"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
+<project name="solr-clustering" default="build">
+
+ <property name="solr-path" value="../.." />
+
+ <import file="../../common-build.xml"/>
+
+ <description>
+ Clustering Integraton
+ </description>
+
+ <path id="common.classpath">
+ <fileset dir="lib"/>
+ <fileset dir="lib/downloads"/>
+ <pathelement location="${solr-path}/build/solr" />
+ <pathelement location="${solr-path}/build/solrj" />
+ <fileset dir="${solr-path}/lib" includes="*.jar"/>
+ </path>
+
+ <path id="test.classpath">
+ <pathelement path="${dest}/classes" />
+ <pathelement path="${dest}/test-classes" />
+ <pathelement path="${java.class.path}"/>
+ <path refid="common.classpath" />
+ </path>
+
+ <target name="clean">
+ <delete failonerror="false" dir="${dest}"/>
+
+ </target>
+ <target name="clean-downloads">
+ <delete dir="lib/downloads"/>
+ </target>
+
+ <target name="init">
+ <mkdir dir="${dest}/classes"/>
+ <mkdir dir="${build.javadoc}" />
+ <ant dir="../../" inheritall="false" target="compile" />
+ <ant dir="../../" inheritall="false" target="make-manifest" />
+ <mkdir dir="lib/downloads" />
+ </target>
+
+ <target name="check-files">
+ <available file="lib/downloads/colt-1.2.0.jar" property="colt.exists"/>
+ <available file="lib/downloads/pcj-1.2.jar" property="pcj.exists"/>
+ <available file="lib/downloads/nni-1.0.0.jar" property="nni.exists"/>
+ </target>
+ <target name="get-colt" depends="check-files" unless="colt.exists">
+ <!-- Get the LGPL deps and put them in a separate dir -->
+ <get src="http://repo1.maven.org/maven2/colt/colt/1.2.0/colt-1.2.0.jar" dest="lib/downloads/colt-1.2.0.jar"/>
+ </target>
+ <target name="get-nni" depends="check-files" unless="nni.exists">
+ <!-- Get the LGPL deps and put them in a separate dir -->
+ <get src="http://download.carrot2.org/maven2/org/carrot2/nni/1.0.0/nni-1.0.0.jar" dest="lib/downloads/nni-1.0.0.jar"/>
+
+ </target>
+ <target name="get-pcj" depends="check-files" unless="pcj.exists">
+ <!-- Get the LGPL deps and put them in a separate dir -->
+ <get src="http://repo1.maven.org/maven2/pcj/pcj/1.2/pcj-1.2.jar" dest="lib/downloads/pcj-1.2.jar"/>
+ </target>
+ <target name="get-libraries" depends="init, get-colt, get-pcj, get-nni"/>
+
+ <target name="compile" depends="init, get-libraries">
+ <solr-javac destdir="${dest}/classes"
+ classpathref="common.classpath">
+ <src path="src/main/java" />
+ </solr-javac>
+ </target>
+
+ <target name="build" depends="compile">
+ <solr-jar destfile="${dest}/${fullnamever}.jar" basedir="${dest}/classes"
+ manifest="${common.dir}/${dest}/META-INF/MANIFEST.MF" />
+ </target>
+
+ <target name="compileTests" depends="compile">
+ <solr-javac destdir="${dest}/test-classes"
+ classpathref="test.classpath">
+ <src path="src/test/java" />
+ </solr-javac>
+ </target>
+
+ <target name="example" depends="build">
+ <!-- Copy the jar into example/solr/lib -->
+ <copy file="${dest}/${fullnamever}.jar" todir="${example}/clustering/solr/lib"/>
+ <copy todir="${example}/clustering/solr/lib">
+ <fileset dir="lib">
+ <include name="*.jar"/>
+ </fileset>
+ <fileset dir="lib/downloads">
+ <include name="*.jar"/>
+ </fileset>
+ </copy>
+ <copy todir="${example}/clustering/solr/lib">
+ <fileset dir="lib/downloads">
+ <include name="*.jar"/>
+ </fileset>
+ </copy>
+ </target>
+
+
+ <target name="test" depends="compileTests">
+ <mkdir dir="${junit.output.dir}"/>
+
+ <junit printsummary="on"
+ haltonfailure="no"
+ errorProperty="tests.failed"
+ failureProperty="tests.failed"
+ dir="src/test/resources/"
+ >
+ <formatter type="brief" usefile="false" if="junit.details"/>
+ <classpath refid="test.classpath"/>
+ <formatter type="xml"/>
+ <batchtest fork="yes" todir="${junit.output.dir}" unless="testcase" >
+ <fileset dir="src/test/java" includes="${junit.includes}">
+ <exclude name="**/AbstractClusteringTest*"/>
+ </fileset>
+ </batchtest>
+ <batchtest fork="yes" todir="${junit.output.dir}" if="testcase">
+ <fileset dir="src/test/java" includes="**/${testcase}.java"/>
+ </batchtest>
+ </junit>
+
+ <fail if="tests.failed">Tests failed!</fail>
+ </target>
+
+ <target name="dist" depends="build">
+ <copy file="${dest}/${fullnamever}.jar" todir="${solr-path}/build/web/WEB-INF/lib"/>
+ <copy todir="${solr-path}/build/web/WEB-INF/lib" flatten="true">
+ <fileset dir="lib">
+ <include name="**/*.jar"/>
+ </fileset>
+ </copy>
+ <copy file="${dest}/${fullnamever}.jar" todir="${solr-path}/dist"/>
+ </target>
+
+ <target name="javadoc">
+ <sequential>
+ <mkdir dir="${build.javadoc}/contrib-${name}"/>
+
+ <path id="javadoc.classpath">
+ <path refid="common.classpath"/>
+ </path>
+
+ <invoke-javadoc
+ destdir="${build.javadoc}/contrib-${name}"
+ title="${Name} ${version} contrib-${fullnamever} API">
+ <sources>
+ <packageset dir="src/main/java"/>
+ </sources>
+ </invoke-javadoc>
+ </sequential>
+ </target>
+
+</project>
Propchange: lucene/solr/trunk/contrib/clustering/lib/
------------------------------------------------------------------------------
--- svn:ignore (added)
+++ svn:ignore Wed May 20 13:29:52 2009
@@ -0,0 +1 @@
+downloads
Added: lucene/solr/trunk/contrib/clustering/lib/carrot2-mini-3.0.1.jar
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/clustering/lib/carrot2-mini-3.0.1.jar?rev=776692&view=auto
==============================================================================
Binary file - no diff available.
Propchange: lucene/solr/trunk/contrib/clustering/lib/carrot2-mini-3.0.1.jar
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: lucene/solr/trunk/contrib/clustering/lib/commons-lang-2.4.jar
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/clustering/lib/commons-lang-2.4.jar?rev=776692&view=auto
==============================================================================
Binary file - no diff available.
Propchange: lucene/solr/trunk/contrib/clustering/lib/commons-lang-2.4.jar
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: lucene/solr/trunk/contrib/clustering/lib/ehcache-1.6.0-beta3.jar
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/clustering/lib/ehcache-1.6.0-beta3.jar?rev=776692&view=auto
==============================================================================
Binary file - no diff available.
Propchange: lucene/solr/trunk/contrib/clustering/lib/ehcache-1.6.0-beta3.jar
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: lucene/solr/trunk/contrib/clustering/lib/google-collections.jar
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/clustering/lib/google-collections.jar?rev=776692&view=auto
==============================================================================
Binary file - no diff available.
Propchange: lucene/solr/trunk/contrib/clustering/lib/google-collections.jar
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: lucene/solr/trunk/contrib/clustering/lib/jackson-0.9.2-patched.jar
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/clustering/lib/jackson-0.9.2-patched.jar?rev=776692&view=auto
==============================================================================
Binary file - no diff available.
Propchange: lucene/solr/trunk/contrib/clustering/lib/jackson-0.9.2-patched.jar
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: lucene/solr/trunk/contrib/clustering/lib/log4j-1.2.14.jar
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/clustering/lib/log4j-1.2.14.jar?rev=776692&view=auto
==============================================================================
Binary file - no diff available.
Propchange: lucene/solr/trunk/contrib/clustering/lib/log4j-1.2.14.jar
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: lucene/solr/trunk/contrib/clustering/solr-clustering-pom.xml.template
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/clustering/solr-clustering-pom.xml.template?rev=776692&view=auto
==============================================================================
--- lucene/solr/trunk/contrib/clustering/solr-clustering-pom.xml.template (added)
+++ lucene/solr/trunk/contrib/clustering/solr-clustering-pom.xml.template Wed May 20 13:29:52 2009
@@ -0,0 +1,63 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+ <!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+ -->
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.solr</groupId>
+ <artifactId>solr-parent</artifactId>
+ <version>@maven_version@</version>
+ </parent>
+
+ <groupId>org.apache.solr</groupId>
+ <artifactId>solr-clustering</artifactId>
+ <name>Apache Solr Clustering</name>
+ <version>@maven_version@</version>
+ <description>Apache Solr Clustering</description>
+ <packaging>jar</packaging>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.solr</groupId>
+ <artifactId>solr-common</artifactId>
+ <version>@maven_version@</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.solr</groupId>
+ <artifactId>solr-core</artifactId>
+ <version>@maven_version@</version>
+ </dependency>
+ <!-- Carrot -->
+ <dependency>
+ <groupId>org.carrot2</groupId>
+ <artifactId>carrot2-mini</artifactId>
+ <version>3.0.1</version>
+ </dependency>
+ </dependencies>
+ <repositories>
+ <repository>
+ <id>carrot2.org</id>
+ <name>Carrot2 Maven2 repository</name>
+ <url>http://download.carrot2.org/maven2/</url>
+ </repository>
+ </repositories>
+</project>
Added: lucene/solr/trunk/contrib/clustering/src/main/java/org/apache/solr/handler/clustering/ClusteringComponent.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/clustering/src/main/java/org/apache/solr/handler/clustering/ClusteringComponent.java?rev=776692&view=auto
==============================================================================
--- lucene/solr/trunk/contrib/clustering/src/main/java/org/apache/solr/handler/clustering/ClusteringComponent.java (added)
+++ lucene/solr/trunk/contrib/clustering/src/main/java/org/apache/solr/handler/clustering/ClusteringComponent.java Wed May 20 13:29:52 2009
@@ -0,0 +1,190 @@
+package org.apache.solr.handler.clustering;
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.core.SolrResourceLoader;
+import org.apache.solr.handler.clustering.carrot2.CarrotClusteringEngine;
+import org.apache.solr.handler.component.ResponseBuilder;
+import org.apache.solr.handler.component.SearchComponent;
+import org.apache.solr.search.DocListAndSet;
+import org.apache.solr.util.plugin.SolrCoreAware;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+
+/**
+ * Provide a plugin for clustering results. Can either be for search results (i.e. via Carrot2) or for
+ * clustering documents (i.e. via Mahout)
+ * <p/>
+ * This engine is experimental. Output from this engine is subject to change in future releases.
+ *
+ */
+public class ClusteringComponent extends SearchComponent implements SolrCoreAware {
+ private transient static Logger log = LoggerFactory.getLogger(ClusteringComponent.class);
+
+ private Map<String, SearchClusteringEngine> searchClusteringEngines = new HashMap<String, SearchClusteringEngine>();
+ private Map<String, DocumentClusteringEngine> documentClusteringEngines = new HashMap<String, DocumentClusteringEngine>();
+ /**
+ * Base name for all spell checker query parameters. This name is also used to
+ * register this component with SearchHandler.
+ */
+ public static final String COMPONENT_NAME = "clustering";
+ private NamedList initParams;
+
+
+ public void prepare(ResponseBuilder rb) throws IOException {
+ SolrParams params = rb.req.getParams();
+ if (!params.getBool(COMPONENT_NAME, false)) {
+ return;
+ }
+ }
+
+ public void process(ResponseBuilder rb) throws IOException {
+ SolrParams params = rb.req.getParams();
+ if (!params.getBool(COMPONENT_NAME, false)) {
+ return;
+ }
+ String name = params.get(ClusteringParams.ENGINE_NAME, ClusteringEngine.DEFAULT_ENGINE_NAME);
+ boolean useResults = params.getBool(ClusteringParams.USE_SEARCH_RESULTS, false);
+ if (useResults == true) {
+ SearchClusteringEngine engine = searchClusteringEngines.get(name);
+ if (engine != null) {
+ DocListAndSet results = rb.getResults();
+ NamedList nl = engine.cluster(rb.getQuery(), results.docList, params);
+ rb.rsp.add("clusters", nl);
+ } else {
+ log.warn("No engine for: " + name);
+ }
+ }
+ boolean useCollection = params.getBool(ClusteringParams.USE_COLLECTION, false);
+ if (useCollection == true) {
+ DocumentClusteringEngine engine = documentClusteringEngines.get(name);
+ if (engine != null) {
+ boolean useDocSet = params.getBool(ClusteringParams.USE_DOC_SET, false);
+ NamedList nl = null;
+
+ //TODO: This likely needs to be made into a background task that runs in an executor
+ if (useDocSet == true) {
+ nl = engine.cluster(rb.getResults().docSet, params);
+ } else {
+ nl = engine.cluster(params);
+ }
+ rb.rsp.add("clusters", nl);
+ } else {
+ log.warn("No engine for " + name);
+ }
+ }
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public void init(NamedList args) {
+ super.init(args);
+ this.initParams = args;
+ }
+
+ public void inform(SolrCore core) {
+ if (initParams != null) {
+ log.info("Initializing Clustering Engines");
+ boolean searchHasDefault = false;
+ boolean documentHasDefault = false;
+ for (int i = 0; i < initParams.size(); i++) {
+ if (initParams.getName(i).equals("engine")) {
+ NamedList engineNL = (NamedList) initParams.getVal(i);
+ String className = (String) engineNL.get("classname");
+ if (className == null) {
+ className = CarrotClusteringEngine.class.getName();
+ }
+ SolrResourceLoader loader = core.getResourceLoader();
+ ClusteringEngine clusterer = (ClusteringEngine) loader.newInstance(className);
+ if (clusterer != null) {
+ String name = clusterer.init(engineNL, core);
+ if (name != null) {
+ boolean isDefault = name.equals(ClusteringEngine.DEFAULT_ENGINE_NAME);
+ if (clusterer instanceof SearchClusteringEngine) {
+ if (isDefault == true && searchHasDefault == false) {
+ searchHasDefault = true;
+ } else if (isDefault == true && searchHasDefault == true) {
+ throw new RuntimeException("More than one engine is missing name: " + engineNL);
+ }
+ searchClusteringEngines.put(name, (SearchClusteringEngine) clusterer);
+ } else if (clusterer instanceof DocumentClusteringEngine) {
+ if (isDefault == true && documentHasDefault == false) {
+ searchHasDefault = true;
+ } else if (isDefault == true && documentHasDefault == true) {
+ throw new RuntimeException("More than one engine is missing name: " + engineNL);
+ }
+ documentClusteringEngines.put(name, (DocumentClusteringEngine) clusterer);
+ }
+ } else {
+ if (clusterer instanceof SearchClusteringEngine && searchHasDefault == false) {
+ searchClusteringEngines.put(ClusteringEngine.DEFAULT_ENGINE_NAME, (SearchClusteringEngine) clusterer);
+ searchHasDefault = true;
+ } else if (clusterer instanceof DocumentClusteringEngine && documentHasDefault == false) {
+ documentClusteringEngines.put(ClusteringEngine.DEFAULT_ENGINE_NAME, (DocumentClusteringEngine) clusterer);
+ documentHasDefault = true;
+ } else {
+ throw new RuntimeException("More than one engine is missing name: " + engineNL);
+ }
+ }
+ }
+ }
+ }
+ log.info("Finished Initializing Clustering Engines");
+ }
+ }
+
+ /*
+ * @return Unmodifiable Map of the engines, key is the name from the config, value is the engine
+ * */
+ public Map<String, SearchClusteringEngine> getSearchClusteringEngines() {
+ return Collections.unmodifiableMap(searchClusteringEngines);
+ }
+
+ // ///////////////////////////////////////////
+ // / SolrInfoMBean
+ // //////////////////////////////////////////
+
+ @Override
+ public String getDescription() {
+ return "A Clustering component";
+ }
+
+ @Override
+ public String getVersion() {
+ return "$Revision:$";
+ }
+
+ @Override
+ public String getSourceId() {
+ return "$Id:$";
+ }
+
+ @Override
+ public String getSource() {
+ return "$URL:$";
+ }
+
+}
Added: lucene/solr/trunk/contrib/clustering/src/main/java/org/apache/solr/handler/clustering/ClusteringEngine.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/clustering/src/main/java/org/apache/solr/handler/clustering/ClusteringEngine.java?rev=776692&view=auto
==============================================================================
--- lucene/solr/trunk/contrib/clustering/src/main/java/org/apache/solr/handler/clustering/ClusteringEngine.java (added)
+++ lucene/solr/trunk/contrib/clustering/src/main/java/org/apache/solr/handler/clustering/ClusteringEngine.java Wed May 20 13:29:52 2009
@@ -0,0 +1,25 @@
+package org.apache.solr.handler.clustering;
+
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.core.SolrCore;
+
+
+/**
+ *
+ *
+ **/
+public class ClusteringEngine {
+ private String name;
+ public static final String ENGINE_NAME = "name";
+ public static final String DEFAULT_ENGINE_NAME = "default";
+
+ public String init(NamedList config, SolrCore core) {
+ name = (String) config.get(ENGINE_NAME);
+
+ return name;
+ }
+
+ public String getName() {
+ return name;
+ }
+}
Added: lucene/solr/trunk/contrib/clustering/src/main/java/org/apache/solr/handler/clustering/ClusteringParams.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/clustering/src/main/java/org/apache/solr/handler/clustering/ClusteringParams.java?rev=776692&view=auto
==============================================================================
--- lucene/solr/trunk/contrib/clustering/src/main/java/org/apache/solr/handler/clustering/ClusteringParams.java (added)
+++ lucene/solr/trunk/contrib/clustering/src/main/java/org/apache/solr/handler/clustering/ClusteringParams.java Wed May 20 13:29:52 2009
@@ -0,0 +1,37 @@
+package org.apache.solr.handler.clustering;
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+/**
+ *
+ *
+ **/
+public interface ClusteringParams {
+
+ public static final String CLUSTERING_PREFIX = "clustering.";
+
+ public static final String ENGINE_NAME = CLUSTERING_PREFIX + "engine";
+
+ public static final String USE_SEARCH_RESULTS = CLUSTERING_PREFIX + "results";
+
+ public static final String USE_COLLECTION = CLUSTERING_PREFIX + "collection";
+ /**
+ * When document clustering, cluster on the Doc Set
+ */
+ public static final String USE_DOC_SET = CLUSTERING_PREFIX + "docs.useDocSet";
+}
Added: lucene/solr/trunk/contrib/clustering/src/main/java/org/apache/solr/handler/clustering/DocumentClusteringEngine.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/clustering/src/main/java/org/apache/solr/handler/clustering/DocumentClusteringEngine.java?rev=776692&view=auto
==============================================================================
--- lucene/solr/trunk/contrib/clustering/src/main/java/org/apache/solr/handler/clustering/DocumentClusteringEngine.java (added)
+++ lucene/solr/trunk/contrib/clustering/src/main/java/org/apache/solr/handler/clustering/DocumentClusteringEngine.java Wed May 20 13:29:52 2009
@@ -0,0 +1,54 @@
+package org.apache.solr.handler.clustering;
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.search.DocList;
+import org.apache.solr.search.DocSet;
+import org.apache.lucene.search.Query;
+
+
+/**
+ * Experimental. Subject to change before the next release.
+ *
+ **/
+public abstract class DocumentClusteringEngine extends ClusteringEngine {
+
+ /**
+ * Experimental. Subject to change before the next release
+ *
+ * Cluster all the documents in the index. Clustering is often an expensive task that can take a long time.
+ * @param solrParams The params controlling clustering
+ * @return The clustering results
+ */
+ public abstract NamedList cluster(SolrParams solrParams);
+
+ /**
+ * Experimental. Subject to change before the next release
+ *
+ *
+ * Cluster the set of docs. Clustering of documents is often an expensive task that can take a long time.
+ * @param docs The docs to cluster. If null, cluster all docs as in {@link #cluster(org.apache.solr.common.params.SolrParams)}
+ * @param solrParams The params controlling the clustering
+ * @return The results.
+ */
+ public abstract NamedList cluster(DocSet docs, SolrParams solrParams);
+
+
+}
Added: lucene/solr/trunk/contrib/clustering/src/main/java/org/apache/solr/handler/clustering/SearchClusteringEngine.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/clustering/src/main/java/org/apache/solr/handler/clustering/SearchClusteringEngine.java?rev=776692&view=auto
==============================================================================
--- lucene/solr/trunk/contrib/clustering/src/main/java/org/apache/solr/handler/clustering/SearchClusteringEngine.java (added)
+++ lucene/solr/trunk/contrib/clustering/src/main/java/org/apache/solr/handler/clustering/SearchClusteringEngine.java Wed May 20 13:29:52 2009
@@ -0,0 +1,36 @@
+package org.apache.solr.handler.clustering;
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.search.DocList;
+import org.apache.lucene.search.Query;
+
+
+/**
+ *
+ *
+ **/
+public abstract class SearchClusteringEngine extends ClusteringEngine {
+
+
+ public abstract NamedList cluster(Query query, DocList docList, SolrParams solrParams);
+
+
+}
Added: lucene/solr/trunk/contrib/clustering/src/main/java/org/apache/solr/handler/clustering/carrot2/CarrotClusteringEngine.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/clustering/src/main/java/org/apache/solr/handler/clustering/carrot2/CarrotClusteringEngine.java?rev=776692&view=auto
==============================================================================
--- lucene/solr/trunk/contrib/clustering/src/main/java/org/apache/solr/handler/clustering/carrot2/CarrotClusteringEngine.java (added)
+++ lucene/solr/trunk/contrib/clustering/src/main/java/org/apache/solr/handler/clustering/carrot2/CarrotClusteringEngine.java Wed May 20 13:29:52 2009
@@ -0,0 +1,254 @@
+package org.apache.solr.handler.clustering.carrot2;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.*;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.lucene.document.FieldSelector;
+import org.apache.lucene.document.SetBasedFieldSelector;
+import org.apache.lucene.search.Query;
+import org.apache.solr.common.params.HighlightParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.handler.clustering.SearchClusteringEngine;
+import org.apache.solr.highlight.SolrHighlighter;
+import org.apache.solr.request.LocalSolrQueryRequest;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.search.*;
+import org.apache.solr.util.RefCounted;
+import org.carrot2.core.*;
+import org.carrot2.core.attribute.AttributeNames;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Sets;
+
+/**
+ * Search results clustering engine based on Carrot2 clustering algorithms.
+ *
+ * Output from this class is subject to change.
+ *
+ * @link http://project.carrot2.org
+ */
+@SuppressWarnings("unchecked")
+public class CarrotClusteringEngine extends SearchClusteringEngine {
+ private transient static Logger log = LoggerFactory
+ .getLogger(CarrotClusteringEngine.class);
+
+ /** Carrot2 controller that manages instances of clustering algorithms */
+ private CachingController controller = new CachingController();
+ private Class<? extends IClusteringAlgorithm> clusteringAlgorithmClass;
+
+ private SolrCore core;
+ private String idFieldName;
+
+ public NamedList cluster(Query query, DocList docList, SolrParams solrParams) {
+ try {
+ // Prepare attributes for Carrot2 clustering call
+ Map<String, Object> attributes = new HashMap<String, Object>();
+ List<Document> documents = getDocuments(docList, core, query, solrParams);
+ attributes.put(AttributeNames.DOCUMENTS, documents);
+ attributes.put(AttributeNames.QUERY, query.toString());
+
+ // Pass extra overriding attributes from the request, if any
+ extractCarrotAttributes(solrParams, attributes);
+
+ // Perform clustering and convert to named list
+ return clustersToNamedList(controller.process(attributes,
+ clusteringAlgorithmClass).getClusters(), solrParams);
+ } catch (Exception e) {
+ log.error("Carrot2 clustering failed", e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public String init(NamedList config, final SolrCore core) {
+ String result = super.init(config, core);
+ SolrParams initParams = SolrParams.toSolrParams(config);
+
+ // Initialize Carrot2 controller. Pass initialization attributes, if any.
+ HashMap<String, Object> initAttributes = new HashMap<String, Object>();
+ extractCarrotAttributes(initParams, initAttributes);
+ this.controller.init(initAttributes);
+
+ this.core = core;
+ this.idFieldName = core.getSchema().getUniqueKeyField().getName();
+
+ // Make sure the requested Carrot2 clustering algorithm class is available
+ String carrotAlgorithmClassName = initParams.get(CarrotParams.ALGORITHM);
+ try {
+ Class<?> algorithmClass = Thread.currentThread().getContextClassLoader()
+ .loadClass(carrotAlgorithmClassName);
+ if (!IClusteringAlgorithm.class.isAssignableFrom(algorithmClass)) {
+ throw new IllegalArgumentException("Class provided as "
+ + CarrotParams.ALGORITHM + " must implement "
+ + IClusteringAlgorithm.class.getName());
+ }
+ this.clusteringAlgorithmClass = (Class<? extends IClusteringAlgorithm>) algorithmClass;
+ } catch (ClassNotFoundException e) {
+ throw new RuntimeException(
+ "Failed to load Carrot clustering algorithm class", e);
+ }
+
+ return result;
+ }
+
+ /**
+ * Prepares Carrot2 documents for clustering.
+ */
+ private List<Document> getDocuments(DocList docList, SolrCore core,
+ Query query, SolrParams solrParams) {
+ SolrHighlighter highligher = null;
+
+ // Names of fields to deliver content for clustering
+ String urlField = solrParams.get(CarrotParams.URL_FIELD_NAME, "url");
+ String titleField = solrParams.get(CarrotParams.TITLE_FIELD_NAME, "title");
+ String snippetField = solrParams.get(CarrotParams.SNIPPET_FIELD_NAME,
+ titleField);
+ if (StringUtils.isBlank(snippetField)) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, CarrotParams.SNIPPET_FIELD_NAME
+ + " must not be blank.");
+ }
+ Set<String> fieldsToLoad = Sets.newHashSet(urlField, titleField,
+ snippetField, idFieldName);
+
+ // Get the documents
+ DocIterator docsIter = docList.iterator();
+ boolean produceSummary = solrParams.getBool(CarrotParams.PRODUCE_SUMMARY,
+ false);
+
+ SolrQueryRequest req = null;
+ String[] snippetFieldAry = null;
+ if (produceSummary == true) {
+ highligher = core.getHighlighter();
+ Map args = new HashMap();
+ snippetFieldAry = new String[] { snippetField };
+ args.put(HighlightParams.FIELDS, snippetFieldAry);
+ args.put(HighlightParams.HIGHLIGHT, "true");
+ req = new LocalSolrQueryRequest(core, query.toString(), "", 0, 1, args);
+ }
+
+ RefCounted<SolrIndexSearcher> refCounter = core.getSearcher();
+ SolrIndexSearcher searcher = refCounter.get();
+ List<Document> result = new ArrayList<Document>(docList.size());
+ try {
+ FieldSelector fieldSelector = new SetBasedFieldSelector(fieldsToLoad,
+ Collections.emptySet());
+ float[] scores = { 1.0f };
+ int[] docsHolder = new int[1];
+ Query theQuery = query;
+
+ while (docsIter.hasNext()) {
+ Integer id = docsIter.next();
+ org.apache.lucene.document.Document doc = searcher.doc(id,
+ fieldSelector);
+ String snippet = getValue(doc, snippetField);
+ if (produceSummary == true) {
+ docsHolder[0] = id.intValue();
+ DocList docAsList = new DocSlice(0, 1, docsHolder, scores, 1, 1.0f);
+ highligher.doHighlighting(docAsList, theQuery, req, snippetFieldAry);
+ }
+ Document carrotDocument = new Document(getValue(doc, titleField),
+ snippet, doc.get(urlField));
+ carrotDocument.addField("solrId", doc.get(idFieldName));
+ result.add(carrotDocument);
+ }
+ } catch (IOException e) {
+ log.error("IOException", e);
+ } finally {
+ refCounter.decref();
+ }
+ return result;
+ }
+
+ protected String getValue(org.apache.lucene.document.Document doc,
+ String field) {
+ StringBuilder result = new StringBuilder();
+ String[] vals = doc.getValues(field);
+ for (int i = 0; i < vals.length; i++) {
+ // Join multiple values with a period so that Carrot2 does not pick up
+ // phrases that cross field value boundaries (in most cases it would
+ // create useless phrases).
+ result.append(vals[i]).append(" . ");
+ }
+ return result.toString().trim();
+ }
+
+ private NamedList clustersToNamedList(List<Cluster> carrotClusters,
+ SolrParams solrParams) {
+ NamedList result = new NamedList();
+ clustersToNamedList(carrotClusters, result, solrParams.getBool(
+ CarrotParams.OUTPUT_SUB_CLUSTERS, false), solrParams.getInt(
+ CarrotParams.NUM_DESCRIPTIONS, Integer.MAX_VALUE));
+ return result;
+ }
+
+ private void clustersToNamedList(List<Cluster> outputClusters,
+ NamedList parent, boolean outputSubClusters, int maxLabels) {
+ for (Cluster outCluster : outputClusters) {
+ NamedList cluster = new NamedList();
+ parent.add("cluster", cluster);
+
+ List<String> labels = outCluster.getPhrases();
+ NamedList labelsNL = new NamedList();
+ cluster.add("labels", labelsNL);
+ int labelsAdded = 0;
+ for (String label : labels) {
+ if (++labelsAdded > maxLabels) {
+ break;
+ }
+ labelsNL.add("label", label);
+ }
+
+ List<Document> docs = outCluster.getDocuments();
+ NamedList docsNL = new NamedList();
+ cluster.add("docs", docsNL);
+ for (Document doc : docs) {
+ docsNL.add("doc", doc.getField("solrId"));
+ }
+
+ if (outputSubClusters) {
+ NamedList subclustersNL = new NamedList();
+ cluster.add("clusters", subclustersNL);
+ clustersToNamedList(outCluster.getSubclusters(), subclustersNL,
+ outputSubClusters, maxLabels);
+ }
+ }
+ }
+
+ /**
+ * Extracts parameters that can possibly match some attributes of Carrot2 algorithms.
+ */
+ private void extractCarrotAttributes(SolrParams solrParams,
+ Map<String, Object> attributes) {
+ // Extract all non-predefined parameters. This way, we'll be able to set all
+ // parameters of Carrot2 algorithms without defining their names as constants.
+ for (Iterator<String> paramNames = solrParams.getParameterNamesIterator(); paramNames
+ .hasNext();) {
+ String paramName = paramNames.next();
+ if (!CarrotParams.CARROT_PARAM_NAMES.contains(paramName)) {
+ attributes.put(paramName, solrParams.get(paramName));
+ }
+ }
+ }
+}
Added: lucene/solr/trunk/contrib/clustering/src/main/java/org/apache/solr/handler/clustering/carrot2/CarrotParams.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/clustering/src/main/java/org/apache/solr/handler/clustering/carrot2/CarrotParams.java?rev=776692&view=auto
==============================================================================
--- lucene/solr/trunk/contrib/clustering/src/main/java/org/apache/solr/handler/clustering/carrot2/CarrotParams.java (added)
+++ lucene/solr/trunk/contrib/clustering/src/main/java/org/apache/solr/handler/clustering/carrot2/CarrotParams.java Wed May 20 13:29:52 2009
@@ -0,0 +1,42 @@
+package org.apache.solr.handler.clustering.carrot2;
+
+import java.util.Set;
+
+import com.google.common.collect.ImmutableSet;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ *
+ */
+public interface CarrotParams {
+
+ String CARROT_PREFIX = "carrot.";
+
+ String ALGORITHM = CARROT_PREFIX + "algorithm";
+ String TITLE_FIELD_NAME = CARROT_PREFIX + "title";
+ String URL_FIELD_NAME = CARROT_PREFIX + "url";
+ String SNIPPET_FIELD_NAME = CARROT_PREFIX + "snippet";
+ String PRODUCE_SUMMARY = CARROT_PREFIX + "produceSummary";
+ String NUM_DESCRIPTIONS = CARROT_PREFIX + "numDescriptions";
+ String OUTPUT_SUB_CLUSTERS = CARROT_PREFIX + "outputSubClusters";
+
+ public static final Set<String> CARROT_PARAM_NAMES = ImmutableSet.of(
+ ALGORITHM, TITLE_FIELD_NAME, URL_FIELD_NAME, SNIPPET_FIELD_NAME,
+ PRODUCE_SUMMARY, NUM_DESCRIPTIONS, OUTPUT_SUB_CLUSTERS);
+}
Added: lucene/solr/trunk/contrib/clustering/src/test/java/org/apache/solr/handler/clustering/AbstractClusteringTest.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/clustering/src/test/java/org/apache/solr/handler/clustering/AbstractClusteringTest.java?rev=776692&view=auto
==============================================================================
--- lucene/solr/trunk/contrib/clustering/src/test/java/org/apache/solr/handler/clustering/AbstractClusteringTest.java (added)
+++ lucene/solr/trunk/contrib/clustering/src/test/java/org/apache/solr/handler/clustering/AbstractClusteringTest.java Wed May 20 13:29:52 2009
@@ -0,0 +1,198 @@
+package org.apache.solr.handler.clustering;
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.solr.util.AbstractSolrTestCase;
+
+
+/**
+ *
+ */
+public class AbstractClusteringTest extends AbstractSolrTestCase {
+ protected int numberOfDocs = 0;
+
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+
+ numberOfDocs = 0;
+ for (String[] doc : DOCUMENTS) {
+ assertU("add failed", adoc("id", Integer.toString(numberOfDocs), "url", doc[0], "title", doc[1], "snippet", doc[2]));
+ numberOfDocs++;
+ }
+ assertU("commit", commit());
+ }
+
+ public String getSchemaFile() {
+ return "schema.xml";
+ }
+
+ public String getSolrConfigFile() {
+ return "solrconfig.xml";
+ }
+
+ final String [][] DOCUMENTS = new String[][] {
+ { "http://en.wikipedia.org/wiki/Data_mining",
+ "Data Mining - Wikipedia",
+ "Article about knowledge-discovery in databases (KDD), the practice of automatically searching large stores of data for patterns." },
+
+
+ { "http://en.wikipedia.org/wiki/Datamining",
+ "Data mining - Wikipedia, the free encyclopedia",
+ "Data mining is the entire process of applying computer-based methodology, ... Moreover, some data-mining systems such as neural networks are inherently geared ..." },
+
+
+ { "http://www.statsoft.com/textbook/stdatmin.html",
+ "Electronic Statistics Textbook: Data Mining Techniques",
+ "Outlines the crucial concepts in data mining, defines the data warehousing process, and offers examples of computational and graphical exploratory data analysis techniques." },
+
+
+ { "http://www.thearling.com/text/dmwhite/dmwhite.htm",
+ "An Introduction to Data Mining",
+ "Data mining, the extraction of hidden predictive information from large ... Data mining tools predict future trends and behaviors, allowing businesses to ..." },
+
+
+ { "http://www.anderson.ucla.edu/faculty/jason.frand/teacher/technologies/palace/datamining.htm",
+ "Data Mining: What is Data Mining?",
+ "Outlines what knowledge discovery, the process of analyzing data from different perspectives and summarizing it into useful information, can do and how it works." },
+
+
+ { "http://www.spss.com/datamine",
+ "Data Mining Software, Data Mining Applications and Data Mining Solutions",
+ "The patterns uncovered using data mining help organizations make better and ... data mining customer ... Data mining applications, on the other hand, embed ..." },
+
+
+ { "http://www.kdnuggets.com/",
+ "KD Nuggets",
+ "Newsletter on the data mining and knowledge industries, offering information on data mining, knowledge discovery, text mining, and web mining software, courses, jobs, publications, and meetings." },
+
+
+ { "http://www.answers.com/topic/data-mining",
+ "data mining: Definition from Answers.com",
+ "data mining n. The automatic extraction of useful, often previously unknown information from large databases or data ... Data Mining For Investing ..." },
+
+
+ { "http://www.statsoft.com/products/dataminer.htm",
+ "STATISTICA Data Mining and Predictive Modeling Solutions",
+ "GRC site-wide menuing system research and development. ... Contact a Data Mining Solutions Consultant. News and Success Stories. Events ..." },
+
+
+ { "http://datamining.typepad.com/",
+ "Data Mining: Text Mining, Visualization and Social Media",
+ "Commentary on text mining, data mining, social media and data visualization. ... While mining Twitter data for business and marketing intelligence (trend/buzz ..." },
+
+
+ { "http://www.twocrows.com/",
+ "Two Crows Corporation",
+ "Dedicated to the development, marketing, sales and support of tools for knowledge discovery to make data mining accessible and easy to use." },
+
+
+ { "http://www.thearling.com/",
+ "Thearling.com",
+ "Kurt Thearling's site dedicated to sharing information about data mining, the automated extraction of hidden predictive information from databases, and other analytic technologies." },
+
+
+ { "http://www.ccsu.edu/datamining/",
+ "CCSU - Data Mining",
+ "Offers degrees and certificates in data mining. Allows students to explore cutting-edge data mining techniques and applications: market basket analysis, decision trees, neural networks, machine learning, web mining, and data modeling." },
+
+
+ { "http://www.oracle.com/technology/products/bi/odm",
+ "Oracle Data Mining",
+ "Oracle Data Mining Product Center ... New Oracle Data Mining Powers New Social CRM Application (more information ... Mining High-Dimensional Data for ..." },
+
+
+ { "http://databases.about.com/od/datamining/a/datamining.htm",
+ "Data Mining: An Introduction",
+ "About.com article on how businesses are discovering new trends and patterns of behavior that previously went unnoticed through data mining, automated statistical analysis techniques." },
+
+
+ { "http://www.dmoz.org/Computers/Software/Databases/Data_Mining/",
+ "Open Directory - Computers: Software: Databases: Data Mining",
+ "Data Mining and Knowledge Discovery - A peer-reviewed journal publishing ... Data mining creates information assets that an organization can leverage to ..." },
+
+
+ { "http://www.cs.wisc.edu/dmi/",
+ "DMI:Data Mining Institute",
+ "Data Mining Institute at UW-Madison ... The Data Mining Institute (DMI) was started on June 1, 1999 at the Computer ... of the Data Mining Group of Microsoft ..." },
+
+
+ { "http://www.the-data-mine.com/",
+ "The Data Mine",
+ "Provides information about data mining also known as knowledge discovery in databases (KDD) or simply knowledge discovery. List software, events, organizations, and people working in data mining." },
+
+
+ { "http://www.statserv.com/datamining.html",
+ "St@tServ - About Data Mining",
+ "St@tServ Data Mining page ... Data mining in molecular biology, by Alvis Brazma. Graham Williams page. Knowledge Discovery and Data Mining Resources, ..." },
+
+
+ { "http://ocw.mit.edu/OcwWeb/Sloan-School-of-Management/15-062Data-MiningSpring2003/CourseHome/index.htm",
+ "MIT OpenCourseWare | Sloan School of Management | 15.062 Data Mining ...",
+ "Introduces students to a class of methods known as data mining that assists managers in recognizing patterns and making intelligent use of massive amounts of ..." },
+
+
+ { "http://www.pentaho.com/products/data_mining/",
+ "Pentaho Commercial Open Source Business Intelligence: Data Mining",
+ "For example, data mining can warn you there's a high probability a specific ... Pentaho Data Mining is differentiated by its open, standards-compliant nature, ..." },
+
+
+ { "http://www.investorhome.com/mining.htm",
+ "Investor Home - Data Mining",
+ "Data Mining or Data Snooping is the practice of searching for relationships and ... Data mining involves searching through databases for correlations and patterns ..." },
+
+
+ { "http://www.datamining.com/",
+ "Predictive Modeling and Predictive Analytics Solutions | Enterprise ...",
+ "Insightful Enterprise Miner - Enterprise data mining for predictive modeling and predictive analytics." },
+
+
+ { "http://www.sourcewatch.org/index.php?title=Data_mining",
+ "Data mining - SourceWatch",
+ "These agencies reported 199 data mining projects, of which 68 ... Office, \"DATA MINING. ... powerful technology known as data mining -- and how, in the ..." },
+
+
+ { "http://www.autonlab.org/tutorials/",
+ "Statistical Data Mining Tutorials",
+ "Includes a set of tutorials on many aspects of statistical data mining, including the foundations of probability, the foundations of statistical data analysis, and most of the classic machine learning and data mining algorithms." },
+
+
+ { "http://www.microstrategy.com/data-mining/index.asp",
+ "Data Mining",
+ "With MicroStrategy, data mining scoring is fully integrated into mainstream ... The integration of data mining models from other applications is accomplished by ..." },
+
+
+ { "http://www.datamininglab.com/",
+ "Elder Research",
+ "Provides consulting and short courses in data mining and pattern discovery patterns in data." },
+
+
+ { "http://www.sqlserverdatamining.com/",
+ "SQL Server Data Mining > Home",
+ "SQL Server Data Mining Portal ... Data Mining as an Application Platform (Whitepaper) Creating a Web Cross-sell Application with SQL Server 2005 Data Mining (Article) ..." },
+
+
+ { "http://databases.about.com/cs/datamining/g/dmining.htm",
+ "Data Mining",
+ "What is data mining? Find out here! ... Book Review: Data Mining and Statistical Analysis Using SQL. What is Data Mining, and What Does it Have to Do with ..." },
+
+
+ { "http://www.sas.com/technologies/analytics/datamining/index.html",
+ "Data Mining Software and Text Mining | SAS",
+ "... raw data to smarter ... Data Mining is an iterative process of creating ... The knowledge gleaned from data and text mining can be used to fuel ..." }
+ };
+}
Added: lucene/solr/trunk/contrib/clustering/src/test/java/org/apache/solr/handler/clustering/ClusteringComponentTest.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/clustering/src/test/java/org/apache/solr/handler/clustering/ClusteringComponentTest.java?rev=776692&view=auto
==============================================================================
--- lucene/solr/trunk/contrib/clustering/src/test/java/org/apache/solr/handler/clustering/ClusteringComponentTest.java (added)
+++ lucene/solr/trunk/contrib/clustering/src/test/java/org/apache/solr/handler/clustering/ClusteringComponentTest.java Wed May 20 13:29:52 2009
@@ -0,0 +1,82 @@
+package org.apache.solr.handler.clustering;
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.solr.util.AbstractSolrTestCase;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.handler.component.SearchComponent;
+import org.apache.solr.handler.component.SpellCheckComponent;
+import org.apache.solr.handler.component.QueryComponent;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.SimpleOrderedMap;
+import org.apache.solr.request.SolrRequestHandler;
+import org.apache.solr.request.SolrQueryResponse;
+import org.apache.solr.request.LocalSolrQueryRequest;
+
+
+/**
+ *
+ *
+ **/
+public class ClusteringComponentTest extends AbstractClusteringTest {
+
+ public void testComponent() throws Exception {
+ SolrCore core = h.getCore();
+
+ SearchComponent sc = core.getSearchComponent("clustering");
+ assertTrue("sc is null and it shouldn't be", sc != null);
+ ModifiableSolrParams params = new ModifiableSolrParams();
+
+ params.add(ClusteringComponent.COMPONENT_NAME, "true");
+ params.add(CommonParams.Q, "*:*");
+
+ params.add(ClusteringParams.USE_SEARCH_RESULTS, "true");
+
+
+ SolrRequestHandler handler = core.getRequestHandler("standard");
+ SolrQueryResponse rsp;
+ rsp = new SolrQueryResponse();
+ rsp.add("responseHeader", new SimpleOrderedMap());
+ handler.handleRequest(new LocalSolrQueryRequest(core, params), rsp);
+ NamedList values = rsp.getValues();
+ NamedList clusters = (NamedList) values.get("clusters");
+ //System.out.println("Clusters: " + clusters);
+ assertTrue("clusters is null and it shouldn't be", clusters != null);
+
+
+ params = new ModifiableSolrParams();
+ params.add(ClusteringComponent.COMPONENT_NAME, "true");
+ params.add(ClusteringParams.ENGINE_NAME, "mock");
+ params.add(ClusteringParams.USE_COLLECTION, "true");
+ params.add(QueryComponent.COMPONENT_NAME, "false");
+
+ handler = core.getRequestHandler("docClustering");
+
+ rsp = new SolrQueryResponse();
+ rsp.add("responseHeader", new SimpleOrderedMap());
+ handler.handleRequest(new LocalSolrQueryRequest(core, params), rsp);
+ values = rsp.getValues();
+ clusters = (NamedList) values.get("clusters");
+ //System.out.println("Clusters: " + clusters);
+ assertTrue("clusters is null and it shouldn't be", clusters != null);
+
+
+ }
+
+}
Added: lucene/solr/trunk/contrib/clustering/src/test/java/org/apache/solr/handler/clustering/MockDocumentClusteringEngine.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/clustering/src/test/java/org/apache/solr/handler/clustering/MockDocumentClusteringEngine.java?rev=776692&view=auto
==============================================================================
--- lucene/solr/trunk/contrib/clustering/src/test/java/org/apache/solr/handler/clustering/MockDocumentClusteringEngine.java (added)
+++ lucene/solr/trunk/contrib/clustering/src/test/java/org/apache/solr/handler/clustering/MockDocumentClusteringEngine.java Wed May 20 13:29:52 2009
@@ -0,0 +1,22 @@
+package org.apache.solr.handler.clustering;
+
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.search.DocSet;
+
+
+/**
+ *
+ *
+ **/
+public class MockDocumentClusteringEngine extends DocumentClusteringEngine{
+ public NamedList cluster(DocSet docs, SolrParams solrParams) {
+ NamedList result = new NamedList();
+ return result;
+ }
+
+ public NamedList cluster(SolrParams solrParams) {
+ NamedList result = new NamedList();
+ return result;
+ }
+}
Added: lucene/solr/trunk/contrib/clustering/src/test/java/org/apache/solr/handler/clustering/carrot2/CarrotClusteringEngineTest.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/clustering/src/test/java/org/apache/solr/handler/clustering/carrot2/CarrotClusteringEngineTest.java?rev=776692&view=auto
==============================================================================
--- lucene/solr/trunk/contrib/clustering/src/test/java/org/apache/solr/handler/clustering/carrot2/CarrotClusteringEngineTest.java (added)
+++ lucene/solr/trunk/contrib/clustering/src/test/java/org/apache/solr/handler/clustering/carrot2/CarrotClusteringEngineTest.java Wed May 20 13:29:52 2009
@@ -0,0 +1,168 @@
+package org.apache.solr.handler.clustering.carrot2;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+
+import org.apache.lucene.search.*;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.handler.clustering.AbstractClusteringTest;
+import org.apache.solr.handler.clustering.ClusteringComponent;
+import org.apache.solr.search.DocList;
+import org.apache.solr.search.SolrIndexSearcher;
+import org.apache.solr.util.RefCounted;
+import org.carrot2.util.attribute.AttributeUtils;
+
+/**
+ *
+ */
+@SuppressWarnings("unchecked")
+public class CarrotClusteringEngineTest extends AbstractClusteringTest {
+ public void testCarrotLingo() throws Exception {
+ checkEngine(getClusteringEngine("default"), 9);
+ }
+
+ public void testCarrotStc() throws Exception {
+ checkEngine(getClusteringEngine("stc"), 2);
+ }
+
+ public void testWithoutSubclusters() throws Exception {
+ checkClusters(checkEngine(getClusteringEngine("mock"), this.numberOfDocs),
+ 1, 1, 0);
+ }
+
+ public void testWithSubclusters() throws Exception {
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.set(CarrotParams.OUTPUT_SUB_CLUSTERS, true);
+ checkClusters(checkEngine(getClusteringEngine("mock"), this.numberOfDocs,
+ params), 1, 1, 2);
+ }
+
+ public void testNumDescriptions() throws Exception {
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.set(AttributeUtils.getKey(MockClusteringAlgorithm.class, "labels"), 5);
+ params.set(CarrotParams.NUM_DESCRIPTIONS, 3);
+ checkClusters(checkEngine(getClusteringEngine("mock"), this.numberOfDocs,
+ params), 1, 3, 0);
+ }
+
+ public void testCarrotAttributePassing() throws Exception {
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.set(AttributeUtils.getKey(MockClusteringAlgorithm.class, "depth"), 1);
+ params.set(AttributeUtils.getKey(MockClusteringAlgorithm.class, "labels"), 3);
+ checkClusters(checkEngine(getClusteringEngine("mock"), this.numberOfDocs,
+ params), 1, 3, 0);
+ }
+
+ private CarrotClusteringEngine getClusteringEngine(String engineName) {
+ ClusteringComponent comp = (ClusteringComponent) h.getCore()
+ .getSearchComponent("clustering");
+ assertNotNull("clustering component should not be null", comp);
+ CarrotClusteringEngine engine = (CarrotClusteringEngine) comp
+ .getSearchClusteringEngines().get(engineName);
+ assertNotNull("clustering engine for name: " + engineName
+ + " should not be null", engine);
+ return engine;
+ }
+
+ private NamedList checkEngine(CarrotClusteringEngine engine,
+ int expectedNumClusters) throws IOException {
+ return checkEngine(engine, expectedNumClusters, new ModifiableSolrParams());
+ }
+
+ private NamedList checkEngine(CarrotClusteringEngine engine,
+ int expectedNumClusters, SolrParams clusteringParams) throws IOException {
+ // Get all documents to cluster
+ RefCounted<SolrIndexSearcher> ref = h.getCore().getSearcher();
+ MatchAllDocsQuery query = new MatchAllDocsQuery();
+ DocList docList;
+ try {
+ SolrIndexSearcher searcher = ref.get();
+ docList = searcher.getDocList(query, (Query) null, new Sort(), 0,
+ numberOfDocs);
+ assertEquals("docList size", this.numberOfDocs, docList.matches());
+ } finally {
+ ref.decref();
+ }
+
+ ModifiableSolrParams solrParams = new ModifiableSolrParams();
+ solrParams.add(CarrotParams.PRODUCE_SUMMARY, "true");
+ solrParams.add(clusteringParams);
+
+ // Perform clustering
+ NamedList results = engine.cluster(query, docList, solrParams);
+ assertEquals("number of clusters", expectedNumClusters, results.size());
+ checkClusters(results, false);
+ return results;
+ }
+
+ private void checkClusters(NamedList results, int expectedDocCount,
+ int expectedLabelCount, int expectedSubclusterCount) {
+ for (int i = 0; i < results.size(); i++) {
+ if (results.getName(i).equals("cluster")) {
+ NamedList cluster = (NamedList) results.getVal(i);
+ checkCluster(cluster, expectedDocCount, expectedLabelCount,
+ expectedSubclusterCount);
+ }
+ }
+ }
+
+ private void checkClusters(NamedList results, boolean hasSubclusters) {
+ for (int i = 0; i < results.size(); i++) {
+ if (results.getName(i).equals("cluster")) {
+ NamedList cluster = (NamedList) results.getVal(i);
+ checkCluster(cluster, hasSubclusters);
+ }
+ }
+ }
+
+ private void checkCluster(NamedList cluster, boolean hasSubclusters) {
+ NamedList docs = (NamedList) cluster.get("docs");
+ assertNotNull("docs is null and it shouldn't be", docs);
+ for (int j = 0; j < docs.size(); j++) {
+ String id = (String) docs.getVal(j);
+ assertNotNull("id is null and it shouldn't be", id);
+ }
+
+ NamedList labels = (NamedList) cluster.get("labels");
+ assertNotNull("labels is null but it shouldn't be", labels);
+
+ if (hasSubclusters) {
+ NamedList subclusters = (NamedList) cluster.get("clusters");
+ assertNotNull("subclusters is null but it shouldn't be", subclusters);
+ }
+ }
+
+ private void checkCluster(NamedList cluster, int expectedDocCount,
+ int expectedLabelCount, int expectedSubclusterCount) {
+ checkCluster(cluster, expectedSubclusterCount > 0);
+ assertEquals("number of docs in cluster", expectedDocCount,
+ ((NamedList) cluster.get("docs")).size());
+ assertEquals("number of labels in cluster", expectedLabelCount,
+ ((NamedList) cluster.get("labels")).size());
+
+ if (expectedSubclusterCount > 0) {
+ NamedList subclusters = (NamedList) cluster.get("clusters");
+ assertEquals("numClusters", expectedSubclusterCount, subclusters.size());
+ assertEquals("number of subclusters in cluster",
+ expectedSubclusterCount, subclusters.size());
+ }
+ }
+}
Added: lucene/solr/trunk/contrib/clustering/src/test/java/org/apache/solr/handler/clustering/carrot2/MockClusteringAlgorithm.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/clustering/src/test/java/org/apache/solr/handler/clustering/carrot2/MockClusteringAlgorithm.java?rev=776692&view=auto
==============================================================================
--- lucene/solr/trunk/contrib/clustering/src/test/java/org/apache/solr/handler/clustering/carrot2/MockClusteringAlgorithm.java (added)
+++ lucene/solr/trunk/contrib/clustering/src/test/java/org/apache/solr/handler/clustering/carrot2/MockClusteringAlgorithm.java Wed May 20 13:29:52 2009
@@ -0,0 +1,69 @@
+package org.apache.solr.handler.clustering.carrot2;
+
+import java.util.List;
+
+import org.carrot2.core.*;
+import org.carrot2.core.attribute.AttributeNames;
+import org.carrot2.core.attribute.Processing;
+import org.carrot2.util.attribute.*;
+import org.carrot2.util.attribute.constraint.IntRange;
+
+import com.google.common.collect.Lists;
+
+@Bindable(prefix = "MockClusteringAlgorithm")
+public class MockClusteringAlgorithm extends ProcessingComponentBase implements
+ IClusteringAlgorithm {
+ @Input
+ @Processing
+ @Attribute(key = AttributeNames.DOCUMENTS)
+ private List<Document> documents;
+
+ @Output
+ @Processing
+ @Attribute(key = AttributeNames.CLUSTERS)
+ private List<Cluster> clusters;
+
+ @Input
+ @Processing
+ @Attribute
+ @IntRange(min = 1, max = 5)
+ private int depth = 2;
+
+ @Input
+ @Processing
+ @Attribute
+ @IntRange(min = 1, max = 5)
+ private int labels = 1;
+
+ @Override
+ public void process() throws ProcessingException {
+ clusters = Lists.newArrayList();
+ if (documents == null) {
+ return;
+ }
+
+ int documentIndex = 1;
+ for (Document document : documents) {
+ StringBuilder label = new StringBuilder("Cluster " + documentIndex);
+ Cluster cluster = createCluster(label.toString(), document);
+ clusters.add(cluster);
+ for (int i = 1; i <= depth; i++) {
+ label.append(".");
+ label.append(i);
+ Cluster newCluster = createCluster(label.toString(), document);
+ cluster.addSubclusters(createCluster(label.toString(), document), newCluster);
+ cluster = newCluster;
+ }
+ documentIndex++;
+ }
+ }
+
+ private Cluster createCluster(String labelBase, Document... documents) {
+ Cluster cluster = new Cluster();
+ for (int i = 0; i < labels; i++) {
+ cluster.addPhrases(labelBase + "#" + (i + 1));
+ }
+ cluster.addDocuments(documents);
+ return cluster;
+ }
+}
Added: lucene/solr/trunk/contrib/clustering/src/test/resources/solr/conf/mapping-ISOLatin1Accent.txt
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/clustering/src/test/resources/solr/conf/mapping-ISOLatin1Accent.txt?rev=776692&view=auto
==============================================================================
--- lucene/solr/trunk/contrib/clustering/src/test/resources/solr/conf/mapping-ISOLatin1Accent.txt (added)
+++ lucene/solr/trunk/contrib/clustering/src/test/resources/solr/conf/mapping-ISOLatin1Accent.txt Wed May 20 13:29:52 2009
@@ -0,0 +1,246 @@
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Syntax:
+# "source" => "target"
+# "source".length() > 0 (source cannot be empty.)
+# "target".length() >= 0 (target can be empty.)
+
+# example:
+# "Ã" => "A"
+# "\u00C0" => "A"
+# "\u00C0" => "\u0041"
+# "Ã" => "ss"
+# "\t" => " "
+# "\n" => ""
+
+# Ã => A
+"\u00C0" => "A"
+
+# Ã => A
+"\u00C1" => "A"
+
+# Ã => A
+"\u00C2" => "A"
+
+# Ã => A
+"\u00C3" => "A"
+
+# Ã => A
+"\u00C4" => "A"
+
+# Ã
=> A
+"\u00C5" => "A"
+
+# Ã => AE
+"\u00C6" => "AE"
+
+# Ã => C
+"\u00C7" => "C"
+
+# Ã => E
+"\u00C8" => "E"
+
+# Ã => E
+"\u00C9" => "E"
+
+# Ã => E
+"\u00CA" => "E"
+
+# Ã => E
+"\u00CB" => "E"
+
+# Ã => I
+"\u00CC" => "I"
+
+# Ã => I
+"\u00CD" => "I"
+
+# Ã => I
+"\u00CE" => "I"
+
+# Ã => I
+"\u00CF" => "I"
+
+# IJ => IJ
+"\u0132" => "IJ"
+
+# Ã => D
+"\u00D0" => "D"
+
+# Ã => N
+"\u00D1" => "N"
+
+# Ã => O
+"\u00D2" => "O"
+
+# Ã => O
+"\u00D3" => "O"
+
+# Ã => O
+"\u00D4" => "O"
+
+# Ã => O
+"\u00D5" => "O"
+
+# Ã => O
+"\u00D6" => "O"
+
+# Ã => O
+"\u00D8" => "O"
+
+# Å => OE
+"\u0152" => "OE"
+
+# Ã
+"\u00DE" => "TH"
+
+# Ã => U
+"\u00D9" => "U"
+
+# Ã => U
+"\u00DA" => "U"
+
+# Ã => U
+"\u00DB" => "U"
+
+# Ã => U
+"\u00DC" => "U"
+
+# Ã => Y
+"\u00DD" => "Y"
+
+# Ÿ => Y
+"\u0178" => "Y"
+
+# Ã => a
+"\u00E0" => "a"
+
+# á => a
+"\u00E1" => "a"
+
+# â => a
+"\u00E2" => "a"
+
+# ã => a
+"\u00E3" => "a"
+
+# ä => a
+"\u00E4" => "a"
+
+# å => a
+"\u00E5" => "a"
+
+# æ => ae
+"\u00E6" => "ae"
+
+# ç => c
+"\u00E7" => "c"
+
+# è => e
+"\u00E8" => "e"
+
+# é => e
+"\u00E9" => "e"
+
+# ê => e
+"\u00EA" => "e"
+
+# ë => e
+"\u00EB" => "e"
+
+# ì => i
+"\u00EC" => "i"
+
+# Ã => i
+"\u00ED" => "i"
+
+# î => i
+"\u00EE" => "i"
+
+# ï => i
+"\u00EF" => "i"
+
+# ij => ij
+"\u0133" => "ij"
+
+# ð => d
+"\u00F0" => "d"
+
+# ñ => n
+"\u00F1" => "n"
+
+# ò => o
+"\u00F2" => "o"
+
+# ó => o
+"\u00F3" => "o"
+
+# ô => o
+"\u00F4" => "o"
+
+# õ => o
+"\u00F5" => "o"
+
+# ö => o
+"\u00F6" => "o"
+
+# ø => o
+"\u00F8" => "o"
+
+# Å => oe
+"\u0153" => "oe"
+
+# Ã => ss
+"\u00DF" => "ss"
+
+# þ => th
+"\u00FE" => "th"
+
+# ù => u
+"\u00F9" => "u"
+
+# ú => u
+"\u00FA" => "u"
+
+# û => u
+"\u00FB" => "u"
+
+# ü => u
+"\u00FC" => "u"
+
+# ý => y
+"\u00FD" => "y"
+
+# ÿ => y
+"\u00FF" => "y"
+
+# ï¬ => ff
+"\uFB00" => "ff"
+
+# ï¬ => fi
+"\uFB01" => "fi"
+
+# ï¬ => fl
+"\uFB02" => "fl"
+
+# ï¬ => ffi
+"\uFB03" => "ffi"
+
+# ï¬ => ffl
+"\uFB04" => "ffl"
+
+# ï¬
=> ft
+"\uFB05" => "ft"
+
+# ï¬ => st
+"\uFB06" => "st"