You are viewing a plain text version of this content. The canonical link for it is here.
Posted to java-commits@lucene.apache.org by rm...@apache.org on 2009/10/28 18:49:54 UTC
svn commit: r830699 - in /lucene/java/trunk/contrib: ./
memory/src/java/org/apache/lucene/index/memory/
memory/src/test/org/apache/lucene/index/memory/
wordnet/src/java/org/apache/lucene/wordnet/
wordnet/src/test/org/apache/lucene/wordnet/
Author: rmuir
Date: Wed Oct 28 17:49:53 2009
New Revision: 830699
URL: http://svn.apache.org/viewvc?rev=830699&view=rev
Log:
LUCENE-1904: Move wordnet synonym code from contrib/memory to contrib/wordnet
Added:
lucene/java/trunk/contrib/wordnet/src/java/org/apache/lucene/wordnet/AnalyzerUtil.java
- copied, changed from r830669, lucene/java/trunk/contrib/memory/src/java/org/apache/lucene/index/memory/AnalyzerUtil.java
lucene/java/trunk/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynonymMap.java
- copied, changed from r830669, lucene/java/trunk/contrib/memory/src/java/org/apache/lucene/index/memory/SynonymMap.java
lucene/java/trunk/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynonymTokenFilter.java
- copied, changed from r830669, lucene/java/trunk/contrib/memory/src/java/org/apache/lucene/index/memory/SynonymTokenFilter.java
lucene/java/trunk/contrib/wordnet/src/test/org/apache/lucene/wordnet/TestSynonymTokenFilter.java
- copied, changed from r830669, lucene/java/trunk/contrib/memory/src/test/org/apache/lucene/index/memory/TestSynonymTokenFilter.java
Removed:
lucene/java/trunk/contrib/memory/src/java/org/apache/lucene/index/memory/AnalyzerUtil.java
lucene/java/trunk/contrib/memory/src/java/org/apache/lucene/index/memory/SynonymMap.java
lucene/java/trunk/contrib/memory/src/java/org/apache/lucene/index/memory/SynonymTokenFilter.java
lucene/java/trunk/contrib/memory/src/test/org/apache/lucene/index/memory/TestSynonymTokenFilter.java
lucene/java/trunk/contrib/memory/src/test/org/apache/lucene/index/memory/testSynonyms.txt
Modified:
lucene/java/trunk/contrib/CHANGES.txt
lucene/java/trunk/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynExpand.java
lucene/java/trunk/contrib/wordnet/src/java/org/apache/lucene/wordnet/package.html
Modified: lucene/java/trunk/contrib/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/CHANGES.txt?rev=830699&r1=830698&r2=830699&view=diff
==============================================================================
--- lucene/java/trunk/contrib/CHANGES.txt (original)
+++ lucene/java/trunk/contrib/CHANGES.txt Wed Oct 28 17:49:53 2009
@@ -89,6 +89,9 @@
Build
+ * LUCENE-1904: Moved wordnet-based synonym support from contrib/memory
+ into contrib/wordnet. (Robert Muir)
+
Test Cases
======================= Release 2.9.0 2009-09-23 =======================
Copied: lucene/java/trunk/contrib/wordnet/src/java/org/apache/lucene/wordnet/AnalyzerUtil.java (from r830669, lucene/java/trunk/contrib/memory/src/java/org/apache/lucene/index/memory/AnalyzerUtil.java)
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/wordnet/src/java/org/apache/lucene/wordnet/AnalyzerUtil.java?p2=lucene/java/trunk/contrib/wordnet/src/java/org/apache/lucene/wordnet/AnalyzerUtil.java&p1=lucene/java/trunk/contrib/memory/src/java/org/apache/lucene/index/memory/AnalyzerUtil.java&r1=830669&r2=830699&rev=830699&view=diff
==============================================================================
--- lucene/java/trunk/contrib/memory/src/java/org/apache/lucene/index/memory/AnalyzerUtil.java (original)
+++ lucene/java/trunk/contrib/wordnet/src/java/org/apache/lucene/wordnet/AnalyzerUtil.java Wed Oct 28 17:49:53 2009
@@ -1,4 +1,4 @@
-package org.apache.lucene.index.memory;
+package org.apache.lucene.wordnet;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
Modified: lucene/java/trunk/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynExpand.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynExpand.java?rev=830699&r1=830698&r2=830699&view=diff
==============================================================================
--- lucene/java/trunk/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynExpand.java (original)
+++ lucene/java/trunk/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynExpand.java Wed Oct 28 17:49:53 2009
@@ -97,7 +97,7 @@
*
* @param a optional analyzer used to parse the users query else {@link StandardAnalyzer} is used
*
- * @param field optional field name to search in or null if you want the default of "contents"
+ * @param f optional field name to search in or null if you want the default of "contents"
*
* @param boost optional boost applied to synonyms else no boost is applied
*
Copied: lucene/java/trunk/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynonymMap.java (from r830669, lucene/java/trunk/contrib/memory/src/java/org/apache/lucene/index/memory/SynonymMap.java)
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynonymMap.java?p2=lucene/java/trunk/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynonymMap.java&p1=lucene/java/trunk/contrib/memory/src/java/org/apache/lucene/index/memory/SynonymMap.java&r1=830669&r2=830699&rev=830699&view=diff
==============================================================================
--- lucene/java/trunk/contrib/memory/src/java/org/apache/lucene/index/memory/SynonymMap.java (original)
+++ lucene/java/trunk/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynonymMap.java Wed Oct 28 17:49:53 2009
@@ -1,4 +1,4 @@
-package org.apache.lucene.index.memory;
+package org.apache.lucene.wordnet;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
Copied: lucene/java/trunk/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynonymTokenFilter.java (from r830669, lucene/java/trunk/contrib/memory/src/java/org/apache/lucene/index/memory/SynonymTokenFilter.java)
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynonymTokenFilter.java?p2=lucene/java/trunk/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynonymTokenFilter.java&p1=lucene/java/trunk/contrib/memory/src/java/org/apache/lucene/index/memory/SynonymTokenFilter.java&r1=830669&r2=830699&rev=830699&view=diff
==============================================================================
--- lucene/java/trunk/contrib/memory/src/java/org/apache/lucene/index/memory/SynonymTokenFilter.java (original)
+++ lucene/java/trunk/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynonymTokenFilter.java Wed Oct 28 17:49:53 2009
@@ -1,4 +1,4 @@
-package org.apache.lucene.index.memory;
+package org.apache.lucene.wordnet;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
Modified: lucene/java/trunk/contrib/wordnet/src/java/org/apache/lucene/wordnet/package.html
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/wordnet/src/java/org/apache/lucene/wordnet/package.html?rev=830699&r1=830698&r2=830699&view=diff
==============================================================================
--- lucene/java/trunk/contrib/wordnet/src/java/org/apache/lucene/wordnet/package.html (original)
+++ lucene/java/trunk/contrib/wordnet/src/java/org/apache/lucene/wordnet/package.html Wed Oct 28 17:49:53 2009
@@ -21,8 +21,14 @@
</head>
<body>
- This package uses synonyms defined by <a href="http://www.cogsci.princeton.edu/~wn/">WordNet</a> to build a
- Lucene index storing them, which in turn can be used for query expansion.
+ This package uses synonyms defined by <a href="http://www.cogsci.princeton.edu/~wn/">WordNet</a>.
+ There are two methods: query expansion and analysis.
+
+ Both methods first require you to download the <a href="http://www.cogsci.princeton.edu/2.0/WNprolog-2.0.tar.gz">WordNet prolog database</a>
+ Inside this archive is a file named wn_s.pl, which contains the WordNet synonyms.
+
+ <h1>Query Expansion Method</h1>
+ This method creates Lucene index storing the synonyms, which in turn can be used for query expansion.
You normally run {@link org.apache.lucene.wordnet.Syns2Index} once to build the query index/"database", and then call
{@link org.apache.lucene.wordnet.SynExpand#expand SynExpand.expand(...)} to expand a query.
@@ -31,12 +37,21 @@
<h3> Instructions </h3>
<ol>
- <li> Download the <a href="http://www.cogsci.princeton.edu/2.0/WNprolog-2.0.tar.gz">WordNet prolog database</a> , gunzip, untar etc.
<li> Invoke Syn2Index as appropriate to build a synonym index.
- It'll take 2 arguments, the path to wn_s.pl from that WordNet download, and the index name.
+ It'll take 2 arguments, the path to wn_s.pl from the WordNet download, and the index name.
<li> Update your UI so that as appropriate you call SynExpand.expand(...) to expand user queries with synonyms.
</ol>
+
+ <h1>Analysis Method</h1>
+ This method injects additional synonym tokens for tokens from a child {@link org.apache.lucene.analysis.TokenStream}.
+
+ <h3> Instructions </h3>
+ <ol>
+ <li>Create a {@link org.apache.lucene.wordnet.SynonymMap}, passing in the path to wn_s.pl
+ <li>Add a {@link org.apache.lucene.wordnet.SynonymTokenFilter} to your analyzer. Note: SynonymTokenFilter should be after LowerCaseFilter,
+ because it expects terms to already be in lowercase.
+ </ol>
</body>
</html>
\ No newline at end of file
Copied: lucene/java/trunk/contrib/wordnet/src/test/org/apache/lucene/wordnet/TestSynonymTokenFilter.java (from r830669, lucene/java/trunk/contrib/memory/src/test/org/apache/lucene/index/memory/TestSynonymTokenFilter.java)
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/wordnet/src/test/org/apache/lucene/wordnet/TestSynonymTokenFilter.java?p2=lucene/java/trunk/contrib/wordnet/src/test/org/apache/lucene/wordnet/TestSynonymTokenFilter.java&p1=lucene/java/trunk/contrib/memory/src/test/org/apache/lucene/index/memory/TestSynonymTokenFilter.java&r1=830669&r2=830699&rev=830699&view=diff
==============================================================================
--- lucene/java/trunk/contrib/memory/src/test/org/apache/lucene/index/memory/TestSynonymTokenFilter.java (original)
+++ lucene/java/trunk/contrib/wordnet/src/test/org/apache/lucene/wordnet/TestSynonymTokenFilter.java Wed Oct 28 17:49:53 2009
@@ -1,4 +1,4 @@
-package org.apache.lucene.index.memory;
+package org.apache.lucene.wordnet;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -32,7 +32,7 @@
public class TestSynonymTokenFilter extends BaseTokenStreamTestCase {
File dataDir = new File(System.getProperty("dataDir", "./bin"));
- File testFile = new File(dataDir, "org/apache/lucene/index/memory/testSynonyms.txt");
+ File testFile = new File(dataDir, "org/apache/lucene/wordnet/testSynonyms.txt");
public void testSynonyms() throws Exception {
SynonymMap map = new SynonymMap(new FileInputStream(testFile));