You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ofbiz.apache.org by er...@apache.org on 2010/12/19 12:12:54 UTC
svn commit: r1050811 - in /ofbiz/trunk: ./ applications/content/lib/
applications/content/src/org/ofbiz/content/search/
applications/content/webapp/content/WEB-INF/actions/cms/
specialpurpose/ecommerce/webapp/ecommerce/WEB-INF/actions/content/
Author: erwan
Date: Sun Dec 19 11:12:53 2010
New Revision: 1050811
URL: http://svn.apache.org/viewvc?rev=1050811&view=rev
Log:
Migrating from lucene 2.4.1 to lucene 3.0.3
Added:
ofbiz/trunk/applications/content/lib/lucene-core-3.0.3.jar (with props)
Removed:
ofbiz/trunk/applications/content/lib/lucene-core-2.4.1.jar
Modified:
ofbiz/trunk/.classpath
ofbiz/trunk/LICENSE
ofbiz/trunk/applications/content/src/org/ofbiz/content/search/SearchServices.java
ofbiz/trunk/applications/content/src/org/ofbiz/content/search/SearchWorker.java
ofbiz/trunk/applications/content/webapp/content/WEB-INF/actions/cms/Search.groovy
ofbiz/trunk/specialpurpose/ecommerce/webapp/ecommerce/WEB-INF/actions/content/Search.groovy
Modified: ofbiz/trunk/.classpath
URL: http://svn.apache.org/viewvc/ofbiz/trunk/.classpath?rev=1050811&r1=1050810&r2=1050811&view=diff
==============================================================================
--- ofbiz/trunk/.classpath (original)
+++ ofbiz/trunk/.classpath Sun Dec 19 11:12:53 2010
@@ -2,7 +2,7 @@
<classpath>
<classpathentry kind="con" path="GROOVY_SUPPORT" exported="true"/>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
- <classpathentry kind="lib" path="applications/content/lib/lucene-core-2.4.1.jar"/>
+ <classpathentry kind="lib" path="applications/content/lib/lucene-core-3.0.3.jar"/>
<classpathentry kind="lib" path="applications/content/lib/poi-3.2-FINAL-20081019.jar"/>
<classpathentry kind="lib" path="framework/base/lib/ant-1.7.1.jar"/>
<classpathentry kind="lib" path="framework/base/lib/ant-junit-1.7.1.jar"/>
Modified: ofbiz/trunk/LICENSE
URL: http://svn.apache.org/viewvc/ofbiz/trunk/LICENSE?rev=1050811&r1=1050810&r2=1050811&view=diff
==============================================================================
--- ofbiz/trunk/LICENSE (original)
+++ ofbiz/trunk/LICENSE Sun Dec 19 11:12:53 2010
@@ -151,7 +151,7 @@ ofbiz/trunk/framework/webslinger/lib/web
ofbiz/trunk/framework/webslinger/lib/webslinger-extension-velocity-20091211-3897-7ab22baea4b6.jar
ofbiz/trunk/framework/webslinger/lib/webslinger-extension-wiki-20091211-3897-7ab22baea4b6.jar
ofbiz/trunk/framework/webslinger/lib/webslinger-launcher-20091211-3897-7ab22baea4b6.jar
-ofbiz/trunk/applications/content/lib/lucene-core-2.4.1.jar
+ofbiz/trunk/applications/content/lib/lucene-core-3.0.3.jar
ofbiz/trunk/applications/content/lib/poi-3.2-FINAL-20081019.jar
ofbiz/trunk/specialpurpose/googlecheckout/lib/checkout-sdk-0.8.8.jar
=========================================================================
Added: ofbiz/trunk/applications/content/lib/lucene-core-3.0.3.jar
URL: http://svn.apache.org/viewvc/ofbiz/trunk/applications/content/lib/lucene-core-3.0.3.jar?rev=1050811&view=auto
==============================================================================
Binary file - no diff available.
Propchange: ofbiz/trunk/applications/content/lib/lucene-core-3.0.3.jar
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Modified: ofbiz/trunk/applications/content/src/org/ofbiz/content/search/SearchServices.java
URL: http://svn.apache.org/viewvc/ofbiz/trunk/applications/content/src/org/ofbiz/content/search/SearchServices.java?rev=1050811&r1=1050810&r2=1050811&view=diff
==============================================================================
--- ofbiz/trunk/applications/content/src/org/ofbiz/content/search/SearchServices.java (original)
+++ ofbiz/trunk/applications/content/src/org/ofbiz/content/search/SearchServices.java Sun Dec 19 11:12:53 2010
@@ -18,6 +18,7 @@
*******************************************************************************/
package org.ofbiz.content.search;
+import java.util.Date;
import java.util.List;
import java.util.Map;
@@ -39,11 +40,15 @@ public class SearchServices {
public static final String module = SearchServices.class.getName();
public static Map<String, Object> indexTree(DispatchContext dctx, Map<String, ? extends Object> context) {
+ Date start = new Date();
LocalDispatcher dispatcher = dctx.getDispatcher();
Delegator delegator = dctx.getDelegator();
String siteId = (String) context.get("contentId");
String path = (String) context.get("path");
+ if (path == null) {
+ path = SearchWorker.getIndexPath(path);
+ }
Map<String, Object> envContext = FastMap.newInstance();
if (Debug.infoOn()) Debug.logInfo("in indexTree, siteId:" + siteId, module);
@@ -58,7 +63,9 @@ public class SearchServices {
Debug.logError(e, module);
return ServiceUtil.returnError("Error indexing tree: " + e.toString());
}
+ Date end = new Date();
if (Debug.infoOn()) Debug.logInfo("in indexTree, results:" + results, module);
+ if (Debug.infoOn()) Debug.logInfo("Indexing done in: " + (end.getTime()-start.getTime()) + " ms", module);
return results;
}
}
Modified: ofbiz/trunk/applications/content/src/org/ofbiz/content/search/SearchWorker.java
URL: http://svn.apache.org/viewvc/ofbiz/trunk/applications/content/src/org/ofbiz/content/search/SearchWorker.java?rev=1050811&r1=1050810&r2=1050811&view=diff
==============================================================================
--- ofbiz/trunk/applications/content/src/org/ofbiz/content/search/SearchWorker.java (original)
+++ ofbiz/trunk/applications/content/src/org/ofbiz/content/search/SearchWorker.java Sun Dec 19 11:12:53 2010
@@ -18,6 +18,7 @@
*******************************************************************************/
package org.ofbiz.content.search;
+import java.io.File;
import java.io.FileNotFoundException;
import java.util.Iterator;
import java.util.List;
@@ -43,6 +44,9 @@ import org.apache.lucene.document.Docume
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.Term;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.util.Version;
@@ -99,16 +103,16 @@ public class SearchWorker {
indexContentList(dispatcher, delegator, context, idList, path);
}
- public static void indexContentList(LocalDispatcher dispatcher, Delegator delegator, Map<String, Object> context, List<String> idList, String path) throws Exception {
- String indexAllPath = getIndexPath(path);
- if (Debug.infoOn()) Debug.logInfo("in indexContent, indexAllPath:" + indexAllPath, module);
+ public static void indexContentList(LocalDispatcher dispatcher, Delegator delegator, Map<String, Object> context,List<String> idList, String path) throws Exception {
+ Directory directory = FSDirectory.open(new File(getIndexPath(path)));
+ if (Debug.infoOn()) Debug.logInfo("in indexContent, indexAllPath: " + directory.toString(), module);
GenericValue content = null;
// Delete existing documents
Iterator<String> iter = null;
List<GenericValue> contentList = null;
IndexReader reader = null;
try {
- reader = IndexReader.open(indexAllPath);
+ reader = IndexReader.open(directory, false);
} catch (Exception e) {
// ignore
}
@@ -137,9 +141,9 @@ public class SearchWorker {
// Now create
IndexWriter writer = null;
try {
- writer = new IndexWriter(indexAllPath, new StandardAnalyzer(), false, IndexWriter.MaxFieldLength.UNLIMITED);
+ writer = new IndexWriter(directory, new StandardAnalyzer(Version.LUCENE_30), false, IndexWriter.MaxFieldLength.UNLIMITED);
} catch (Exception e) {
- writer = new IndexWriter(indexAllPath, new StandardAnalyzer(), true, IndexWriter.MaxFieldLength.UNLIMITED);
+ writer = new IndexWriter(directory, new StandardAnalyzer(Version.LUCENE_30), true, IndexWriter.MaxFieldLength.UNLIMITED);
}
Iterator<GenericValue> contentListIter = contentList.iterator();
@@ -152,9 +156,8 @@ public class SearchWorker {
}
public static void deleteContentDocument(GenericValue content, String path) throws Exception {
- String indexAllPath = null;
- indexAllPath = getIndexPath(path);
- IndexReader reader = IndexReader.open(indexAllPath);
+ Directory directory = FSDirectory.open(new File(getIndexPath(path)));
+ IndexReader reader = IndexReader.open(directory);
deleteContentDocument(content, reader);
reader.close();
}
@@ -179,14 +182,14 @@ public class SearchWorker {
}
public static void indexContent(LocalDispatcher dispatcher, Delegator delegator, Map<String, Object> context, GenericValue content, String path) throws Exception {
- String indexAllPath = getIndexPath(path);
+ Directory directory = FSDirectory.open(new File(getIndexPath(path)));
IndexWriter writer = null;
try {
- writer = new IndexWriter(indexAllPath, new StandardAnalyzer(), false, IndexWriter.MaxFieldLength.UNLIMITED);
- if (Debug.infoOn()) Debug.logInfo("Used old directory:" + indexAllPath, module);
+ writer = new IndexWriter(directory, new StandardAnalyzer(Version.LUCENE_30), false, IndexWriter.MaxFieldLength.UNLIMITED);
+ if (Debug.infoOn()) Debug.logInfo("Used old directory:" + directory.toString(), module);
} catch (FileNotFoundException e) {
- writer = new IndexWriter(indexAllPath, new StandardAnalyzer(), true, IndexWriter.MaxFieldLength.UNLIMITED);
- if (Debug.infoOn()) Debug.logInfo("Created new directory:" + indexAllPath, module);
+ writer = new IndexWriter(directory, new StandardAnalyzer(Version.LUCENE_30), true, IndexWriter.MaxFieldLength.UNLIMITED);
+ if (Debug.infoOn()) Debug.logInfo("Created new directory:" + directory.toString(), module);
}
indexContent(dispatcher, delegator, context, content, writer);
@@ -200,8 +203,7 @@ public class SearchWorker {
if (doc != null) {
writer.addDocument(doc);
Integer goodIndexCount = (Integer)context.get("goodIndexCount");
- int newCount = goodIndexCount.intValue() + 1;
- Integer newIndexCount = Integer.valueOf(newCount);
+ Integer newIndexCount = goodIndexCount + 1;
context.put("goodIndexCount", newIndexCount);
}
/*
@@ -218,12 +220,12 @@ public class SearchWorker {
}
public static void indexDataResource(Delegator delegator, Map<String, Object> context, String id, String path) throws Exception {
- String indexAllPath = getIndexPath(path);
+ Directory directory = FSDirectory.open(new File(getIndexPath(path)));
IndexWriter writer = null;
try {
- writer = new IndexWriter(indexAllPath, new StandardAnalyzer(), false, IndexWriter.MaxFieldLength.UNLIMITED);
+ writer = new IndexWriter(directory, new StandardAnalyzer(Version.LUCENE_30), false, IndexWriter.MaxFieldLength.UNLIMITED);
} catch (FileNotFoundException e) {
- writer = new IndexWriter(indexAllPath, new StandardAnalyzer(), true, IndexWriter.MaxFieldLength.UNLIMITED);
+ writer = new IndexWriter(directory, new StandardAnalyzer(Version.LUCENE_30), true, IndexWriter.MaxFieldLength.UNLIMITED);
}
indexDataResource(delegator, context, id, writer);
writer.optimize();
Modified: ofbiz/trunk/applications/content/webapp/content/WEB-INF/actions/cms/Search.groovy
URL: http://svn.apache.org/viewvc/ofbiz/trunk/applications/content/webapp/content/WEB-INF/actions/cms/Search.groovy?rev=1050811&r1=1050810&r2=1050811&view=diff
==============================================================================
--- ofbiz/trunk/applications/content/webapp/content/WEB-INF/actions/cms/Search.groovy (original)
+++ ofbiz/trunk/applications/content/webapp/content/WEB-INF/actions/cms/Search.groovy Sun Dec 19 11:12:53 2010
@@ -20,47 +20,43 @@
import org.apache.lucene.analysis.Analyzer
import org.apache.lucene.analysis.standard.StandardAnalyzer
import org.apache.lucene.document.Document
-import org.apache.lucene.search.Searcher
-import org.apache.lucene.search.IndexSearcher
import org.apache.lucene.index.IndexReader
-import org.apache.lucene.search.Query
-import org.apache.lucene.search.TermQuery
-import org.apache.lucene.search.BooleanQuery
-import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.index.Term
-import org.apache.lucene.search.Hits
import org.apache.lucene.queryParser.QueryParser
-import org.ofbiz.base.util.UtilHttp
+import org.apache.lucene.store.FSDirectory
+import org.apache.lucene.util.Version
import org.ofbiz.base.util.Debug
-import org.ofbiz.product.feature.ParametricSearch
+import org.ofbiz.base.util.UtilHttp
import org.ofbiz.content.search.SearchWorker
+import org.ofbiz.product.feature.ParametricSearch
+import org.apache.lucene.search.*
+import org.apache.lucene.store.Directory
paramMap = UtilHttp.getParameterMap(request);
-queryLine = paramMap.queryLine;
+queryLine = paramMap.queryLine.toString();
Debug.logInfo("in search, queryLine:" + queryLine, "");
siteId = paramMap.lcSiteId;
Debug.logInfo("in search, siteId:" + siteId, "");
-searchFeature1 = paramMap.SEARCH_FEAT;
-searchFeature2 = paramMap.SEARCH_FEAT2;
-searchFeature3 = paramMap.SEARCH_FEAT3;
+searchFeature1 = (String) paramMap.SEARCH_FEAT;
+searchFeature2 = (String) paramMap.SEARCH_FEAT2;
+searchFeature3 = (String) paramMap.SEARCH_FEAT3;
featureIdByType = ParametricSearch.makeFeatureIdByTypeMap(paramMap);
Debug.logInfo("in search, featureIdByType:" + featureIdByType, "");
-
combQuery = new BooleanQuery();
-indexPath = null;
+Directory directory = FSDirectory.open(new File(SearchWorker.getIndexPath(null)));
+IndexReader reader = IndexReader.open(directory, true); // only searching, so read-only=true
Searcher searcher = null;
Analyzer analyzer = null;
try {
- indexPath = SearchWorker.getIndexPath(null);
- Debug.logInfo("in search, indexPath:" + indexPath, "");
- searcher = new IndexSearcher(indexPath);
+ Debug.logInfo("in search, indexPath:" + directory.toString(), "");
+ searcher = new IndexSearcher(reader);
Debug.logInfo("in search, searcher:" + searcher, "");
- analyzer = new StandardAnalyzer();
+ analyzer = new StandardAnalyzer(Version.LUCENE_30);
} catch (java.io.FileNotFoundException e) {
request.setAttribute("errorMsgReq", "No index file exists.");
Debug.logError("in search, error:" + e.getMessage(), "");
@@ -70,14 +66,14 @@ try {
if (queryLine || siteId) {
Query query = null;
if (queryLine) {
- queryParser = new QueryParser("content", analyzer);
- query = queryParser.parse(queryLine);
+ QueryParser parser = new QueryParser(Version.LUCENE_30, "content", analyzer);
+ query = parser.parse(queryLine);
combQuery.add(query, BooleanClause.Occur.MUST);
}
Debug.logInfo("in search, combQuery(0):" + combQuery, "");
if (siteId) {
- termQuery = new TermQuery(new Term("site", siteId));
+ termQuery = new TermQuery(new Term("site", siteId.toString()));
combQuery.add(termQuery, BooleanClause.Occur.MUST);
Debug.logInfo("in search, termQuery:" + termQuery.toString(), "");
}
@@ -109,32 +105,32 @@ if (searchFeature1 || searchFeature2 ||
Debug.logInfo("in search searchFeature3, termQuery:" + termQuery.toString(), "");
}
- if (!featureIdByType.isEmpty()) {
- values = featureIdByType.values();
- values.each { val ->
- termQuery = new TermQuery(new Term("feature", val));
+ if (featureIdByType) {
+ featureIdByType.each { key, value ->
+ termQuery = new TermQuery(new Term("feature", value));
featureQuery.add(termQuery, featuresRequired);
Debug.logInfo("in search searchFeature3, termQuery:" + termQuery.toString(), "");
}
- combQuery.add(featureQuery, featuresRequired);
+ combQuery.add(featureQuery, featuresRequired);
}
}
if (searcher) {
Debug.logInfo("in search searchFeature3, combQuery:" + combQuery.toString(), "");
- Hits hits = searcher.search(combQuery);
- Debug.logInfo("in search, hits:" + hits.length(), "");
+ TopScoreDocCollector collector = TopScoreDocCollector.create(100, false); //defaulting to 100 results
+ searcher.search(combQuery, collector);
+ ScoreDoc[] hits = collector.topDocs().scoreDocs;
+ Debug.logInfo("in search, hits:" + collector.getTotalHits(), "");
contentList = [] as ArrayList;
hitSet = [:] as HashSet;
- for (int start = 0; start < hits.length(); start++) {
- doc = hits.doc(start);
- contentId = doc.contentId;
- content = delegator.findOne("Content", [contentId : contentId], true);
- if (!hitSet.contains(contentId)) {
- contentList.add(content);
- hitSet.add(contentId);
- }
+ for (int start = 0; start < collector.getTotalHits(); start++) {
+ Document doc = searcher.doc(hits[start].doc)
+ contentId = doc.get("contentId");
+ content = delegator.findOne("Content", [contentId : contentId], true);
+ if (!hitSet.contains(contentId)) {
+ contentList.add(content);
+ hitSet.add(contentId);
+ }
}
context.queryResults = contentList;
}
-
Modified: ofbiz/trunk/specialpurpose/ecommerce/webapp/ecommerce/WEB-INF/actions/content/Search.groovy
URL: http://svn.apache.org/viewvc/ofbiz/trunk/specialpurpose/ecommerce/webapp/ecommerce/WEB-INF/actions/content/Search.groovy?rev=1050811&r1=1050810&r2=1050811&view=diff
==============================================================================
--- ofbiz/trunk/specialpurpose/ecommerce/webapp/ecommerce/WEB-INF/actions/content/Search.groovy (original)
+++ ofbiz/trunk/specialpurpose/ecommerce/webapp/ecommerce/WEB-INF/actions/content/Search.groovy Sun Dec 19 11:12:53 2010
@@ -17,24 +17,23 @@
* under the License.
*/
-import org.ofbiz.base.util.UtilHttp;
-import org.ofbiz.base.util.UtilMisc;
-import org.ofbiz.base.util.Debug;
-import org.ofbiz.base.util.UtilValidate;
-import org.ofbiz.base.util.StringUtil;
-import org.ofbiz.content.search.SearchWorker;
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.standard.StandardAnalyzer;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.search.*;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.queryParser.QueryParser;
-import org.ofbiz.widget.html.HtmlFormWrapper;
-import org.ofbiz.product.feature.ParametricSearch;
+import org.apache.lucene.analysis.Analyzer
+import org.apache.lucene.analysis.standard.StandardAnalyzer
+import org.apache.lucene.document.Document
+import org.apache.lucene.index.IndexReader
+import org.apache.lucene.index.Term
+import org.apache.lucene.queryParser.QueryParser
+import org.apache.lucene.store.FSDirectory
+import org.apache.lucene.util.Version
+import org.ofbiz.base.util.Debug
+import org.ofbiz.base.util.UtilHttp
+import org.ofbiz.content.search.SearchWorker
+import org.ofbiz.product.feature.ParametricSearch
+import org.ofbiz.widget.html.HtmlFormWrapper
+import org.apache.lucene.search.*
paramMap = UtilHttp.getParameterMap(request);
-queryLine = paramMap.queryLine;
+queryLine = paramMap.queryLine.toString();
//Debug.logInfo("in search, queryLine:" + queryLine, "");
formDefFile = page.formDefFile;
@@ -53,57 +52,58 @@ featureIdByType = ParametricSearch.makeF
//Debug.logInfo("in search, featureIdByType:" + featureIdByType, "");
combQuery = new BooleanQuery();
-indexPath = null;
-searcher = null;
-analyzer = null;
+IndexReader reader = IndexReader.open(FSDirectory.open(new File(SearchWorker.getIndexPath(null))), true); // only searching, so read-only=true
+Searcher searcher = null;
+Analyzer analyzer = null;
try {
- indexPath = SearchWorker.getIndexPath(null);
- searcher = new IndexSearcher(indexPath);
- analyzer = new StandardAnalyzer();
+ searcher = new IndexSearcher(reader);
+ analyzer = new StandardAnalyzer(Version.LUCENE_30);
} catch (java.io.FileNotFoundException e) {
Debug.logError(e, "Search.groovy");
request.setAttribute("errorMsgReq", "No index file exists.");
}
-termQuery = new TermQuery(new Term("site", siteId.toLowerCase()));
+termQuery = new TermQuery(new Term("site", siteId.toString()));
combQuery.add(termQuery, BooleanClause.Occur.MUST);
//Debug.logInfo("in search, termQuery:" + termQuery.toString(), "");
//Debug.logInfo("in search, combQuery(1):" + combQuery, "");
if (queryLine && analyzer) {
Query query = null;
- queryParser = new QueryParser("content", analyzer);
- query = queryParser.parse(queryLine);
- combQuery.add(query, true, false);
+ QueryParser parser = new QueryParser(Version.LUCENE_30, "content", analyzer);
+ query = parser.parse(queryLine);
+ combQuery.add(query, BooleanClause.Occur.MUST);
}
if (featureIdByType) {
featureQuery = new BooleanQuery();
- anyOrAll = paramMap.any_or_all;
- featuresRequired = true;
- if ("any".equals(anyOrAll)) {
- featuresRequired = false;
+ featuresRequired = BooleanClause.Occur.MUST;
+ if ("any".equals(paramMap.anyOrAll)) {
+ featuresRequired = BooleanClause.Occur.SHOULD;
}
if (featureIdByType) {
featureIdByType.each { key, value ->
termQuery = new TermQuery(new Term("feature", value));
- featureQuery.add(termQuery, featuresRequired, false);
+ featureQuery.add(termQuery, featuresRequired);
//Debug.logInfo("in search searchFeature3, termQuery:" + termQuery.toString(), "");
}
}
- combQuery.add(featureQuery, featuresRequired, false);
+ combQuery.add(featureQuery, featuresRequired);
}
if (searcher) {
Debug.logInfo("in search searchFeature3, combQuery:" + combQuery.toString(), "");
- hits = searcher.search(combQuery);
- Debug.logInfo("in search, hits:" + hits.length(), "");
- contentList = [];
- hitSet = new HashSet();
- for (start = 0; start < hits.length(); start++) {
- doc = hits.doc(start);
- contentId = doc.contentId;
- content = delegator.findByPrimaryKeyCache("Content", [contentId : contentId]);
+ TopScoreDocCollector collector = TopScoreDocCollector.create(100, false); //defaulting to 100 results
+ searcher.search(combQuery, collector);
+ ScoreDoc[] hits = collector.topDocs().scoreDocs;
+ Debug.logInfo("in search, hits:" + collector.getTotalHits(), "");
+
+ contentList = [] as ArrayList;
+ hitSet = [:] as HashSet;
+ for (int start = 0; start < collector.getTotalHits(); start++) {
+ Document doc = searcher.doc(hits[start].doc)
+ contentId = doc.get("contentId");
+ content = delegator.findOne("Content", [contentId : contentId], true);
if (!hitSet.contains(contentId)) {
contentList.add(content);
hitSet.add(contentId);