You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ofbiz.apache.org by ja...@apache.org on 2013/10/15 12:13:11 UTC
svn commit: r1532273 - in /ofbiz/trunk/specialpurpose/lucene: servicedef/
src/org/ofbiz/content/search/
Author: jacopoc
Date: Tue Oct 15 10:13:11 2013
New Revision: 1532273
URL: http://svn.apache.org/r1532273
Log:
Refactored and simplified the old code to index Content record: it now leverages the new LuceneDocument interface and index handling facilities.
Modified:
ofbiz/trunk/specialpurpose/lucene/servicedef/services.xml
ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/ContentDocument.java
ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/SearchEvents.java
ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/SearchServices.java
ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/SearchWorker.java
Modified: ofbiz/trunk/specialpurpose/lucene/servicedef/services.xml
URL: http://svn.apache.org/viewvc/ofbiz/trunk/specialpurpose/lucene/servicedef/services.xml?rev=1532273&r1=1532272&r2=1532273&view=diff
==============================================================================
--- ofbiz/trunk/specialpurpose/lucene/servicedef/services.xml (original)
+++ ofbiz/trunk/specialpurpose/lucene/servicedef/services.xml Tue Oct 15 10:13:11 2013
@@ -27,8 +27,6 @@ under the License.
location="org.ofbiz.content.search.SearchServices" invoke="indexTree">
<description>Index content under publish point</description>
<attribute mode="IN" name="contentId" optional="false" type="String"/>
- <attribute mode="OUT" name="badIndexList" optional="true" type="List"/>
- <attribute mode="OUT" name="goodIndexCount" optional="true" type="Integer"/>
</service>
<service name="indexProduct" engine="java" location="org.ofbiz.content.search.SearchServices" invoke="indexProduct">
Modified: ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/ContentDocument.java
URL: http://svn.apache.org/viewvc/ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/ContentDocument.java?rev=1532273&r1=1532272&r2=1532273&view=diff
==============================================================================
--- ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/ContentDocument.java (original)
+++ ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/ContentDocument.java Tue Oct 15 10:13:11 2013
@@ -26,6 +26,7 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
+import org.apache.lucene.index.Term;
import org.ofbiz.base.util.Debug;
import org.ofbiz.base.util.GeneralException;
import org.ofbiz.base.util.StringUtil;
@@ -48,12 +49,29 @@ import org.apache.lucene.document.TextFi
* ContentDocument Class
*/
-public class ContentDocument {
+public class ContentDocument implements LuceneDocument {
- public static final String module = ContentDocument.class.getName();
+ private static final String module = ContentDocument.class.getName();
+ private final Term documentIdentifier;
+ private final LocalDispatcher dispatcher;
+ private final GenericValue content;
+
+ public ContentDocument(GenericValue content, LocalDispatcher dispatcher) {
+ this.content = content;
+ this.dispatcher = dispatcher;
+ this.documentIdentifier = new Term("contentId", content.getString("contentId"));
+ }
- public static Document Document(GenericValue content, Map<String, Object> context, LocalDispatcher dispatcher) throws InterruptedException {
+ @Override
+ public String toString() {
+ return getDocumentIdentifier().toString();
+ }
+ public Term getDocumentIdentifier() {
+ return documentIdentifier;
+ }
+
+ public Document prepareDocument(Delegator delegator) {
Document doc;
// make a new, empty document
doc = new Document();
@@ -82,26 +100,23 @@ public class ContentDocument {
Field field = new StringField("site", ancestorString, Store.NO);
doc.add(field);
}
- boolean retVal = indexDataResource(content, doc, context, dispatcher);
- if (!retVal)
+ boolean retVal = indexDataResource(doc);
+ if (!retVal) {
doc = null;
+ }
return doc;
}
- public static boolean indexDataResource(GenericValue content, Document doc, Map<String, Object> context, LocalDispatcher dispatcher) {
+ private boolean indexDataResource(Document doc) {
String contentId = content.getString("contentId");
GenericValue dataResource;
try {
dataResource = content.getRelatedOne("DataResource", true);
} catch (GenericEntityException e) {
Debug.logError(e, module);
- List<String> badIndexList = UtilGenerics.checkList(context.get("badIndexList"));
- badIndexList.add(contentId + " - " + e.getMessage());
return false;
}
if (dataResource == null) {
- List<String> badIndexList = UtilGenerics.checkList(context.get("badIndexList"));
- badIndexList.add(contentId + " - dataResource is null.");
return false;
}
String mimeTypeId = dataResource.getString("mimeTypeId");
@@ -115,16 +130,12 @@ public class ContentDocument {
}
String text;
try {
- text = ContentWorker.renderContentAsText(dispatcher, content.getDelegator(), contentId, context, locale, mimeTypeId, true);
+ text = ContentWorker.renderContentAsText(dispatcher, content.getDelegator(), contentId, null, locale, mimeTypeId, true);
} catch (GeneralException e) {
Debug.logError(e, module);
- List<String> badIndexList = UtilGenerics.checkList(context.get("badIndexList"));
- badIndexList.add(contentId + " - " + e.getMessage());
return false;
} catch (IOException e2) {
Debug.logError(e2, module);
- List<String> badIndexList = UtilGenerics.checkList(context.get("badIndexList"));
- badIndexList.add(contentId + " - " + e2.getMessage());
return false;
}
if (UtilValidate.isNotEmpty(text)) {
@@ -136,8 +147,6 @@ public class ContentDocument {
featureDataResourceList = content.getRelated("ProductFeatureDataResource", null, null, true);
} catch (GenericEntityException e) {
Debug.logError(e, module);
- List<String> badIndexList = UtilGenerics.checkList(context.get("badIndexList"));
- badIndexList.add(contentId + " - " + e.getMessage());
return false;
}
List<String> featureList = new ArrayList<String>();
Modified: ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/SearchEvents.java
URL: http://svn.apache.org/viewvc/ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/SearchEvents.java?rev=1532273&r1=1532272&r2=1532273&view=diff
==============================================================================
--- ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/SearchEvents.java (original)
+++ ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/SearchEvents.java Tue Oct 15 10:13:11 2013
@@ -21,15 +21,12 @@ package org.ofbiz.content.search;
import java.lang.Object;
import java.lang.String;
import java.util.HashMap;
-import java.util.List;
import java.util.Map;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.servlet.http.HttpSession;
import org.ofbiz.base.util.Debug;
-import org.ofbiz.base.util.StringUtil;
-import org.ofbiz.base.util.UtilGenerics;
import org.ofbiz.base.util.UtilHttp;
import org.ofbiz.base.util.UtilValidate;
import org.ofbiz.entity.GenericValue;
@@ -64,16 +61,7 @@ public class SearchEvents {
return "error";
}
String errMsg = ServiceUtil.getErrorMessage(result);
- if (Debug.infoOn()) Debug.logInfo("errMsg:" + errMsg, module);
- if (Debug.infoOn()) Debug.logInfo("result:" + result, module);
if (UtilValidate.isEmpty(errMsg)) {
- List<String> badIndexList = UtilGenerics.checkList(result.get("badIndexList"));
- if (Debug.infoOn()) Debug.logInfo("badIndexList:" + badIndexList, module);
- String badIndexMsg = StringUtil.join(badIndexList, "\n") + badIndexList.size() + " entities not indexed";
- Integer goodIndexCount = (Integer)result.get("goodIndexCount");
- String goodIndexMsg = goodIndexCount + " entities indexed.";
- if (Debug.infoOn()) Debug.logInfo("goodIndexCount:" + goodIndexCount, module);
- ServiceUtil.setMessages(request, badIndexMsg, goodIndexMsg, null);
return "success";
} else {
ServiceUtil.setMessages(request, errMsg, null, null);
Modified: ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/SearchServices.java
URL: http://svn.apache.org/viewvc/ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/SearchServices.java?rev=1532273&r1=1532272&r2=1532273&view=diff
==============================================================================
--- ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/SearchServices.java (original)
+++ ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/SearchServices.java Tue Oct 15 10:13:11 2013
@@ -20,9 +20,6 @@ package org.ofbiz.content.search;
import java.lang.Object;
import java.lang.String;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
@@ -49,30 +46,18 @@ public class SearchServices {
public static final String resource = "ContentUiLabels";
public static Map<String, Object> indexTree(DispatchContext dctx, Map<String, ? extends Object> context) {
- Date start = new Date();
LocalDispatcher dispatcher = dctx.getDispatcher();
Delegator delegator = dctx.getDelegator();
String siteId = (String) context.get("contentId");
Locale locale = (Locale) context.get("locale");
- Map<String, Object> envContext = new HashMap<String, Object>();
-
- if (Debug.infoOn()) Debug.logInfo("in indexTree, siteId:" + siteId, module);
- List<String> badIndexList = new ArrayList<String>();
- envContext.put("badIndexList", badIndexList);
- envContext.put("goodIndexCount", Integer.valueOf(0));
-
- Map<String, Object> results;
try {
- results = SearchWorker.indexTree(dispatcher, delegator, siteId, envContext);
+ SearchWorker.indexTree(dispatcher, delegator, siteId);
} catch (Exception e) {
Debug.logError(e, module);
return ServiceUtil.returnError(UtilProperties.getMessage(resource,
"ContentIndexingTreeError", UtilMisc.toMap("errorString", e.toString()), locale));
}
- Date end = new Date();
- if (Debug.infoOn()) Debug.logInfo("in indexTree, results:" + results, module);
- if (Debug.infoOn()) Debug.logInfo("Indexing done in: " + (end.getTime()-start.getTime()) + " ms", module);
- return results;
+ return ServiceUtil.returnSuccess();
}
public static Map<String, Object> indexProduct(DispatchContext dctx, Map<String, ? extends Object> context) {
Modified: ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/SearchWorker.java
URL: http://svn.apache.org/viewvc/ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/SearchWorker.java?rev=1532273&r1=1532272&r2=1532273&view=diff
==============================================================================
--- ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/SearchWorker.java (original)
+++ ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/SearchWorker.java Tue Oct 15 10:13:11 2013
@@ -18,16 +18,12 @@
*******************************************************************************/
package org.ofbiz.content.search;
-import java.io.File;
-import java.io.IOException;
import java.lang.String;
import java.util.ArrayList;
import java.util.List;
-import java.util.Map;
import org.ofbiz.base.util.Debug;
import org.ofbiz.base.util.UtilDateTime;
-import org.ofbiz.base.util.UtilGenerics;
import org.ofbiz.base.util.UtilMisc;
import org.ofbiz.base.util.UtilProperties;
import org.ofbiz.base.util.UtilValidate;
@@ -37,17 +33,6 @@ import org.ofbiz.entity.GenericEntityExc
import org.ofbiz.entity.GenericValue;
import org.ofbiz.service.LocalDispatcher;
-import org.apache.lucene.analysis.standard.StandardAnalyzer;
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.index.CorruptIndexException;
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.FSDirectory;
-import org.apache.lucene.store.LockObtainFailedException;
import org.apache.lucene.util.Version;
/**
@@ -59,9 +44,8 @@ public class SearchWorker {
public static final Version LUCENE_VERSION = Version.LUCENE_45;
- public static Map<String, Object> indexTree(LocalDispatcher dispatcher, Delegator delegator, String siteId, Map<String, Object> context) throws Exception {
+ public static void indexTree(LocalDispatcher dispatcher, Delegator delegator, String siteId) throws Exception {
GenericValue content = delegator.makeValue("Content", UtilMisc.toMap("contentId", siteId));
- if (Debug.infoOn()) Debug.logInfo("in indexTree, siteId:" + siteId + " content:" + content, module);
List<GenericValue> siteList = ContentWorker.getAssociatedContent(content, "To", UtilMisc.toList("SUBSITE", "PUBLISH_LINK", "SUB_CONTENT"), null, UtilDateTime.nowTimestamp().toString(), null);
if (siteList != null) {
@@ -74,19 +58,11 @@ public class SearchWorker {
for (GenericValue subContent : subContentList) {
contentIdList.add(subContent.getString("contentId"));
}
- indexContentList(dispatcher, delegator, context, contentIdList);
- indexTree(dispatcher, delegator, siteContentId, context);
- } else {
- List<String> badIndexList = UtilGenerics.checkList(context.get("badIndexList"));
- badIndexList.add(siteContentId + " had no sub-entities.");
+ indexContentList(dispatcher, delegator, contentIdList);
+ indexTree(dispatcher, delegator, siteContentId);
}
}
- } else {
- List<String> badIndexList = UtilGenerics.checkList(context.get("badIndexList"));
- badIndexList.add(siteId + " had no sub-entities.");
}
-
- return UtilMisc.toMap("badIndexList", context.get("badIndexList"), "goodIndexCount", context.get("goodIndexCount"));
}
public static String getIndexPath(String path) {
@@ -94,40 +70,13 @@ public class SearchWorker {
return (UtilValidate.isNotEmpty(path)? basePath + "/" + path: basePath);
}
- private static IndexWriter getDefaultIndexWriter(Directory directory) {
- IndexWriter writer = null;
- long savedWriteLockTimeout = IndexWriterConfig.getDefaultWriteLockTimeout();
- Analyzer analyzer = new StandardAnalyzer(LUCENE_VERSION);
- IndexWriterConfig conf = new IndexWriterConfig(LUCENE_VERSION, analyzer);
- IndexWriterConfig.setDefaultWriteLockTimeout(2000);
- try {
- writer = new IndexWriter(directory, conf);
- } catch (CorruptIndexException e) {
- Debug.logError("Corrupted lucene index: " + e.getMessage(), module);
- } catch (LockObtainFailedException e) {
- Debug.logError("Could not obtain Lock on lucene index " + e.getMessage(), module);
- } catch (IOException e) {
- Debug.logError(e.getMessage(), module);
- } finally {
- IndexWriterConfig.setDefaultWriteLockTimeout(savedWriteLockTimeout);
- }
- return writer;
- }
-
- public static void indexContentList(LocalDispatcher dispatcher, Delegator delegator, Map<String, Object> context,List<String> idList) throws Exception {
- Directory directory = FSDirectory.open(new File(getIndexPath("content")));
- if (Debug.infoOn()) Debug.logInfo("in indexContentList, indexAllPath: " + directory.toString(), module);
- // Delete existing documents
- IndexWriter writer = getDefaultIndexWriter(directory);
+ public static void indexContentList(LocalDispatcher dispatcher, Delegator delegator, List<String> idList) throws Exception {
+ DocumentIndexer indexer = DocumentIndexer.getInstance(delegator, "content");
List<GenericValue> contentList = new ArrayList<GenericValue>();
for (String id : idList) {
- if (Debug.infoOn()) Debug.logInfo("in indexContentList, id:" + id, module);
try {
GenericValue content = delegator.findOne("Content", UtilMisc .toMap("contentId", id), true);
if (content != null) {
- if (writer != null) {
- deleteContentDocuments(content, writer);
- }
contentList.add(content);
}
} catch (GenericEntityException e) {
@@ -135,50 +84,8 @@ public class SearchWorker {
return;
}
}
- for (GenericValue gv : contentList) {
- indexContent(dispatcher, context, gv, writer);
+ for (GenericValue content : contentList) {
+ indexer.queue(new ContentDocument(content, dispatcher));
}
- try {
- writer.forceMerge(1);
- } catch (NullPointerException e) {
- Debug.logError(e, module);
- }
- writer.close();
}
-
- private static void deleteContentDocuments(GenericValue content, IndexWriter writer) throws Exception {
- String contentId = content.getString("contentId");
- Term term = new Term("contentId", contentId);
- deleteDocumentsByTerm(term, writer);
- String dataResourceId = content.getString("dataResourceId");
- if (dataResourceId != null) {
- term = new Term("dataResourceId", dataResourceId);
- deleteDocumentsByTerm(term, writer);
- }
- }
-
- private static void deleteDocumentsByTerm(Term term, IndexWriter writer) throws Exception {
- DirectoryReader reader = DirectoryReader.open(writer, false);
- int qtyBefore = reader.docFreq(term);
-
- //deletes documents, all the rest is for logging
- writer.deleteDocuments(term);
-
- int qtyAfter = reader.docFreq(term);
- reader.close();
-
- if (Debug.infoOn()) Debug.logInfo("For term " + term.toString() + ", documents deleted: " + qtyBefore + ", remaining: " + qtyAfter, module);
- }
-
- private static void indexContent(LocalDispatcher dispatcher, Map<String, Object> context, GenericValue content, IndexWriter writer) throws Exception {
- Document doc = ContentDocument.Document(content, context, dispatcher);
-
- if (doc != null) {
- writer.addDocument(doc);
- Integer goodIndexCount = (Integer)context.get("goodIndexCount");
- Integer newIndexCount = goodIndexCount + 1;
- context.put("goodIndexCount", newIndexCount);
- }
- }
-
}