You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ofbiz.apache.org by ja...@apache.org on 2013/10/15 10:28:08 UTC
svn commit: r1532241 - in /ofbiz/trunk/specialpurpose/lucene:
src/org/ofbiz/content/search/ webapp/content/WEB-INF/actions/
Author: jacopoc
Date: Tue Oct 15 08:28:08 2013
New Revision: 1532241
URL: http://svn.apache.org/r1532241
Log:
Refactoring of Lucene product indexing: separated logic that is product specific from indexing logic that is independent from products; in this way it will be easy to index other content or customize the existing one, by implementing new LuceneDocument classes.
Added:
ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/DocumentIndexer.java (contents, props changed)
- copied, changed from r1531918, ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/ProductIndexer.java
ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/LuceneDocument.java (with props)
ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/ProductDocument.java (with props)
Removed:
ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/ProductIndexer.java
Modified:
ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/SearchServices.java
ofbiz/trunk/specialpurpose/lucene/webapp/content/WEB-INF/actions/IndexProducts.groovy
Copied: ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/DocumentIndexer.java (from r1531918, ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/ProductIndexer.java)
URL: http://svn.apache.org/viewvc/ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/DocumentIndexer.java?p2=ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/DocumentIndexer.java&p1=ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/ProductIndexer.java&r1=1531918&r2=1532241&rev=1532241&view=diff
==============================================================================
--- ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/ProductIndexer.java (original)
+++ ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/DocumentIndexer.java Tue Oct 15 08:28:08 2013
@@ -20,36 +20,15 @@ package org.ofbiz.content.search;
import java.io.File;
import java.io.IOException;
-import java.sql.Timestamp;
import java.util.HashMap;
-import java.util.List;
import java.util.Map;
-import java.util.Set;
-import java.util.TreeSet;
import java.util.concurrent.LinkedBlockingQueue;
import org.ofbiz.base.util.Debug;
-import org.ofbiz.base.util.GeneralException;
-import org.ofbiz.base.util.UtilDateTime;
-import org.ofbiz.base.util.UtilMisc;
-import org.ofbiz.base.util.UtilProperties;
-import org.ofbiz.base.util.UtilValidate;
-import org.ofbiz.content.data.DataResourceWorker;
import org.ofbiz.entity.Delegator;
-import org.ofbiz.entity.GenericEntityException;
-import org.ofbiz.entity.GenericValue;
-import org.ofbiz.entity.condition.EntityCondition;
-import org.ofbiz.entity.condition.EntityOperator;
-import org.ofbiz.entity.util.EntityUtil;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
-import org.apache.lucene.document.DoubleField;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.Field.Store;
-import org.apache.lucene.document.LongField;
-import org.apache.lucene.document.StringField;
-import org.apache.lucene.document.TextField;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
@@ -58,22 +37,21 @@ import org.apache.lucene.store.Directory
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.LockObtainFailedException;
-public class ProductIndexer extends Thread {
+public class DocumentIndexer extends Thread {
- public static final String module = ProductIndexer.class.getName();
+ public static final String module = DocumentIndexer.class.getName();
- private static Map<Delegator, ProductIndexer> productIndexerMap = new HashMap<Delegator, ProductIndexer>();
- private LinkedBlockingQueue<String> productIndexQueue = new LinkedBlockingQueue<String>();
+ private static Map<String, DocumentIndexer> documentIndexerMap = new HashMap<String, DocumentIndexer>();
+ private LinkedBlockingQueue<LuceneDocument> documentIndexQueue = new LinkedBlockingQueue<LuceneDocument>();
private Delegator delegator;
private Directory indexDirectory;
- private static final String NULL_STRING = "NULL";
// TODO: Move to property file
private static final int UNCOMMITTED_DOC_LIMIT = 100;
- private ProductIndexer(Delegator delegator) {
+ private DocumentIndexer(Delegator delegator, String indexName) {
this.delegator = delegator;
try {
- this.indexDirectory = FSDirectory.open(new File(SearchWorker.getIndexPath("products")));
+ this.indexDirectory = FSDirectory.open(new File(SearchWorker.getIndexPath(indexName)));
} catch (CorruptIndexException e) {
Debug.logError("Corrupted lucene index: " + e.getMessage(), module);
} catch (LockObtainFailedException e) {
@@ -83,15 +61,16 @@ public class ProductIndexer extends Thre
}
}
- public static synchronized ProductIndexer getInstance(Delegator delegator) {
- ProductIndexer productIndexer = productIndexerMap.get(delegator);
- if (productIndexer == null) {
- productIndexer = new ProductIndexer(delegator);
- productIndexer.setName("ProductIndexer_" + delegator.getDelegatorName());
- productIndexer.start();
- productIndexerMap.put(delegator, productIndexer);
+ public static synchronized DocumentIndexer getInstance(Delegator delegator, String indexName) {
+ String documentIndexerId = delegator.getDelegatorName() + "_" + indexName;
+ DocumentIndexer documentIndexer = documentIndexerMap.get(documentIndexerId);
+ if (documentIndexer == null) {
+ documentIndexer = new DocumentIndexer(delegator, indexName);
+ documentIndexer.setName("DocumentIndexer_" + delegator.getDelegatorName() + "_" + indexName);
+ documentIndexer.start();
+ documentIndexerMap.put(documentIndexerId, documentIndexer);
}
- return productIndexer;
+ return documentIndexer;
}
@Override
@@ -99,10 +78,10 @@ public class ProductIndexer extends Thre
IndexWriter indexWriter = null;
int uncommittedDocs = 0;
while (true) {
- String productId;
+ LuceneDocument ofbizDocument;
try {
- // Execution will pause here until the queue receives a product for indexing
- productId = productIndexQueue.take();
+ // Execution will pause here until the queue receives a LuceneDocument for indexing
+ ofbizDocument = documentIndexQueue.take();
} catch (InterruptedException e) {
Debug.logError(e, module);
if (indexWriter != null) {
@@ -115,8 +94,8 @@ public class ProductIndexer extends Thre
}
break;
}
- Document productDocument = this.prepareProductDocument(productId);
- Term documentIdentifier = new Term("productId", productId);
+ Term documentIdentifier = ofbizDocument.getDocumentIdentifier();
+ Document document = ofbizDocument.prepareDocument(this.delegator);
if (indexWriter == null) {
try {
indexWriter = new IndexWriter(this.indexDirectory, new IndexWriterConfig(SearchWorker.LUCENE_VERSION, new StandardAnalyzer(SearchWorker.LUCENE_VERSION)));
@@ -133,16 +112,16 @@ public class ProductIndexer extends Thre
}
}
try {
- if (productDocument == null) {
+ if (document == null) {
indexWriter.deleteDocuments(documentIdentifier);
- if (Debug.infoOn()) Debug.logInfo("Deleted Lucene document for product: " + productId, module);
+ if (Debug.infoOn()) Debug.logInfo(getName() + ": deleted Lucene document: " + ofbizDocument, module);
} else {
- indexWriter.updateDocument(documentIdentifier, productDocument);
- if (Debug.infoOn()) Debug.logInfo("Indexed Lucene document for product: " + productId, module);
+ indexWriter.updateDocument(documentIdentifier, document);
+ if (Debug.infoOn()) Debug.logInfo(getName() + ": indexed Lucene document: " + ofbizDocument, module);
}
} catch(Exception e) {
- Debug.logError(e, "Error processing Lucene document for product: " + productId, module);
- if (productIndexQueue.peek() == null) {
+ Debug.logError(e, getName() + ": error processing Lucene document: " + ofbizDocument, module);
+ if (documentIndexQueue.peek() == null) {
try {
indexWriter.close();
indexWriter = null;
@@ -153,7 +132,7 @@ public class ProductIndexer extends Thre
continue;
}
uncommittedDocs++;
- if (uncommittedDocs == UNCOMMITTED_DOC_LIMIT || productIndexQueue.peek() == null) {
+ if (uncommittedDocs == UNCOMMITTED_DOC_LIMIT || documentIndexQueue.peek() == null) {
// limit reached or queue empty, time to commit
try {
indexWriter.commit();
@@ -162,7 +141,7 @@ public class ProductIndexer extends Thre
}
uncommittedDocs = 0;
}
- if (productIndexQueue.peek() == null) {
+ if (documentIndexQueue.peek() == null) {
try {
indexWriter.close();
indexWriter = null;
@@ -173,375 +152,7 @@ public class ProductIndexer extends Thre
}
}
- public boolean queue(String productId) {
- return productIndexQueue.add(productId);
- }
-
- private Document prepareProductDocument(String productId) {
- try {
- GenericValue product = delegator.findOne("Product", true, "productId", productId);
- if (product == null) {
- // Return a null document (we will remove the document from the index)
- return null;
- } else {
- if ("Y".equals(product.getString("isVariant")) && "true".equals(UtilProperties.getPropertyValue("prodsearch", "index.ignore.variants"))) {
- return null;
- }
- Document doc = new Document();
- Timestamp nextReIndex = null;
-
- // Product Fields
- doc.add(new StringField("productId", productId, Store.YES));
- this.addTextFieldByWeight(doc, "productName", product.getString("productName"), "index.weight.Product.productName", 0, false, "fullText");
- this.addTextFieldByWeight(doc, "internalName", product.getString("internalName"), "index.weight.Product.internalName", 0, false, "fullText");
- this.addTextFieldByWeight(doc, "brandName", product.getString("brandName"), "index.weight.Product.brandName", 0, false, "fullText");
- this.addTextFieldByWeight(doc, "description", product.getString("description"), "index.weight.Product.description", 0, false, "fullText");
- this.addTextFieldByWeight(doc, "longDescription", product.getString("longDescription"), "index.weight.Product.longDescription", 0, false, "fullText");
- //doc.add(new StringField("introductionDate", checkValue(product.getString("introductionDate")), Store.NO));
- doc.add(new LongField("introductionDate", quantizeTimestampToDays(product.getTimestamp("introductionDate")), Store.NO));
- nextReIndex = this.checkSetNextReIndex(product.getTimestamp("introductionDate"), nextReIndex);
- doc.add(new LongField("salesDiscontinuationDate", quantizeTimestampToDays(product.getTimestamp("salesDiscontinuationDate")), Store.NO));
- nextReIndex = this.checkSetNextReIndex(product.getTimestamp("salesDiscontinuationDate"), nextReIndex);
- doc.add(new StringField("isVariant", product.get("isVariant") != null && product.getBoolean("isVariant") ? "true" : "false", Store.NO));
-
- // ProductFeature Fields, check that at least one of the fields is set to be indexed
- if (!"0".equals(UtilProperties.getPropertyValue("prodsearch", "index.weight.ProductFeatureAndAppl.description", "0")) ||
- !"0".equals(UtilProperties.getPropertyValue("prodsearch", "index.weight.ProductFeatureAndAppl.abbrev", "0")) ||
- !"0".equals(UtilProperties.getPropertyValue("prodsearch", "index.weight.ProductFeatureAndAppl.idCode", "0"))) {
-
- List<GenericValue> productFeatureAndAppls = delegator.findByAnd("ProductFeatureAndAppl", UtilMisc.toMap("productId", productId), null, false);
- productFeatureAndAppls = this.filterByThruDate(productFeatureAndAppls);
-
- for (GenericValue productFeatureAndAppl: productFeatureAndAppls) {
- Timestamp fromDate = productFeatureAndAppl.getTimestamp("fromDate");
- Timestamp thruDate = productFeatureAndAppl.getTimestamp("thruDate");
- if (fromDate != null && fromDate.after(UtilDateTime.nowTimestamp())) {
- // fromDate is after now, update reindex date but don't index the feature
- nextReIndex = this.checkSetNextReIndex(fromDate, nextReIndex);
- continue;
- } else if (thruDate != null) {
- nextReIndex = this.checkSetNextReIndex(thruDate, nextReIndex);
- }
- doc.add(new StringField("productFeatureId", productFeatureAndAppl.getString("productFeatureId"), Store.NO));
- doc.add(new StringField("productFeatureCategoryId", productFeatureAndAppl.getString("productFeatureCategoryId"), Store.NO));
- doc.add(new StringField("productFeatureTypeId", productFeatureAndAppl.getString("productFeatureTypeId"), Store.NO));
- this.addTextFieldByWeight(doc, "featureDescription", productFeatureAndAppl.getString("description"), "index.weight.ProductFeatureAndAppl.description", 0, false, "fullText");
- this.addTextFieldByWeight(doc, "featureAbbreviation", productFeatureAndAppl.getString("abbrev"), "index.weight.ProductFeatureAndAppl.abbrev", 0, false, "fullText");
- this.addTextFieldByWeight(doc, "featureCode", productFeatureAndAppl.getString("idCode"), "index.weight.ProductFeatureAndAppl.idCode", 0, false, "fullText");
- // Get the ProductFeatureGroupIds
- List<GenericValue> productFeatureGroupAppls = delegator.findByAnd("ProductFeatureGroupAppl", UtilMisc.toMap("productFeatureId", productFeatureAndAppl.get("productFeatureId")), null, false);
- productFeatureGroupAppls = this.filterByThruDate(productFeatureGroupAppls);
- for (GenericValue productFeatureGroupAppl : productFeatureGroupAppls) {
- fromDate = productFeatureGroupAppl.getTimestamp("fromDate");
- thruDate = productFeatureGroupAppl.getTimestamp("thruDate");
- if (fromDate != null && fromDate.after(UtilDateTime.nowTimestamp())) {
- // fromDate is after now, update reindex date but don't index the feature
- nextReIndex = this.checkSetNextReIndex(fromDate, nextReIndex);
- continue;
- } else if (thruDate != null) {
- nextReIndex = this.checkSetNextReIndex(thruDate, nextReIndex);
- }
- doc.add(new StringField("productFeatureGroupId", productFeatureGroupAppl.getString("productFeatureGroupId"), Store.NO));
- }
- }
- }
-
- // ProductAttribute Fields
- if (!"0".equals(UtilProperties.getPropertyValue("prodsearch", "index.weight.ProductAttribute.attrName", "0")) ||
- !"0".equals(UtilProperties.getPropertyValue("prodsearch", "index.weight.ProductAttribute.attrValue", "0"))) {
-
- List<GenericValue> productAttributes = delegator.findByAnd("ProductAttribute", UtilMisc.toMap("productId", productId), null, false);
- for (GenericValue productAttribute: productAttributes) {
- this.addTextFieldByWeight(doc, "attributeName", productAttribute.getString("attrName"), "index.weight.ProductAttribute.attrName", 0, false, "fullText");
- this.addTextFieldByWeight(doc, "attributeValue", productAttribute.getString("attrValue"), "index.weight.ProductAttribute.attrValue", 0, false, "fullText");
- }
- }
-
- // GoodIdentification
- if (!"0".equals(UtilProperties.getPropertyValue("prodsearch", "index.weight.GoodIdentification.idValue", "0"))) {
- List<GenericValue> goodIdentifications = delegator.findByAnd("GoodIdentification", UtilMisc.toMap("productId", productId), null, false);
- for (GenericValue goodIdentification: goodIdentifications) {
- String goodIdentificationTypeId = goodIdentification.getString("goodIdentificationTypeId");
- String idValue = goodIdentification.getString("idValue");
- doc.add(new StringField("goodIdentificationTypeId", goodIdentificationTypeId, Store.NO));
- doc.add(new StringField("goodIdentificationIdValue", idValue, Store.NO));
- doc.add(new StringField(goodIdentificationTypeId + "_GoodIdentification", idValue, Store.NO));
- this.addTextFieldByWeight(doc, "identificationValue", idValue, "index.weight.GoodIdentification.idValue", 0, false, "fullText");
- }
- }
-
- // Virtual ProductIds
- if ("Y".equals(product.getString("isVirtual"))) {
- if (!"0".equals(UtilProperties.getPropertyValue("prodsearch", "index.weight.Variant.Product.productId", "0"))) {
- List<GenericValue> variantProductAssocs = delegator.findByAnd("ProductAssoc", UtilMisc.toMap("productId", productId, "productAssocTypeId", "PRODUCT_VARIANT"), null, false);
- variantProductAssocs = this.filterByThruDate(variantProductAssocs);
- for (GenericValue variantProductAssoc: variantProductAssocs) {
- Timestamp fromDate = variantProductAssoc.getTimestamp("fromDate");
- Timestamp thruDate = variantProductAssoc.getTimestamp("thruDate");
- if (fromDate != null && fromDate.after(UtilDateTime.nowTimestamp())) {
- // fromDate is after now, update reindex date but don't index the feature
- nextReIndex = this.checkSetNextReIndex(fromDate, nextReIndex);
- continue;
- } else if (thruDate != null) {
- nextReIndex = this.checkSetNextReIndex(thruDate, nextReIndex);
- }
- this.addTextFieldByWeight(doc, "variantProductId", variantProductAssoc.getString("productIdTo"), "index.weight.Variant.Product.productId", 0, false, "fullText");
- }
- }
- }
-
- // Index product content
- String productContentTypes = UtilProperties.getPropertyValue("prodsearch", "index.include.ProductContentTypes");
- for (String productContentTypeId: productContentTypes.split(",")) {
- int weight = 1;
- try {
- // this is defaulting to a weight of 1 because you specified you wanted to index this type
- weight = Integer.parseInt(UtilProperties.getPropertyValue("prodsearch", "index.weight.ProductContent." + productContentTypeId, "1"));
- } catch (Exception e) {
- Debug.logWarning("Could not parse weight number: " + e.toString(), module);
- }
-
- List<GenericValue> productContentAndInfos = delegator.findByAnd("ProductContentAndInfo", UtilMisc.toMap("productId", productId, "productContentTypeId", productContentTypeId), null, false);
- productContentAndInfos = this.filterByThruDate(productContentAndInfos);
- for (GenericValue productContentAndInfo: productContentAndInfos) {
- Timestamp fromDate = productContentAndInfo.getTimestamp("fromDate");
- Timestamp thruDate = productContentAndInfo.getTimestamp("thruDate");
- if (fromDate != null && fromDate.after(UtilDateTime.nowTimestamp())) {
- // fromDate is after now, update reindex date but don't index the feature
- nextReIndex = this.checkSetNextReIndex(fromDate, nextReIndex);
- continue;
- } else if (thruDate != null) {
- nextReIndex = this.checkSetNextReIndex(thruDate, nextReIndex);
- }
- try {
- Map<String, Object> drContext = UtilMisc.<String, Object>toMap("product", product);
- String contentText = DataResourceWorker.renderDataResourceAsText(delegator, productContentAndInfo.getString("dataResourceId"), drContext, null, null, false);
- this.addTextFieldByWeight(doc, "content", contentText, null, weight, false, "fullText");
- } catch (IOException e1) {
- Debug.logError(e1, "Error getting content text to index", module);
- } catch (GeneralException e1) {
- Debug.logError(e1, "Error getting content text to index", module);
- }
-
- // TODO: Not indexing alternate locales, needs special handling
- /*
- List<GenericValue> alternateViews = productContentAndInfo.getRelated("ContentAssocDataResourceViewTo", UtilMisc.toMap("caContentAssocTypeId", "ALTERNATE_LOCALE"), UtilMisc.toList("-caFromDate"));
- alternateViews = EntityUtil.filterByDate(alternateViews, UtilDateTime.nowTimestamp(), "caFromDate", "caThruDate", true);
- for (GenericValue thisView: alternateViews) {
- }
- */
- }
- }
-
- // Index the product's directProductCategoryIds (direct parents), productCategoryIds (all ancestors) and prodCatalogIds
- this.populateCategoryData(doc, product);
-
- // Index ProductPrices, uses dynamic fields in the format ${productPriceTypeId}_${productPricePurposeId}_${currencyUomId}_${productStoreGroupId}_price
- List<GenericValue> productPrices = product.getRelated("ProductPrice", null, null, false);
- productPrices = this.filterByThruDate(productPrices);
- for (GenericValue productPrice : productPrices) {
- Timestamp fromDate = productPrice.getTimestamp("fromDate");
- Timestamp thruDate = productPrice.getTimestamp("thruDate");
- if (fromDate != null && fromDate.after(UtilDateTime.nowTimestamp())) {
- // fromDate is after now, update reindex date but don't index the feature
- nextReIndex = this.checkSetNextReIndex(fromDate, nextReIndex);
- continue;
- } else if (thruDate != null) {
- nextReIndex = this.checkSetNextReIndex(thruDate, nextReIndex);
- }
- StringBuilder fieldNameSb = new StringBuilder();
- fieldNameSb.append(productPrice.getString("productPriceTypeId"));
- fieldNameSb.append('_');
- fieldNameSb.append(productPrice.getString("productPricePurposeId"));
- fieldNameSb.append('_');
- fieldNameSb.append(productPrice.getString("currencyUomId"));
- fieldNameSb.append('_');
- fieldNameSb.append(productPrice.getString("productStoreGroupId"));
- fieldNameSb.append("_price");
- doc.add(new DoubleField(fieldNameSb.toString(), productPrice.getDouble("price"), Store.NO));
- }
-
- // Index ProductSuppliers
- List<GenericValue> supplierProducts = product.getRelated("SupplierProduct", null, null, false);
- supplierProducts = this.filterByThruDate(supplierProducts, "availableThruDate");
- Set<String> supplierPartyIds = new TreeSet<String>();
- for (GenericValue supplierProduct : supplierProducts) {
- Timestamp fromDate = supplierProduct.getTimestamp("availableFromDate");
- Timestamp thruDate = supplierProduct.getTimestamp("availableThruDate");
- if (fromDate != null && fromDate.after(UtilDateTime.nowTimestamp())) {
- // fromDate is after now, update reindex date but don't index the feature
- nextReIndex = this.checkSetNextReIndex(fromDate, nextReIndex);
- continue;
- } else if (thruDate != null) {
- nextReIndex = this.checkSetNextReIndex(thruDate, nextReIndex);
- }
- supplierPartyIds.add(supplierProduct.getString("partyId"));
- }
- for (String supplierPartyId : supplierPartyIds) {
- doc.add(new StringField("supplierPartyId", supplierPartyId, Store.NO));
- }
-
- // TODO: Add the nextReIndex timestamp to the document for when the product should be automatically re-indexed outside of any ECAs
- // based on the next known from/thru date whose passing will cause a change to the document. Need to build a scheduled service to look for these.
- return doc;
- }
- } catch (GenericEntityException e) {
- Debug.logError(e, module);
- }
- return null;
- }
-
- // An attempt to boost/weight values in a similar manner to what OFBiz product search does.
- private void addTextFieldByWeight(Document doc, String fieldName, String value, String property, int defaultWeight, boolean store, String fullTextFieldName) {
- if (fieldName == null) return;
-
- float weight = 0;
- if (property != null) {
- try {
- weight = Float.parseFloat(UtilProperties.getPropertyValue("prodsearch", property, "0"));
- } catch (Exception e) {
- Debug.logWarning("Could not parse weight number: " + e.toString(), module);
- }
- } else if (defaultWeight > 0) {
- weight = defaultWeight;
- }
- if (weight == 0 && !store) {
- return;
- }
- Field field = new TextField(fieldName, checkValue(value), (store? Store.YES: Store.NO));
- if (weight > 0 && weight != 1) {
- field.setBoost(weight);
- }
- doc.add(field);
- if (fullTextFieldName != null) {
- doc.add(new TextField(fullTextFieldName, checkValue(value), Store.NO));
- }
- }
-
- private String checkValue(String value) {
- if (UtilValidate.isEmpty(value)) {
- return NULL_STRING;
- }
- return value;
- }
-
- private Timestamp checkSetNextReIndex(Timestamp nextValue, Timestamp currentValue) {
- // nextValue is null, stick with what we've got
- if (nextValue == null) return currentValue;
- // currentValue is null so use nextValue
- if (currentValue == null) return nextValue;
- // currentValue is after nextValue so use nextValue
- if (currentValue.after(nextValue)) return nextValue;
- // stick with current value
- return currentValue;
- }
-
- private static final EntityCondition THRU_DATE_ONLY_CONDITION = EntityCondition.makeCondition(
- EntityCondition.makeCondition("thruDate", EntityOperator.EQUALS, null),
- EntityOperator.OR,
- EntityCondition.makeCondition("thruDate", EntityOperator.GREATER_THAN, UtilDateTime.nowTimestamp())
- );
-
- private List<GenericValue> filterByThruDate(List<GenericValue> values) {
- return EntityUtil.filterByCondition(values, THRU_DATE_ONLY_CONDITION);
- }
-
- private List<GenericValue> filterByThruDate(List<GenericValue> values, String thruDateName) {
- return EntityUtil.filterByCondition(values, EntityCondition.makeCondition(
- EntityCondition.makeCondition(thruDateName, EntityOperator.EQUALS, null),
- EntityOperator.OR,
- EntityCondition.makeCondition(thruDateName, EntityOperator.GREATER_THAN, UtilDateTime.nowTimestamp())
- ));
- }
-
- private Timestamp populateCategoryData(Document doc, GenericValue product) throws GenericEntityException {
- Timestamp nextReIndex = null;
- Set<String> indexedCategoryIds = new TreeSet<String>();
- List<GenericValue> productCategoryMembers = product.getRelated("ProductCategoryMember", null, null, false);
- productCategoryMembers = this.filterByThruDate(productCategoryMembers);
-
- for (GenericValue productCategoryMember: productCategoryMembers) {
- String productCategoryId = productCategoryMember.getString("productCategoryId");
- doc.add(new StringField("productCategoryId", productCategoryId, Store.NO));
- doc.add(new StringField("directProductCategoryId", productCategoryId, Store.NO));
- indexedCategoryIds.add(productCategoryId);
- Timestamp fromDate = productCategoryMember.getTimestamp("fromDate");
- Timestamp thruDate = productCategoryMember.getTimestamp("thruDate");
- if (fromDate != null && fromDate.after(UtilDateTime.nowTimestamp())) {
- // fromDate is after now, update reindex date but don't index the feature
- nextReIndex = this.checkSetNextReIndex(fromDate, nextReIndex);
- continue;
- } else if (thruDate != null) {
- nextReIndex = this.checkSetNextReIndex(thruDate, nextReIndex);
- }
- nextReIndex = this.checkSetNextReIndex(
- this.getParentCategories(doc, productCategoryMember.getRelatedOne("ProductCategory", false), indexedCategoryIds),
- nextReIndex);
- }
- return nextReIndex;
- }
-
- private Timestamp getParentCategories(Document doc, GenericValue productCategory, Set<String> indexedCategoryIds) throws GenericEntityException {
- return this.getParentCategories(doc, productCategory, indexedCategoryIds, new TreeSet<String>());
- }
-
- private Timestamp getParentCategories(Document doc, GenericValue productCategory, Set<String> indexedCategoryIds, Set<String> indexedCatalogIds) throws GenericEntityException {
- Timestamp nextReIndex = null;
- nextReIndex = this.getCategoryCatalogs(doc, productCategory, indexedCatalogIds);
- List<GenericValue> productCategoryRollups = productCategory.getRelated("CurrentProductCategoryRollup", null, null, false);
- productCategoryRollups = this.filterByThruDate(productCategoryRollups);
- for (GenericValue productCategoryRollup : productCategoryRollups) {
- Timestamp fromDate = productCategoryRollup.getTimestamp("fromDate");
- Timestamp thruDate = productCategoryRollup.getTimestamp("thruDate");
- if (fromDate != null && fromDate.after(UtilDateTime.nowTimestamp())) {
- // fromDate is after now, update reindex date but don't index now
- nextReIndex = this.checkSetNextReIndex(fromDate, nextReIndex);
- continue;
- } else if (thruDate != null) {
- nextReIndex = this.checkSetNextReIndex(thruDate, nextReIndex);
- }
- // Skip if we've done this category already
- if (!indexedCategoryIds.add(productCategoryRollup.getString("parentProductCategoryId"))) {
- continue;
- }
- GenericValue parentProductCategory = productCategoryRollup.getRelatedOne("ParentProductCategory", false);
- doc.add(new StringField("productCategoryId", parentProductCategory.getString("productCategoryId"), Store.NO));
- nextReIndex = this.checkSetNextReIndex(
- this.getParentCategories(doc, parentProductCategory, indexedCategoryIds),
- nextReIndex
- );
- }
- return nextReIndex;
- }
-
- private Timestamp getCategoryCatalogs(Document doc, GenericValue productCategory, Set<String> indexedCatalogIds) throws GenericEntityException {
- Timestamp nextReIndex = null;
- List<GenericValue> prodCatalogCategories = productCategory.getRelated("ProdCatalogCategory", null, null, false);
- prodCatalogCategories = this.filterByThruDate(prodCatalogCategories);
- for (GenericValue prodCatalogCategory : prodCatalogCategories) {
- Timestamp fromDate = prodCatalogCategory.getTimestamp("fromDate");
- Timestamp thruDate = prodCatalogCategory.getTimestamp("thruDate");
- if (fromDate != null && fromDate.after(UtilDateTime.nowTimestamp())) {
- // fromDate is after now, update reindex date but don't index now
- nextReIndex = this.checkSetNextReIndex(fromDate, nextReIndex);
- continue;
- } else if (thruDate != null) {
- nextReIndex = this.checkSetNextReIndex(thruDate, nextReIndex);
- }
- // Skip if we've done this catalog already
- if (!indexedCatalogIds.add(prodCatalogCategory.getString("prodCatalogId"))) {
- continue;
- }
- doc.add(new StringField("prodCatalogId", prodCatalogCategory.getString("prodCatalogId"), Store.NO));
- }
- return nextReIndex;
- }
-
- private long quantizeTimestampToDays(Timestamp date) {
- long quantizedDate = 0;
- if (date != null) {
- quantizedDate = date.getTime()/24/3600;
- }
- return quantizedDate;
+ public boolean queue(LuceneDocument document) {
+ return documentIndexQueue.add(document);
}
}
Propchange: ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/DocumentIndexer.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/DocumentIndexer.java
------------------------------------------------------------------------------
svn:keywords = Date Rev Author URL Id
Propchange: ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/DocumentIndexer.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/LuceneDocument.java
URL: http://svn.apache.org/viewvc/ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/LuceneDocument.java?rev=1532241&view=auto
==============================================================================
--- ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/LuceneDocument.java (added)
+++ ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/LuceneDocument.java Tue Oct 15 08:28:08 2013
@@ -0,0 +1,31 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *******************************************************************************/
+package org.ofbiz.content.search;
+
+import org.ofbiz.entity.Delegator;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.index.Term;
+
+public interface LuceneDocument {
+
+ public Term getDocumentIdentifier();
+
+ public Document prepareDocument(Delegator delegator);
+}
Propchange: ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/LuceneDocument.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/LuceneDocument.java
------------------------------------------------------------------------------
svn:keywords = Date Rev Author URL Id
Propchange: ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/LuceneDocument.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/ProductDocument.java
URL: http://svn.apache.org/viewvc/ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/ProductDocument.java?rev=1532241&view=auto
==============================================================================
--- ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/ProductDocument.java (added)
+++ ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/ProductDocument.java Tue Oct 15 08:28:08 2013
@@ -0,0 +1,437 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *******************************************************************************/
+package org.ofbiz.content.search;
+
+import org.ofbiz.base.util.Debug;
+import org.ofbiz.base.util.GeneralException;
+import org.ofbiz.base.util.UtilDateTime;
+import org.ofbiz.base.util.UtilMisc;
+import org.ofbiz.base.util.UtilProperties;
+import org.ofbiz.base.util.UtilValidate;
+import org.ofbiz.content.data.DataResourceWorker;
+import org.ofbiz.entity.Delegator;
+import org.ofbiz.entity.GenericEntityException;
+import org.ofbiz.entity.GenericValue;
+import org.ofbiz.entity.condition.EntityCondition;
+import org.ofbiz.entity.condition.EntityOperator;
+import org.ofbiz.entity.util.EntityUtil;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.DoubleField;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.LongField;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.Term;
+
+import java.io.IOException;
+import java.sql.Timestamp;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+
+public class ProductDocument implements LuceneDocument {
+ private static final String module = ProductDocument.class.getName();
+ private static final String NULL_STRING = "NULL";
+ private final Term documentIdentifier;
+
+ public ProductDocument(String productId) {
+ this.documentIdentifier = new Term("productId", productId);
+ }
+
+ @Override
+ public String toString() {
+ return getDocumentIdentifier().toString();
+ }
+
+ public Term getDocumentIdentifier() {
+ return documentIdentifier;
+ }
+
+ public Document prepareDocument(Delegator delegator) {
+ String productId = getDocumentIdentifier().text();
+ try {
+ GenericValue product = delegator.findOne("Product", true, "productId", productId);
+ if (product == null) {
+ // Return a null document (we will remove the document from the index)
+ return null;
+ } else {
+ if ("Y".equals(product.getString("isVariant")) && "true".equals(UtilProperties.getPropertyValue("prodsearch", "index.ignore.variants"))) {
+ return null;
+ }
+ Document doc = new Document();
+ Timestamp nextReIndex = null;
+
+ // Product Fields
+ doc.add(new StringField("productId", productId, Field.Store.YES));
+ this.addTextFieldByWeight(doc, "productName", product.getString("productName"), "index.weight.Product.productName", 0, false, "fullText");
+ this.addTextFieldByWeight(doc, "internalName", product.getString("internalName"), "index.weight.Product.internalName", 0, false, "fullText");
+ this.addTextFieldByWeight(doc, "brandName", product.getString("brandName"), "index.weight.Product.brandName", 0, false, "fullText");
+ this.addTextFieldByWeight(doc, "description", product.getString("description"), "index.weight.Product.description", 0, false, "fullText");
+ this.addTextFieldByWeight(doc, "longDescription", product.getString("longDescription"), "index.weight.Product.longDescription", 0, false, "fullText");
+ //doc.add(new StringField("introductionDate", checkValue(product.getString("introductionDate")), Store.NO));
+ doc.add(new LongField("introductionDate", quantizeTimestampToDays(product.getTimestamp("introductionDate")), Field.Store.NO));
+ nextReIndex = this.checkSetNextReIndex(product.getTimestamp("introductionDate"), nextReIndex);
+ doc.add(new LongField("salesDiscontinuationDate", quantizeTimestampToDays(product.getTimestamp("salesDiscontinuationDate")), Field.Store.NO));
+ nextReIndex = this.checkSetNextReIndex(product.getTimestamp("salesDiscontinuationDate"), nextReIndex);
+ doc.add(new StringField("isVariant", product.get("isVariant") != null && product.getBoolean("isVariant") ? "true" : "false", Field.Store.NO));
+
+ // ProductFeature Fields, check that at least one of the fields is set to be indexed
+ if (!"0".equals(UtilProperties.getPropertyValue("prodsearch", "index.weight.ProductFeatureAndAppl.description", "0")) ||
+ !"0".equals(UtilProperties.getPropertyValue("prodsearch", "index.weight.ProductFeatureAndAppl.abbrev", "0")) ||
+ !"0".equals(UtilProperties.getPropertyValue("prodsearch", "index.weight.ProductFeatureAndAppl.idCode", "0"))) {
+
+ List<GenericValue> productFeatureAndAppls = delegator.findByAnd("ProductFeatureAndAppl", UtilMisc.toMap("productId", productId), null, false);
+ productFeatureAndAppls = this.filterByThruDate(productFeatureAndAppls);
+
+ for (GenericValue productFeatureAndAppl: productFeatureAndAppls) {
+ Timestamp fromDate = productFeatureAndAppl.getTimestamp("fromDate");
+ Timestamp thruDate = productFeatureAndAppl.getTimestamp("thruDate");
+ if (fromDate != null && fromDate.after(UtilDateTime.nowTimestamp())) {
+ // fromDate is after now, update reindex date but don't index the feature
+ nextReIndex = this.checkSetNextReIndex(fromDate, nextReIndex);
+ continue;
+ } else if (thruDate != null) {
+ nextReIndex = this.checkSetNextReIndex(thruDate, nextReIndex);
+ }
+ doc.add(new StringField("productFeatureId", productFeatureAndAppl.getString("productFeatureId"), Field.Store.NO));
+ doc.add(new StringField("productFeatureCategoryId", productFeatureAndAppl.getString("productFeatureCategoryId"), Field.Store.NO));
+ doc.add(new StringField("productFeatureTypeId", productFeatureAndAppl.getString("productFeatureTypeId"), Field.Store.NO));
+ this.addTextFieldByWeight(doc, "featureDescription", productFeatureAndAppl.getString("description"), "index.weight.ProductFeatureAndAppl.description", 0, false, "fullText");
+ this.addTextFieldByWeight(doc, "featureAbbreviation", productFeatureAndAppl.getString("abbrev"), "index.weight.ProductFeatureAndAppl.abbrev", 0, false, "fullText");
+ this.addTextFieldByWeight(doc, "featureCode", productFeatureAndAppl.getString("idCode"), "index.weight.ProductFeatureAndAppl.idCode", 0, false, "fullText");
+ // Get the ProductFeatureGroupIds
+ List<GenericValue> productFeatureGroupAppls = delegator.findByAnd("ProductFeatureGroupAppl", UtilMisc.toMap("productFeatureId", productFeatureAndAppl.get("productFeatureId")), null, false);
+ productFeatureGroupAppls = this.filterByThruDate(productFeatureGroupAppls);
+ for (GenericValue productFeatureGroupAppl : productFeatureGroupAppls) {
+ fromDate = productFeatureGroupAppl.getTimestamp("fromDate");
+ thruDate = productFeatureGroupAppl.getTimestamp("thruDate");
+ if (fromDate != null && fromDate.after(UtilDateTime.nowTimestamp())) {
+ // fromDate is after now, update reindex date but don't index the feature
+ nextReIndex = this.checkSetNextReIndex(fromDate, nextReIndex);
+ continue;
+ } else if (thruDate != null) {
+ nextReIndex = this.checkSetNextReIndex(thruDate, nextReIndex);
+ }
+ doc.add(new StringField("productFeatureGroupId", productFeatureGroupAppl.getString("productFeatureGroupId"), Field.Store.NO));
+ }
+ }
+ }
+
+ // ProductAttribute Fields
+ if (!"0".equals(UtilProperties.getPropertyValue("prodsearch", "index.weight.ProductAttribute.attrName", "0")) ||
+ !"0".equals(UtilProperties.getPropertyValue("prodsearch", "index.weight.ProductAttribute.attrValue", "0"))) {
+
+ List<GenericValue> productAttributes = delegator.findByAnd("ProductAttribute", UtilMisc.toMap("productId", productId), null, false);
+ for (GenericValue productAttribute: productAttributes) {
+ this.addTextFieldByWeight(doc, "attributeName", productAttribute.getString("attrName"), "index.weight.ProductAttribute.attrName", 0, false, "fullText");
+ this.addTextFieldByWeight(doc, "attributeValue", productAttribute.getString("attrValue"), "index.weight.ProductAttribute.attrValue", 0, false, "fullText");
+ }
+ }
+
+ // GoodIdentification
+ if (!"0".equals(UtilProperties.getPropertyValue("prodsearch", "index.weight.GoodIdentification.idValue", "0"))) {
+ List<GenericValue> goodIdentifications = delegator.findByAnd("GoodIdentification", UtilMisc.toMap("productId", productId), null, false);
+ for (GenericValue goodIdentification: goodIdentifications) {
+ String goodIdentificationTypeId = goodIdentification.getString("goodIdentificationTypeId");
+ String idValue = goodIdentification.getString("idValue");
+ doc.add(new StringField("goodIdentificationTypeId", goodIdentificationTypeId, Field.Store.NO));
+ doc.add(new StringField("goodIdentificationIdValue", idValue, Field.Store.NO));
+ doc.add(new StringField(goodIdentificationTypeId + "_GoodIdentification", idValue, Field.Store.NO));
+ this.addTextFieldByWeight(doc, "identificationValue", idValue, "index.weight.GoodIdentification.idValue", 0, false, "fullText");
+ }
+ }
+
+ // Virtual ProductIds
+ if ("Y".equals(product.getString("isVirtual"))) {
+ if (!"0".equals(UtilProperties.getPropertyValue("prodsearch", "index.weight.Variant.Product.productId", "0"))) {
+ List<GenericValue> variantProductAssocs = delegator.findByAnd("ProductAssoc", UtilMisc.toMap("productId", productId, "productAssocTypeId", "PRODUCT_VARIANT"), null, false);
+ variantProductAssocs = this.filterByThruDate(variantProductAssocs);
+ for (GenericValue variantProductAssoc: variantProductAssocs) {
+ Timestamp fromDate = variantProductAssoc.getTimestamp("fromDate");
+ Timestamp thruDate = variantProductAssoc.getTimestamp("thruDate");
+ if (fromDate != null && fromDate.after(UtilDateTime.nowTimestamp())) {
+ // fromDate is after now, update reindex date but don't index the feature
+ nextReIndex = this.checkSetNextReIndex(fromDate, nextReIndex);
+ continue;
+ } else if (thruDate != null) {
+ nextReIndex = this.checkSetNextReIndex(thruDate, nextReIndex);
+ }
+ this.addTextFieldByWeight(doc, "variantProductId", variantProductAssoc.getString("productIdTo"), "index.weight.Variant.Product.productId", 0, false, "fullText");
+ }
+ }
+ }
+
+ // Index product content
+ String productContentTypes = UtilProperties.getPropertyValue("prodsearch", "index.include.ProductContentTypes");
+ for (String productContentTypeId: productContentTypes.split(",")) {
+ int weight = 1;
+ try {
+ // this is defaulting to a weight of 1 because you specified you wanted to index this type
+ weight = Integer.parseInt(UtilProperties.getPropertyValue("prodsearch", "index.weight.ProductContent." + productContentTypeId, "1"));
+ } catch (Exception e) {
+ Debug.logWarning("Could not parse weight number: " + e.toString(), module);
+ }
+
+ List<GenericValue> productContentAndInfos = delegator.findByAnd("ProductContentAndInfo", UtilMisc.toMap("productId", productId, "productContentTypeId", productContentTypeId), null, false);
+ productContentAndInfos = this.filterByThruDate(productContentAndInfos);
+ for (GenericValue productContentAndInfo: productContentAndInfos) {
+ Timestamp fromDate = productContentAndInfo.getTimestamp("fromDate");
+ Timestamp thruDate = productContentAndInfo.getTimestamp("thruDate");
+ if (fromDate != null && fromDate.after(UtilDateTime.nowTimestamp())) {
+ // fromDate is after now, update reindex date but don't index the feature
+ nextReIndex = this.checkSetNextReIndex(fromDate, nextReIndex);
+ continue;
+ } else if (thruDate != null) {
+ nextReIndex = this.checkSetNextReIndex(thruDate, nextReIndex);
+ }
+ try {
+ Map<String, Object> drContext = UtilMisc.<String, Object>toMap("product", product);
+ String contentText = DataResourceWorker.renderDataResourceAsText(delegator, productContentAndInfo.getString("dataResourceId"), drContext, null, null, false);
+ this.addTextFieldByWeight(doc, "content", contentText, null, weight, false, "fullText");
+ } catch (IOException e1) {
+ Debug.logError(e1, "Error getting content text to index", module);
+ } catch (GeneralException e1) {
+ Debug.logError(e1, "Error getting content text to index", module);
+ }
+
+ // TODO: Not indexing alternate locales, needs special handling
+ /*
+ List<GenericValue> alternateViews = productContentAndInfo.getRelated("ContentAssocDataResourceViewTo", UtilMisc.toMap("caContentAssocTypeId", "ALTERNATE_LOCALE"), UtilMisc.toList("-caFromDate"));
+ alternateViews = EntityUtil.filterByDate(alternateViews, UtilDateTime.nowTimestamp(), "caFromDate", "caThruDate", true);
+ for (GenericValue thisView: alternateViews) {
+ }
+ */
+ }
+ }
+
+ // Index the product's directProductCategoryIds (direct parents), productCategoryIds (all ancestors) and prodCatalogIds
+ this.populateCategoryData(doc, product);
+
+ // Index ProductPrices, uses dynamic fields in the format ${productPriceTypeId}_${productPricePurposeId}_${currencyUomId}_${productStoreGroupId}_price
+ List<GenericValue> productPrices = product.getRelated("ProductPrice", null, null, false);
+ productPrices = this.filterByThruDate(productPrices);
+ for (GenericValue productPrice : productPrices) {
+ Timestamp fromDate = productPrice.getTimestamp("fromDate");
+ Timestamp thruDate = productPrice.getTimestamp("thruDate");
+ if (fromDate != null && fromDate.after(UtilDateTime.nowTimestamp())) {
+ // fromDate is after now, update reindex date but don't index the feature
+ nextReIndex = this.checkSetNextReIndex(fromDate, nextReIndex);
+ continue;
+ } else if (thruDate != null) {
+ nextReIndex = this.checkSetNextReIndex(thruDate, nextReIndex);
+ }
+ StringBuilder fieldNameSb = new StringBuilder();
+ fieldNameSb.append(productPrice.getString("productPriceTypeId"));
+ fieldNameSb.append('_');
+ fieldNameSb.append(productPrice.getString("productPricePurposeId"));
+ fieldNameSb.append('_');
+ fieldNameSb.append(productPrice.getString("currencyUomId"));
+ fieldNameSb.append('_');
+ fieldNameSb.append(productPrice.getString("productStoreGroupId"));
+ fieldNameSb.append("_price");
+ doc.add(new DoubleField(fieldNameSb.toString(), productPrice.getDouble("price"), Field.Store.NO));
+ }
+
+ // Index ProductSuppliers
+ List<GenericValue> supplierProducts = product.getRelated("SupplierProduct", null, null, false);
+ supplierProducts = this.filterByThruDate(supplierProducts, "availableThruDate");
+ Set<String> supplierPartyIds = new TreeSet<String>();
+ for (GenericValue supplierProduct : supplierProducts) {
+ Timestamp fromDate = supplierProduct.getTimestamp("availableFromDate");
+ Timestamp thruDate = supplierProduct.getTimestamp("availableThruDate");
+ if (fromDate != null && fromDate.after(UtilDateTime.nowTimestamp())) {
+ // fromDate is after now, update reindex date but don't index the feature
+ nextReIndex = this.checkSetNextReIndex(fromDate, nextReIndex);
+ continue;
+ } else if (thruDate != null) {
+ nextReIndex = this.checkSetNextReIndex(thruDate, nextReIndex);
+ }
+ supplierPartyIds.add(supplierProduct.getString("partyId"));
+ }
+ for (String supplierPartyId : supplierPartyIds) {
+ doc.add(new StringField("supplierPartyId", supplierPartyId, Field.Store.NO));
+ }
+
+ // TODO: Add the nextReIndex timestamp to the document for when the product should be automatically re-indexed outside of any ECAs
+ // based on the next known from/thru date whose passing will cause a change to the document. Need to build a scheduled service to look for these.
+ return doc;
+ }
+ } catch (GenericEntityException e) {
+ Debug.logError(e, module);
+ }
+ return null;
+ }
+
+ // An attempt to boost/weight values in a similar manner to what OFBiz product search does.
+ private void addTextFieldByWeight(Document doc, String fieldName, String value, String property, int defaultWeight, boolean store, String fullTextFieldName) {
+ if (fieldName == null) return;
+
+ float weight = 0;
+ if (property != null) {
+ try {
+ weight = Float.parseFloat(UtilProperties.getPropertyValue("prodsearch", property, "0"));
+ } catch (Exception e) {
+ Debug.logWarning("Could not parse weight number: " + e.toString(), module);
+ }
+ } else if (defaultWeight > 0) {
+ weight = defaultWeight;
+ }
+ if (weight == 0 && !store) {
+ return;
+ }
+ Field field = new TextField(fieldName, checkValue(value), (store? Field.Store.YES: Field.Store.NO));
+ if (weight > 0 && weight != 1) {
+ field.setBoost(weight);
+ }
+ doc.add(field);
+ if (fullTextFieldName != null) {
+ doc.add(new TextField(fullTextFieldName, checkValue(value), Field.Store.NO));
+ }
+ }
+
+ private String checkValue(String value) {
+ if (UtilValidate.isEmpty(value)) {
+ return NULL_STRING;
+ }
+ return value;
+ }
+
+ private Timestamp checkSetNextReIndex(Timestamp nextValue, Timestamp currentValue) {
+ // nextValue is null, stick with what we've got
+ if (nextValue == null) return currentValue;
+ // currentValue is null so use nextValue
+ if (currentValue == null) return nextValue;
+ // currentValue is after nextValue so use nextValue
+ if (currentValue.after(nextValue)) return nextValue;
+ // stick with current value
+ return currentValue;
+ }
+
+ private static final EntityCondition THRU_DATE_ONLY_CONDITION = EntityCondition.makeCondition(
+ EntityCondition.makeCondition("thruDate", EntityOperator.EQUALS, null),
+ EntityOperator.OR,
+ EntityCondition.makeCondition("thruDate", EntityOperator.GREATER_THAN, UtilDateTime.nowTimestamp())
+ );
+
+ private List<GenericValue> filterByThruDate(List<GenericValue> values) {
+ return EntityUtil.filterByCondition(values, THRU_DATE_ONLY_CONDITION);
+ }
+
+ private List<GenericValue> filterByThruDate(List<GenericValue> values, String thruDateName) {
+ return EntityUtil.filterByCondition(values, EntityCondition.makeCondition(
+ EntityCondition.makeCondition(thruDateName, EntityOperator.EQUALS, null),
+ EntityOperator.OR,
+ EntityCondition.makeCondition(thruDateName, EntityOperator.GREATER_THAN, UtilDateTime.nowTimestamp())
+ ));
+ }
+
+ private Timestamp populateCategoryData(Document doc, GenericValue product) throws GenericEntityException {
+ Timestamp nextReIndex = null;
+ Set<String> indexedCategoryIds = new TreeSet<String>();
+ List<GenericValue> productCategoryMembers = product.getRelated("ProductCategoryMember", null, null, false);
+ productCategoryMembers = this.filterByThruDate(productCategoryMembers);
+
+ for (GenericValue productCategoryMember: productCategoryMembers) {
+ String productCategoryId = productCategoryMember.getString("productCategoryId");
+ doc.add(new StringField("productCategoryId", productCategoryId, Field.Store.NO));
+ doc.add(new StringField("directProductCategoryId", productCategoryId, Field.Store.NO));
+ indexedCategoryIds.add(productCategoryId);
+ Timestamp fromDate = productCategoryMember.getTimestamp("fromDate");
+ Timestamp thruDate = productCategoryMember.getTimestamp("thruDate");
+ if (fromDate != null && fromDate.after(UtilDateTime.nowTimestamp())) {
+ // fromDate is after now, update reindex date but don't index the feature
+ nextReIndex = this.checkSetNextReIndex(fromDate, nextReIndex);
+ continue;
+ } else if (thruDate != null) {
+ nextReIndex = this.checkSetNextReIndex(thruDate, nextReIndex);
+ }
+ nextReIndex = this.checkSetNextReIndex(
+ this.getParentCategories(doc, productCategoryMember.getRelatedOne("ProductCategory", false), indexedCategoryIds),
+ nextReIndex);
+ }
+ return nextReIndex;
+ }
+
+ private Timestamp getParentCategories(Document doc, GenericValue productCategory, Set<String> indexedCategoryIds) throws GenericEntityException {
+ return this.getParentCategories(doc, productCategory, indexedCategoryIds, new TreeSet<String>());
+ }
+
+ private Timestamp getParentCategories(Document doc, GenericValue productCategory, Set<String> indexedCategoryIds, Set<String> indexedCatalogIds) throws GenericEntityException {
+ Timestamp nextReIndex = null;
+ nextReIndex = this.getCategoryCatalogs(doc, productCategory, indexedCatalogIds);
+ List<GenericValue> productCategoryRollups = productCategory.getRelated("CurrentProductCategoryRollup", null, null, false);
+ productCategoryRollups = this.filterByThruDate(productCategoryRollups);
+ for (GenericValue productCategoryRollup : productCategoryRollups) {
+ Timestamp fromDate = productCategoryRollup.getTimestamp("fromDate");
+ Timestamp thruDate = productCategoryRollup.getTimestamp("thruDate");
+ if (fromDate != null && fromDate.after(UtilDateTime.nowTimestamp())) {
+ // fromDate is after now, update reindex date but don't index now
+ nextReIndex = this.checkSetNextReIndex(fromDate, nextReIndex);
+ continue;
+ } else if (thruDate != null) {
+ nextReIndex = this.checkSetNextReIndex(thruDate, nextReIndex);
+ }
+ // Skip if we've done this category already
+ if (!indexedCategoryIds.add(productCategoryRollup.getString("parentProductCategoryId"))) {
+ continue;
+ }
+ GenericValue parentProductCategory = productCategoryRollup.getRelatedOne("ParentProductCategory", false);
+ doc.add(new StringField("productCategoryId", parentProductCategory.getString("productCategoryId"), Field.Store.NO));
+ nextReIndex = this.checkSetNextReIndex(
+ this.getParentCategories(doc, parentProductCategory, indexedCategoryIds),
+ nextReIndex
+ );
+ }
+ return nextReIndex;
+ }
+
+ private Timestamp getCategoryCatalogs(Document doc, GenericValue productCategory, Set<String> indexedCatalogIds) throws GenericEntityException {
+ Timestamp nextReIndex = null;
+ List<GenericValue> prodCatalogCategories = productCategory.getRelated("ProdCatalogCategory", null, null, false);
+ prodCatalogCategories = this.filterByThruDate(prodCatalogCategories);
+ for (GenericValue prodCatalogCategory : prodCatalogCategories) {
+ Timestamp fromDate = prodCatalogCategory.getTimestamp("fromDate");
+ Timestamp thruDate = prodCatalogCategory.getTimestamp("thruDate");
+ if (fromDate != null && fromDate.after(UtilDateTime.nowTimestamp())) {
+ // fromDate is after now, update reindex date but don't index now
+ nextReIndex = this.checkSetNextReIndex(fromDate, nextReIndex);
+ continue;
+ } else if (thruDate != null) {
+ nextReIndex = this.checkSetNextReIndex(thruDate, nextReIndex);
+ }
+ // Skip if we've done this catalog already
+ if (!indexedCatalogIds.add(prodCatalogCategory.getString("prodCatalogId"))) {
+ continue;
+ }
+ doc.add(new StringField("prodCatalogId", prodCatalogCategory.getString("prodCatalogId"), Field.Store.NO));
+ }
+ return nextReIndex;
+ }
+
+ private long quantizeTimestampToDays(Timestamp date) {
+ long quantizedDate = 0;
+ if (date != null) {
+ quantizedDate = date.getTime()/24/3600;
+ }
+ return quantizedDate;
+ }
+
+}
Propchange: ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/ProductDocument.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/ProductDocument.java
------------------------------------------------------------------------------
svn:keywords = Date Rev Author URL Id
Propchange: ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/ProductDocument.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Modified: ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/SearchServices.java
URL: http://svn.apache.org/viewvc/ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/SearchServices.java?rev=1532241&r1=1532240&r2=1532241&view=diff
==============================================================================
--- ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/SearchServices.java (original)
+++ ofbiz/trunk/specialpurpose/lucene/src/org/ofbiz/content/search/SearchServices.java Tue Oct 15 08:28:08 2013
@@ -78,8 +78,8 @@ public class SearchServices {
public static Map<String, Object> indexProduct(DispatchContext dctx, Map<String, ? extends Object> context) {
Delegator delegator = dctx.getDelegator();
String productId = (String) context.get("productId");
- ProductIndexer indexer = ProductIndexer.getInstance(delegator);
- indexer.queue(productId);
+ DocumentIndexer indexer = DocumentIndexer.getInstance(delegator, "products");
+ indexer.queue(new ProductDocument(productId));
return ServiceUtil.returnSuccess();
}
Modified: ofbiz/trunk/specialpurpose/lucene/webapp/content/WEB-INF/actions/IndexProducts.groovy
URL: http://svn.apache.org/viewvc/ofbiz/trunk/specialpurpose/lucene/webapp/content/WEB-INF/actions/IndexProducts.groovy?rev=1532241&r1=1532240&r2=1532241&view=diff
==============================================================================
--- ofbiz/trunk/specialpurpose/lucene/webapp/content/WEB-INF/actions/IndexProducts.groovy (original)
+++ ofbiz/trunk/specialpurpose/lucene/webapp/content/WEB-INF/actions/IndexProducts.groovy Tue Oct 15 08:28:08 2013
@@ -17,11 +17,13 @@
* under the License.
*/
-import org.ofbiz.content.search.ProductIndexer
+
+import org.ofbiz.content.search.ProductDocument
+import org.ofbiz.content.search.DocumentIndexer
import org.ofbiz.entity.transaction.TransactionUtil
import org.ofbiz.entity.util.EntityListIterator
-ProductIndexer pi = ProductIndexer.getInstance(delegator)
+DocumentIndexer pi = DocumentIndexer.getInstance(delegator, 'products')
if (pi) {
productsCounter = 0
beganTransaction = TransactionUtil.begin()
@@ -29,7 +31,7 @@ if (pi) {
try {
products = delegator.find('Product', null, null, new TreeSet(['productId']), null, null)
while (product = products.next()) {
- pi.queue(product.productId)
+ pi.queue(new ProductDocument(product.productId))
productsCounter++
}
} catch(Exception e) {