You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sh...@apache.org on 2014/01/29 17:23:28 UTC
svn commit: r1562499 - in /lucene/dev/branches/branch_4x: ./ lucene/
lucene/benchmark/
lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/
lucene/facet/ lucene/facet/src/java/org/apache/lucene/facet/
lucene/replicator/ lucene/replicator...
Author: shaie
Date: Wed Jan 29 16:23:28 2014
New Revision: 1562499
URL: http://svn.apache.org/r1562499
Log:
LUCENE-5387: Improve FacetConfig.build
Modified:
lucene/dev/branches/branch_4x/ (props changed)
lucene/dev/branches/branch_4x/lucene/ (props changed)
lucene/dev/branches/branch_4x/lucene/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_4x/lucene/benchmark/ (props changed)
lucene/dev/branches/branch_4x/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/AddDocTask.java
lucene/dev/branches/branch_4x/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/AddFacetedDocTask.java
lucene/dev/branches/branch_4x/lucene/facet/ (props changed)
lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/FacetField.java
lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/FacetsConfig.java
lucene/dev/branches/branch_4x/lucene/replicator/ (props changed)
lucene/dev/branches/branch_4x/lucene/replicator/src/test/org/apache/lucene/replicator/IndexAndTaxonomyReplicationClientTest.java
lucene/dev/branches/branch_4x/lucene/replicator/src/test/org/apache/lucene/replicator/IndexAndTaxonomyRevisionTest.java
Modified: lucene/dev/branches/branch_4x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/CHANGES.txt?rev=1562499&r1=1562498&r2=1562499&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/lucene/CHANGES.txt Wed Jan 29 16:23:28 2014
@@ -152,7 +152,9 @@ Bug fixes
API Changes
* LUCENE-5339: The facet module was simplified/reworked to make the
- APIs more approachable to new users. (Shai Erera, Gilad Barkai, Rob
+ APIs more approachable to new users. Note: when migrating to the new
+ API, you must pass the Document that is returned from FacetConfig.build()
+ to IndexWriter.addDocument(). (Shai Erera, Gilad Barkai, Rob
Muir, Mike McCandless)
* LUCENE-5405: Make ShingleAnalzyerWrapper.getWrappedAnalyzer() public final (gsingers)
Modified: lucene/dev/branches/branch_4x/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/AddDocTask.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/AddDocTask.java?rev=1562499&r1=1562498&r2=1562499&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/AddDocTask.java (original)
+++ lucene/dev/branches/branch_4x/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/AddDocTask.java Wed Jan 29 16:23:28 2014
@@ -17,7 +17,6 @@ package org.apache.lucene.benchmark.byTa
* limitations under the License.
*/
-import java.text.NumberFormat;
import java.util.Locale;
import org.apache.lucene.benchmark.byTask.PerfRunData;
Modified: lucene/dev/branches/branch_4x/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/AddFacetedDocTask.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/AddFacetedDocTask.java?rev=1562499&r1=1562498&r2=1562499&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/AddFacetedDocTask.java (original)
+++ lucene/dev/branches/branch_4x/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/AddFacetedDocTask.java Wed Jan 29 16:23:28 2014
@@ -22,12 +22,8 @@ import java.util.List;
import org.apache.lucene.benchmark.byTask.PerfRunData;
import org.apache.lucene.benchmark.byTask.feeds.FacetSource;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
import org.apache.lucene.facet.FacetField;
import org.apache.lucene.facet.FacetsConfig;
-import org.apache.lucene.facet.taxonomy.FacetLabel;
-import org.apache.lucene.index.IndexableField;
/**
* Add a faceted document.
@@ -81,7 +77,7 @@ public class AddFacetedDocTask extends A
List<FacetField> facets = new ArrayList<FacetField>();
getRunData().getFacetSource().getNextFacets(facets);
for(FacetField ff : facets) {
- ((Document) doc).add(ff);
+ doc.add(ff);
}
doc = config.build(getRunData().getTaxonomyWriter(), doc);
}
Modified: lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/FacetField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/FacetField.java?rev=1562499&r1=1562498&r2=1562499&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/FacetField.java (original)
+++ lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/FacetField.java Wed Jan 29 16:23:28 2014
@@ -19,12 +19,17 @@ package org.apache.lucene.facet;
import java.util.Arrays;
-import org.apache.lucene.document.Document; // javadoc
+import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
-/** Add an instance of this to your {@link Document} for
- * every facet label. */
+/**
+ * Add an instance of this to your {@link Document} for every facet label.
+ *
+ * <p>
+ * <b>NOTE:</b> you must call {@link FacetsConfig#build(Document)} before
+ * you add the document to IndexWriter.
+ */
public class FacetField extends Field {
static final FieldType TYPE = new FieldType();
static {
Modified: lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/FacetsConfig.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/FacetsConfig.java?rev=1562499&r1=1562498&r2=1562499&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/FacetsConfig.java (original)
+++ lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/FacetsConfig.java Wed Jan 29 16:23:28 2014
@@ -165,16 +165,26 @@ public class FacetsConfig {
seenDims.add(dim);
}
- /** Translates any added {@link FacetField}s into normal
- * fields for indexing; only use this version if you
- * did not add any taxonomy-based fields ({@link
- * FacetField} or {@link AssociationFacetField}) */
+ /**
+ * Translates any added {@link FacetField}s into normal fields for indexing;
+ * only use this version if you did not add any taxonomy-based fields (
+ * {@link FacetField} or {@link AssociationFacetField}).
+ *
+ * <p>
+ * <b>NOTE:</b> you should add the returned document to IndexWriter, not the
+ * input one!
+ */
public Document build(Document doc) throws IOException {
return build(null, doc);
}
- /** Translates any added {@link FacetField}s into normal
- * fields for indexing. */
+ /**
+ * Translates any added {@link FacetField}s into normal fields for indexing.
+ *
+ * <p>
+ * <b>NOTE:</b> you should add the returned document to IndexWriter, not the
+ * input one!
+ */
public Document build(TaxonomyWriter taxoWriter, Document doc) throws IOException {
// Find all FacetFields, collated by the actual field:
Map<String,List<FacetField>> byField = new HashMap<String,List<FacetField>>();
@@ -187,7 +197,7 @@ public class FacetsConfig {
Set<String> seenDims = new HashSet<String>();
- for(IndexableField field : doc.getFields()) {
+ for (IndexableField field : doc.getFields()) {
if (field.fieldType() == FacetField.TYPE) {
FacetField facetField = (FacetField) field;
FacetsConfig.DimConfig dimConfig = getDimConfig(facetField.dim);
@@ -259,35 +269,25 @@ public class FacetsConfig {
}
}
- List<Field> addedIndexedFields = new ArrayList<Field>();
- List<Field> addedStoredFields = new ArrayList<Field>();
+ Document result = new Document();
- processFacetFields(taxoWriter, byField, addedIndexedFields, addedStoredFields);
- processSSDVFacetFields(dvByField, addedIndexedFields, addedStoredFields);
- processAssocFacetFields(taxoWriter, assocByField, addedIndexedFields, addedStoredFields);
+ processFacetFields(taxoWriter, byField, result);
+ processSSDVFacetFields(dvByField, result);
+ processAssocFacetFields(taxoWriter, assocByField, result);
//System.out.println("add stored: " + addedStoredFields);
- Document result = new Document();
- for(IndexableField field : doc.getFields()) {
+ for (IndexableField field : doc.getFields()) {
IndexableFieldType ft = field.fieldType();
if (ft != FacetField.TYPE && ft != SortedSetDocValuesFacetField.TYPE && ft != AssociationFacetField.TYPE) {
result.add(field);
}
}
- for(IndexableField field : addedIndexedFields) {
- result.add(field);
- }
- for(IndexableField field : addedStoredFields) {
- result.add(field);
- }
- //System.out.println("all indexed: " + allIndexedFields);
- //System.out.println("all stored: " + allStoredFields);
return result;
}
- private void processFacetFields(TaxonomyWriter taxoWriter, Map<String,List<FacetField>> byField, List<Field> addedIndexedFields, List<Field> addedStoredFields) throws IOException {
+ private void processFacetFields(TaxonomyWriter taxoWriter, Map<String,List<FacetField>> byField, Document doc) throws IOException {
for(Map.Entry<String,List<FacetField>> ent : byField.entrySet()) {
@@ -332,18 +332,18 @@ public class FacetsConfig {
}
// Drill down:
- for(int i=1;i<=cp.length;i++) {
- addedIndexedFields.add(new StringField(indexFieldName, pathToString(cp.components, i), Field.Store.NO));
+ for (int i=1;i<=cp.length;i++) {
+ doc.add(new StringField(indexFieldName, pathToString(cp.components, i), Field.Store.NO));
}
}
// Facet counts:
// DocValues are considered stored fields:
- addedStoredFields.add(new BinaryDocValuesField(indexFieldName, dedupAndEncode(ordinals)));
+ doc.add(new BinaryDocValuesField(indexFieldName, dedupAndEncode(ordinals)));
}
}
- private void processSSDVFacetFields(Map<String,List<SortedSetDocValuesFacetField>> byField, List<Field> addedIndexedFields, List<Field> addedStoredFields) throws IOException {
+ private void processSSDVFacetFields(Map<String,List<SortedSetDocValuesFacetField>> byField, Document doc) throws IOException {
//System.out.println("process SSDV: " + byField);
for(Map.Entry<String,List<SortedSetDocValuesFacetField>> ent : byField.entrySet()) {
@@ -356,18 +356,19 @@ public class FacetsConfig {
//System.out.println("add " + fullPath);
// For facet counts:
- addedStoredFields.add(new SortedSetDocValuesField(indexFieldName, new BytesRef(fullPath)));
+ doc.add(new SortedSetDocValuesField(indexFieldName, new BytesRef(fullPath)));
// For drill-down:
- addedIndexedFields.add(new StringField(indexFieldName, fullPath, Field.Store.NO));
- addedIndexedFields.add(new StringField(indexFieldName, facetField.dim, Field.Store.NO));
+ doc.add(new StringField(indexFieldName, fullPath, Field.Store.NO));
+ doc.add(new StringField(indexFieldName, facetField.dim, Field.Store.NO));
}
}
}
- private void processAssocFacetFields(TaxonomyWriter taxoWriter, Map<String,List<AssociationFacetField>> byField,
- List<Field> addedIndexedFields, List<Field> addedStoredFields) throws IOException {
- for(Map.Entry<String,List<AssociationFacetField>> ent : byField.entrySet()) {
+ private void processAssocFacetFields(TaxonomyWriter taxoWriter,
+ Map<String,List<AssociationFacetField>> byField, Document doc)
+ throws IOException {
+ for (Map.Entry<String,List<AssociationFacetField>> ent : byField.entrySet()) {
byte[] bytes = new byte[16];
int upto = 0;
String indexFieldName = ent.getKey();
@@ -389,7 +390,7 @@ public class FacetsConfig {
System.arraycopy(field.assoc.bytes, field.assoc.offset, bytes, upto, field.assoc.length);
upto += field.assoc.length;
}
- addedStoredFields.add(new BinaryDocValuesField(indexFieldName, new BytesRef(bytes, 0, upto)));
+ doc.add(new BinaryDocValuesField(indexFieldName, new BytesRef(bytes, 0, upto)));
}
}
Modified: lucene/dev/branches/branch_4x/lucene/replicator/src/test/org/apache/lucene/replicator/IndexAndTaxonomyReplicationClientTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/replicator/src/test/org/apache/lucene/replicator/IndexAndTaxonomyReplicationClientTest.java?rev=1562499&r1=1562498&r2=1562499&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/replicator/src/test/org/apache/lucene/replicator/IndexAndTaxonomyReplicationClientTest.java (original)
+++ lucene/dev/branches/branch_4x/lucene/replicator/src/test/org/apache/lucene/replicator/IndexAndTaxonomyReplicationClientTest.java Wed Jan 29 16:23:28 2014
@@ -20,7 +20,6 @@ package org.apache.lucene.replicator;
import java.io.Closeable;
import java.io.File;
import java.io.IOException;
-import java.util.Collections;
import java.util.HashMap;
import java.util.concurrent.Callable;
import java.util.concurrent.atomic.AtomicInteger;
@@ -29,7 +28,6 @@ import org.apache.lucene.document.Docume
import org.apache.lucene.document.Document;
import org.apache.lucene.facet.DrillDownQuery;
import org.apache.lucene.facet.FacetField;
-import org.apache.lucene.facet.FacetResult;
import org.apache.lucene.facet.Facets;
import org.apache.lucene.facet.FacetsCollector;
import org.apache.lucene.facet.FacetsConfig;
Modified: lucene/dev/branches/branch_4x/lucene/replicator/src/test/org/apache/lucene/replicator/IndexAndTaxonomyRevisionTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/replicator/src/test/org/apache/lucene/replicator/IndexAndTaxonomyRevisionTest.java?rev=1562499&r1=1562498&r2=1562499&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/replicator/src/test/org/apache/lucene/replicator/IndexAndTaxonomyRevisionTest.java (original)
+++ lucene/dev/branches/branch_4x/lucene/replicator/src/test/org/apache/lucene/replicator/IndexAndTaxonomyRevisionTest.java Wed Jan 29 16:23:28 2014
@@ -19,16 +19,14 @@ package org.apache.lucene.replicator;
import java.io.IOException;
import java.io.InputStream;
-import java.util.Collections;
import java.util.List;
-import java.util.Map.Entry;
import java.util.Map;
+import java.util.Map.Entry;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Document;
import org.apache.lucene.facet.FacetField;
import org.apache.lucene.facet.FacetsConfig;
-import org.apache.lucene.facet.taxonomy.FacetLabel;
import org.apache.lucene.facet.taxonomy.TaxonomyWriter;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.IndexWriter;