You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2014/11/27 12:34:48 UTC
svn commit: r1642110 [5/12] - in /lucene/dev/branches/lucene6005/lucene:
analysis/uima/src/test/org/apache/lucene/analysis/uima/
backward-codecs/src/test/org/apache/lucene/index/
benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/ benchmark/sr...
Modified: lucene/dev/branches/lucene6005/lucene/demo/src/java/org/apache/lucene/demo/facet/SimpleSortedSetFacetsExample.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/demo/src/java/org/apache/lucene/demo/facet/SimpleSortedSetFacetsExample.java?rev=1642110&r1=1642109&r2=1642110&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/demo/src/java/org/apache/lucene/demo/facet/SimpleSortedSetFacetsExample.java (original)
+++ lucene/dev/branches/lucene6005/lucene/demo/src/java/org/apache/lucene/demo/facet/SimpleSortedSetFacetsExample.java Thu Nov 27 11:34:43 2014
@@ -22,6 +22,7 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
+import org.apache.lucene.document.Document2;
import org.apache.lucene.document.Document;
import org.apache.lucene.facet.DrillDownQuery;
import org.apache.lucene.facet.FacetResult;
@@ -34,8 +35,8 @@ import org.apache.lucene.facet.sortedset
import org.apache.lucene.facet.sortedset.SortedSetDocValuesReaderState;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
+import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.store.Directory;
@@ -58,27 +59,27 @@ public class SimpleSortedSetFacetsExampl
private void index() throws IOException {
IndexWriter indexWriter = new IndexWriter(indexDir, new IndexWriterConfig(
new WhitespaceAnalyzer()).setOpenMode(OpenMode.CREATE));
- Document doc = new Document();
+ Document2 doc = indexWriter.newDocument();
doc.add(new SortedSetDocValuesFacetField("Author", "Bob"));
doc.add(new SortedSetDocValuesFacetField("Publish Year", "2010"));
indexWriter.addDocument(config.build(doc));
- doc = new Document();
+ doc = indexWriter.newDocument();
doc.add(new SortedSetDocValuesFacetField("Author", "Lisa"));
doc.add(new SortedSetDocValuesFacetField("Publish Year", "2010"));
indexWriter.addDocument(config.build(doc));
- doc = new Document();
+ doc = indexWriter.newDocument();
doc.add(new SortedSetDocValuesFacetField("Author", "Lisa"));
doc.add(new SortedSetDocValuesFacetField("Publish Year", "2012"));
indexWriter.addDocument(config.build(doc));
- doc = new Document();
+ doc = indexWriter.newDocument();
doc.add(new SortedSetDocValuesFacetField("Author", "Susan"));
doc.add(new SortedSetDocValuesFacetField("Publish Year", "2012"));
indexWriter.addDocument(config.build(doc));
- doc = new Document();
+ doc = indexWriter.newDocument();
doc.add(new SortedSetDocValuesFacetField("Author", "Frank"));
doc.add(new SortedSetDocValuesFacetField("Publish Year", "1999"));
indexWriter.addDocument(config.build(doc));
Modified: lucene/dev/branches/lucene6005/lucene/expressions/src/test/org/apache/lucene/expressions/TestDemoExpressions.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/expressions/src/test/org/apache/lucene/expressions/TestDemoExpressions.java?rev=1642110&r1=1642109&r2=1642110&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/expressions/src/test/org/apache/lucene/expressions/TestDemoExpressions.java (original)
+++ lucene/dev/branches/lucene6005/lucene/expressions/src/test/org/apache/lucene/expressions/TestDemoExpressions.java Thu Nov 27 11:34:43 2014
@@ -1,5 +1,6 @@
package org.apache.lucene.expressions;
+import org.apache.lucene.document.Document2;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.NumericDocValuesField;
@@ -23,9 +24,9 @@ import org.apache.lucene.search.TopField
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
+import static org.apache.lucene.expressions.js.VariableContext.Type.INT_INDEX;
import static org.apache.lucene.expressions.js.VariableContext.Type.MEMBER;
import static org.apache.lucene.expressions.js.VariableContext.Type.STR_INDEX;
-import static org.apache.lucene.expressions.js.VariableContext.Type.INT_INDEX;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -56,28 +57,28 @@ public class TestDemoExpressions extend
dir = newDirectory();
RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
- Document doc = new Document();
- doc.add(newStringField("id", "1", Field.Store.YES));
- doc.add(newTextField("body", "some contents and more contents", Field.Store.NO));
- doc.add(new NumericDocValuesField("popularity", 5));
- doc.add(new NumericDocValuesField("latitude", Double.doubleToRawLongBits(40.759011)));
- doc.add(new NumericDocValuesField("longitude", Double.doubleToRawLongBits(-73.9844722)));
+ Document2 doc = iw.newDocument();
+ doc.addUniqueAtom("id", "1");
+ doc.addLargeText("body", "some contents and more contents");
+ doc.addInt("popularity", 5);
+ doc.addDouble("latitude", 40.759011);
+ doc.addDouble("longitude", -73.9844722);
iw.addDocument(doc);
- doc = new Document();
- doc.add(newStringField("id", "2", Field.Store.YES));
- doc.add(newTextField("body", "another document with different contents", Field.Store.NO));
- doc.add(new NumericDocValuesField("popularity", 20));
- doc.add(new NumericDocValuesField("latitude", Double.doubleToRawLongBits(40.718266)));
- doc.add(new NumericDocValuesField("longitude", Double.doubleToRawLongBits(-74.007819)));
+ doc = iw.newDocument();
+ doc.addUniqueAtom("id", "2");
+ doc.addLargeText("body", "another document with different contents");
+ doc.addInt("popularity", 20);
+ doc.addDouble("latitude", 40.718266);
+ doc.addDouble("longitude", -74.007819);
iw.addDocument(doc);
- doc = new Document();
- doc.add(newStringField("id", "3", Field.Store.YES));
- doc.add(newTextField("body", "crappy contents", Field.Store.NO));
- doc.add(new NumericDocValuesField("popularity", 2));
- doc.add(new NumericDocValuesField("latitude", Double.doubleToRawLongBits(40.7051157)));
- doc.add(new NumericDocValuesField("longitude", Double.doubleToRawLongBits(-74.0088305)));
+ doc = iw.newDocument();
+ doc.addUniqueAtom("id", "3");
+ doc.addLargeText("body", "crappy contents");
+ doc.addInt("popularity", 2);
+ doc.addDouble("latitude", 40.7051157);
+ doc.addDouble("longitude", -74.0088305);
iw.addDocument(doc);
reader = iw.getReader();
Modified: lucene/dev/branches/lucene6005/lucene/expressions/src/test/org/apache/lucene/expressions/TestExpressionSorts.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/expressions/src/test/org/apache/lucene/expressions/TestExpressionSorts.java?rev=1642110&r1=1642109&r2=1642110&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/expressions/src/test/org/apache/lucene/expressions/TestExpressionSorts.java (original)
+++ lucene/dev/branches/lucene6005/lucene/expressions/src/test/org/apache/lucene/expressions/TestExpressionSorts.java Thu Nov 27 11:34:43 2014
@@ -20,13 +20,10 @@ package org.apache.lucene.expressions;
import java.util.Arrays;
import java.util.Collections;
+import org.apache.lucene.document.Document2;
import org.apache.lucene.document.Document;
-import org.apache.lucene.document.DoubleField;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FloatDocValuesField;
-import org.apache.lucene.document.FloatField;
-import org.apache.lucene.document.IntField;
-import org.apache.lucene.document.LongField;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.expressions.js.JavascriptCompiler;
import org.apache.lucene.index.IndexReader;
@@ -51,7 +48,7 @@ import org.apache.lucene.util.TestUtil;
/**
* Tests some basic expressions against different queries,
- * and fieldcache/docvalues fields against an equivalent sort.
+ * and docvalues fields against an equivalent sort.
*/
public class TestExpressionSorts extends LuceneTestCase {
private Directory dir;
@@ -65,19 +62,13 @@ public class TestExpressionSorts extends
RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
int numDocs = TestUtil.nextInt(random(), 2049, 4000);
for (int i = 0; i < numDocs; i++) {
- Document document = new Document();
- document.add(newTextField("english", English.intToEnglish(i), Field.Store.NO));
- document.add(newTextField("oddeven", (i % 2 == 0) ? "even" : "odd", Field.Store.NO));
- document.add(newStringField("byte", "" + ((byte) random().nextInt()), Field.Store.NO));
- document.add(newStringField("short", "" + ((short) random().nextInt()), Field.Store.NO));
- document.add(new IntField("int", random().nextInt(), Field.Store.NO));
- document.add(new LongField("long", random().nextLong(), Field.Store.NO));
-
- document.add(new FloatField("float", random().nextFloat(), Field.Store.NO));
- document.add(new DoubleField("double", random().nextDouble(), Field.Store.NO));
-
- document.add(new NumericDocValuesField("intdocvalues", random().nextInt()));
- document.add(new FloatDocValuesField("floatdocvalues", random().nextFloat()));
+ Document2 document = iw.newDocument();
+ document.addLargeText("english", English.intToEnglish(i));
+ document.addLargeText("oddeven", (i % 2 == 0) ? "even" : "odd");
+ document.addAtom("byte", "" + ((byte) random().nextInt()));
+ document.addAtom("short", "" + ((short) random().nextInt()));
+ document.addInt("intdocvalues", random().nextInt());
+ document.addFloat("floatdocvalues", random().nextFloat());
iw.addDocument(document);
}
reader = iw.getReader();
@@ -115,10 +106,6 @@ public class TestExpressionSorts extends
for (int i = 0; i < 10; i++) {
boolean reversed = random().nextBoolean();
SortField fields[] = new SortField[] {
- new SortField("int", SortField.Type.INT, reversed),
- new SortField("long", SortField.Type.LONG, reversed),
- new SortField("float", SortField.Type.FLOAT, reversed),
- new SortField("double", SortField.Type.DOUBLE, reversed),
new SortField("intdocvalues", SortField.Type.INT, reversed),
new SortField("floatdocvalues", SortField.Type.FLOAT, reversed),
new SortField("score", SortField.Type.SCORE)
Modified: lucene/dev/branches/lucene6005/lucene/facet/src/java/org/apache/lucene/facet/DrillDownQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/facet/src/java/org/apache/lucene/facet/DrillDownQuery.java?rev=1642110&r1=1642109&r2=1642110&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/facet/src/java/org/apache/lucene/facet/DrillDownQuery.java (original)
+++ lucene/dev/branches/lucene6005/lucene/facet/src/java/org/apache/lucene/facet/DrillDownQuery.java Thu Nov 27 11:34:43 2014
@@ -130,7 +130,7 @@ public final class DrillDownQuery extend
String indexedField = config.getDimConfig(dim).indexFieldName;
BooleanQuery bq = (BooleanQuery) q.getQuery();
- bq.add(new TermQuery(term(indexedField, dim, path)), Occur.SHOULD);
+ bq.add(new TermQuery(term(FacetsConfig.drillDownFieldName(indexedField), dim, path)), Occur.SHOULD);
}
/** Adds one dimension of drill downs; if you pass the same
@@ -146,7 +146,7 @@ public final class DrillDownQuery extend
String indexedField = config.getDimConfig(dim).indexFieldName;
BooleanQuery bq = new BooleanQuery(true); // disable coord
- bq.add(new TermQuery(term(indexedField, dim, path)), Occur.SHOULD);
+ bq.add(new TermQuery(term(FacetsConfig.drillDownFieldName(indexedField), dim, path)), Occur.SHOULD);
add(dim, bq);
}
@@ -286,7 +286,6 @@ public final class DrillDownQuery extend
for(Filter filter : filters) {
wrapped = new FilteredQuery(wrapped, filter, FilteredQuery.QUERY_FIRST_FILTER_STRATEGY);
}
-
return wrapped;
}
}
Modified: lucene/dev/branches/lucene6005/lucene/facet/src/java/org/apache/lucene/facet/FacetsConfig.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/facet/src/java/org/apache/lucene/facet/FacetsConfig.java?rev=1642110&r1=1642109&r2=1642110&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/facet/src/java/org/apache/lucene/facet/FacetsConfig.java (original)
+++ lucene/dev/branches/lucene6005/lucene/facet/src/java/org/apache/lucene/facet/FacetsConfig.java Thu Nov 27 11:34:43 2014
@@ -28,9 +28,11 @@ import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.lucene.document.BinaryDocValuesField;
+import org.apache.lucene.document.Document2;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.FieldTypes;
import org.apache.lucene.document.SortedSetDocValuesField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.facet.sortedset.SortedSetDocValuesFacetField;
@@ -39,6 +41,8 @@ import org.apache.lucene.facet.taxonomy.
import org.apache.lucene.facet.taxonomy.FloatAssociationFacetField;
import org.apache.lucene.facet.taxonomy.IntAssociationFacetField;
import org.apache.lucene.facet.taxonomy.TaxonomyWriter;
+import org.apache.lucene.index.DocValuesType;
+import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.IndexableFieldType;
import org.apache.lucene.util.ArrayUtil;
@@ -192,6 +196,19 @@ public class FacetsConfig {
}
/**
+ * Translates any added {@link FacetField}s into normal fields for indexing;
+ * only use this version if you did not add any taxonomy-based fields (
+ * {@link FacetField} or {@link AssociationFacetField}).
+ *
+ * <p>
+ * <b>NOTE:</b> you should add the returned document to IndexWriter, not the
+ * input one!
+ */
+ public Document2 build(Document2 doc) throws IOException {
+ return build(null, doc);
+ }
+
+ /**
* Translates any added {@link FacetField}s into normal fields for indexing.
*
* <p>
@@ -303,6 +320,118 @@ public class FacetsConfig {
return result;
}
+ /**
+ * Translates any added {@link FacetField}s into normal fields for indexing.
+ *
+ * <p>
+ * <b>NOTE:</b> you should add the returned document to IndexWriter, not the
+ * input one!
+ */
+ public Document2 build(TaxonomyWriter taxoWriter, Document2 doc) throws IOException {
+ // Find all FacetFields, collated by the actual field:
+ Map<String,List<FacetField>> byField = new HashMap<>();
+
+ // ... and also all SortedSetDocValuesFacetFields:
+ Map<String,List<SortedSetDocValuesFacetField>> dvByField = new HashMap<>();
+
+ // ... and also all AssociationFacetFields
+ Map<String,List<AssociationFacetField>> assocByField = new HashMap<>();
+
+ Set<String> seenDims = new HashSet<>();
+
+ for (IndexableField field : doc) {
+ if (field.fieldType() == FacetField.TYPE) {
+ FacetField facetField = (FacetField) field;
+ FacetsConfig.DimConfig dimConfig = getDimConfig(facetField.dim);
+ if (dimConfig.multiValued == false) {
+ checkSeen(seenDims, facetField.dim);
+ }
+ String indexFieldName = dimConfig.indexFieldName;
+ List<FacetField> fields = byField.get(indexFieldName);
+ if (fields == null) {
+ fields = new ArrayList<>();
+ byField.put(indexFieldName, fields);
+ }
+ fields.add(facetField);
+ }
+
+ if (field.fieldType() == SortedSetDocValuesFacetField.TYPE) {
+ SortedSetDocValuesFacetField facetField = (SortedSetDocValuesFacetField) field;
+ FacetsConfig.DimConfig dimConfig = getDimConfig(facetField.dim);
+ if (dimConfig.multiValued == false) {
+ checkSeen(seenDims, facetField.dim);
+ }
+ String indexFieldName = dimConfig.indexFieldName;
+ List<SortedSetDocValuesFacetField> fields = dvByField.get(indexFieldName);
+ if (fields == null) {
+ fields = new ArrayList<>();
+ dvByField.put(indexFieldName, fields);
+ }
+ fields.add(facetField);
+ }
+
+ if (field.fieldType() == AssociationFacetField.TYPE) {
+ AssociationFacetField facetField = (AssociationFacetField) field;
+ FacetsConfig.DimConfig dimConfig = getDimConfig(facetField.dim);
+ if (dimConfig.multiValued == false) {
+ checkSeen(seenDims, facetField.dim);
+ }
+ if (dimConfig.hierarchical) {
+ throw new IllegalArgumentException("AssociationFacetField cannot be hierarchical (dim=\"" + facetField.dim + "\")");
+ }
+ if (dimConfig.requireDimCount) {
+ throw new IllegalArgumentException("AssociationFacetField cannot requireDimCount (dim=\"" + facetField.dim + "\")");
+ }
+
+ String indexFieldName = dimConfig.indexFieldName;
+ List<AssociationFacetField> fields = assocByField.get(indexFieldName);
+ if (fields == null) {
+ fields = new ArrayList<>();
+ assocByField.put(indexFieldName, fields);
+ }
+ fields.add(facetField);
+
+ // Best effort: detect mis-matched types in same
+ // indexed field:
+ String type;
+ if (facetField instanceof IntAssociationFacetField) {
+ type = "int";
+ } else if (facetField instanceof FloatAssociationFacetField) {
+ type = "float";
+ } else {
+ type = "bytes";
+ }
+ // NOTE: not thread safe, but this is just best effort:
+ String curType = assocDimTypes.get(indexFieldName);
+ if (curType == null) {
+ assocDimTypes.put(indexFieldName, type);
+ } else if (!curType.equals(type)) {
+ throw new IllegalArgumentException("mixing incompatible types of AssocationFacetField (" + curType + " and " + type + ") in indexed field \"" + indexFieldName + "\"; use FacetsConfig to change the indexFieldName for each dimension");
+ }
+ }
+ }
+
+ Document2 result = new Document2(doc.getFieldTypes());
+
+ processFacetFields(taxoWriter, byField, result);
+ processSSDVFacetFields(dvByField, result);
+ processAssocFacetFields(taxoWriter, assocByField, result);
+
+ //System.out.println("add stored: " + addedStoredFields);
+
+ for (IndexableField field : doc.getFields()) {
+ IndexableFieldType ft = field.fieldType();
+ if (ft != FacetField.TYPE && ft != SortedSetDocValuesFacetField.TYPE && ft != AssociationFacetField.TYPE) {
+ result.add(field);
+ }
+ }
+
+ //System.out.println("all indexed: " + allIndexedFields);
+ //System.out.println("all stored: " + allStoredFields);
+
+ return result;
+ }
+
private void processFacetFields(TaxonomyWriter taxoWriter, Map<String,List<FacetField>> byField, Document doc) throws IOException {
for(Map.Entry<String,List<FacetField>> ent : byField.entrySet()) {
@@ -353,6 +482,64 @@ public class FacetsConfig {
}
}
+ private void processFacetFields(TaxonomyWriter taxoWriter, Map<String,List<FacetField>> byField, Document2 doc) throws IOException {
+
+ FieldTypes fieldTypes = doc.getFieldTypes();
+
+ for(Map.Entry<String,List<FacetField>> ent : byField.entrySet()) {
+ String indexFieldName = ent.getKey();
+ String drillDownFieldName = drillDownFieldName(indexFieldName);
+
+ //System.out.println(" indexFieldName=" + indexFieldName + " fields=" + ent.getValue());
+ fieldTypes.setMultiValued(drillDownFieldName);
+ fieldTypes.disableStored(drillDownFieldName);
+ fieldTypes.disableSorting(drillDownFieldName);
+ fieldTypes.disableSorting(indexFieldName);
+ fieldTypes.disableStored(indexFieldName);
+
+ IntsRefBuilder ordinals = new IntsRefBuilder();
+ for(FacetField facetField : ent.getValue()) {
+
+ FacetsConfig.DimConfig ft = getDimConfig(facetField.dim);
+ if (facetField.path.length > 1 && ft.hierarchical == false) {
+ throw new IllegalArgumentException("dimension \"" + facetField.dim + "\" is not hierarchical yet has " + facetField.path.length + " components");
+ }
+
+ FacetLabel cp = new FacetLabel(facetField.dim, facetField.path);
+
+ checkTaxoWriter(taxoWriter);
+ int ordinal = taxoWriter.addCategory(cp);
+ ordinals.append(ordinal);
+ //System.out.println("ords[" + (ordinals.length-1) + "]=" + ordinal);
+ //System.out.println(" add cp=" + cp);
+
+ if (ft.multiValued && (ft.hierarchical || ft.requireDimCount)) {
+ //System.out.println(" add parents");
+ // Add all parents too:
+ int parent = taxoWriter.getParent(ordinal);
+ while (parent > 0) {
+ ordinals.append(parent);
+ parent = taxoWriter.getParent(parent);
+ }
+
+ if (ft.requireDimCount == false) {
+ // Remove last (dimension) ord:
+ ordinals.setLength(ordinals.length() - 1);
+ }
+ }
+
+ // Drill down:
+ for (int i=1;i<=cp.length;i++) {
+ doc.addAtom(drillDownFieldName, pathToString(cp.components, i));
+ }
+ }
+
+ // Facet counts:
+ // DocValues are considered stored fields:
+ doc.addBinary(indexFieldName, dedupAndEncode(ordinals.get()));
+ }
+ }
+
private void processSSDVFacetFields(Map<String,List<SortedSetDocValuesFacetField>> byField, Document doc) throws IOException {
//System.out.println("process SSDV: " + byField);
for(Map.Entry<String,List<SortedSetDocValuesFacetField>> ent : byField.entrySet()) {
@@ -375,6 +562,41 @@ public class FacetsConfig {
}
}
+ private void processSSDVFacetFields(Map<String,List<SortedSetDocValuesFacetField>> byField, Document2 doc) throws IOException {
+ //System.out.println("process SSDV: " + byField);
+ FieldTypes fieldTypes = doc.getFieldTypes();
+ for(Map.Entry<String,List<SortedSetDocValuesFacetField>> ent : byField.entrySet()) {
+
+ String indexFieldName = ent.getKey();
+ String drillDownFieldName = drillDownFieldName(indexFieldName);
+ fieldTypes.setMultiValued(indexFieldName);
+ fieldTypes.setIndexOptions(indexFieldName, IndexOptions.NONE);
+ fieldTypes.disableSorting(drillDownFieldName);
+ fieldTypes.setMultiValued(drillDownFieldName);
+
+ //System.out.println(" field=" + indexFieldName);
+
+ for(SortedSetDocValuesFacetField facetField : ent.getValue()) {
+ FacetLabel cp = new FacetLabel(facetField.dim, facetField.label);
+ String fullPath = pathToString(cp.components, cp.length);
+ //System.out.println("add " + fullPath);
+
+ // For facet counts:
+ doc.addAtom(indexFieldName, fullPath);
+
+ // For drill down:
+ doc.addAtom(drillDownFieldName, fullPath);
+
+ // nocommit why were we doing this...?
+ //doc.add(new StringField(indexFieldName, facetField.dim, Field.Store.NO));
+ }
+ }
+ }
+
+ static String drillDownFieldName(String fieldName) {
+ return fieldName + ".drilldown";
+ }
+
private void processAssocFacetFields(TaxonomyWriter taxoWriter,
Map<String,List<AssociationFacetField>> byField, Document doc)
throws IOException {
@@ -410,6 +632,54 @@ public class FacetsConfig {
}
}
+ private void processAssocFacetFields(TaxonomyWriter taxoWriter,
+ Map<String,List<AssociationFacetField>> byField, Document2 doc)
+ throws IOException {
+
+ FieldTypes fieldTypes = doc.getFieldTypes();
+
+ for (Map.Entry<String,List<AssociationFacetField>> ent : byField.entrySet()) {
+ byte[] bytes = new byte[16];
+ int upto = 0;
+ String indexFieldName = ent.getKey();
+ String drillDownFieldName = drillDownFieldName(indexFieldName);
+ fieldTypes.setMultiValued(drillDownFieldName);
+ fieldTypes.disableSorting(drillDownFieldName);
+ fieldTypes.disableStored(drillDownFieldName);
+ fieldTypes.setDocValuesType(drillDownFieldName, DocValuesType.NONE);
+
+ fieldTypes.disableSorting(indexFieldName);
+ fieldTypes.disableStored(indexFieldName);
+
+ for(AssociationFacetField field : ent.getValue()) {
+ // NOTE: we don't add parents for associations
+ checkTaxoWriter(taxoWriter);
+ FacetLabel label = new FacetLabel(field.dim, field.path);
+ int ordinal = taxoWriter.addCategory(label);
+ if (upto + 4 > bytes.length) {
+ bytes = ArrayUtil.grow(bytes, upto+4);
+ }
+ // big-endian:
+ bytes[upto++] = (byte) (ordinal >> 24);
+ bytes[upto++] = (byte) (ordinal >> 16);
+ bytes[upto++] = (byte) (ordinal >> 8);
+ bytes[upto++] = (byte) ordinal;
+ if (upto + field.assoc.length > bytes.length) {
+ bytes = ArrayUtil.grow(bytes, upto+field.assoc.length);
+ }
+ System.arraycopy(field.assoc.bytes, field.assoc.offset, bytes, upto, field.assoc.length);
+ upto += field.assoc.length;
+
+ // Drill down:
+ for (int i = 1; i <= label.length; i++) {
+ doc.addAtom(drillDownFieldName, pathToString(label.components, i));
+ }
+ }
+
+ doc.addBinary(indexFieldName, new BytesRef(bytes, 0, upto));
+ }
+ }
+
/** Encodes ordinals into a BytesRef; expert: subclass can
* override this to change encoding. */
protected BytesRef dedupAndEncode(IntsRef ordinals) {
Modified: lucene/dev/branches/lucene6005/lucene/facet/src/test/org/apache/lucene/facet/TestDrillDownQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/facet/src/test/org/apache/lucene/facet/TestDrillDownQuery.java?rev=1642110&r1=1642109&r2=1642110&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/facet/src/test/org/apache/lucene/facet/TestDrillDownQuery.java (original)
+++ lucene/dev/branches/lucene6005/lucene/facet/src/test/org/apache/lucene/facet/TestDrillDownQuery.java Thu Nov 27 11:34:43 2014
@@ -22,8 +22,10 @@ import java.util.Random;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.analysis.MockTokenizer;
+import org.apache.lucene.document.Document2;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
+import org.apache.lucene.document.FieldTypes;
import org.apache.lucene.document.TextField;
import org.apache.lucene.facet.taxonomy.TaxonomyWriter;
import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader;
@@ -68,6 +70,9 @@ public class TestDrillDownQuery extends
RandomIndexWriter writer = new RandomIndexWriter(r, dir,
newIndexWriterConfig(new MockAnalyzer(r, MockTokenizer.KEYWORD, false)));
+ FieldTypes fieldTypes = writer.getFieldTypes();
+ fieldTypes.setMultiValued("content");
+
taxoDir = newDirectory();
TaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
config = new FacetsConfig();
@@ -88,12 +93,12 @@ public class TestDrillDownQuery extends
config.setRequireDimCount("b", true);
for (int i = 0; i < 100; i++) {
- Document doc = new Document();
+ Document2 doc = writer.newDocument();
if (i % 2 == 0) { // 50
- doc.add(new TextField("content", "foo", Field.Store.NO));
+ doc.addLargeText("content", "foo");
}
if (i % 3 == 0) { // 33
- doc.add(new TextField("content", "bar", Field.Store.NO));
+ doc.addLargeText("content", "bar");
}
if (i % 4 == 0) { // 25
if (r.nextBoolean()) {
Modified: lucene/dev/branches/lucene6005/lucene/facet/src/test/org/apache/lucene/facet/TestDrillSideways.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/facet/src/test/org/apache/lucene/facet/TestDrillSideways.java?rev=1642110&r1=1642109&r2=1642110&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/facet/src/test/org/apache/lucene/facet/TestDrillSideways.java (original)
+++ lucene/dev/branches/lucene6005/lucene/facet/src/test/org/apache/lucene/facet/TestDrillSideways.java Thu Nov 27 11:34:43 2014
@@ -28,8 +28,10 @@ import java.util.Map;
import java.util.Set;
import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.document.Document2;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
+import org.apache.lucene.document.FieldTypes;
import org.apache.lucene.document.SortedDocValuesField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.facet.DrillSideways.DrillSidewaysResult;
@@ -39,9 +41,10 @@ import org.apache.lucene.facet.sortedset
import org.apache.lucene.facet.taxonomy.TaxonomyReader;
import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader;
import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter;
-import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.DocIdSet;
@@ -56,9 +59,9 @@ import org.apache.lucene.search.SortFiel
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BitDocIdSet;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.BitDocIdSet;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.InPlaceMergeSorter;
@@ -80,27 +83,27 @@ public class TestDrillSideways extends F
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
- Document doc = new Document();
+ Document2 doc = writer.newDocument();
doc.add(new FacetField("Author", "Bob"));
doc.add(new FacetField("Publish Date", "2010", "10", "15"));
writer.addDocument(config.build(taxoWriter, doc));
- doc = new Document();
+ doc = writer.newDocument();
doc.add(new FacetField("Author", "Lisa"));
doc.add(new FacetField("Publish Date", "2010", "10", "20"));
writer.addDocument(config.build(taxoWriter, doc));
- doc = new Document();
+ doc = writer.newDocument();
doc.add(new FacetField("Author", "Lisa"));
doc.add(new FacetField("Publish Date", "2012", "1", "1"));
writer.addDocument(config.build(taxoWriter, doc));
- doc = new Document();
+ doc = writer.newDocument();
doc.add(new FacetField("Author", "Susan"));
doc.add(new FacetField("Publish Date", "2012", "1", "7"));
writer.addDocument(config.build(taxoWriter, doc));
- doc = new Document();
+ doc = writer.newDocument();
doc.add(new FacetField("Author", "Frank"));
doc.add(new FacetField("Publish Date", "1999", "5", "5"));
writer.addDocument(config.build(taxoWriter, doc));
@@ -257,12 +260,12 @@ public class TestDrillSideways extends F
FacetsConfig config = new FacetsConfig();
config.setHierarchical("Publish Date", true);
- Document doc = new Document();
+ Document2 doc = writer.newDocument();
doc.add(new FacetField("Author", "Bob"));
doc.add(new FacetField("Publish Date", "2010", "10", "15"));
writer.addDocument(config.build(taxoWriter, doc));
- doc = new Document();
+ doc = writer.newDocument();
doc.add(new FacetField("Author", "Lisa"));
doc.add(new FacetField("Publish Date", "2010", "10", "20"));
writer.addDocument(config.build(taxoWriter, doc));
@@ -270,7 +273,7 @@ public class TestDrillSideways extends F
writer.commit();
// 2nd segment has no Author:
- doc = new Document();
+ doc = writer.newDocument();
doc.add(new FacetField("Foobar", "Lisa"));
doc.add(new FacetField("Publish Date", "2012", "1", "1"));
writer.addDocument(config.build(taxoWriter, doc));
@@ -312,27 +315,27 @@ public class TestDrillSideways extends F
FacetsConfig config = new FacetsConfig();
config.setHierarchical("dim", true);
- Document doc = new Document();
+ Document2 doc = writer.newDocument();
doc.add(new FacetField("dim", "a", "x"));
writer.addDocument(config.build(taxoWriter, doc));
- doc = new Document();
+ doc = writer.newDocument();
doc.add(new FacetField("dim", "a", "y"));
writer.addDocument(config.build(taxoWriter, doc));
- doc = new Document();
+ doc = writer.newDocument();
doc.add(new FacetField("dim", "a", "z"));
writer.addDocument(config.build(taxoWriter, doc));
- doc = new Document();
+ doc = writer.newDocument();
doc.add(new FacetField("dim", "b"));
writer.addDocument(config.build(taxoWriter, doc));
- doc = new Document();
+ doc = writer.newDocument();
doc.add(new FacetField("dim", "c"));
writer.addDocument(config.build(taxoWriter, doc));
- doc = new Document();
+ doc = writer.newDocument();
doc.add(new FacetField("dim", "d"));
writer.addDocument(config.build(taxoWriter, doc));
@@ -480,22 +483,29 @@ public class TestDrillSideways extends F
Directory d = newDirectory();
Directory td = newDirectory();
+ boolean doUseDV = random().nextBoolean();
+
IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
iwc.setInfoStream(InfoStream.NO_OUTPUT);
RandomIndexWriter w = new RandomIndexWriter(random(), d, iwc);
+ FieldTypes fieldTypes = w.getFieldTypes();
+ for(int dim=0;dim<numDims;dim++) {
+ fieldTypes.setMultiValued("dim" + dim);
+ if (doUseDV == false) {
+ fieldTypes.setDocValuesType("dim" + dim, DocValuesType.NONE);
+ }
+ }
+
DirectoryTaxonomyWriter tw = new DirectoryTaxonomyWriter(td, IndexWriterConfig.OpenMode.CREATE);
FacetsConfig config = new FacetsConfig();
for(int i=0;i<numDims;i++) {
config.setMultiValued("dim"+i, true);
}
- boolean doUseDV = random().nextBoolean();
-
for(Doc rawDoc : docs) {
- Document doc = new Document();
- doc.add(newStringField("id", rawDoc.id, Field.Store.YES));
- doc.add(new SortedDocValuesField("id", new BytesRef(rawDoc.id)));
- doc.add(newStringField("content", rawDoc.contentToken, Field.Store.NO));
+ Document2 doc = w.newDocument();
+ doc.addAtom("id", rawDoc.id);
+ doc.addAtom("content", rawDoc.contentToken);
if (VERBOSE) {
System.out.println(" doc id=" + rawDoc.id + " token=" + rawDoc.contentToken);
@@ -503,30 +513,29 @@ public class TestDrillSideways extends F
for(int dim=0;dim<numDims;dim++) {
int dimValue = rawDoc.dims[dim];
if (dimValue != -1) {
+ doc.addAtom("dim" + dim, dimValues[dim][dimValue]);
if (doUseDV) {
doc.add(new SortedSetDocValuesFacetField("dim" + dim, dimValues[dim][dimValue]));
} else {
doc.add(new FacetField("dim" + dim, dimValues[dim][dimValue]));
}
- doc.add(new StringField("dim" + dim, dimValues[dim][dimValue], Field.Store.YES));
if (VERBOSE) {
System.out.println(" dim" + dim + "=" + new BytesRef(dimValues[dim][dimValue]));
}
}
int dimValue2 = rawDoc.dims2[dim];
if (dimValue2 != -1) {
+ doc.addAtom("dim" + dim, dimValues[dim][dimValue2]);
if (doUseDV) {
doc.add(new SortedSetDocValuesFacetField("dim" + dim, dimValues[dim][dimValue2]));
} else {
doc.add(new FacetField("dim" + dim, dimValues[dim][dimValue2]));
}
- doc.add(new StringField("dim" + dim, dimValues[dim][dimValue2], Field.Store.YES));
if (VERBOSE) {
System.out.println(" dim" + dim + "=" + new BytesRef(dimValues[dim][dimValue2]));
}
}
}
-
w.addDocument(config.build(tw, doc));
}
Modified: lucene/dev/branches/lucene6005/lucene/facet/src/test/org/apache/lucene/facet/range/TestRangeFacetCounts.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/facet/src/test/org/apache/lucene/facet/range/TestRangeFacetCounts.java?rev=1642110&r1=1642109&r2=1642110&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/facet/src/test/org/apache/lucene/facet/range/TestRangeFacetCounts.java (original)
+++ lucene/dev/branches/lucene6005/lucene/facet/src/test/org/apache/lucene/facet/range/TestRangeFacetCounts.java Thu Nov 27 11:34:43 2014
@@ -22,13 +22,12 @@ import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.lucene.document.Document2;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.DoubleDocValuesField;
-import org.apache.lucene.document.DoubleField;
import org.apache.lucene.document.Field;
+import org.apache.lucene.document.FieldTypes;
import org.apache.lucene.document.FloatDocValuesField;
-import org.apache.lucene.document.FloatField;
-import org.apache.lucene.document.LongField;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.facet.DrillDownQuery;
import org.apache.lucene.facet.DrillSideways.DrillSidewaysResult;
@@ -44,10 +43,10 @@ import org.apache.lucene.facet.MultiFace
import org.apache.lucene.facet.taxonomy.TaxonomyReader;
import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader;
import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter;
-import org.apache.lucene.index.LeafReader;
-import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
@@ -56,6 +55,7 @@ import org.apache.lucene.queries.functio
import org.apache.lucene.queries.function.valuesource.FloatFieldSource;
import org.apache.lucene.queries.function.valuesource.LongFieldSource;
import org.apache.lucene.search.CachingWrapperFilter;
+import org.apache.lucene.search.ConstantScoreQuery;
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.Filter;
@@ -76,16 +76,15 @@ public class TestRangeFacetCounts extend
public void testBasicLong() throws Exception {
Directory d = newDirectory();
RandomIndexWriter w = new RandomIndexWriter(random(), d);
- Document doc = new Document();
- NumericDocValuesField field = new NumericDocValuesField("field", 0L);
- doc.add(field);
for(long l=0;l<100;l++) {
- field.setLongValue(l);
+ Document2 doc = w.newDocument();
+ doc.addLong("field", l);
w.addDocument(doc);
}
// Also add Long.MAX_VALUE
- field.setLongValue(Long.MAX_VALUE);
+ Document2 doc = w.newDocument();
+ doc.addLong("field", Long.MAX_VALUE);
w.addDocument(doc);
IndexReader r = w.getReader();
@@ -142,14 +141,17 @@ public class TestRangeFacetCounts extend
Directory d = newDirectory();
RandomIndexWriter w = new RandomIndexWriter(random(), d);
- Document doc = new Document();
- NumericDocValuesField field = new NumericDocValuesField("field", 0L);
- doc.add(field);
- field.setLongValue(Long.MIN_VALUE);
+
+ Document2 doc = w.newDocument();
+ doc.addLong("field", Long.MIN_VALUE);
w.addDocument(doc);
- field.setLongValue(0);
+
+ doc = w.newDocument();
+ doc.addLong("field", 0L);
w.addDocument(doc);
- field.setLongValue(Long.MAX_VALUE);
+
+ doc = w.newDocument();
+ doc.addLong("field", Long.MAX_VALUE);
w.addDocument(doc);
IndexReader r = w.getReader();
@@ -178,14 +180,13 @@ public class TestRangeFacetCounts extend
public void testOverlappedEndStart() throws Exception {
Directory d = newDirectory();
RandomIndexWriter w = new RandomIndexWriter(random(), d);
- Document doc = new Document();
- NumericDocValuesField field = new NumericDocValuesField("field", 0L);
- doc.add(field);
for(long l=0;l<100;l++) {
- field.setLongValue(l);
+ Document2 doc = w.newDocument();
+ doc.addLong("field", l);
w.addDocument(doc);
}
- field.setLongValue(Long.MAX_VALUE);
+ Document2 doc = w.newDocument();
+ doc.addLong("field", Long.MAX_VALUE);
w.addDocument(doc);
IndexReader r = w.getReader();
@@ -220,11 +221,9 @@ public class TestRangeFacetCounts extend
FacetsConfig config = new FacetsConfig();
for (long l = 0; l < 100; l++) {
- Document doc = new Document();
- // For computing range facet counts:
- doc.add(new NumericDocValuesField("field", l));
- // For drill down by numeric range:
- doc.add(new LongField("field", l, Field.Store.NO));
+ Document2 doc = w.newDocument();
+ // For computing range facet counts and drill down by numeric range:
+ doc.addLong("field", l);
if ((l&3) == 0) {
doc.add(new FacetField("dim", "a"));
@@ -235,6 +234,7 @@ public class TestRangeFacetCounts extend
}
final IndexReader r = w.getReader();
+ FieldTypes fieldTypes = r.getFieldTypes();
final TaxonomyReader tr = new DirectoryTaxonomyReader(tw);
@@ -300,7 +300,7 @@ public class TestRangeFacetCounts extend
// Third search, drill down on "less than or equal to 10":
ddq = new DrillDownQuery(config);
- ddq.add("field", NumericRangeQuery.newLongRange("field", 0L, 10L, true, true));
+ ddq.add("field", new ConstantScoreQuery(fieldTypes.newRangeFilter("field", 0L, true, 10L, true)));
dsr = ds.search(null, ddq, 10);
assertEquals(11, dsr.hits.totalHits);
@@ -314,11 +314,9 @@ public class TestRangeFacetCounts extend
public void testBasicDouble() throws Exception {
Directory d = newDirectory();
RandomIndexWriter w = new RandomIndexWriter(random(), d);
- Document doc = new Document();
- DoubleDocValuesField field = new DoubleDocValuesField("field", 0.0);
- doc.add(field);
for(long l=0;l<100;l++) {
- field.setDoubleValue(l);
+ Document2 doc = w.newDocument();
+ doc.addDouble("field", l);
w.addDocument(doc);
}
@@ -344,11 +342,9 @@ public class TestRangeFacetCounts extend
public void testBasicFloat() throws Exception {
Directory d = newDirectory();
RandomIndexWriter w = new RandomIndexWriter(random(), d);
- Document doc = new Document();
- FloatDocValuesField field = new FloatDocValuesField("field", 0.0f);
- doc.add(field);
for(long l=0;l<100;l++) {
- field.setFloatValue(l);
+ Document2 doc = w.newDocument();
+ doc.addFloat("field", l);
w.addDocument(doc);
}
@@ -384,16 +380,16 @@ public class TestRangeFacetCounts extend
long minValue = Long.MAX_VALUE;
long maxValue = Long.MIN_VALUE;
for(int i=0;i<numDocs;i++) {
- Document doc = new Document();
+ Document2 doc = w.newDocument();
long v = random().nextLong();
values[i] = v;
- doc.add(new NumericDocValuesField("field", v));
- doc.add(new LongField("field", v, Field.Store.NO));
+ doc.addLong("field", v);
w.addDocument(doc);
minValue = Math.min(minValue, v);
maxValue = Math.max(maxValue, v);
}
IndexReader r = w.getReader();
+ FieldTypes fieldTypes = r.getFieldTypes();
IndexSearcher s = newSearcher(r);
FacetsConfig config = new FacetsConfig();
@@ -480,9 +476,9 @@ public class TestRangeFacetCounts extend
Filter fastMatchFilter;
if (random().nextBoolean()) {
if (random().nextBoolean()) {
- fastMatchFilter = NumericRangeFilter.newLongRange("field", minValue, maxValue, true, true);
+ fastMatchFilter = fieldTypes.newRangeFilter("field", minValue, true, maxValue, true);
} else {
- fastMatchFilter = NumericRangeFilter.newLongRange("field", minAcceptedValue, maxAcceptedValue, true, true);
+ fastMatchFilter = fieldTypes.newRangeFilter("field", minAcceptedValue, true, maxAcceptedValue, true);
}
} else {
fastMatchFilter = null;
@@ -505,9 +501,9 @@ public class TestRangeFacetCounts extend
DrillDownQuery ddq = new DrillDownQuery(config);
if (random().nextBoolean()) {
if (random().nextBoolean()) {
- ddq.add("field", NumericRangeFilter.newLongRange("field", range.min, range.max, range.minInclusive, range.maxInclusive));
+ ddq.add("field", fieldTypes.newRangeFilter("field", range.min, range.minInclusive, range.max, range.maxInclusive));
} else {
- ddq.add("field", NumericRangeQuery.newLongRange("field", range.min, range.max, range.minInclusive, range.maxInclusive));
+ ddq.add("field", new ConstantScoreQuery(fieldTypes.newRangeFilter("field", range.min, range.minInclusive, range.max, range.maxInclusive)));
}
} else {
ddq.add("field", range.getFilter(fastMatchFilter, vs));
@@ -529,16 +525,16 @@ public class TestRangeFacetCounts extend
float minValue = Float.POSITIVE_INFINITY;
float maxValue = Float.NEGATIVE_INFINITY;
for(int i=0;i<numDocs;i++) {
- Document doc = new Document();
+ Document2 doc = w.newDocument();
float v = random().nextFloat();
values[i] = v;
- doc.add(new FloatDocValuesField("field", v));
- doc.add(new FloatField("field", v, Field.Store.NO));
+ doc.addFloat("field", v);
w.addDocument(doc);
minValue = Math.min(minValue, v);
maxValue = Math.max(maxValue, v);
}
IndexReader r = w.getReader();
+ FieldTypes fieldTypes = r.getFieldTypes();
IndexSearcher s = newSearcher(r);
FacetsConfig config = new FacetsConfig();
@@ -639,9 +635,9 @@ public class TestRangeFacetCounts extend
Filter fastMatchFilter;
if (random().nextBoolean()) {
if (random().nextBoolean()) {
- fastMatchFilter = NumericRangeFilter.newFloatRange("field", minValue, maxValue, true, true);
+ fastMatchFilter = fieldTypes.newRangeFilter("field", minValue, true, maxValue, true);
} else {
- fastMatchFilter = NumericRangeFilter.newFloatRange("field", minAcceptedValue, maxAcceptedValue, true, true);
+ fastMatchFilter = fieldTypes.newRangeFilter("field", minAcceptedValue, true, maxAcceptedValue, true);
}
} else {
fastMatchFilter = null;
@@ -664,9 +660,9 @@ public class TestRangeFacetCounts extend
DrillDownQuery ddq = new DrillDownQuery(config);
if (random().nextBoolean()) {
if (random().nextBoolean()) {
- ddq.add("field", NumericRangeFilter.newFloatRange("field", (float) range.min, (float) range.max, range.minInclusive, range.maxInclusive));
+ ddq.add("field", fieldTypes.newRangeFilter("field", (float) range.min, range.minInclusive, (float) range.max, range.maxInclusive));
} else {
- ddq.add("field", NumericRangeQuery.newFloatRange("field", (float) range.min, (float) range.max, range.minInclusive, range.maxInclusive));
+ ddq.add("field", new ConstantScoreQuery(fieldTypes.newRangeFilter("field", (float) range.min, range.minInclusive, (float) range.max, range.maxInclusive)));
}
} else {
ddq.add("field", range.getFilter(fastMatchFilter, vs));
@@ -688,16 +684,16 @@ public class TestRangeFacetCounts extend
double minValue = Double.POSITIVE_INFINITY;
double maxValue = Double.NEGATIVE_INFINITY;
for(int i=0;i<numDocs;i++) {
- Document doc = new Document();
+ Document2 doc = w.newDocument();
double v = random().nextDouble();
values[i] = v;
- doc.add(new DoubleDocValuesField("field", v));
- doc.add(new DoubleField("field", v, Field.Store.NO));
+ doc.addDouble("field", v);
w.addDocument(doc);
minValue = Math.min(minValue, v);
maxValue = Math.max(maxValue, v);
}
IndexReader r = w.getReader();
+ FieldTypes fieldTypes = r.getFieldTypes();
IndexSearcher s = newSearcher(r);
FacetsConfig config = new FacetsConfig();
@@ -782,9 +778,9 @@ public class TestRangeFacetCounts extend
Filter fastMatchFilter;
if (random().nextBoolean()) {
if (random().nextBoolean()) {
- fastMatchFilter = NumericRangeFilter.newDoubleRange("field", minValue, maxValue, true, true);
+ fastMatchFilter = fieldTypes.newRangeFilter("field", minValue, true, maxValue, true);
} else {
- fastMatchFilter = NumericRangeFilter.newDoubleRange("field", minAcceptedValue, maxAcceptedValue, true, true);
+ fastMatchFilter = fieldTypes.newRangeFilter("field", minAcceptedValue, true, maxAcceptedValue, true);
}
} else {
fastMatchFilter = null;
@@ -807,9 +803,9 @@ public class TestRangeFacetCounts extend
DrillDownQuery ddq = new DrillDownQuery(config);
if (random().nextBoolean()) {
if (random().nextBoolean()) {
- ddq.add("field", NumericRangeFilter.newDoubleRange("field", range.min, range.max, range.minInclusive, range.maxInclusive));
+ ddq.add("field", fieldTypes.newRangeFilter("field", range.min, range.minInclusive, range.max, range.maxInclusive));
} else {
- ddq.add("field", NumericRangeQuery.newDoubleRange("field", range.min, range.max, range.minInclusive, range.maxInclusive));
+ ddq.add("field", new ConstantScoreQuery(fieldTypes.newRangeFilter("field", range.min, range.minInclusive, range.max, range.maxInclusive)));
}
} else {
ddq.add("field", range.getFilter(fastMatchFilter, vs));
@@ -827,16 +823,14 @@ public class TestRangeFacetCounts extend
public void testMissingValues() throws Exception {
Directory d = newDirectory();
RandomIndexWriter w = new RandomIndexWriter(random(), d);
- Document doc = new Document();
- NumericDocValuesField field = new NumericDocValuesField("field", 0L);
- doc.add(field);
for(long l=0;l<100;l++) {
+ // Every 5th doc is missing the value:
if (l % 5 == 0) {
- // Every 5th doc is missing the value:
- w.addDocument(new Document());
+ w.addDocument(w.newDocument());
continue;
}
- field.setLongValue(l);
+ Document2 doc = w.newDocument();
+ doc.addLong("field", l);
w.addDocument(doc);
}
@@ -864,7 +858,7 @@ public class TestRangeFacetCounts extend
Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
- Document doc = new Document();
+ Document2 doc = writer.newDocument();
writer.addDocument(doc);
writer.addDocument(doc);
writer.addDocument(doc);
Modified: lucene/dev/branches/lucene6005/lucene/facet/src/test/org/apache/lucene/facet/sortedset/TestSortedSetDocValuesFacets.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/facet/src/test/org/apache/lucene/facet/sortedset/TestSortedSetDocValuesFacets.java?rev=1642110&r1=1642109&r2=1642110&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/facet/src/test/org/apache/lucene/facet/sortedset/TestSortedSetDocValuesFacets.java (original)
+++ lucene/dev/branches/lucene6005/lucene/facet/src/test/org/apache/lucene/facet/sortedset/TestSortedSetDocValuesFacets.java Thu Nov 27 11:34:43 2014
@@ -22,6 +22,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import org.apache.lucene.document.Document2;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.facet.DrillDownQuery;
@@ -55,7 +56,7 @@ public class TestSortedSetDocValuesFacet
config.setMultiValued("a", true);
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
- Document doc = new Document();
+ Document2 doc = writer.newDocument();
doc.add(new SortedSetDocValuesFacetField("a", "foo"));
doc.add(new SortedSetDocValuesFacetField("a", "bar"));
doc.add(new SortedSetDocValuesFacetField("a", "zoo"));
@@ -65,7 +66,7 @@ public class TestSortedSetDocValuesFacet
writer.commit();
}
- doc = new Document();
+ doc = writer.newDocument();
doc.add(new SortedSetDocValuesFacetField("a", "foo"));
writer.addDocument(config.build(doc));
Modified: lucene/dev/branches/lucene6005/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestTaxonomyFacetAssociations.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestTaxonomyFacetAssociations.java?rev=1642110&r1=1642109&r2=1642110&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestTaxonomyFacetAssociations.java (original)
+++ lucene/dev/branches/lucene6005/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestTaxonomyFacetAssociations.java Thu Nov 27 11:34:43 2014
@@ -18,6 +18,7 @@ package org.apache.lucene.facet.taxonomy
*/
+import org.apache.lucene.document.Document2;
import org.apache.lucene.document.Document;
import org.apache.lucene.facet.DrillDownQuery;
import org.apache.lucene.facet.FacetTestCase;
@@ -64,7 +65,7 @@ public class TestTaxonomyFacetAssociatio
// index documents, 50% have only 'b' and all have 'a'
for (int i = 0; i < 110; i++) {
- Document doc = new Document();
+ Document2 doc = writer.newDocument();
// every 11th document is added empty, this used to cause the association
// aggregators to go into an infinite loop
if (i % 11 != 0) {
@@ -235,5 +236,4 @@ public class TestTaxonomyFacetAssociatio
assertEquals("Wrong count for category 'a'!", 100, facets.getSpecificValue("int", "a").intValue());
assertEquals("Wrong count for category 'b'!", 150, facets.getSpecificValue("int", "b").intValue());
}
-
}
Modified: lucene/dev/branches/lucene6005/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestTaxonomyFacetCounts.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestTaxonomyFacetCounts.java?rev=1642110&r1=1642109&r2=1642110&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestTaxonomyFacetCounts.java (original)
+++ lucene/dev/branches/lucene6005/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestTaxonomyFacetCounts.java Thu Nov 27 11:34:43 2014
@@ -27,6 +27,7 @@ import java.util.Map;
import java.util.Set;
import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.document.Document2;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.StringField;
@@ -72,27 +73,27 @@ public class TestTaxonomyFacetCounts ext
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
- Document doc = new Document();
+ Document2 doc = writer.newDocument();
doc.add(new FacetField("Author", "Bob"));
doc.add(new FacetField("Publish Date", "2010", "10", "15"));
writer.addDocument(config.build(taxoWriter, doc));
- doc = new Document();
+ doc = writer.newDocument();
doc.add(new FacetField("Author", "Lisa"));
doc.add(new FacetField("Publish Date", "2010", "10", "20"));
writer.addDocument(config.build(taxoWriter, doc));
- doc = new Document();
+ doc = writer.newDocument();
doc.add(new FacetField("Author", "Lisa"));
doc.add(new FacetField("Publish Date", "2012", "1", "1"));
writer.addDocument(config.build(taxoWriter, doc));
- doc = new Document();
+ doc = writer.newDocument();
doc.add(new FacetField("Author", "Susan"));
doc.add(new FacetField("Publish Date", "2012", "1", "7"));
writer.addDocument(config.build(taxoWriter, doc));
- doc = new Document();
+ doc = writer.newDocument();
doc.add(new FacetField("Author", "Frank"));
doc.add(new FacetField("Publish Date", "1999", "5", "5"));
writer.addDocument(config.build(taxoWriter, doc));
@@ -157,7 +158,7 @@ public class TestTaxonomyFacetCounts ext
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
FacetsConfig config = new FacetsConfig();
- Document doc = new Document();
+ Document2 doc = writer.newDocument();
doc.add(new FacetField("a", "foo1"));
writer.addDocument(config.build(taxoWriter, doc));
@@ -165,7 +166,7 @@ public class TestTaxonomyFacetCounts ext
writer.commit();
}
- doc = new Document();
+ doc = writer.newDocument();
doc.add(new FacetField("a", "foo2"));
doc.add(new FacetField("b", "bar1"));
writer.addDocument(config.build(taxoWriter, doc));
@@ -174,7 +175,7 @@ public class TestTaxonomyFacetCounts ext
writer.commit();
}
- doc = new Document();
+ doc = writer.newDocument();
doc.add(new FacetField("a", "foo3"));
doc.add(new FacetField("b", "bar2"));
doc.add(new FacetField("c", "baz1"));
@@ -215,7 +216,7 @@ public class TestTaxonomyFacetCounts ext
config.setIndexFieldName("a", "$facets2");
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
- Document doc = new Document();
+ Document2 doc = writer.newDocument();
doc.add(new FacetField("a", "foo1"));
writer.addDocument(config.build(taxoWriter, doc));
@@ -279,7 +280,7 @@ public class TestTaxonomyFacetCounts ext
RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc);
FacetsConfig config = new FacetsConfig();
- Document doc = new Document();
+ Document2 doc = writer.newDocument();
doc.add(newTextField("field", "text", Field.Store.NO));
doc.add(new FacetField("a", "path"));
writer.addDocument(config.build(taxoWriter, doc));
@@ -296,7 +297,7 @@ public class TestTaxonomyFacetCounts ext
config.setMultiValued("a", true);
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
- Document doc = new Document();
+ Document2 doc = writer.newDocument();
doc.add(newTextField("field", "text", Field.Store.NO));
doc.add(new FacetField("a", "path", "x"));
doc.add(new FacetField("a", "path", "y"));
@@ -342,7 +343,7 @@ public class TestTaxonomyFacetCounts ext
FacetsConfig config = new FacetsConfig();
config.setMultiValued("dim", true);
- Document doc = new Document();
+ Document2 doc = writer.newDocument();
doc.add(newTextField("field", "text", Field.Store.NO));
doc.add(new FacetField("dim", "test\u001Fone"));
doc.add(new FacetField("dim", "test\u001Etwo"));
@@ -383,7 +384,7 @@ public class TestTaxonomyFacetCounts ext
config.setHierarchical("dim3", true);
config.setRequireDimCount("dim3", true);
- Document doc = new Document();
+ Document2 doc = writer.newDocument();
doc.add(newTextField("field", "text", Field.Store.NO));
doc.add(new FacetField("dim", "a"));
doc.add(new FacetField("dim2", "a"));
@@ -432,7 +433,7 @@ public class TestTaxonomyFacetCounts ext
int numLabels = TestUtil.nextInt(random(), 40000, 100000);
- Document doc = new Document();
+ Document2 doc = writer.newDocument();
doc.add(newTextField("field", "text", Field.Store.NO));
for (int i = 0; i < numLabels; i++) {
doc.add(new FacetField("dim", "" + i));
@@ -477,7 +478,7 @@ public class TestTaxonomyFacetCounts ext
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
FacetsConfig config = new FacetsConfig();
- Document doc = new Document();
+ Document2 doc = writer.newDocument();
doc.add(newTextField("field", "text", Field.Store.NO));
doc.add(new FacetField("a", "path", "other"));
try {
@@ -499,7 +500,7 @@ public class TestTaxonomyFacetCounts ext
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
FacetsConfig config = new FacetsConfig();
- Document doc = new Document();
+ Document2 doc = writer.newDocument();
doc.add(newTextField("field", "text", Field.Store.NO));
doc.add(new FacetField("a", "path"));
doc.add(new FacetField("a", "path2"));
@@ -523,8 +524,8 @@ public class TestTaxonomyFacetCounts ext
config.setIndexFieldName("b", "$b");
for(int i = atLeast(30); i > 0; --i) {
- Document doc = new Document();
- doc.add(new StringField("f", "v", Field.Store.NO));
+ Document2 doc = iw.newDocument();
+ doc.addAtom("f", "v");
doc.add(new FacetField("a", "1"));
doc.add(new FacetField("b", "1"));
iw.addDocument(config.build(taxoWriter, doc));
@@ -552,7 +553,7 @@ public class TestTaxonomyFacetCounts ext
IndexWriter iw = new IndexWriter(indexDir, newIndexWriterConfig(new MockAnalyzer(random())));
FacetsConfig config = new FacetsConfig();
for(int i = atLeast(30); i > 0; --i) {
- Document doc = new Document();
+ Document2 doc = iw.newDocument();
doc.add(new FacetField("a", "1"));
doc.add(new FacetField("b", "1"));
iw.addDocument(config.build(taxoWriter, doc));
@@ -581,7 +582,7 @@ public class TestTaxonomyFacetCounts ext
IndexWriter iw = new IndexWriter(indexDir, newIndexWriterConfig(new MockAnalyzer(random())));
FacetsConfig config = new FacetsConfig();
- Document doc = new Document();
+ Document2 doc = iw.newDocument();
doc.add(new FacetField("a", "1"));
doc.add(new FacetField("b", "1"));
iw.addDocument(config.build(taxoWriter, doc));
@@ -610,7 +611,7 @@ public class TestTaxonomyFacetCounts ext
IndexWriter iw = new IndexWriter(indexDir, newIndexWriterConfig(new MockAnalyzer(random())));
FacetsConfig config = new FacetsConfig();
for (int i = 0; i < 10; i++) {
- Document doc = new Document();
+ Document2 doc = iw.newDocument();
doc.add(new FacetField("a", Integer.toString(i)));
iw.addDocument(config.build(taxoWriter, doc));
}
@@ -630,9 +631,9 @@ public class TestTaxonomyFacetCounts ext
private void indexTwoDocs(TaxonomyWriter taxoWriter, IndexWriter indexWriter, FacetsConfig config, boolean withContent) throws Exception {
for (int i = 0; i < 2; i++) {
- Document doc = new Document();
+ Document2 doc = indexWriter.newDocument();
if (withContent) {
- doc.add(new StringField("f", "a", Field.Store.NO));
+ doc.addAtom("f", "a");
}
if (config != null) {
doc.add(new FacetField("A", Integer.toString(i)));
@@ -696,8 +697,8 @@ public class TestTaxonomyFacetCounts ext
int numDims = TestUtil.nextInt(random(), 1, 7);
List<TestDoc> testDocs = getRandomDocs(tokens, numDocs, numDims);
for(TestDoc testDoc : testDocs) {
- Document doc = new Document();
- doc.add(newStringField("content", testDoc.content, Field.Store.NO));
+ Document2 doc = w.newDocument();
+ doc.addAtom("content", testDoc.content);
for(int j=0;j<numDims;j++) {
if (testDoc.dims[j] != null) {
doc.add(new FacetField("dim" + j, testDoc.dims[j]));
Modified: lucene/dev/branches/lucene6005/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestTaxonomyFacetSumValueSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestTaxonomyFacetSumValueSource.java?rev=1642110&r1=1642109&r2=1642110&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestTaxonomyFacetSumValueSource.java (original)
+++ lucene/dev/branches/lucene6005/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestTaxonomyFacetSumValueSource.java Thu Nov 27 11:34:43 2014
@@ -24,10 +24,10 @@ import java.util.List;
import java.util.Map;
import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.document.Document2;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FloatDocValuesField;
-import org.apache.lucene.document.IntField;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.facet.FacetField;
@@ -39,10 +39,10 @@ import org.apache.lucene.facet.FacetsCon
import org.apache.lucene.facet.LabelAndValue;
import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader;
import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter;
-import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.queries.function.FunctionQuery;
@@ -209,7 +209,7 @@ public class TestTaxonomyFacetSumValueSo
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
Document doc = new Document();
- doc.add(new IntField("num", 10, Field.Store.NO));
+ doc.add(new NumericDocValuesField("num", 10));
doc.add(new FacetField("a", "foo1"));
writer.addDocument(config.build(taxoWriter, doc));
Modified: lucene/dev/branches/lucene6005/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestDirectoryTaxonomyWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestDirectoryTaxonomyWriter.java?rev=1642110&r1=1642109&r2=1642110&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestDirectoryTaxonomyWriter.java (original)
+++ lucene/dev/branches/lucene6005/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestDirectoryTaxonomyWriter.java Thu Nov 27 11:34:43 2014
@@ -8,17 +8,18 @@ import java.util.concurrent.ConcurrentHa
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.document.Document2;
import org.apache.lucene.document.Document;
+import org.apache.lucene.facet.DrillDownQuery;
import org.apache.lucene.facet.FacetField;
import org.apache.lucene.facet.FacetTestCase;
import org.apache.lucene.facet.FacetsConfig;
-import org.apache.lucene.facet.DrillDownQuery;
import org.apache.lucene.facet.taxonomy.FacetLabel;
import org.apache.lucene.facet.taxonomy.TaxonomyReader;
import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter.MemoryOrdinalMap;
-import org.apache.lucene.facet.taxonomy.writercache.TaxonomyWriterCache;
import org.apache.lucene.facet.taxonomy.writercache.Cl2oTaxonomyWriterCache;
import org.apache.lucene.facet.taxonomy.writercache.LruTaxonomyWriterCache;
+import org.apache.lucene.facet.taxonomy.writercache.TaxonomyWriterCache;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
@@ -448,7 +449,7 @@ public class TestDirectoryTaxonomyWriter
FacetField ff = new FacetField("dim", bigs);
FacetLabel cp = new FacetLabel("dim", bigs);
ordinal = taxoWriter.addCategory(cp);
- Document doc = new Document();
+ Document2 doc = indexWriter.newDocument();
doc.add(ff);
indexWriter.addDocument(config.build(taxoWriter, doc));
@@ -456,7 +457,7 @@ public class TestDirectoryTaxonomyWriter
for (int i = 0; i < 3; i++) {
String s = TestUtil.randomSimpleString(random(), 1, 10);
taxoWriter.addCategory(new FacetLabel("dim", s));
- doc = new Document();
+ doc = indexWriter.newDocument();
doc.add(new FacetField("dim", s));
indexWriter.addDocument(config.build(taxoWriter, doc));
}
Modified: lucene/dev/branches/lucene6005/lucene/grouping/src/test/org/apache/lucene/search/grouping/AllGroupHeadsCollectorTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/grouping/src/test/org/apache/lucene/search/grouping/AllGroupHeadsCollectorTest.java?rev=1642110&r1=1642109&r2=1642110&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/grouping/src/test/org/apache/lucene/search/grouping/AllGroupHeadsCollectorTest.java (original)
+++ lucene/dev/branches/lucene6005/lucene/grouping/src/test/org/apache/lucene/search/grouping/AllGroupHeadsCollectorTest.java Thu Nov 27 11:34:43 2014
@@ -29,9 +29,9 @@ import java.util.Map;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.BinaryDocValuesField;
+import org.apache.lucene.document.Document2;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
-import org.apache.lucene.document.IntField;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.SortedDocValuesField;
import org.apache.lucene.index.DirectoryReader;
@@ -220,29 +220,6 @@ public class AllGroupHeadsCollectorTest
newIndexWriterConfig(new MockAnalyzer(random())));
DocValuesType valueType = DocValuesType.SORTED;
- Document doc = new Document();
- Document docNoGroup = new Document();
- Field valuesField = null;
- valuesField = new SortedDocValuesField("group", new BytesRef());
- doc.add(valuesField);
- Field sort1 = new SortedDocValuesField("sort1", new BytesRef());
- doc.add(sort1);
- docNoGroup.add(sort1);
- Field sort2 = new SortedDocValuesField("sort2", new BytesRef());
- doc.add(sort2);
- docNoGroup.add(sort2);
- Field sort3 = new SortedDocValuesField("sort3", new BytesRef());
- doc.add(sort3);
- docNoGroup.add(sort3);
- Field content = newTextField("content", "", Field.Store.NO);
- doc.add(content);
- docNoGroup.add(content);
- IntField id = new IntField("id", 0, Field.Store.NO);
- doc.add(id);
- docNoGroup.add(id);
- NumericDocValuesField idDV = new NumericDocValuesField("id", 0);
- doc.add(idDV);
- docNoGroup.add(idDV);
final GroupDoc[] groupDocs = new GroupDoc[numDocs];
for (int i = 0; i < numDocs; i++) {
final BytesRef groupValue;
@@ -268,20 +245,17 @@ public class AllGroupHeadsCollectorTest
}
groupDocs[i] = groupDoc;
+
+ Document2 doc = w.newDocument();
if (groupDoc.group != null) {
- valuesField.setBytesValue(new BytesRef(groupDoc.group.utf8ToString()));
- }
- sort1.setBytesValue(groupDoc.sort1);
- sort2.setBytesValue(groupDoc.sort2);
- sort3.setBytesValue(groupDoc.sort3);
- content.setStringValue(groupDoc.content);
- id.setIntValue(groupDoc.id);
- idDV.setLongValue(groupDoc.id);
- if (groupDoc.group == null) {
- w.addDocument(docNoGroup);
- } else {
- w.addDocument(doc);
+ doc.addAtom("group", new BytesRef(groupDoc.group.utf8ToString()));
}
+ doc.addAtom("sort1", groupDoc.sort1);
+ doc.addAtom("sort2", groupDoc.sort2);
+ doc.addAtom("sort3", groupDoc.sort3);
+ doc.addLargeText("content", groupDoc.content);
+ doc.addInt("id", groupDoc.id);
+ w.addDocument(doc);
}
final DirectoryReader r = w.getReader();
Modified: lucene/dev/branches/lucene6005/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestGrouping.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestGrouping.java?rev=1642110&r1=1642109&r2=1642110&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestGrouping.java (original)
+++ lucene/dev/branches/lucene6005/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestGrouping.java Thu Nov 27 11:34:43 2014
@@ -548,35 +548,26 @@ public class TestGrouping extends Lucene
dir,
newIndexWriterConfig(new MockAnalyzer(random())));
- final List<List<Document>> updateDocs = new ArrayList<>();
-
- FieldType groupEndType = new FieldType(StringField.TYPE_NOT_STORED);
- groupEndType.setIndexOptions(IndexOptions.DOCS);
- groupEndType.setOmitNorms(true);
+ final List<List<Document2>> updateDocs = new ArrayList<>();
//System.out.println("TEST: index groups");
for(BytesRef group : groupValues) {
- final List<Document> docs = new ArrayList<>();
+ final List<Document2> docs = new ArrayList<>();
//System.out.println("TEST: group=" + (group == null ? "null" : group.utf8ToString()));
for(GroupDoc groupValue : groupMap.get(group)) {
- Document doc = new Document();
+ Document2 doc = w.newDocument();
docs.add(doc);
if (groupValue.group != null) {
- doc.add(newStringField("group", groupValue.group.utf8ToString(), Field.Store.YES));
- doc.add(new SortedDocValuesField("group", BytesRef.deepCopyOf(groupValue.group)));
+ doc.addAtom("group", groupValue.group.utf8ToString());
}
- doc.add(newStringField("sort1", groupValue.sort1.utf8ToString(), Field.Store.NO));
- doc.add(new SortedDocValuesField("sort1", BytesRef.deepCopyOf(groupValue.sort1)));
- doc.add(newStringField("sort2", groupValue.sort2.utf8ToString(), Field.Store.NO));
- doc.add(new SortedDocValuesField("sort2", BytesRef.deepCopyOf(groupValue.sort2)));
- doc.add(new IntField("id", groupValue.id, Field.Store.NO));
- doc.add(new NumericDocValuesField("id", groupValue.id));
- doc.add(newTextField("content", groupValue.content, Field.Store.NO));
+ doc.addAtom("sort1", groupValue.sort1.utf8ToString());
+ doc.addAtom("sort2", groupValue.sort2.utf8ToString());
+ doc.addInt("id", groupValue.id);
+ doc.addLargeText("content", groupValue.content);
//System.out.println("TEST: doc content=" + groupValue.content + " group=" + (groupValue.group == null ? "null" : groupValue.group.utf8ToString()) + " sort1=" + groupValue.sort1.utf8ToString() + " id=" + groupValue.id);
}
// So we can pull filter marking last doc in block:
- final Field groupEnd = newField("groupend", "x", groupEndType);
- docs.get(docs.size()-1).add(groupEnd);
+ docs.get(docs.size()-1).addAtom("groupend", "x");
// Add as a doc block:
w.addDocuments(docs);
if (group != null && random().nextInt(7) == 4) {
@@ -584,9 +575,9 @@ public class TestGrouping extends Lucene
}
}
- for(List<Document> docs : updateDocs) {
+ for(List<Document2> docs : updateDocs) {
// Just replaces docs w/ same docs:
- w.updateDocuments(new Term("group", docs.get(0).get("group")), docs);
+ w.updateDocuments(new Term("group", docs.get(0).getString("group")), docs);
}
final DirectoryReader r = w.getReader();
@@ -667,29 +658,6 @@ public class TestGrouping extends Lucene
random(),
dir,
newIndexWriterConfig(new MockAnalyzer(random())));
- Document doc = new Document();
- Document docNoGroup = new Document();
- Field idvGroupField = new SortedDocValuesField("group", new BytesRef());
- doc.add(idvGroupField);
- docNoGroup.add(idvGroupField);
-
- Field group = newStringField("group", "", Field.Store.NO);
- doc.add(group);
- Field sort1 = new SortedDocValuesField("sort1", new BytesRef());
- doc.add(sort1);
- docNoGroup.add(sort1);
- Field sort2 = new SortedDocValuesField("sort2", new BytesRef());
- doc.add(sort2);
- docNoGroup.add(sort2);
- Field content = newTextField("content", "", Field.Store.NO);
- doc.add(content);
- docNoGroup.add(content);
- IntField id = new IntField("id", 0, Field.Store.NO);
- doc.add(id);
- NumericDocValuesField idDV = new NumericDocValuesField("id", 0);
- doc.add(idDV);
- docNoGroup.add(id);
- docNoGroup.add(idDV);
final GroupDoc[] groupDocs = new GroupDoc[numDocs];
for(int i=0;i<numDocs;i++) {
final BytesRef groupValue;
@@ -710,23 +678,16 @@ public class TestGrouping extends Lucene
}
groupDocs[i] = groupDoc;
+ Document2 doc = w.newDocument();
if (groupDoc.group != null) {
- group.setStringValue(groupDoc.group.utf8ToString());
- idvGroupField.setBytesValue(BytesRef.deepCopyOf(groupDoc.group));
- } else {
- // TODO: not true
- // Must explicitly set empty string, else eg if
- // the segment has all docs missing the field then
- // we get null back instead of empty BytesRef:
- idvGroupField.setBytesValue(new BytesRef());
- }
- sort1.setBytesValue(BytesRef.deepCopyOf(groupDoc.sort1));
- sort2.setBytesValue(BytesRef.deepCopyOf(groupDoc.sort2));
- content.setStringValue(groupDoc.content);
- id.setIntValue(groupDoc.id);
- idDV.setLongValue(groupDoc.id);
+ doc.addAtom("group", groupDoc.group.utf8ToString());
+ }
+ doc.addAtom("sort1", BytesRef.deepCopyOf(groupDoc.sort1));
+ doc.addAtom("sort2", BytesRef.deepCopyOf(groupDoc.sort2));
+ doc.addLargeText("content", groupDoc.content);
+ doc.addInt("id", groupDoc.id);
if (groupDoc.group == null) {
- w.addDocument(docNoGroup);
+ w.addDocument(doc);
} else {
w.addDocument(doc);
}
@@ -1253,7 +1214,7 @@ public class TestGrouping extends Lucene
final GroupDocs<BytesRef> actualGroup = actual.groups[groupIDX];
if (verifyGroupValues) {
if (idvBasedImplsUsed) {
- if (actualGroup.groupValue.length == 0) {
+ if (actualGroup.groupValue == null || actualGroup.groupValue.length == 0) {
assertNull(expectedGroup.groupValue);
} else {
assertEquals(expectedGroup.groupValue, actualGroup.groupValue);
Modified: lucene/dev/branches/lucene6005/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java?rev=1642110&r1=1642109&r2=1642110&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java (original)
+++ lucene/dev/branches/lucene6005/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java Thu Nov 27 11:34:43 2014
@@ -37,7 +37,6 @@ import org.apache.lucene.analysis.tokena
import org.apache.lucene.document.Document2;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
-import org.apache.lucene.document.IntField;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.DirectoryReader;
@@ -499,7 +498,7 @@ public class HighlighterTest extends Bas
public void testNumericRangeQuery() throws Exception {
// doesn't currently highlight, but make sure it doesn't cause exception either
- query = NumericRangeQuery.newIntRange(NUMERIC_FIELD_NAME, 2, 6, true, true);
+ query = new ConstantScoreQuery(reader.getFieldTypes().newRangeFilter(NUMERIC_FIELD_NAME, 2, true, 6, true));
searcher = newSearcher(reader);
hits = searcher.search(query, 100);
int maxNumFragmentsRequired = 2;
@@ -1938,29 +1937,25 @@ public class HighlighterTest extends Bas
analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET);
dir = newDirectory();
ramDir = newDirectory();
- IndexWriter writer = new IndexWriter(ramDir, newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET)));
+ IndexWriter writer = new IndexWriter(ramDir, newIndexWriterConfig(analyzer));
for (String text : texts) {
addDoc(writer, text);
}
- Document doc = new Document();
- doc.add(new IntField(NUMERIC_FIELD_NAME, 1, Field.Store.NO));
- doc.add(new StoredField(NUMERIC_FIELD_NAME, 1));
- writer.addDocument(doc, analyzer);
-
- doc = new Document();
- doc.add(new IntField(NUMERIC_FIELD_NAME, 3, Field.Store.NO));
- doc.add(new StoredField(NUMERIC_FIELD_NAME, 3));
- writer.addDocument(doc, analyzer);
-
- doc = new Document();
- doc.add(new IntField(NUMERIC_FIELD_NAME, 5, Field.Store.NO));
- doc.add(new StoredField(NUMERIC_FIELD_NAME, 5));
- writer.addDocument(doc, analyzer);
-
- doc = new Document();
- doc.add(new IntField(NUMERIC_FIELD_NAME, 7, Field.Store.NO));
- doc.add(new StoredField(NUMERIC_FIELD_NAME, 7));
- writer.addDocument(doc, analyzer);
+ Document2 doc = writer.newDocument();
+ doc.addInt(NUMERIC_FIELD_NAME, 1);
+ writer.addDocument(doc);
+
+ doc = writer.newDocument();
+ doc.addInt(NUMERIC_FIELD_NAME, 3);
+ writer.addDocument(doc);
+
+ doc = writer.newDocument();
+ doc.addInt(NUMERIC_FIELD_NAME, 5);
+ writer.addDocument(doc);
+
+ doc = writer.newDocument();
+ doc.addInt(NUMERIC_FIELD_NAME, 7);
+ writer.addDocument(doc);
Document childDoc = doc(FIELD_NAME, "child document");
Document parentDoc = doc(FIELD_NAME, "parent document");
Modified: lucene/dev/branches/lucene6005/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java?rev=1642110&r1=1642109&r2=1642110&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java (original)
+++ lucene/dev/branches/lucene6005/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java Thu Nov 27 11:34:43 2014
@@ -26,9 +26,11 @@ import org.apache.lucene.analysis.*;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute;
+import org.apache.lucene.document.Document2;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.FieldTypes;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
@@ -349,13 +351,16 @@ public abstract class AbstractTestCase e
// make 1 doc with multi valued field
protected void make1dmfIndex( Analyzer analyzer, String... values ) throws Exception {
IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(analyzer).setOpenMode(OpenMode.CREATE));
- Document doc = new Document();
- FieldType customType = new FieldType(TextField.TYPE_STORED);
- customType.setStoreTermVectors(true);
- customType.setStoreTermVectorOffsets(true);
- customType.setStoreTermVectorPositions(true);
+ FieldTypes fieldTypes = writer.getFieldTypes();
+ fieldTypes.enableTermVectors(F);
+ fieldTypes.enableTermVectorOffsets(F);
+ fieldTypes.enableTermVectorPositions(F);
+ fieldTypes.setMultiValued(F);
+
+ Document2 doc = writer.newDocument();
+
for( String value: values ) {
- doc.add( new Field( F, value, customType) );
+ doc.addLargeText(F, value);
}
writer.addDocument( doc );
writer.close();
@@ -366,13 +371,14 @@ public abstract class AbstractTestCase e
// make 1 doc with multi valued & not analyzed field
protected void make1dmfIndexNA( String... values ) throws Exception {
IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(analyzerK).setOpenMode(OpenMode.CREATE));
- Document doc = new Document();
- FieldType customType = new FieldType(TextField.TYPE_STORED);
- customType.setStoreTermVectors(true);
- customType.setStoreTermVectorOffsets(true);
- customType.setStoreTermVectorPositions(true);
+ FieldTypes fieldTypes = writer.getFieldTypes();
+ fieldTypes.enableTermVectors(F);
+ fieldTypes.enableTermVectorOffsets(F);
+ fieldTypes.enableTermVectorPositions(F);
+ fieldTypes.setMultiValued(F);
+ Document2 doc = writer.newDocument();
for( String value: values ) {
- doc.add( new Field( F, value, customType));
+ doc.addLargeText(F, value);
//doc.add( new Field( F, value, Store.YES, Index.NOT_ANALYZED, TermVector.WITH_POSITIONS_OFFSETS ) );
}
writer.addDocument( doc );