You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2014/01/04 13:33:29 UTC
svn commit: r1555342 [3/3] - in /lucene/dev/branches/branch_4x: ./
dev-tools/ lucene/ lucene/analysis/ lucene/backwards/ lucene/benchmark/
lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/
lucene/benchmark/src/java/org/apache/lucene/b...
Modified: lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestConcurrentFacetedIndexing.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestConcurrentFacetedIndexing.java?rev=1555342&r1=1555341&r2=1555342&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestConcurrentFacetedIndexing.java (original)
+++ lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestConcurrentFacetedIndexing.java Sat Jan 4 12:33:26 2014
@@ -1,19 +1,18 @@
package org.apache.lucene.facet.taxonomy.directory;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
import java.util.Random;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.lucene.document.Document;
+import org.apache.lucene.facet.FacetField;
import org.apache.lucene.facet.FacetTestCase;
-import org.apache.lucene.facet.index.FacetFields;
-import org.apache.lucene.facet.taxonomy.CategoryPath;
+import org.apache.lucene.facet.FacetsConfig;
+import org.apache.lucene.facet.taxonomy.FacetLabel;
import org.apache.lucene.facet.taxonomy.writercache.TaxonomyWriterCache;
-import org.apache.lucene.facet.taxonomy.writercache.cl2o.Cl2oTaxonomyWriterCache;
-import org.apache.lucene.facet.taxonomy.writercache.lru.LruTaxonomyWriterCache;
+import org.apache.lucene.facet.taxonomy.writercache.Cl2oTaxonomyWriterCache;
+import org.apache.lucene.facet.taxonomy.writercache.LruTaxonomyWriterCache;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.store.Directory;
@@ -45,9 +44,9 @@ public class TestConcurrentFacetedIndexi
@Override
public void close() {}
@Override
- public int get(CategoryPath categoryPath) { return -1; }
+ public int get(FacetLabel categoryPath) { return -1; }
@Override
- public boolean put(CategoryPath categoryPath, int ordinal) { return true; }
+ public boolean put(FacetLabel categoryPath, int ordinal) { return true; }
@Override
public boolean isFull() { return true; }
@Override
@@ -55,12 +54,12 @@ public class TestConcurrentFacetedIndexi
};
- static CategoryPath newCategory() {
+ static FacetField newCategory() {
Random r = random();
String l1 = "l1." + r.nextInt(10); // l1.0-l1.9 (10 categories)
String l2 = "l2." + r.nextInt(30); // l2.0-l2.29 (30 categories)
String l3 = "l3." + r.nextInt(100); // l3.0-l3.99 (100 categories)
- return new CategoryPath(l1, l2, l3);
+ return new FacetField(l1, l2, l3);
}
static TaxonomyWriterCache newTaxoWriterCache(int ndocs) {
@@ -86,10 +85,14 @@ public class TestConcurrentFacetedIndexi
final IndexWriter iw = new IndexWriter(indexDir, newIndexWriterConfig(TEST_VERSION_CURRENT, null));
final DirectoryTaxonomyWriter tw = new DirectoryTaxonomyWriter(taxoDir, OpenMode.CREATE, newTaxoWriterCache(numDocs.get()));
final Thread[] indexThreads = new Thread[atLeast(4)];
+ final FacetsConfig config = new FacetsConfig();
+ for(int i=0;i<10;i++) {
+ config.setHierarchical("l1." + i, true);
+ config.setMultiValued("l1." + i, true);
+ }
for (int i = 0; i < indexThreads.length; i++) {
indexThreads[i] = new Thread() {
- private final FacetFields facetFields = new FacetFields(tw);
@Override
public void run() {
@@ -98,20 +101,20 @@ public class TestConcurrentFacetedIndexi
try {
Document doc = new Document();
int numCats = random.nextInt(3) + 1; // 1-3
- List<CategoryPath> cats = new ArrayList<CategoryPath>(numCats);
while (numCats-- > 0) {
- CategoryPath cp = newCategory();
- cats.add(cp);
+ FacetField ff = newCategory();
+ doc.add(ff);
+
+ FacetLabel label = new FacetLabel(ff.dim, ff.path);
// add all prefixes to values
- int level = cp.length;
+ int level = label.length;
while (level > 0) {
- String s = cp.subpath(level).toString('/');
+ String s = FacetsConfig.pathToString(label.components, level);
values.put(s, s);
--level;
}
}
- facetFields.addFields(doc, cats);
- iw.addDocument(doc);
+ iw.addDocument(config.build(tw, doc));
} catch (IOException e) {
throw new RuntimeException(e);
}
@@ -124,14 +127,23 @@ public class TestConcurrentFacetedIndexi
for (Thread t : indexThreads) t.join();
DirectoryTaxonomyReader tr = new DirectoryTaxonomyReader(tw);
- assertEquals("mismatch number of categories", values.size() + 1, tr.getSize()); // +1 for root category
+ // +1 for root category
+ if (values.size() + 1 != tr.getSize()) {
+ for(String value : values.keySet()) {
+ FacetLabel label = new FacetLabel(FacetsConfig.stringToPath(value));
+ if (tr.getOrdinal(label) == -1) {
+ System.out.println("FAIL: path=" + label + " not recognized");
+ }
+ }
+ fail("mismatch number of categories");
+ }
int[] parents = tr.getParallelTaxonomyArrays().parents();
for (String cat : values.keySet()) {
- CategoryPath cp = new CategoryPath(cat, '/');
+ FacetLabel cp = new FacetLabel(FacetsConfig.stringToPath(cat));
assertTrue("category not found " + cp, tr.getOrdinal(cp) > 0);
int level = cp.length;
int parentOrd = 0; // for root, parent is always virtual ROOT (ord=0)
- CategoryPath path = CategoryPath.EMPTY;
+ FacetLabel path = null;
for (int i = 0; i < level; i++) {
path = cp.subpath(i + 1);
int ord = tr.getOrdinal(path);
@@ -139,9 +151,8 @@ public class TestConcurrentFacetedIndexi
parentOrd = ord; // next level should have this parent
}
}
- tr.close();
- IOUtils.close(tw, iw, taxoDir, indexDir);
+ IOUtils.close(tw, iw, tr, taxoDir, indexDir);
}
}
Modified: lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestDirectoryTaxonomyReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestDirectoryTaxonomyReader.java?rev=1555342&r1=1555341&r2=1555342&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestDirectoryTaxonomyReader.java (original)
+++ lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestDirectoryTaxonomyReader.java Sat Jan 4 12:33:26 2014
@@ -8,7 +8,7 @@ import java.util.Set;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.facet.FacetTestCase;
-import org.apache.lucene.facet.taxonomy.CategoryPath;
+import org.apache.lucene.facet.taxonomy.FacetLabel;
import org.apache.lucene.facet.taxonomy.TaxonomyReader;
import org.apache.lucene.facet.taxonomy.TaxonomyReader.ChildrenIterator;
import org.apache.lucene.facet.taxonomy.TaxonomyWriter;
@@ -46,7 +46,7 @@ public class TestDirectoryTaxonomyReader
public void testCloseAfterIncRef() throws Exception {
Directory dir = newDirectory();
DirectoryTaxonomyWriter ltw = new DirectoryTaxonomyWriter(dir);
- ltw.addCategory(new CategoryPath("a"));
+ ltw.addCategory(new FacetLabel("a"));
ltw.close();
DirectoryTaxonomyReader ltr = new DirectoryTaxonomyReader(dir);
@@ -64,7 +64,7 @@ public class TestDirectoryTaxonomyReader
public void testCloseTwice() throws Exception {
Directory dir = newDirectory();
DirectoryTaxonomyWriter ltw = new DirectoryTaxonomyWriter(dir);
- ltw.addCategory(new CategoryPath("a"));
+ ltw.addCategory(new FacetLabel("a"));
ltw.close();
DirectoryTaxonomyReader ltr = new DirectoryTaxonomyReader(dir);
@@ -84,13 +84,13 @@ public class TestDirectoryTaxonomyReader
dir = newDirectory();
ltw = new DirectoryTaxonomyWriter(dir);
- ltw.addCategory(new CategoryPath("a"));
+ ltw.addCategory(new FacetLabel("a"));
ltw.commit();
ltr = new DirectoryTaxonomyReader(dir);
assertNull("Nothing has changed", TaxonomyReader.openIfChanged(ltr));
- ltw.addCategory(new CategoryPath("b"));
+ ltw.addCategory(new FacetLabel("b"));
ltw.commit();
DirectoryTaxonomyReader newtr = TaxonomyReader.openIfChanged(ltr);
@@ -106,7 +106,7 @@ public class TestDirectoryTaxonomyReader
public void testAlreadyClosed() throws Exception {
Directory dir = newDirectory();
DirectoryTaxonomyWriter ltw = new DirectoryTaxonomyWriter(dir);
- ltw.addCategory(new CategoryPath("a"));
+ ltw.addCategory(new FacetLabel("a"));
ltw.close();
DirectoryTaxonomyReader ltr = new DirectoryTaxonomyReader(dir);
@@ -140,16 +140,16 @@ public class TestDirectoryTaxonomyReader
// prepare a few categories
int n = 10;
- CategoryPath[] cp = new CategoryPath[n];
+ FacetLabel[] cp = new FacetLabel[n];
for (int i=0; i<n; i++) {
- cp[i] = new CategoryPath("a", Integer.toString(i));
+ cp[i] = new FacetLabel("a", Integer.toString(i));
}
try {
dir = newDirectory();
tw = new DirectoryTaxonomyWriter(dir);
- tw.addCategory(new CategoryPath("a"));
+ tw.addCategory(new FacetLabel("a"));
tw.close();
tr = new DirectoryTaxonomyReader(dir);
@@ -183,7 +183,7 @@ public class TestDirectoryTaxonomyReader
Directory dir = new RAMDirectory(); // no need for random directories here
DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(dir);
- taxoWriter.addCategory(new CategoryPath("a"));
+ taxoWriter.addCategory(new FacetLabel("a"));
taxoWriter.commit();
TaxonomyReader taxoReader = new DirectoryTaxonomyReader(dir);
@@ -192,7 +192,7 @@ public class TestDirectoryTaxonomyReader
taxoReader.incRef();
assertEquals("wrong refCount", 2, taxoReader.getRefCount());
- taxoWriter.addCategory(new CategoryPath("a", "b"));
+ taxoWriter.addCategory(new FacetLabel("a", "b"));
taxoWriter.commit();
TaxonomyReader newtr = TaxonomyReader.openIfChanged(taxoReader);
assertNotNull(newtr);
@@ -226,7 +226,7 @@ public class TestDirectoryTaxonomyReader
for (int i = 0; i < numRounds; i++) {
int numCats = random().nextInt(4) + 1;
for (int j = 0; j < numCats; j++) {
- writer.addCategory(new CategoryPath(Integer.toString(i), Integer.toString(j)));
+ writer.addCategory(new FacetLabel(Integer.toString(i), Integer.toString(j)));
}
numCategories += numCats + 1 /* one for round-parent */;
TaxonomyReader newtr = TaxonomyReader.openIfChanged(reader);
@@ -236,11 +236,11 @@ public class TestDirectoryTaxonomyReader
// assert categories
assertEquals(numCategories, reader.getSize());
- int roundOrdinal = reader.getOrdinal(new CategoryPath(Integer.toString(i)));
+ int roundOrdinal = reader.getOrdinal(new FacetLabel(Integer.toString(i)));
int[] parents = reader.getParallelTaxonomyArrays().parents();
assertEquals(0, parents[roundOrdinal]); // round's parent is root
for (int j = 0; j < numCats; j++) {
- int ord = reader.getOrdinal(new CategoryPath(Integer.toString(i), Integer.toString(j)));
+ int ord = reader.getOrdinal(new FacetLabel(Integer.toString(i), Integer.toString(j)));
assertEquals(roundOrdinal, parents[ord]); // round's parent is root
}
}
@@ -276,7 +276,7 @@ public class TestDirectoryTaxonomyReader
// add category and call forceMerge -- this should flush IW and merge segments down to 1
// in ParentArray.initFromReader, this used to fail assuming there are no parents.
- writer.addCategory(new CategoryPath("1"));
+ writer.addCategory(new FacetLabel("1"));
iw.forceMerge(1);
// now calling openIfChanged should trip on the bug
@@ -315,7 +315,7 @@ public class TestDirectoryTaxonomyReader
// add a category so that the following DTR open will cause a flush and
// a new segment will be created
- writer.addCategory(new CategoryPath("a"));
+ writer.addCategory(new FacetLabel("a"));
TaxonomyReader reader = new DirectoryTaxonomyReader(writer);
assertEquals(2, reader.getSize());
@@ -342,7 +342,7 @@ public class TestDirectoryTaxonomyReader
// tests that if the taxonomy is recreated, no data is reused from the previous taxonomy
Directory dir = newDirectory();
DirectoryTaxonomyWriter writer = new DirectoryTaxonomyWriter(dir);
- CategoryPath cp_a = new CategoryPath("a");
+ FacetLabel cp_a = new FacetLabel("a");
writer.addCategory(cp_a);
writer.close();
@@ -353,7 +353,7 @@ public class TestDirectoryTaxonomyReader
// now recreate, add a different category
writer = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE);
- CategoryPath cp_b = new CategoryPath("b");
+ FacetLabel cp_b = new FacetLabel("b");
writer.addCategory(cp_b);
writer.close();
@@ -384,7 +384,7 @@ public class TestDirectoryTaxonomyReader
Directory dir = newDirectory();
DirectoryTaxonomyWriter writer = new DirectoryTaxonomyWriter(dir);
- CategoryPath cp_a = new CategoryPath("a");
+ FacetLabel cp_a = new FacetLabel("a");
writer.addCategory(cp_a);
if (!nrt) writer.commit();
@@ -393,7 +393,7 @@ public class TestDirectoryTaxonomyReader
assertEquals(1, r1.getOrdinal(cp_a));
assertEquals(cp_a, r1.getPath(1));
- CategoryPath cp_b = new CategoryPath("b");
+ FacetLabel cp_b = new FacetLabel("b");
writer.addCategory(cp_b);
if (!nrt) writer.commit();
@@ -421,7 +421,7 @@ public class TestDirectoryTaxonomyReader
// only can work with NRT as well
Directory src = newDirectory();
DirectoryTaxonomyWriter w = new DirectoryTaxonomyWriter(src);
- CategoryPath cp_b = new CategoryPath("b");
+ FacetLabel cp_b = new FacetLabel("b");
w.addCategory(cp_b);
w.close();
@@ -429,7 +429,7 @@ public class TestDirectoryTaxonomyReader
Directory dir = newDirectory();
DirectoryTaxonomyWriter writer = new DirectoryTaxonomyWriter(dir);
- CategoryPath cp_a = new CategoryPath("a");
+ FacetLabel cp_a = new FacetLabel("a");
writer.addCategory(cp_a);
if (!nrt) writer.commit();
@@ -474,29 +474,29 @@ public class TestDirectoryTaxonomyReader
int numA = 0, numB = 0;
Random random = random();
// add the two categories for which we'll also add children (so asserts are simpler)
- taxoWriter.addCategory(new CategoryPath("a"));
- taxoWriter.addCategory(new CategoryPath("b"));
+ taxoWriter.addCategory(new FacetLabel("a"));
+ taxoWriter.addCategory(new FacetLabel("b"));
for (int i = 0; i < numCategories; i++) {
if (random.nextBoolean()) {
- taxoWriter.addCategory(new CategoryPath("a", Integer.toString(i)));
+ taxoWriter.addCategory(new FacetLabel("a", Integer.toString(i)));
++numA;
} else {
- taxoWriter.addCategory(new CategoryPath("b", Integer.toString(i)));
+ taxoWriter.addCategory(new FacetLabel("b", Integer.toString(i)));
++numB;
}
}
// add category with no children
- taxoWriter.addCategory(new CategoryPath("c"));
+ taxoWriter.addCategory(new FacetLabel("c"));
taxoWriter.close();
DirectoryTaxonomyReader taxoReader = new DirectoryTaxonomyReader(dir);
// non existing category
- ChildrenIterator it = taxoReader.getChildren(taxoReader.getOrdinal(new CategoryPath("invalid")));
+ ChildrenIterator it = taxoReader.getChildren(taxoReader.getOrdinal(new FacetLabel("invalid")));
assertEquals(TaxonomyReader.INVALID_ORDINAL, it.next());
// a category with no children
- it = taxoReader.getChildren(taxoReader.getOrdinal(new CategoryPath("c")));
+ it = taxoReader.getChildren(taxoReader.getOrdinal(new FacetLabel("c")));
assertEquals(TaxonomyReader.INVALID_ORDINAL, it.next());
// arbitrary negative ordinal
@@ -507,20 +507,20 @@ public class TestDirectoryTaxonomyReader
Set<String> roots = new HashSet<String>(Arrays.asList("a", "b", "c"));
it = taxoReader.getChildren(TaxonomyReader.ROOT_ORDINAL);
while (!roots.isEmpty()) {
- CategoryPath root = taxoReader.getPath(it.next());
+ FacetLabel root = taxoReader.getPath(it.next());
assertEquals(1, root.length);
assertTrue(roots.remove(root.components[0]));
}
assertEquals(TaxonomyReader.INVALID_ORDINAL, it.next());
for (int i = 0; i < 2; i++) {
- CategoryPath cp = i == 0 ? new CategoryPath("a") : new CategoryPath("b");
+ FacetLabel cp = i == 0 ? new FacetLabel("a") : new FacetLabel("b");
int ordinal = taxoReader.getOrdinal(cp);
it = taxoReader.getChildren(ordinal);
int numChildren = 0;
int child;
while ((child = it.next()) != TaxonomyReader.INVALID_ORDINAL) {
- CategoryPath path = taxoReader.getPath(child);
+ FacetLabel path = taxoReader.getPath(child);
assertEquals(2, path.length);
assertEquals(path.components[0], i == 0 ? "a" : "b");
++numChildren;
Modified: lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestDirectoryTaxonomyWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestDirectoryTaxonomyWriter.java?rev=1555342&r1=1555341&r2=1555342&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestDirectoryTaxonomyWriter.java (original)
+++ lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestDirectoryTaxonomyWriter.java Sat Jan 4 12:33:26 2014
@@ -7,21 +7,29 @@ import java.util.Random;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.facet.FacetField;
import org.apache.lucene.facet.FacetTestCase;
-import org.apache.lucene.facet.taxonomy.CategoryPath;
+import org.apache.lucene.facet.FacetsConfig;
+import org.apache.lucene.facet.DrillDownQuery;
+import org.apache.lucene.facet.taxonomy.FacetLabel;
import org.apache.lucene.facet.taxonomy.TaxonomyReader;
import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter.MemoryOrdinalMap;
import org.apache.lucene.facet.taxonomy.writercache.TaxonomyWriterCache;
-import org.apache.lucene.facet.taxonomy.writercache.cl2o.Cl2oTaxonomyWriterCache;
-import org.apache.lucene.facet.taxonomy.writercache.lru.LruTaxonomyWriterCache;
+import org.apache.lucene.facet.taxonomy.writercache.Cl2oTaxonomyWriterCache;
+import org.apache.lucene.facet.taxonomy.writercache.LruTaxonomyWriterCache;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
+import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.SegmentInfos;
+import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util._TestUtil;
import org.junit.Test;
/*
@@ -50,9 +58,9 @@ public class TestDirectoryTaxonomyWriter
@Override
public void close() {}
@Override
- public int get(CategoryPath categoryPath) { return -1; }
+ public int get(FacetLabel categoryPath) { return -1; }
@Override
- public boolean put(CategoryPath categoryPath, int ordinal) { return true; }
+ public boolean put(FacetLabel categoryPath, int ordinal) { return true; }
@Override
public boolean isFull() { return true; }
@Override
@@ -68,7 +76,7 @@ public class TestDirectoryTaxonomyWriter
DirectoryTaxonomyWriter ltw = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE_OR_APPEND, NO_OP_CACHE);
assertFalse(DirectoryReader.indexExists(dir));
ltw.commit(); // first commit, so that an index will be created
- ltw.addCategory(new CategoryPath("a"));
+ ltw.addCategory(new FacetLabel("a"));
IndexReader r = DirectoryReader.open(dir);
assertEquals("No categories should have been committed to the underlying directory", 1, r.numDocs());
@@ -82,8 +90,8 @@ public class TestDirectoryTaxonomyWriter
// Verifies taxonomy commit data
Directory dir = newDirectory();
DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE_OR_APPEND, NO_OP_CACHE);
- taxoWriter.addCategory(new CategoryPath("a"));
- taxoWriter.addCategory(new CategoryPath("b"));
+ taxoWriter.addCategory(new FacetLabel("a"));
+ taxoWriter.addCategory(new FacetLabel("b"));
Map<String, String> userCommitData = new HashMap<String, String>();
userCommitData.put("testing", "1 2 3");
taxoWriter.setCommitData(userCommitData);
@@ -100,7 +108,7 @@ public class TestDirectoryTaxonomyWriter
// in the commit data, otherwise DirTaxoReader.refresh() might not detect
// that the taxonomy index has been recreated.
taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE_OR_APPEND, NO_OP_CACHE);
- taxoWriter.addCategory(new CategoryPath("c")); // add a category so that commit will happen
+ taxoWriter.addCategory(new FacetLabel("c")); // add a category so that commit will happen
taxoWriter.setCommitData(new HashMap<String, String>(){{
put("just", "data");
}});
@@ -124,10 +132,10 @@ public class TestDirectoryTaxonomyWriter
// Verifies that if rollback is called, DTW is closed.
Directory dir = newDirectory();
DirectoryTaxonomyWriter dtw = new DirectoryTaxonomyWriter(dir);
- dtw.addCategory(new CategoryPath("a"));
+ dtw.addCategory(new FacetLabel("a"));
dtw.rollback();
try {
- dtw.addCategory(new CategoryPath("a"));
+ dtw.addCategory(new FacetLabel("a"));
fail("should not have succeeded to add a category following rollback.");
} catch (AlreadyClosedException e) {
// expected
@@ -155,7 +163,7 @@ public class TestDirectoryTaxonomyWriter
DirectoryTaxonomyWriter dtw = new DirectoryTaxonomyWriter(dir);
dtw.close();
try {
- dtw.addCategory(new CategoryPath("a"));
+ dtw.addCategory(new FacetLabel("a"));
fail("should not have succeeded to add a category following close.");
} catch (AlreadyClosedException e) {
// expected
@@ -163,7 +171,7 @@ public class TestDirectoryTaxonomyWriter
dir.close();
}
- private void touchTaxo(DirectoryTaxonomyWriter taxoWriter, CategoryPath cp) throws IOException {
+ private void touchTaxo(DirectoryTaxonomyWriter taxoWriter, FacetLabel cp) throws IOException {
taxoWriter.addCategory(cp);
taxoWriter.setCommitData(new HashMap<String, String>(){{
put("just", "data");
@@ -179,11 +187,11 @@ public class TestDirectoryTaxonomyWriter
Directory dir = newDirectory();
DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE_OR_APPEND, NO_OP_CACHE);
- touchTaxo(taxoWriter, new CategoryPath("a"));
+ touchTaxo(taxoWriter, new FacetLabel("a"));
TaxonomyReader taxoReader = new DirectoryTaxonomyReader(dir);
- touchTaxo(taxoWriter, new CategoryPath("b"));
+ touchTaxo(taxoWriter, new FacetLabel("b"));
TaxonomyReader newtr = TaxonomyReader.openIfChanged(taxoReader);
taxoReader.close();
@@ -193,11 +201,11 @@ public class TestDirectoryTaxonomyWriter
// now recreate the taxonomy, and check that the epoch is preserved after opening DirTW again.
taxoWriter.close();
taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE, NO_OP_CACHE);
- touchTaxo(taxoWriter, new CategoryPath("c"));
+ touchTaxo(taxoWriter, new FacetLabel("c"));
taxoWriter.close();
taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE_OR_APPEND, NO_OP_CACHE);
- touchTaxo(taxoWriter, new CategoryPath("d"));
+ touchTaxo(taxoWriter, new FacetLabel("d"));
taxoWriter.close();
newtr = TaxonomyReader.openIfChanged(taxoReader);
@@ -248,6 +256,9 @@ public class TestDirectoryTaxonomyWriter
// this is slower than CL2O, but less memory consuming, and exercises finding categories on disk too.
cache = new LruTaxonomyWriterCache(ncats / 10);
}
+ if (VERBOSE) {
+ System.out.println("TEST: use cache=" + cache);
+ }
final DirectoryTaxonomyWriter tw = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE, cache);
Thread[] addThreads = new Thread[atLeast(4)];
for (int z = 0; z < addThreads.length; z++) {
@@ -258,14 +269,14 @@ public class TestDirectoryTaxonomyWriter
while (numCats.decrementAndGet() > 0) {
try {
int value = random.nextInt(range);
- CategoryPath cp = new CategoryPath(Integer.toString(value / 1000), Integer.toString(value / 10000),
+ FacetLabel cp = new FacetLabel(Integer.toString(value / 1000), Integer.toString(value / 10000),
Integer.toString(value / 100000), Integer.toString(value));
int ord = tw.addCategory(cp);
assertTrue("invalid parent for ordinal " + ord + ", category " + cp, tw.getParent(ord) != -1);
- String l1 = cp.subpath(1).toString('/');
- String l2 = cp.subpath(2).toString('/');
- String l3 = cp.subpath(3).toString('/');
- String l4 = cp.subpath(4).toString('/');
+ String l1 = FacetsConfig.pathToString(cp.components, 1);
+ String l2 = FacetsConfig.pathToString(cp.components, 2);
+ String l3 = FacetsConfig.pathToString(cp.components, 3);
+ String l4 = FacetsConfig.pathToString(cp.components, 4);
values.put(l1, l1);
values.put(l2, l2);
values.put(l3, l3);
@@ -283,14 +294,24 @@ public class TestDirectoryTaxonomyWriter
tw.close();
DirectoryTaxonomyReader dtr = new DirectoryTaxonomyReader(dir);
- assertEquals("mismatch number of categories", values.size() + 1, dtr.getSize()); // +1 for root category
+ // +1 for root category
+ if (values.size() + 1 != dtr.getSize()) {
+ for(String value : values.keySet()) {
+ FacetLabel label = new FacetLabel(FacetsConfig.stringToPath(value));
+ if (dtr.getOrdinal(label) == -1) {
+ System.out.println("FAIL: path=" + label + " not recognized");
+ }
+ }
+ fail("mismatch number of categories");
+ }
+
int[] parents = dtr.getParallelTaxonomyArrays().parents();
for (String cat : values.keySet()) {
- CategoryPath cp = new CategoryPath(cat, '/');
+ FacetLabel cp = new FacetLabel(FacetsConfig.stringToPath(cat));
assertTrue("category not found " + cp, dtr.getOrdinal(cp) > 0);
int level = cp.length;
int parentOrd = 0; // for root, parent is always virtual ROOT (ord=0)
- CategoryPath path = CategoryPath.EMPTY;
+ FacetLabel path = new FacetLabel();
for (int i = 0; i < level; i++) {
path = cp.subpath(i + 1);
int ord = dtr.getOrdinal(path);
@@ -298,9 +319,8 @@ public class TestDirectoryTaxonomyWriter
parentOrd = ord; // next level should have this parent
}
}
- dtr.close();
-
- dir.close();
+
+ IOUtils.close(dtr, dir);
}
private long getEpoch(Directory taxoDir) throws IOException {
@@ -313,13 +333,13 @@ public class TestDirectoryTaxonomyWriter
public void testReplaceTaxonomy() throws Exception {
Directory input = newDirectory();
DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(input);
- int ordA = taxoWriter.addCategory(new CategoryPath("a"));
+ int ordA = taxoWriter.addCategory(new FacetLabel("a"));
taxoWriter.close();
Directory dir = newDirectory();
taxoWriter = new DirectoryTaxonomyWriter(dir);
- int ordB = taxoWriter.addCategory(new CategoryPath("b"));
- taxoWriter.addCategory(new CategoryPath("c"));
+ int ordB = taxoWriter.addCategory(new FacetLabel("b"));
+ taxoWriter.addCategory(new FacetLabel("c"));
taxoWriter.commit();
long origEpoch = getEpoch(dir);
@@ -330,10 +350,10 @@ public class TestDirectoryTaxonomyWriter
// LUCENE-4633: make sure that category "a" is not added again in any case
taxoWriter.addTaxonomy(input, new MemoryOrdinalMap());
assertEquals("no categories should have been added", 2, taxoWriter.getSize()); // root + 'a'
- assertEquals("category 'a' received new ordinal?", ordA, taxoWriter.addCategory(new CategoryPath("a")));
+ assertEquals("category 'a' received new ordinal?", ordA, taxoWriter.addCategory(new FacetLabel("a")));
// add the same category again -- it should not receive the same ordinal !
- int newOrdB = taxoWriter.addCategory(new CategoryPath("b"));
+ int newOrdB = taxoWriter.addCategory(new FacetLabel("b"));
assertNotSame("new ordinal cannot be the original ordinal", ordB, newOrdB);
assertEquals("ordinal should have been 2 since only one category was added by replaceTaxonomy", 2, newOrdB);
@@ -353,8 +373,8 @@ public class TestDirectoryTaxonomyWriter
// is being added.
Directory dir = newDirectory();
DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE, NO_OP_CACHE);
- int o1 = taxoWriter.addCategory(new CategoryPath("a"));
- int o2 = taxoWriter.addCategory(new CategoryPath("a"));
+ int o1 = taxoWriter.addCategory(new FacetLabel("a"));
+ int o2 = taxoWriter.addCategory(new FacetLabel("a"));
assertTrue("ordinal for same category that is added twice should be the same !", o1 == o2);
taxoWriter.close();
dir.close();
@@ -365,7 +385,7 @@ public class TestDirectoryTaxonomyWriter
// LUCENE-4972: DTW used to create empty commits even if no changes were made
Directory dir = newDirectory();
DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(dir);
- taxoWriter.addCategory(new CategoryPath("a"));
+ taxoWriter.addCategory(new FacetLabel("a"));
taxoWriter.commit();
long gen1 = SegmentInfos.getLastCommitGeneration(dir);
@@ -382,7 +402,7 @@ public class TestDirectoryTaxonomyWriter
// LUCENE-4972: DTW used to create empty commits even if no changes were made
Directory dir = newDirectory();
DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(dir);
- taxoWriter.addCategory(new CategoryPath("a"));
+ taxoWriter.addCategory(new FacetLabel("a"));
taxoWriter.commit();
long gen1 = SegmentInfos.getLastCommitGeneration(dir);
@@ -399,7 +419,7 @@ public class TestDirectoryTaxonomyWriter
// LUCENE-4972: DTW used to create empty commits even if no changes were made
Directory dir = newDirectory();
DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(dir);
- taxoWriter.addCategory(new CategoryPath("a"));
+ taxoWriter.addCategory(new FacetLabel("a"));
taxoWriter.prepareCommit();
taxoWriter.commit();
@@ -412,7 +432,50 @@ public class TestDirectoryTaxonomyWriter
taxoWriter.close();
dir.close();
}
-
+
+ @Test
+ public void testHugeLabel() throws Exception {
+ Directory indexDir = newDirectory(), taxoDir = newDirectory();
+ IndexWriter indexWriter = new IndexWriter(indexDir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())));
+ DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir, OpenMode.CREATE, new Cl2oTaxonomyWriterCache(2, 1f, 1));
+ FacetsConfig config = new FacetsConfig();
+
+ // Add one huge label:
+ String bigs = null;
+ int ordinal = -1;
+
+ int len = FacetLabel.MAX_CATEGORY_PATH_LENGTH - 4; // for the dimension and separator
+ bigs = _TestUtil.randomSimpleString(random(), len, len);
+ FacetField ff = new FacetField("dim", bigs);
+ FacetLabel cp = new FacetLabel("dim", bigs);
+ ordinal = taxoWriter.addCategory(cp);
+ Document doc = new Document();
+ doc.add(ff);
+ indexWriter.addDocument(config.build(taxoWriter, doc));
+
+ // Add tiny ones to cause a re-hash
+ for (int i = 0; i < 3; i++) {
+ String s = _TestUtil.randomSimpleString(random(), 1, 10);
+ taxoWriter.addCategory(new FacetLabel("dim", s));
+ doc = new Document();
+ doc.add(new FacetField("dim", s));
+ indexWriter.addDocument(config.build(taxoWriter, doc));
+ }
+
+ // when too large components were allowed to be added, this resulted in a new added category
+ assertEquals(ordinal, taxoWriter.addCategory(cp));
+
+ IOUtils.close(indexWriter, taxoWriter);
+
+ DirectoryReader indexReader = DirectoryReader.open(indexDir);
+ TaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoDir);
+ IndexSearcher searcher = new IndexSearcher(indexReader);
+ DrillDownQuery ddq = new DrillDownQuery(new FacetsConfig());
+ ddq.add("dim", bigs);
+ assertEquals(1, searcher.search(ddq, 10).totalHits);
+
+ IOUtils.close(indexReader, taxoReader, indexDir, taxoDir);
+ }
@Test
public void testReplaceTaxoWithLargeTaxonomy() throws Exception {
@@ -420,11 +483,11 @@ public class TestDirectoryTaxonomyWriter
// build source, large, taxonomy
DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(srcTaxoDir);
- int ord = taxoWriter.addCategory(new CategoryPath("A/1/1/1/1/1/1", '/'));
+ int ord = taxoWriter.addCategory(new FacetLabel("A", "1", "1", "1", "1", "1", "1"));
taxoWriter.close();
taxoWriter = new DirectoryTaxonomyWriter(targetTaxoDir);
- int ordinal = taxoWriter.addCategory(new CategoryPath("B/1", '/'));
+ int ordinal = taxoWriter.addCategory(new FacetLabel("B", "1"));
assertEquals(1, taxoWriter.getParent(ordinal)); // call getParent to initialize taxoArrays
taxoWriter.commit();
Modified: lucene/dev/branches/branch_4x/lucene/replicator/src/test/org/apache/lucene/replicator/IndexAndTaxonomyReplicationClientTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/replicator/src/test/org/apache/lucene/replicator/IndexAndTaxonomyReplicationClientTest.java?rev=1555342&r1=1555341&r2=1555342&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/replicator/src/test/org/apache/lucene/replicator/IndexAndTaxonomyReplicationClientTest.java (original)
+++ lucene/dev/branches/branch_4x/lucene/replicator/src/test/org/apache/lucene/replicator/IndexAndTaxonomyReplicationClientTest.java Sat Jan 4 12:33:26 2014
@@ -26,13 +26,14 @@ import java.util.concurrent.Callable;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.lucene.document.Document;
-import org.apache.lucene.facet.index.FacetFields;
-import org.apache.lucene.facet.params.FacetIndexingParams;
-import org.apache.lucene.facet.params.FacetSearchParams;
-import org.apache.lucene.facet.search.CountFacetRequest;
-import org.apache.lucene.facet.search.DrillDownQuery;
-import org.apache.lucene.facet.search.FacetsCollector;
-import org.apache.lucene.facet.taxonomy.CategoryPath;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.facet.DrillDownQuery;
+import org.apache.lucene.facet.FacetField;
+import org.apache.lucene.facet.FacetResult;
+import org.apache.lucene.facet.Facets;
+import org.apache.lucene.facet.FacetsCollector;
+import org.apache.lucene.facet.FacetsConfig;
+import org.apache.lucene.facet.taxonomy.FastTaxonomyFacetCounts;
import org.apache.lucene.facet.taxonomy.TaxonomyReader;
import org.apache.lucene.facet.taxonomy.TaxonomyWriter;
import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader;
@@ -63,11 +64,14 @@ public class IndexAndTaxonomyReplication
private final Directory indexDir, taxoDir;
private DirectoryReader indexReader;
private DirectoryTaxonomyReader taxoReader;
+ private FacetsConfig config;
private long lastIndexGeneration = -1;
public IndexAndTaxonomyReadyCallback(Directory indexDir, Directory taxoDir) throws IOException {
this.indexDir = indexDir;
this.taxoDir = taxoDir;
+ config = new FacetsConfig();
+ config.setHierarchical("A", true);
if (DirectoryReader.indexExists(indexDir)) {
indexReader = DirectoryReader.open(indexDir);
lastIndexGeneration = indexReader.getIndexCommit().getGeneration();
@@ -102,14 +106,14 @@ public class IndexAndTaxonomyReplication
// verify faceted search
int id = Integer.parseInt(indexReader.getIndexCommit().getUserData().get(VERSION_ID), 16);
- CategoryPath cp = new CategoryPath("A", Integer.toString(id, 16));
IndexSearcher searcher = new IndexSearcher(indexReader);
- FacetsCollector fc = FacetsCollector.create(new FacetSearchParams(new CountFacetRequest(cp, 10)), indexReader, taxoReader);
+ FacetsCollector fc = new FacetsCollector();
searcher.search(new MatchAllDocsQuery(), fc);
- assertEquals(1, (int) fc.getFacetResults().get(0).getFacetResultNode().value);
+ Facets facets = new FastTaxonomyFacetCounts(taxoReader, config, fc);
+ assertEquals(1, facets.getSpecificValue("A", Integer.toString(id, 16)).intValue());
- DrillDownQuery drillDown = new DrillDownQuery(FacetIndexingParams.DEFAULT);
- drillDown.add(cp);
+ DrillDownQuery drillDown = new DrillDownQuery(config);
+ drillDown.add("A", Integer.toString(id, 16));
TopDocs docs = searcher.search(drillDown, 10);
assertEquals(1, docs.totalHits);
}
@@ -130,6 +134,7 @@ public class IndexAndTaxonomyReplication
private ReplicationHandler handler;
private IndexWriter publishIndexWriter;
private SnapshotDirectoryTaxonomyWriter publishTaxoWriter;
+ private FacetsConfig config;
private IndexAndTaxonomyReadyCallback callback;
private File clientWorkDir;
@@ -177,9 +182,8 @@ public class IndexAndTaxonomyReplication
private Document newDocument(TaxonomyWriter taxoWriter, int id) throws IOException {
Document doc = new Document();
- FacetFields facetFields = new FacetFields(taxoWriter);
- facetFields.addFields(doc, Collections.singleton(new CategoryPath("A", Integer.toString(id, 16))));
- return doc;
+ doc.add(new FacetField("A", Integer.toString(id, 16)));
+ return config.build(publishTaxoWriter, doc);
}
@Override
@@ -201,6 +205,8 @@ public class IndexAndTaxonomyReplication
conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
publishIndexWriter = new IndexWriter(publishIndexDir, conf);
publishTaxoWriter = new SnapshotDirectoryTaxonomyWriter(publishTaxoDir);
+ config = new FacetsConfig();
+ config.setHierarchical("A", true);
}
@After
Modified: lucene/dev/branches/branch_4x/lucene/replicator/src/test/org/apache/lucene/replicator/IndexAndTaxonomyRevisionTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/replicator/src/test/org/apache/lucene/replicator/IndexAndTaxonomyRevisionTest.java?rev=1555342&r1=1555341&r2=1555342&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/replicator/src/test/org/apache/lucene/replicator/IndexAndTaxonomyRevisionTest.java (original)
+++ lucene/dev/branches/branch_4x/lucene/replicator/src/test/org/apache/lucene/replicator/IndexAndTaxonomyRevisionTest.java Sat Jan 4 12:33:26 2014
@@ -21,12 +21,14 @@ import java.io.IOException;
import java.io.InputStream;
import java.util.Collections;
import java.util.List;
-import java.util.Map;
import java.util.Map.Entry;
+import java.util.Map;
import org.apache.lucene.document.Document;
-import org.apache.lucene.facet.index.FacetFields;
-import org.apache.lucene.facet.taxonomy.CategoryPath;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.facet.FacetField;
+import org.apache.lucene.facet.FacetsConfig;
+import org.apache.lucene.facet.taxonomy.FacetLabel;
import org.apache.lucene.facet.taxonomy.TaxonomyWriter;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.IndexWriter;
@@ -42,10 +44,10 @@ import org.junit.Test;
public class IndexAndTaxonomyRevisionTest extends ReplicatorTestCase {
private Document newDocument(TaxonomyWriter taxoWriter) throws IOException {
+ FacetsConfig config = new FacetsConfig();
Document doc = new Document();
- FacetFields ff = new FacetFields(taxoWriter);
- ff.addFields(doc, Collections.singleton(new CategoryPath("A")));
- return doc;
+ doc.add(new FacetField("A", "1"));
+ return config.build(taxoWriter, doc);
}
@Test