You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2013/11/17 22:57:43 UTC
svn commit: r1542843 - in /lucene/dev/branches/lucene5339: ./
lucene/facet/src/java/org/apache/lucene/facet/simple/
lucene/facet/src/java/org/apache/lucene/facet/taxonomy/
Author: mikemccand
Date: Sun Nov 17 21:57:42 2013
New Revision: 1542843
URL: http://svn.apache.org/r1542843
Log:
LUCENE-5339: remove delim char
Removed:
lucene/dev/branches/lucene5339/lucene/facet/src/java/org/apache/lucene/facet/simple/Constants.java
Modified:
lucene/dev/branches/lucene5339/TODO
lucene/dev/branches/lucene5339/lucene/facet/src/java/org/apache/lucene/facet/simple/FacetIndexWriter.java
lucene/dev/branches/lucene5339/lucene/facet/src/java/org/apache/lucene/facet/simple/FacetsConfig.java
lucene/dev/branches/lucene5339/lucene/facet/src/java/org/apache/lucene/facet/simple/SimpleDrillDownQuery.java
lucene/dev/branches/lucene5339/lucene/facet/src/java/org/apache/lucene/facet/simple/SortedSetDocValuesFacetCounts.java
lucene/dev/branches/lucene5339/lucene/facet/src/java/org/apache/lucene/facet/simple/SortedSetDocValuesReaderState.java
lucene/dev/branches/lucene5339/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/FacetLabel.java
Modified: lucene/dev/branches/lucene5339/TODO
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/TODO?rev=1542843&r1=1542842&r2=1542843&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/TODO (original)
+++ lucene/dev/branches/lucene5339/TODO Sun Nov 17 21:57:42 2013
@@ -2,10 +2,9 @@ nocommit this!
TODO
- associations
- - nuke delimChar? can we escape instead?
- - wrap an IW instead of extending one?
+ - cutover taxo writer/reader to pathToString/stringToPath
+ - wrap an IW instead of extending one? or, FacetDocument?
- re-enable ALL_BUT_DIM somehow?
- - SSDVValueSourceFacets?
- we could put more stuff into the "schema", e.g. this field is
sorted-set-DV and that one is taxo?
- standardize on facet or facets (e.g. FacetIndexWriter)
@@ -13,9 +12,8 @@ TODO
- how to do avg() agg?
- test needsScores=true / valuesource associations
- make FieldTypes optional (if all your dims are flat)?
- - add hierarchy to ssdv facets?
- - sparse faceting: allow skipping of certain dims?
- - maybe an interface/abstract class for "FacetResults"? has common
- API, ie to get top facets under a path, get all dims; then DS can
- use this?
- consistently name things "dimension"? calling these fields is CONFUSING
+ - later
+ - SSDVValueSourceFacets?
+ - add hierarchy to ssdv facets?
+ - sparse faceting: allow skipping of certain dims?
Modified: lucene/dev/branches/lucene5339/lucene/facet/src/java/org/apache/lucene/facet/simple/FacetIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/lucene/facet/src/java/org/apache/lucene/facet/simple/FacetIndexWriter.java?rev=1542843&r1=1542842&r2=1542843&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/lucene/facet/src/java/org/apache/lucene/facet/simple/FacetIndexWriter.java (original)
+++ lucene/dev/branches/lucene5339/lucene/facet/src/java/org/apache/lucene/facet/simple/FacetIndexWriter.java Sun Nov 17 21:57:42 2013
@@ -44,17 +44,11 @@ import org.apache.lucene.util.IntsRef;
public class FacetIndexWriter extends IndexWriter {
private final TaxonomyWriter taxoWriter;
- private final char facetDelimChar;
private final FacetsConfig facetsConfig;
public FacetIndexWriter(Directory d, IndexWriterConfig conf, TaxonomyWriter taxoWriter, FacetsConfig facetsConfig) throws IOException {
- this(d, conf, taxoWriter, facetsConfig, Constants.DEFAULT_DELIM_CHAR);
- }
-
- public FacetIndexWriter(Directory d, IndexWriterConfig conf, TaxonomyWriter taxoWriter, FacetsConfig facetsConfig, char facetDelimChar) throws IOException {
super(d, conf);
this.taxoWriter = taxoWriter;
- this.facetDelimChar = facetDelimChar;
this.facetsConfig = facetsConfig;
}
@@ -175,7 +169,7 @@ public class FacetIndexWriter extends In
// Drill down:
for(int i=2;i<=cp.length;i++) {
- addedIndexedFields.add(new StringField(indexedFieldName, cp.subpath(i).toString(facetDelimChar), Field.Store.NO));
+ addedIndexedFields.add(new StringField(indexedFieldName, pathToString(cp.components, i), Field.Store.NO));
}
}
@@ -194,7 +188,7 @@ public class FacetIndexWriter extends In
for(SortedSetDocValuesFacetField facetField : ent.getValue()) {
FacetLabel cp = new FacetLabel(facetField.dim, facetField.label);
- String fullPath = cp.toString(facetDelimChar);
+ String fullPath = pathToString(cp.components, cp.length);
//System.out.println("add " + fullPath);
// For facet counts:
@@ -255,4 +249,72 @@ public class FacetIndexWriter extends In
return new BytesRef(bytes, 0, upto);
}
+ // nocommit move these constants / methods to Util?
+
+ // Joins the path components together:
+ private static final char DELIM_CHAR = '\u001F';
+
+ // Escapes any occurrence of the path component inside the label:
+ private static final char ESCAPE_CHAR = '\u001E';
+
+ /** Turns a path into a string without stealing any
+ * characters. */
+ public static String pathToString(String dim, String[] path) {
+ String[] fullPath = new String[1+path.length];
+ fullPath[0] = dim;
+ System.arraycopy(path, 0, fullPath, 1, path.length);
+ return pathToString(fullPath, fullPath.length);
+ }
+
+ public static String pathToString(String[] path) {
+ return pathToString(path, path.length);
+ }
+
+ public static String pathToString(String[] path, int length) {
+ StringBuilder sb = new StringBuilder();
+ for(int i=0;i<length;i++) {
+ String s = path[i];
+ int numChars = s.length();
+ for(int j=0;j<numChars;j++) {
+ char ch = s.charAt(j);
+ if (ch == DELIM_CHAR || ch == ESCAPE_CHAR) {
+ sb.append(ESCAPE_CHAR);
+ }
+ sb.append(ch);
+ }
+ sb.append(DELIM_CHAR);
+ }
+
+ // Trim off last DELIM_CHAR:
+ sb.setLength(sb.length()-1);
+ return sb.toString();
+ }
+
+ /** Turns a result from previous call to {@link
+ * #pathToString} back into the original {@code String[]}
+ * without stealing any characters. */
+ public static String[] stringToPath(String s) {
+ List<String> parts = new ArrayList<String>();
+ int length = s.length();
+ char[] buffer = new char[length];
+
+ int upto = 0;
+ boolean lastEscape = false;
+ for(int i=0;i<length;i++) {
+ char ch = s.charAt(i);
+ if (lastEscape) {
+ buffer[upto++] = ch;
+ lastEscape = false;
+ } else if (ch == ESCAPE_CHAR) {
+ lastEscape = true;
+ } else if (ch == DELIM_CHAR) {
+ parts.add(new String(buffer, 0, upto));
+ upto = 0;
+ } else {
+ buffer[upto++] = ch;
+ }
+ }
+ assert !lastEscape;
+ return parts.toArray(new String[parts.size()]);
+ }
}
\ No newline at end of file
Modified: lucene/dev/branches/lucene5339/lucene/facet/src/java/org/apache/lucene/facet/simple/FacetsConfig.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/lucene/facet/src/java/org/apache/lucene/facet/simple/FacetsConfig.java?rev=1542843&r1=1542842&r2=1542843&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/lucene/facet/src/java/org/apache/lucene/facet/simple/FacetsConfig.java (original)
+++ lucene/dev/branches/lucene5339/lucene/facet/src/java/org/apache/lucene/facet/simple/FacetsConfig.java Sun Nov 17 21:57:42 2013
@@ -26,7 +26,6 @@ public class FacetsConfig {
public static final String DEFAULT_INDEXED_FIELD_NAME = "$facets";
- // nocommit pull the delim char into there?
// nocommit pull DimType into here (shai?)
private final Map<String,DimConfig> fieldTypes = new ConcurrentHashMap<String,DimConfig>();
Modified: lucene/dev/branches/lucene5339/lucene/facet/src/java/org/apache/lucene/facet/simple/SimpleDrillDownQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/lucene/facet/src/java/org/apache/lucene/facet/simple/SimpleDrillDownQuery.java?rev=1542843&r1=1542842&r2=1542843&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/lucene/facet/src/java/org/apache/lucene/facet/simple/SimpleDrillDownQuery.java (original)
+++ lucene/dev/branches/lucene5339/lucene/facet/src/java/org/apache/lucene/facet/simple/SimpleDrillDownQuery.java Sun Nov 17 21:57:42 2013
@@ -18,6 +18,7 @@ package org.apache.lucene.facet.simple;
*/
import java.io.IOException;
+import java.util.Arrays;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -51,8 +52,8 @@ import org.apache.lucene.search.TermQuer
*/
public final class SimpleDrillDownQuery extends Query {
- private static Term term(String field, char delimChar, String dim, String[] path) {
- return new Term(field, FacetLabel.create(dim, path).toString(delimChar));
+ private static Term term(String field, String dim, String[] path) {
+ return new Term(field, FacetIndexWriter.pathToString(dim, path));
}
private final FacetsConfig config;
@@ -134,10 +135,8 @@ public final class SimpleDrillDownQuery
}
String indexedField = config.getDimConfig(dim).indexedFieldName;
- // nocommit pull this from FacetsConfig
- char delimChar = Constants.DEFAULT_DELIM_CHAR;
BooleanQuery bq = (BooleanQuery) q.getQuery();
- bq.add(new TermQuery(term(indexedField, delimChar, dim, path)), Occur.SHOULD);
+ bq.add(new TermQuery(term(indexedField, dim, path)), Occur.SHOULD);
}
/** Adds one dimension of drill downs; if you pass the same
@@ -153,13 +152,11 @@ public final class SimpleDrillDownQuery
}
String indexedField = config.getDimConfig(dim).indexedFieldName;
- // nocommit pull this from FacetsConfig
- char delimChar = Constants.DEFAULT_DELIM_CHAR;
BooleanQuery bq = new BooleanQuery(true); // disable coord
if (path.length == 0) {
throw new IllegalArgumentException("must have at least one facet label under dim");
}
- bq.add(new TermQuery(term(indexedField, delimChar, dim, path)), Occur.SHOULD);
+ bq.add(new TermQuery(term(indexedField, dim, path)), Occur.SHOULD);
add(dim, bq);
}
Modified: lucene/dev/branches/lucene5339/lucene/facet/src/java/org/apache/lucene/facet/simple/SortedSetDocValuesFacetCounts.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/lucene/facet/src/java/org/apache/lucene/facet/simple/SortedSetDocValuesFacetCounts.java?rev=1542843&r1=1542842&r2=1542843&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/lucene/facet/src/java/org/apache/lucene/facet/simple/SortedSetDocValuesFacetCounts.java (original)
+++ lucene/dev/branches/lucene5339/lucene/facet/src/java/org/apache/lucene/facet/simple/SortedSetDocValuesFacetCounts.java Sun Nov 17 21:57:42 2013
@@ -230,7 +230,7 @@ public class SortedSetDocValuesFacetCoun
throw new IllegalArgumentException("path must be length=1");
}
- int ord = (int) dv.lookupTerm(new BytesRef(dim + state.separator + path[0]));
+ int ord = (int) dv.lookupTerm(new BytesRef(FacetIndexWriter.pathToString(dim, path)));
if (ord < 0) {
return -1;
}
Modified: lucene/dev/branches/lucene5339/lucene/facet/src/java/org/apache/lucene/facet/simple/SortedSetDocValuesReaderState.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/lucene/facet/src/java/org/apache/lucene/facet/simple/SortedSetDocValuesReaderState.java?rev=1542843&r1=1542842&r2=1542843&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/lucene/facet/src/java/org/apache/lucene/facet/simple/SortedSetDocValuesReaderState.java (original)
+++ lucene/dev/branches/lucene5339/lucene/facet/src/java/org/apache/lucene/facet/simple/SortedSetDocValuesReaderState.java Sun Nov 17 21:57:42 2013
@@ -52,8 +52,6 @@ public final class SortedSetDocValuesRea
private final AtomicReader topReader;
private final int valueCount;
public final IndexReader origReader;
- public final char separator;
- final String separatorRegex;
/** Holds start/end range of ords, which maps to one
* dimension (someday we may generalize it to map to
@@ -74,21 +72,15 @@ public final class SortedSetDocValuesRea
private final Map<String,OrdRange> prefixToOrdRange = new HashMap<String,OrdRange>();
public SortedSetDocValuesReaderState(IndexReader reader) throws IOException {
- this(reader, FacetsConfig.DEFAULT_INDEXED_FIELD_NAME, Constants.DEFAULT_DELIM_CHAR);
- }
-
- public SortedSetDocValuesReaderState(IndexReader reader, String dvField) throws IOException {
- this(reader, dvField, Constants.DEFAULT_DELIM_CHAR);
+ this(reader, FacetsConfig.DEFAULT_INDEXED_FIELD_NAME);
}
/** Create an instance, scanning the {@link
* SortedSetDocValues} from the provided reader, with
* default {@link FacetIndexingParams}. */
- public SortedSetDocValuesReaderState(IndexReader reader, String field, char delimChar) throws IOException {
+ public SortedSetDocValuesReaderState(IndexReader reader, String field) throws IOException {
this.field = field;
- this.separator = delimChar;
- this.separatorRegex = Pattern.quote(Character.toString(separator));
this.origReader = reader;
// We need this to create thread-safe MultiSortedSetDV
@@ -116,7 +108,7 @@ public final class SortedSetDocValuesRea
// support arbitrary hierarchy:
for(int ord=0;ord<valueCount;ord++) {
dv.lookupOrd(ord, spare);
- String[] components = spare.utf8ToString().split(separatorRegex, 2);
+ String[] components = FacetIndexWriter.stringToPath(spare.utf8ToString());
if (components.length != 2) {
throw new IllegalArgumentException("this class can only handle 2 level hierarchy (dim/value); got: " + spare.utf8ToString());
}
Modified: lucene/dev/branches/lucene5339/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/FacetLabel.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/FacetLabel.java?rev=1542843&r1=1542842&r2=1542843&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/FacetLabel.java (original)
+++ lucene/dev/branches/lucene5339/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/FacetLabel.java Sun Nov 17 21:57:42 2013
@@ -274,6 +274,7 @@ public class FacetLabel implements Compa
* Returns a string representation of the path, separating components with the
* given delimiter.
*/
+ // nocommit remove
public String toString(char delimiter) {
if (length == 0) return "";