You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2011/05/09 17:24:23 UTC
svn commit: r1101062 [19/21] - in /lucene/dev/branches/bulkpostings: ./
dev-tools/ dev-tools/eclipse/ dev-tools/idea/.idea/
dev-tools/idea/lucene/contrib/ant/ dev-tools/idea/lucene/contrib/db/bdb-je/
dev-tools/idea/lucene/contrib/db/bdb/ dev-tools/idea...
Modified: lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/handler/component/FacetComponent.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/handler/component/FacetComponent.java?rev=1101062&r1=1101061&r2=1101062&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/handler/component/FacetComponent.java (original)
+++ lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/handler/component/FacetComponent.java Mon May 9 15:24:04 2011
@@ -17,23 +17,23 @@
package org.apache.solr.handler.component;
-import java.io.IOException;
-import java.net.URL;
-import java.util.*;
-
+import org.apache.lucene.queryParser.ParseException;
+import org.apache.lucene.util.OpenBitSet;
+import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.FacetParams;
-import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.common.util.StrUtils;
-import org.apache.solr.common.SolrException;
import org.apache.solr.request.SimpleFacets;
-import org.apache.lucene.util.OpenBitSet;
-import org.apache.solr.search.QueryParsing;
import org.apache.solr.schema.FieldType;
-import org.apache.lucene.queryParser.ParseException;
+import org.apache.solr.search.QueryParsing;
+
+import java.io.IOException;
+import java.net.URL;
+import java.util.*;
/**
* TODO!
@@ -222,11 +222,37 @@ public class FacetComponent extends Sear
sreq.params.remove(paramStart + FacetParams.FACET_MINCOUNT);
sreq.params.remove(paramStart + FacetParams.FACET_OFFSET);
- dff.initialLimit = dff.offset + dff.limit;
+ dff.initialLimit = dff.limit <= 0 ? dff.limit : dff.offset + dff.limit;
- if(dff.sort.equals(FacetParams.FACET_SORT_COUNT) && dff.limit > 0) {
- // set the initial limit higher to increase accuracy
- dff.initialLimit = (int)(dff.initialLimit * 1.5) + 10;
+ if (dff.sort.equals(FacetParams.FACET_SORT_COUNT)) {
+ if (dff.limit > 0) {
+ // set the initial limit higher to increase accuracy
+ dff.initialLimit = (int)(dff.initialLimit * 1.5) + 10;
+ dff.initialMincount = 0; // TODO: we could change this to 1, but would then need more refinement for small facet result sets?
+ } else {
+ // if limit==-1, then no need to artificially lower mincount to 0 if it's 1
+ dff.initialMincount = Math.min(dff.minCount, 1);
+ }
+ } else {
+ // we're sorting by index order.
+ // if minCount==0, we should always be able to get accurate results w/o over-requesting or refining
+ // if minCount==1, we should be able to get accurate results w/o over-requesting, but we'll need to refine
+ // if minCount==n (>1), we can set the initialMincount to minCount/nShards, rounded up.
+ // For example, we know that if minCount=10 and we have 3 shards, then at least one shard must have a count of 4 for the term
+ // For the minCount>1 case, we can generate too short of a list (miss terms at the end of the list) unless limit==-1
+ // For example: each shard could produce a list of top 10, but some of those could fail to make it into the combined list (i.e.
+ // we needed to go beyond the top 10 to generate the top 10 combined). Overrequesting can help a little here, but not as
+ // much as when sorting by count.
+ if (dff.minCount <= 1) {
+ dff.initialMincount = dff.minCount;
+ } else {
+ dff.initialMincount = (int)Math.ceil((double)dff.minCount / rb.slices.length);
+ // dff.initialMincount = 1;
+ }
+ }
+
+ if (dff.initialMincount != 0) {
+ sreq.params.set(paramStart + FacetParams.FACET_MINCOUNT, dff.initialMincount);
}
// Currently this is for testing only and allows overriding of the
@@ -286,8 +312,95 @@ public class FacetComponent extends Sear
dff.add(shardNum, (NamedList)facet_fields.get(dff.getKey()), dff.initialLimit);
}
}
- }
+ // Distributed facet_dates
+ //
+ // The implementation below uses the first encountered shard's
+ // facet_dates as the basis for subsequent shards' data to be merged.
+ // (the "NOW" param should ensure consistency)
+ @SuppressWarnings("unchecked")
+ SimpleOrderedMap<SimpleOrderedMap<Object>> facet_dates =
+ (SimpleOrderedMap<SimpleOrderedMap<Object>>)
+ facet_counts.get("facet_dates");
+
+ if (facet_dates != null) {
+
+ // go through each facet_date
+ for (Map.Entry<String,SimpleOrderedMap<Object>> entry : facet_dates) {
+ final String field = entry.getKey();
+ if (fi.dateFacets.get(field) == null) {
+ // first time we've seen this field, no merging
+ fi.dateFacets.add(field, entry.getValue());
+
+ } else {
+ // not the first time, merge current field
+
+ SimpleOrderedMap<Object> shardFieldValues
+ = entry.getValue();
+ SimpleOrderedMap<Object> existFieldValues
+ = fi.dateFacets.get(field);
+
+ for (Map.Entry<String,Object> existPair : existFieldValues) {
+ final String key = existPair.getKey();
+ if (key.equals("gap") ||
+ key.equals("end") ||
+ key.equals("start")) {
+ // we can skip these, must all be the same across shards
+ continue;
+ }
+ // can be null if inconsistencies in shards responses
+ Integer newValue = (Integer) shardFieldValues.get(key);
+ if (null != newValue) {
+ Integer oldValue = ((Integer) existPair.getValue());
+ existPair.setValue(oldValue + newValue);
+ }
+ }
+ }
+ }
+ }
+
+ // Distributed facet_ranges
+ //
+ // The implementation below uses the first encountered shard's
+ // facet_ranges as the basis for subsequent shards' data to be merged.
+ @SuppressWarnings("unchecked")
+ SimpleOrderedMap<SimpleOrderedMap<Object>> facet_ranges =
+ (SimpleOrderedMap<SimpleOrderedMap<Object>>)
+ facet_counts.get("facet_ranges");
+
+ if (facet_ranges != null) {
+
+ // go through each facet_range
+ for (Map.Entry<String,SimpleOrderedMap<Object>> entry : facet_ranges) {
+ final String field = entry.getKey();
+ if (fi.rangeFacets.get(field) == null) {
+ // first time we've seen this field, no merging
+ fi.rangeFacets.add(field, entry.getValue());
+
+ } else {
+ // not the first time, merge current field counts
+
+ @SuppressWarnings("unchecked")
+ NamedList<Integer> shardFieldValues
+ = (NamedList<Integer>) entry.getValue().get("counts");
+
+ @SuppressWarnings("unchecked")
+ NamedList<Integer> existFieldValues
+ = (NamedList<Integer>) fi.rangeFacets.get(field).get("counts");
+
+ for (Map.Entry<String,Integer> existPair : existFieldValues) {
+ final String key = existPair.getKey();
+ // can be null if inconsistencies in shards responses
+ Integer newValue = shardFieldValues.get(key);
+ if (null != newValue) {
+ Integer oldValue = existPair.getValue();
+ existPair.setValue(oldValue + newValue);
+ }
+ }
+ }
+ }
+ }
+ }
//
// This code currently assumes that there will be only a single
@@ -296,15 +409,18 @@ public class FacetComponent extends Sear
//
for (DistribFieldFacet dff : fi.facets.values()) {
- if (dff.limit <= 0) continue; // no need to check these facets for refinement
- if (dff.minCount <= 1 && dff.sort.equals(FacetParams.FACET_SORT_INDEX)) continue;
+ // no need to check these facets for refinement
+ if (dff.initialLimit <= 0 && dff.initialMincount == 0) continue;
- @SuppressWarnings("unchecked") // generic array's are anoying
+ // only other case where index-sort doesn't need refinement is if minCount==0
+ if (dff.minCount == 0 && dff.sort.equals(FacetParams.FACET_SORT_INDEX)) continue;
+
+ @SuppressWarnings("unchecked") // generic array's are annoying
List<String>[] tmp = (List<String>[]) new List[rb.shards.length];
dff._toRefine = tmp;
ShardFacetCount[] counts = dff.getCountSorted();
- int ntop = Math.min(counts.length, dff.offset + dff.limit);
+ int ntop = Math.min(counts.length, dff.limit >= 0 ? dff.offset + dff.limit : Integer.MAX_VALUE);
long smallestCount = counts.length == 0 ? 0 : counts[ntop-1].count;
for (int i=0; i<counts.length; i++) {
@@ -313,8 +429,11 @@ public class FacetComponent extends Sear
if (i<ntop) {
// automatically flag the top values for refinement
+ // this should always be true for facet.sort=index
needRefinement = true;
} else {
+ // this logic should only be invoked for facet.sort=index (for now)
+
// calculate the maximum value that this term may have
// and if it is >= smallestCount, then flag for refinement
long maxCount = sfc.count;
@@ -422,13 +541,32 @@ public class FacetComponent extends Sear
counts = dff.getLexSorted();
}
- int end = dff.limit < 0 ? counts.length : Math.min(dff.offset + dff.limit, counts.length);
- for (int i=dff.offset; i<end; i++) {
- if (counts[i].count < dff.minCount) {
- if (countSorted) break; // if sorted by count, we can break out of loop early
- else continue;
+ if (countSorted) {
+ int end = dff.limit < 0 ? counts.length : Math.min(dff.offset + dff.limit, counts.length);
+ for (int i=dff.offset; i<end; i++) {
+ if (counts[i].count < dff.minCount) {
+ break;
+ }
+ fieldCounts.add(counts[i].name, num(counts[i].count));
+ }
+ } else {
+ int off = dff.offset;
+ int lim = dff.limit >= 0 ? dff.limit : Integer.MAX_VALUE;
+
+ // index order...
+ for (int i=0; i<counts.length; i++) {
+ long count = counts[i].count;
+ if (count < dff.minCount) continue;
+ if (off > 0) {
+ off--;
+ continue;
+ }
+ if (lim <= 0) {
+ break;
+ }
+ lim--;
+ fieldCounts.add(counts[i].name, num(count));
}
- fieldCounts.add(counts[i].name, num(counts[i].count));
}
if (dff.missing) {
@@ -436,9 +574,8 @@ public class FacetComponent extends Sear
}
}
- // TODO: facet dates & numbers
- facet_counts.add("facet_dates", new SimpleOrderedMap());
- facet_counts.add("facet_ranges", new SimpleOrderedMap());
+ facet_counts.add("facet_dates", fi.dateFacets);
+ facet_counts.add("facet_ranges", fi.rangeFacets);
rb.rsp.add("facet_counts", facet_counts);
@@ -490,8 +627,14 @@ public class FacetComponent extends Sear
* <b>This API is experimental and subject to change</b>
*/
public static class FacetInfo {
+
public LinkedHashMap<String,QueryFacet> queryFacets;
public LinkedHashMap<String,DistribFieldFacet> facets;
+ public SimpleOrderedMap<SimpleOrderedMap<Object>> dateFacets
+ = new SimpleOrderedMap<SimpleOrderedMap<Object>>();
+ public SimpleOrderedMap<SimpleOrderedMap<Object>> rangeFacets
+ = new SimpleOrderedMap<SimpleOrderedMap<Object>>();
+
public List<String> exceptionList;
void parse(SolrParams params, ResponseBuilder rb) {
@@ -631,7 +774,8 @@ public class FacetComponent extends Sear
public HashMap<String,ShardFacetCount> counts = new HashMap<String,ShardFacetCount>(128);
public int termNum;
- public int initialLimit; // how many terms requested in first phase
+ public int initialLimit; // how many terms requested in first phase
+ public int initialMincount; // mincount param sent to each shard
public boolean needRefinements;
public ShardFacetCount[] countSorted;
@@ -671,11 +815,10 @@ public class FacetComponent extends Sear
}
}
- // the largest possible missing term is 0 if we received less
- // than the number requested (provided mincount==0 like it should be for
- // a shard request)
+ // the largest possible missing term is initialMincount if we received less
+ // than the number requested.
if (numRequested<0 || numRequested != 0 && numReceived < numRequested) {
- last = 0;
+ last = initialMincount;
}
missingMaxPossible += last;
Modified: lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/handler/component/SpellCheckComponent.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/handler/component/SpellCheckComponent.java?rev=1101062&r1=1101061&r2=1101062&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/handler/component/SpellCheckComponent.java (original)
+++ lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/handler/component/SpellCheckComponent.java Mon May 9 15:24:04 2011
@@ -162,7 +162,7 @@ public class SpellCheckComponent extends
} else {
throw new SolrException(SolrException.ErrorCode.NOT_FOUND,
- "Specified dictionary does not exist.");
+ "Specified dictionary does not exist: " + getDictionaryName(params));
}
}
}
Modified: lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/handler/component/TermVectorComponent.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/handler/component/TermVectorComponent.java?rev=1101062&r1=1101061&r2=1101062&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/handler/component/TermVectorComponent.java (original)
+++ lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/handler/component/TermVectorComponent.java Mon May 9 15:24:04 2011
@@ -208,7 +208,7 @@ public class TermVectorComponent extends
if (keyField != null) {
Document document = reader.document(docId, fieldSelector);
- Fieldable uniqId = document.getField(uniqFieldName);
+ Fieldable uniqId = document.getFieldable(uniqFieldName);
String uniqVal = null;
if (uniqId != null) {
uniqVal = keyField.getType().storedToReadable(uniqId);
Modified: lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java?rev=1101062&r1=1101061&r2=1101062&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java (original)
+++ lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java Mon May 9 15:24:04 2011
@@ -401,13 +401,24 @@ public class DefaultSolrHighlighter exte
private void doHighlightingByHighlighter( Query query, SolrQueryRequest req, NamedList docSummaries,
int docId, Document doc, String fieldName ) throws IOException {
+ final SolrIndexSearcher searcher = req.getSearcher();
+ final IndexSchema schema = searcher.getSchema();
+
+ // TODO: Currently in trunk highlighting numeric fields is broken (Lucene) -
+ // so we disable them until fixed (see LUCENE-3080)!
+ // BEGIN: Hack
+ final SchemaField schemaField = schema.getFieldOrNull(fieldName);
+ if (schemaField != null && (
+ (schemaField.getType() instanceof org.apache.solr.schema.TrieField) ||
+ (schemaField.getType() instanceof org.apache.solr.schema.TrieDateField)
+ )) return;
+ // END: Hack
+
SolrParams params = req.getParams();
String[] docTexts = doc.getValues(fieldName);
// according to Document javadoc, doc.getValues() never returns null. check empty instead of null
if (docTexts.length == 0) return;
- SolrIndexSearcher searcher = req.getSearcher();
- IndexSchema schema = searcher.getSchema();
TokenStream tstream = null;
int numFragments = getMaxSnippets(fieldName, params);
boolean mergeContiguousFragments = isMergeContiguousFragments(fieldName, params);
@@ -435,12 +446,20 @@ public class DefaultSolrHighlighter exte
// fall back to analyzer
tstream = createAnalyzerTStream(schema, fieldName, docTexts[j]);
}
-
+
+ int maxCharsToAnalyze = params.getFieldInt(fieldName,
+ HighlightParams.MAX_CHARS,
+ Highlighter.DEFAULT_MAX_CHARS_TO_ANALYZE);
+
Highlighter highlighter;
if (Boolean.valueOf(req.getParams().get(HighlightParams.USE_PHRASE_HIGHLIGHTER, "true"))) {
// TODO: this is not always necessary - eventually we would like to avoid this wrap
// when it is not needed.
- tstream = new CachingTokenFilter(tstream);
+ if (maxCharsToAnalyze < 0) {
+ tstream = new CachingTokenFilter(tstream);
+ } else {
+ tstream = new CachingTokenFilter(new OffsetLimitTokenFilter(tstream, maxCharsToAnalyze));
+ }
// get highlighter
highlighter = getPhraseHighlighter(query, fieldName, req, (CachingTokenFilter) tstream);
@@ -453,9 +472,6 @@ public class DefaultSolrHighlighter exte
highlighter = getHighlighter(query, fieldName, req);
}
- int maxCharsToAnalyze = params.getFieldInt(fieldName,
- HighlightParams.MAX_CHARS,
- Highlighter.DEFAULT_MAX_CHARS_TO_ANALYZE);
if (maxCharsToAnalyze < 0) {
highlighter.setMaxDocCharsToAnalyze(docTexts[j].length());
} else {
Modified: lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/response/JSONResponseWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/response/JSONResponseWriter.java?rev=1101062&r1=1101061&r2=1101062&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/response/JSONResponseWriter.java (original)
+++ lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/response/JSONResponseWriter.java Mon May 9 15:24:04 2011
@@ -316,7 +316,8 @@ class JSONWriter extends TextResponseWri
if( idx > 0 ) {
writeArraySeparator();
}
-
+
+ indent();
writeMapOpener(doc.size());
incLevel();
Modified: lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/response/PageTool.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/response/PageTool.java?rev=1101062&r1=1101061&r2=1101062&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/response/PageTool.java (original)
+++ lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/response/PageTool.java Mon May 9 15:24:04 2011
@@ -19,8 +19,10 @@ package org.apache.solr.response;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.search.DocList;
import org.apache.solr.search.DocSlice;
import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.common.SolrException;
public class PageTool {
private long start;
@@ -42,10 +44,16 @@ public class PageTool {
DocSlice doc_slice = (DocSlice) docs;
results_found = doc_slice.matches();
start = doc_slice.offset();
- } else {
+ } else if(docs instanceof ResultContext) {
+ DocList dl = ((ResultContext) docs).docs;
+ results_found = dl.matches();
+ start = dl.offset();
+ } else if(docs instanceof SolrDocumentList) {
SolrDocumentList doc_list = (SolrDocumentList) docs;
results_found = doc_list.getNumFound();
start = doc_list.getStart();
+ } else {
+ throw new SolrException(SolrException.ErrorCode.UNKNOWN, "Unknown response type "+docs+". Expected one of DocSlice, ResultContext or SolrDocumentList");
}
}
Modified: lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/response/transform/ValueSourceAugmenter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/response/transform/ValueSourceAugmenter.java?rev=1101062&r1=1101061&r2=1101062&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/response/transform/ValueSourceAugmenter.java (original)
+++ lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/response/transform/ValueSourceAugmenter.java Mon May 9 15:24:04 2011
@@ -90,9 +90,10 @@ public class ValueSourceAugmenter extend
}
int localId = docid - rcontext.docBase;
- float val = values.floatVal(localId); // TODO: handle all types -- see: SOLR-2443
-
- doc.setField( name, val );
+ Object val = values.objectVal(localId);
+ if (val != null) {
+ doc.setField( name, val );
+ }
} catch (IOException e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "exception at docid " + docid + " for valuesource " + valueSource, e, false);
}
Modified: lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/DateField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/DateField.java?rev=1101062&r1=1101061&r2=1101062&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/DateField.java (original)
+++ lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/DateField.java Mon May 9 15:24:04 2011
@@ -487,6 +487,17 @@ class DateFieldSource extends FieldCache
}
@Override
+ public Object objectVal(int doc) {
+ int ord=termsIndex.getOrd(doc);
+ if (ord == 0) {
+ return null;
+ } else {
+ BytesRef br = termsIndex.lookup(ord, new BytesRef());
+ return ft.toObject(null, br);
+ }
+ }
+
+ @Override
public String toString(int doc) {
return description() + '=' + intVal(doc);
}
Modified: lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/IndexSchema.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/IndexSchema.java?rev=1101062&r1=1101061&r2=1101062&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/IndexSchema.java (original)
+++ lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/IndexSchema.java Mon May 9 15:24:04 2011
@@ -797,19 +797,23 @@ public final class IndexSchema {
NamedNodeMap attrs = node.getAttributes();
String analyzerName = DOMUtil.getAttr(attrs,"class");
if (analyzerName != null) {
- // No need to be core-aware as Analyzers are not in the core-aware list
- final Class<? extends Analyzer> clazz = loader.findClass(analyzerName).asSubclass(Analyzer.class);
try {
+ // No need to be core-aware as Analyzers are not in the core-aware list
+ final Class<? extends Analyzer> clazz = loader.findClass
+ (analyzerName).asSubclass(Analyzer.class);
+
try {
- // first try to use a ctor with version parameter (needed for many new Analyzers that have no default one anymore)
+ // first try to use a ctor with version parameter
+ // (needed for many new Analyzers that have no default one anymore)
Constructor<? extends Analyzer> cnstr = clazz.getConstructor(Version.class);
final String matchVersionStr = DOMUtil.getAttr(attrs, LUCENE_MATCH_VERSION_PARAM);
final Version luceneMatchVersion = (matchVersionStr == null) ?
solrConfig.luceneMatchVersion : Config.parseLuceneVersionString(matchVersionStr);
if (luceneMatchVersion == null) {
- throw new SolrException( SolrException.ErrorCode.SERVER_ERROR,
- "Configuration Error: Analyzer '" + clazz.getName() +
- "' needs a 'luceneMatchVersion' parameter");
+ throw new SolrException
+ ( SolrException.ErrorCode.SERVER_ERROR,
+ "Configuration Error: Analyzer '" + clazz.getName() +
+ "' needs a 'luceneMatchVersion' parameter");
}
return cnstr.newInstance(luceneMatchVersion);
} catch (NoSuchMethodException nsme) {
@@ -817,8 +821,9 @@ public final class IndexSchema {
return clazz.newInstance();
}
} catch (Exception e) {
+ log.error("Cannot load analyzer: "+analyzerName, e);
throw new SolrException( SolrException.ErrorCode.SERVER_ERROR,
- "Cannot load analyzer: "+analyzerName );
+ "Cannot load analyzer: "+analyzerName, e );
}
}
Modified: lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/RandomSortField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/RandomSortField.java?rev=1101062&r1=1101061&r2=1101062&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/RandomSortField.java (original)
+++ lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/RandomSortField.java Mon May 9 15:24:04 2011
@@ -28,6 +28,7 @@ import org.apache.lucene.util.ReaderUtil
import org.apache.solr.response.TextResponseWriter;
import org.apache.solr.search.QParser;
import org.apache.solr.search.function.DocValues;
+import org.apache.solr.search.function.IntDocValues;
import org.apache.solr.search.function.ValueSource;
/**
@@ -157,37 +158,12 @@ public class RandomSortField extends Fie
@Override
public DocValues getValues(Map context, final AtomicReaderContext readerContext) throws IOException {
- return new DocValues() {
+ return new IntDocValues(this) {
private final int seed = getSeed(field, readerContext);
@Override
- public float floatVal(int doc) {
- return (float)hash(doc+seed);
- }
-
- @Override
public int intVal(int doc) {
return hash(doc+seed);
}
-
- @Override
- public long longVal(int doc) {
- return (long)hash(doc+seed);
- }
-
- @Override
- public double doubleVal(int doc) {
- return (double)hash(doc+seed);
- }
-
- @Override
- public String strVal(int doc) {
- return Integer.toString(hash(doc+seed));
- }
-
- @Override
- public String toString(int doc) {
- return description() + '=' + intVal(doc);
- }
};
}
Modified: lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/SchemaField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/SchemaField.java?rev=1101062&r1=1101061&r2=1101062&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/SchemaField.java (original)
+++ lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/SchemaField.java Mon May 9 15:24:04 2011
@@ -19,7 +19,6 @@ package org.apache.solr.schema;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
-import org.apache.lucene.document.Field;
import org.apache.lucene.document.Fieldable;
import org.apache.lucene.search.SortField;
import org.apache.solr.search.QParser;
Modified: lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/SortableDoubleField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/SortableDoubleField.java?rev=1101062&r1=1101061&r2=1101062&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/SortableDoubleField.java (original)
+++ lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/SortableDoubleField.java Mon May 9 15:24:04 2011
@@ -149,6 +149,12 @@ class SortableDoubleFieldSource extends
}
@Override
+ public Object objectVal(int doc) {
+ int ord=termsIndex.getOrd(doc);
+ return ord==0 ? null : NumberUtils.SortableStr2double(termsIndex.lookup(ord, spare));
+ }
+
+ @Override
public String toString(int doc) {
return description() + '=' + doubleVal(doc);
}
Modified: lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/SortableFloatField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/SortableFloatField.java?rev=1101062&r1=1101061&r2=1101062&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/SortableFloatField.java (original)
+++ lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/SortableFloatField.java Mon May 9 15:24:04 2011
@@ -154,6 +154,12 @@ class SortableFloatFieldSource extends F
}
@Override
+ public Object objectVal(int doc) {
+ int ord=termsIndex.getOrd(doc);
+ return ord==0 ? null : NumberUtils.SortableStr2float(termsIndex.lookup(ord, spare));
+ }
+
+ @Override
public ValueFiller getValueFiller() {
return new ValueFiller() {
private final MutableValueFloat mval = new MutableValueFloat();
Modified: lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/SortableIntField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/SortableIntField.java?rev=1101062&r1=1101061&r2=1101062&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/SortableIntField.java (original)
+++ lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/SortableIntField.java Mon May 9 15:24:04 2011
@@ -156,6 +156,12 @@ class SortableIntFieldSource extends Fie
}
@Override
+ public Object objectVal(int doc) {
+ int ord=termsIndex.getOrd(doc);
+ return ord==0 ? null : NumberUtils.SortableStr2int(termsIndex.lookup(ord, spare));
+ }
+
+ @Override
public ValueFiller getValueFiller() {
return new ValueFiller() {
private final MutableValueInt mval = new MutableValueInt();
Modified: lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/SortableLongField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/SortableLongField.java?rev=1101062&r1=1101061&r2=1101062&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/SortableLongField.java (original)
+++ lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/SortableLongField.java Mon May 9 15:24:04 2011
@@ -150,6 +150,12 @@ class SortableLongFieldSource extends Fi
}
@Override
+ public Object objectVal(int doc) {
+ int ord=termsIndex.getOrd(doc);
+ return ord==0 ? null : NumberUtils.SortableStr2long(termsIndex.lookup(ord, spare));
+ }
+
+ @Override
public String toString(int doc) {
return description() + '=' + longVal(doc);
}
Modified: lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/StrFieldSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/StrFieldSource.java?rev=1101062&r1=1101061&r2=1101062&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/StrFieldSource.java (original)
+++ lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/StrFieldSource.java Mon May 9 15:24:04 2011
@@ -19,9 +19,11 @@ package org.apache.solr.schema;
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
import org.apache.lucene.util.BytesRef;
+import org.apache.noggit.CharArr;
import org.apache.solr.search.function.DocValues;
import org.apache.solr.search.function.FieldCacheSource;
import org.apache.solr.search.function.StringIndexDocValues;
+import org.apache.solr.util.ByteUtils;
import java.io.IOException;
import java.util.Map;
@@ -40,33 +42,13 @@ public class StrFieldSource extends Fiel
@Override
public DocValues getValues(Map context, AtomicReaderContext readerContext) throws IOException {
return new StringIndexDocValues(this, readerContext, field) {
+
@Override
protected String toTerm(String readableValue) {
return readableValue;
}
@Override
- public float floatVal(int doc) {
- return (float)intVal(doc);
- }
-
- @Override
- public int intVal(int doc) {
- int ord=termsIndex.getOrd(doc);
- return ord;
- }
-
- @Override
- public long longVal(int doc) {
- return (long)intVal(doc);
- }
-
- @Override
- public double doubleVal(int doc) {
- return (double)intVal(doc);
- }
-
- @Override
public int ordVal(int doc) {
return termsIndex.getOrd(doc);
}
@@ -77,13 +59,8 @@ public class StrFieldSource extends Fiel
}
@Override
- public String strVal(int doc) {
- int ord=termsIndex.getOrd(doc);
- if (ord == 0) {
- return null;
- } else {
- return termsIndex.lookup(ord, new BytesRef()).utf8ToString();
- }
+ public Object objectVal(int doc) {
+ return strVal(doc);
}
@Override
Modified: lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/TrieDateField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/TrieDateField.java?rev=1101062&r1=1101061&r2=1101062&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/TrieDateField.java (original)
+++ lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/TrieDateField.java Mon May 9 15:24:04 2011
@@ -18,210 +18,125 @@
package org.apache.solr.schema;
import org.apache.noggit.CharArr;
-import org.apache.solr.common.SolrException;
-import org.apache.solr.analysis.CharFilterFactory;
-import org.apache.solr.analysis.TokenFilterFactory;
-import org.apache.solr.analysis.TokenizerChain;
-import org.apache.solr.analysis.TrieTokenizerFactory;
-import org.apache.solr.search.function.*;
+import org.apache.solr.search.function.ValueSource;
import org.apache.solr.search.QParser;
import org.apache.solr.response.TextResponseWriter;
import org.apache.lucene.document.Fieldable;
-import org.apache.lucene.document.Field;
import org.apache.lucene.search.SortField;
-import org.apache.lucene.search.FieldCache;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.NumericRangeQuery;
-import org.apache.lucene.search.cache.CachedArrayCreator;
-import org.apache.lucene.search.cache.LongValuesCreator;
import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.NumericUtils;
-import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.NumericTokenStream;
import java.util.Map;
import java.util.Date;
import java.io.IOException;
public class TrieDateField extends DateField {
- protected int precisionStepArg = TrieField.DEFAULT_PRECISION_STEP; // the one passed in or defaulted
- protected int precisionStep = precisionStepArg; // normalized
+
+ final TrieField wrappedField = new TrieField() {{
+ type = TrieTypes.DATE;
+ }};
@Override
protected void init(IndexSchema schema, Map<String, String> args) {
- String p = args.remove("precisionStep");
- if (p != null) {
- precisionStepArg = Integer.parseInt(p);
- }
- // normalize the precisionStep
- precisionStep = precisionStepArg;
- if (precisionStep<=0 || precisionStep>=64) precisionStep=Integer.MAX_VALUE;
-
- CharFilterFactory[] filterFactories = new CharFilterFactory[0];
- TokenFilterFactory[] tokenFilterFactories = new TokenFilterFactory[0];
- analyzer = new TokenizerChain(filterFactories, new TrieTokenizerFactory(TrieField.TrieTypes.DATE, precisionStep), tokenFilterFactories);
- // for query time we only need one token, so we use the biggest possible precisionStep:
- queryAnalyzer = new TokenizerChain(filterFactories, new TrieTokenizerFactory(TrieField.TrieTypes.DATE, Integer.MAX_VALUE), tokenFilterFactories);
+ wrappedField.init(schema, args);
+ analyzer = wrappedField.analyzer;
+ queryAnalyzer = wrappedField.queryAnalyzer;
}
@Override
public Date toObject(Fieldable f) {
- byte[] arr = f.getBinaryValue();
- if (arr==null) throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,TrieField.badFieldString(f));
- return new Date(TrieFieldHelper.toLong(arr));
+ return (Date) wrappedField.toObject(f);
}
@Override
public Object toObject(SchemaField sf, BytesRef term) {
- return new Date(NumericUtils.prefixCodedToLong(term));
+ return wrappedField.toObject(sf, term);
}
@Override
public SortField getSortField(SchemaField field, boolean top) {
- field.checkSortability();
-
- int flags = CachedArrayCreator.CACHE_VALUES_AND_BITS;
- boolean sortMissingLast = field.sortMissingLast();
- boolean sortMissingFirst = field.sortMissingFirst();
-
- Object missingValue = null;
- if( sortMissingLast ) {
- missingValue = top ? Long.MIN_VALUE : Long.MAX_VALUE;
- } else if( sortMissingFirst ) {
- missingValue = top ? Long.MAX_VALUE : Long.MIN_VALUE;
- }
- return new SortField(new LongValuesCreator(field.getName(), FieldCache.NUMERIC_UTILS_LONG_PARSER, flags), top).setMissingValue(missingValue);
+ return wrappedField.getSortField(field, top);
}
@Override
public ValueSource getValueSource(SchemaField field, QParser parser) {
- field.checkFieldCacheSource(parser);
- return new TrieDateFieldSource( new LongValuesCreator( field.getName(), FieldCache.NUMERIC_UTILS_LONG_PARSER, CachedArrayCreator.CACHE_VALUES_AND_BITS ));
+ return wrappedField.getValueSource(field, parser);
+ }
+
+ /**
+ * @return the precisionStep used to index values into the field
+ */
+ public int getPrecisionStep() {
+ return wrappedField.getPrecisionStep();
}
+
@Override
public void write(TextResponseWriter writer, String name, Fieldable f) throws IOException {
- byte[] arr = f.getBinaryValue();
- if (arr==null) {
- writer.writeStr(name, TrieField.badFieldString(f),true);
- return;
- }
-
- writer.writeDate(name,new Date(TrieFieldHelper.toLong(arr)));
+ wrappedField.write(writer, name, f);
}
@Override
public boolean isTokenized() {
- return true;
+ return wrappedField.isTokenized();
}
- /**
- * @return the precisionStep used to index values into the field
- */
- public int getPrecisionStep() {
- return precisionStepArg;
+ @Override
+ public boolean multiValuedFieldCache() {
+ return wrappedField.multiValuedFieldCache();
}
-
-
@Override
public String storedToReadable(Fieldable f) {
- return toExternal(f);
+ return wrappedField.storedToReadable(f);
}
@Override
public String readableToIndexed(String val) {
- // TODO: Numeric should never be handled as String, that may break in future lucene versions! Change to use BytesRef for term texts!
- BytesRef bytes = new BytesRef(NumericUtils.BUF_SIZE_LONG);
- NumericUtils.longToPrefixCoded(super.parseMath(null, val).getTime(), 0, bytes);
- return bytes.utf8ToString();
+ return wrappedField.readableToIndexed(val);
}
@Override
public String toInternal(String val) {
- return readableToIndexed(val);
+ return wrappedField.toInternal(val);
}
@Override
public String toExternal(Fieldable f) {
- byte[] arr = f.getBinaryValue();
- if (arr==null) return TrieField.badFieldString(f);
- return super.toExternal(new Date(TrieFieldHelper.toLong(arr)));
+ return wrappedField.toExternal(f);
}
@Override
public String indexedToReadable(String _indexedForm) {
- final BytesRef indexedForm = new BytesRef(_indexedForm);
- return super.toExternal( new Date(NumericUtils.prefixCodedToLong(indexedForm)) );
+ return wrappedField.indexedToReadable(_indexedForm);
}
@Override
public void indexedToReadable(BytesRef input, CharArr out) {
- String ext = super.toExternal( new Date(NumericUtils.prefixCodedToLong(input)) );
- out.write(ext);
+ wrappedField.indexedToReadable(input, out);
}
@Override
public String storedToIndexed(Fieldable f) {
- // TODO: optimize to remove redundant string conversion
- return readableToIndexed(storedToReadable(f));
+ return wrappedField.storedToIndexed(f);
}
@Override
public Fieldable createField(SchemaField field, Object value, float boost) {
- boolean indexed = field.indexed();
- boolean stored = field.stored();
-
- if (!indexed && !stored) {
- if (log.isTraceEnabled())
- log.trace("Ignoring unindexed/unstored field: " + field);
- return null;
- }
-
- int ps = precisionStep;
-
- byte[] arr=null;
- TokenStream ts=null;
-
- long time = (value instanceof Date)
- ? ((Date)value).getTime()
- : super.parseMath(null, value.toString()).getTime();
-
- if (stored) arr = TrieFieldHelper.toArr(time);
- if (indexed) ts = new NumericTokenStream(ps).setLongValue(time);
-
- Field f;
- if (stored) {
- f = new Field(field.getName(), arr);
- if (indexed) f.setTokenStream(ts);
- } else {
- f = new Field(field.getName(), ts);
- }
-
- // term vectors aren't supported
-
- f.setOmitNorms(field.omitNorms());
- f.setOmitTermFreqAndPositions(field.omitTf());
- f.setBoost(boost);
- return f;
+ return wrappedField.createField(field, value, boost);
}
@Override
public Query getRangeQuery(QParser parser, SchemaField field, String min, String max, boolean minInclusive, boolean maxInclusive) {
- return getRangeQuery(parser, field,
- min==null ? null : super.parseMath(null,min),
- max==null ? null : super.parseMath(null,max),
- minInclusive, maxInclusive);
+ return wrappedField.getRangeQuery(parser, field, min, max, minInclusive, maxInclusive);
}
@Override
public Query getRangeQuery(QParser parser, SchemaField sf, Date min, Date max, boolean minInclusive, boolean maxInclusive) {
- int ps = precisionStep;
- Query query = NumericRangeQuery.newLongRange(sf.getName(), ps,
+ return NumericRangeQuery.newLongRange(sf.getName(), wrappedField.precisionStep,
min == null ? null : min.getTime(),
max == null ? null : max.getTime(),
minInclusive, maxInclusive);
-
- return query;
}
}
Modified: lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/TrieField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/TrieField.java?rev=1101062&r1=1101061&r2=1101062&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/TrieField.java (original)
+++ lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/schema/TrieField.java Mon May 9 15:24:04 2011
@@ -17,6 +17,8 @@
package org.apache.solr.schema;
import org.apache.lucene.document.Fieldable;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.NumericField;
import org.apache.lucene.search.*;
import org.apache.lucene.search.cache.CachedArrayCreator;
import org.apache.lucene.search.cache.DoubleValuesCreator;
@@ -40,17 +42,17 @@ import java.util.Map;
import java.util.Date;
/**
- * Provides field types to support for Lucene's Trie Range Queries.
+ * Provides field types to support for Lucene's {@link NumericField}.
* See {@link org.apache.lucene.search.NumericRangeQuery} for more details.
* It supports integer, float, long, double and date types.
* <p/>
* For each number being added to this field, multiple terms are generated as per the algorithm described in the above
- * link. The possible number of terms increases dramatically with higher precision steps (factor 2^precisionStep). For
+ * link. The possible number of terms increases dramatically with lower precision steps. For
* the fast range search to work, trie fields must be indexed.
* <p/>
* Trie fields are sortable in numerical order and can be used in function queries.
* <p/>
- * Note that if you use a precisionStep of 32 for int/float and 64 for long/double, then multiple terms will not be
+ * Note that if you use a precisionStep of 32 for int/float and 64 for long/double/date, then multiple terms will not be
* generated, range search will be no faster than any other number field, but sorting will still be possible.
*
* @version $Id$
@@ -101,21 +103,28 @@ public class TrieField extends FieldType
@Override
public Object toObject(Fieldable f) {
- byte[] arr = f.getBinaryValue();
- if (arr==null) return badFieldString(f);
- switch (type) {
- case INTEGER:
- return TrieFieldHelper.toInt(arr);
- case FLOAT:
- return TrieFieldHelper.toFloat(arr);
- case LONG:
- return TrieFieldHelper.toLong(arr);
- case DOUBLE:
- return TrieFieldHelper.toDouble(arr);
- case DATE:
- return new Date(TrieFieldHelper.toLong(arr));
- default:
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown type for trie field: " + f.name());
+ if (f instanceof NumericField) {
+ final Number val = ((NumericField) f).getNumericValue();
+ if (val==null) return badFieldString(f);
+ return (type == TrieTypes.DATE) ? new Date(val.longValue()) : val;
+ } else {
+ // the following code is "deprecated" and only to support pre-3.2 indexes using the old BinaryField encoding:
+ final byte[] arr = f.getBinaryValue();
+ if (arr==null) return badFieldString(f);
+ switch (type) {
+ case INTEGER:
+ return toInt(arr);
+ case FLOAT:
+ return Float.intBitsToFloat(toInt(arr));
+ case LONG:
+ return toLong(arr);
+ case DOUBLE:
+ return Double.longBitsToDouble(toLong(arr));
+ case DATE:
+ return new Date(toLong(arr));
+ default:
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown type for trie field: " + f.name());
+ }
}
}
@@ -198,30 +207,7 @@ public class TrieField extends FieldType
@Override
public void write(TextResponseWriter writer, String name, Fieldable f) throws IOException {
- byte[] arr = f.getBinaryValue();
- if (arr==null) {
- writer.writeStr(name, badFieldString(f),true);
- return;
- }
- switch (type) {
- case INTEGER:
- writer.writeInt(name,TrieFieldHelper.toInt(arr));
- break;
- case FLOAT:
- writer.writeFloat(name,TrieFieldHelper.toFloat(arr));
- break;
- case LONG:
- writer.writeLong(name,TrieFieldHelper.toLong(arr));
- break;
- case DOUBLE:
- writer.writeDouble(name,TrieFieldHelper.toDouble(arr));
- break;
- case DATE:
- writer.writeDate(name,new Date(TrieFieldHelper.toLong(arr)));
- break;
- default:
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown type for trie field: " + f.name());
- }
+ writer.writeVal(name, toObject(f));
}
@Override
@@ -290,6 +276,17 @@ public class TrieField extends FieldType
return query;
}
+ @Deprecated
+ static int toInt(byte[] arr) {
+ return (arr[0]<<24) | ((arr[1]&0xff)<<16) | ((arr[2]&0xff)<<8) | (arr[3]&0xff);
+ }
+
+ @Deprecated
+ static long toLong(byte[] arr) {
+ int high = (arr[0]<<24) | ((arr[1]&0xff)<<16) | ((arr[2]&0xff)<<8) | (arr[3]&0xff);
+ int low = (arr[4]<<24) | ((arr[5]&0xff)<<16) | ((arr[6]&0xff)<<8) | (arr[7]&0xff);
+ return (((long)high)<<32) | (low&0x0ffffffffL);
+ }
@Override
public String storedToReadable(Fieldable f) {
@@ -341,22 +338,9 @@ public class TrieField extends FieldType
@Override
public String toExternal(Fieldable f) {
- byte[] arr = f.getBinaryValue();
- if (arr==null) return badFieldString(f);
- switch (type) {
- case INTEGER:
- return Integer.toString(TrieFieldHelper.toInt(arr));
- case FLOAT:
- return Float.toString(TrieFieldHelper.toFloat(arr));
- case LONG:
- return Long.toString(TrieFieldHelper.toLong(arr));
- case DOUBLE:
- return Double.toString(TrieFieldHelper.toDouble(arr));
- case DATE:
- return dateField.formatDate(new Date(TrieFieldHelper.toLong(arr)));
- default:
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown type for trie field: " + f.name());
- }
+ return (type == TrieTypes.DATE)
+ ? dateField.toExternal((Date) toObject(f))
+ : toObject(f).toString();
}
@Override
@@ -372,7 +356,7 @@ public class TrieField extends FieldType
case DOUBLE:
return Double.toString( NumericUtils.sortableLongToDouble(NumericUtils.prefixCodedToLong(indexedForm)) );
case DATE:
- return dateField.formatDate( new Date(NumericUtils.prefixCodedToLong(indexedForm)) );
+ return dateField.toExternal( new Date(NumericUtils.prefixCodedToLong(indexedForm)) );
default:
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown type for trie field: " + type);
}
@@ -397,7 +381,7 @@ public class TrieField extends FieldType
s = Double.toString( NumericUtils.sortableLongToDouble(NumericUtils.prefixCodedToLong(indexedForm)) );
break;
case DATE:
- s = dateField.formatDate( new Date(NumericUtils.prefixCodedToLong(indexedForm)) );
+ s = dateField.toExternal( new Date(NumericUtils.prefixCodedToLong(indexedForm)) );
break;
default:
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown type for trie field: " + type);
@@ -426,59 +410,117 @@ public class TrieField extends FieldType
@Override
public String storedToIndexed(Fieldable f) {
- // TODO: optimize to remove redundant string conversion
- return readableToIndexed(storedToReadable(f));
+ final BytesRef bytes = new BytesRef(NumericUtils.BUF_SIZE_LONG);
+ if (f instanceof NumericField) {
+ final Number val = ((NumericField) f).getNumericValue();
+ if (val==null)
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Invalid field contents: "+f.name());
+ switch (type) {
+ case INTEGER:
+ NumericUtils.intToPrefixCoded(val.intValue(), 0, bytes);
+ break;
+ case FLOAT:
+ NumericUtils.intToPrefixCoded(NumericUtils.floatToSortableInt(val.floatValue()), 0, bytes);
+ break;
+ case LONG: //fallthrough!
+ case DATE:
+ NumericUtils.longToPrefixCoded(val.longValue(), 0, bytes);
+ break;
+ case DOUBLE:
+ NumericUtils.longToPrefixCoded(NumericUtils.doubleToSortableLong(val.doubleValue()), 0, bytes);
+ break;
+ default:
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown type for trie field: " + f.name());
+ }
+ } else {
+ // the following code is "deprecated" and only to support pre-3.2 indexes using the old BinaryField encoding:
+ final byte[] arr = f.getBinaryValue();
+ if (arr==null)
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Invalid field contents: "+f.name());
+ switch (type) {
+ case INTEGER:
+ NumericUtils.intToPrefixCoded(toInt(arr), 0, bytes);
+ break;
+ case FLOAT: {
+ // WARNING: Code Duplication! Keep in sync with o.a.l.util.NumericUtils!
+ // copied from NumericUtils to not convert to/from float two times
+ // code in next 2 lines is identical to: int v = NumericUtils.floatToSortableInt(Float.intBitsToFloat(toInt(arr)));
+ int v = toInt(arr);
+ if (v<0) v ^= 0x7fffffff;
+ NumericUtils.intToPrefixCoded(v, 0, bytes);
+ break;
+ }
+ case LONG: //fallthrough!
+ case DATE:
+ NumericUtils.longToPrefixCoded(toLong(arr), 0, bytes);
+ break;
+ case DOUBLE: {
+ // WARNING: Code Duplication! Keep in sync with o.a.l.util.NumericUtils!
+ // copied from NumericUtils to not convert to/from double two times
+ // code in next 2 lines is identical to: long v = NumericUtils.doubleToSortableLong(Double.longBitsToDouble(toLong(arr)));
+ long v = toLong(arr);
+ if (v<0) v ^= 0x7fffffffffffffffL;
+ NumericUtils.longToPrefixCoded(v, 0, bytes);
+ break;
+ }
+ default:
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown type for trie field: " + f.name());
+ }
+ }
+ return bytes.utf8ToString();
}
@Override
public Fieldable createField(SchemaField field, Object value, float boost) {
- TrieFieldHelper.FieldInfo info = new TrieFieldHelper.FieldInfo();
- info.index = field.indexed();
- info.store = field.stored();
- info.precisionStep = precisionStep;
- info.omitNorms = field.omitNorms();
- info.omitTF = field.omitTf();
-
- if (!info.index && !info.store) {
+ boolean indexed = field.indexed();
+ boolean stored = field.stored();
+
+ if (!indexed && !stored) {
if (log.isTraceEnabled())
log.trace("Ignoring unindexed/unstored field: " + field);
return null;
}
+ final NumericField f = new NumericField(field.getName(), precisionStep, stored ? Field.Store.YES : Field.Store.NO, indexed);
switch (type) {
case INTEGER:
int i = (value instanceof Number)
? ((Number)value).intValue()
: Integer.parseInt(value.toString());
- return TrieFieldHelper.createIntField(field.getName(), i, info, boost);
-
+ f.setIntValue(i);
+ break;
case FLOAT:
- float f = (value instanceof Number)
+ float fl = (value instanceof Number)
? ((Number)value).floatValue()
: Float.parseFloat(value.toString());
- return TrieFieldHelper.createFloatField(field.getName(), f, info, boost);
-
+ f.setFloatValue(fl);
+ break;
case LONG:
long l = (value instanceof Number)
? ((Number)value).longValue()
: Long.parseLong(value.toString());
- return TrieFieldHelper.createLongField(field.getName(), l, info, boost);
-
+ f.setLongValue(l);
+ break;
case DOUBLE:
double d = (value instanceof Number)
? ((Number)value).doubleValue()
: Double.parseDouble(value.toString());
- return TrieFieldHelper.createDoubleField(field.getName(), d, info, boost);
-
+ f.setDoubleValue(d);
+ break;
case DATE:
Date date = (value instanceof Date)
? ((Date)value)
: dateField.parseMath(null, value.toString());
- return TrieFieldHelper.createDateField(field.getName(), date, info, boost);
-
+ f.setLongValue(date.getTime());
+ break;
default:
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown type for trie field: " + type);
}
+
+ f.setOmitNorms(field.omitNorms());
+ f.setOmitTermFreqAndPositions(field.omitTf());
+ f.setBoost(boost);
+ return f;
}
public enum TrieTypes {
@@ -498,14 +540,12 @@ public class TrieField extends FieldType
* that indexes multiple precisions per value.
*/
public static String getMainValuePrefix(FieldType ft) {
- if (ft instanceof TrieDateField) {
- int step = ((TrieDateField)ft).getPrecisionStep();
- if (step <= 0 || step >=64) return null;
- return LONG_PREFIX;
- } else if (ft instanceof TrieField) {
- TrieField trie = (TrieField)ft;
- if (trie.precisionStep == Integer.MAX_VALUE) return null;
-
+ if (ft instanceof TrieDateField)
+ ft = ((TrieDateField) ft).wrappedField;
+ if (ft instanceof TrieField) {
+ final TrieField trie = (TrieField)ft;
+ if (trie.precisionStep == Integer.MAX_VALUE)
+ return null;
switch (trie.type) {
case INTEGER:
case FLOAT:
@@ -539,6 +579,11 @@ class TrieDateFieldSource extends LongFi
}
@Override
+ public Object longToObject(long val) {
+ return new Date(val);
+ }
+
+ @Override
public long externalToLong(String extVal) {
return TrieField.dateField.parseMath(null, extVal).getTime();
}
Modified: lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/ExtendedDismaxQParserPlugin.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/ExtendedDismaxQParserPlugin.java?rev=1101062&r1=1101061&r2=1101062&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/ExtendedDismaxQParserPlugin.java (original)
+++ lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/ExtendedDismaxQParserPlugin.java Mon May 9 15:24:04 2011
@@ -238,6 +238,7 @@ class ExtendedDismaxQParser extends QPar
try {
up.setRemoveStopFilter(!stopwords);
+ up.exceptions = true;
parsedUserQuery = up.parse(mainUserQuery);
if (stopwords && isEmpty(parsedUserQuery)) {
@@ -247,6 +248,7 @@ class ExtendedDismaxQParser extends QPar
}
} catch (Exception e) {
// ignore failure and reparse later after escaping reserved chars
+ up.exceptions = false;
}
if (parsedUserQuery != null && doMinMatched) {
@@ -785,12 +787,19 @@ class ExtendedDismaxQParser extends QPar
RANGE
}
+
+ static final RuntimeException unknownField = new RuntimeException("UnknownField");
+ static {
+ unknownField.fillInStackTrace();
+ }
+
/**
* A subclass of SolrQueryParser that supports aliasing fields for
* constructing DisjunctionMaxQueries.
*/
class ExtendedSolrQueryParser extends SolrQueryParser {
+
/** A simple container for storing alias info
*/
protected class Alias {
@@ -803,6 +812,7 @@ class ExtendedDismaxQParser extends QPar
boolean allowWildcard=true;
int minClauseSize = 0; // minimum number of clauses per phrase query...
// used when constructing boosting part of query via sloppy phrases
+ boolean exceptions; // allow exceptions to be thrown (for example on a missing field)
ExtendedAnalyzer analyzer;
@@ -982,6 +992,15 @@ class ExtendedDismaxQParser extends QPar
return q;
}
} else {
+
+ // verify that a fielded query is actually on a field that exists... if not,
+ // then throw an exception to get us out of here, and we'll treat it like a
+ // literal when we try the escape+re-parse.
+ if (exceptions) {
+ FieldType ft = schema.getFieldTypeNoEx(field);
+ if (ft == null) throw unknownField;
+ }
+
return getQuery();
}
}
Modified: lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/QueryParsing.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/QueryParsing.java?rev=1101062&r1=1101061&r2=1101062&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/QueryParsing.java (original)
+++ lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/QueryParsing.java Mon May 9 15:24:04 2011
@@ -306,7 +306,7 @@ public class QueryParsing {
Boolean top = sp.getSortDirection();
if (null == top) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
- "Can't determine Sort Order: " + sp);
+ "Can't determine a Sort Order (asc or desc) in sort spec " + sp);
}
if (SCORE.equals(field)) {
@@ -638,7 +638,11 @@ public class QueryParsing {
}
String v = val.substring(start,pos);
- return flt ? Double.parseDouble(v) : Long.parseLong(v);
+ if (flt) {
+ return Double.parseDouble(v);
+ } else {
+ return Long.parseLong(v);
+ }
}
double getDouble() throws ParseException {
@@ -693,7 +697,8 @@ public class QueryParsing {
pos++;
while (pos < end) {
ch = val.charAt(pos);
- if (!Character.isJavaIdentifierPart(ch) && ch != '.' && ch != ':') {
+// if (!Character.isJavaIdentifierPart(ch) && ch != '.' && ch != ':') {
+ if (!Character.isJavaIdentifierPart(ch) && ch != '.') {
break;
}
pos++;
Modified: lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/ReturnFields.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/ReturnFields.java?rev=1101062&r1=1101061&r2=1101062&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/ReturnFields.java (original)
+++ lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/ReturnFields.java Mon May 9 15:24:04 2011
@@ -90,12 +90,10 @@ public class ReturnFields
parseFieldList( new String[]{fl}, req);
}
}
- SolrCore.log.info("fields=" + fields + "\t globs="+globs + "\t transformer="+transformer);
}
public ReturnFields(String[] fl, SolrQueryRequest req) {
parseFieldList(fl, req);
- SolrCore.log.info("fields=" + fields + "\t globs="+globs + "\t transformer="+transformer);
}
private void parseFieldList(String[] fl, SolrQueryRequest req) {
@@ -162,7 +160,7 @@ public class ReturnFields
char ch = sp.ch();
if (field != null) {
- if (sp.opt('=')) {
+ if (sp.opt(':')) {
// this was a key, not a field name
key = field;
field = null;
@@ -180,7 +178,7 @@ public class ReturnFields
}
if (key != null) {
- // we read "key = "
+ // we read "key : "
field = sp.getId(null);
ch = sp.ch();
if (field != null && (ch==' ' || ch == ',' || ch==0)) {
Modified: lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/SolrIndexSearcher.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/SolrIndexSearcher.java?rev=1101062&r1=1101061&r2=1101062&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/SolrIndexSearcher.java (original)
+++ lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/SolrIndexSearcher.java Mon May 9 15:24:04 2011
@@ -1603,14 +1603,12 @@ public class SolrIndexSearcher extends I
DocIterator iter = set.iterator();
int base=0;
int end=0;
- int readerIndex = -1;
-
- AtomicReaderContext leaf = null;
+ int readerIndex = 0;
- for (int i = 0; i < leafContexts.length; i++) {
+ while (iter.hasNext()) {
int doc = iter.nextDoc();
while (doc>=end) {
- leaf = leafContexts[i++];
+ AtomicReaderContext leaf = leafContexts[readerIndex++];
base = leaf.docBase;
end = base + leaf.reader.maxDoc();
topCollector.setNextReader(leaf);
Modified: lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/SolrQueryParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/SolrQueryParser.java?rev=1101062&r1=1101061&r2=1101062&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/SolrQueryParser.java (original)
+++ lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/SolrQueryParser.java Mon May 9 15:24:04 2011
@@ -67,7 +67,7 @@ public class SolrQueryParser extends Que
}
public SolrQueryParser(QParser parser, String defaultField, Analyzer analyzer) {
- super(parser.getReq().getCore().getSolrConfig().getLuceneVersion("luceneMatchVersion", Version.LUCENE_30), defaultField, analyzer);
+ super(parser.getReq().getCore().getSolrConfig().luceneMatchVersion, defaultField, analyzer);
this.schema = parser.getReq().getSchema();
this.parser = parser;
this.defaultField = defaultField;
Modified: lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/ValueSourceParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/ValueSourceParser.java?rev=1101062&r1=1101061&r2=1101062&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/ValueSourceParser.java (original)
+++ lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/ValueSourceParser.java Mon May 9 15:24:04 2011
@@ -335,6 +335,15 @@ public abstract class ValueSourceParser
return new StringDistanceFunction(str1, str2, dist);
}
});
+ addParser("field", new ValueSourceParser() {
+ @Override
+ public ValueSource parse(FunctionQParser fp) throws ParseException {
+
+ String fieldName = fp.parseArg();
+ SchemaField f = fp.getReq().getSchema().getField(fieldName);
+ return f.getType().getValueSource(f, fp);
+ }
+ });
addParser(new DoubleParser("rad") {
@Override
@@ -784,7 +793,7 @@ class LongConstValueSource extends Const
@Override
public DocValues getValues(Map context, AtomicReaderContext readerContext) throws IOException {
- return new DocValues() {
+ return new LongDocValues(this) {
@Override
public float floatVal(int doc) {
return fv;
@@ -806,11 +815,6 @@ class LongConstValueSource extends Const
}
@Override
- public String strVal(int doc) {
- return Long.toString(constant);
- }
-
- @Override
public String toString(int doc) {
return description();
}
@@ -892,28 +896,12 @@ abstract class DoubleParser extends Name
@Override
public DocValues getValues(Map context, AtomicReaderContext readerContext) throws IOException {
final DocValues vals = source.getValues(context, readerContext);
- return new DocValues() {
- @Override
- public float floatVal(int doc) {
- return (float)doubleVal(doc);
- }
- @Override
- public int intVal(int doc) {
- return (int)doubleVal(doc);
- }
- @Override
- public long longVal(int doc) {
- return (long)doubleVal(doc);
- }
+ return new DoubleDocValues(this) {
@Override
public double doubleVal(int doc) {
return func(doc, vals);
}
@Override
- public String strVal(int doc) {
- return Double.toString(doubleVal(doc));
- }
- @Override
public String toString(int doc) {
return name() + '(' + vals.toString(doc) + ')';
}
@@ -957,28 +945,12 @@ abstract class Double2Parser extends Nam
public DocValues getValues(Map context, AtomicReaderContext readerContext) throws IOException {
final DocValues aVals = a.getValues(context, readerContext);
final DocValues bVals = b.getValues(context, readerContext);
- return new DocValues() {
- @Override
- public float floatVal(int doc) {
- return (float)doubleVal(doc);
- }
- @Override
- public int intVal(int doc) {
- return (int)doubleVal(doc);
- }
- @Override
- public long longVal(int doc) {
- return (long)doubleVal(doc);
- }
- @Override
+ return new DoubleDocValues(this) {
+ @Override
public double doubleVal(int doc) {
return func(doc, aVals, bVals);
}
@Override
- public String strVal(int doc) {
- return Double.toString(doubleVal(doc));
- }
- @Override
public String toString(int doc) {
return name() + '(' + aVals.toString(doc) + ',' + bVals.toString(doc) + ')';
}
Modified: lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/ByteFieldSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/ByteFieldSource.java?rev=1101062&r1=1101061&r2=1101062&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/ByteFieldSource.java (original)
+++ lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/ByteFieldSource.java Mon May 9 15:24:04 2011
@@ -88,6 +88,11 @@ public class ByteFieldSource extends Num
return description() + '=' + byteVal(doc);
}
+ @Override
+ public Object objectVal(int doc) {
+ return arr[doc]; // TODO: valid?
+ }
+
};
}
}
Modified: lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/ConstValueSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/ConstValueSource.java?rev=1101062&r1=1101061&r2=1101062&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/ConstValueSource.java (original)
+++ lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/ConstValueSource.java Mon May 9 15:24:04 2011
@@ -41,7 +41,7 @@ public class ConstValueSource extends Co
@Override
public DocValues getValues(Map context, AtomicReaderContext readerContext) throws IOException {
- return new DocValues() {
+ return new FloatDocValues(this) {
@Override
public float floatVal(int doc) {
return constant;
@@ -59,13 +59,13 @@ public class ConstValueSource extends Co
return dv;
}
@Override
- public String strVal(int doc) {
- return Float.toString(constant);
- }
- @Override
public String toString(int doc) {
return description();
}
+ @Override
+ public Object objectVal(int doc) {
+ return constant;
+ }
};
}
Modified: lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/DocFreqValueSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/DocFreqValueSource.java?rev=1101062&r1=1101061&r2=1101062&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/DocFreqValueSource.java (original)
+++ lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/DocFreqValueSource.java Mon May 9 15:24:04 2011
@@ -21,14 +21,13 @@ import org.apache.lucene.index.IndexRead
import org.apache.lucene.index.Term;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.util.BytesRef;
-import org.apache.solr.search.MutableValueInt;
-import org.apache.solr.search.MutableValue;
+import org.apache.solr.search.*;
import java.io.IOException;
import java.util.Map;
-class ConstIntDocValues extends DocValues {
+class ConstIntDocValues extends IntDocValues {
final int ival;
final float fval;
final double dval;
@@ -37,6 +36,7 @@ class ConstIntDocValues extends DocValue
final ValueSource parent;
ConstIntDocValues(int val, ValueSource parent) {
+ super(parent);
ival = val;
fval = val;
dval = val;
@@ -71,7 +71,7 @@ class ConstIntDocValues extends DocValue
}
}
-class ConstDoubleDocValues extends DocValues {
+class ConstDoubleDocValues extends DoubleDocValues {
final int ival;
final float fval;
final double dval;
@@ -80,6 +80,7 @@ class ConstDoubleDocValues extends DocVa
final ValueSource parent;
ConstDoubleDocValues(double val, ValueSource parent) {
+ super(parent);
ival = (int)val;
fval = (float)val;
dval = val;
@@ -114,115 +115,6 @@ class ConstDoubleDocValues extends DocVa
}
}
-abstract class FloatDocValues extends DocValues {
- protected final ValueSource vs;
-
- public FloatDocValues(ValueSource vs) {
- this.vs = vs;
- }
-
- @Override
- public byte byteVal(int doc) {
- return (byte)floatVal(doc);
- }
-
- @Override
- public short shortVal(int doc) {
- return (short)floatVal(doc);
- }
-
- @Override
- public abstract float floatVal(int doc);
-
- @Override
- public int intVal(int doc) {
- return (int)floatVal(doc);
- }
-
- @Override
- public long longVal(int doc) {
- return (long)floatVal(doc);
- }
-
- @Override
- public double doubleVal(int doc) {
- return (double)floatVal(doc);
- }
-
- @Override
- public String strVal(int doc) {
- return Float.toString(floatVal(doc));
- }
-
- @Override
- public String toString(int doc) {
- return vs.description() + '=' + strVal(doc);
- }
-}
-
-abstract class IntDocValues extends DocValues {
- protected final ValueSource vs;
-
- public IntDocValues(ValueSource vs) {
- this.vs = vs;
- }
-
- @Override
- public byte byteVal(int doc) {
- return (byte)intVal(doc);
- }
-
- @Override
- public short shortVal(int doc) {
- return (short)intVal(doc);
- }
-
- @Override
- public float floatVal(int doc) {
- return (float)intVal(doc);
- }
-
- @Override
- public abstract int intVal(int doc);
-
- @Override
- public long longVal(int doc) {
- return (long)intVal(doc);
- }
-
- @Override
- public double doubleVal(int doc) {
- return (double)intVal(doc);
- }
-
- @Override
- public String strVal(int doc) {
- return Integer.toString(intVal(doc));
- }
-
- @Override
- public String toString(int doc) {
- return vs.description() + '=' + strVal(doc);
- }
-
- @Override
- public ValueFiller getValueFiller() {
- return new ValueFiller() {
- private final MutableValueInt mval = new MutableValueInt();
-
- @Override
- public MutableValue getValue() {
- return mval;
- }
-
- @Override
- public void fillValue(int doc) {
- mval.value = intVal(doc);
- }
- };
- }
-}
-
/**
* <code>DocFreqValueSource</code> returns the number of documents containing the term.
Modified: lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/DocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/DocValues.java?rev=1101062&r1=1101061&r2=1101062&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/DocValues.java (original)
+++ lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/DocValues.java Mon May 9 15:24:04 2011
@@ -19,6 +19,7 @@ package org.apache.solr.search.function;
import org.apache.lucene.search.*;
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.util.BytesRef;
import org.apache.solr.search.MutableValue;
import org.apache.solr.search.MutableValueFloat;
@@ -47,6 +48,28 @@ public abstract class DocValues {
// TODO: should we make a termVal, returns BytesRef?
public String strVal(int doc) { throw new UnsupportedOperationException(); }
+ /** returns the bytes representation of the string val - TODO: should this return the indexed raw bytes not? */
+ public boolean bytesVal(int doc, BytesRef target) {
+ String s = strVal(doc);
+ if (s==null) {
+ target.length = 0;
+ return false;
+ }
+ target.copy(s);
+ return true;
+ };
+
+ /** Native Java Object representation of the value */
+ public Object objectVal(int doc) {
+ // most DocValues are functions, so by default return a Float()
+ return floatVal(doc);
+ }
+
+ /** Returns true if there is a value for this document */
+ public boolean exists(int doc) {
+ return true;
+ }
+
/**
* @param doc The doc to retrieve to sort ordinal for
* @return the sort ordinal for the specified doc
Modified: lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/DoubleConstValueSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/DoubleConstValueSource.java?rev=1101062&r1=1101061&r2=1101062&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/DoubleConstValueSource.java (original)
+++ lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/DoubleConstValueSource.java Mon May 9 15:24:04 2011
@@ -40,7 +40,7 @@ public class DoubleConstValueSource exte
@Override
public DocValues getValues(Map context, AtomicReaderContext readerContext) throws IOException {
- return new DocValues() {
+ return new DoubleDocValues(this) {
@Override
public float floatVal(int doc) {
return fv;
@@ -67,6 +67,11 @@ public class DoubleConstValueSource exte
}
@Override
+ public Object objectVal(int doc) {
+ return constant;
+ }
+
+ @Override
public String toString(int doc) {
return description();
}
Modified: lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/DoubleFieldSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/DoubleFieldSource.java?rev=1101062&r1=1101061&r2=1101062&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/DoubleFieldSource.java (original)
+++ lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/DoubleFieldSource.java Mon May 9 15:24:04 2011
@@ -51,7 +51,7 @@ public class DoubleFieldSource extends N
public DocValues getValues(Map context, AtomicReaderContext readerContext) throws IOException {
final DoubleValues vals = cache.getDoubles(readerContext.reader, field, creator);
final double[] arr = vals.values;
- final Bits valid = vals.valid;
+ final Bits valid = vals.valid;
return new DocValues() {
@Override
@@ -80,6 +80,11 @@ public class DoubleFieldSource extends N
}
@Override
+ public Object objectVal(int doc) {
+ return valid.get(doc) ? arr[doc] : null;
+ }
+
+ @Override
public String toString(int doc) {
return description() + '=' + doubleVal(doc);
}
Modified: lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/DualFloatFunction.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/DualFloatFunction.java?rev=1101062&r1=1101061&r2=1101062&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/DualFloatFunction.java (original)
+++ lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/DualFloatFunction.java Mon May 9 15:24:04 2011
@@ -48,28 +48,12 @@ public abstract class DualFloatFunction
public DocValues getValues(Map context, AtomicReaderContext readerContext) throws IOException {
final DocValues aVals = a.getValues(context, readerContext);
final DocValues bVals = b.getValues(context, readerContext);
- return new DocValues() {
+ return new FloatDocValues(this) {
@Override
public float floatVal(int doc) {
return func(doc, aVals, bVals);
}
@Override
- public int intVal(int doc) {
- return (int)floatVal(doc);
- }
- @Override
- public long longVal(int doc) {
- return (long)floatVal(doc);
- }
- @Override
- public double doubleVal(int doc) {
- return floatVal(doc);
- }
- @Override
- public String strVal(int doc) {
- return Float.toString(floatVal(doc));
- }
- @Override
public String toString(int doc) {
return name() + '(' + aVals.toString(doc) + ',' + bVals.toString(doc) + ')';
}
Modified: lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/FileFloatSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/FileFloatSource.java?rev=1101062&r1=1101061&r2=1101062&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/FileFloatSource.java (original)
+++ lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/FileFloatSource.java Mon May 9 15:24:04 2011
@@ -65,35 +65,15 @@ public class FileFloatSource extends Val
final int off = offset;
final float[] arr = getCachedFloats(topLevelContext.reader);
- return new DocValues() {
+ return new FloatDocValues(this) {
@Override
public float floatVal(int doc) {
return arr[doc + off];
}
@Override
- public int intVal(int doc) {
- return (int)arr[doc + off];
- }
-
- @Override
- public long longVal(int doc) {
- return (long)arr[doc + off];
- }
-
- @Override
- public double doubleVal(int doc) {
- return (double)arr[doc + off];
- }
-
- @Override
- public String strVal(int doc) {
- return Float.toString(arr[doc + off]);
- }
-
- @Override
- public String toString(int doc) {
- return description() + '=' + floatVal(doc);
+ public Object objectVal(int doc) {
+ return floatVal(doc); // TODO: keep track of missing values
}
};
}
Modified: lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/FloatFieldSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/FloatFieldSource.java?rev=1101062&r1=1101061&r2=1101062&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/FloatFieldSource.java (original)
+++ lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/FloatFieldSource.java Mon May 9 15:24:04 2011
@@ -50,37 +50,22 @@ public class FloatFieldSource extends Nu
public DocValues getValues(Map context, AtomicReaderContext readerContext) throws IOException {
final FloatValues vals = cache.getFloats(readerContext.reader, field, creator);
final float[] arr = vals.values;
- final Bits valid = vals.valid;
+ final Bits valid = vals.valid;
- return new DocValues() {
+ return new FloatDocValues(this) {
@Override
public float floatVal(int doc) {
return arr[doc];
}
@Override
- public int intVal(int doc) {
- return (int)arr[doc];
+ public Object objectVal(int doc) {
+ return valid.get(doc) ? arr[doc] : null;
}
@Override
- public long longVal(int doc) {
- return (long)arr[doc];
- }
-
- @Override
- public double doubleVal(int doc) {
- return (double)arr[doc];
- }
-
- @Override
- public String strVal(int doc) {
- return Float.toString(arr[doc]);
- }
-
- @Override
- public String toString(int doc) {
- return description() + '=' + floatVal(doc);
+ public boolean exists(int doc) {
+ return valid.get(doc);
}
@Override
Modified: lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/IntFieldSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/IntFieldSource.java?rev=1101062&r1=1101061&r2=1101062&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/IntFieldSource.java (original)
+++ lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/search/function/IntFieldSource.java Mon May 9 15:24:04 2011
@@ -51,9 +51,9 @@ public class IntFieldSource extends Nume
public DocValues getValues(Map context, AtomicReaderContext readerContext) throws IOException {
final IntValues vals = cache.getInts(readerContext.reader, field, creator);
final int[] arr = vals.values;
- final Bits valid = vals.valid;
+ final Bits valid = vals.valid;
- return new DocValues() {
+ return new IntDocValues(this) {
final MutableValueInt val = new MutableValueInt();
@Override
@@ -82,6 +82,16 @@ public class IntFieldSource extends Nume
}
@Override
+ public Object objectVal(int doc) {
+ return valid.get(doc) ? arr[doc] : null;
+ }
+
+ @Override
+ public boolean exists(int doc) {
+ return valid.get(doc);
+ }
+
+ @Override
public String toString(int doc) {
return description() + '=' + intVal(doc);
}