You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ab...@apache.org on 2017/11/09 21:19:52 UTC
[46/50] [abbrv] lucene-solr:jira/solr-11285-sim: SOLR-11610: Move
SOLR to PayloadDecoder
SOLR-11610: Move SOLR to PayloadDecoder
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/1a80bc76
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/1a80bc76
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/1a80bc76
Branch: refs/heads/jira/solr-11285-sim
Commit: 1a80bc76b12e74a3fea065ac6989a9a72662f5f4
Parents: 5c9bcc9
Author: Alan Woodward <ro...@apache.org>
Authored: Mon Nov 6 11:52:13 2017 +0000
Committer: Alan Woodward <ro...@apache.org>
Committed: Thu Nov 9 14:05:34 2017 +0000
----------------------------------------------------------------------
solr/CHANGES.txt | 3 +
.../org/apache/solr/schema/IndexSchema.java | 16 +++-
.../solr/search/FloatPayloadValueSource.java | 4 +-
.../solr/search/PayloadScoreQParserPlugin.java | 4 +-
.../apache/solr/search/ValueSourceParser.java | 7 +-
.../PayloadScoringSimilarityWrapper.java | 80 --------------------
.../similarities/SchemaSimilarityFactory.java | 22 +-----
.../org/apache/solr/util/PayloadDecoder.java | 27 -------
.../java/org/apache/solr/util/PayloadUtils.java | 6 +-
9 files changed, 32 insertions(+), 137 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1a80bc76/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 9b96be3..28abb89 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -150,6 +150,9 @@ Other Changes
* SOLR-11628: Add documentation of maxRamMB for filter cache and query result cache. (shalin)
+* SOLR-11610: Refactored payload handling to use lucene's PayloadDecoder
+ framework (Alan Woodward)
+
================== 7.1.0 ==================
Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1a80bc76/solr/core/src/java/org/apache/solr/schema/IndexSchema.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/schema/IndexSchema.java b/solr/core/src/java/org/apache/solr/schema/IndexSchema.java
index 94884a2..7fb4f79 100644
--- a/solr/core/src/java/org/apache/solr/schema/IndexSchema.java
+++ b/solr/core/src/java/org/apache/solr/schema/IndexSchema.java
@@ -49,9 +49,10 @@ import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.MultiFields;
+import org.apache.lucene.queries.payloads.PayloadDecoder;
import org.apache.lucene.search.similarities.Similarity;
-import org.apache.solr.uninverting.UninvertingReader;
import org.apache.lucene.util.Version;
+import org.apache.solr.common.MapSerializable;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.params.CommonParams;
@@ -62,14 +63,15 @@ import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.Pair;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.core.Config;
-import org.apache.solr.common.MapSerializable;
import org.apache.solr.core.SolrConfig;
import org.apache.solr.core.SolrResourceLoader;
import org.apache.solr.request.LocalSolrQueryRequest;
import org.apache.solr.response.SchemaXmlWriter;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.search.similarities.SchemaSimilarityFactory;
+import org.apache.solr.uninverting.UninvertingReader;
import org.apache.solr.util.DOMUtil;
+import org.apache.solr.util.PayloadUtils;
import org.apache.solr.util.plugin.SolrCoreAware;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -150,6 +152,8 @@ public class IndexSchema {
protected DynamicCopy[] dynamicCopyFields;
public DynamicCopy[] getDynamicCopyFields() { return dynamicCopyFields; }
+ private Map<FieldType, PayloadDecoder> decoders = new HashMap<>(); // cache to avoid scanning token filters repeatedly, unnecessarily
+
/**
* keys are all fields copied to, count is num of copyField
* directives that target them.
@@ -1937,4 +1941,12 @@ public class IndexSchema {
null != rootType &&
rootType.getTypeName().equals(uniqueKeyFieldType.getTypeName()));
}
+
+ public PayloadDecoder getPayloadDecoder(String field) {
+ FieldType ft = getFieldType(field);
+ if (ft == null)
+ return null;
+ return decoders.computeIfAbsent(ft, f -> PayloadUtils.getPayloadDecoder(ft));
+ }
+
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1a80bc76/solr/core/src/java/org/apache/solr/search/FloatPayloadValueSource.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/FloatPayloadValueSource.java b/solr/core/src/java/org/apache/solr/search/FloatPayloadValueSource.java
index 9b2d969..c548418 100644
--- a/solr/core/src/java/org/apache/solr/search/FloatPayloadValueSource.java
+++ b/solr/core/src/java/org/apache/solr/search/FloatPayloadValueSource.java
@@ -27,10 +27,10 @@ import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.docvalues.FloatDocValues;
+import org.apache.lucene.queries.payloads.PayloadDecoder;
import org.apache.lucene.queries.payloads.PayloadFunction;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.util.BytesRef;
-import org.apache.solr.util.PayloadDecoder;
public class FloatPayloadValueSource extends ValueSource {
protected final String field;
@@ -167,7 +167,7 @@ public class FloatPayloadValueSource extends ValueSource {
docs.nextPosition();
BytesRef payload = docs.getPayload();
if (payload != null) {
- float payloadVal = decoder.decode(atDoc, docs.startOffset(), docs.endOffset(), payload);
+ float payloadVal = decoder.computePayloadFactor(payload);
// payloadFunction = null represents "first"
if (payloadFunction == null) return payloadVal;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1a80bc76/solr/core/src/java/org/apache/solr/search/PayloadScoreQParserPlugin.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/PayloadScoreQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/PayloadScoreQParserPlugin.java
index 7042cda..45f39c7 100644
--- a/solr/core/src/java/org/apache/solr/search/PayloadScoreQParserPlugin.java
+++ b/solr/core/src/java/org/apache/solr/search/PayloadScoreQParserPlugin.java
@@ -20,6 +20,7 @@ package org.apache.solr.search;
import java.io.IOException;
import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.queries.payloads.PayloadDecoder;
import org.apache.lucene.queries.payloads.PayloadFunction;
import org.apache.lucene.queries.payloads.PayloadScoreQuery;
import org.apache.lucene.search.Query;
@@ -83,7 +84,8 @@ public class PayloadScoreQParserPlugin extends QParserPlugin {
PayloadFunction payloadFunction = PayloadUtils.getPayloadFunction(func);
if (payloadFunction == null) throw new SyntaxError("Unknown payload function: " + func);
- return new PayloadScoreQuery(query, payloadFunction, includeSpanScore);
+ PayloadDecoder payloadDecoder = req.getCore().getLatestSchema().getPayloadDecoder(field);
+ return new PayloadScoreQuery(query, payloadFunction, payloadDecoder, includeSpanScore);
}
};
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1a80bc76/solr/core/src/java/org/apache/solr/search/ValueSourceParser.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/ValueSourceParser.java b/solr/core/src/java/org/apache/solr/search/ValueSourceParser.java
index 51048d2..5944dc5 100644
--- a/solr/core/src/java/org/apache/solr/search/ValueSourceParser.java
+++ b/solr/core/src/java/org/apache/solr/search/ValueSourceParser.java
@@ -34,6 +34,7 @@ import org.apache.lucene.queries.function.docvalues.BoolDocValues;
import org.apache.lucene.queries.function.docvalues.DoubleDocValues;
import org.apache.lucene.queries.function.docvalues.LongDocValues;
import org.apache.lucene.queries.function.valuesource.*;
+import org.apache.lucene.queries.payloads.PayloadDecoder;
import org.apache.lucene.queries.payloads.PayloadFunction;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
@@ -49,6 +50,7 @@ import org.apache.solr.common.util.NamedList;
import org.apache.solr.request.SolrRequestInfo;
import org.apache.solr.schema.CurrencyFieldType;
import org.apache.solr.schema.FieldType;
+import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.schema.StrField;
import org.apache.solr.schema.TextField;
@@ -77,7 +79,6 @@ import org.apache.solr.search.function.distance.StringDistanceFunction;
import org.apache.solr.search.function.distance.VectorDistanceFunction;
import org.apache.solr.search.join.ChildFieldValueSourceParser;
import org.apache.solr.util.DateMathParser;
-import org.apache.solr.util.PayloadDecoder;
import org.apache.solr.util.PayloadUtils;
import org.apache.solr.util.plugin.NamedListInitializedPlugin;
import org.locationtech.spatial4j.distance.DistanceUtils;
@@ -737,8 +738,8 @@ public abstract class ValueSourceParser implements NamedListInitializedPlugin {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Invalid payload function: " + func);
}
- FieldType fieldType = fp.getReq().getCore().getLatestSchema().getFieldTypeNoEx(tinfo.field);
- PayloadDecoder decoder = PayloadUtils.getPayloadDecoder(fieldType);
+ IndexSchema schema = fp.getReq().getCore().getLatestSchema();
+ PayloadDecoder decoder = schema.getPayloadDecoder(tinfo.field);
if (decoder==null) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No payload decoder found for field: " + tinfo.field);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1a80bc76/solr/core/src/java/org/apache/solr/search/similarities/PayloadScoringSimilarityWrapper.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/similarities/PayloadScoringSimilarityWrapper.java b/solr/core/src/java/org/apache/solr/search/similarities/PayloadScoringSimilarityWrapper.java
deleted file mode 100644
index 9141620..0000000
--- a/solr/core/src/java/org/apache/solr/search/similarities/PayloadScoringSimilarityWrapper.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.search.similarities;
-
-import java.io.IOException;
-
-import org.apache.lucene.index.FieldInvertState;
-import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.search.CollectionStatistics;
-import org.apache.lucene.search.TermStatistics;
-import org.apache.lucene.search.similarities.Similarity;
-import org.apache.lucene.util.BytesRef;
-import org.apache.solr.util.PayloadDecoder;
-
-/**
- * The computation Lucene's PayloadScoreQuery uses is SimScorer#computePayloadFactor.
- * This wrapper delegates to a main similarity except for this one method.
- */
-public class PayloadScoringSimilarityWrapper extends Similarity {
- private Similarity delegate;
- private PayloadDecoder decoder;
-
- public PayloadScoringSimilarityWrapper(Similarity delegate, PayloadDecoder decoder) {
- this.delegate = delegate;
- this.decoder = decoder;
- }
-
- @Override
- public String toString() {
- return "PayloadScoring(" + delegate.toString() + ", decoder=" + decoder.toString() + ")";
- }
-
- @Override
- public long computeNorm(FieldInvertState state) {
- return delegate.computeNorm(state);
- }
-
- @Override
- public SimWeight computeWeight(float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
- return delegate.computeWeight(boost, collectionStats, termStats);
- }
-
- @Override
- public SimScorer simScorer(SimWeight weight, LeafReaderContext context) throws IOException {
- final SimScorer simScorer = delegate.simScorer(weight,context);
- SimScorer payloadSimScorer = new SimScorer() {
- @Override
- public float score(int doc, float freq) throws IOException {
- return simScorer.score(doc,freq);
- }
-
- @Override
- public float computeSlopFactor(int distance) {
- return simScorer.computeSlopFactor(distance);
- }
-
- @Override
- public float computePayloadFactor(int doc, int start, int end, BytesRef payload) {
- return decoder.decode(doc, start, end, payload);
- }
- };
-
- return payloadSimScorer;
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1a80bc76/solr/core/src/java/org/apache/solr/search/similarities/SchemaSimilarityFactory.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/similarities/SchemaSimilarityFactory.java b/solr/core/src/java/org/apache/solr/search/similarities/SchemaSimilarityFactory.java
index 0aea8ed..378197c 100644
--- a/solr/core/src/java/org/apache/solr/search/similarities/SchemaSimilarityFactory.java
+++ b/solr/core/src/java/org/apache/solr/search/similarities/SchemaSimilarityFactory.java
@@ -16,22 +16,17 @@
*/
package org.apache.solr.search.similarities;
-import java.util.HashMap;
-
-import org.apache.lucene.search.similarities.ClassicSimilarity;
import org.apache.lucene.search.similarities.BM25Similarity;
+import org.apache.lucene.search.similarities.ClassicSimilarity;
import org.apache.lucene.search.similarities.PerFieldSimilarityWrapper;
import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.util.Version;
-
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.core.SolrCore;
import org.apache.solr.schema.FieldType;
import org.apache.solr.schema.SimilarityFactory;
-import org.apache.solr.util.PayloadDecoder;
-import org.apache.solr.util.PayloadUtils;
import org.apache.solr.util.plugin.SolrCoreAware;
/**
@@ -138,7 +133,6 @@ public class SchemaSimilarityFactory extends SimilarityFactory implements SolrCo
private class SchemaSimilarity extends PerFieldSimilarityWrapper {
private Similarity defaultSimilarity;
- private HashMap<FieldType,PayloadDecoder> decoders; // cache to avoid scanning token filters repeatedly, unnecessarily
public SchemaSimilarity(Similarity defaultSimilarity) {
this.defaultSimilarity = defaultSimilarity;
@@ -151,19 +145,7 @@ public class SchemaSimilarityFactory extends SimilarityFactory implements SolrCo
return defaultSimilarity;
} else {
Similarity similarity = fieldType.getSimilarity();
- similarity = similarity == null ? defaultSimilarity : similarity;
-
- // Payload score handling: if field type has index-time payload encoding, wrap and computePayloadFactor accordingly
- if (decoders == null) decoders = new HashMap<>();
- PayloadDecoder decoder;
- if (!decoders.containsKey(fieldType)) {
- decoders.put(fieldType, PayloadUtils.getPayloadDecoder(fieldType));
- }
- decoder = decoders.get(fieldType);
-
- if (decoder != null) similarity = new PayloadScoringSimilarityWrapper(similarity, decoder);
-
- return similarity;
+ return similarity == null ? defaultSimilarity : similarity;
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1a80bc76/solr/core/src/java/org/apache/solr/util/PayloadDecoder.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/util/PayloadDecoder.java b/solr/core/src/java/org/apache/solr/util/PayloadDecoder.java
deleted file mode 100644
index e083d21..0000000
--- a/solr/core/src/java/org/apache/solr/util/PayloadDecoder.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.util;
-
-import org.apache.lucene.util.BytesRef;
-
-/**
- * Mirrors SimScorer#computePayloadFactor's signature
- */
-public interface PayloadDecoder {
- float decode(int doc, int start, int end, BytesRef payload);
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1a80bc76/solr/core/src/java/org/apache/solr/util/PayloadUtils.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/util/PayloadUtils.java b/solr/core/src/java/org/apache/solr/util/PayloadUtils.java
index 2de7307..2e404df 100644
--- a/solr/core/src/java/org/apache/solr/util/PayloadUtils.java
+++ b/solr/core/src/java/org/apache/solr/util/PayloadUtils.java
@@ -32,6 +32,7 @@ import org.apache.lucene.index.Term;
import org.apache.lucene.queries.payloads.AveragePayloadFunction;
import org.apache.lucene.queries.payloads.MaxPayloadFunction;
import org.apache.lucene.queries.payloads.MinPayloadFunction;
+import org.apache.lucene.queries.payloads.PayloadDecoder;
import org.apache.lucene.queries.payloads.PayloadFunction;
import org.apache.lucene.queries.payloads.SumPayloadFunction;
import org.apache.lucene.search.spans.SpanNearQuery;
@@ -44,6 +45,7 @@ import org.apache.solr.schema.FieldType;
import org.apache.solr.search.PayloadScoreQParserPlugin;
public class PayloadUtils {
+
public static String getPayloadEncoder(FieldType fieldType) {
// TODO: support custom payload encoding fields too somehow - maybe someone has a custom component that encodes payloads as floats
String encoder = null;
@@ -75,10 +77,10 @@ public class PayloadUtils {
String encoder = getPayloadEncoder(fieldType);
if ("integer".equals(encoder)) {
- decoder = (int doc, int start, int end, BytesRef payload) -> PayloadHelper.decodeInt(payload.bytes, payload.offset);
+ decoder = (BytesRef payload) -> payload == null ? 1 : PayloadHelper.decodeInt(payload.bytes, payload.offset);
}
if ("float".equals(encoder)) {
- decoder = (int doc, int start, int end, BytesRef payload) -> PayloadHelper.decodeFloat(payload.bytes, payload.offset);
+ decoder = (BytesRef payload) -> payload == null ? 1 : PayloadHelper.decodeFloat(payload.bytes, payload.offset);
}
// encoder could be "identity" at this point, in the case of DelimitedTokenFilterFactory encoder="identity"