You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sa...@apache.org on 2017/07/28 15:08:16 UTC
[3/3] lucene-solr:branch_7_0: SOLR-11155: /analysis/field and
/analysis/document requests should support points fields.
SOLR-11155: /analysis/field and /analysis/document requests should support points fields.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/717a6a8d
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/717a6a8d
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/717a6a8d
Branch: refs/heads/branch_7_0
Commit: 717a6a8dd089f62d972b2d7cfdc95d34362b5d50
Parents: 35f8d55
Author: Steve Rowe <sa...@apache.org>
Authored: Fri Jul 28 11:07:44 2017 -0400
Committer: Steve Rowe <sa...@apache.org>
Committed: Fri Jul 28 11:08:08 2017 -0400
----------------------------------------------------------------------
solr/CHANGES.txt | 3 ++
.../handler/AnalysisRequestHandlerBase.java | 11 ++++--
.../java/org/apache/solr/schema/FieldType.java | 11 ++++--
.../DocumentAnalysisRequestHandlerTest.java | 37 +++++++++++++-------
.../FieldAnalysisRequestHandlerTest.java | 28 +++++++++++----
.../solr/client/solrj/SolrExampleTests.java | 9 ++---
6 files changed, 69 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/717a6a8d/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 10513e9..9f15665 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -472,6 +472,9 @@ Other Changes
* SOLR-10846: ExternalFileField/FloatFieldSource should throw a clear exception on initialization with
a Points-based keyField, which is not supported. (hossman, Steve Rowe)
+
+* SOLR-11155: /analysis/field and /analysis/document requests should support points fields.
+ (Jason Gerlowski, Steve Rowe)
================== 6.7.0 ==================
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/717a6a8d/solr/core/src/java/org/apache/solr/handler/AnalysisRequestHandlerBase.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/AnalysisRequestHandlerBase.java b/solr/core/src/java/org/apache/solr/handler/AnalysisRequestHandlerBase.java
index 0770bd4..579e5ef 100644
--- a/solr/core/src/java/org/apache/solr/handler/AnalysisRequestHandlerBase.java
+++ b/solr/core/src/java/org/apache/solr/handler/AnalysisRequestHandlerBase.java
@@ -33,6 +33,7 @@ import org.apache.commons.lang.ArrayUtils;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.tokenattributes.BytesTermAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
@@ -248,8 +249,14 @@ public abstract class AnalysisRequestHandlerBase extends RequestHandlerBase {
for (int i = 0; i < tokens.length; i++) {
AttributeSource token = tokens[i];
final NamedList<Object> tokenNamedList = new SimpleOrderedMap<>();
- final TermToBytesRefAttribute termAtt = token.getAttribute(TermToBytesRefAttribute.class);
- BytesRef rawBytes = termAtt.getBytesRef();
+ final BytesRef rawBytes;
+ if (token.hasAttribute(BytesTermAttribute.class)) {
+ final BytesTermAttribute bytesAtt = token.getAttribute(BytesTermAttribute.class);
+ rawBytes = bytesAtt.getBytesRef();
+ } else {
+ final TermToBytesRefAttribute termAtt = token.getAttribute(TermToBytesRefAttribute.class);
+ rawBytes = termAtt.getBytesRef();
+ }
final String text = fieldType.indexedToReadable(rawBytes, new CharsRefBuilder()).toString();
tokenNamedList.add("text", text);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/717a6a8d/solr/core/src/java/org/apache/solr/schema/FieldType.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/schema/FieldType.java b/solr/core/src/java/org/apache/solr/schema/FieldType.java
index 26fc657..3d38844 100644
--- a/solr/core/src/java/org/apache/solr/schema/FieldType.java
+++ b/solr/core/src/java/org/apache/solr/schema/FieldType.java
@@ -30,6 +30,7 @@ import java.util.Set;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.tokenattributes.BytesTermAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.util.CharFilterFactory;
@@ -479,14 +480,20 @@ public abstract class FieldType extends FieldProperties {
Tokenizer ts = new Tokenizer() {
final char[] cbuf = new char[maxChars];
final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+ final BytesTermAttribute bytesAtt = isPointField() ? addAttribute(BytesTermAttribute.class) : null;
final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
@Override
public boolean incrementToken() throws IOException {
clearAttributes();
int n = input.read(cbuf,0,maxChars);
if (n<=0) return false;
- String s = toInternal(new String(cbuf,0,n));
- termAtt.setEmpty().append(s);
+ if (isPointField()) {
+ BytesRef b = ((PointField)FieldType.this).toInternalByteRef(new String(cbuf, 0, n));
+ bytesAtt.setBytesRef(b);
+ } else {
+ String s = toInternal(new String(cbuf, 0, n));
+ termAtt.setEmpty().append(s);
+ }
offsetAtt.setOffset(correctOffset(0),correctOffset(n));
return true;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/717a6a8d/solr/core/src/test/org/apache/solr/handler/DocumentAnalysisRequestHandlerTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/DocumentAnalysisRequestHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/DocumentAnalysisRequestHandlerTest.java
index d3b0ab0..7f19526 100644
--- a/solr/core/src/test/org/apache/solr/handler/DocumentAnalysisRequestHandlerTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/DocumentAnalysisRequestHandlerTest.java
@@ -210,6 +210,7 @@ public class DocumentAnalysisRequestHandlerTest extends AnalysisRequestHandlerTe
document.addField("id", 1);
document.addField("whitetok", "Jumping Jack");
document.addField("text", "The Fox Jumped Over The Dogs");
+ document.addField("number_l_p", 88L);
DocumentAnalysisRequest request = new DocumentAnalysisRequest()
.setQuery("JUMPING")
@@ -221,35 +222,45 @@ public class DocumentAnalysisRequestHandlerTest extends AnalysisRequestHandlerTe
NamedList<NamedList<NamedList<Object>>> documentResult = (NamedList<NamedList<NamedList<Object>>>) result.get("1");
assertNotNull("An analysis for document with key '1' should be returned", documentResult);
- // the id field
- NamedList<NamedList<Object>> idResult = documentResult.get("id");
- assertNotNull("an analysis for the 'id' field should be returned", idResult);
-
NamedList<Object> queryResult;
List<NamedList> tokenList;
NamedList<Object> indexResult;
NamedList<List<NamedList>> valueResult;
+ String name;
- /*** Much of this test seems invalid for a numeric "id" field
- NamedList<Object> queryResult = idResult.get("query");
+ // the id field
+ NamedList<NamedList<Object>> idResult = documentResult.get("id");
+ assertNotNull("an analysis for the 'id' field should be returned", idResult);
+ queryResult = idResult.get("query");
assertEquals("Only the default analyzer should be applied", 1, queryResult.size());
- String name = queryResult.getName(0);
+ name = queryResult.getName(0);
assertTrue("Only the default analyzer should be applied", name.matches("org.apache.solr.schema.FieldType\\$DefaultAnalyzer.*"));
- List<NamedList> tokenList = (List<NamedList>) queryResult.getVal(0);
+ tokenList = (List<NamedList>) queryResult.getVal(0);
assertEquals("Query has only one token", 1, tokenList.size());
assertToken(tokenList.get(0), new TokenInfo("JUMPING", null, "word", 0, 7, 1, new int[]{1}, null, false));
- NamedList<Object> indexResult = idResult.get("index");
-
+ indexResult = idResult.get("index");
assertEquals("The id field has only a single value", 1, indexResult.size());
- NamedList<List<NamedList>> valueResult = (NamedList<List<NamedList>>) indexResult.get("1");
+ valueResult = (NamedList<List<NamedList>>) indexResult.get("1");
assertEquals("Only the default analyzer should be applied", 1, valueResult.size());
name = queryResult.getName(0);
assertTrue("Only the default analyzer should be applied", name.matches("org.apache.solr.schema.FieldType\\$DefaultAnalyzer.*"));
tokenList = valueResult.getVal(0);
assertEquals("The 'id' field value has only one token", 1, tokenList.size());
assertToken(tokenList.get(0), new TokenInfo("1", null, "word", 0, 1, 1, new int[]{1}, null, false));
- ***/
-
+
+ // the number_l_p field
+ NamedList<NamedList<Object>> number_l_p_Result = documentResult.get("number_l_p");
+ assertNotNull("an analysis for the 'number_l_p' field should be returned", number_l_p_Result);
+ indexResult = number_l_p_Result.get("index");
+ assertEquals("The number_l_p field has only a single value", 1, indexResult.size());
+ valueResult = (NamedList<List<NamedList>>) indexResult.get("88");
+ assertEquals("Only the default analyzer should be applied", 1, valueResult.size());
+ name = queryResult.getName(0);
+ assertTrue("Only the default analyzer should be applied", name.matches("org.apache.solr.schema.FieldType\\$DefaultAnalyzer.*"));
+ tokenList = valueResult.getVal(0);
+ assertEquals("The 'number_l_p' field value has only one token", 1, tokenList.size());
+ assertToken(tokenList.get(0), new TokenInfo("88", null, "word", 0, 2, 1, new int[]{1}, null, false));
+
// the name field
NamedList<NamedList<Object>> whitetokResult = documentResult.get("whitetok");
assertNotNull("an analysis for the 'whitetok' field should be returned", whitetokResult);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/717a6a8d/solr/core/src/test/org/apache/solr/handler/FieldAnalysisRequestHandlerTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/FieldAnalysisRequestHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/FieldAnalysisRequestHandlerTest.java
index 2b2fba9..260dc4a 100644
--- a/solr/core/src/test/org/apache/solr/handler/FieldAnalysisRequestHandlerTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/FieldAnalysisRequestHandlerTest.java
@@ -16,6 +16,11 @@
*/
package org.apache.solr.handler;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenFilter;
@@ -29,12 +34,12 @@ import org.apache.lucene.analysis.util.TokenFilterFactory;
import org.apache.lucene.analysis.util.TokenizerFactory;
import org.apache.lucene.util.AttributeFactory;
import org.apache.solr.analysis.TokenizerChain;
+import org.apache.solr.client.solrj.request.FieldAnalysisRequest;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.AnalysisParams;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.NamedList;
-import org.apache.solr.client.solrj.request.FieldAnalysisRequest;
import org.apache.solr.request.LocalSolrQueryRequest;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.schema.FieldType;
@@ -43,11 +48,6 @@ import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
/**
* A test for {@link FieldAnalysisRequestHandler}.
*
@@ -55,7 +55,7 @@ import java.util.List;
* @since solr 1.4
*/
public class FieldAnalysisRequestHandlerTest extends AnalysisRequestHandlerTestBase {
-
+
private FieldAnalysisRequestHandler handler;
@Override
@@ -69,6 +69,20 @@ public class FieldAnalysisRequestHandlerTest extends AnalysisRequestHandlerTestB
public static void beforeClass() throws Exception {
initCore("solrconfig.xml", "schema.xml");
}
+
+ @Test
+ public void testPointField() throws Exception {
+ FieldAnalysisRequest request = new FieldAnalysisRequest();
+ request.addFieldType("pint");
+ request.setFieldValue("5");
+
+ NamedList<NamedList> nl = handler.handleAnalysisRequest(request, h.getCore().getLatestSchema());
+ NamedList pintNL = (NamedList)nl.get("field_types").get("pint");
+ NamedList indexNL = (NamedList)pintNL.get("index");
+ ArrayList analyzerNL = (ArrayList)indexNL.get("org.apache.solr.schema.FieldType$DefaultAnalyzer$1");
+ String text = (String)((NamedList)analyzerNL.get(0)).get("text");
+ assertEquals("5", text);
+ }
/**
* Tests the {@link FieldAnalysisRequestHandler#resolveAnalysisRequest(org.apache.solr.request.SolrQueryRequest)}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/717a6a8d/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleTests.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleTests.java b/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleTests.java
index 39158cb..63bd72c 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleTests.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleTests.java
@@ -416,14 +416,11 @@ abstract public class SolrExampleTests extends SolrExampleTestsBase
query.set(AnalysisParams.FIELD_VALUE, "ignore_exception");
try {
client.query( query );
- Assert.fail("should have a server exception");
+ Assert.fail("should have a number format exception");
}
catch(SolrException ex) {
- assertEquals(500, ex.code());
- assertThat(ex.getMessage(), containsString(" Can't generate internal string in PointField. use PointField.toInternalByteRef"));
- }
- catch (SolrServerException ex) {
- assertThat(ex.getMessage(), containsString(" Can't generate internal string in PointField. use PointField.toInternalByteRef"));
+ assertEquals(400, ex.code());
+ assertThat(ex.getMessage(), containsString("Invalid Number: ignore_exception"));
}
catch(Throwable t) {
t.printStackTrace();