You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sa...@apache.org on 2013/03/18 04:39:51 UTC
svn commit: r1457628 - in /lucene/dev/branches/lucene_solr_4_2: ./ solr/
solr/core/ solr/core/src/java/org/apache/solr/rest/
solr/core/src/java/org/apache/solr/schema/
solr/core/src/test-files/solr/collection1/conf/
solr/core/src/test/org/apache/solr/r...
Author: sarowe
Date: Mon Mar 18 03:39:50 2013
New Revision: 1457628
URL: http://svn.apache.org/r1457628
Log:
SOLR-4567: copyField source glob matching explicit field(s) stopped working in Solr 4.2. (merged branch_4x r1455837 and r1456289)
Modified:
lucene/dev/branches/lucene_solr_4_2/ (props changed)
lucene/dev/branches/lucene_solr_4_2/solr/ (props changed)
lucene/dev/branches/lucene_solr_4_2/solr/CHANGES.txt (contents, props changed)
lucene/dev/branches/lucene_solr_4_2/solr/core/ (props changed)
lucene/dev/branches/lucene_solr_4_2/solr/core/src/java/org/apache/solr/rest/CopyFieldCollectionResource.java
lucene/dev/branches/lucene_solr_4_2/solr/core/src/java/org/apache/solr/schema/IndexSchema.java
lucene/dev/branches/lucene_solr_4_2/solr/core/src/test-files/solr/collection1/conf/schema-copyfield-test.xml
lucene/dev/branches/lucene_solr_4_2/solr/core/src/test-files/solr/collection1/conf/schema-rest.xml
lucene/dev/branches/lucene_solr_4_2/solr/core/src/test/org/apache/solr/rest/TestCopyFieldCollectionResource.java
lucene/dev/branches/lucene_solr_4_2/solr/core/src/test/org/apache/solr/schema/CopyFieldTest.java
Modified: lucene/dev/branches/lucene_solr_4_2/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_2/solr/CHANGES.txt?rev=1457628&r1=1457627&r2=1457628&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_2/solr/CHANGES.txt (original)
+++ lucene/dev/branches/lucene_solr_4_2/solr/CHANGES.txt Mon Mar 18 03:39:50 2013
@@ -33,6 +33,12 @@ Apache ZooKeeper 3.4.5
Detailed Change List
----------------------
+Bug Fixes
+----------------------
+
+* SOLR-4567: copyField source glob matching explicit field(s) stopped working
+ in Solr 4.2. (Alexandre Rafalovitch, Steve Rowe)
+
Other Changes
----------------------
Modified: lucene/dev/branches/lucene_solr_4_2/solr/core/src/java/org/apache/solr/rest/CopyFieldCollectionResource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_2/solr/core/src/java/org/apache/solr/rest/CopyFieldCollectionResource.java?rev=1457628&r1=1457627&r2=1457628&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_2/solr/core/src/java/org/apache/solr/rest/CopyFieldCollectionResource.java (original)
+++ lucene/dev/branches/lucene_solr_4_2/solr/core/src/java/org/apache/solr/rest/CopyFieldCollectionResource.java Mon Mar 18 03:39:50 2013
@@ -34,7 +34,10 @@ import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.SortedMap;
+import java.util.SortedSet;
import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.regex.Pattern;
/**
* This class responds to requests at /solr/(corename)/schema/copyfields
@@ -63,9 +66,10 @@ public class CopyFieldCollectionResource
private static final String MAX_CHARS = "maxChars";
private static final String SOURCE_DYNAMIC_BASE = "sourceDynamicBase";
private static final String DESTINATION_DYNAMIC_BASE = "destDynamicBase";
+ private static final String SOURCE_EXPLICIT_FIELDS = "sourceExplicitFields";
- private Set<String> sourceFields;
- private Set<String> destinationFields;
+ private Set<String> requestedSourceFields;
+ private Set<String> requestedDestinationFields;
public CopyFieldCollectionResource() {
super();
@@ -79,16 +83,16 @@ public class CopyFieldCollectionResource
if (null != sourceFieldListParam) {
String[] fields = sourceFieldListParam.trim().split("[,\\s]+");
if (fields.length > 0) {
- sourceFields = new HashSet<String>(Arrays.asList(fields));
- sourceFields.remove(""); // Remove empty values, if any
+ requestedSourceFields = new HashSet<String>(Arrays.asList(fields));
+ requestedSourceFields.remove(""); // Remove empty values, if any
}
}
String destinationFieldListParam = getSolrRequest().getParams().get(DESTINATION_FIELD_LIST);
if (null != destinationFieldListParam) {
String[] fields = destinationFieldListParam.trim().split("[,\\s]+");
if (fields.length > 0) {
- destinationFields = new HashSet<String>(Arrays.asList(fields));
- destinationFields.remove(""); // Remove empty values, if any
+ requestedDestinationFields = new HashSet<String>(Arrays.asList(fields));
+ requestedDestinationFields.remove(""); // Remove empty values, if any
}
}
}
@@ -98,8 +102,7 @@ public class CopyFieldCollectionResource
public Representation get() {
try {
final List<SimpleOrderedMap<Object>> props = new ArrayList<SimpleOrderedMap<Object>>();
- SortedMap<String,List<CopyField>> sortedCopyFields
- = new TreeMap<String, List<CopyField>>(getSchema().getCopyFieldsMap());
+ SortedMap<String,List<CopyField>> sortedCopyFields = new TreeMap<String, List<CopyField>>(getSchema().getCopyFieldsMap());
for (List<CopyField> copyFields : sortedCopyFields.values()) {
Collections.sort(copyFields, new Comparator<CopyField>() {
@Override
@@ -111,8 +114,8 @@ public class CopyFieldCollectionResource
for (CopyField copyField : copyFields) {
final String source = copyField.getSource().getName();
final String destination = copyField.getDestination().getName();
- if ( (null == sourceFields || sourceFields.contains(source))
- && (null == destinationFields || destinationFields.contains(destination))) {
+ if ( (null == requestedSourceFields || requestedSourceFields.contains(source))
+ && (null == requestedDestinationFields || requestedDestinationFields.contains(destination))) {
SimpleOrderedMap<Object> copyFieldProps = new SimpleOrderedMap<Object>();
copyFieldProps.add(SOURCE, source);
copyFieldProps.add(DESTINATION, destination);
@@ -126,14 +129,26 @@ public class CopyFieldCollectionResource
for (IndexSchema.DynamicCopy dynamicCopy : getSchema().getDynamicCopyFields()) {
final String source = dynamicCopy.getRegex();
final String destination = dynamicCopy.getDestFieldName();
- if ( (null == sourceFields || sourceFields.contains(source))
- && (null == destinationFields || destinationFields.contains(destination))) {
+ if ( (null == requestedSourceFields || requestedSourceFields.contains(source))
+ && (null == requestedDestinationFields || requestedDestinationFields.contains(destination))) {
SimpleOrderedMap<Object> dynamicCopyProps = new SimpleOrderedMap<Object>();
-
+
dynamicCopyProps.add(SOURCE, dynamicCopy.getRegex());
IndexSchema.DynamicField sourceDynamicBase = dynamicCopy.getSourceDynamicBase();
if (null != sourceDynamicBase) {
dynamicCopyProps.add(SOURCE_DYNAMIC_BASE, sourceDynamicBase.getRegex());
+ } else if (source.contains("*")) {
+ List<String> sourceExplicitFields = new ArrayList<String>();
+ Pattern pattern = Pattern.compile(source.replace("*", ".*")); // glob->regex
+ for (String field : getSchema().getFields().keySet()) {
+ if (pattern.matcher(field).matches()) {
+ sourceExplicitFields.add(field);
+ }
+ }
+ if (sourceExplicitFields.size() > 0) {
+ Collections.sort(sourceExplicitFields);
+ dynamicCopyProps.add(SOURCE_EXPLICIT_FIELDS, sourceExplicitFields);
+ }
}
dynamicCopyProps.add(DESTINATION, dynamicCopy.getDestFieldName());
Modified: lucene/dev/branches/lucene_solr_4_2/solr/core/src/java/org/apache/solr/schema/IndexSchema.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_2/solr/core/src/java/org/apache/solr/schema/IndexSchema.java?rev=1457628&r1=1457627&r2=1457628&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_2/solr/core/src/java/org/apache/solr/schema/IndexSchema.java (original)
+++ lucene/dev/branches/lucene_solr_4_2/solr/core/src/java/org/apache/solr/schema/IndexSchema.java Mon Mar 18 03:39:50 2013
@@ -53,6 +53,7 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.regex.Pattern;
/**
* <code>IndexSchema</code> contains information about the valid fields in an index
@@ -418,7 +419,7 @@ public final class IndexSchema {
requiredFields.add(f);
}
} else if (node.getNodeName().equals("dynamicField")) {
- if (isValidDynamicFieldName(name)) {
+ if (isValidFieldGlob(name)) {
// make sure nothing else has the same path
addDynamicField(dFields, f);
} else {
@@ -586,7 +587,7 @@ public final class IndexSchema {
}
/** Returns true if the given name has exactly one asterisk either at the start or end of the name */
- private boolean isValidDynamicFieldName(String name) {
+ private static boolean isValidFieldGlob(String name) {
if (name.startsWith("*") || name.endsWith("*")) {
int count = 0;
for (int pos = 0 ; pos < name.length() && -1 != (pos = name.indexOf('*', pos)) ; ++pos) ++count;
@@ -658,11 +659,22 @@ public final class IndexSchema {
DynamicField destDynamicBase = null;
boolean sourceIsDynamicFieldReference = false;
+ boolean sourceIsExplicitFieldGlob = false;
- if (null == destSchemaField || null == sourceSchemaField) {
+ if (null == sourceSchemaField && isValidFieldGlob(source)) {
+ Pattern pattern = Pattern.compile(source.replace("*", ".*")); // glob->regex
+ for (String field : fields.keySet()) {
+ if (pattern.matcher(field).matches()) {
+ sourceIsExplicitFieldGlob = true;
+ break;
+ }
+ }
+ }
+
+ if (null == destSchemaField || (null == sourceSchemaField && ! sourceIsExplicitFieldGlob)) {
// Go through dynamicFields array only once, collecting info for both source and dest fields, if needed
for (DynamicField dynamicField : dynamicFields) {
- if (null == sourceSchemaField && ! sourceIsDynamicFieldReference) {
+ if (null == sourceSchemaField && ! sourceIsDynamicFieldReference && ! sourceIsExplicitFieldGlob) {
if (dynamicField.matches(source)) {
sourceIsDynamicFieldReference = true;
if ( ! source.equals(dynamicField.getRegex())) {
@@ -680,19 +692,22 @@ public final class IndexSchema {
destDynamicBase = dynamicField;
}
}
- if (null != destSchemaField && (null != sourceSchemaField || sourceIsDynamicFieldReference)) break;
+ if (null != destSchemaField
+ && (null != sourceSchemaField || sourceIsDynamicFieldReference || sourceIsExplicitFieldGlob)) {
+ break;
+ }
}
}
- if (null == sourceSchemaField && ! sourceIsDynamicFieldReference) {
- String msg = "copyField source :'" + source + "' is not an explicit field and doesn't match a dynamicField.";
+ if (null == sourceSchemaField && ! sourceIsDynamicFieldReference && ! sourceIsExplicitFieldGlob) {
+ String msg = "copyField source :'" + source + "' doesn't match any explicit field or dynamicField.";
throw new SolrException(ErrorCode.SERVER_ERROR, msg);
}
if (null == destSchemaField) {
String msg = "copyField dest :'" + dest + "' is not an explicit field and doesn't match a dynamicField.";
throw new SolrException(ErrorCode.SERVER_ERROR, msg);
}
- if (sourceIsDynamicFieldReference) {
- if (null != destDynamicField) { // source & dest: dynamic field references
+ if (sourceIsDynamicFieldReference || sourceIsExplicitFieldGlob) {
+ if (null != destDynamicField) { // source: dynamic field ref or explicit field glob; dest: dynamic field ref
registerDynamicCopyField(new DynamicCopy(source, destDynamicField, maxChars, sourceDynamicBase, destDynamicBase));
incrementCopyFieldTargetCount(destSchemaField);
} else { // source: dynamic field reference; dest: explicit field
@@ -708,7 +723,7 @@ public final class IndexSchema {
incrementCopyFieldTargetCount(destSchemaField);
} else {
String msg = "copyField only supports a dynamic destination with an asterisk "
- + "if the source is also dynamic with an asterisk";
+ + "if the source also has an asterisk";
throw new SolrException(ErrorCode.SERVER_ERROR, msg);
}
} else { // source & dest: explicit fields
Modified: lucene/dev/branches/lucene_solr_4_2/solr/core/src/test-files/solr/collection1/conf/schema-copyfield-test.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_2/solr/core/src/test-files/solr/collection1/conf/schema-copyfield-test.xml?rev=1457628&r1=1457627&r2=1457628&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_2/solr/core/src/test-files/solr/collection1/conf/schema-copyfield-test.xml (original)
+++ lucene/dev/branches/lucene_solr_4_2/solr/core/src/test-files/solr/collection1/conf/schema-copyfield-test.xml Mon Mar 18 03:39:50 2013
@@ -460,7 +460,13 @@
<copyField source="text_fr" dest="highlight" maxChars="25" />
<copyField source="text_en" dest="highlight" maxChars="25" />
<copyField source="text_*" dest="highlight" maxChars="25" />
-
+
+ <!-- test source glob matching multiple explicit fields: sku1 and sku2 -->
+ <copyField source="sku*" dest="text"/>
+ <copyField source="sku*" dest="*_s"/>
+ <copyField source="sku*" dest="*_dest_sub_s"/>
+ <copyField source="sku*" dest="dest_sub_no_ast_s"/>
+
<!-- Similarity is the scoring routine for each document vs a query.
A custom similarity may be specified here, but the default is fine
for most applications.
Modified: lucene/dev/branches/lucene_solr_4_2/solr/core/src/test-files/solr/collection1/conf/schema-rest.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_2/solr/core/src/test-files/solr/collection1/conf/schema-rest.xml?rev=1457628&r1=1457627&r2=1457628&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_2/solr/core/src/test-files/solr/collection1/conf/schema-rest.xml (original)
+++ lucene/dev/branches/lucene_solr_4_2/solr/core/src/test-files/solr/collection1/conf/schema-rest.xml Mon Mar 18 03:39:50 2013
@@ -616,4 +616,11 @@
<copyField source="src_sub_no_ast_i" dest="*_s"/>
<copyField source="src_sub_no_ast_i" dest="*_dest_sub_s"/>
<copyField source="src_sub_no_ast_i" dest="dest_sub_no_ast_s"/>
+
+ <!-- test source glob matching multiple explicit fields: title_stemmed and title_lettertok -->
+ <copyField source="title_*" dest="text"/>
+ <copyField source="title_*" dest="*_s"/>
+ <copyField source="title_*" dest="*_dest_sub_s"/>
+ <copyField source="title_*" dest="dest_sub_no_ast_s"/>
+
</schema>
Modified: lucene/dev/branches/lucene_solr_4_2/solr/core/src/test/org/apache/solr/rest/TestCopyFieldCollectionResource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_2/solr/core/src/test/org/apache/solr/rest/TestCopyFieldCollectionResource.java?rev=1457628&r1=1457627&r2=1457628&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_2/solr/core/src/test/org/apache/solr/rest/TestCopyFieldCollectionResource.java (original)
+++ lucene/dev/branches/lucene_solr_4_2/solr/core/src/test/org/apache/solr/rest/TestCopyFieldCollectionResource.java Mon Mar 18 03:39:50 2013
@@ -78,7 +78,27 @@ public class TestCopyFieldCollectionReso
"/response/arr[@name='copyfields']/lst[ str[@name='source'][.='src_sub_no_ast_i']"
+" and str[@name='sourceDynamicBase'][.='*_i']"
+" and str[@name='dest'][.='dest_sub_no_ast_s']"
- +" and str[@name='destDynamicBase'][.='*_s']]");
+ +" and str[@name='destDynamicBase'][.='*_s']]",
+
+ "/response/arr[@name='copyfields']/lst[ str[@name='source'][.='title_*']"
+ +" and arr[@name='sourceExplicitFields']/str[.='title_stemmed']"
+ +" and arr[@name='sourceExplicitFields']/str[.='title_lettertok']"
+ +" and str[@name='dest'][.='text']]",
+
+ "/response/arr[@name='copyfields']/lst[ str[@name='source'][.='title_*']"
+ +" and arr[@name='sourceExplicitFields']/str[.='title_stemmed']"
+ +" and arr[@name='sourceExplicitFields']/str[.='title_lettertok']"
+ +" and str[@name='dest'][.='*_s']]",
+
+ "/response/arr[@name='copyfields']/lst[ str[@name='source'][.='title_*']"
+ +" and arr[@name='sourceExplicitFields']/str[.='title_stemmed']"
+ +" and arr[@name='sourceExplicitFields']/str[.='title_lettertok']"
+ +" and str[@name='dest'][.='*_dest_sub_s']]",
+
+ "/response/arr[@name='copyfields']/lst[ str[@name='source'][.='title_*']"
+ +" and arr[@name='sourceExplicitFields']/str[.='title_stemmed']"
+ +" and arr[@name='sourceExplicitFields']/str[.='title_lettertok']"
+ +" and str[@name='dest'][.='dest_sub_no_ast_s']]");
}
@Test
@@ -116,11 +136,11 @@ public class TestCopyFieldCollectionReso
@Test
public void testRestrictDest() throws Exception {
assertQ("/schema/copyfields/?indent=on&wt=xml&dest.fl=title,*_s,*_dest_sub_s,dest_sub_no_ast_s",
- "count(/response/arr[@name='copyfields']/lst) = 13", // 3 + 3 + 3 + 4
+ "count(/response/arr[@name='copyfields']/lst) = 16", // 3 + 4 + 4 + 5
"count(/response/arr[@name='copyfields']/lst/str[@name='dest'][.='title']) = 3",
- "count(/response/arr[@name='copyfields']/lst/str[@name='dest'][.='*_s']) = 3",
- "count(/response/arr[@name='copyfields']/lst/str[@name='dest'][.='*_dest_sub_s']) = 3",
- "count(/response/arr[@name='copyfields']/lst/str[@name='dest'][.='dest_sub_no_ast_s']) = 4");
+ "count(/response/arr[@name='copyfields']/lst/str[@name='dest'][.='*_s']) = 4",
+ "count(/response/arr[@name='copyfields']/lst/str[@name='dest'][.='*_dest_sub_s']) = 4",
+ "count(/response/arr[@name='copyfields']/lst/str[@name='dest'][.='dest_sub_no_ast_s']) = 5");
}
@Test
Modified: lucene/dev/branches/lucene_solr_4_2/solr/core/src/test/org/apache/solr/schema/CopyFieldTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_2/solr/core/src/test/org/apache/solr/schema/CopyFieldTest.java?rev=1457628&r1=1457627&r2=1457628&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_2/solr/core/src/test/org/apache/solr/schema/CopyFieldTest.java (original)
+++ lucene/dev/branches/lucene_solr_4_2/solr/core/src/test/org/apache/solr/schema/CopyFieldTest.java Mon Mar 18 03:39:50 2013
@@ -162,4 +162,64 @@ public class CopyFieldTest extends SolrT
assertQ("dynamic source", req
,"//*[@numFound='0']");
}
+
+ @Test
+ public void testExplicitSourceGlob()
+ {
+ SolrCore core = h.getCore();
+ IndexSchema schema = core.getSchema();
+
+ assertTrue("schema should contain explicit field 'sku1'", schema.getFields().containsKey("sku1"));
+ assertTrue("schema should contain explicit field 'sku2'", schema.getFields().containsKey("sku2"));
+ assertNull("'sku*' should not be (or match) a dynamic field", schema.getDynamicPattern("sku*"));
+
+ assertTrue("schema should contain dynamic field '*_s'", schema.getDynamicPattern("*_s").equals("*_s"));
+
+ final String subsetPattern = "*_dest_sub_s";
+ final String dynamicPattern1 = schema.getDynamicPattern(subsetPattern);
+ assertTrue("'" + subsetPattern + "' should match dynamic field '*_s', but instead matches '" + dynamicPattern1 + "'",
+ dynamicPattern1.equals("*_s"));
+
+ final String dest_sub_no_ast_s = "dest_sub_no_ast_s";
+ assertFalse(schema.getFields().containsKey(dest_sub_no_ast_s)); // Should not be an explicit field
+ final String dynamicPattern2 = schema.getDynamicPattern(dest_sub_no_ast_s);
+ assertTrue("'" + dest_sub_no_ast_s + "' should match dynamic field '*_s', but instead matches '" + dynamicPattern2 + "'",
+ dynamicPattern2.equals("*_s"));
+
+ assertU(adoc("id", "A5", "sku1", "10-1839ACX-93", "sku2", "AAM46"));
+ assertU(commit());
+
+ Map<String,String> args = new HashMap<String, String>();
+ args.put( CommonParams.Q, "text:AAM46" );
+ args.put( "indent", "true" );
+ SolrQueryRequest req = new LocalSolrQueryRequest( core, new MapSolrParams( args) );
+ assertQ("sku2 copied to text", req
+ ,"//*[@numFound='1']"
+ ,"//result/doc[1]/str[@name='id'][.='A5']"
+ );
+
+ args = new HashMap<String, String>();
+ args.put( CommonParams.Q, "1_s:10-1839ACX-93" );
+ args.put( "indent", "true" );
+ req = new LocalSolrQueryRequest( core, new MapSolrParams( args) );
+ assertQ("sku1 copied to dynamic dest *_s", req
+ ,"//*[@numFound='1']"
+ ,"//result/doc[1]/str[@name='id'][.='A5']"
+ ,"//result/doc[1]/arr[@name='sku1']/str[.='10-1839ACX-93']"
+ );
+
+ args = new HashMap<String, String>();
+ args.put( CommonParams.Q, "1_dest_sub_s:10-1839ACX-93" );
+ args.put( "indent", "true" );
+ req = new LocalSolrQueryRequest( core, new MapSolrParams( args) );
+ assertQ("sku1 copied to *_dest_sub_s (*_s subset pattern)", req
+ ,"//*[@numFound='1']");
+
+ args = new HashMap<String, String>();
+ args.put( CommonParams.Q, "dest_sub_no_ast_s:AAM46" );
+ args.put( "indent", "true" );
+ req = new LocalSolrQueryRequest( core, new MapSolrParams( args) );
+ assertQ("sku2 copied to dest_sub_no_ast_s (*_s subset pattern no asterisk)", req
+ ,"//*[@numFound='1']");
+ }
}