You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by er...@apache.org on 2015/01/05 21:50:40 UTC
svn commit: r1649657 - in /lucene/dev/trunk/solr: ./
core/src/java/org/apache/solr/schema/
core/src/test/org/apache/solr/rest/schema/
Author: erick
Date: Mon Jan 5 20:50:39 2015
New Revision: 1649657
URL: http://svn.apache.org/r1649657
Log:
SOLR-6666: Dynamic copy fields are considering all dynamic fields, causing a significant performance impact on indexing documents
Modified:
lucene/dev/trunk/solr/CHANGES.txt
lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/IndexSchema.java
lucene/dev/trunk/solr/core/src/test/org/apache/solr/rest/schema/TestCopyFieldCollectionResource.java
lucene/dev/trunk/solr/core/src/test/org/apache/solr/rest/schema/TestSchemaResource.java
Modified: lucene/dev/trunk/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/CHANGES.txt?rev=1649657&r1=1649656&r2=1649657&view=diff
==============================================================================
--- lucene/dev/trunk/solr/CHANGES.txt (original)
+++ lucene/dev/trunk/solr/CHANGES.txt Mon Jan 5 20:50:39 2015
@@ -419,6 +419,10 @@ Optimizations
hl.usePhraseHighlighter, and can be more efficient handling data from term vectors.
(David Smiley)
+* SOLR-6666: Dynamic copy fields are considering all dynamic fields, causing
+ a significant performance impact on indexing documents. (Liram Vardi via Erick
+ Erickson, Steve Rowe)
+
Other Changes
----------------------
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/IndexSchema.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/IndexSchema.java?rev=1649657&r1=1649656&r2=1649657&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/IndexSchema.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/IndexSchema.java Mon Jan 5 20:50:39 2015
@@ -901,37 +901,49 @@ public class IndexSchema {
String msg = "copyField dest :'" + dest + "' is not an explicit field and doesn't match a dynamicField.";
throw new SolrException(ErrorCode.SERVER_ERROR, msg);
}
- if (sourceIsDynamicFieldReference || sourceIsGlob) {
- if (null != destDynamicField) { // source: glob or no-asterisk dynamic field ref; dest: dynamic field ref
+ if (sourceIsGlob) {
+ if (null != destDynamicField) { // source: glob ; dest: dynamic field ref
registerDynamicCopyField(new DynamicCopy(source, destDynamicField, maxChars, sourceDynamicBase, destDynamicBase));
incrementCopyFieldTargetCount(destSchemaField);
- } else { // source: glob or no-asterisk dynamic field ref; dest: explicit field
+ } else { // source: glob ; dest: explicit field
destDynamicField = new DynamicField(destSchemaField);
registerDynamicCopyField(new DynamicCopy(source, destDynamicField, maxChars, sourceDynamicBase, null));
incrementCopyFieldTargetCount(destSchemaField);
}
- } else {
- if (null != destDynamicField) { // source: explicit field; dest: dynamic field reference
+ } else if (sourceIsDynamicFieldReference) {
+ if (null != destDynamicField) { // source: no-asterisk dynamic field ref ; dest: dynamic field ref
+ registerDynamicCopyField(new DynamicCopy(source, destDynamicField, maxChars, sourceDynamicBase, destDynamicBase));
+ incrementCopyFieldTargetCount(destSchemaField);
+ } else { // source: no-asterisk dynamic field ref ; dest: explicit field
+ sourceSchemaField = getField(source);
+ registerExplicitSrcAndDestFields(source, maxChars, destSchemaField, sourceSchemaField);
+ }
+ } else {
+ if (null != destDynamicField) { // source: explicit field ; dest: dynamic field reference
if (destDynamicField.pattern instanceof DynamicReplacement.DynamicPattern.NameEquals) {
// Dynamic dest with no asterisk is acceptable
registerDynamicCopyField(new DynamicCopy(source, destDynamicField, maxChars, sourceDynamicBase, destDynamicBase));
incrementCopyFieldTargetCount(destSchemaField);
- } else {
+ } else { // source: explicit field ; dest: dynamic field with an asterisk
String msg = "copyField only supports a dynamic destination with an asterisk "
+ "if the source also has an asterisk";
throw new SolrException(ErrorCode.SERVER_ERROR, msg);
}
- } else { // source & dest: explicit fields
- List<CopyField> copyFieldList = copyFieldsMap.get(source);
- if (copyFieldList == null) {
- copyFieldList = new ArrayList<>();
- copyFieldsMap.put(source, copyFieldList);
- }
- copyFieldList.add(new CopyField(sourceSchemaField, destSchemaField, maxChars));
- incrementCopyFieldTargetCount(destSchemaField);
+ } else { // source & dest: explicit fields
+ registerExplicitSrcAndDestFields(source, maxChars, destSchemaField, sourceSchemaField);
}
}
}
+
+ private void registerExplicitSrcAndDestFields(String source, int maxChars, SchemaField destSchemaField, SchemaField sourceSchemaField) {
+ List<CopyField> copyFieldList = copyFieldsMap.get(source);
+ if (copyFieldList == null) {
+ copyFieldList = new ArrayList<>();
+ copyFieldsMap.put(source, copyFieldList);
+ }
+ copyFieldList.add(new CopyField(sourceSchemaField, destSchemaField, maxChars));
+ incrementCopyFieldTargetCount(destSchemaField);
+ }
private void incrementCopyFieldTargetCount(SchemaField dest) {
copyFieldTargetCounts.put(dest, copyFieldTargetCounts.containsKey(dest) ? copyFieldTargetCounts.get(dest) + 1 : 1);
Modified: lucene/dev/trunk/solr/core/src/test/org/apache/solr/rest/schema/TestCopyFieldCollectionResource.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/rest/schema/TestCopyFieldCollectionResource.java?rev=1649657&r1=1649656&r2=1649657&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/rest/schema/TestCopyFieldCollectionResource.java (original)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/rest/schema/TestCopyFieldCollectionResource.java Mon Jan 5 20:50:39 2015
@@ -23,7 +23,10 @@ public class TestCopyFieldCollectionReso
@Test
public void testGetAllCopyFields() throws Exception {
assertQ("/schema/copyfields?indent=on&wt=xml",
- "/response/arr[@name='copyFields']/lst[ str[@name='source'][.='title']"
+ "/response/arr[@name='copyFields']/lst[ str[@name='source'][.='src_sub_no_ast_i']"
+ +" and str[@name='dest'][.='title']]",
+
+ "/response/arr[@name='copyFields']/lst[ str[@name='source'][.='title']"
+" and str[@name='dest'][.='title_stemmed']"
+" and int[@name='maxChars'][.='200']]",
@@ -65,10 +68,6 @@ public class TestCopyFieldCollectionReso
"/response/arr[@name='copyFields']/lst[ str[@name='source'][.='src_sub_no_ast_i']"
+" and str[@name='sourceDynamicBase'][.='*_i']"
- +" and str[@name='dest'][.='title']]",
-
- "/response/arr[@name='copyFields']/lst[ str[@name='source'][.='src_sub_no_ast_i']"
- +" and str[@name='sourceDynamicBase'][.='*_i']"
+" and str[@name='dest'][.='*_s']]",
"/response/arr[@name='copyFields']/lst[ str[@name='source'][.='src_sub_no_ast_i']"
@@ -105,19 +104,19 @@ public class TestCopyFieldCollectionReso
@Test
public void testJsonGetAllCopyFields() throws Exception {
assertJQ("/schema/copyfields?indent=on&wt=json",
- "/copyFields/[6]=={'source':'title','dest':'dest_sub_no_ast_s','destDynamicBase':'*_s'}",
+ "/copyFields/[1]=={'source':'src_sub_no_ast_i','dest':'title'}",
+ "/copyFields/[7]=={'source':'title','dest':'dest_sub_no_ast_s','destDynamicBase':'*_s'}",
- "/copyFields/[7]=={'source':'*_i','dest':'title'}",
- "/copyFields/[8]=={'source':'*_i','dest':'*_s'}",
- "/copyFields/[9]=={'source':'*_i','dest':'*_dest_sub_s','destDynamicBase':'*_s'}",
- "/copyFields/[10]=={'source':'*_i','dest':'dest_sub_no_ast_s','destDynamicBase':'*_s'}",
-
- "/copyFields/[11]=={'source':'*_src_sub_i','sourceDynamicBase':'*_i','dest':'title'}",
- "/copyFields/[12]=={'source':'*_src_sub_i','sourceDynamicBase':'*_i','dest':'*_s'}",
- "/copyFields/[13]=={'source':'*_src_sub_i','sourceDynamicBase':'*_i','dest':'*_dest_sub_s','destDynamicBase':'*_s'}",
- "/copyFields/[14]=={'source':'*_src_sub_i','sourceDynamicBase':'*_i','dest':'dest_sub_no_ast_s','destDynamicBase':'*_s'}",
+ "/copyFields/[8]=={'source':'*_i','dest':'title'}",
+ "/copyFields/[9]=={'source':'*_i','dest':'*_s'}",
+ "/copyFields/[10]=={'source':'*_i','dest':'*_dest_sub_s','destDynamicBase':'*_s'}",
+ "/copyFields/[11]=={'source':'*_i','dest':'dest_sub_no_ast_s','destDynamicBase':'*_s'}",
+
+ "/copyFields/[12]=={'source':'*_src_sub_i','sourceDynamicBase':'*_i','dest':'title'}",
+ "/copyFields/[13]=={'source':'*_src_sub_i','sourceDynamicBase':'*_i','dest':'*_s'}",
+ "/copyFields/[14]=={'source':'*_src_sub_i','sourceDynamicBase':'*_i','dest':'*_dest_sub_s','destDynamicBase':'*_s'}",
+ "/copyFields/[15]=={'source':'*_src_sub_i','sourceDynamicBase':'*_i','dest':'dest_sub_no_ast_s','destDynamicBase':'*_s'}",
- "/copyFields/[15]=={'source':'src_sub_no_ast_i','sourceDynamicBase':'*_i','dest':'title'}",
"/copyFields/[16]=={'source':'src_sub_no_ast_i','sourceDynamicBase':'*_i','dest':'*_s'}",
"/copyFields/[17]=={'source':'src_sub_no_ast_i','sourceDynamicBase':'*_i','dest':'*_dest_sub_s','destDynamicBase':'*_s'}",
"/copyFields/[18]=={'source':'src_sub_no_ast_i','sourceDynamicBase':'*_i','dest':'dest_sub_no_ast_s','destDynamicBase':'*_s'}");
Modified: lucene/dev/trunk/solr/core/src/test/org/apache/solr/rest/schema/TestSchemaResource.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/rest/schema/TestSchemaResource.java?rev=1649657&r1=1649656&r2=1649657&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/rest/schema/TestSchemaResource.java (original)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/rest/schema/TestSchemaResource.java Mon Jan 5 20:50:39 2015
@@ -131,19 +131,19 @@ public class TestSchemaResource extends
"/schema/dynamicFields/[1]/name=='ignored_*'",
"/schema/dynamicFields/[2]/name=='*_mfacet'",
- "/schema/copyFields/[6]=={'source':'title','dest':'dest_sub_no_ast_s'}",
+ "/schema/copyFields/[1]=={'source':'src_sub_no_ast_i','dest':'title'}",
- "/schema/copyFields/[7]=={'source':'*_i','dest':'title'}",
- "/schema/copyFields/[8]=={'source':'*_i','dest':'*_s'}",
- "/schema/copyFields/[9]=={'source':'*_i','dest':'*_dest_sub_s'}",
- "/schema/copyFields/[10]=={'source':'*_i','dest':'dest_sub_no_ast_s'}",
+ "/schema/copyFields/[7]=={'source':'title','dest':'dest_sub_no_ast_s'}",
+ "/schema/copyFields/[8]=={'source':'*_i','dest':'title'}",
+ "/schema/copyFields/[9]=={'source':'*_i','dest':'*_s'}",
+ "/schema/copyFields/[10]=={'source':'*_i','dest':'*_dest_sub_s'}",
+ "/schema/copyFields/[11]=={'source':'*_i','dest':'dest_sub_no_ast_s'}",
- "/schema/copyFields/[11]=={'source':'*_src_sub_i','dest':'title'}",
- "/schema/copyFields/[12]=={'source':'*_src_sub_i','dest':'*_s'}",
- "/schema/copyFields/[13]=={'source':'*_src_sub_i','dest':'*_dest_sub_s'}",
- "/schema/copyFields/[14]=={'source':'*_src_sub_i','dest':'dest_sub_no_ast_s'}",
+ "/schema/copyFields/[12]=={'source':'*_src_sub_i','dest':'title'}",
+ "/schema/copyFields/[13]=={'source':'*_src_sub_i','dest':'*_s'}",
+ "/schema/copyFields/[14]=={'source':'*_src_sub_i','dest':'*_dest_sub_s'}",
+ "/schema/copyFields/[15]=={'source':'*_src_sub_i','dest':'dest_sub_no_ast_s'}",
- "/schema/copyFields/[15]=={'source':'src_sub_no_ast_i','dest':'title'}",
"/schema/copyFields/[16]=={'source':'src_sub_no_ast_i','dest':'*_s'}",
"/schema/copyFields/[17]=={'source':'src_sub_no_ast_i','dest':'*_dest_sub_s'}",
"/schema/copyFields/[18]=={'source':'src_sub_no_ast_i','dest':'dest_sub_no_ast_s'}");