You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sh...@apache.org on 2013/12/03 15:13:13 UTC
svn commit: r1547394 - in /lucene/dev/trunk/solr: ./
contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/
contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/
contrib/dataimporthandler/src/test/org/apache/solr...
Author: shalin
Date: Tue Dec 3 14:13:13 2013
New Revision: 1547394
URL: http://svn.apache.org/r1547394
Log:
SOLR-5527: DIH logs spurious warning for special commands
Modified:
lucene/dev/trunk/solr/CHANGES.txt
lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java
lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/EntityProcessorBase.java
lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/XPathEntityProcessor.java
lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/DIHConfiguration.java
lucene/dev/trunk/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDocBuilder2.java
Modified: lucene/dev/trunk/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/CHANGES.txt?rev=1547394&r1=1547393&r2=1547394&view=diff
==============================================================================
--- lucene/dev/trunk/solr/CHANGES.txt (original)
+++ lucene/dev/trunk/solr/CHANGES.txt Tue Dec 3 14:13:13 2013
@@ -187,6 +187,8 @@ Bug Fixes
* SOLR-5204: StatsComponent and SpellCheckComponent do not support the
shards.tolerant=true parameter. (Anca Kopetz, shalin)
+* SOLR-5527: DIH logs spurious warning for special commands. (shalin)
+
Optimizations
----------------------
Modified: lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java?rev=1547394&r1=1547393&r2=1547394&view=diff
==============================================================================
--- lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java (original)
+++ lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java Tue Dec 3 14:13:13 2013
@@ -51,6 +51,11 @@ public class DocBuilder {
private static final Logger LOG = LoggerFactory.getLogger(DocBuilder.class);
private static final Date EPOCH = new Date(0);
+ public static final String DELETE_DOC_BY_ID = "$deleteDocById";
+ public static final String DELETE_DOC_BY_QUERY = "$deleteDocByQuery";
+ public static final String DOC_BOOST = "$docBoost";
+ public static final String SKIP_DOC = "$skipDoc";
+ public static final String SKIP_ROW = "$skipRow";
DataImporter dataImporter;
@@ -568,7 +573,7 @@ public class DocBuilder {
}
private void handleSpecialCommands(Map<String, Object> arow, DocWrapper doc) {
- Object value = arow.get("$deleteDocById");
+ Object value = arow.get(DELETE_DOC_BY_ID);
if (value != null) {
if (value instanceof Collection) {
Collection collection = (Collection) value;
@@ -581,7 +586,7 @@ public class DocBuilder {
importStatistics.deletedDocCount.incrementAndGet();
}
}
- value = arow.get("$deleteDocByQuery");
+ value = arow.get(DELETE_DOC_BY_QUERY);
if (value != null) {
if (value instanceof Collection) {
Collection collection = (Collection) value;
@@ -594,7 +599,7 @@ public class DocBuilder {
importStatistics.deletedDocCount.incrementAndGet();
}
}
- value = arow.get("$docBoost");
+ value = arow.get(DOC_BOOST);
if (value != null) {
float value1 = 1.0f;
if (value instanceof Number) {
@@ -605,7 +610,7 @@ public class DocBuilder {
doc.setDocumentBoost(value1);
}
- value = arow.get("$skipDoc");
+ value = arow.get(SKIP_DOC);
if (value != null) {
if (Boolean.parseBoolean(value.toString())) {
throw new DataImportHandlerException(DataImportHandlerException.SKIP,
@@ -613,7 +618,7 @@ public class DocBuilder {
}
}
- value = arow.get("$skipRow");
+ value = arow.get(SKIP_ROW);
if (value != null) {
if (Boolean.parseBoolean(value.toString())) {
throw new DataImportHandlerException(DataImportHandlerException.SKIP_ROW);
Modified: lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/EntityProcessorBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/EntityProcessorBase.java?rev=1547394&r1=1547393&r2=1547394&view=diff
==============================================================================
--- lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/EntityProcessorBase.java (original)
+++ lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/EntityProcessorBase.java Tue Dec 3 14:13:13 2013
@@ -153,7 +153,4 @@ public class EntityProcessorBase extends
public static final String CONTINUE = "continue";
public static final String SKIP = "skip";
-
- public static final String SKIP_DOC = "$skipDoc";
-
}
Modified: lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/XPathEntityProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/XPathEntityProcessor.java?rev=1547394&r1=1547393&r2=1547394&view=diff
==============================================================================
--- lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/XPathEntityProcessor.java (original)
+++ lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/XPathEntityProcessor.java Tue Dec 3 14:13:13 2013
@@ -330,7 +330,7 @@ public class XPathEntityProcessor extend
} else if (SKIP.equals(onError)) {
LOG.warn(msg, e);
Map<String, Object> map = new HashMap<String, Object>();
- map.put(SKIP_DOC, Boolean.TRUE);
+ map.put(DocBuilder.SKIP_DOC, Boolean.TRUE);
rows.add(map);
} else if (CONTINUE.equals(onError)) {
LOG.warn(msg, e);
Modified: lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/DIHConfiguration.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/DIHConfiguration.java?rev=1547394&r1=1547393&r2=1547394&view=diff
==============================================================================
--- lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/DIHConfiguration.java (original)
+++ lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/DIHConfiguration.java Tue Dec 3 14:13:13 2013
@@ -8,6 +8,7 @@ import java.util.Locale;
import java.util.Map;
import org.apache.solr.handler.dataimport.DataImporter;
+import org.apache.solr.handler.dataimport.DocBuilder;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.SchemaField;
import org.slf4j.Logger;
@@ -111,7 +112,7 @@ public class DIHConfiguration {
for (Map.Entry<String,EntityField> entry : fields.entrySet()) {
EntityField fld = entry.getValue();
SchemaField field = getSchemaField(fld.getName());
- if (field == null) {
+ if (field == null && !isSpecialCommand(fld.getName())) {
LOG.info("The field :" + fld.getName() + " present in DataConfig does not have a counterpart in Solr Schema");
}
}
@@ -178,4 +179,13 @@ public class DIHConfiguration {
public IndexSchema getSchema() {
return schema;
}
+
+ public static boolean isSpecialCommand(String fld) {
+ return DocBuilder.DELETE_DOC_BY_ID.equals(fld) ||
+ DocBuilder.DELETE_DOC_BY_QUERY.equals(fld) ||
+ DocBuilder.DOC_BOOST.equals(fld) ||
+ DocBuilder.SKIP_DOC.equals(fld) ||
+ DocBuilder.SKIP_ROW.equals(fld);
+
+ }
}
\ No newline at end of file
Modified: lucene/dev/trunk/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDocBuilder2.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDocBuilder2.java?rev=1547394&r1=1547393&r2=1547394&view=diff
==============================================================================
--- lucene/dev/trunk/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDocBuilder2.java (original)
+++ lucene/dev/trunk/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDocBuilder2.java Tue Dec 3 14:13:13 2013
@@ -132,7 +132,7 @@ public class TestDocBuilder2 extends Abs
public void testSkipDoc() throws Exception {
List rows = new ArrayList();
rows.add(createMap("id", "1", "desc", "one"));
- rows.add(createMap("id", "2", "desc", "two", "$skipDoc", "true"));
+ rows.add(createMap("id", "2", "desc", "two", DocBuilder.SKIP_DOC, "true"));
MockDataSource.setIterator("select * from x", rows.iterator());
runFullImport(dataConfigWithDynamicTransformer);
@@ -146,7 +146,7 @@ public class TestDocBuilder2 extends Abs
public void testSkipRow() throws Exception {
List rows = new ArrayList();
rows.add(createMap("id", "1", "desc", "one"));
- rows.add(createMap("id", "2", "desc", "two", "$skipRow", "true"));
+ rows.add(createMap("id", "2", "desc", "two", DocBuilder.SKIP_ROW, "true"));
MockDataSource.setIterator("select * from x", rows.iterator());
runFullImport(dataConfigWithDynamicTransformer);
@@ -166,7 +166,7 @@ public class TestDocBuilder2 extends Abs
MockDataSource.setIterator("3", rows.iterator());
rows = new ArrayList();
- rows.add(createMap("name_s", "xyz", "$skipRow", "true"));
+ rows.add(createMap("name_s", "xyz", DocBuilder.SKIP_ROW, "true"));
MockDataSource.setIterator("4", rows.iterator());
runFullImport(dataConfigWithTwoEntities);
@@ -197,7 +197,7 @@ public class TestDocBuilder2 extends Abs
List rows = new ArrayList();
rows.add(createMap("id", "1", "desc", "one"));
rows.add(createMap("id", "2", "desc", "two"));
- rows.add(createMap("id", "3", "desc", "two", "$deleteDocById", "2"));
+ rows.add(createMap("id", "3", "desc", "two", DocBuilder.DELETE_DOC_BY_ID, "2"));
MockDataSource.setIterator("select * from x", rows.iterator());
runFullImport(dataConfigForSkipTransform);
@@ -213,7 +213,7 @@ public class TestDocBuilder2 extends Abs
rows = new ArrayList();
rows.add(createMap("id", "1", "desc", "one"));
rows.add(createMap("id", "2", "desc", "one"));
- rows.add(createMap("id", "3", "desc", "two", "$deleteDocByQuery", "desc:one"));
+ rows.add(createMap("id", "3", "desc", "two", DocBuilder.DELETE_DOC_BY_QUERY, "desc:one"));
MockDataSource.setIterator("select * from x", rows.iterator());
runFullImport(dataConfigForSkipTransform);
@@ -227,7 +227,7 @@ public class TestDocBuilder2 extends Abs
MockDataSource.clearCache();
rows = new ArrayList();
- rows.add(createMap("$deleteDocById", "3"));
+ rows.add(createMap(DocBuilder.DELETE_DOC_BY_ID, "3"));
MockDataSource.setIterator("select * from x", rows.iterator());
runFullImport(dataConfigForSkipTransform, createMap("clean","false"));
assertQ(req("id:3"), "//*[@numFound='0']");