You are viewing a plain text version of this content. The canonical link for it is here.
Posted to solr-commits@lucene.apache.org by sh...@apache.org on 2009/04/13 11:01:01 UTC
svn commit: r764379 - in
/lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport:
DataConfig.java DataImportHandlerException.java DocBuilder.java
Author: shalin
Date: Mon Apr 13 09:00:59 2009
New Revision: 764379
URL: http://svn.apache.org/viewvc?rev=764379&view=rev
Log:
SOLR-1059 -- Fixing bug where skipping a row containing nested entities did not skip the nested entities. Handling special flag variables is in one method now.
Modified:
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataConfig.java
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImportHandlerException.java
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java
Modified: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataConfig.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataConfig.java?rev=764379&r1=764378&r2=764379&view=diff
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataConfig.java (original)
+++ lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataConfig.java Mon Apr 13 09:00:59 2009
@@ -87,7 +87,7 @@
public boolean isDocRoot = false;
- public List<Field> fields;
+ public List<Field> fields = new ArrayList<Field>();
public List<Map<String, String>> allFieldsList = new ArrayList<Map<String, String>>();
@@ -104,7 +104,7 @@
public Script script;
- public Map<String, List<Field>> colNameVsField;
+ public Map<String, List<Field>> colNameVsField = new HashMap<String, List<Field>>();
public Entity() {
}
@@ -121,8 +121,6 @@
dataSource = getStringAttribute(element, DataImporter.DATA_SRC, null);
allAttributes = getAllAttributes(element);
List<Element> n = getChildNodes(element, "field");
- fields = new ArrayList<Field>();
- colNameVsField = new HashMap<String, List<Field>>();
for (Element elem : n) {
Field field = new Field(elem);
fields.add(field);
Modified: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImportHandlerException.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImportHandlerException.java?rev=764379&r1=764378&r2=764379&view=diff
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImportHandlerException.java (original)
+++ lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImportHandlerException.java Mon Apr 13 09:00:59 2009
@@ -31,7 +31,7 @@
public boolean debugged = false;
- public static final int SEVERE = 500, WARN = 400, SKIP = 300;
+ public static final int SEVERE = 500, WARN = 400, SKIP = 300, SKIP_ROW =301;
public DataImportHandlerException(int err) {
super();
Modified: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java?rev=764379&r1=764378&r2=764379&view=diff
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java (original)
+++ lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java Mon Apr 13 09:00:59 2009
@@ -354,7 +354,8 @@
writer.log(SolrWriter.ENTITY_OUT, entity.name, arow);
}
importStatistics.rowsCount.incrementAndGet();
- if (entity.fields != null && doc != null) {
+ if (doc != null) {
+ handleSpecialCommands(arow);
addFields(entity, doc, arow);
}
if (isRoot)
@@ -382,6 +383,9 @@
if (verboseDebug) {
writer.log(SolrWriter.ENTITY_EXCEPTION, entity.name, e);
}
+ if(e.getErrCode() == DataImportHandlerException.SKIP_ROW){
+ continue;
+ }
if (isRoot) {
if (e.getErrCode() == DataImportHandlerException.SKIP) {
importStatistics.skipDocCount.getAndIncrement();
@@ -424,37 +428,53 @@
doc.setDocumentBoost(value);
}
+ private void handleSpecialCommands(Map<String, Object> arow) {
+ Object value = arow.get("$deleteDocById");
+ if (value != null) {
+ if (value instanceof Collection) {
+ Collection collection = (Collection) value;
+ for (Object o : collection) {
+ writer.deleteDoc(o.toString());
+ }
+ } else {
+ writer.deleteDoc(value);
+ }
+ }
+ value = arow.get("$deleteDocByQuery");
+ if (value != null) {
+ if (value instanceof Collection) {
+ Collection collection = (Collection) value;
+ for (Object o : collection) {
+ writer.deleteByQuery(o.toString());
+ }
+
+ } else {
+ writer.deleteByQuery(value.toString());
+ }
+ }
+ value = arow.get("$skipDoc");
+ if (value != null) {
+ if (Boolean.parseBoolean(value.toString())) {
+ throw new DataImportHandlerException(DataImportHandlerException.SKIP,
+ "Document skipped :" + arow);
+ }
+ }
+
+ value = arow.get("$skipRow");
+ if (value != null) {
+ if (Boolean.parseBoolean(value.toString())) {
+ throw new DataImportHandlerException(DataImportHandlerException.SKIP_ROW);
+ }
+ }
+ }
+
@SuppressWarnings("unchecked")
private void addFields(DataConfig.Entity entity, SolrInputDocument doc, Map<String, Object> arow) {
- Object s = arow.get("$skipRow");
- if (s != null && Boolean.parseBoolean(s.toString())) {
- return;
- }
for (Map.Entry<String, Object> entry : arow.entrySet()) {
String key = entry.getKey();
Object value = entry.getValue();
if (value == null) continue;
- if (key.startsWith("$")) {
- if ("$deleteDocById".equals(key)) {
- if (value instanceof Collection) {
- Collection collection = (Collection) value;
- for (Object o : collection) {
- writer.deleteDoc(o.toString());
- }
- } else {
- writer.deleteDoc(value);
- }
- }
- if ("$deleteDocByQuery".equals(key)) {
- writer.deleteByQuery(entry.getValue().toString());
- }
- if ("$skipDoc".equals(key) && Boolean.parseBoolean(value.toString())) {
- throw new DataImportHandlerException(DataImportHandlerException.SKIP,
- "Document skipped :" + arow);
- }
- // All fields starting with $ are special values and don't need to be added
- continue;
- }
+ if (key.startsWith("$")) continue;
List<DataConfig.Field> field = entity.colNameVsField.get(key);
if (field == null && dataImporter.getSchema() != null) {
// This can be a dynamic field or a field which does not have an entry in data-config ( an implicit field)