You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ko...@apache.org on 2010/11/30 17:10:29 UTC
svn commit: r1040608 - in /lucene/dev/trunk/solr: ./
contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/
contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/
Author: koji
Date: Tue Nov 30 16:10:29 2010
New Revision: 1040608
URL: http://svn.apache.org/viewvc?rev=1040608&view=rev
Log:
SOLR-2252: when child entity is the root in nested entities, delta-import doesn't work
Added:
lucene/dev/trunk/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta3.java
Modified:
lucene/dev/trunk/solr/CHANGES.txt
lucene/dev/trunk/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataConfig.java
lucene/dev/trunk/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java
lucene/dev/trunk/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta.java
lucene/dev/trunk/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta2.java
Modified: lucene/dev/trunk/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/CHANGES.txt?rev=1040608&r1=1040607&r2=1040608&view=diff
==============================================================================
--- lucene/dev/trunk/solr/CHANGES.txt (original)
+++ lucene/dev/trunk/solr/CHANGES.txt Tue Nov 30 16:10:29 2010
@@ -259,7 +259,7 @@ New Features
allows you to customize how WordDelimiterFilter tokenizes text with
a configuration file. (Peter Karich, rmuir)
-* SOLR-1665: Add debug component options for timings, results and query info only (gsingers, hossman, yonik)
+* SOLR-1665: Add debug component options for timings, results and query info only (gsingers, hossman, yonik)
* SOLR-2099: Add ability to throttle rsync based replication using rsync option --bwlimit.
(Brandon Evans via koji)
@@ -555,6 +555,9 @@ Bug Fixes
true/on/yes (for TRUE) and false/off/no (for FALSE) can be used for sub-options
(debug, verbose, synchronous, commit, clean, optimize) for full/delta-import commands. (koji)
+* SOLR-2252: When a child entity in nested entities is rootEntity="true", delta-import doesn't work.
+ (koji)
+
Other Changes
----------------------
Modified: lucene/dev/trunk/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataConfig.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataConfig.java?rev=1040608&r1=1040607&r2=1040608&view=diff
==============================================================================
--- lucene/dev/trunk/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataConfig.java (original)
+++ lucene/dev/trunk/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataConfig.java Tue Nov 30 16:10:29 2010
@@ -175,6 +175,10 @@ public class DataConfig {
public String getPk(){
return pk == null ? pkMappingFromSchema : pk;
}
+
+ public String getSchemaPk(){
+ return pkMappingFromSchema != null ? pkMappingFromSchema : pk;
+ }
}
public static class Script {
Modified: lucene/dev/trunk/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java?rev=1040608&r1=1040607&r2=1040608&view=diff
==============================================================================
--- lucene/dev/trunk/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java (original)
+++ lucene/dev/trunk/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java Tue Nov 30 16:10:29 2010
@@ -314,9 +314,10 @@ public class DocBuilder {
Iterator<Map<String, Object>> iter = deletedKeys.iterator();
while (iter.hasNext()) {
Map<String, Object> map = iter.next();
- Object key = map.get(root.getPk());
+ String keyName = root.isDocRoot ? root.getPk() : root.getSchemaPk();
+ Object key = map.get(keyName);
if(key == null) {
- LOG.warn("no key was available for deleteted pk query");
+ LOG.warn("no key was available for deleteted pk query. keyName = " + keyName);
continue;
}
writer.deleteDoc(key);
@@ -607,7 +608,8 @@ public class DocBuilder {
if (entity.entities != null) {
vr.addNamespace(entity.name, arow);
for (DataConfig.Entity child : entity.entities) {
- buildDocument(vr, doc, null, child, false, ctx);
+ buildDocument(vr, doc,
+ child.isDocRoot ? pk : null, child, false, ctx);
}
vr.removeNamespace(entity.name);
}
@@ -910,8 +912,9 @@ public class DocBuilder {
if (entity.isDocRoot)
deletedRows.addAll(deletedSet);
- return entity.isDocRoot ? myModifiedPks : new HashSet<Map<String, Object>>(
- parentKeyList);
+ // Do not use entity.isDocRoot here because one of descendant entities may set rootEntity="true"
+ return entity.parentEntity == null ?
+ myModifiedPks : new HashSet<Map<String, Object>>(parentKeyList);
}
private void getModifiedParentRows(VariableResolverImpl resolver,
Modified: lucene/dev/trunk/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta.java?rev=1040608&r1=1040607&r2=1040608&view=diff
==============================================================================
--- lucene/dev/trunk/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta.java (original)
+++ lucene/dev/trunk/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta.java Tue Nov 30 16:10:29 2010
@@ -40,6 +40,23 @@ public class TestSqlEntityProcessorDelta
private static final String DELETED_PK_QUERY = "select id from x where last_modified > NOW AND deleted='true'";
+ private static final String dataConfig_delta =
+ "<dataConfig>" +
+ " <dataSource type=\"MockDataSource\"/>\n" +
+ " <document>\n" +
+ " <entity name=\"x\" transformer=\"TemplateTransformer\"" +
+ " query=\"" + FULLIMPORT_QUERY + "\"" +
+ " deletedPkQuery=\"" + DELETED_PK_QUERY + "\"" +
+ " deltaImportQuery=\"select * from x where id='${dih.delta.id}'\"" +
+ " deltaQuery=\"" + DELTA_QUERY + "\">\n" +
+ " <field column=\"id\" name=\"id\"/>\n" +
+ " <entity name=\"y\" query=\"select * from y where y.A='${x.id}'\">\n" +
+ " <field column=\"desc\" />\n" +
+ " </entity>\n" +
+ " </entity>\n" +
+ " </document>\n" +
+ "</dataConfig>\n";
+
@BeforeClass
public static void beforeClass() throws Exception {
initCore("dataimport-solrconfig.xml", "dataimport-schema.xml");
@@ -115,8 +132,8 @@ public class TestSqlEntityProcessorDelta
List childRow = new ArrayList();
childRow.add(createMap("desc", "hello"));
- MockDataSource.setIterator("select * from y where y.A='1'", childRow
- .iterator());
+ MockDataSource.setIterator("select * from y where y.A='1'",
+ childRow.iterator());
runDeltaImport(dataConfig_delta);
@@ -270,18 +287,4 @@ public class TestSqlEntityProcessorDelta
assertQ(req("desc:hello"), "//*[@numFound='0']");
assertQ(req("desc:goodbye"), "//*[@numFound='1']");
}
-
- private static String dataConfig_delta = "<dataConfig><dataSource type=\"MockDataSource\"/>\n"
- + " <document>\n"
- + " <entity name=\"x\" transformer=\"TemplateTransformer\""
- + " query=\"" + FULLIMPORT_QUERY + "\""
- + " deletedPkQuery=\"" + DELETED_PK_QUERY + "\""
- + " deltaImportQuery=\"select * from x where id='${dataimporter.delta.id}'\""
- + " deltaQuery=\"" + DELTA_QUERY + "\">\n"
- + " <field column=\"id\" name=\"id\"/>\n"
- + " <entity name=\"y\" query=\"select * from y where y.A='${x.id}'\">\n"
- + " <field column=\"desc\" />\n"
- + " </entity>\n" + " </entity>\n"
- + " </document>\n" + "</dataConfig>\n";
-
}
Modified: lucene/dev/trunk/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta2.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta2.java?rev=1040608&r1=1040607&r2=1040608&view=diff
==============================================================================
--- lucene/dev/trunk/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta2.java (original)
+++ lucene/dev/trunk/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta2.java Tue Nov 30 16:10:29 2010
@@ -40,6 +40,23 @@ public class TestSqlEntityProcessorDelta
private static final String DELETED_PK_QUERY = "select id from x where last_modified > NOW AND deleted='true'";
+ private static final String dataConfig_delta2 =
+ "<dataConfig>" +
+ " <dataSource type=\"MockDataSource\"/>\n" +
+ " <document>\n" +
+ " <entity name=\"x\" transformer=\"TemplateTransformer\"" +
+ " query=\"" + FULLIMPORT_QUERY + "\"" +
+ " deletedPkQuery=\"" + DELETED_PK_QUERY + "\"" +
+ " deltaImportQuery=\"select * from x where id='${dih.delta.id}'\"" +
+ " deltaQuery=\"" + DELTA_QUERY + "\">\n" +
+ " <field column=\"tmpid\" template=\"prefix-${x.id}\" name=\"solr_id\"/>\n" +
+ " <entity name=\"y\" query=\"select * from y where y.A='${x.id}'\">\n" +
+ " <field column=\"desc\" />\n" +
+ " </entity>\n" +
+ " </entity>\n" +
+ " </document>\n" +
+ "</dataConfig>\n";
+
@BeforeClass
public static void beforeClass() throws Exception {
initCore("dataimport-solrconfig.xml", "dataimport-solr_id-schema.xml");
@@ -266,18 +283,4 @@ public class TestSqlEntityProcessorDelta
assertQ(req("desc:hello"), "//*[@numFound='0']");
assertQ(req("desc:goodbye"), "//*[@numFound='1']");
}
-
- private static String dataConfig_delta2 = "<dataConfig><dataSource type=\"MockDataSource\"/>\n"
- + " <document>\n"
- + " <entity name=\"x\" transformer=\"TemplateTransformer\""
- + " query=\"" + FULLIMPORT_QUERY + "\""
- + " deletedPkQuery=\"" + DELETED_PK_QUERY + "\""
- + " deltaImportQuery=\"select * from x where id='${dataimporter.delta.id}'\""
- + " deltaQuery=\"" + DELTA_QUERY + "\">\n"
- + " <field column=\"tmpid\" template=\"prefix-${x.id}\" name=\"solr_id\"/>\n"
- + " <entity name=\"y\" query=\"select * from y where y.A='${x.id}'\">\n"
- + " <field column=\"desc\" />\n"
- + " </entity>\n" + " </entity>\n"
- + " </document>\n" + "</dataConfig>\n";
-
}
Added: lucene/dev/trunk/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta3.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta3.java?rev=1040608&view=auto
==============================================================================
--- lucene/dev/trunk/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta3.java (added)
+++ lucene/dev/trunk/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta3.java Tue Nov 30 16:10:29 2010
@@ -0,0 +1,234 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+public class TestSqlEntityProcessorDelta3 extends AbstractDataImportHandlerTestCase {
+ private static final String P_FULLIMPORT_QUERY = "select * from parent";
+ private static final String P_DELTA_QUERY = "select parent_id from parent where last_modified > NOW";
+ private static final String P_DELTAIMPORT_QUERY = "select * from parent where last_modified > NOW AND parent_id=${dih.delta.parent_id}";
+
+ private static final String C_FULLIMPORT_QUERY = "select * from child";
+ private static final String C_DELETED_PK_QUERY = "select id from child where last_modified > NOW AND deleted='true'";
+ private static final String C_DELTA_QUERY = "select id from child where last_modified > NOW";
+ private static final String C_PARENTDELTA_QUERY = "select parent_id from child where id=${child.id}";
+ private static final String C_DELTAIMPORT_QUERY = "select * from child where last_modified > NOW AND parent_id=${dih.delta.parent_id}";
+
+ private static final String dataConfig_delta =
+ "<dataConfig>" +
+ " <dataSource type=\"MockDataSource\"/>\n" +
+ " <document>" +
+ " <entity name=\"parent\" pk=\"parent_id\" rootEntity=\"false\"" +
+ " query=\"" + P_FULLIMPORT_QUERY + "\"" +
+ " deltaQuery=\"" + P_DELTA_QUERY + "\"" +
+ " deltaImportQuery=\"" + P_DELTAIMPORT_QUERY + "\">" +
+ " <field column=\"desc\" name=\"desc\"/>" +
+ " <entity name=\"child\" pk=\"id\" rootEntity=\"true\"" +
+ " query=\"" + C_FULLIMPORT_QUERY + "\"" +
+ " deletedPkQuery=\"" + C_DELETED_PK_QUERY + "\"" +
+ " deltaQuery=\"" + C_DELTA_QUERY + "\"" +
+ " parentDeltaQuery=\"" + C_PARENTDELTA_QUERY + "\"" +
+ " deltaImportQuery=\"" + C_DELTAIMPORT_QUERY + "\">" +
+ " <field column=\"id\" name=\"id\" />" +
+ " </entity>" +
+ " </entity>" +
+ " </document>" +
+ "</dataConfig>\n";
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ initCore("dataimport-solrconfig.xml", "dataimport-schema.xml");
+ }
+
+ @Before @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ clearIndex();
+ assertU(commit());
+ }
+
+ @SuppressWarnings("unchecked")
+ private void add1document() throws Exception {
+ List parentRow = new ArrayList();
+ parentRow.add(createMap("parent_id", "1", "desc", "d1"));
+ MockDataSource.setIterator(P_FULLIMPORT_QUERY, parentRow.iterator());
+
+ List childRow = new ArrayList();
+ childRow.add(createMap("id", "2"));
+ MockDataSource.setIterator(C_FULLIMPORT_QUERY, childRow.iterator());
+
+ runFullImport(dataConfig_delta);
+
+ assertQ(req("*:* OR add1document"), "//*[@numFound='1']");
+ assertQ(req("id:1"), "//*[@numFound='0']");
+ assertQ(req("id:2"), "//*[@numFound='1']");
+ assertQ(req("desc:d1"), "//*[@numFound='1']");
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testCompositePk_FullImport() throws Exception {
+ add1document();
+ }
+
+ // WORKS
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testCompositePk_DeltaImport_delete() throws Exception {
+ add1document();
+ List deletedRow = new ArrayList();
+ deletedRow.add(createMap("id", "2"));
+ MockDataSource.setIterator(C_DELETED_PK_QUERY, deletedRow.iterator());
+ MockDataSource.setIterator(C_DELTA_QUERY, Collections.EMPTY_LIST.iterator());
+
+ List deletedParentRow = new ArrayList();
+ deletedParentRow.add(createMap("parent_id", "1"));
+ MockDataSource.setIterator("select parent_id from child where id=2", deletedParentRow.iterator());
+
+ runDeltaImport(dataConfig_delta);
+ assertQ(req("*:* OR testCompositePk_DeltaImport_delete"), "//*[@numFound='0']");
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testCompositePk_DeltaImport_empty() throws Exception {
+ List childDeltaRow = new ArrayList();
+ childDeltaRow.add(createMap("id", "2"));
+ MockDataSource.setIterator(C_DELTA_QUERY, childDeltaRow.iterator());
+ MockDataSource.setIterator(C_DELETED_PK_QUERY, Collections.EMPTY_LIST.iterator());
+
+ List childParentDeltaRow = new ArrayList();
+ childParentDeltaRow.add(createMap("parent_id", "1"));
+ MockDataSource.setIterator("select parent_id from child where id=2", childParentDeltaRow.iterator());
+
+ MockDataSource.setIterator(P_DELTA_QUERY, Collections.EMPTY_LIST.iterator());
+
+ List parentDeltaImportRow = new ArrayList();
+ parentDeltaImportRow.add(createMap("parent_id", "1", "desc", "d1"));
+ MockDataSource.setIterator("select * from parent where last_modified > NOW AND parent_id=1",
+ parentDeltaImportRow.iterator());
+
+ List childDeltaImportRow = new ArrayList();
+ childDeltaImportRow.add(createMap("id", "2"));
+ MockDataSource.setIterator("select * from child where last_modified > NOW AND parent_id=1",
+ childDeltaImportRow.iterator());
+
+ runDeltaImport(dataConfig_delta);
+
+ assertQ(req("*:* OR testCompositePk_DeltaImport_empty"), "//*[@numFound='1']");
+ assertQ(req("id:2"), "//*[@numFound='1']");
+ assertQ(req("desc:d1"), "//*[@numFound='1']");
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testCompositePk_DeltaImport_replace_nodelete() throws Exception {
+ add1document();
+ MockDataSource.clearCache();
+
+ List deltaRow = new ArrayList();
+ deltaRow.add(createMap("parent_id", "1"));
+ MockDataSource.setIterator(P_DELTA_QUERY,
+ deltaRow.iterator());
+
+ List parentRow = new ArrayList();
+ parentRow.add(createMap("parent_id", "1", "desc", "d2"));
+ MockDataSource.setIterator("select * from parent where last_modified > NOW AND parent_id=1",
+ parentRow.iterator());
+
+ List childRow = new ArrayList();
+ childRow.add(createMap("id", "2"));
+ MockDataSource.setIterator("select * from child where last_modified > NOW AND parent_id=1",
+ childRow.iterator());
+
+ MockDataSource.setIterator(C_DELETED_PK_QUERY, Collections
+ .EMPTY_LIST.iterator());
+
+ runDeltaImport(dataConfig_delta);
+
+ assertQ(req("*:* OR XtestCompositePk_DeltaImport_replace_nodelete"), "//*[@numFound='1']");
+ assertQ(req("id:2"), "//*[@numFound='1']");
+ assertQ(req("desc:s1 OR XtestCompositePk_DeltaImport_replace_nodelete"), "//*[@numFound='0']");
+ assertQ(req("desc:d2"), "//*[@numFound='1']");
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testCompositePk_DeltaImport_add() throws Exception {
+ add1document();
+ MockDataSource.clearCache();
+
+ List parentDeltaRow = new ArrayList();
+ parentDeltaRow.add(createMap("parent_id", "1"));
+ MockDataSource.setIterator(P_DELTA_QUERY,
+ parentDeltaRow.iterator());
+
+ List parentRow = new ArrayList();
+ parentRow.add(createMap("parent_id", "1", "desc", "d1"));
+ MockDataSource.setIterator("select * from parent where last_modified > NOW AND parent_id=1",
+ parentRow.iterator());
+
+ List childDeltaRow = new ArrayList();
+ childDeltaRow.add(createMap("id", "3"));
+ MockDataSource.setIterator(C_DELTA_QUERY,
+ childDeltaRow.iterator());
+
+ List childParentDeltaRow = new ArrayList();
+ childParentDeltaRow.add(createMap("parent_id", "1"));
+ MockDataSource.setIterator("select parent_id from child where id='3'",
+ childParentDeltaRow.iterator());
+
+ List childRow = new ArrayList();
+ childRow.add(createMap("id", "3"));
+ MockDataSource.setIterator("select * from child where last_modified > NOW AND parent_id=1",
+ childRow.iterator());
+
+ runDeltaImport(dataConfig_delta);
+
+ assertQ(req("*:* OR testCompositePk_DeltaImport_add"), "//*[@numFound='2']");
+ assertQ(req("id:2"), "//*[@numFound='1']");
+ assertQ(req("id:3"), "//*[@numFound='1']");
+ assertQ(req("desc:d1"), "//*[@numFound='2']");
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testCompositePk_DeltaImport_nodelta() throws Exception {
+ add1document();
+ MockDataSource.clearCache();
+
+ MockDataSource.setIterator(P_DELTA_QUERY,
+ Collections.EMPTY_LIST.iterator());
+
+ MockDataSource.setIterator(C_DELTA_QUERY,
+ Collections.EMPTY_LIST.iterator());
+
+ runDeltaImport(dataConfig_delta);
+
+ assertQ(req("*:* OR testCompositePk_DeltaImport_nodelta"), "//*[@numFound='1']");
+ assertQ(req("id:2 OR testCompositePk_DeltaImport_nodelta"), "//*[@numFound='1']");
+ assertQ(req("desc:d1 OR testCompositePk_DeltaImport_nodelta"), "//*[@numFound='1']");
+ }
+}