You are viewing a plain text version of this content. The canonical link for it is here.
Posted to solr-commits@lucene.apache.org by eh...@apache.org on 2009/07/10 16:55:29 UTC
svn commit: r792963 - in /lucene/solr/trunk/contrib/dataimporthandler: ./
src/main/java/org/apache/solr/handler/dataimport/
src/test/java/org/apache/solr/handler/dataimport/
src/test/resources/solr/conf/
Author: ehatcher
Date: Fri Jul 10 14:55:28 2009
New Revision: 792963
URL: http://svn.apache.org/viewvc?rev=792963&view=rev
Log:
SOLR-1229: fixes for deletedPkQuery
Added:
lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta.java
lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta2.java
lucene/solr/trunk/contrib/dataimporthandler/src/test/resources/solr/conf/dataimport-solr_id-schema.xml
Modified:
lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/AbstractDataImportHandlerTest.java
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataConfig.java
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImporter.java
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java
Modified: lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt?rev=792963&r1=792962&r2=792963&view=diff
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt (original)
+++ lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt Fri Jul 10 14:55:28 2009
@@ -246,6 +246,8 @@
26.SOLR-1146: ConcurrentModificationException in DataImporter.getStatusMessages
(Walter Ferrara, Noble Paul via shalin)
+27.SOLR-1229: Fixes for deletedPkQuery, particularly when using transformed Solr unique id's
+ (Lance Norskog, Noble Paul via ehatcher)
Documentation
Modified: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/AbstractDataImportHandlerTest.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/AbstractDataImportHandlerTest.java?rev=792963&r1=792962&r2=792963&view=diff
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/AbstractDataImportHandlerTest.java (original)
+++ lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/AbstractDataImportHandlerTest.java Fri Jul 10 14:55:28 2009
@@ -68,7 +68,7 @@
protected void runDeltaImport(String dataConfig) throws Exception {
LocalSolrQueryRequest request = lrf.makeRequest("command", "delta-import",
- "debug", "on", "clean", "true", "commit", "true", "dataConfig",
+ "debug", "on", "clean", "false", "commit", "true", "dataConfig",
dataConfig);
h.query("/dataimport", request);
}
Modified: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataConfig.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataConfig.java?rev=792963&r1=792962&r2=792963&view=diff
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataConfig.java (original)
+++ lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataConfig.java Fri Jul 10 14:55:28 2009
@@ -78,6 +78,8 @@
public String pk;
+ public String pkMappingFromSchema;
+
public String dataSource;
public Map<String, String> allAttributes;
Modified: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImporter.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImporter.java?rev=792963&r1=792962&r2=792963&view=diff
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImporter.java (original)
+++ lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImporter.java Fri Jul 10 14:55:28 2009
@@ -106,7 +106,8 @@
Map<String, DataConfig.Field> fields = new HashMap<String, DataConfig.Field>();
initEntity(e, fields, false);
verifyWithSchema(fields);
- }
+ identifyPk(e);
+ }
}
private void verifyWithSchema(Map<String, DataConfig.Field> fields) {
@@ -145,6 +146,22 @@
}
}
+ private void identifyPk(DataConfig.Entity entity) {
+ String schemaPk = schema.getUniqueKeyField().getName();
+ //if no fields are mentioned . solr uniqeKey is same as dih 'pk'
+ entity.pkMappingFromSchema = schemaPk;
+ for (DataConfig.Field field : entity.fields) {
+ if(field.getName().equals(schemaPk)) {
+ entity.pkMappingFromSchema = field.column;
+ //get the corresponding column mapping for the solr uniqueKey
+ // But if there are multiple columns mapping to the solr uniqueKey, it will fail
+ // so , in one off cases we may need pk
+ break;
+ }
+ }
+
+ }
+
void loadDataConfig(String configFile) {
try {
Modified: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java?rev=792963&r1=792962&r2=792963&view=diff
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java (original)
+++ lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java Fri Jul 10 14:55:28 2009
@@ -273,16 +273,11 @@
while (iter.hasNext()) {
Map<String, Object> map = iter.next();
Object key = null;
- if(root.pk != null){
- key = map.get(root.pk);
- }
- if(key == null && map.size() ==1){
- //iterating through the map just to get the first and only item
- for (Map.Entry<String, Object> e : map.entrySet()) {
- key = e.getValue();
- break;
- }
- }
+ if(root.pkMappingFromSchema != null){
+ key = map.get(root.pkMappingFromSchema);
+ } else if(root.pk != null){
+ key = map.get(root.pk);
+ }
if(key == null) {
LOG.warn("no key was available for deleteted pk query");
continue;
Added: lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta.java?rev=792963&view=auto
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta.java (added)
+++ lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta.java Fri Jul 10 14:55:28 2009
@@ -0,0 +1,295 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * <p>
+ * Test for SqlEntityProcessor which checks variations in primary key names and deleted ids
+ * </p>
+ *
+ *
+ * @version $Id: TestSqlEntityProcessor2.java 723824 2008-12-05 19:14:11Z shalin $
+ * @since solr 1.3
+ */
+public class TestSqlEntityProcessorDelta extends AbstractDataImportHandlerTest {
+ private static final String FULLIMPORT_QUERY = "select * from x";
+
+ private static final String DELTA_QUERY = "select id from x where last_modified > NOW";
+
+ private static final String DELETED_PK_QUERY = "select id from x where last_modified > NOW AND deleted='true'";
+
+ @Override
+ public String getSchemaFile() {
+ return "dataimport-schema.xml";
+ }
+
+ @Override
+ public String getSolrConfigFile() {
+ return "dataimport-solrconfig.xml";
+ }
+
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ }
+
+ @Override
+ public void tearDown() throws Exception {
+ super.tearDown();
+ }
+
+
+ @SuppressWarnings("unchecked")
+ private void add1document() throws Exception {
+ List parentRow = new ArrayList();
+ parentRow.add(createMap("id", "1"));
+ MockDataSource.setIterator(FULLIMPORT_QUERY, parentRow.iterator());
+
+ List childRow = new ArrayList();
+ childRow.add(createMap("desc", "hello"));
+ MockDataSource.setIterator("select * from y where y.A='1'", childRow
+ .iterator());
+
+ super.runFullImport(dataConfig_delta);
+
+ assertQ(req("*:* OR add1document"), "//*[@numFound='1']");
+ assertQ(req("id:1"), "//*[@numFound='1']");
+ assertQ(req("desc:hello"), "//*[@numFound='1']");
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testCompositePk_FullImport() throws Exception {
+ add1document();
+ }
+
+ // WORKS
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testCompositePk_DeltaImport_delete() throws Exception {
+ add1document();
+ List deletedRow = new ArrayList();
+ deletedRow.add(createMap("id", "1"));
+ MockDataSource.setIterator(DELETED_PK_QUERY, deletedRow.iterator());
+
+ MockDataSource.setIterator(DELTA_QUERY, Collections
+ .EMPTY_LIST.iterator());
+
+ List childRow = new ArrayList();
+ childRow.add(createMap("desc", "hello"));
+ MockDataSource.setIterator("select * from y where y.A='1'", childRow
+ .iterator());
+
+ super.runDeltaImport(dataConfig_delta);
+ assertQ(req("*:* OR testCompositePk_DeltaImport_delete"), "//*[@numFound='0']");
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testCompositePk_DeltaImport_empty() throws Exception {
+ List deltaRow = new ArrayList();
+ deltaRow.add(createMap("id", "1"));
+ MockDataSource.setIterator(DELTA_QUERY, deltaRow.iterator());
+
+ MockDataSource.setIterator(DELETED_PK_QUERY, Collections
+ .EMPTY_LIST.iterator());
+
+ List parentRow = new ArrayList();
+ parentRow.add(createMap("id", "1"));
+ MockDataSource.setIterator("select * from x where id='1'", parentRow
+ .iterator());
+
+ List childRow = new ArrayList();
+ childRow.add(createMap("desc", "hello"));
+ MockDataSource.setIterator("select * from y where y.A='1'", childRow
+ .iterator());
+
+ super.runDeltaImport(dataConfig_delta);
+
+ assertQ(req("*:* OR testCompositePk_DeltaImport_empty"), "//*[@numFound='1']");
+ assertQ(req("id:1"), "//*[@numFound='1']");
+ assertQ(req("desc:hello"), "//*[@numFound='1']");
+ }
+
+ // WORKS
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void XtestCompositePk_DeltaImport_replace_delete() throws Exception {
+ add1document();
+ MockDataSource.clearCache();
+
+ List deltaRow = new ArrayList();
+ deltaRow.add(createMap("id", "1"));
+ MockDataSource.setIterator(DELTA_QUERY,
+ deltaRow.iterator());
+
+ List deletedRow = new ArrayList();
+ deletedRow.add(createMap("id", "1"));
+ MockDataSource.setIterator(DELETED_PK_QUERY,
+ deletedRow.iterator());
+
+ List parentRow = new ArrayList();
+ parentRow.add(createMap("id", "1"));
+ MockDataSource.setIterator("select * from x where id='1'", parentRow
+ .iterator());
+
+ List childRow = new ArrayList();
+ childRow.add(createMap("desc", "goodbye"));
+ MockDataSource.setIterator("select * from y where y.A='1'", childRow
+ .iterator());
+
+ super.runDeltaImport(dataConfig_delta);
+
+ assertQ(req("*:* OR testCompositePk_DeltaImport_replace_delete"), "//*[@numFound='0']");
+ }
+
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testCompositePk_DeltaImport_replace_nodelete() throws Exception {
+ add1document();
+ MockDataSource.clearCache();
+
+ List deltaRow = new ArrayList();
+ deltaRow.add(createMap("id", "1"));
+ MockDataSource.setIterator(DELTA_QUERY,
+ deltaRow.iterator());
+
+ MockDataSource.setIterator(DELETED_PK_QUERY, Collections
+ .EMPTY_LIST.iterator());
+
+ List parentRow = new ArrayList();
+ parentRow.add(createMap("id", "1"));
+ MockDataSource.setIterator("select * from x where id='1'", parentRow
+ .iterator());
+
+ List childRow = new ArrayList();
+ childRow.add(createMap("desc", "goodbye"));
+ MockDataSource.setIterator("select * from y where y.A='1'", childRow
+ .iterator());
+
+ super.runDeltaImport(dataConfig_delta);
+
+ assertQ(req("*:* OR XtestCompositePk_DeltaImport_replace_nodelete"), "//*[@numFound='1']");
+ assertQ(req("id:1"), "//*[@numFound='1']");
+ assertQ(req("desc:hello OR XtestCompositePk_DeltaImport_replace_nodelete"), "//*[@numFound='0']");
+ assertQ(req("desc:goodbye"), "//*[@numFound='1']");
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testCompositePk_DeltaImport_add() throws Exception {
+ add1document();
+ MockDataSource.clearCache();
+
+ List deltaRow = new ArrayList();
+ deltaRow.add(createMap("id", "2"));
+ MockDataSource.setIterator(DELTA_QUERY,
+ deltaRow.iterator());
+
+ List parentRow = new ArrayList();
+ parentRow.add(createMap("id", "2"));
+ MockDataSource.setIterator("select * from x where id='2'", parentRow
+ .iterator());
+
+ List childRow = new ArrayList();
+ childRow.add(createMap("desc", "goodbye"));
+ MockDataSource.setIterator("select * from y where y.A='2'", childRow
+ .iterator());
+
+ super.runDeltaImport(dataConfig_delta);
+
+ assertQ(req("*:* OR testCompositePk_DeltaImport_add"), "//*[@numFound='2']");
+ assertQ(req("id:1"), "//*[@numFound='1']");
+ assertQ(req("id:2"), "//*[@numFound='1']");
+ assertQ(req("desc:hello"), "//*[@numFound='1']");
+ assertQ(req("desc:goodbye"), "//*[@numFound='1']");
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testCompositePk_DeltaImport_nodelta() throws Exception {
+ add1document();
+ MockDataSource.clearCache();
+
+ MockDataSource.setIterator(DELTA_QUERY,
+ Collections.EMPTY_LIST.iterator());
+
+ super.runDeltaImport(dataConfig_delta);
+
+ assertQ(req("*:* OR testCompositePk_DeltaImport_nodelta"), "//*[@numFound='1']");
+ assertQ(req("id:1 OR testCompositePk_DeltaImport_nodelta"), "//*[@numFound='1']");
+ assertQ(req("desc:hello OR testCompositePk_DeltaImport_nodelta"), "//*[@numFound='1']");
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testCompositePk_DeltaImport_add_delete() throws Exception {
+ add1document();
+ MockDataSource.clearCache();
+
+ List deltaRow = new ArrayList();
+ deltaRow.add(createMap("id", "2"));
+ MockDataSource.setIterator(DELTA_QUERY,
+ deltaRow.iterator());
+
+ List deletedRow = new ArrayList();
+ deletedRow.add(createMap("id", "1"));
+ MockDataSource.setIterator(DELETED_PK_QUERY,
+ deletedRow.iterator());
+
+ List parentRow = new ArrayList();
+ parentRow.add(createMap("id", "2"));
+ MockDataSource.setIterator("select * from x where id='2'", parentRow
+ .iterator());
+
+ List childRow = new ArrayList();
+ childRow.add(createMap("desc", "goodbye"));
+ MockDataSource.setIterator("select * from y where y.A='2'", childRow
+ .iterator());
+
+ super.runDeltaImport(dataConfig_delta);
+
+ assertQ(req("*:* OR XtestCompositePk_DeltaImport_add_delete"), "//*[@numFound='1']");
+ assertQ(req("id:2"), "//*[@numFound='1']");
+ assertQ(req("desc:hello"), "//*[@numFound='0']");
+ assertQ(req("desc:goodbye"), "//*[@numFound='1']");
+ }
+
+ private static String dataConfig_delta = "<dataConfig>\n"
+ + " <document>\n"
+ + " <entity name=\"x\" pk=\"id\" transformer=\"TemplateTransformer\""
+ + " query=\"" + FULLIMPORT_QUERY + "\""
+ + " deletedPkQuery=\"" + DELETED_PK_QUERY + "\""
+ + " deltaImportQuery=\"select * from x where id='${dataimporter.delta.id}'\""
+ + " deltaQuery=\"" + DELTA_QUERY + "\">\n"
+ + " <field column=\"id\" name=\"id\"/>\n"
+ + " <entity name=\"y\" query=\"select * from y where y.A='${x.id}'\">\n"
+ + " <field column=\"desc\" />\n"
+ + " </entity>\n" + " </entity>\n"
+ + " </document>\n" + "</dataConfig>\n";
+
+}
Added: lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta2.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta2.java?rev=792963&view=auto
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta2.java (added)
+++ lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta2.java Fri Jul 10 14:55:28 2009
@@ -0,0 +1,291 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * <p>
+ * Test for SqlEntityProcessor which checks variations in primary key names and deleted ids
+ * </p>
+ *
+ *
+ * @version $Id: TestSqlEntityProcessor2.java 723824 2008-12-05 19:14:11Z shalin $
+ * @since solr 1.3
+ */
+public class TestSqlEntityProcessorDelta2 extends AbstractDataImportHandlerTest {
+ private static final String FULLIMPORT_QUERY = "select * from x";
+
+ private static final String DELTA_QUERY = "select id from x where last_modified > NOW";
+
+ private static final String DELETED_PK_QUERY = "select id from x where last_modified > NOW AND deleted='true'";
+
+ @Override
+ public String getSchemaFile() {
+ return "dataimport-solr_id-schema.xml";
+ }
+
+ @Override
+ public String getSolrConfigFile() {
+ return "dataimport-solrconfig.xml";
+ }
+
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ }
+
+ @Override
+ public void tearDown() throws Exception {
+ super.tearDown();
+ }
+
+
+ @SuppressWarnings("unchecked")
+ private void add1document() throws Exception {
+ List parentRow = new ArrayList();
+ parentRow.add(createMap("id", "1"));
+ MockDataSource.setIterator(FULLIMPORT_QUERY, parentRow.iterator());
+
+ List childRow = new ArrayList();
+ childRow.add(createMap("desc", "hello"));
+ MockDataSource.setIterator("select * from y where y.A='1'", childRow
+ .iterator());
+
+ super.runFullImport(dataConfig_delta2);
+
+ assertQ(req("*:* OR add1document"), "//*[@numFound='1']");
+ assertQ(req("solr_id:prefix-1"), "//*[@numFound='1']");
+ assertQ(req("desc:hello"), "//*[@numFound='1']");
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testCompositePk_FullImport() throws Exception {
+ add1document();
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testCompositePk_DeltaImport_delete() throws Exception {
+ add1document();
+ List deletedRow = new ArrayList();
+ deletedRow.add(createMap("id", "1"));
+ MockDataSource.setIterator(DELETED_PK_QUERY, deletedRow.iterator());
+
+ MockDataSource.setIterator(DELTA_QUERY, Collections
+ .EMPTY_LIST.iterator());
+
+ List childRow = new ArrayList();
+ childRow.add(createMap("desc", "hello"));
+ MockDataSource.setIterator("select * from y where y.A='1'", childRow
+ .iterator());
+
+ super.runDeltaImport(dataConfig_delta2);
+ assertQ(req("*:* OR testCompositePk_DeltaImport_delete"), "//*[@numFound='0']");
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testCompositePk_DeltaImport_empty() throws Exception {
+ List deltaRow = new ArrayList();
+ deltaRow.add(createMap("id", "1"));
+ MockDataSource.setIterator(DELTA_QUERY, deltaRow.iterator());
+
+ MockDataSource.setIterator(DELETED_PK_QUERY, Collections
+ .EMPTY_LIST.iterator());
+
+ List parentRow = new ArrayList();
+ parentRow.add(createMap("id", "1"));
+ MockDataSource.setIterator("select * from x where id='1'", parentRow
+ .iterator());
+
+ List childRow = new ArrayList();
+ childRow.add(createMap("desc", "hello"));
+ MockDataSource.setIterator("select * from y where y.A='1'", childRow
+ .iterator());
+
+ super.runDeltaImport(dataConfig_delta2);
+
+ assertQ(req("*:* OR testCompositePk_DeltaImport_empty"), "//*[@numFound='1']");
+ assertQ(req("solr_id:prefix-1"), "//*[@numFound='1']");
+ assertQ(req("desc:hello"), "//*[@numFound='1']");
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void XtestCompositePk_DeltaImport_replace_delete() throws Exception {
+ add1document();
+ MockDataSource.clearCache();
+
+ List deltaRow = new ArrayList();
+ deltaRow.add(createMap("id", "1"));
+ MockDataSource.setIterator(DELTA_QUERY,
+ deltaRow.iterator());
+
+ List deletedRow = new ArrayList();
+ deletedRow.add(createMap("id", "1"));
+ MockDataSource.setIterator(DELETED_PK_QUERY,
+ deletedRow.iterator());
+
+ List parentRow = new ArrayList();
+ parentRow.add(createMap("id", "1"));
+ MockDataSource.setIterator("select * from x where id='1'", parentRow
+ .iterator());
+
+ List childRow = new ArrayList();
+ childRow.add(createMap("desc", "goodbye"));
+ MockDataSource.setIterator("select * from y where y.A='1'", childRow
+ .iterator());
+
+ super.runDeltaImport(dataConfig_delta2);
+
+ assertQ(req("*:* OR testCompositePk_DeltaImport_replace_delete"), "//*[@numFound='0']");
+ }
+
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testCompositePk_DeltaImport_replace_nodelete() throws Exception {
+ add1document();
+ MockDataSource.clearCache();
+
+ List deltaRow = new ArrayList();
+ deltaRow.add(createMap("id", "1"));
+ MockDataSource.setIterator(DELTA_QUERY,
+ deltaRow.iterator());
+
+ MockDataSource.setIterator(DELETED_PK_QUERY, Collections
+ .EMPTY_LIST.iterator());
+
+ List parentRow = new ArrayList();
+ parentRow.add(createMap("id", "1"));
+ MockDataSource.setIterator("select * from x where id='1'", parentRow
+ .iterator());
+
+ List childRow = new ArrayList();
+ childRow.add(createMap("desc", "goodbye"));
+ MockDataSource.setIterator("select * from y where y.A='1'", childRow
+ .iterator());
+
+ super.runDeltaImport(dataConfig_delta2);
+
+ assertQ(req("*:* OR XtestCompositePk_DeltaImport_replace_nodelete"), "//*[@numFound='1']");
+ assertQ(req("solr_id:prefix-1"), "//*[@numFound='1']");
+ assertQ(req("desc:hello OR XtestCompositePk_DeltaImport_replace_nodelete"), "//*[@numFound='0']");
+ assertQ(req("desc:goodbye"), "//*[@numFound='1']");
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testCompositePk_DeltaImport_add() throws Exception {
+ add1document();
+ MockDataSource.clearCache();
+
+ List deltaRow = new ArrayList();
+ deltaRow.add(createMap("id", "2"));
+ MockDataSource.setIterator(DELTA_QUERY,
+ deltaRow.iterator());
+
+ List parentRow = new ArrayList();
+ parentRow.add(createMap("id", "2"));
+ MockDataSource.setIterator("select * from x where id='2'", parentRow
+ .iterator());
+
+ List childRow = new ArrayList();
+ childRow.add(createMap("desc", "goodbye"));
+ MockDataSource.setIterator("select * from y where y.A='2'", childRow
+ .iterator());
+
+ super.runDeltaImport(dataConfig_delta2);
+
+ assertQ(req("*:* OR testCompositePk_DeltaImport_add"), "//*[@numFound='2']");
+ assertQ(req("solr_id:prefix-1"), "//*[@numFound='1']");
+ assertQ(req("solr_id:prefix-2"), "//*[@numFound='1']");
+ assertQ(req("desc:hello"), "//*[@numFound='1']");
+ assertQ(req("desc:goodbye"), "//*[@numFound='1']");
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testCompositePk_DeltaImport_nodelta() throws Exception {
+ add1document();
+ MockDataSource.clearCache();
+
+ MockDataSource.setIterator(DELTA_QUERY,
+ Collections.EMPTY_LIST.iterator());
+
+ super.runDeltaImport(dataConfig_delta2);
+
+ assertQ(req("*:* OR testCompositePk_DeltaImport_nodelta"), "//*[@numFound='1']");
+ assertQ(req("solr_id:prefix-1 OR testCompositePk_DeltaImport_nodelta"), "//*[@numFound='1']");
+ assertQ(req("desc:hello OR testCompositePk_DeltaImport_nodelta"), "//*[@numFound='1']");
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testCompositePk_DeltaImport_add_delete() throws Exception {
+ add1document();
+ MockDataSource.clearCache();
+
+ List deltaRow = new ArrayList();
+ deltaRow.add(createMap("id", "2"));
+ MockDataSource.setIterator(DELTA_QUERY,
+ deltaRow.iterator());
+
+ List deletedRow = new ArrayList();
+ deletedRow.add(createMap("id", "1"));
+ MockDataSource.setIterator(DELETED_PK_QUERY,
+ deletedRow.iterator());
+
+ List parentRow = new ArrayList();
+ parentRow.add(createMap("id", "2"));
+ MockDataSource.setIterator("select * from x where id='2'", parentRow
+ .iterator());
+
+ List childRow = new ArrayList();
+ childRow.add(createMap("desc", "goodbye"));
+ MockDataSource.setIterator("select * from y where y.A='2'", childRow
+ .iterator());
+
+ super.runDeltaImport(dataConfig_delta2);
+
+ assertQ(req("*:* OR XtestCompositePk_DeltaImport_add_delete"), "//*[@numFound='1']");
+ assertQ(req("solr_id:prefix-2"), "//*[@numFound='1']");
+ assertQ(req("desc:hello"), "//*[@numFound='0']");
+ assertQ(req("desc:goodbye"), "//*[@numFound='1']");
+ }
+
+ private static String dataConfig_delta2 = "<dataConfig>\n"
+ + " <document>\n"
+ + " <entity name=\"x\" pk=\"id\" transformer=\"TemplateTransformer\""
+ + " query=\"" + FULLIMPORT_QUERY + "\""
+ + " deletedPkQuery=\"" + DELETED_PK_QUERY + "\""
+ + " deltaImportQuery=\"select * from x where id='${dataimporter.delta.id}'\""
+ + " deltaQuery=\"" + DELTA_QUERY + "\">\n"
+ + " <field column=\"solr_id\" template=\"prefix-${x.id}\"/>\n"
+ + " <entity name=\"y\" query=\"select * from y where y.A='${x.id}'\">\n"
+ + " <field column=\"desc\" />\n"
+ + " </entity>\n" + " </entity>\n"
+ + " </document>\n" + "</dataConfig>\n";
+
+}
Added: lucene/solr/trunk/contrib/dataimporthandler/src/test/resources/solr/conf/dataimport-solr_id-schema.xml
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/test/resources/solr/conf/dataimport-solr_id-schema.xml?rev=792963&view=auto
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/src/test/resources/solr/conf/dataimport-solr_id-schema.xml (added)
+++ lucene/solr/trunk/contrib/dataimporthandler/src/test/resources/solr/conf/dataimport-solr_id-schema.xml Fri Jul 10 14:55:28 2009
@@ -0,0 +1,304 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!--
+ This is the Solr schema file. This file should be named "schema.xml" and
+ should be in the conf directory under the solr home
+ (i.e. ./solr/conf/schema.xml by default)
+ or located where the classloader for the Solr webapp can find it.
+
+ This example schema is the recommended starting point for users.
+ It should be kept correct and concise, usable out-of-the-box.
+
+ For more information, on how to customize this file, please see
+ http://wiki.apache.org/solr/SchemaXml
+-->
+
+<schema name="test" version="1.1">
+ <!-- attribute "name" is the name of this schema and is only used for display purposes.
+ Applications should change this to reflect the nature of the search collection.
+ version="1.1" is Solr's version number for the schema syntax and semantics. It should
+ not normally be changed by applications.
+ 1.0: multiValued attribute did not exist, all fields are multiValued by nature
+ 1.1: multiValued attribute introduced, false by default -->
+
+ <types>
+ <!-- field type definitions. The "name" attribute is
+ just a label to be used by field definitions. The "class"
+ attribute and any other attributes determine the real
+ behavior of the fieldType.
+ Class names starting with "solr" refer to java classes in the
+ org.apache.solr.analysis package.
+ -->
+
+ <!-- The StrField type is not analyzed, but indexed/stored verbatim.
+ - StrField and TextField support an optional compressThreshold which
+ limits compression (if enabled in the derived fields) to values which
+ exceed a certain size (in characters).
+ -->
+ <fieldType name="string" class="solr.StrField" sortMissingLast="true" omitNorms="true"/>
+
+ <!-- boolean type: "true" or "false" -->
+ <fieldType name="boolean" class="solr.BoolField" sortMissingLast="true" omitNorms="true"/>
+
+ <!-- The optional sortMissingLast and sortMissingFirst attributes are
+ currently supported on types that are sorted internally as strings.
+ - If sortMissingLast="true", then a sort on this field will cause documents
+ without the field to come after documents with the field,
+ regardless of the requested sort order (asc or desc).
+ - If sortMissingFirst="true", then a sort on this field will cause documents
+ without the field to come before documents with the field,
+ regardless of the requested sort order.
+ - If sortMissingLast="false" and sortMissingFirst="false" (the default),
+ then default lucene sorting will be used which places docs without the
+ field first in an ascending sort and last in a descending sort.
+ -->
+
+
+ <!-- numeric field types that store and index the text
+ value verbatim (and hence don't support range queries, since the
+ lexicographic ordering isn't equal to the numeric ordering) -->
+ <fieldType name="integer" class="solr.IntField" omitNorms="true"/>
+ <fieldType name="long" class="solr.LongField" omitNorms="true"/>
+ <fieldType name="float" class="solr.FloatField" omitNorms="true"/>
+ <fieldType name="double" class="solr.DoubleField" omitNorms="true"/>
+
+
+ <!-- Numeric field types that manipulate the value into
+ a string value that isn't human-readable in its internal form,
+ but with a lexicographic ordering the same as the numeric ordering,
+ so that range queries work correctly. -->
+ <fieldType name="sint" class="solr.SortableIntField" sortMissingLast="true" omitNorms="true"/>
+ <fieldType name="slong" class="solr.SortableLongField" sortMissingLast="true" omitNorms="true"/>
+ <fieldType name="sfloat" class="solr.SortableFloatField" sortMissingLast="true" omitNorms="true"/>
+ <fieldType name="sdouble" class="solr.SortableDoubleField" sortMissingLast="true" omitNorms="true"/>
+
+
+ <!-- The format for this date field is of the form 1995-12-31T23:59:59Z, and
+ is a more restricted form of the canonical representation of dateTime
+ http://www.w3.org/TR/xmlschema-2/#dateTime
+ The trailing "Z" designates UTC time and is mandatory.
+ Optional fractional seconds are allowed: 1995-12-31T23:59:59.999Z
+ All other components are mandatory.
+
+ Expressions can also be used to denote calculations that should be
+ performed relative to "NOW" to determine the value, ie...
+
+ NOW/HOUR
+ ... Round to the start of the current hour
+ NOW-1DAY
+ ... Exactly 1 day prior to now
+ NOW/DAY+6MONTHS+3DAYS
+ ... 6 months and 3 days in the future from the start of
+ the current day
+
+ Consult the DateField javadocs for more information.
+ -->
+ <fieldType name="date" class="solr.DateField" sortMissingLast="true" omitNorms="true"/>
+
+
+ <!-- The "RandomSortField" is not used to store or search any
+ data. You can declare fields of this type it in your schema
+ to generate psuedo-random orderings of your docs for sorting
+ purposes. The ordering is generated based on the field name
+ and the version of the index, As long as the index version
+ remains unchanged, and the same field name is reused,
+ the ordering of the docs will be consistent.
+ If you want differend psuedo-random orderings of documents,
+ for the same version of the index, use a dynamicField and
+ change the name
+ -->
+ <fieldType name="random" class="solr.RandomSortField" indexed="true" />
+
+ <!-- solr.TextField allows the specification of custom text analyzers
+ specified as a tokenizer and a list of token filters. Different
+ analyzers may be specified for indexing and querying.
+
+ The optional positionIncrementGap puts space between multiple fields of
+ this type on the same document, with the purpose of preventing false phrase
+ matching across fields.
+
+ For more info on customizing your analyzer chain, please see
+ http://wiki.apache.org/solr/AnalyzersTokenizersTokenFilters
+ -->
+
+ <!-- One can also specify an existing Analyzer class that has a
+ default constructor via the class attribute on the analyzer element
+ <fieldType name="text_greek" class="solr.TextField">
+ <analyzer class="org.apache.lucene.analysis.el.GreekAnalyzer"/>
+ </fieldType>
+ -->
+
+ <!-- A text field that only splits on whitespace for exact matching of words -->
+ <fieldType name="text_ws" class="solr.TextField" positionIncrementGap="100">
+ <analyzer>
+ <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+ </analyzer>
+ </fieldType>
+
+ <!-- A text field that uses WordDelimiterFilter to enable splitting and matching of
+ words on case-change, alpha numeric boundaries, and non-alphanumeric chars,
+ so that a query of "wifi" or "wi fi" could match a document containing "Wi-Fi".
+ Synonyms and stopwords are customized by external files, and stemming is enabled.
+ Duplicate tokens at the same position (which may result from Stemmed Synonyms or
+ WordDelim parts) are removed.
+ -->
+ <fieldType name="text" class="solr.TextField" positionIncrementGap="100">
+ <analyzer type="index">
+ <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+ <!-- in this example, we will only use synonyms at query time
+ <filter class="solr.SynonymFilterFactory" synonyms="index_synonyms.txt" ignoreCase="true" expand="false"/>
+ -->
+ <!--<filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt"/>-->
+ <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="1"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <!--<filter class="solr.EnglishPorterFilterFactory" protected="protwords.txt"/>-->
+ <filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
+ </analyzer>
+ <analyzer type="query">
+ <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+ <!--<filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>-->
+ <!--<filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt"/>-->
+ <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="0" catenateNumbers="0" catenateAll="0" splitOnCaseChange="1"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <!--<filter class="solr.EnglishPorterFilterFactory" protected="protwords.txt"/>-->
+ <filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
+ </analyzer>
+ </fieldType>
+
+
+ <!-- Less flexible matching, but less false matches. Probably not ideal for product names,
+ but may be good for SKUs. Can insert dashes in the wrong place and still match. -->
+ <fieldType name="textTight" class="solr.TextField" positionIncrementGap="100" >
+ <analyzer>
+ <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+ <!--<filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="false"/>-->
+ <!--<filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt"/>-->
+ <filter class="solr.WordDelimiterFilterFactory" generateWordParts="0" generateNumberParts="0" catenateWords="1" catenateNumbers="1" catenateAll="0"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <!--<filter class="solr.EnglishPorterFilterFactory" protected="protwords.txt"/>-->
+ <filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
+ </analyzer>
+ </fieldType>
+
+ <!-- This is an example of using the KeywordTokenizer along
+ With various TokenFilterFactories to produce a sortable field
+ that does not include some properties of the source text
+ -->
+ <fieldType name="alphaOnlySort" class="solr.TextField" sortMissingLast="true" omitNorms="true">
+ <analyzer>
+ <!-- KeywordTokenizer does no actual tokenizing, so the entire
+ input string is preserved as a single token
+ -->
+ <tokenizer class="solr.KeywordTokenizerFactory"/>
+ <!-- The LowerCase TokenFilter does what you expect, which can be
+ when you want your sorting to be case insensitive
+ -->
+ <filter class="solr.LowerCaseFilterFactory" />
+ <!-- The TrimFilter removes any leading or trailing whitespace -->
+ <filter class="solr.TrimFilterFactory" />
+ <!-- The PatternReplaceFilter gives you the flexibility to use
+ Java Regular expression to replace any sequence of characters
+ matching a pattern with an arbitrary replacement string,
+ which may include back refrences to portions of the orriginal
+ string matched by the pattern.
+
+ See the Java Regular Expression documentation for more
+ infomation on pattern and replacement string syntax.
+
+ http://java.sun.com/j2se/1.5.0/docs/api/java/util/regex/package-summary.html
+ -->
+ <filter class="solr.PatternReplaceFilterFactory"
+ pattern="([^a-z])" replacement="" replace="all"
+ />
+ </analyzer>
+ </fieldType>
+
+ <!-- since fields of this type are by default not stored or indexed, any data added to
+ them will be ignored outright
+ -->
+ <fieldtype name="ignored" stored="false" indexed="false" class="solr.StrField" />
+
+ </types>
+
+
+ <fields>
+ <!-- Valid attributes for fields:
+ name: mandatory - the name for the field
+ type: mandatory - the name of a previously defined type from the <types> section
+ indexed: true if this field should be indexed (searchable or sortable)
+ stored: true if this field should be retrievable
+ compressed: [false] if this field should be stored using gzip compression
+ (this will only apply if the field type is compressable; among
+ the standard field types, only TextField and StrField are)
+ multiValued: true if this field may contain multiple values per document
+ omitNorms: (expert) set to true to omit the norms associated with
+ this field (this disables length normalization and index-time
+ boosting for the field, and saves some memory). Only full-text
+ fields or fields that need an index-time boost need norms.
+ termVectors: [false] set to true to store the term vector for a given field.
+ When using MoreLikeThis, fields used for similarity should be stored for
+ best performance.
+ -->
+
+ <field name="solr_id" type="string" indexed="true" stored="true" required="true" />
+ <field name="desc" type="string" indexed="true" stored="true" multiValued="true" />
+
+ <field name="date" type="date" indexed="true" stored="true" />
+
+ <field name="timestamp" type="date" indexed="true" stored="true" default="NOW" multiValued="false"/>
+
+
+ <!-- Dynamic field definitions. If a field name is not found, dynamicFields
+ will be used if the name matches any of the patterns.
+ RESTRICTION: the glob-like pattern in the name attribute must have
+ a "*" only at the start or the end.
+ EXAMPLE: name="*_i" will match any field ending in _i (like myid_i, z_i)
+ Longer patterns will be matched first. if equal size patterns
+ both match, the first appearing in the schema will be used. -->
+ <dynamicField name="*_i" type="sint" indexed="true" stored="true"/>
+ <dynamicField name="*_s" type="string" indexed="true" stored="true"/>
+ <dynamicField name="*_l" type="slong" indexed="true" stored="true"/>
+ <dynamicField name="*_t" type="text" indexed="true" stored="true"/>
+ <dynamicField name="*_b" type="boolean" indexed="true" stored="true"/>
+ <dynamicField name="*_f" type="sfloat" indexed="true" stored="true"/>
+ <dynamicField name="*_d" type="sdouble" indexed="true" stored="true"/>
+ <dynamicField name="*_dt" type="date" indexed="true" stored="true"/>
+
+ <dynamicField name="random*" type="random" />
+
+ <!-- uncomment the following to ignore any fields that don't already match an existing
+ field name or dynamic field, rather than reporting them as an error.
+ alternately, change the type="ignored" to some other type e.g. "text" if you want
+ unknown fields indexed and/or stored by default -->
+ <!--dynamicField name="*" type="ignored" /-->
+
+ </fields>
+
+ <!-- Field to use to determine and enforce document uniqueness.
+ Unless this field is marked with required="false", it will be a required field
+ -->
+ <uniqueKey>solr_id</uniqueKey>
+
+ <!-- field for the QueryParser to use when an explicit fieldname is absent -->
+ <defaultSearchField>desc</defaultSearchField>
+
+ <!-- SolrQueryParser configuration: defaultOperator="AND|OR" -->
+ <solrQueryParser defaultOperator="OR"/>
+
+</schema>