You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by da...@apache.org on 2018/11/02 11:33:23 UTC
[02/25] lucene-solr:jira/gradle: Adding dataimporthandler-extras
module
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestXPathEntityProcessor.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestXPathEntityProcessor.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestXPathEntityProcessor.java
deleted file mode 100644
index 72da77a..0000000
--- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestXPathEntityProcessor.java
+++ /dev/null
@@ -1,491 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import java.io.File;
-import java.io.Reader;
-import java.io.StringReader;
-import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.concurrent.TimeUnit;
-
-import org.junit.Test;
-
-/**
- * <p>
- * Test for XPathEntityProcessor
- * </p>
- *
- *
- * @since solr 1.3
- */
-public class TestXPathEntityProcessor extends AbstractDataImportHandlerTestCase {
- boolean simulateSlowReader;
- boolean simulateSlowResultProcessor;
- int rowsToRead = -1;
-
- @Test
- public void withFieldsAndXpath() throws Exception {
- File tmpdir = createTempDir().toFile();
-
- createFile(tmpdir, "x.xsl", xsl.getBytes(StandardCharsets.UTF_8), false);
- Map entityAttrs = createMap("name", "e", "url", "cd.xml",
- XPathEntityProcessor.FOR_EACH, "/catalog/cd");
- List fields = new ArrayList();
- fields.add(createMap("column", "title", "xpath", "/catalog/cd/title"));
- fields.add(createMap("column", "artist", "xpath", "/catalog/cd/artist"));
- fields.add(createMap("column", "year", "xpath", "/catalog/cd/year"));
- Context c = getContext(null,
- new VariableResolver(), getDataSource(cdData), Context.FULL_DUMP, fields, entityAttrs);
- XPathEntityProcessor xPathEntityProcessor = new XPathEntityProcessor();
- xPathEntityProcessor.init(c);
- List<Map<String, Object>> result = new ArrayList<>();
- while (true) {
- Map<String, Object> row = xPathEntityProcessor.nextRow();
- if (row == null)
- break;
- result.add(row);
- }
- assertEquals(3, result.size());
- assertEquals("Empire Burlesque", result.get(0).get("title"));
- assertEquals("Bonnie Tyler", result.get(1).get("artist"));
- assertEquals("1982", result.get(2).get("year"));
- }
-
- @Test
- public void testMultiValued() throws Exception {
- Map entityAttrs = createMap("name", "e", "url", "testdata.xml",
- XPathEntityProcessor.FOR_EACH, "/root");
- List fields = new ArrayList();
- fields.add(createMap("column", "a", "xpath", "/root/a", DataImporter.MULTI_VALUED, "true"));
- Context c = getContext(null,
- new VariableResolver(), getDataSource(testXml), Context.FULL_DUMP, fields, entityAttrs);
- XPathEntityProcessor xPathEntityProcessor = new XPathEntityProcessor();
- xPathEntityProcessor.init(c);
- List<Map<String, Object>> result = new ArrayList<>();
- while (true) {
- Map<String, Object> row = xPathEntityProcessor.nextRow();
- if (row == null)
- break;
- result.add(row);
- }
- List l = (List)result.get(0).get("a");
- assertEquals(3, l.size());
- assertEquals("1", l.get(0));
- assertEquals("2", l.get(1));
- assertEquals("ΓΌ", l.get(2));
- }
-
- @SuppressWarnings({"rawtypes", "unchecked"})
- @Test
- public void testMultiValuedWithMultipleDocuments() throws Exception {
- Map entityAttrs = createMap("name", "e", "url", "testdata.xml", XPathEntityProcessor.FOR_EACH, "/documents/doc");
- List fields = new ArrayList();
- fields.add(createMap("column", "id", "xpath", "/documents/doc/id", DataImporter.MULTI_VALUED, "false"));
- fields.add(createMap("column", "a", "xpath", "/documents/doc/a", DataImporter.MULTI_VALUED, "true"));
- fields.add(createMap("column", "s1dataA", "xpath", "/documents/doc/sec1/s1dataA", DataImporter.MULTI_VALUED, "true"));
- fields.add(createMap("column", "s1dataB", "xpath", "/documents/doc/sec1/s1dataB", DataImporter.MULTI_VALUED, "true"));
- fields.add(createMap("column", "s1dataC", "xpath", "/documents/doc/sec1/s1dataC", DataImporter.MULTI_VALUED, "true"));
-
- Context c = getContext(null,
- new VariableResolver(), getDataSource(textMultipleDocuments), Context.FULL_DUMP, fields, entityAttrs);
- XPathEntityProcessor xPathEntityProcessor = new XPathEntityProcessor();
- xPathEntityProcessor.init(c);
- List<Map<String, Object>> result = new ArrayList<>();
- while (true) {
- Map<String, Object> row = xPathEntityProcessor.nextRow();
- if (row == null)
- break;
- result.add(row);
- }
- {
- assertEquals("1", result.get(0).get("id"));
- List a = (List)result.get(0).get("a");
- List s1dataA = (List)result.get(0).get("s1dataA");
- List s1dataB = (List)result.get(0).get("s1dataB");
- List s1dataC = (List)result.get(0).get("s1dataC");
- assertEquals(2, a.size());
- assertEquals("id1-a1", a.get(0));
- assertEquals("id1-a2", a.get(1));
- assertEquals(3, s1dataA.size());
- assertEquals("id1-s1dataA-1", s1dataA.get(0));
- assertNull(s1dataA.get(1));
- assertEquals("id1-s1dataA-3", s1dataA.get(2));
- assertEquals(3, s1dataB.size());
- assertEquals("id1-s1dataB-1", s1dataB.get(0));
- assertEquals("id1-s1dataB-2", s1dataB.get(1));
- assertEquals("id1-s1dataB-3", s1dataB.get(2));
- assertEquals(3, s1dataC.size());
- assertNull(s1dataC.get(0));
- assertNull(s1dataC.get(1));
- assertNull(s1dataC.get(2));
- }
- {
- assertEquals("2", result.get(1).get("id"));
- List a = (List)result.get(1).get("a");
- List s1dataA = (List)result.get(1).get("s1dataA");
- List s1dataB = (List)result.get(1).get("s1dataB");
- List s1dataC = (List)result.get(1).get("s1dataC");
- assertTrue(a==null || a.size()==0);
- assertEquals(1, s1dataA.size());
- assertNull(s1dataA.get(0));
- assertEquals(1, s1dataB.size());
- assertEquals("id2-s1dataB-1", s1dataB.get(0));
- assertEquals(1, s1dataC.size());
- assertNull(s1dataC.get(0));
- }
- {
- assertEquals("3", result.get(2).get("id"));
- List a = (List)result.get(2).get("a");
- List s1dataA = (List)result.get(2).get("s1dataA");
- List s1dataB = (List)result.get(2).get("s1dataB");
- List s1dataC = (List)result.get(2).get("s1dataC");
- assertTrue(a==null || a.size()==0);
- assertEquals(1, s1dataA.size());
- assertEquals("id3-s1dataA-1", s1dataA.get(0));
- assertEquals(1, s1dataB.size());
- assertNull(s1dataB.get(0));
- assertEquals(1, s1dataC.size());
- assertNull(s1dataC.get(0));
- }
- {
- assertEquals("4", result.get(3).get("id"));
- List a = (List)result.get(3).get("a");
- List s1dataA = (List)result.get(3).get("s1dataA");
- List s1dataB = (List)result.get(3).get("s1dataB");
- List s1dataC = (List)result.get(3).get("s1dataC");
- assertTrue(a==null || a.size()==0);
- assertEquals(1, s1dataA.size());
- assertEquals("id4-s1dataA-1", s1dataA.get(0));
- assertEquals(1, s1dataB.size());
- assertEquals("id4-s1dataB-1", s1dataB.get(0));
- assertEquals(1, s1dataC.size());
- assertEquals("id4-s1dataC-1", s1dataC.get(0));
- }
- {
- assertEquals("5", result.get(4).get("id"));
- List a = (List)result.get(4).get("a");
- List s1dataA = (List)result.get(4).get("s1dataA");
- List s1dataB = (List)result.get(4).get("s1dataB");
- List s1dataC = (List)result.get(4).get("s1dataC");
- assertTrue(a==null || a.size()==0);
- assertEquals(1, s1dataA.size());
- assertNull(s1dataA.get(0));
- assertEquals(1, s1dataB.size());
- assertNull(s1dataB.get(0));
- assertEquals(1, s1dataC.size());
- assertEquals("id5-s1dataC-1", s1dataC.get(0));
- }
- {
- assertEquals("6", result.get(5).get("id"));
- List a = (List)result.get(5).get("a");
- List s1dataA = (List)result.get(5).get("s1dataA");
- List s1dataB = (List)result.get(5).get("s1dataB");
- List s1dataC = (List)result.get(5).get("s1dataC");
- assertTrue(a==null || a.size()==0);
- assertEquals(3, s1dataA.size());
- assertEquals("id6-s1dataA-1", s1dataA.get(0));
- assertEquals("id6-s1dataA-2", s1dataA.get(1));
- assertNull(s1dataA.get(2));
- assertEquals(3, s1dataB.size());
- assertEquals("id6-s1dataB-1", s1dataB.get(0));
- assertEquals("id6-s1dataB-2", s1dataB.get(1));
- assertEquals("id6-s1dataB-3", s1dataB.get(2));
- assertEquals(3, s1dataC.size());
- assertEquals("id6-s1dataC-1", s1dataC.get(0));
- assertNull(s1dataC.get(1));
- assertEquals("id6-s1dataC-3", s1dataC.get(2));
- }
- }
-
- @Test
- public void testMultiValuedFlatten() throws Exception {
- Map entityAttrs = createMap("name", "e", "url", "testdata.xml",
- XPathEntityProcessor.FOR_EACH, "/root");
- List fields = new ArrayList();
- fields.add(createMap("column", "a", "xpath", "/root/a" ,"flatten","true"));
- Context c = getContext(null,
- new VariableResolver(), getDataSource(testXmlFlatten), Context.FULL_DUMP, fields, entityAttrs);
- XPathEntityProcessor xPathEntityProcessor = new XPathEntityProcessor();
- xPathEntityProcessor.init(c);
- Map<String, Object> result = null;
- while (true) {
- Map<String, Object> row = xPathEntityProcessor.nextRow();
- if (row == null)
- break;
- result = row;
- }
- assertEquals("1B2", result.get("a"));
- }
-
- @Test
- public void withFieldsAndXpathStream() throws Exception {
- final Object monitor = new Object();
- final boolean[] done = new boolean[1];
-
- Map entityAttrs = createMap("name", "e", "url", "cd.xml",
- XPathEntityProcessor.FOR_EACH, "/catalog/cd", "stream", "true", "batchSize","1");
- List fields = new ArrayList();
- fields.add(createMap("column", "title", "xpath", "/catalog/cd/title"));
- fields.add(createMap("column", "artist", "xpath", "/catalog/cd/artist"));
- fields.add(createMap("column", "year", "xpath", "/catalog/cd/year"));
- Context c = getContext(null,
- new VariableResolver(), getDataSource(cdData), Context.FULL_DUMP, fields, entityAttrs);
- XPathEntityProcessor xPathEntityProcessor = new XPathEntityProcessor() {
- private int count;
-
- @Override
- protected Map<String, Object> readRow(Map<String, Object> record,
- String xpath) {
- synchronized (monitor) {
- if (simulateSlowReader && !done[0]) {
- try {
- monitor.wait(100);
- } catch (InterruptedException e) {
- throw new RuntimeException(e);
- }
- }
- }
-
- return super.readRow(record, xpath);
- }
- };
-
- if (simulateSlowResultProcessor) {
- xPathEntityProcessor.blockingQueueSize = 1;
- }
- xPathEntityProcessor.blockingQueueTimeOut = 1;
- xPathEntityProcessor.blockingQueueTimeOutUnits = TimeUnit.MICROSECONDS;
-
- xPathEntityProcessor.init(c);
- List<Map<String, Object>> result = new ArrayList<>();
- while (true) {
- if (rowsToRead >= 0 && result.size() >= rowsToRead) {
- Thread.currentThread().interrupt();
- }
- Map<String, Object> row = xPathEntityProcessor.nextRow();
- if (row == null)
- break;
- result.add(row);
- if (simulateSlowResultProcessor) {
- synchronized (xPathEntityProcessor.publisherThread) {
- if (xPathEntityProcessor.publisherThread.isAlive()) {
- xPathEntityProcessor.publisherThread.wait(1000);
- }
- }
- }
- }
-
- synchronized (monitor) {
- done[0] = true;
- monitor.notify();
- }
-
- // confirm that publisher thread stops.
- xPathEntityProcessor.publisherThread.join(1000);
- assertEquals("Expected thread to stop", false, xPathEntityProcessor.publisherThread.isAlive());
-
- assertEquals(rowsToRead < 0 ? 3 : rowsToRead, result.size());
-
- if (rowsToRead < 0) {
- assertEquals("Empire Burlesque", result.get(0).get("title"));
- assertEquals("Bonnie Tyler", result.get(1).get("artist"));
- assertEquals("1982", result.get(2).get("year"));
- }
- }
-
- @Test
- public void withFieldsAndXpathStreamContinuesOnTimeout() throws Exception {
- simulateSlowReader = true;
- withFieldsAndXpathStream();
- }
-
- @Test
- public void streamWritesMessageAfterBlockedAttempt() throws Exception {
- simulateSlowResultProcessor = true;
- withFieldsAndXpathStream();
- }
-
- @Test
- public void streamStopsAfterInterrupt() throws Exception {
- simulateSlowResultProcessor = true;
- rowsToRead = 1;
- withFieldsAndXpathStream();
- }
-
- @Test
- public void withDefaultSolrAndXsl() throws Exception {
- File tmpdir = createTempDir().toFile();
- AbstractDataImportHandlerTestCase.createFile(tmpdir, "x.xsl", xsl.getBytes(StandardCharsets.UTF_8),
- false);
-
- Map entityAttrs = createMap("name", "e",
- XPathEntityProcessor.USE_SOLR_ADD_SCHEMA, "true", "xsl", ""
- + new File(tmpdir, "x.xsl").toURI(), "url", "cd.xml");
- Context c = getContext(null,
- new VariableResolver(), getDataSource(cdData), Context.FULL_DUMP, null, entityAttrs);
- XPathEntityProcessor xPathEntityProcessor = new XPathEntityProcessor();
- xPathEntityProcessor.init(c);
- List<Map<String, Object>> result = new ArrayList<>();
- while (true) {
- Map<String, Object> row = xPathEntityProcessor.nextRow();
- if (row == null)
- break;
- result.add(row);
- }
- assertEquals(3, result.size());
- assertEquals("Empire Burlesque", result.get(0).get("title"));
- assertEquals("Bonnie Tyler", result.get(1).get("artist"));
- assertEquals("1982", result.get(2).get("year"));
- }
-
- private DataSource<Reader> getDataSource(final String xml) {
- return new DataSource<Reader>() {
-
- @Override
- public void init(Context context, Properties initProps) {
- }
-
- @Override
- public void close() {
- }
-
- @Override
- public Reader getData(String query) {
- return new StringReader(xml);
- }
- };
- }
-
- private static final String xsl = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
- + "<xsl:stylesheet version=\"1.0\"\n"
- + "xmlns:xsl=\"http://www.w3.org/1999/XSL/Transform\">\n"
- + "<xsl:output version='1.0' method='xml' encoding='UTF-8' indent='yes'/>\n"
- + "\n"
- + "<xsl:template match=\"/\">\n"
- + " <add> \n"
- + " <xsl:for-each select=\"catalog/cd\">\n"
- + " <doc>\n"
- + " <field name=\"title\"><xsl:value-of select=\"title\"/></field>\n"
- + " <field name=\"artist\"><xsl:value-of select=\"artist\"/></field>\n"
- + " <field name=\"country\"><xsl:value-of select=\"country\"/></field>\n"
- + " <field name=\"company\"><xsl:value-of select=\"company\"/></field> \n"
- + " <field name=\"price\"><xsl:value-of select=\"price\"/></field>\n"
- + " <field name=\"year\"><xsl:value-of select=\"year\"/></field> \n"
- + " </doc>\n"
- + " </xsl:for-each>\n"
- + " </add> \n"
- + "</xsl:template>\n" + "</xsl:stylesheet>";
-
- private static final String cdData = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
- + "<?xml-stylesheet type=\"text/xsl\" href=\"solr.xsl\"?>\n"
- + "<catalog>\n"
- + "\t<cd>\n"
- + "\t\t<title>Empire Burlesque</title>\n"
- + "\t\t<artist>Bob Dylan</artist>\n"
- + "\t\t<country>USA</country>\n"
- + "\t\t<company>Columbia</company>\n"
- + "\t\t<price>10.90</price>\n"
- + "\t\t<year>1985</year>\n"
- + "\t</cd>\n"
- + "\t<cd>\n"
- + "\t\t<title>Hide your heart</title>\n"
- + "\t\t<artist>Bonnie Tyler</artist>\n"
- + "\t\t<country>UK</country>\n"
- + "\t\t<company>CBS Records</company>\n"
- + "\t\t<price>9.90</price>\n"
- + "\t\t<year>1988</year>\n"
- + "\t</cd>\n"
- + "\t<cd>\n"
- + "\t\t<title>Greatest Hits</title>\n"
- + "\t\t<artist>Dolly Parton</artist>\n"
- + "\t\t<country>USA</country>\n"
- + "\t\t<company>RCA</company>\n"
- + "\t\t<price>9.90</price>\n"
- + "\t\t<year>1982</year>\n" + "\t</cd>\n" + "</catalog>\t";
-
- private static final String testXml = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<!DOCTYPE root [\n<!ENTITY uuml \"ü\" >\n]>\n<root><a>1</a><a>2</a><a>ü</a></root>";
-
- private static final String testXmlFlatten = "<?xml version=\"1.0\" encoding=\"UTF-8\"?><root><a>1<b>B</b>2</a></root>";
-
- private static final String textMultipleDocuments =
- "<?xml version=\"1.0\" ?>" +
- "<documents>" +
- " <doc>" +
- " <id>1</id>" +
- " <a>id1-a1</a>" +
- " <a>id1-a2</a>" +
- " <sec1>" +
- " <s1dataA>id1-s1dataA-1</s1dataA>" +
- " <s1dataB>id1-s1dataB-1</s1dataB>" +
- " </sec1>" +
- " <sec1>" +
- " <s1dataB>id1-s1dataB-2</s1dataB>" +
- " </sec1>" +
- " <sec1>" +
- " <s1dataA>id1-s1dataA-3</s1dataA>" +
- " <s1dataB>id1-s1dataB-3</s1dataB>" +
- " </sec1>" +
- " </doc>" +
- " <doc>" +
- " <id>2</id>" +
- " <sec1>" +
- " <s1dataB>id2-s1dataB-1</s1dataB>" +
- " </sec1>" +
- " </doc>" +
- " <doc>" +
- " <id>3</id>" +
- " <sec1>" +
- " <s1dataA>id3-s1dataA-1</s1dataA>" +
- " </sec1>" +
- " </doc>" +
- " <doc>" +
- " <id>4</id>" +
- " <sec1>" +
- " <s1dataA>id4-s1dataA-1</s1dataA>" +
- " <s1dataB>id4-s1dataB-1</s1dataB>" +
- " <s1dataC>id4-s1dataC-1</s1dataC>" +
- " </sec1>" +
- " </doc>" +
- " <doc>" +
- " <id>5</id>" +
- " <sec1>" +
- " <s1dataC>id5-s1dataC-1</s1dataC>" +
- " </sec1>" +
- " </doc>" +
- " <doc>" +
- " <id>6</id>" +
- " <sec1>" +
- " <s1dataA>id6-s1dataA-1</s1dataA>" +
- " <s1dataB>id6-s1dataB-1</s1dataB>" +
- " <s1dataC>id6-s1dataC-1</s1dataC>" +
- " </sec1>" +
- " <sec1>" +
- " <s1dataA>id6-s1dataA-2</s1dataA>" +
- " <s1dataB>id6-s1dataB-2</s1dataB>" +
- " </sec1>" +
- " <sec1>" +
- " <s1dataB>id6-s1dataB-3</s1dataB>" +
- " <s1dataC>id6-s1dataC-3</s1dataC>" +
- " </sec1>" +
- " </doc>" +
- "</documents>"
- ;
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestXPathRecordReader.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestXPathRecordReader.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestXPathRecordReader.java
deleted file mode 100644
index d8e3cbe..0000000
--- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestXPathRecordReader.java
+++ /dev/null
@@ -1,598 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import java.io.StringReader;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-import org.junit.Test;
-
-/**
- * <p> Test for XPathRecordReader </p>
- *
- *
- * @since solr 1.3
- */
-public class TestXPathRecordReader extends AbstractDataImportHandlerTestCase {
- @Test
- public void testBasic() {
- String xml="<root>\n"
- + " <b><c>Hello C1</c>\n"
- + " <c>Hello C1</c>\n"
- + " </b>\n"
- + " <b><c>Hello C2</c>\n"
- + " </b>\n"
- + "</root>";
- XPathRecordReader rr = new XPathRecordReader("/root/b");
- rr.addField("c", "/root/b/c", true);
- List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
- assertEquals(2, l.size());
- assertEquals(2, ((List) l.get(0).get("c")).size());
- assertEquals(1, ((List) l.get(1).get("c")).size());
- }
-
- @Test
- public void testAttributes() {
- String xml="<root>\n"
- + " <b a=\"x0\" b=\"y0\" />\n"
- + " <b a=\"x1\" b=\"y1\" />\n"
- + " <b a=\"x2\" b=\"y2\" />\n"
- + "</root>";
- XPathRecordReader rr = new XPathRecordReader("/root/b");
- rr.addField("a", "/root/b/@a", false);
- rr.addField("b", "/root/b/@b", false);
- List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
- assertEquals(3, l.size());
- assertEquals("x0", l.get(0).get("a"));
- assertEquals("x1", l.get(1).get("a"));
- assertEquals("x2", l.get(2).get("a"));
- assertEquals("y0", l.get(0).get("b"));
- assertEquals("y1", l.get(1).get("b"));
- assertEquals("y2", l.get(2).get("b"));
- }
-
- @Test
- public void testAttrInRoot(){
- String xml="<r>\n" +
- "<merchantProduct id=\"814636051\" mid=\"189973\">\n" +
- " <in_stock type=\"stock-4\" />\n" +
- " <condition type=\"cond-0\" />\n" +
- " <price>301.46</price>\n" +
- " </merchantProduct>\n" +
- "<merchantProduct id=\"814636052\" mid=\"189974\">\n" +
- " <in_stock type=\"stock-5\" />\n" +
- " <condition type=\"cond-1\" />\n" +
- " <price>302.46</price>\n" +
- " </merchantProduct>\n" +
- "\n" +
- "</r>";
- XPathRecordReader rr = new XPathRecordReader("/r/merchantProduct");
- rr.addField("id", "/r/merchantProduct/@id", false);
- rr.addField("mid", "/r/merchantProduct/@mid", false);
- rr.addField("price", "/r/merchantProduct/price", false);
- rr.addField("conditionType", "/r/merchantProduct/condition/@type", false);
- List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
- Map<String, Object> m = l.get(0);
- assertEquals("814636051", m.get("id"));
- assertEquals("189973", m.get("mid"));
- assertEquals("301.46", m.get("price"));
- assertEquals("cond-0", m.get("conditionType"));
-
- m = l.get(1);
- assertEquals("814636052", m.get("id"));
- assertEquals("189974", m.get("mid"));
- assertEquals("302.46", m.get("price"));
- assertEquals("cond-1", m.get("conditionType"));
- }
-
- @Test
- public void testAttributes2Level() {
- String xml="<root>\n"
- + "<a>\n <b a=\"x0\" b=\"y0\" />\n"
- + " <b a=\"x1\" b=\"y1\" />\n"
- + " <b a=\"x2\" b=\"y2\" />\n"
- + " </a>"
- + "</root>";
- XPathRecordReader rr = new XPathRecordReader("/root/a/b");
- rr.addField("a", "/root/a/b/@a", false);
- rr.addField("b", "/root/a/b/@b", false);
- List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
- assertEquals(3, l.size());
- assertEquals("x0", l.get(0).get("a"));
- assertEquals("y1", l.get(1).get("b"));
- }
-
- @Test
- public void testAttributes2LevelHetero() {
- String xml="<root>\n"
- + "<a>\n <b a=\"x0\" b=\"y0\" />\n"
- + " <b a=\"x1\" b=\"y1\" />\n"
- + " <b a=\"x2\" b=\"y2\" />\n"
- + " </a>"
- + "<x>\n <b a=\"x4\" b=\"y4\" />\n"
- + " <b a=\"x5\" b=\"y5\" />\n"
- + " <b a=\"x6\" b=\"y6\" />\n"
- + " </x>"
- + "</root>";
- XPathRecordReader rr = new XPathRecordReader("/root/a | /root/x");
- rr.addField("a", "/root/a/b/@a", false);
- rr.addField("b", "/root/a/b/@b", false);
- rr.addField("a", "/root/x/b/@a", false);
- rr.addField("b", "/root/x/b/@b", false);
-
- final List<Map<String, Object>> a = new ArrayList<>();
- final List<Map<String, Object>> x = new ArrayList<>();
- rr.streamRecords(new StringReader(xml), (record, xpath) -> {
- if (record == null) return;
- if (xpath.equals("/root/a")) a.add(record);
- if (xpath.equals("/root/x")) x.add(record);
- });
-
- assertEquals(1, a.size());
- assertEquals(1, x.size());
- }
-
- @Test
- public void testAttributes2LevelMissingAttrVal() {
- String xml="<root>\n"
- + "<a>\n <b a=\"x0\" b=\"y0\" />\n"
- + " <b a=\"x1\" b=\"y1\" />\n"
- + " </a>"
- + "<a>\n <b a=\"x3\" />\n"
- + " <b b=\"y4\" />\n"
- + " </a>"
- + "</root>";
- XPathRecordReader rr = new XPathRecordReader("/root/a");
- rr.addField("a", "/root/a/b/@a", true);
- rr.addField("b", "/root/a/b/@b", true);
- List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
- assertEquals(2, l.size());
- assertNull(((List) l.get(1).get("a")).get(1));
- assertNull(((List) l.get(1).get("b")).get(0));
- }
-
- @Test
- public void testElems2LevelMissing() {
- String xml="<root>\n"
- + "\t<a>\n"
- + "\t <b>\n\t <x>x0</x>\n"
- + "\t <y>y0</y>\n"
- + "\t </b>\n"
- + "\t <b>\n\t <x>x1</x>\n"
- + "\t <y>y1</y>\n"
- + "\t </b>\n"
- + "\t </a>\n"
- + "\t<a>\n"
- + "\t <b>\n\t <x>x3</x>\n\t </b>\n"
- + "\t <b>\n\t <y>y4</y>\n\t </b>\n"
- + "\t </a>\n"
- + "</root>";
- XPathRecordReader rr = new XPathRecordReader("/root/a");
- rr.addField("a", "/root/a/b/x", true);
- rr.addField("b", "/root/a/b/y", true);
- List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
- assertEquals(2, l.size());
- assertNull(((List) l.get(1).get("a")).get(1));
- assertNull(((List) l.get(1).get("b")).get(0));
- }
-
- @Test
- public void testElems2LevelEmpty() {
- String xml="<root>\n"
- + "\t<a>\n"
- + "\t <b>\n\t <x>x0</x>\n"
- + "\t <y>y0</y>\n"
- + "\t </b>\n"
- + "\t <b>\n\t <x></x>\n" // empty
- + "\t <y>y1</y>\n"
- + "\t </b>\n"
- + "\t</a>\n"
- + "</root>";
- XPathRecordReader rr = new XPathRecordReader("/root/a");
- rr.addField("a", "/root/a/b/x", true);
- rr.addField("b", "/root/a/b/y", true);
- List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
- assertEquals(1, l.size());
- assertEquals("x0",((List) l.get(0).get("a")).get(0));
- assertEquals("y0",((List) l.get(0).get("b")).get(0));
- assertEquals("",((List) l.get(0).get("a")).get(1));
- assertEquals("y1",((List) l.get(0).get("b")).get(1));
- }
-
- @Test
- public void testMixedContent() {
- String xml = "<xhtml:p xmlns:xhtml=\"http://xhtml.com/\" >This text is \n" +
- " <xhtml:b>bold</xhtml:b> and this text is \n" +
- " <xhtml:u>underlined</xhtml:u>!\n" +
- "</xhtml:p>";
- XPathRecordReader rr = new XPathRecordReader("/p");
- rr.addField("p", "/p", true);
- rr.addField("b", "/p/b", true);
- rr.addField("u", "/p/u", true);
- List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
- Map<String, Object> row = l.get(0);
-
- assertEquals("bold", ((List) row.get("b")).get(0));
- assertEquals("underlined", ((List) row.get("u")).get(0));
- String p = (String) ((List) row.get("p")).get(0);
- assertTrue(p.contains("This text is"));
- assertTrue(p.contains("and this text is"));
- assertTrue(p.contains("!"));
- // Should not contain content from child elements
- assertFalse(p.contains("bold"));
- }
-
- @Test
- public void testMixedContentFlattened() {
- String xml = "<xhtml:p xmlns:xhtml=\"http://xhtml.com/\" >This text is \n" +
- " <xhtml:b>bold</xhtml:b> and this text is \n" +
- " <xhtml:u>underlined</xhtml:u>!\n" +
- "</xhtml:p>";
- XPathRecordReader rr = new XPathRecordReader("/p");
- rr.addField("p", "/p", false, XPathRecordReader.FLATTEN);
- List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
- Map<String, Object> row = l.get(0);
- assertEquals("This text is \n" +
- " bold and this text is \n" +
- " underlined!", ((String)row.get("p")).trim() );
- }
-
- @Test
- public void testElems2LevelWithAttrib() {
- String xml = "<root>\n\t<a>\n\t <b k=\"x\">\n"
- + "\t <x>x0</x>\n"
- + "\t <y></y>\n" // empty
- + "\t </b>\n"
- + "\t <b k=\"y\">\n"
- + "\t <x></x>\n" // empty
- + "\t <y>y1</y>\n"
- + "\t </b>\n"
- + "\t <b k=\"z\">\n"
- + "\t <x>x2</x>\n"
- + "\t <y>y2</y>\n"
- + "\t </b>\n"
- + "\t </a>\n"
- + "\t <a>\n\t <b>\n"
- + "\t <x>x3</x>\n"
- + "\t </b>\n"
- + "\t <b>\n"
- + "\t <y>y4</y>\n"
- + "\t </b>\n"
- + "\t </a>\n"
- + "</root>";
- XPathRecordReader rr = new XPathRecordReader("/root/a");
- rr.addField("x", "/root/a/b[@k]/x", true);
- rr.addField("y", "/root/a/b[@k]/y", true);
- List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
- assertEquals(2, l.size());
- assertEquals(3, ((List) l.get(0).get("x")).size());
- assertEquals(3, ((List) l.get(0).get("y")).size());
- assertEquals("x0", ((List) l.get(0).get("x")).get(0));
- assertEquals("", ((List) l.get(0).get("y")).get(0));
- assertEquals("", ((List) l.get(0).get("x")).get(1));
- assertEquals("y1", ((List) l.get(0).get("y")).get(1));
- assertEquals("x2", ((List) l.get(0).get("x")).get(2));
- assertEquals("y2", ((List) l.get(0).get("y")).get(2));
- assertEquals(0, l.get(1).size());
- }
-
- @Test
- public void testElems2LevelWithAttribMultiple() {
- String xml="<root>\n"
- + "\t<a>\n\t <b k=\"x\" m=\"n\" >\n"
- + "\t <x>x0</x>\n"
- + "\t <y>y0</y>\n"
- + "\t </b>\n"
- + "\t <b k=\"y\" m=\"p\">\n"
- + "\t <x>x1</x>\n"
- + "\t <y>y1</y>\n"
- + "\t </b>\n"
- + "\t </a>\n"
- + "\t<a>\n\t <b k=\"x\">\n"
- + "\t <x>x3</x>\n"
- + "\t </b>\n"
- + "\t <b m=\"n\">\n"
- + "\t <y>y4</y>\n"
- + "\t </b>\n"
- + "\t </a>\n"
- + "</root>";
- XPathRecordReader rr = new XPathRecordReader("/root/a");
- rr.addField("x", "/root/a/b[@k][@m='n']/x", true);
- rr.addField("y", "/root/a/b[@k][@m='n']/y", true);
- List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
- assertEquals(2, l.size());
- assertEquals(1, ((List) l.get(0).get("x")).size());
- assertEquals(1, ((List) l.get(0).get("y")).size());
- assertEquals(0, l.get(1).size());
- }
-
- @Test
- public void testElems2LevelWithAttribVal() {
- String xml="<root>\n\t<a>\n <b k=\"x\">\n"
- + "\t <x>x0</x>\n"
- + "\t <y>y0</y>\n"
- + "\t </b>\n"
- + "\t <b k=\"y\">\n"
- + "\t <x>x1</x>\n"
- + "\t <y>y1</y>\n"
- + "\t </b>\n"
- + "\t </a>\n"
- + "\t <a>\n <b><x>x3</x></b>\n"
- + "\t <b><y>y4</y></b>\n"
- + "\t</a>\n" + "</root>";
- XPathRecordReader rr = new XPathRecordReader("/root/a");
- rr.addField("x", "/root/a/b[@k='x']/x", true);
- rr.addField("y", "/root/a/b[@k='x']/y", true);
- List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
- assertEquals(2, l.size());
- assertEquals(1, ((List) l.get(0).get("x")).size());
- assertEquals(1, ((List) l.get(0).get("y")).size());
- assertEquals(0, l.get(1).size());
- }
-
- @Test
- public void testAttribValWithSlash() {
- String xml = "<root><b>\n" +
- " <a x=\"a/b\" h=\"hello-A\"/> \n" +
- "</b></root>";
- XPathRecordReader rr = new XPathRecordReader("/root/b");
- rr.addField("x", "/root/b/a[@x='a/b']/@h", false);
- List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
- assertEquals(1, l.size());
- Map<String, Object> m = l.get(0);
- assertEquals("hello-A", m.get("x"));
- }
-
- @Test
- public void testUnsupported_Xpaths() {
- String xml = "<root><b><a x=\"a/b\" h=\"hello-A\"/> </b></root>";
- XPathRecordReader rr=null;
- try {
- rr = new XPathRecordReader("//b");
- fail("A RuntimeException was expected: //b forEach cannot begin with '//'.");
- }
- catch (RuntimeException ex) { }
- try {
- rr.addField("bold" ,"b", false);
- fail("A RuntimeException was expected: 'b' xpaths must begin with '/'.");
- }
- catch (RuntimeException ex) { }
- }
-
- @Test
- public void testAny_decendent_from_root() {
- XPathRecordReader rr = new XPathRecordReader("/anyd/contenido");
- rr.addField("descdend", "//boo", true);
- rr.addField("inr_descd","//boo/i", false);
- rr.addField("cont", "/anyd/contenido", false);
- rr.addField("id", "/anyd/contenido/@id", false);
- rr.addField("status", "/anyd/status", false);
- rr.addField("title", "/anyd/contenido/titulo", false,XPathRecordReader.FLATTEN);
- rr.addField("resume", "/anyd/contenido/resumen",false);
- rr.addField("text", "/anyd/contenido/texto", false);
-
- String xml="<anyd>\n"
- + " this <boo>top level</boo> is ignored because it is external to the forEach\n"
- + " <status>as is <boo>this element</boo></status>\n"
- + " <contenido id=\"10097\" idioma=\"cat\">\n"
- + " This one is <boo>not ignored as it's</boo> inside a forEach\n"
- + " <antetitulo><i> big <boo>antler</boo></i></antetitulo>\n"
- + " <titulo> My <i>flattened <boo>title</boo></i> </titulo>\n"
- + " <resumen> My summary <i>skip this!</i> </resumen>\n"
- + " <texto> <boo>Within the body of</boo>My text</texto>\n"
- + " <p>Access <boo>inner <i>sub clauses</i> as well</boo></p>\n"
- + " </contenido>\n"
- + "</anyd>";
-
- List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
- assertEquals(1, l.size());
- Map<String, Object> m = l.get(0);
- assertEquals("This one is inside a forEach", m.get("cont").toString().trim());
- assertEquals("10097" ,m.get("id"));
- assertEquals("My flattened title" ,m.get("title").toString().trim());
- assertEquals("My summary" ,m.get("resume").toString().trim());
- assertEquals("My text" ,m.get("text").toString().trim());
- assertEquals("not ignored as it's",(String) ((List) m.get("descdend")).get(0) );
- assertEquals("antler" ,(String) ((List) m.get("descdend")).get(1) );
- assertEquals("Within the body of" ,(String) ((List) m.get("descdend")).get(2) );
- assertEquals("inner as well" ,(String) ((List) m.get("descdend")).get(3) );
- assertEquals("sub clauses" ,m.get("inr_descd").toString().trim());
- }
-
- @Test
- public void testAny_decendent_of_a_child1() {
- XPathRecordReader rr = new XPathRecordReader("/anycd");
- rr.addField("descdend", "/anycd//boo", true);
-
- // same test string as above but checking to see if *all* //boo's are collected
- String xml="<anycd>\n"
- + " this <boo>top level</boo> is ignored because it is external to the forEach\n"
- + " <status>as is <boo>this element</boo></status>\n"
- + " <contenido id=\"10097\" idioma=\"cat\">\n"
- + " This one is <boo>not ignored as it's</boo> inside a forEach\n"
- + " <antetitulo><i> big <boo>antler</boo></i></antetitulo>\n"
- + " <titulo> My <i>flattened <boo>title</boo></i> </titulo>\n"
- + " <resumen> My summary <i>skip this!</i> </resumen>\n"
- + " <texto> <boo>Within the body of</boo>My text</texto>\n"
- + " <p>Access <boo>inner <i>sub clauses</i> as well</boo></p>\n"
- + " </contenido>\n"
- + "</anycd>";
-
- List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
- assertEquals(1, l.size());
- Map<String, Object> m = l.get(0);
- assertEquals("top level" ,(String) ((List) m.get("descdend")).get(0) );
- assertEquals("this element" ,(String) ((List) m.get("descdend")).get(1) );
- assertEquals("not ignored as it's",(String) ((List) m.get("descdend")).get(2) );
- assertEquals("antler" ,(String) ((List) m.get("descdend")).get(3) );
- assertEquals("title" ,(String) ((List) m.get("descdend")).get(4) );
- assertEquals("Within the body of" ,(String) ((List) m.get("descdend")).get(5) );
- assertEquals("inner as well" ,(String) ((List) m.get("descdend")).get(6) );
- }
-
- @Test
- public void testAny_decendent_of_a_child2() {
- XPathRecordReader rr = new XPathRecordReader("/anycd");
- rr.addField("descdend", "/anycd/contenido//boo", true);
-
- // same test string as above but checking to see if *some* //boo's are collected
- String xml="<anycd>\n"
- + " this <boo>top level</boo> is ignored because it is external to the forEach\n"
- + " <status>as is <boo>this element</boo></status>\n"
- + " <contenido id=\"10097\" idioma=\"cat\">\n"
- + " This one is <boo>not ignored as it's</boo> inside a forEach\n"
- + " <antetitulo><i> big <boo>antler</boo></i></antetitulo>\n"
- + " <titulo> My <i>flattened <boo>title</boo></i> </titulo>\n"
- + " <resumen> My summary <i>skip this!</i> </resumen>\n"
- + " <texto> <boo>Within the body of</boo>My text</texto>\n"
- + " <p>Access <boo>inner <i>sub clauses</i> as well</boo></p>\n"
- + " </contenido>\n"
- + "</anycd>";
-
- List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
- assertEquals(1, l.size());
- Map<String, Object> m = l.get(0);
- assertEquals("not ignored as it's",((List) m.get("descdend")).get(0) );
- assertEquals("antler" ,((List) m.get("descdend")).get(1) );
- assertEquals("title" ,((List) m.get("descdend")).get(2) );
- assertEquals("Within the body of" ,((List) m.get("descdend")).get(3) );
- assertEquals("inner as well" ,((List) m.get("descdend")).get(4) );
- }
-
- @Test
- public void testAnother() {
- String xml="<root>\n"
- + " <contenido id=\"10097\" idioma=\"cat\">\n"
- + " <antetitulo></antetitulo>\n"
- + " <titulo> This is my title </titulo>\n"
- + " <resumen> This is my summary </resumen>\n"
- + " <texto> This is the body of my text </texto>\n"
- + " </contenido>\n"
- + "</root>";
- XPathRecordReader rr = new XPathRecordReader("/root/contenido");
- rr.addField("id", "/root/contenido/@id", false);
- rr.addField("title", "/root/contenido/titulo", false);
- rr.addField("resume","/root/contenido/resumen",false);
- rr.addField("text", "/root/contenido/texto", false);
-
- List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
- assertEquals(1, l.size());
- Map<String, Object> m = l.get(0);
- assertEquals("10097", m.get("id"));
- assertEquals("This is my title", m.get("title").toString().trim());
- assertEquals("This is my summary", m.get("resume").toString().trim());
- assertEquals("This is the body of my text", m.get("text").toString()
- .trim());
- }
-
- @Test
- public void testSameForEachAndXpath(){
- String xml="<root>\n" +
- " <cat>\n" +
- " <name>hello</name>\n" +
- " </cat>\n" +
- " <item name=\"item name\"/>\n" +
- "</root>";
- XPathRecordReader rr = new XPathRecordReader("/root/cat/name");
- rr.addField("catName", "/root/cat/name",false);
- List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
- assertEquals("hello",l.get(0).get("catName"));
- }
-
- @Test
- public void testPutNullTest(){
- String xml = "<root>\n" +
- " <i>\n" +
- " <x>\n" +
- " <a>A.1.1</a>\n" +
- " <b>B.1.1</b>\n" +
- " </x>\n" +
- " <x>\n" +
- " <b>B.1.2</b>\n" +
- " <c>C.1.2</c>\n" +
- " </x>\n" +
- " </i>\n" +
- " <i>\n" +
- " <x>\n" +
- " <a>A.2.1</a>\n" +
- " <c>C.2.1</c>\n" +
- " </x>\n" +
- " <x>\n" +
- " <b>B.2.2</b>\n" +
- " <c>C.2.2</c>\n" +
- " </x>\n" +
- " </i>\n" +
- "</root>";
- XPathRecordReader rr = new XPathRecordReader("/root/i");
- rr.addField("a", "/root/i/x/a", true);
- rr.addField("b", "/root/i/x/b", true);
- rr.addField("c", "/root/i/x/c", true);
- List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
- Map<String, Object> map = l.get(0);
- List<String> a = (List<String>) map.get("a");
- List<String> b = (List<String>) map.get("b");
- List<String> c = (List<String>) map.get("c");
-
- assertEquals("A.1.1",a.get(0));
- assertEquals("B.1.1",b.get(0));
- assertNull(c.get(0));
-
- assertNull(a.get(1));
- assertEquals("B.1.2",b.get(1));
- assertEquals("C.1.2",c.get(1));
-
- map = l.get(1);
- a = (List<String>) map.get("a");
- b = (List<String>) map.get("b");
- c = (List<String>) map.get("c");
- assertEquals("A.2.1",a.get(0));
- assertNull(b.get(0));
- assertEquals("C.2.1",c.get(0));
-
- assertNull(a.get(1));
- assertEquals("B.2.2",b.get(1));
- assertEquals("C.2.2",c.get(1));
- }
-
-
- @Test
- public void testError(){
- String malformedXml = "<root>\n" +
- " <node>\n" +
- " <id>1</id>\n" +
- " <desc>test1</desc>\n" +
- " </node>\n" +
- " <node>\n" +
- " <id>2</id>\n" +
- " <desc>test2</desc>\n" +
- " </node>\n" +
- " <node>\n" +
- " <id/>3</id>\n" + // invalid XML
- " <desc>test3</desc>\n" +
- " </node>\n" +
- "</root>";
- XPathRecordReader rr = new XPathRecordReader("/root/node");
- rr.addField("id", "/root/node/id", true);
- rr.addField("desc", "/root/node/desc", true);
- try {
- rr.getAllRecords(new StringReader(malformedXml));
- fail("A RuntimeException was expected: the input XML is invalid.");
- } catch (Exception e) { }
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestZKPropertiesWriter.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestZKPropertiesWriter.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestZKPropertiesWriter.java
deleted file mode 100644
index c8727d0..0000000
--- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestZKPropertiesWriter.java
+++ /dev/null
@@ -1,164 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import java.lang.invoke.MethodHandles;
-
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-
-import org.apache.solr.cloud.AbstractZkTestCase;
-import org.apache.solr.cloud.ZkTestServer;
-import org.apache.solr.common.params.ModifiableSolrParams;
-import org.apache.solr.common.util.SuppressForbidden;
-import org.apache.solr.core.CoreContainer;
-import org.apache.solr.request.LocalSolrQueryRequest;
-import org.apache.solr.request.SolrQueryRequest;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TestZKPropertiesWriter extends AbstractDataImportHandlerTestCase {
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
- protected static ZkTestServer zkServer;
-
- protected static String zkDir;
-
- private static CoreContainer cc;
-
- private String dateFormat = "yyyy-MM-dd HH:mm:ss.SSSSSS";
-
- @BeforeClass
- public static void dihZk_beforeClass() throws Exception {
- zkDir = createTempDir("zkData").toFile().getAbsolutePath();
- zkServer = new ZkTestServer(zkDir);
- zkServer.run();
-
- System.setProperty("solrcloud.skip.autorecovery", "true");
- System.setProperty("zkHost", zkServer.getZkAddress());
- System.setProperty("jetty.port", "0000");
-
- AbstractZkTestCase.buildZooKeeper(zkServer.getZkHost(), zkServer.getZkAddress(), getFile("dih/solr"),
- "dataimport-solrconfig.xml", "dataimport-schema.xml");
-
- //initCore("solrconfig.xml", "schema.xml", getFile("dih/solr").getAbsolutePath());
- cc = createDefaultCoreContainer(getFile("dih/solr").toPath());
- }
-
- @Before
- public void beforeDihZKTest() throws Exception {
-
- }
-
- @After
- public void afterDihZkTest() throws Exception {
- MockDataSource.clearCache();
- }
-
-
- @AfterClass
- public static void dihZk_afterClass() throws Exception {
- cc.shutdown();
-
- zkServer.shutdown();
-
- zkServer = null;
- zkDir = null;
- cc = null;
- }
-
- @SuppressForbidden(reason = "Needs currentTimeMillis to construct date stamps")
- @Test
- public void testZKPropertiesWriter() throws Exception {
- // test using ZooKeeper
- assertTrue("Not using ZooKeeper", h.getCoreContainer().isZooKeeperAware());
-
- // for the really slow/busy computer, we wait to make sure we have a leader before starting
- h.getCoreContainer().getZkController().getZkStateReader().getLeaderUrl("collection1", "shard1", 30000);
-
- assertQ("test query on empty index", request("qlkciyopsbgzyvkylsjhchghjrdf"),
- "//result[@numFound='0']");
-
- SimpleDateFormat errMsgFormat = new SimpleDateFormat(dateFormat, Locale.ROOT);
-
- delQ("*:*");
- commit();
- SimpleDateFormat df = new SimpleDateFormat(dateFormat, Locale.ROOT);
- Date oneSecondAgo = new Date(System.currentTimeMillis() - 1000);
-
- Map<String, String> init = new HashMap<>();
- init.put("dateFormat", dateFormat);
- ZKPropertiesWriter spw = new ZKPropertiesWriter();
- spw.init(new DataImporter(h.getCore(), "dataimport"), init);
- Map<String, Object> props = new HashMap<>();
- props.put("SomeDates.last_index_time", oneSecondAgo);
- props.put("last_index_time", oneSecondAgo);
- spw.persist(props);
-
- List rows = new ArrayList();
- rows.add(createMap("id", "1", "year_s", "2013"));
- MockDataSource.setIterator("select " + df.format(oneSecondAgo) + " from dummy", rows.iterator());
-
- h.query("/dataimport", lrf.makeRequest("command", "full-import", "dataConfig",
- generateConfig(), "clean", "true", "commit", "true", "synchronous",
- "true", "indent", "true"));
- props = spw.readIndexerProperties();
- Date entityDate = df.parse((String) props.get("SomeDates.last_index_time"));
- Date docDate = df.parse((String) props.get("last_index_time"));
-
- Assert.assertTrue("This date: " + errMsgFormat.format(oneSecondAgo) + " should be prior to the document date: " + errMsgFormat.format(docDate), docDate.getTime() - oneSecondAgo.getTime() > 0);
- Assert.assertTrue("This date: " + errMsgFormat.format(oneSecondAgo) + " should be prior to the entity date: " + errMsgFormat.format(entityDate), entityDate.getTime() - oneSecondAgo.getTime() > 0);
- assertQ(request("*:*"), "//*[@numFound='1']", "//doc/str[@name=\"year_s\"]=\"2013\"");
-
- }
-
- public SolrQueryRequest request(String... q) {
- LocalSolrQueryRequest req = lrf.makeRequest(q);
- ModifiableSolrParams params = new ModifiableSolrParams();
- params.add(req.getParams());
- params.set("distrib", true);
- req.setParams(params);
- return req;
- }
-
- protected String generateConfig() {
- StringBuilder sb = new StringBuilder();
- sb.append("<dataConfig> \n");
- sb.append("<propertyWriter dateFormat=\"" + dateFormat + "\" type=\"ZKPropertiesWriter\" />\n");
- sb.append("<dataSource name=\"mock\" type=\"MockDataSource\"/>\n");
- sb.append("<document name=\"TestSimplePropertiesWriter\"> \n");
- sb.append("<entity name=\"SomeDates\" processor=\"SqlEntityProcessor\" dataSource=\"mock\" ");
- sb.append("query=\"select ${dih.last_index_time} from dummy\" >\n");
- sb.append("<field column=\"AYEAR_S\" name=\"year_s\" /> \n");
- sb.append("</entity>\n");
- sb.append("</document> \n");
- sb.append("</dataConfig> \n");
- String config = sb.toString();
- log.debug(config);
- return config;
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TripleThreatTransformer.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TripleThreatTransformer.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TripleThreatTransformer.java
deleted file mode 100644
index 2d0aadb..0000000
--- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TripleThreatTransformer.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * This transformer does 3 things
- * <ul>
- * <li>It turns every row into 3 rows,
- * modifying any "id" column to ensure duplicate entries in the index
- * <li>The 2nd Row has 2x values for every column,
- * with the added one being backwards of the original
- * <li>The 3rd Row has an added static value
- * </ul>
- *
- * Also, this does not extend Transformer.
- */
-public class TripleThreatTransformer {
- public Object transformRow(Map<String, Object> row) {
- List<Map<String, Object>> rows = new ArrayList<>(3);
- rows.add(row);
- rows.add(addDuplicateBackwardsValues(row));
- rows.add(new LinkedHashMap<>(row));
- rows.get(2).put("AddAColumn_s", "Added");
- modifyIdColumn(rows.get(1), 1);
- modifyIdColumn(rows.get(2), 2);
- return rows;
- }
- private LinkedHashMap<String,Object> addDuplicateBackwardsValues(Map<String, Object> row) {
- LinkedHashMap<String,Object> n = new LinkedHashMap<>();
- for(Map.Entry<String,Object> entry : row.entrySet()) {
- String key = entry.getKey();
- if(!"id".equalsIgnoreCase(key)) {
- String[] vals = new String[2];
- vals[0] = entry.getValue()==null ? "null" : entry.getValue().toString();
- vals[1] = new StringBuilder(vals[0]).reverse().toString();
- n.put(key, Arrays.asList(vals));
- } else {
- n.put(key, entry.getValue());
- }
- }
- return n;
- }
-
- private void modifyIdColumn(Map<String, Object> row, int num) {
- Object o = row.remove("ID");
- if(o==null) {
- o = row.remove("id");
- }
- if(o!=null) {
- String id = o.toString();
- id = "TripleThreat-" + num + "-" + id;
- row.put("id", id);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/contentstream-solrconfig.xml
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/contentstream-solrconfig.xml b/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/contentstream-solrconfig.xml
new file mode 100644
index 0000000..a07ab78
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/contentstream-solrconfig.xml
@@ -0,0 +1,293 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<config>
+ <luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>
+ <indexConfig>
+ <useCompoundFile>${useCompoundFile:false}</useCompoundFile>
+ </indexConfig>
+
+ <!-- Used to specify an alternate directory to hold all index data
+ other than the default ./data under the Solr home.
+ If replication is in use, this should match the replication configuration. -->
+ <dataDir>${solr.data.dir:}</dataDir>
+
+ <directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.RAMDirectoryFactory}"/>
+ <schemaFactory class="ClassicIndexSchemaFactory"/>
+
+ <!-- the default high-performance update handler -->
+ <updateHandler class="solr.DirectUpdateHandler2">
+
+ <!-- A prefix of "solr." for class names is an alias that
+ causes solr to search appropriate packages, including
+ org.apache.solr.(search|update|request|core|analysis)
+ -->
+
+ <!-- Limit the number of deletions Solr will buffer during doc updating.
+
+ Setting this lower can help bound memory use during indexing.
+ -->
+ <maxPendingDeletes>100000</maxPendingDeletes>
+
+ </updateHandler>
+
+
+ <query>
+ <!-- Maximum number of clauses in a boolean query... can affect
+ range or prefix queries that expand to big boolean
+ queries. An exception is thrown if exceeded. -->
+ <maxBooleanClauses>1024</maxBooleanClauses>
+
+
+ <!-- Cache used by SolrIndexSearcher for filters (DocSets),
+ unordered sets of *all* documents that match a query.
+ When a new searcher is opened, its caches may be prepopulated
+ or "autowarmed" using data from caches in the old searcher.
+ autowarmCount is the number of items to prepopulate. For LRUCache,
+ the autowarmed items will be the most recently accessed items.
+ Parameters:
+ class - the SolrCache implementation (currently only LRUCache)
+ size - the maximum number of entries in the cache
+ initialSize - the initial capacity (number of entries) of
+ the cache. (seel java.util.HashMap)
+ autowarmCount - the number of entries to prepopulate from
+ and old cache.
+ -->
+ <filterCache
+ class="solr.LRUCache"
+ size="512"
+ initialSize="512"
+ autowarmCount="256"/>
+
+ <!-- queryResultCache caches results of searches - ordered lists of
+ document ids (DocList) based on a query, a sort, and the range
+ of documents requested. -->
+ <queryResultCache
+ class="solr.LRUCache"
+ size="512"
+ initialSize="512"
+ autowarmCount="256"/>
+
+ <!-- documentCache caches Lucene Document objects (the stored fields for each document).
+ Since Lucene internal document ids are transient, this cache will not be autowarmed. -->
+ <documentCache
+ class="solr.LRUCache"
+ size="512"
+ initialSize="512"
+ autowarmCount="0"/>
+
+ <!-- If true, stored fields that are not requested will be loaded lazily.
+
+ This can result in a significant speed improvement if the usual case is to
+ not load all stored fields, especially if the skipped fields are large compressed
+ text fields.
+ -->
+ <enableLazyFieldLoading>true</enableLazyFieldLoading>
+
+ <!-- Example of a generic cache. These caches may be accessed by name
+ through SolrIndexSearcher.getCache(),cacheLookup(), and cacheInsert().
+ The purpose is to enable easy caching of user/application level data.
+ The regenerator argument should be specified as an implementation
+ of solr.search.CacheRegenerator if autowarming is desired. -->
+ <!--
+ <cache name="myUserCache"
+ class="solr.LRUCache"
+ size="4096"
+ initialSize="1024"
+ autowarmCount="1024"
+ regenerator="org.mycompany.mypackage.MyRegenerator"
+ />
+ -->
+
+ <!-- An optimization that attempts to use a filter to satisfy a search.
+ If the requested sort does not include score, then the filterCache
+ will be checked for a filter matching the query. If found, the filter
+ will be used as the source of document ids, and then the sort will be
+ applied to that.
+ <useFilterForSortedQuery>true</useFilterForSortedQuery>
+ -->
+
+ <!-- An optimization for use with the queryResultCache. When a search
+ is requested, a superset of the requested number of document ids
+ are collected. For example, if a search for a particular query
+ requests matching documents 10 through 19, and queryWindowSize is 50,
+ then documents 0 through 49 will be collected and cached. Any further
+ requests in that range can be satisfied via the cache. -->
+ <queryResultWindowSize>50</queryResultWindowSize>
+
+ <!-- Maximum number of documents to cache for any entry in the
+ queryResultCache. -->
+ <queryResultMaxDocsCached>200</queryResultMaxDocsCached>
+
+ <!-- This entry enables an int hash representation for filters (DocSets)
+ when the number of items in the set is less than maxSize. For smaller
+ sets, this representation is more memory efficient, more efficient to
+ iterate over, and faster to take intersections. -->
+ <HashDocSet maxSize="3000" loadFactor="0.75"/>
+
+ <!-- a newSearcher event is fired whenever a new searcher is being prepared
+ and there is a current searcher handling requests (aka registered). -->
+ <!-- QuerySenderListener takes an array of NamedList and executes a
+ local query request for each NamedList in sequence. -->
+ <!--<listener event="newSearcher" class="solr.QuerySenderListener">-->
+ <!--<arr name="queries">-->
+ <!--<lst> <str name="q">solr</str> <str name="start">0</str> <str name="rows">10</str> </lst>-->
+ <!--<lst> <str name="q">rocks</str> <str name="start">0</str> <str name="rows">10</str> </lst>-->
+ <!--<lst><str name="q">static newSearcher warming query from solrconfig.xml</str></lst>-->
+ <!--</arr>-->
+ <!--</listener>-->
+
+ <!-- a firstSearcher event is fired whenever a new searcher is being
+ prepared but there is no current registered searcher to handle
+ requests or to gain autowarming data from. -->
+ <!--<listener event="firstSearcher" class="solr.QuerySenderListener">-->
+ <!--<arr name="queries">-->
+ <!--</arr>-->
+ <!--</listener>-->
+
+ <!-- If a search request comes in and there is no current registered searcher,
+ then immediately register the still warming searcher and use it. If
+ "false" then all requests will block until the first searcher is done
+ warming. -->
+ <useColdSearcher>false</useColdSearcher>
+
+ <!-- Maximum number of searchers that may be warming in the background
+ concurrently. An error is returned if this limit is exceeded. Recommend
+ 1-2 for read-only slaves, higher for masters w/o cache warming. -->
+ <maxWarmingSearchers>4</maxWarmingSearchers>
+
+ </query>
+
+ <requestDispatcher>
+ <!--Make sure your system has some authentication before enabling remote streaming!
+ <requestParsers enableRemoteStreaming="false" multipartUploadLimitInKB="-1" />
+ -->
+
+ <!-- Set HTTP caching related parameters (for proxy caches and clients).
+
+ To get the behaviour of Solr 1.2 (ie: no caching related headers)
+ use the never304="true" option and do not specify a value for
+ <cacheControl>
+ -->
+ <httpCaching never304="true">
+ <!--httpCaching lastModifiedFrom="openTime"
+ etagSeed="Solr"-->
+ <!-- lastModFrom="openTime" is the default, the Last-Modified value
+ (and validation against If-Modified-Since requests) will all be
+ relative to when the current Searcher was opened.
+ You can change it to lastModFrom="dirLastMod" if you want the
+ value to exactly corrispond to when the physical index was last
+ modified.
+
+ etagSeed="..." is an option you can change to force the ETag
+ header (and validation against If-None-Match requests) to be
+ differnet even if the index has not changed (ie: when making
+ significant changes to your config file)
+
+ lastModifiedFrom and etagSeed are both ignored if you use the
+ never304="true" option.
+ -->
+ <!-- If you include a <cacheControl> directive, it will be used to
+ generate a Cache-Control header, as well as an Expires header
+ if the value contains "max-age="
+
+ By default, no Cache-Control header is generated.
+
+ You can use the <cacheControl> option even if you have set
+ never304="true"
+ -->
+ <!-- <cacheControl>max-age=30, public</cacheControl> -->
+ </httpCaching>
+ </requestDispatcher>
+
+ <requestHandler name="/select" class="solr.SearchHandler">
+ <!-- default values for query parameters -->
+ <lst name="defaults">
+ <str name="echoParams">explicit</str>
+ <!--
+ <int name="rows">10</int>
+ <str name="fl">*</str>
+ <str name="version">2.1</str>
+ -->
+ </lst>
+ </requestHandler>
+
+ <requestHandler name="/dataimport" class="org.apache.solr.handler.dataimport.DataImportHandler">
+ <lst name="defaults">
+ <str name="config">data-config.xml</str>
+
+ </lst>
+ </requestHandler>
+
+ <!--
+
+ Search components are registered to SolrCore and used by Search Handlers
+
+ By default, the following components are avaliable:
+
+ <searchComponent name="query" class="org.apache.solr.handler.component.QueryComponent" />
+ <searchComponent name="facet" class="org.apache.solr.handler.component.FacetComponent" />
+ <searchComponent name="mlt" class="org.apache.solr.handler.component.MoreLikeThisComponent" />
+ <searchComponent name="highlight" class="org.apache.solr.handler.component.HighlightComponent" />
+ <searchComponent name="debug" class="org.apache.solr.handler.component.DebugComponent" />
+
+ If you register a searchComponent to one of the standard names, that will be used instead.
+
+ -->
+
+ <requestHandler name="/search" class="org.apache.solr.handler.component.SearchHandler">
+ <lst name="defaults">
+ <str name="echoParams">explicit</str>
+ </lst>
+ <!--
+ By default, this will register the following components:
+
+ <arr name="components">
+ <str>query</str>
+ <str>facet</str>
+ <str>mlt</str>
+ <str>highlight</str>
+ <str>debug</str>
+ </arr>
+
+ To insert handlers before or after the 'standard' components, use:
+
+ <arr name="first-components">
+ <str>first</str>
+ </arr>
+
+ <arr name="last-components">
+ <str>last</str>
+ </arr>
+
+ -->
+ </requestHandler>
+
+ <!-- config for the admin interface -->
+ <admin>
+ <defaultQuery>*:*</defaultQuery>
+ </admin>
+
+ <updateRequestProcessorChain key="contentstream" default="true">
+ <processor class="org.apache.solr.handler.dataimport.AbstractDataImportHandlerTestCase$TestUpdateRequestProcessorFactory"/>
+ <processor class="solr.RunUpdateProcessorFactory"/>
+ <processor class="solr.LogUpdateProcessorFactory"/>
+ </updateRequestProcessorChain>
+
+</config>
+
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/data-config-end-to-end.xml
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/data-config-end-to-end.xml b/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/data-config-end-to-end.xml
new file mode 100644
index 0000000..a582112
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/data-config-end-to-end.xml
@@ -0,0 +1,41 @@
+<dataConfig>
+ <dataSource name="hsqldb" driver="org.hsqldb.jdbcDriver" url="jdbc:hsqldb:mem:." />
+ <document name="dih_end_to_end">
+ <entity
+ name="People"
+ processor="SqlEntityProcessor"
+ dataSource="hsqldb"
+ query="SELECT ID, NAME, COUNTRY_CODES FROM PEOPLE"
+ transformer="RegexTransformer"
+ >
+ <field column="ID" name="id" />
+ <field column="COUNTRY_CODE" sourceColName="COUNTRY_CODES" splitBy="," />
+
+<!--
+ Instead of using 'cachePk'/'cacheLookup' as done below, we could have done:
+ where="CODE=People.COUNTRY_CODE"
+-->
+ <entity
+ name="Countries"
+ processor="SqlEntityProcessor"
+ dataSource="hsqldb"
+ cacheImpl="SortedMapBackedCache"
+ cacheKey="CODE"
+ cacheLookup="People.COUNTRY_CODE"
+
+ query="SELECT CODE, COUNTRY_NAME FROM COUNTRIES"
+ >
+ <field column="CODE" name="DO_NOT_INDEX" />
+ </entity>
+
+ <entity
+ name="Sports"
+ processor="SqlEntityProcessor"
+ dataSource="hsqldb"
+ query="SELECT PERSON_ID, SPORT_NAME FROM PEOPLE_SPORTS WHERE PERSON_ID=${People.ID}"
+ />
+
+ </entity>
+ </document>
+</dataConfig>
+
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/data-config-with-datasource.xml
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/data-config-with-datasource.xml b/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/data-config-with-datasource.xml
new file mode 100644
index 0000000..46a6603
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/data-config-with-datasource.xml
@@ -0,0 +1,9 @@
+<dataConfig>
+ <dataSource type="MockDataSource" />
+ <document>
+ <entity name="x" query="select * from x">
+ <field column="id" />
+ <field column="desc" />
+ </entity>
+ </document>
+</dataConfig>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/data-config-with-transformer.xml
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/data-config-with-transformer.xml b/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/data-config-with-transformer.xml
new file mode 100644
index 0000000..925e6c2
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/data-config-with-transformer.xml
@@ -0,0 +1,10 @@
+<dataConfig>
+ <dataSource type="MockDataSource" />
+ <dataSource name="mockDs" type="TestDocBuilder2$MockDataSource2" />
+ <document>
+ <entity name="x" query="select * from x" transformer="TestDocBuilder2$MockTransformer">
+ <field column="id" />
+ <field column="desc" />
+ </entity>
+ </document>
+</dataConfig>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/dataconfig-contentstream.xml
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/dataconfig-contentstream.xml b/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/dataconfig-contentstream.xml
new file mode 100644
index 0000000..7520e74
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/dataconfig-contentstream.xml
@@ -0,0 +1,10 @@
+<dataConfig>
+ <dataSource type="ContentStreamDataSource" name="c"/>
+ <document>
+ <entity name="b" dataSource="c" processor="XPathEntityProcessor"
+ forEach="/root/b">
+ <field column="desc" xpath="/root/b/c"/>
+ <field column="id" xpath="/root/b/id"/>
+ </entity>
+ </document>
+</dataConfig>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/dataimport-nodatasource-solrconfig.xml
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/dataimport-nodatasource-solrconfig.xml b/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/dataimport-nodatasource-solrconfig.xml
new file mode 100644
index 0000000..6754f9e
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/resources/dih/solr/collection1/conf/dataimport-nodatasource-solrconfig.xml
@@ -0,0 +1,285 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<config>
+ <luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>
+
+ <!-- Used to specify an alternate directory to hold all index data
+ other than the default ./data under the Solr home.
+ If replication is in use, this should match the replication configuration. -->
+ <dataDir>${solr.data.dir:}</dataDir>
+
+ <directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.RAMDirectoryFactory}"/>
+ <schemaFactory class="ClassicIndexSchemaFactory"/>
+
+ <indexConfig>
+ <lockType>single</lockType>
+ <useCompoundFile>${useCompoundFile:false}</useCompoundFile>
+ </indexConfig>
+
+ <!-- the default high-performance update handler -->
+ <updateHandler class="solr.DirectUpdateHandler2">
+
+ <!-- A prefix of "solr." for class names is an alias that
+ causes solr to search appropriate packages, including
+ org.apache.solr.(search|update|request|core|analysis)
+ -->
+
+ <!-- Limit the number of deletions Solr will buffer during doc updating.
+
+ Setting this lower can help bound memory use during indexing.
+ -->
+ <maxPendingDeletes>100000</maxPendingDeletes>
+
+ </updateHandler>
+
+
+ <query>
+ <!-- Maximum number of clauses in a boolean query... can affect
+ range or prefix queries that expand to big boolean
+ queries. An exception is thrown if exceeded. -->
+ <maxBooleanClauses>1024</maxBooleanClauses>
+
+
+ <!-- Cache used by SolrIndexSearcher for filters (DocSets),
+ unordered sets of *all* documents that match a query.
+ When a new searcher is opened, its caches may be prepopulated
+ or "autowarmed" using data from caches in the old searcher.
+ autowarmCount is the number of items to prepopulate. For LRUCache,
+ the autowarmed items will be the most recently accessed items.
+ Parameters:
+ class - the SolrCache implementation (currently only LRUCache)
+ size - the maximum number of entries in the cache
+ initialSize - the initial capacity (number of entries) of
+ the cache. (seel java.util.HashMap)
+ autowarmCount - the number of entries to prepopulate from
+ and old cache.
+ -->
+ <filterCache
+ class="solr.LRUCache"
+ size="512"
+ initialSize="512"
+ autowarmCount="256"/>
+
+ <!-- queryResultCache caches results of searches - ordered lists of
+ document ids (DocList) based on a query, a sort, and the range
+ of documents requested. -->
+ <queryResultCache
+ class="solr.LRUCache"
+ size="512"
+ initialSize="512"
+ autowarmCount="256"/>
+
+ <!-- documentCache caches Lucene Document objects (the stored fields for each document).
+ Since Lucene internal document ids are transient, this cache will not be autowarmed. -->
+ <documentCache
+ class="solr.LRUCache"
+ size="512"
+ initialSize="512"
+ autowarmCount="0"/>
+
+ <!-- If true, stored fields that are not requested will be loaded lazily.
+
+ This can result in a significant speed improvement if the usual case is to
+ not load all stored fields, especially if the skipped fields are large compressed
+ text fields.
+ -->
+ <enableLazyFieldLoading>true</enableLazyFieldLoading>
+
+ <!-- Example of a generic cache. These caches may be accessed by name
+ through SolrIndexSearcher.getCache(),cacheLookup(), and cacheInsert().
+ The purpose is to enable easy caching of user/application level data.
+ The regenerator argument should be specified as an implementation
+ of solr.search.CacheRegenerator if autowarming is desired. -->
+ <!--
+ <cache name="myUserCache"
+ class="solr.LRUCache"
+ size="4096"
+ initialSize="1024"
+ autowarmCount="1024"
+ regenerator="org.mycompany.mypackage.MyRegenerator"
+ />
+ -->
+
+ <!-- An optimization that attempts to use a filter to satisfy a search.
+ If the requested sort does not include score, then the filterCache
+ will be checked for a filter matching the query. If found, the filter
+ will be used as the source of document ids, and then the sort will be
+ applied to that.
+ <useFilterForSortedQuery>true</useFilterForSortedQuery>
+ -->
+
+ <!-- An optimization for use with the queryResultCache. When a search
+ is requested, a superset of the requested number of document ids
+ are collected. For example, if a search for a particular query
+ requests matching documents 10 through 19, and queryWindowSize is 50,
+ then documents 0 through 49 will be collected and cached. Any further
+ requests in that range can be satisfied via the cache. -->
+ <queryResultWindowSize>50</queryResultWindowSize>
+
+ <!-- Maximum number of documents to cache for any entry in the
+ queryResultCache. -->
+ <queryResultMaxDocsCached>200</queryResultMaxDocsCached>
+
+ <!-- This entry enables an int hash representation for filters (DocSets)
+ when the number of items in the set is less than maxSize. For smaller
+ sets, this representation is more memory efficient, more efficient to
+ iterate over, and faster to take intersections. -->
+ <HashDocSet maxSize="3000" loadFactor="0.75"/>
+
+ <!-- a newSearcher event is fired whenever a new searcher is being prepared
+ and there is a current searcher handling requests (aka registered). -->
+ <!-- QuerySenderListener takes an array of NamedList and executes a
+ local query request for each NamedList in sequence. -->
+ <!--<listener event="newSearcher" class="solr.QuerySenderListener">-->
+ <!--<arr name="queries">-->
+ <!--<lst> <str name="q">solr</str> <str name="start">0</str> <str name="rows">10</str> </lst>-->
+ <!--<lst> <str name="q">rocks</str> <str name="start">0</str> <str name="rows">10</str> </lst>-->
+ <!--<lst><str name="q">static newSearcher warming query from solrconfig.xml</str></lst>-->
+ <!--</arr>-->
+ <!--</listener>-->
+
+ <!-- a firstSearcher event is fired whenever a new searcher is being
+ prepared but there is no current registered searcher to handle
+ requests or to gain autowarming data from. -->
+ <!--<listener event="firstSearcher" class="solr.QuerySenderListener">-->
+ <!--<arr name="queries">-->
+ <!--</arr>-->
+ <!--</listener>-->
+
+ <!-- If a search request comes in and there is no current registered searcher,
+ then immediately register the still warming searcher and use it. If
+ "false" then all requests will block until the first searcher is done
+ warming. -->
+ <useColdSearcher>false</useColdSearcher>
+
+ <!-- Maximum number of searchers that may be warming in the background
+ concurrently. An error is returned if this limit is exceeded. Recommend
+ 1-2 for read-only slaves, higher for masters w/o cache warming. -->
+ <maxWarmingSearchers>4</maxWarmingSearchers>
+
+ </query>
+
+ <requestDispatcher>
+ <!--Make sure your system has some authentication before enabling remote streaming!
+ <requestParsers enableRemoteStreaming="false" multipartUploadLimitInKB="-1" />
+ -->
+
+ <!-- Set HTTP caching related parameters (for proxy caches and clients).
+
+ To get the behaviour of Solr 1.2 (ie: no caching related headers)
+ use the never304="true" option and do not specify a value for
+ <cacheControl>
+ -->
+ <httpCaching never304="true">
+ <!--httpCaching lastModifiedFrom="openTime"
+ etagSeed="Solr"-->
+ <!-- lastModFrom="openTime" is the default, the Last-Modified value
+ (and validation against If-Modified-Since requests) will all be
+ relative to when the current Searcher was opened.
+ You can change it to lastModFrom="dirLastMod" if you want the
+ value to exactly corrispond to when the physical index was last
+ modified.
+
+ etagSeed="..." is an option you can change to force the ETag
+ header (and validation against If-None-Match requests) to be
+ differnet even if the index has not changed (ie: when making
+ significant changes to your config file)
+
+ lastModifiedFrom and etagSeed are both ignored if you use the
+ never304="true" option.
+ -->
+ <!-- If you include a <cacheControl> directive, it will be used to
+ generate a Cache-Control header, as well as an Expires header
+ if the value contains "max-age="
+
+ By default, no Cache-Control header is generated.
+
+ You can use the <cacheControl> option even if you have set
+ never304="true"
+ -->
+ <!-- <cacheControl>max-age=30, public</cacheControl> -->
+ </httpCaching>
+ </requestDispatcher>
+
+ <requestHandler name="/select" class="solr.SearchHandler">
+ <!-- default values for query parameters -->
+ <lst name="defaults">
+ <str name="echoParams">explicit</str>
+ <!--
+ <int name="rows">10</int>
+ <str name="fl">*</str>
+ <str name="version">2.1</str>
+ -->
+ </lst>
+ </requestHandler>
+
+ <requestHandler name="/dataimport" class="org.apache.solr.handler.dataimport.DataImportHandler">
+ </requestHandler>
+
+ <!--
+
+ Search components are registered to SolrCore and used by Search Handlers
+
+ By default, the following components are avaliable:
+
+ <searchComponent name="query" class="org.apache.solr.handler.component.QueryComponent" />
+ <searchComponent name="facet" class="org.apache.solr.handler.component.FacetComponent" />
+ <searchComponent name="mlt" class="org.apache.solr.handler.component.MoreLikeThisComponent" />
+ <searchComponent name="highlight" class="org.apache.solr.handler.component.HighlightComponent" />
+ <searchComponent name="debug" class="org.apache.solr.handler.component.DebugComponent" />
+
+ If you register a searchComponent to one of the standard names, that will be used instead.
+
+ -->
+
+ <requestHandler name="/search" class="org.apache.solr.handler.component.SearchHandler">
+ <lst name="defaults">
+ <str name="echoParams">explicit</str>
+ </lst>
+ <!--
+ By default, this will register the following components:
+
+ <arr name="components">
+ <str>query</str>
+ <str>facet</str>
+ <str>mlt</str>
+ <str>highlight</str>
+ <str>debug</str>
+ </arr>
+
+ To insert handlers before or after the 'standard' components, use:
+
+ <arr name="first-components">
+ <str>first</str>
+ </arr>
+
+ <arr name="last-components">
+ <str>last</str>
+ </arr>
+
+ -->
+ </requestHandler>
+
+ <!-- config for the admin interface -->
+ <admin>
+ <defaultQuery>*:*</defaultQuery>
+ </admin>
+
+</config>
+