You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by da...@apache.org on 2018/11/02 11:33:26 UTC
[05/25] lucene-solr:jira/gradle: Adding dataimporthandler-extras
module
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDataConfig.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDataConfig.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDataConfig.java
deleted file mode 100644
index 7a56fd2..0000000
--- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDataConfig.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import org.apache.solr.handler.dataimport.config.DIHConfiguration;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.w3c.dom.Document;
-import org.xml.sax.InputSource;
-
-import javax.xml.parsers.DocumentBuilderFactory;
-import java.io.StringReader;
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * <p>
- * Test for DataConfig
- * </p>
- *
- *
- * @since solr 1.3
- */
-public class TestDataConfig extends AbstractDataImportHandlerTestCase {
- @BeforeClass
- public static void beforeClass() throws Exception {
- initCore("dataimport-nodatasource-solrconfig.xml", "dataimport-schema.xml");
- }
-
- @Test
- @SuppressWarnings("unchecked")
- public void testDataConfigWithDataSource() throws Exception {
- List rows = new ArrayList();
- rows.add(createMap("id", "1", "desc", "one"));
- MockDataSource.setIterator("select * from x", rows.iterator());
-
- runFullImport(loadDataConfig("data-config-with-datasource.xml"));
-
- assertQ(req("id:1"), "//*[@numFound='1']");
- }
-
- @Test
- public void testBasic() throws Exception {
- javax.xml.parsers.DocumentBuilder builder = DocumentBuilderFactory
- .newInstance().newDocumentBuilder();
- Document doc = builder.parse(new InputSource(new StringReader(xml)));
- DataImporter di = new DataImporter();
- DIHConfiguration dc = di.readFromXml(doc);
- assertEquals("atrimlisting", dc.getEntities().get(0).getName());
- }
-
- private static final String xml = "<dataConfig>\n"
- + "\t<document name=\"autos\" >\n"
- + "\t\t<entity name=\"atrimlisting\" pk=\"acode\"\n"
- + "\t\t\tquery=\"select acode,make,model,year,msrp,category,image,izmo_image_url,price_range_low,price_range_high,invoice_range_low,invoice_range_high from atrimlisting\"\n"
- + "\t\t\tdeltaQuery=\"select acode from atrimlisting where last_modified > '${indexer.last_index_time}'\">\n"
- +
-
- "\t\t</entity>\n" +
-
- "\t</document>\n" + "</dataConfig>";
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDateFormatTransformer.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDateFormatTransformer.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDateFormatTransformer.java
deleted file mode 100644
index a1e85d7..0000000
--- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDateFormatTransformer.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import org.junit.Test;
-
-import java.text.SimpleDateFormat;
-import java.util.*;
-
-/**
- * <p>
- * Test for DateFormatTransformer
- * </p>
- *
- *
- * @since solr 1.3
- */
-public class TestDateFormatTransformer extends AbstractDataImportHandlerTestCase {
-
- @Test
- @SuppressWarnings("unchecked")
- public void testTransformRow_SingleRow() throws Exception {
- List<Map<String, String>> fields = new ArrayList<>();
- fields.add(createMap(DataImporter.COLUMN, "lastModified"));
- fields.add(createMap(DataImporter.COLUMN,
- "dateAdded", RegexTransformer.SRC_COL_NAME, "lastModified",
- DateFormatTransformer.DATE_TIME_FMT, "${xyz.myDateFormat}"));
-
- SimpleDateFormat format = new SimpleDateFormat("MM/dd/yyyy", Locale.ROOT);
- Date now = format.parse(format.format(new Date()));
-
- Map<String,Object> row = createMap("lastModified", format.format(now));
-
- VariableResolver resolver = new VariableResolver();
- resolver.addNamespace("e", row);
- resolver.addNamespace("xyz", createMap("myDateFormat", "MM/dd/yyyy"));
-
- Context context = getContext(null, resolver,
- null, Context.FULL_DUMP, fields, null);
- new DateFormatTransformer().transformRow(row, context);
- assertEquals(now, row.get("dateAdded"));
- }
-
- @Test
- @SuppressWarnings("unchecked")
- public void testTransformRow_MultipleRows() throws Exception {
- List<Map<String, String>> fields = new ArrayList<>();
- fields.add(createMap(DataImporter.COLUMN, "lastModified"));
- fields.add(createMap(DataImporter.COLUMN,
- "dateAdded", RegexTransformer.SRC_COL_NAME, "lastModified",
- DateFormatTransformer.DATE_TIME_FMT, "MM/dd/yyyy hh:mm:ss.SSS"));
-
- SimpleDateFormat format = new SimpleDateFormat("MM/dd/yyyy hh:mm:ss.SSS", Locale.ROOT);
- Date now1 = format.parse(format.format(new Date()));
- Date now2 = format.parse(format.format(new Date()));
-
- Map<String,Object> row = new HashMap<>();
- List<String> list = new ArrayList<>();
- list.add(format.format(now1));
- list.add(format.format(now2));
- row.put("lastModified", list);
-
- VariableResolver resolver = new VariableResolver();
- resolver.addNamespace("e", row);
-
- Context context = getContext(null, resolver,
- null, Context.FULL_DUMP, fields, null);
- new DateFormatTransformer().transformRow(row, context);
- List<Object> output = new ArrayList<>();
- output.add(now1);
- output.add(now2);
- assertEquals(output, row.get("dateAdded"));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDocBuilder.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDocBuilder.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDocBuilder.java
deleted file mode 100644
index 7f7278e..0000000
--- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDocBuilder.java
+++ /dev/null
@@ -1,335 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import org.apache.solr.common.SolrInputDocument;
-import org.apache.solr.handler.dataimport.config.DIHConfiguration;
-import org.apache.solr.handler.dataimport.config.Entity;
-
-import org.junit.After;
-import org.junit.Test;
-
-import java.util.*;
-
-/**
- * <p>
- * Test for DocBuilder
- * </p>
- *
- *
- * @since solr 1.3
- */
-// See: https://issues.apache.org/jira/browse/SOLR-12028 Tests cannot remove files on Windows machines occasionally
-public class TestDocBuilder extends AbstractDataImportHandlerTestCase {
-
- @Override
- @After
- public void tearDown() throws Exception {
- MockDataSource.clearCache();
- MockStringDataSource.clearCache();
- super.tearDown();
- }
-
- @Test
- public void loadClass() throws Exception {
- @SuppressWarnings("unchecked")
- Class<Transformer> clz = DocBuilder.loadClass("RegexTransformer", null);
- assertNotNull(clz);
- }
-
- @Test
- public void singleEntityNoRows() {
- DataImporter di = new DataImporter();
- di.loadAndInit(dc_singleEntity);
- DIHConfiguration cfg = di.getConfig();
- Entity ent = cfg.getEntities().get(0);
- MockDataSource.setIterator("select * from x", new ArrayList<Map<String, Object>>().iterator());
- RequestInfo rp = new RequestInfo(null, createMap("command", "full-import"), null);
- SolrWriterImpl swi = new SolrWriterImpl();
- di.runCmd(rp, swi);
- assertEquals(Boolean.TRUE, swi.deleteAllCalled);
- assertEquals(Boolean.TRUE, swi.commitCalled);
- assertEquals(Boolean.TRUE, swi.finishCalled);
- assertEquals(0, swi.docs.size());
- assertEquals(1, di.getDocBuilder().importStatistics.queryCount.get());
- assertEquals(0, di.getDocBuilder().importStatistics.docCount.get());
- assertEquals(0, di.getDocBuilder().importStatistics.rowsCount.get());
- }
-
- @Test
- public void testDeltaImportNoRows_MustNotCommit() {
- DataImporter di = new DataImporter();
- di.loadAndInit(dc_deltaConfig);
- redirectTempProperties(di);
-
- DIHConfiguration cfg = di.getConfig();
- Entity ent = cfg.getEntities().get(0);
- MockDataSource.setIterator("select * from x", new ArrayList<Map<String, Object>>().iterator());
- MockDataSource.setIterator("select id from x", new ArrayList<Map<String, Object>>().iterator());
- RequestInfo rp = new RequestInfo(null, createMap("command", "delta-import"), null);
- SolrWriterImpl swi = new SolrWriterImpl();
- di.runCmd(rp, swi);
- assertEquals(Boolean.FALSE, swi.deleteAllCalled);
- assertEquals(Boolean.FALSE, swi.commitCalled);
- assertEquals(Boolean.TRUE, swi.finishCalled);
- assertEquals(0, swi.docs.size());
- assertEquals(1, di.getDocBuilder().importStatistics.queryCount.get());
- assertEquals(0, di.getDocBuilder().importStatistics.docCount.get());
- assertEquals(0, di.getDocBuilder().importStatistics.rowsCount.get());
- }
-
- @Test
- public void singleEntityOneRow() {
- DataImporter di = new DataImporter();
- di.loadAndInit(dc_singleEntity);
- DIHConfiguration cfg = di.getConfig();
- Entity ent = cfg.getEntities().get(0);
- List<Map<String, Object>> l = new ArrayList<>();
- l.add(createMap("id", 1, "desc", "one"));
- MockDataSource.setIterator("select * from x", l.iterator());
- RequestInfo rp = new RequestInfo(null, createMap("command", "full-import"), null);
- SolrWriterImpl swi = new SolrWriterImpl();
- di.runCmd(rp, swi);
- assertEquals(Boolean.TRUE, swi.deleteAllCalled);
- assertEquals(Boolean.TRUE, swi.commitCalled);
- assertEquals(Boolean.TRUE, swi.finishCalled);
- assertEquals(1, swi.docs.size());
- assertEquals(1, di.getDocBuilder().importStatistics.queryCount.get());
- assertEquals(1, di.getDocBuilder().importStatistics.docCount.get());
- assertEquals(1, di.getDocBuilder().importStatistics.rowsCount.get());
-
- for (int i = 0; i < l.size(); i++) {
- Map<String, Object> map = l.get(i);
- SolrInputDocument doc = swi.docs.get(i);
- for (Map.Entry<String, Object> entry : map.entrySet()) {
- assertEquals(entry.getValue(), doc.getFieldValue(entry.getKey()));
- }
- }
- }
-
- @Test
- public void testImportCommand() {
- DataImporter di = new DataImporter();
- di.loadAndInit(dc_singleEntity);
- DIHConfiguration cfg = di.getConfig();
- Entity ent = cfg.getEntities().get(0);
- List<Map<String, Object>> l = new ArrayList<>();
- l.add(createMap("id", 1, "desc", "one"));
- MockDataSource.setIterator("select * from x", l.iterator());
- RequestInfo rp = new RequestInfo(null, createMap("command", "import"), null);
- SolrWriterImpl swi = new SolrWriterImpl();
- di.runCmd(rp, swi);
- assertEquals(Boolean.FALSE, swi.deleteAllCalled);
- assertEquals(Boolean.TRUE, swi.commitCalled);
- assertEquals(Boolean.TRUE, swi.finishCalled);
- assertEquals(1, swi.docs.size());
- assertEquals(1, di.getDocBuilder().importStatistics.queryCount.get());
- assertEquals(1, di.getDocBuilder().importStatistics.docCount.get());
- assertEquals(1, di.getDocBuilder().importStatistics.rowsCount.get());
-
- for (int i = 0; i < l.size(); i++) {
- Map<String, Object> map = (Map<String, Object>) l.get(i);
- SolrInputDocument doc = swi.docs.get(i);
- for (Map.Entry<String, Object> entry : map.entrySet()) {
- assertEquals(entry.getValue(), doc.getFieldValue(entry.getKey()));
- }
- }
- }
-
- @Test
- public void singleEntityMultipleRows() {
- DataImporter di = new DataImporter();
- di.loadAndInit(dc_singleEntity);
- DIHConfiguration cfg = di.getConfig();
- Entity ent = cfg.getEntities().get(0);
- RequestInfo rp = new RequestInfo(null, createMap("command", "full-import"), null);
- List<Map<String, Object>> l = new ArrayList<>();
- l.add(createMap("id", 1, "desc", "one"));
- l.add(createMap("id", 2, "desc", "two"));
- l.add(createMap("id", 3, "desc", "three"));
-
- MockDataSource.setIterator("select * from x", l.iterator());
- SolrWriterImpl swi = new SolrWriterImpl();
- di.runCmd(rp, swi);
- assertEquals(Boolean.TRUE, swi.deleteAllCalled);
- assertEquals(Boolean.TRUE, swi.commitCalled);
- assertEquals(Boolean.TRUE, swi.finishCalled);
- assertEquals(3, swi.docs.size());
- for (int i = 0; i < l.size(); i++) {
- Map<String, Object> map = (Map<String, Object>) l.get(i);
- SolrInputDocument doc = swi.docs.get(i);
- for (Map.Entry<String, Object> entry : map.entrySet()) {
- assertEquals(entry.getValue(), doc.getFieldValue(entry.getKey()));
- }
- assertEquals(map.get("desc"), doc.getFieldValue("desc_s"));
- }
- assertEquals(1, di.getDocBuilder().importStatistics.queryCount.get());
- assertEquals(3, di.getDocBuilder().importStatistics.docCount.get());
- assertEquals(3, di.getDocBuilder().importStatistics.rowsCount.get());
- }
-
- @Test
- public void templateXPath() {
- DataImporter di = new DataImporter();
- di.loadAndInit(dc_variableXpath);
- DIHConfiguration cfg = di.getConfig();
-
- RequestInfo rp = new RequestInfo(null, createMap("command", "full-import"), null);
- List<Map<String, Object>> l = new ArrayList<>();
- l.add(createMap("id", 1, "name", "iphone", "manufacturer", "Apple"));
- l.add(createMap("id", 2, "name", "ipad", "manufacturer", "Apple"));
- l.add(createMap("id", 3, "name", "pixel", "manufacturer", "Google"));
-
- MockDataSource.setIterator("select * from x", l.iterator());
-
- List<Map<String,Object>> nestedData = new ArrayList<>();
- nestedData.add(createMap("founded", "Cupertino, California, U.S", "year", "1976", "year2", "1976"));
- nestedData.add(createMap("founded", "Cupertino, California, U.S", "year", "1976", "year2", "1976"));
- nestedData.add(createMap("founded", "Menlo Park, California, U.S", "year", "1998", "year2", "1998"));
-
- MockStringDataSource.setData("companies.xml", xml_attrVariableXpath);
- MockStringDataSource.setData("companies2.xml", xml_variableXpath);
- MockStringDataSource.setData("companies3.xml", xml_variableForEach);
-
- SolrWriterImpl swi = new SolrWriterImpl();
- di.runCmd(rp, swi);
- assertEquals(Boolean.TRUE, swi.deleteAllCalled);
- assertEquals(Boolean.TRUE, swi.commitCalled);
- assertEquals(Boolean.TRUE, swi.finishCalled);
- assertEquals(3, swi.docs.size());
- for (int i = 0; i < l.size(); i++) {
- SolrInputDocument doc = swi.docs.get(i);
-
- Map<String, Object> map = l.get(i);
- for (Map.Entry<String, Object> entry : map.entrySet()) {
- assertEquals(entry.getValue(), doc.getFieldValue(entry.getKey()));
- }
-
- map = nestedData.get(i);
- for (Map.Entry<String, Object> entry : map.entrySet()) {
- assertEquals(entry.getValue(), doc.getFieldValue(entry.getKey()));
- }
- }
- assertEquals(1, di.getDocBuilder().importStatistics.queryCount.get());
- assertEquals(3, di.getDocBuilder().importStatistics.docCount.get());
- }
-
- static class SolrWriterImpl extends SolrWriter {
- List<SolrInputDocument> docs = new ArrayList<>();
-
- Boolean deleteAllCalled = Boolean.FALSE;
-
- Boolean commitCalled = Boolean.FALSE;
-
- Boolean finishCalled = Boolean.FALSE;
-
- public SolrWriterImpl() {
- super(null, null);
- }
-
- @Override
- public boolean upload(SolrInputDocument doc) {
- return docs.add(doc);
- }
-
- @Override
- public void doDeleteAll() {
- deleteAllCalled = Boolean.TRUE;
- }
-
- @Override
- public void commit(boolean b) {
- commitCalled = Boolean.TRUE;
- }
-
- @Override
- public void close() {
- finishCalled = Boolean.TRUE;
- }
- }
-
- public static final String dc_singleEntity = "<dataConfig>\n"
- + "<dataSource type=\"MockDataSource\"/>\n"
- + " <document name=\"X\" >\n"
- + " <entity name=\"x\" query=\"select * from x\">\n"
- + " <field column=\"id\"/>\n"
- + " <field column=\"desc\"/>\n"
- + " <field column=\"desc\" name=\"desc_s\" />" + " </entity>\n"
- + " </document>\n" + "</dataConfig>";
-
- public static final String dc_deltaConfig = "<dataConfig>\n"
- + "<dataSource type=\"MockDataSource\"/>\n"
- + " <document name=\"X\" >\n"
- + " <entity name=\"x\" query=\"select * from x\" deltaQuery=\"select id from x\">\n"
- + " <field column=\"id\"/>\n"
- + " <field column=\"desc\"/>\n"
- + " <field column=\"desc\" name=\"desc_s\" />" + " </entity>\n"
- + " </document>\n" + "</dataConfig>";
-
- public static final String dc_variableXpath = "<dataConfig>\n"
- + "<dataSource type=\"MockDataSource\"/>\n"
- + "<dataSource name=\"xml\" type=\"MockStringDataSource\"/>\n"
- + " <document name=\"X\" >\n"
- + " <entity name=\"x\" query=\"select * from x\">\n"
- + " <field column=\"id\"/>\n"
- + " <field column=\"name\"/>\n"
- + " <field column=\"manufacturer\"/>"
- + " <entity name=\"c1\" url=\"companies.xml\" dataSource=\"xml\" forEach=\"/companies/company\" processor=\"XPathEntityProcessor\">"
- + " <field column=\"year\" xpath=\"/companies/company/year[@name='p_${x.manufacturer}_s']\" />"
- + " </entity>"
- + " <entity name=\"c2\" url=\"companies2.xml\" dataSource=\"xml\" forEach=\"/companies/company\" processor=\"XPathEntityProcessor\">"
- + " <field column=\"founded\" xpath=\"/companies/company/p_${x.manufacturer}_s/founded\" />"
- + " </entity>"
- + " <entity name=\"c3\" url=\"companies3.xml\" dataSource=\"xml\" forEach=\"/companies/${x.manufacturer}\" processor=\"XPathEntityProcessor\">"
- + " <field column=\"year2\" xpath=\"/companies/${x.manufacturer}/year\" />"
- + " </entity>"
- + " </entity>\n"
- + " </document>\n" + "</dataConfig>";
-
-
- public static final String xml_variableForEach = "<companies>\n" +
- "\t<Apple>\n" +
- "\t\t<year>1976</year>\n" +
- "\t</Apple>\n" +
- "\t<Google>\n" +
- "\t\t<year>1998</year>\n" +
- "\t</Google>\n" +
- "</companies>";
-
- public static final String xml_variableXpath = "<companies>\n" +
- "\t<company>\n" +
- "\t\t<p_Apple_s>\n" +
- "\t\t\t<founded>Cupertino, California, U.S</founded>\n" +
- "\t\t</p_Apple_s>\t\t\n" +
- "\t</company>\n" +
- "\t<company>\n" +
- "\t\t<p_Google_s>\n" +
- "\t\t\t<founded>Menlo Park, California, U.S</founded>\n" +
- "\t\t</p_Google_s>\n" +
- "\t</company>\n" +
- "</companies>";
-
- public static final String xml_attrVariableXpath = "<companies>\n" +
- "\t<company>\n" +
- "\t\t<year name='p_Apple_s'>1976</year>\n" +
- "\t</company>\n" +
- "\t<company>\n" +
- "\t\t<year name='p_Google_s'>1998</year>\t\t\n" +
- "\t</company>\n" +
- "</companies>";
-
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDocBuilder2.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDocBuilder2.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDocBuilder2.java
deleted file mode 100644
index 27865cd..0000000
--- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDocBuilder2.java
+++ /dev/null
@@ -1,433 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import java.io.File;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.solr.request.LocalSolrQueryRequest;
-import org.junit.BeforeClass;
-import org.junit.Ignore;
-import org.junit.Test;
-
-import java.nio.charset.StandardCharsets;
-
-/**
- * <p>
- * Test for DocBuilder using the test harness
- * </p>
- *
- *
- * @since solr 1.3
- */
-public class TestDocBuilder2 extends AbstractDataImportHandlerTestCase {
-
- @BeforeClass
- public static void beforeClass() throws Exception {
- initCore("dataimport-solrconfig.xml", "dataimport-schema.xml");
- }
-
- @Test
- @SuppressWarnings("unchecked")
- public void testSingleEntity() throws Exception {
- List rows = new ArrayList();
- rows.add(createMap("id", "1", "desc", "one"));
- MockDataSource.setIterator("select * from x", rows.iterator());
-
- runFullImport(loadDataConfig("single-entity-data-config.xml"));
-
- assertQ(req("id:1"), "//*[@numFound='1']");
-
- assertTrue("Update request processor processAdd was not called", TestUpdateRequestProcessor.processAddCalled);
- assertTrue("Update request processor processCommit was not callled", TestUpdateRequestProcessor.processCommitCalled);
- assertTrue("Update request processor finish was not called", TestUpdateRequestProcessor.finishCalled);
- }
-
- @Test
- @SuppressWarnings("unchecked")
- public void testSingleEntity_CaseInsensitive() throws Exception {
- List rows = new ArrayList();
- rows.add(createMap("id", "1", "desC", "one"));
- MockDataSource.setIterator("select * from x", rows.iterator());
-
- runFullImport(DATA_CONFIG_WITH_CASE_INSENSITIVE_FIELDS);
-
- assertQ(req("id:1"), "//*[@numFound='1']");
- assertTrue("Start event listener was not called", StartEventListener.executed);
- assertTrue("End event listener was not called", EndEventListener.executed);
- assertTrue("Update request processor processAdd was not called", TestUpdateRequestProcessor.processAddCalled);
- assertTrue("Update request processor finish was not called", TestUpdateRequestProcessor.finishCalled);
- }
-
- @Test
- public void testErrorHandler() throws Exception {
- List rows = new ArrayList();
- rows.add(createMap("id", "1", "FORCE_ERROR", "true"));
- MockDataSource.setIterator("select * from x", rows.iterator());
-
- runFullImport(DATA_CONFIG_WITH_ERROR_HANDLER);
-
- assertTrue("Error event listener was not called", ErrorEventListener.executed);
- assertTrue(ErrorEventListener.lastException.getMessage().contains("ForcedException"));
- }
-
- @Test
- @SuppressWarnings("unchecked")
- public void testDynamicFields() throws Exception {
- List rows = new ArrayList();
- rows.add(createMap("id", "1", "desc", "one"));
- MockDataSource.setIterator("select * from x", rows.iterator());
-
- runFullImport(DATA_CONFIG_WITH_DYNAMIC_TRANSFORMER);
-
- assertQ(req("id:1"), "//*[@numFound='1']");
- assertQ(req("dynamic_s:test"), "//*[@numFound='1']");
- }
-
- @Test
- @SuppressWarnings("unchecked")
- public void testRequestParamsAsVariable() throws Exception {
- List rows = new ArrayList();
- rows.add(createMap("id", "101", "desc", "ApacheSolr"));
- MockDataSource.setIterator("select * from books where category='search'", rows.iterator());
-
- LocalSolrQueryRequest request = lrf.makeRequest("command", "full-import",
- "debug", "on", "clean", "true", "commit", "true",
- "category", "search",
- "dataConfig", REQUEST_PARAM_AS_VARIABLE);
- h.query("/dataimport", request);
- assertQ(req("desc:ApacheSolr"), "//*[@numFound='1']");
- }
-
- @Test
- @SuppressWarnings("unchecked")
- public void testDynamicFieldNames() throws Exception {
- List rows = new ArrayList();
- rows.add(createMap("mypk", "101", "text", "ApacheSolr"));
- MockDataSource.setIterator("select * from x", rows.iterator());
-
- LocalSolrQueryRequest request = lrf.makeRequest("command", "full-import",
- "debug", "on", "clean", "true", "commit", "true",
- "dataConfig", DATA_CONFIG_WITH_DYNAMIC_FIELD_NAMES);
- h.query("/dataimport", request);
- assertQ(req("id:101"), "//*[@numFound='1']", "//*[@name='101_s']");
- }
-
- @Test
- @SuppressWarnings("unchecked")
- public void testRequestParamsAsFieldName() throws Exception {
- List rows = new ArrayList();
- rows.add(createMap("mypk", "101", "text", "ApacheSolr"));
- MockDataSource.setIterator("select * from x", rows.iterator());
-
- LocalSolrQueryRequest request = lrf.makeRequest("command", "full-import",
- "debug", "on", "clean", "true", "commit", "true",
- "mypk", "id", "text", "desc",
- "dataConfig", DATA_CONFIG_WITH_TEMPLATIZED_FIELD_NAMES);
- h.query("/dataimport", request);
- assertQ(req("id:101"), "//*[@numFound='1']");
- }
-
- @Test
- @SuppressWarnings("unchecked")
- public void testContext() throws Exception {
- List rows = new ArrayList();
- rows.add(createMap("id", "1", "desc", "one"));
- MockDataSource.setIterator("select * from x", rows.iterator());
-
- runFullImport(loadDataConfig("data-config-with-transformer.xml"));
- }
-
- @Test
- @SuppressWarnings("unchecked")
- public void testSkipDoc() throws Exception {
- List rows = new ArrayList();
- rows.add(createMap("id", "1", "desc", "one"));
- rows.add(createMap("id", "2", "desc", "two", DocBuilder.SKIP_DOC, "true"));
- MockDataSource.setIterator("select * from x", rows.iterator());
-
- runFullImport(DATA_CONFIG_WITH_DYNAMIC_TRANSFORMER);
-
- assertQ(req("id:1"), "//*[@numFound='1']");
- assertQ(req("id:2"), "//*[@numFound='0']");
- }
-
- @Test
- @SuppressWarnings("unchecked")
- public void testSkipRow() throws Exception {
- List rows = new ArrayList();
- rows.add(createMap("id", "1", "desc", "one"));
- rows.add(createMap("id", "2", "desc", "two", DocBuilder.SKIP_ROW, "true"));
- MockDataSource.setIterator("select * from x", rows.iterator());
-
- runFullImport(DATA_CONFIG_WITH_DYNAMIC_TRANSFORMER);
-
- assertQ(req("id:1"), "//*[@numFound='1']");
- assertQ(req("id:2"), "//*[@numFound='0']");
-
- MockDataSource.clearCache();
-
- rows = new ArrayList();
- rows.add(createMap("id", "3", "desc", "one"));
- rows.add(createMap("id", "4", "desc", "two"));
- MockDataSource.setIterator("select * from x", rows.iterator());
-
- rows = new ArrayList();
- rows.add(createMap("name_s", "abcd"));
- MockDataSource.setIterator("3", rows.iterator());
-
- rows = new ArrayList();
- rows.add(createMap("name_s", "xyz", DocBuilder.SKIP_ROW, "true"));
- MockDataSource.setIterator("4", rows.iterator());
-
- runFullImport(DATA_CONFIG_WITH_TWO_ENTITIES);
- assertQ(req("id:3"), "//*[@numFound='1']");
- assertQ(req("id:4"), "//*[@numFound='1']");
- assertQ(req("name_s:abcd"), "//*[@numFound='1']");
- assertQ(req("name_s:xyz"), "//*[@numFound='0']");
- }
-
- @Test
- @SuppressWarnings("unchecked")
- public void testStopTransform() throws Exception {
- List rows = new ArrayList();
- rows.add(createMap("id", "1", "desc", "one"));
- rows.add(createMap("id", "2", "desc", "two", "$stopTransform", "true"));
- MockDataSource.setIterator("select * from x", rows.iterator());
-
- runFullImport(DATA_CONFIG_FOR_SKIP_TRANSFORM);
-
- assertQ(req("id:1"), "//*[@numFound='1']");
- assertQ(req("id:2"), "//*[@numFound='1']");
- assertQ(req("name_s:xyz"), "//*[@numFound='1']");
- }
-
- @Test
- @SuppressWarnings("unchecked")
- public void testDeleteDocs() throws Exception {
- List rows = new ArrayList();
- rows.add(createMap("id", "1", "desc", "one"));
- rows.add(createMap("id", "2", "desc", "two"));
- rows.add(createMap("id", "3", "desc", "two", DocBuilder.DELETE_DOC_BY_ID, "2"));
- MockDataSource.setIterator("select * from x", rows.iterator());
-
- runFullImport(DATA_CONFIG_FOR_SKIP_TRANSFORM);
-
- assertQ(req("id:1"), "//*[@numFound='1']");
- assertQ(req("id:2"), "//*[@numFound='0']");
- assertQ(req("id:3"), "//*[@numFound='1']");
-
- assertTrue("Update request processor processDelete was not called", TestUpdateRequestProcessor.processDeleteCalled);
- assertTrue("Update request processor finish was not called", TestUpdateRequestProcessor.finishCalled);
-
- MockDataSource.clearCache();
- rows = new ArrayList();
- rows.add(createMap("id", "1", "desc", "one"));
- rows.add(createMap("id", "2", "desc", "one"));
- rows.add(createMap("id", "3", "desc", "two", DocBuilder.DELETE_DOC_BY_QUERY, "desc:one"));
- MockDataSource.setIterator("select * from x", rows.iterator());
-
- runFullImport(DATA_CONFIG_FOR_SKIP_TRANSFORM);
-
- assertQ(req("id:1"), "//*[@numFound='0']");
- assertQ(req("id:2"), "//*[@numFound='0']");
- assertQ(req("id:3"), "//*[@numFound='1']");
-
- assertTrue("Update request processor processDelete was not called", TestUpdateRequestProcessor.processDeleteCalled);
- assertTrue("Update request processor finish was not called", TestUpdateRequestProcessor.finishCalled);
-
- MockDataSource.clearCache();
- rows = new ArrayList();
- rows.add(createMap(DocBuilder.DELETE_DOC_BY_ID, "3"));
- MockDataSource.setIterator("select * from x", rows.iterator());
- runFullImport(DATA_CONFIG_FOR_SKIP_TRANSFORM, createMap("clean","false"));
- assertQ(req("id:3"), "//*[@numFound='0']");
-
- assertTrue("Update request processor processDelete was not called", TestUpdateRequestProcessor.processDeleteCalled);
- assertTrue("Update request processor finish was not called", TestUpdateRequestProcessor.finishCalled);
-
- }
-
- @Test
- @Ignore("Fix Me. See SOLR-4103.")
- public void testFileListEntityProcessor_lastIndexTime() throws Exception {
- File tmpdir = createTempDir().toFile();
-
- Map<String, String> params = createMap("baseDir", tmpdir.getAbsolutePath());
-
- createFile(tmpdir, "a.xml", "a.xml".getBytes(StandardCharsets.UTF_8), true);
- createFile(tmpdir, "b.xml", "b.xml".getBytes(StandardCharsets.UTF_8), true);
- createFile(tmpdir, "c.props", "c.props".getBytes(StandardCharsets.UTF_8), true);
- runFullImport(DATA_CONFIG_FILE_LIST, params);
- assertQ(req("*:*"), "//*[@numFound='3']");
-
- // Add a new file after a full index is done
- createFile(tmpdir, "t.xml", "t.xml".getBytes(StandardCharsets.UTF_8), false);
- runFullImport(DATA_CONFIG_FILE_LIST, params);
- // we should find only 1 because by default clean=true is passed
- // and this particular import should find only one file t.xml
- assertQ(req("*:*"), "//*[@numFound='1']");
- }
-
- public static class MockTransformer extends Transformer {
- @Override
- public Object transformRow(Map<String, Object> row, Context context) {
- assertTrue("Context gave incorrect data source", context.getDataSource("mockDs") instanceof MockDataSource2);
- return row;
- }
- }
-
- public static class AddDynamicFieldTransformer extends Transformer {
- @Override
- public Object transformRow(Map<String, Object> row, Context context) {
- // Add a dynamic field
- row.put("dynamic_s", "test");
- return row;
- }
- }
-
- public static class ForcedExceptionTransformer extends Transformer {
- @Override
- public Object transformRow(Map<String, Object> row, Context context) {
- throw new DataImportHandlerException(DataImportHandlerException.SEVERE, "ForcedException");
- }
- }
-
- public static class MockDataSource2 extends MockDataSource {
-
- }
-
- public static class StartEventListener implements EventListener {
- public static boolean executed = false;
-
- @Override
- public void onEvent(Context ctx) {
- executed = true;
- }
- }
-
- public static class EndEventListener implements EventListener {
- public static boolean executed = false;
-
- @Override
- public void onEvent(Context ctx) {
- executed = true;
- }
- }
-
- public static class ErrorEventListener implements EventListener {
- public static boolean executed = false;
- public static Exception lastException = null;
-
- @Override
- public void onEvent(Context ctx) {
- executed = true;
- lastException = ((ContextImpl) ctx).getLastException();
- }
- }
-
- private static final String REQUEST_PARAM_AS_VARIABLE = "<dataConfig>\n" +
- " <dataSource type=\"MockDataSource\" />\n" +
- " <document>\n" +
- " <entity name=\"books\" query=\"select * from books where category='${dataimporter.request.category}'\">\n" +
- " <field column=\"id\" />\n" +
- " <field column=\"desc\" />\n" +
- " </entity>\n" +
- " </document>\n" +
- "</dataConfig>";
-
- private static final String DATA_CONFIG_WITH_DYNAMIC_TRANSFORMER = "<dataConfig> <dataSource type=\"MockDataSource\"/>\n" +
- " <document>\n" +
- " <entity name=\"books\" query=\"select * from x\"" +
- " transformer=\"TestDocBuilder2$AddDynamicFieldTransformer\">\n" +
- " <field column=\"id\" />\n" +
- " <field column=\"desc\" />\n" +
- " </entity>\n" +
- " </document>\n" +
- "</dataConfig>";
-
- private static final String DATA_CONFIG_FOR_SKIP_TRANSFORM = "<dataConfig> <dataSource type=\"MockDataSource\"/>\n" +
- " <document>\n" +
- " <entity name=\"books\" query=\"select * from x\"" +
- " transformer=\"TemplateTransformer\">\n" +
- " <field column=\"id\" />\n" +
- " <field column=\"desc\" />\n" +
- " <field column=\"name_s\" template=\"xyz\" />\n" +
- " </entity>\n" +
- " </document>\n" +
- "</dataConfig>";
-
- private static final String DATA_CONFIG_WITH_TWO_ENTITIES = "<dataConfig><dataSource type=\"MockDataSource\"/>\n" +
- " <document>\n" +
- " <entity name=\"books\" query=\"select * from x\">" +
- " <field column=\"id\" />\n" +
- " <field column=\"desc\" />\n" +
- " <entity name=\"authors\" query=\"${books.id}\">" +
- " <field column=\"name_s\" />" +
- " </entity>" +
- " </entity>\n" +
- " </document>\n" +
- "</dataConfig>";
-
- private static final String DATA_CONFIG_WITH_CASE_INSENSITIVE_FIELDS = "<dataConfig> <dataSource type=\"MockDataSource\"/>\n" +
- " <document onImportStart=\"TestDocBuilder2$StartEventListener\" onImportEnd=\"TestDocBuilder2$EndEventListener\">\n" +
- " <entity name=\"books\" query=\"select * from x\">\n" +
- " <field column=\"ID\" />\n" +
- " <field column=\"Desc\" />\n" +
- " </entity>\n" +
- " </document>\n" +
- "</dataConfig>";
-
- private static final String DATA_CONFIG_WITH_ERROR_HANDLER = "<dataConfig> <dataSource type=\"MockDataSource\"/>\n" +
- " <document onError=\"TestDocBuilder2$ErrorEventListener\">\n" +
- " <entity name=\"books\" query=\"select * from x\" transformer=\"TestDocBuilder2$ForcedExceptionTransformer\">\n" +
- " <field column=\"id\" />\n" +
- " <field column=\"FORCE_ERROR\" />\n" +
- " </entity>\n" +
- " </document>\n" +
- "</dataConfig>";
-
- private static final String DATA_CONFIG_WITH_TEMPLATIZED_FIELD_NAMES = "<dataConfig><dataSource type=\"MockDataSource\"/>\n" +
- " <document>\n" +
- " <entity name=\"books\" query=\"select * from x\">\n" +
- " <field column=\"mypk\" name=\"${dih.request.mypk}\" />\n" +
- " <field column=\"text\" name=\"${dih.request.text}\" />\n" +
- " </entity>\n" +
- " </document>\n" +
- "</dataConfig>";
-
- private static final String DATA_CONFIG_WITH_DYNAMIC_FIELD_NAMES = "<dataConfig><dataSource type=\"MockDataSource\"/>\n" +
- " <document>\n" +
- " <entity name=\"books\" query=\"select * from x\">\n" +
- " <field column=\"mypk\" name=\"id\" />\n" +
- " <field column=\"text\" name=\"${books.mypk}_s\" />\n" +
- " </entity>\n" +
- " </document>\n" +
- "</dataConfig>";
-
- private static final String DATA_CONFIG_FILE_LIST = "<dataConfig>\n" +
- "\t<document>\n" +
- "\t\t<entity name=\"x\" processor=\"FileListEntityProcessor\" \n" +
- "\t\t\t\tfileName=\".*\" newerThan=\"${dih.last_index_time}\" \n" +
- "\t\t\t\tbaseDir=\"${dih.request.baseDir}\" transformer=\"TemplateTransformer\">\n" +
- "\t\t\t<field column=\"id\" template=\"${x.file}\" />\n" +
- "\t\t</entity>\n" +
- "\t</document>\n" +
- "</dataConfig>";
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestEntityProcessorBase.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestEntityProcessorBase.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestEntityProcessorBase.java
deleted file mode 100644
index 623e49e..0000000
--- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestEntityProcessorBase.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import org.junit.Test;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * <p>
- * Test for EntityProcessorBase
- * </p>
- *
- *
- * @since solr 1.3
- */
-public class TestEntityProcessorBase extends AbstractDataImportHandlerTestCase {
-
- @Test
- public void multiTransformer() {
- List<Map<String, String>> fields = new ArrayList<>();
- Map<String, String> entity = new HashMap<>();
- entity.put("transformer", T1.class.getName() + "," + T2.class.getName()
- + "," + T3.class.getName());
- fields.add(getField("A", null, null, null, null));
- fields.add(getField("B", null, null, null, null));
-
- Context context = getContext(null, null, new MockDataSource(), Context.FULL_DUMP,
- fields, entity);
- Map<String, Object> src = new HashMap<>();
- src.put("A", "NA");
- src.put("B", "NA");
- EntityProcessorWrapper sep = new EntityProcessorWrapper(new SqlEntityProcessor(), null, null);
- sep.init(context);
- Map<String, Object> res = sep.applyTransformer(src);
- assertNotNull(res.get("T1"));
- assertNotNull(res.get("T2"));
- assertNotNull(res.get("T3"));
- }
-
- static class T1 extends Transformer {
-
- @Override
- public Object transformRow(Map<String, Object> aRow, Context context) {
- aRow.put("T1", "T1 called");
- return aRow;
-
- }
- }
-
- static class T2 extends Transformer {
-
- @Override
- public Object transformRow(Map<String, Object> aRow, Context context) {
- aRow.put("T2", "T2 called");
- return aRow;
- }
- }
-
- static class T3 {
-
- public Object transformRow(Map<String, Object> aRow) {
- aRow.put("T3", "T3 called");
- return aRow;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestEphemeralCache.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestEphemeralCache.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestEphemeralCache.java
deleted file mode 100644
index ddccafe..0000000
--- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestEphemeralCache.java
+++ /dev/null
@@ -1,140 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import java.math.BigDecimal;
-import java.util.ArrayList;
-import java.util.List;
-
-import static org.hamcrest.CoreMatchers.*;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-public class TestEphemeralCache extends AbstractDataImportHandlerTestCase {
-
-
- @BeforeClass
- public static void beforeClass() throws Exception {
- initCore("dataimport-solrconfig.xml", "dataimport-schema.xml");
- }
-
- @Before
- public void reset() {
- DestroyCountCache.destroyed.clear();
- setupMockData();
- }
-
- @Test
- public void test() throws Exception {
- assertFullImport(getDataConfigDotXml());
- }
-
- @SuppressWarnings("unchecked")
- private void setupMockData() {
- List parentRows = new ArrayList();
- parentRows.add(createMap("id", new BigDecimal("1"), "parent_s", "one"));
- parentRows.add(createMap("id", new BigDecimal("2"), "parent_s", "two"));
- parentRows.add(createMap("id", new BigDecimal("3"), "parent_s", "three"));
- parentRows.add(createMap("id", new BigDecimal("4"), "parent_s", "four"));
- parentRows.add(createMap("id", new BigDecimal("5"), "parent_s", "five"));
-
- List child1Rows = new ArrayList();
- child1Rows.add(createMap("id", new BigDecimal("6"), "child1a_mult_s", "this is the number six."));
- child1Rows.add(createMap("id", new BigDecimal("5"), "child1a_mult_s", "this is the number five."));
- child1Rows.add(createMap("id", new BigDecimal("6"), "child1a_mult_s", "let's sing a song of six."));
- child1Rows.add(createMap("id", new BigDecimal("3"), "child1a_mult_s", "three"));
- child1Rows.add(createMap("id", new BigDecimal("3"), "child1a_mult_s", "III"));
- child1Rows.add(createMap("id", new BigDecimal("3"), "child1a_mult_s", "3"));
- child1Rows.add(createMap("id", new BigDecimal("3"), "child1a_mult_s", "|||"));
- child1Rows.add(createMap("id", new BigDecimal("1"), "child1a_mult_s", "one"));
- child1Rows.add(createMap("id", new BigDecimal("1"), "child1a_mult_s", "uno"));
- child1Rows.add(createMap("id", new BigDecimal("2"), "child1b_s", "CHILD1B", "child1a_mult_s", "this is the number two."));
-
- List child2Rows = new ArrayList();
- child2Rows.add(createMap("id", new BigDecimal("6"), "child2a_mult_s", "Child 2 says, 'this is the number six.'"));
- child2Rows.add(createMap("id", new BigDecimal("5"), "child2a_mult_s", "Child 2 says, 'this is the number five.'"));
- child2Rows.add(createMap("id", new BigDecimal("6"), "child2a_mult_s", "Child 2 says, 'let's sing a song of six.'"));
- child2Rows.add(createMap("id", new BigDecimal("3"), "child2a_mult_s", "Child 2 says, 'three'"));
- child2Rows.add(createMap("id", new BigDecimal("3"), "child2a_mult_s", "Child 2 says, 'III'"));
- child2Rows.add(createMap("id", new BigDecimal("3"), "child2b_s", "CHILD2B", "child2a_mult_s", "Child 2 says, '3'"));
- child2Rows.add(createMap("id", new BigDecimal("3"), "child2a_mult_s", "Child 2 says, '|||'"));
- child2Rows.add(createMap("id", new BigDecimal("1"), "child2a_mult_s", "Child 2 says, 'one'"));
- child2Rows.add(createMap("id", new BigDecimal("1"), "child2a_mult_s", "Child 2 says, 'uno'"));
- child2Rows.add(createMap("id", new BigDecimal("2"), "child2a_mult_s", "Child 2 says, 'this is the number two.'"));
-
- MockDataSource.setIterator("SELECT * FROM PARENT", parentRows.iterator());
- MockDataSource.setIterator("SELECT * FROM CHILD_1", child1Rows.iterator());
- MockDataSource.setIterator("SELECT * FROM CHILD_2", child2Rows.iterator());
-
- }
- private String getDataConfigDotXml() {
- return
- "<dataConfig>" +
- " <dataSource type=\"MockDataSource\" />" +
- " <document>" +
- " <entity " +
- " name=\"PARENT\"" +
- " processor=\"SqlEntityProcessor\"" +
- " cacheImpl=\"org.apache.solr.handler.dataimport.DestroyCountCache\"" +
- " cacheName=\"PARENT\"" +
- " query=\"SELECT * FROM PARENT\" " +
- " >" +
- " <entity" +
- " name=\"CHILD_1\"" +
- " processor=\"SqlEntityProcessor\"" +
- " cacheImpl=\"org.apache.solr.handler.dataimport.DestroyCountCache\"" +
- " cacheName=\"CHILD\"" +
- " cacheKey=\"id\"" +
- " cacheLookup=\"PARENT.id\"" +
- " fieldNames=\"id, child1a_mult_s, child1b_s\"" +
- " fieldTypes=\"BIGDECIMAL, STRING, STRING\"" +
- " query=\"SELECT * FROM CHILD_1\" " +
- " />" +
- " <entity" +
- " name=\"CHILD_2\"" +
- " processor=\"SqlEntityProcessor\"" +
- " cacheImpl=\"org.apache.solr.handler.dataimport.DestroyCountCache\"" +
- " cacheKey=\"id\"" +
- " cacheLookup=\"PARENT.id\"" +
- " query=\"SELECT * FROM CHILD_2\" " +
- " />" +
- " </entity>" +
- " </document>" +
- "</dataConfig>"
- ;
- }
-
- private void assertFullImport(String dataConfig) throws Exception {
- runFullImport(dataConfig);
-
- assertQ(req("*:*"), "//*[@numFound='5']");
- assertQ(req("id:1"), "//*[@numFound='1']");
- assertQ(req("id:6"), "//*[@numFound='0']");
- assertQ(req("parent_s:four"), "//*[@numFound='1']");
- assertQ(req("child1a_mult_s:this\\ is\\ the\\ numbe*"), "//*[@numFound='2']");
- assertQ(req("child2a_mult_s:Child\\ 2\\ say*"), "//*[@numFound='4']");
- assertQ(req("child1b_s:CHILD1B"), "//*[@numFound='1']");
- assertQ(req("child2b_s:CHILD2B"), "//*[@numFound='1']");
- assertQ(req("child1a_mult_s:one"), "//*[@numFound='1']");
- assertQ(req("child1a_mult_s:uno"), "//*[@numFound='1']");
- assertQ(req("child1a_mult_s:(uno OR one)"), "//*[@numFound='1']");
-
- assertThat(DestroyCountCache.destroyed.size(), is(3));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestErrorHandling.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestErrorHandling.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestErrorHandling.java
deleted file mode 100644
index 1ea1ad4..0000000
--- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestErrorHandling.java
+++ /dev/null
@@ -1,209 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-import java.io.Reader;
-import java.io.StringReader;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-
-/**
- * Tests exception handling during imports in DataImportHandler
- *
- *
- * @since solr 1.4
- */
-public class TestErrorHandling extends AbstractDataImportHandlerTestCase {
-
- //TODO: fix this test to not require FSDirectory.
- static String savedFactory;
- @BeforeClass
- public static void beforeClass() throws Exception {
- savedFactory = System.getProperty("solr.DirectoryFactory");
- System.setProperty("solr.directoryFactory", "solr.MockFSDirectoryFactory");
- initCore("dataimport-solrconfig.xml", "dataimport-schema.xml");
- ignoreException("Unexpected close tag");
- }
-
- @AfterClass
- public static void afterClass() {
- if (savedFactory == null) {
- System.clearProperty("solr.directoryFactory");
- } else {
- System.setProperty("solr.directoryFactory", savedFactory);
- }
- }
-
- @Before @Override
- public void setUp() throws Exception {
- super.setUp();
- clearIndex();
- assertU(commit());
- }
-
- public void testMalformedStreamingXml() throws Exception {
- StringDataSource.xml = malformedXml;
- runFullImport(dataConfigWithStreaming);
- assertQ(req("id:1"), "//*[@numFound='1']");
- assertQ(req("id:2"), "//*[@numFound='1']");
- }
-
- public void testMalformedNonStreamingXml() throws Exception {
- StringDataSource.xml = malformedXml;
- runFullImport(dataConfigWithoutStreaming);
- assertQ(req("id:1"), "//*[@numFound='1']");
- assertQ(req("id:2"), "//*[@numFound='1']");
- }
-
- public void testAbortOnError() throws Exception {
- StringDataSource.xml = malformedXml;
- runFullImport(dataConfigAbortOnError);
- assertQ(req("*:*"), "//*[@numFound='0']");
- }
-
- public void testTransformerErrorContinue() throws Exception {
- StringDataSource.xml = wellformedXml;
- List<Map<String, Object>> rows = new ArrayList<>();
- rows.add(createMap("id", "3", "desc", "exception-transformer"));
- MockDataSource.setIterator("select * from foo", rows.iterator());
- runFullImport(dataConfigWithTransformer);
- assertQ(req("*:*"), "//*[@numFound='3']");
- }
-
- public void testExternalEntity() throws Exception {
- StringDataSource.xml = wellformedXml;
- // This should not fail as external entities are replaced by an empty string during parsing:
- runFullImport(dataConfigWithEntity);
- assertQ(req("*:*"), "//*[@numFound='3']");
- }
-
- public static class StringDataSource extends DataSource<Reader> {
- public static String xml = "";
-
- @Override
- public void init(Context context, Properties initProps) {
- }
-
- @Override
- public Reader getData(String query) {
- return new StringReader(xml);
- }
-
- @Override
- public void close() {
-
- }
- }
-
- public static class ExceptionTransformer extends Transformer {
- @Override
- public Object transformRow(Map<String, Object> row, Context context) {
- throw new RuntimeException("Test exception");
- }
- }
-
- private String dataConfigWithStreaming = "<dataConfig>\n" +
- " <dataSource name=\"str\" type=\"TestErrorHandling$StringDataSource\" />" +
- " <document>\n" +
- " <entity name=\"node\" dataSource=\"str\" processor=\"XPathEntityProcessor\" url=\"test\" stream=\"true\" forEach=\"/root/node\" onError=\"skip\">\n" +
- " <field column=\"id\" xpath=\"/root/node/id\" />\n" +
- " <field column=\"desc\" xpath=\"/root/node/desc\" />\n" +
- " </entity>\n" +
- " </document>\n" +
- "</dataConfig>";
-
- private String dataConfigWithoutStreaming = "<dataConfig>\n" +
- " <dataSource name=\"str\" type=\"TestErrorHandling$StringDataSource\" />" +
- " <document>\n" +
- " <entity name=\"node\" dataSource=\"str\" processor=\"XPathEntityProcessor\" url=\"test\" forEach=\"/root/node\" onError=\"skip\">\n" +
- " <field column=\"id\" xpath=\"/root/node/id\" />\n" +
- " <field column=\"desc\" xpath=\"/root/node/desc\" />\n" +
- " </entity>\n" +
- " </document>\n" +
- "</dataConfig>";
-
- private String dataConfigAbortOnError = "<dataConfig>\n" +
- " <dataSource name=\"str\" type=\"TestErrorHandling$StringDataSource\" />" +
- " <document>\n" +
- " <entity name=\"node\" dataSource=\"str\" processor=\"XPathEntityProcessor\" url=\"test\" forEach=\"/root/node\" onError=\"abort\">\n" +
- " <field column=\"id\" xpath=\"/root/node/id\" />\n" +
- " <field column=\"desc\" xpath=\"/root/node/desc\" />\n" +
- " </entity>\n" +
- " </document>\n" +
- "</dataConfig>";
-
- private String dataConfigWithTransformer = "<dataConfig>\n" +
- " <dataSource name=\"str\" type=\"TestErrorHandling$StringDataSource\" />" +
- "<dataSource type=\"MockDataSource\"/>" +
- " <document>\n" +
- " <entity name=\"node\" dataSource=\"str\" processor=\"XPathEntityProcessor\" url=\"test\" forEach=\"/root/node\">\n" +
- " <field column=\"id\" xpath=\"/root/node/id\" />\n" +
- " <field column=\"desc\" xpath=\"/root/node/desc\" />\n" +
- " <entity name=\"child\" query=\"select * from foo\" transformer=\"TestErrorHandling$ExceptionTransformer\" onError=\"continue\">\n" +
- " </entity>" +
- " </entity>\n" +
- " </document>\n" +
- "</dataConfig>";
-
- private String dataConfigWithEntity = "<!DOCTYPE dataConfig [\n" +
- " <!ENTITY internalTerm \"node\">\n" +
- " <!ENTITY externalTerm SYSTEM \"foo://bar.xyz/external\">\n" +
- "]><dataConfig>\n" +
- " <dataSource name=\"str\" type=\"TestErrorHandling$StringDataSource\" />" +
- " <document>\n" +
- " <entity name=\"&internalTerm;\" dataSource=\"str\" processor=\"XPathEntityProcessor\" url=\"test\" forEach=\"/root/node\" onError=\"skip\">\n" +
- " <field column=\"id\" xpath=\"/root/node/id\">&externalTerm;</field>\n" +
- " <field column=\"desc\" xpath=\"/root/node/desc\" />\n" +
- " </entity>\n" +
- " </document>\n" +
- "</dataConfig>";
-
- private String malformedXml = "<root>\n" +
- " <node>\n" +
- " <id>1</id>\n" +
- " <desc>test1</desc>\n" +
- " </node>\n" +
- " <node>\n" +
- " <id>2</id>\n" +
- " <desc>test2</desc>\n" +
- " </node>\n" +
- " <node>\n" +
- " <id/>3</id>\n" +
- " <desc>test3</desc>\n" +
- " </node>\n" +
- "</root>";
-
- private String wellformedXml = "<root>\n" +
- " <node>\n" +
- " <id>1</id>\n" +
- " <desc>test1</desc>\n" +
- " </node>\n" +
- " <node>\n" +
- " <id>2</id>\n" +
- " <desc>test2</desc>\n" +
- " </node>\n" +
- " <node>\n" +
- " <id>3</id>\n" +
- " <desc>test3</desc>\n" +
- " </node>\n" +
- "</root>";
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestFieldReader.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestFieldReader.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestFieldReader.java
deleted file mode 100644
index 347a40b..0000000
--- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestFieldReader.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import org.junit.Test;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Test for FieldReaderDataSource
- *
- *
- * @see org.apache.solr.handler.dataimport.FieldReaderDataSource
- * @since 1.4
- */
-public class TestFieldReader extends AbstractDataImportHandlerTestCase {
-
- @Test
- public void simple() {
- DataImporter di = new DataImporter();
- di.loadAndInit(config);
- redirectTempProperties(di);
-
- TestDocBuilder.SolrWriterImpl sw = new TestDocBuilder.SolrWriterImpl();
- RequestInfo rp = new RequestInfo(null, createMap("command", "full-import"), null);
- List<Map<String, Object>> l = new ArrayList<>();
- l.add(createMap("xml", xml));
- MockDataSource.setIterator("select * from a", l.iterator());
- di.runCmd(rp, sw);
- assertEquals(sw.docs.get(0).getFieldValue("y"), "Hello");
- MockDataSource.clearCache();
- }
-
- String config = "<dataConfig>\n" +
- " <dataSource type=\"FieldReaderDataSource\" name=\"f\"/>\n" +
- " <dataSource type=\"MockDataSource\"/>\n" +
- " <document>\n" +
- " <entity name=\"a\" query=\"select * from a\" >\n" +
- " <entity name=\"b\" dataSource=\"f\" processor=\"XPathEntityProcessor\" forEach=\"/x\" dataField=\"a.xml\">\n" +
- " <field column=\"y\" xpath=\"/x/y\"/>\n" +
- " </entity>\n" +
- " </entity>\n" +
- " </document>\n" +
- "</dataConfig>";
-
- String xml = "<x>\n" +
- " <y>Hello</y>\n" +
- "</x>";
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestFileListEntityProcessor.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestFileListEntityProcessor.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestFileListEntityProcessor.java
deleted file mode 100644
index dd2cf72..0000000
--- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestFileListEntityProcessor.java
+++ /dev/null
@@ -1,188 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Date;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.solr.common.util.SuppressForbidden;
-import org.junit.Test;
-
-/**
- * <p>
- * Test for FileListEntityProcessor
- * </p>
- *
- *
- * @since solr 1.3
- */
-public class TestFileListEntityProcessor extends AbstractDataImportHandlerTestCase {
-
- @Test
- @SuppressWarnings("unchecked")
- public void testSimple() throws IOException {
- File tmpdir = createTempDir().toFile();
-
- createFile(tmpdir, "a.xml", "a.xml".getBytes(StandardCharsets.UTF_8), false);
- createFile(tmpdir, "b.xml", "b.xml".getBytes(StandardCharsets.UTF_8), false);
- createFile(tmpdir, "c.props", "c.props".getBytes(StandardCharsets.UTF_8), false);
- Map attrs = createMap(
- FileListEntityProcessor.FILE_NAME, "xml$",
- FileListEntityProcessor.BASE_DIR, tmpdir.getAbsolutePath());
- Context c = getContext(null,
- new VariableResolver(), null, Context.FULL_DUMP, Collections.EMPTY_LIST, attrs);
- FileListEntityProcessor fileListEntityProcessor = new FileListEntityProcessor();
- fileListEntityProcessor.init(c);
- List<String> fList = new ArrayList<>();
- while (true) {
- Map<String, Object> f = fileListEntityProcessor.nextRow();
- if (f == null)
- break;
- fList.add((String) f.get(FileListEntityProcessor.ABSOLUTE_FILE));
- }
- assertEquals(2, fList.size());
- }
-
- @Test
- public void testBiggerSmallerFiles() throws IOException {
- File tmpdir = createTempDir().toFile();
-
- long minLength = Long.MAX_VALUE;
- String smallestFile = "";
- byte[] content = "abcdefgij".getBytes(StandardCharsets.UTF_8);
- createFile(tmpdir, "a.xml", content, false);
- if (minLength > content.length) {
- minLength = content.length;
- smallestFile = "a.xml";
- }
- content = "abcdefgij".getBytes(StandardCharsets.UTF_8);
- createFile(tmpdir, "b.xml", content, false);
- if (minLength > content.length) {
- minLength = content.length;
- smallestFile = "b.xml";
- }
- content = "abc".getBytes(StandardCharsets.UTF_8);
- createFile(tmpdir, "c.props", content, false);
- if (minLength > content.length) {
- minLength = content.length;
- smallestFile = "c.props";
- }
- Map attrs = createMap(
- FileListEntityProcessor.FILE_NAME, ".*",
- FileListEntityProcessor.BASE_DIR, tmpdir.getAbsolutePath(),
- FileListEntityProcessor.BIGGER_THAN, String.valueOf(minLength));
- List<String> fList = getFiles(null, attrs);
- assertEquals(2, fList.size());
- Set<String> l = new HashSet<>();
- l.add(new File(tmpdir, "a.xml").getAbsolutePath());
- l.add(new File(tmpdir, "b.xml").getAbsolutePath());
- assertEquals(l, new HashSet<>(fList));
- attrs = createMap(
- FileListEntityProcessor.FILE_NAME, ".*",
- FileListEntityProcessor.BASE_DIR, tmpdir.getAbsolutePath(),
- FileListEntityProcessor.SMALLER_THAN, String.valueOf(minLength+1));
- fList = getFiles(null, attrs);
- l.clear();
- l.add(new File(tmpdir, smallestFile).getAbsolutePath());
- assertEquals(l, new HashSet<>(fList));
- attrs = createMap(
- FileListEntityProcessor.FILE_NAME, ".*",
- FileListEntityProcessor.BASE_DIR, tmpdir.getAbsolutePath(),
- FileListEntityProcessor.SMALLER_THAN, "${a.x}");
- VariableResolver resolver = new VariableResolver();
- resolver.addNamespace("a", createMap("x", "4"));
- fList = getFiles(resolver, attrs);
- assertEquals(l, new HashSet<>(fList));
- }
-
- @SuppressWarnings("unchecked")
- static List<String> getFiles(VariableResolver resolver, Map attrs) {
- Context c = getContext(null,
- resolver, null, Context.FULL_DUMP, Collections.EMPTY_LIST, attrs);
- FileListEntityProcessor fileListEntityProcessor = new FileListEntityProcessor();
- fileListEntityProcessor.init(c);
- List<String> fList = new ArrayList<>();
- while (true) {
- Map<String, Object> f = fileListEntityProcessor.nextRow();
- if (f == null)
- break;
- fList.add((String) f.get(FileListEntityProcessor.ABSOLUTE_FILE));
- }
- return fList;
- }
-
- @SuppressForbidden(reason = "Needs currentTimeMillis to set last modified time")
- @Test
- public void testNTOT() throws IOException {
- File tmpdir = createTempDir().toFile();
-
- createFile(tmpdir, "a.xml", "a.xml".getBytes(StandardCharsets.UTF_8), true);
- createFile(tmpdir, "b.xml", "b.xml".getBytes(StandardCharsets.UTF_8), true);
- createFile(tmpdir, "c.props", "c.props".getBytes(StandardCharsets.UTF_8), true);
- Map attrs = createMap(
- FileListEntityProcessor.FILE_NAME, "xml$",
- FileListEntityProcessor.BASE_DIR, tmpdir.getAbsolutePath(),
- FileListEntityProcessor.OLDER_THAN, "'NOW'");
- List<String> fList = getFiles(null, attrs);
- assertEquals(2, fList.size());
- attrs = createMap(
- FileListEntityProcessor.FILE_NAME, ".xml$",
- FileListEntityProcessor.BASE_DIR, tmpdir.getAbsolutePath(),
- FileListEntityProcessor.NEWER_THAN, "'NOW-2HOURS'");
- fList = getFiles(null, attrs);
- assertEquals(2, fList.size());
-
- // Use a variable for newerThan
- attrs = createMap(
- FileListEntityProcessor.FILE_NAME, ".xml$",
- FileListEntityProcessor.BASE_DIR, tmpdir.getAbsolutePath(),
- FileListEntityProcessor.NEWER_THAN, "${a.x}");
- VariableResolver resolver = new VariableResolver();
- String lastMod = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.ROOT).format(new Date(System.currentTimeMillis() - 50000));
- resolver.addNamespace("a", createMap("x", lastMod));
- createFile(tmpdir, "t.xml", "t.xml".getBytes(StandardCharsets.UTF_8), false);
- fList = getFiles(resolver, attrs);
- assertEquals(1, fList.size());
- assertEquals("File name must be t.xml", new File(tmpdir, "t.xml").getAbsolutePath(), fList.get(0));
- }
-
- @Test
- public void testRECURSION() throws IOException {
- File tmpdir = createTempDir().toFile();
- File childdir = new File(tmpdir + "/child" );
- childdir.mkdir();
- createFile(childdir, "a.xml", "a.xml".getBytes(StandardCharsets.UTF_8), true);
- createFile(childdir, "b.xml", "b.xml".getBytes(StandardCharsets.UTF_8), true);
- createFile(childdir, "c.props", "c.props".getBytes(StandardCharsets.UTF_8), true);
- Map attrs = createMap(
- FileListEntityProcessor.FILE_NAME, "^.*\\.xml$",
- FileListEntityProcessor.BASE_DIR, childdir.getAbsolutePath(),
- FileListEntityProcessor.RECURSIVE, "true");
- List<String> fList = getFiles(null, attrs);
- assertEquals(2, fList.size());
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestFileListWithLineEntityProcessor.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestFileListWithLineEntityProcessor.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestFileListWithLineEntityProcessor.java
deleted file mode 100644
index aad8e30..0000000
--- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestFileListWithLineEntityProcessor.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import java.io.File;
-import java.nio.charset.StandardCharsets;
-
-import org.apache.lucene.util.LuceneTestCase;
-import org.apache.solr.request.LocalSolrQueryRequest;
-import org.junit.BeforeClass;
-
-public class TestFileListWithLineEntityProcessor extends AbstractDataImportHandlerTestCase {
- @BeforeClass
- public static void beforeClass() throws Exception {
- initCore("dataimport-solrconfig.xml", "dataimport-schema.xml");
- }
-
- public void test() throws Exception {
- File tmpdir = createTempDir(LuceneTestCase.getTestClass().getSimpleName()).toFile();
- createFile(tmpdir, "a.txt", "a line one\na line two\na line three".getBytes(StandardCharsets.UTF_8), false);
- createFile(tmpdir, "b.txt", "b line one\nb line two".getBytes(StandardCharsets.UTF_8), false);
- createFile(tmpdir, "c.txt", "c line one\nc line two\nc line three\nc line four".getBytes(StandardCharsets.UTF_8), false);
-
- String config = generateConfig(tmpdir);
- LocalSolrQueryRequest request = lrf.makeRequest(
- "command", "full-import", "dataConfig", config,
- "clean", "true", "commit", "true", "synchronous", "true", "indent", "true");
- h.query("/dataimport", request);
-
- assertQ(req("*:*"), "//*[@numFound='9']");
- assertQ(req("id:?\\ line\\ one"), "//*[@numFound='3']");
- assertQ(req("id:a\\ line*"), "//*[@numFound='3']");
- assertQ(req("id:b\\ line*"), "//*[@numFound='2']");
- assertQ(req("id:c\\ line*"), "//*[@numFound='4']");
- }
-
- private String generateConfig(File dir) {
- return
- "<dataConfig> \n"+
- "<dataSource type=\"FileDataSource\" encoding=\"UTF-8\" name=\"fds\"/> \n"+
- " <document> \n"+
- " <entity name=\"f\" processor=\"FileListEntityProcessor\" fileName=\".*[.]txt\" baseDir=\"" + dir.getAbsolutePath() + "\" recursive=\"false\" rootEntity=\"false\" transformer=\"TemplateTransformer\"> \n" +
- " <entity name=\"jc\" processor=\"LineEntityProcessor\" url=\"${f.fileAbsolutePath}\" dataSource=\"fds\" rootEntity=\"true\" transformer=\"TemplateTransformer\"> \n" +
- " <field column=\"rawLine\" name=\"id\" /> \n" +
- " </entity> \n"+
- " </entity> \n"+
- " </document> \n"+
- "</dataConfig> \n";
- }
-}