You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by da...@apache.org on 2018/11/02 11:33:31 UTC
[10/25] lucene-solr:jira/gradle: Adding dataimporthandler-extras
module
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/MockInitialContextFactory.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/MockInitialContextFactory.java b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/MockInitialContextFactory.java
new file mode 100644
index 0000000..7568210
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/MockInitialContextFactory.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import java.util.HashMap;
+import java.util.Hashtable;
+import java.util.Map;
+
+import javax.naming.NamingException;
+import javax.naming.spi.InitialContextFactory;
+
+import static org.mockito.Mockito.*;
+
+public class MockInitialContextFactory implements InitialContextFactory {
+ private static final Map<String, Object> objects = new HashMap<>();
+ private final javax.naming.Context context;
+
+ public MockInitialContextFactory() {
+ context = mock(javax.naming.Context.class);
+
+ try {
+ when(context.lookup(anyString())).thenAnswer(invocation -> objects.get(invocation.getArgument(0)));
+
+ } catch (NamingException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public javax.naming.Context getInitialContext(Hashtable env) {
+ return context;
+ }
+
+ public static void bind(String name, Object obj) {
+ objects.put(name, obj);
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/MockSolrEntityProcessor.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/MockSolrEntityProcessor.java b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/MockSolrEntityProcessor.java
new file mode 100644
index 0000000..42e5f7d
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/MockSolrEntityProcessor.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrDocumentList;
+
+import java.util.List;
+
+public class MockSolrEntityProcessor extends SolrEntityProcessor {
+
+ private final List<SolrTestCaseJ4.Doc> docsData;
+// private final int rows;
+ private int queryCount = 0;
+
+ private int rows;
+
+ private int start = 0;
+
+ public MockSolrEntityProcessor(List<SolrTestCaseJ4.Doc> docsData, int rows) {
+ this.docsData = docsData;
+ this.rows = rows;
+ }
+
+ //@Override
+ //protected SolrDocumentList doQuery(int start) {
+ // queryCount++;
+ // return getDocs(start, rows);
+ // }
+
+ @Override
+ protected void buildIterator() {
+ if (rowIterator==null || (!rowIterator.hasNext() && ((SolrDocumentListIterator)rowIterator).hasMoreRows())){
+ queryCount++;
+ SolrDocumentList docs = getDocs(start, rows);
+ rowIterator = new SolrDocumentListIterator(docs);
+ start += docs.size();
+ }
+ }
+
+ private SolrDocumentList getDocs(int start, int rows) {
+ SolrDocumentList docs = new SolrDocumentList();
+ docs.setNumFound(docsData.size());
+ docs.setStart(start);
+
+ int endIndex = start + rows;
+ int end = docsData.size() < endIndex ? docsData.size() : endIndex;
+ for (int i = start; i < end; i++) {
+ SolrDocument doc = new SolrDocument();
+ SolrTestCaseJ4.Doc testDoc = docsData.get(i);
+ doc.addField("id", testDoc.id);
+ doc.addField("description", testDoc.getValues("description"));
+ docs.add(doc);
+ }
+ return docs;
+ }
+
+ public int getQueryCount() {
+ return queryCount;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/MockStringDataSource.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/MockStringDataSource.java b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/MockStringDataSource.java
new file mode 100644
index 0000000..7c9a6d1
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/MockStringDataSource.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.dataimport;
+
+
+import java.io.Reader;
+import java.io.StringReader;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+
+public class MockStringDataSource extends DataSource<Reader> {
+
+ private static Map<String, String> cache = new HashMap<>();
+
+ public static void setData(String query,
+ String data) {
+ cache.put(query, data);
+ }
+
+ public static void clearCache() {
+ cache.clear();
+ }
+ @Override
+ public void init(Context context, Properties initProps) {
+
+ }
+
+ @Override
+ public Reader getData(String query) {
+ return new StringReader(cache.get(query));
+ }
+
+ @Override
+ public void close() {
+ cache.clear();
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestBuiltInEvaluators.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestBuiltInEvaluators.java b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestBuiltInEvaluators.java
new file mode 100644
index 0000000..986a8cd
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestBuiltInEvaluators.java
@@ -0,0 +1,188 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
+import java.text.SimpleDateFormat;
+import java.util.*;
+
+/**
+ * <p> Test for Evaluators </p>
+ *
+ *
+ * @since solr 1.3
+ */
+public class TestBuiltInEvaluators extends AbstractDataImportHandlerTestCase {
+ private static final String ENCODING = StandardCharsets.UTF_8.name();
+
+ VariableResolver resolver;
+
+ Map<String, String> sqlTests;
+
+ Map<String, String> urlTests;
+
+ @Override
+ @Before
+ public void setUp() throws Exception {
+ super.setUp();
+ resolver = new VariableResolver();
+
+ sqlTests = new HashMap<>();
+
+ sqlTests.put("foo\"", "foo\"\"");
+ sqlTests.put("foo\\", "foo\\\\");
+ sqlTests.put("foo'", "foo''");
+ sqlTests.put("foo''", "foo''''");
+ sqlTests.put("'foo\"", "''foo\"\"");
+ sqlTests.put("\"Albert D'souza\"", "\"\"Albert D''souza\"\"");
+
+ urlTests = new HashMap<>();
+
+ urlTests.put("*:*", URLEncoder.encode("*:*", ENCODING));
+ urlTests.put("price:[* TO 200]", URLEncoder.encode("price:[* TO 200]",
+ ENCODING));
+ urlTests.put("review:\"hybrid sedan\"", URLEncoder.encode(
+ "review:\"hybrid sedan\"", ENCODING));
+ }
+
+
+ @Test
+ public void testSqlEscapingEvaluator() {
+ Evaluator sqlEscaper = new SqlEscapingEvaluator();
+ runTests(sqlTests, sqlEscaper);
+ }
+
+
+ @Test
+ public void testUrlEvaluator() throws Exception {
+ Evaluator urlEvaluator = new UrlEvaluator();
+ runTests(urlTests, urlEvaluator);
+ }
+
+ @Test
+ public void parseParams() {
+ Map<String,Object> m = new HashMap<>();
+ m.put("b","B");
+ VariableResolver vr = new VariableResolver();
+ vr.addNamespace("a",m);
+ List<Object> l = (new Evaluator() {
+ @Override
+ public String evaluate(String expression, Context context) {
+ return null;
+ }
+ }).parseParams(" 1 , a.b, 'hello!', 'ds,o,u\'za',",vr);
+ assertEquals(1d,l.get(0));
+ assertEquals("B",((Evaluator.VariableWrapper)l.get(1)).resolve());
+ assertEquals("hello!",l.get(2));
+ assertEquals("ds,o,u'za",l.get(3));
+ }
+
+ @Test
+ public void testEscapeSolrQueryFunction() {
+ final VariableResolver resolver = new VariableResolver();
+ Map<String,Object> m= new HashMap<>();
+ m.put("query","c:t");
+ resolver.setEvaluators(new DataImporter().getEvaluators(Collections.<Map<String,String>>emptyList()));
+
+ resolver.addNamespace("e",m);
+ String s = resolver
+ .replaceTokens("${dataimporter.functions.escapeQueryChars(e.query)}");
+ org.junit.Assert.assertEquals("c\\:t", s);
+
+ }
+
+ private Date twoDaysAgo(Locale l, TimeZone tz) {
+ Calendar calendar = Calendar.getInstance(tz, l);
+ calendar.add(Calendar.DAY_OF_YEAR, -2);
+ return calendar.getTime();
+ }
+
+ @Test
+ public void testDateFormatEvaluator() {
+ Evaluator dateFormatEval = new DateFormatEvaluator();
+ ContextImpl context = new ContextImpl(null, resolver, null,
+ Context.FULL_DUMP, Collections.<String,Object> emptyMap(), null, null);
+
+ Locale rootLocale = Locale.ROOT;
+ Locale defaultLocale = Locale.getDefault();
+ TimeZone defaultTz = TimeZone.getDefault();
+
+ {
+ SimpleDateFormat sdfDate = new SimpleDateFormat("yyyy-MM-dd HH", rootLocale);
+ String sdf = sdfDate.format(twoDaysAgo(rootLocale, defaultTz));
+ String dfe = dateFormatEval.evaluate("'NOW-2DAYS','yyyy-MM-dd HH'", context);
+ assertEquals(sdf,dfe);
+ }
+ {
+ SimpleDateFormat sdfDate = new SimpleDateFormat("yyyy-MM-dd HH", defaultLocale);
+ String sdf = sdfDate.format(twoDaysAgo(defaultLocale, TimeZone.getDefault()));
+ String dfe = dateFormatEval.evaluate(
+ "'NOW-2DAYS','yyyy-MM-dd HH','" + defaultLocale.toLanguageTag() + "'", context);
+ assertEquals(sdf,dfe);
+ for(String tzStr : TimeZone.getAvailableIDs()) {
+ TimeZone tz = TimeZone.getTimeZone(tzStr);
+ sdfDate.setTimeZone(tz);
+ sdf = sdfDate.format(twoDaysAgo(defaultLocale, tz));
+ dfe = dateFormatEval.evaluate(
+ "'NOW-2DAYS','yyyy-MM-dd HH','" + defaultLocale.toLanguageTag() + "','" + tzStr + "'", context);
+ assertEquals(sdf,dfe);
+ }
+ }
+
+ Date d = new Date();
+ Map<String,Object> map = new HashMap<>();
+ map.put("key", d);
+ resolver.addNamespace("A", map);
+
+ assertEquals(
+ new SimpleDateFormat("yyyy-MM-dd HH:mm", rootLocale).format(d),
+ dateFormatEval.evaluate("A.key, 'yyyy-MM-dd HH:mm'", context));
+ assertEquals(
+ new SimpleDateFormat("yyyy-MM-dd HH:mm", defaultLocale).format(d),
+ dateFormatEval.evaluate("A.key, 'yyyy-MM-dd HH:mm','" + defaultLocale.toLanguageTag() + "'", context));
+ SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm", defaultLocale);
+ for(String tzStr : TimeZone.getAvailableIDs()) {
+ TimeZone tz = TimeZone.getTimeZone(tzStr);
+ sdf.setTimeZone(tz);
+ assertEquals(
+ sdf.format(d),
+ dateFormatEval.evaluate(
+ "A.key, 'yyyy-MM-dd HH:mm','" + defaultLocale.toLanguageTag() + "', '" + tzStr + "'", context));
+
+ }
+
+
+ }
+
+ private void runTests(Map<String, String> tests, Evaluator evaluator) {
+ ContextImpl ctx = new ContextImpl(null, resolver, null, Context.FULL_DUMP, Collections.<String, Object>emptyMap(), null, null);
+ for (Map.Entry<String, String> entry : tests.entrySet()) {
+ Map<String, Object> values = new HashMap<>();
+ values.put("key", entry.getKey());
+ resolver.addNamespace("A", values);
+
+ String expected = entry.getValue();
+ String actual = evaluator.evaluate("A.key", ctx);
+ assertEquals(expected, actual);
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestClobTransformer.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestClobTransformer.java b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestClobTransformer.java
new file mode 100644
index 0000000..fe00d49
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestClobTransformer.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import org.junit.Test;
+
+import java.io.StringReader;
+import java.lang.reflect.InvocationHandler;
+import java.lang.reflect.Method;
+import java.lang.reflect.Proxy;
+import java.sql.Clob;
+import java.util.*;
+
+/**
+ * Test for ClobTransformer
+ *
+ *
+ * @see org.apache.solr.handler.dataimport.ClobTransformer
+ * @since solr 1.4
+ */
+public class TestClobTransformer extends AbstractDataImportHandlerTestCase {
+ @Test
+ public void simple() throws Exception {
+ List<Map<String, String>> flds = new ArrayList<>();
+ Map<String, String> f = new HashMap<>();
+ // <field column="dsc" clob="true" name="description" />
+ f.put(DataImporter.COLUMN, "dsc");
+ f.put(ClobTransformer.CLOB, "true");
+ f.put(DataImporter.NAME, "description");
+ flds.add(f);
+ Context ctx = getContext(null, new VariableResolver(), null, Context.FULL_DUMP, flds, Collections.EMPTY_MAP);
+ Transformer t = new ClobTransformer();
+ Map<String, Object> row = new HashMap<>();
+ Clob clob = (Clob) Proxy.newProxyInstance(this.getClass().getClassLoader(), new Class[]{Clob.class}, new InvocationHandler() {
+ @Override
+ public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
+ if (method.getName().equals("getCharacterStream")) {
+ return new StringReader("hello!");
+ }
+ return null;
+ }
+ });
+
+ row.put("dsc", clob);
+ t.transformRow(row, ctx);
+ assertEquals("hello!", row.get("dsc"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestContentStreamDataSource.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestContentStreamDataSource.java b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestContentStreamDataSource.java
new file mode 100644
index 0000000..06fd51c
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestContentStreamDataSource.java
@@ -0,0 +1,193 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.solr.client.solrj.embedded.JettySolrRunner;
+import org.apache.solr.client.solrj.impl.HttpSolrClient;
+import org.apache.solr.client.solrj.request.DirectXmlRequest;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.UpdateParams;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.nio.file.Files;
+import java.util.List;
+import java.util.Properties;
+
+/**
+ * Test for ContentStreamDataSource
+ *
+ *
+ * @since solr 1.4
+ */
+public class TestContentStreamDataSource extends AbstractDataImportHandlerTestCase {
+ private static final String CONF_DIR = "dih/solr/collection1/conf/";
+ private static final String ROOT_DIR = "dih/solr/";
+ SolrInstance instance = null;
+ JettySolrRunner jetty;
+
+ @Override
+ @Before
+ public void setUp() throws Exception {
+ super.setUp();
+ instance = new SolrInstance("inst", null);
+ instance.setUp();
+ jetty = createJetty(instance);
+ }
+
+ @Override
+ @After
+ public void tearDown() throws Exception {
+ jetty.stop();
+ super.tearDown();
+ }
+
+ @Test
+ public void testSimple() throws Exception {
+ DirectXmlRequest req = new DirectXmlRequest("/dataimport", xml);
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.set("command", "full-import");
+ params.set("clean", "false");
+ req.setParams(params);
+ try (HttpSolrClient solrClient = getHttpSolrClient(buildUrl(jetty.getLocalPort(), "/solr/collection1"))) {
+ solrClient.request(req);
+ ModifiableSolrParams qparams = new ModifiableSolrParams();
+ qparams.add("q", "*:*");
+ QueryResponse qres = solrClient.query(qparams);
+ SolrDocumentList results = qres.getResults();
+ assertEquals(2, results.getNumFound());
+ SolrDocument doc = results.get(0);
+ assertEquals("1", doc.getFieldValue("id"));
+ assertEquals("Hello C1", ((List) doc.getFieldValue("desc")).get(0));
+ }
+ }
+
+ @Test
+ public void testCommitWithin() throws Exception {
+ DirectXmlRequest req = new DirectXmlRequest("/dataimport", xml);
+ ModifiableSolrParams params = params("command", "full-import",
+ "clean", "false", UpdateParams.COMMIT, "false",
+ UpdateParams.COMMIT_WITHIN, "1000");
+ req.setParams(params);
+ try (HttpSolrClient solrServer = getHttpSolrClient(buildUrl(jetty.getLocalPort(), "/solr/collection1"))) {
+ solrServer.request(req);
+ Thread.sleep(100);
+ ModifiableSolrParams queryAll = params("q", "*", "df", "desc");
+ QueryResponse qres = solrServer.query(queryAll);
+ SolrDocumentList results = qres.getResults();
+ assertEquals(0, results.getNumFound());
+ Thread.sleep(1000);
+ for (int i = 0; i < 10; i++) {
+ qres = solrServer.query(queryAll);
+ results = qres.getResults();
+ if (2 == results.getNumFound()) {
+ return;
+ }
+ Thread.sleep(500);
+ }
+ }
+ fail("Commit should have occured but it did not");
+ }
+
+ private static class SolrInstance {
+ String name;
+ Integer port;
+ File homeDir;
+ File confDir;
+ File dataDir;
+
+ /**
+ * if masterPort is null, this instance is a master -- otherwise this instance is a slave, and assumes the master is
+ * on localhost at the specified port.
+ */
+ public SolrInstance(String name, Integer port) {
+ this.name = name;
+ this.port = port;
+ }
+
+ public String getHomeDir() {
+ return homeDir.toString();
+ }
+
+ public String getSchemaFile() {
+ return CONF_DIR + "dataimport-schema.xml";
+ }
+
+ public String getConfDir() {
+ return confDir.toString();
+ }
+
+ public String getDataDir() {
+ return dataDir.toString();
+ }
+
+ public String getSolrConfigFile() {
+ return CONF_DIR + "contentstream-solrconfig.xml";
+ }
+
+ public String getSolrXmlFile() {
+ return ROOT_DIR + "solr.xml";
+ }
+
+
+ public void setUp() throws Exception {
+ homeDir = createTempDir("inst").toFile();
+ dataDir = new File(homeDir + "/collection1", "data");
+ confDir = new File(homeDir + "/collection1", "conf");
+
+ homeDir.mkdirs();
+ dataDir.mkdirs();
+ confDir.mkdirs();
+
+ FileUtils.copyFile(getFile(getSolrXmlFile()), new File(homeDir, "solr.xml"));
+ File f = new File(confDir, "solrconfig.xml");
+ FileUtils.copyFile(getFile(getSolrConfigFile()), f);
+ f = new File(confDir, "schema.xml");
+
+ FileUtils.copyFile(getFile(getSchemaFile()), f);
+ f = new File(confDir, "data-config.xml");
+ FileUtils.copyFile(getFile(CONF_DIR + "dataconfig-contentstream.xml"), f);
+
+ Files.createFile(homeDir.toPath().resolve("collection1/core.properties"));
+ }
+
+ }
+
+ private JettySolrRunner createJetty(SolrInstance instance) throws Exception {
+ Properties nodeProperties = new Properties();
+ nodeProperties.setProperty("solr.data.dir", instance.getDataDir());
+ JettySolrRunner jetty = new JettySolrRunner(instance.getHomeDir(), nodeProperties, buildJettyConfig("/solr"));
+ jetty.start();
+ return jetty;
+ }
+
+ static String xml = "<root>\n"
+ + "<b>\n"
+ + " <id>1</id>\n"
+ + " <c>Hello C1</c>\n"
+ + "</b>\n"
+ + "<b>\n"
+ + " <id>2</id>\n"
+ + " <c>Hello C2</c>\n"
+ + "</b>\n" + "</root>";
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestContextImpl.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestContextImpl.java b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestContextImpl.java
new file mode 100644
index 0000000..4193550
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestContextImpl.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import java.util.HashMap;
+
+import org.apache.solr.handler.dataimport.RequestInfo;
+import org.junit.Test;
+
+public class TestContextImpl extends AbstractDataImportHandlerTestCase {
+
+ @Test
+ public void testEntityScope() {
+ ContextImpl ctx = new ContextImpl(null, new VariableResolver(), null, "something", new HashMap<String,Object>(), null, null);
+ String lala = new String("lala");
+ ctx.setSessionAttribute("huhu", lala, Context.SCOPE_ENTITY);
+ Object got = ctx.getSessionAttribute("huhu", Context.SCOPE_ENTITY);
+
+ assertEquals(lala, got);
+
+ }
+ @Test
+ public void testCoreScope() {
+ DataImporter di = new DataImporter();
+ di.loadAndInit("<dataConfig><document /></dataConfig>");
+ DocBuilder db = new DocBuilder(di, new SolrWriter(null, null),new SimplePropertiesWriter(), new RequestInfo(null, new HashMap<String,Object>(), null));
+ ContextImpl ctx = new ContextImpl(null, new VariableResolver(), null, "something", new HashMap<String,Object>(), null, db);
+ String lala = new String("lala");
+ ctx.setSessionAttribute("huhu", lala, Context.SCOPE_SOLR_CORE);
+ Object got = ctx.getSessionAttribute("huhu", Context.SCOPE_SOLR_CORE);
+ assertEquals(lala, got);
+
+ }
+ @Test
+ public void testDocumentScope() {
+ ContextImpl ctx = new ContextImpl(null, new VariableResolver(), null, "something", new HashMap<String,Object>(), null, null);
+ ctx.setDoc(new DocBuilder.DocWrapper());
+ String lala = new String("lala");
+ ctx.setSessionAttribute("huhu", lala, Context.SCOPE_DOC);
+ Object got = ctx.getSessionAttribute("huhu", Context.SCOPE_DOC);
+
+ assertEquals(lala, got);
+
+ }
+ @Test
+ public void testGlobalScope() {
+ ContextImpl ctx = new ContextImpl(null, new VariableResolver(), null, "something", new HashMap<String,Object>(), null, null);
+ String lala = new String("lala");
+ ctx.setSessionAttribute("huhu", lala, Context.SCOPE_GLOBAL);
+ Object got = ctx.getSessionAttribute("huhu", Context.SCOPE_GLOBAL);
+
+ assertEquals(lala, got);
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDataConfig.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDataConfig.java b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDataConfig.java
new file mode 100644
index 0000000..7a56fd2
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDataConfig.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import org.apache.solr.handler.dataimport.config.DIHConfiguration;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.w3c.dom.Document;
+import org.xml.sax.InputSource;
+
+import javax.xml.parsers.DocumentBuilderFactory;
+import java.io.StringReader;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * <p>
+ * Test for DataConfig
+ * </p>
+ *
+ *
+ * @since solr 1.3
+ */
+public class TestDataConfig extends AbstractDataImportHandlerTestCase {
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ initCore("dataimport-nodatasource-solrconfig.xml", "dataimport-schema.xml");
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testDataConfigWithDataSource() throws Exception {
+ List rows = new ArrayList();
+ rows.add(createMap("id", "1", "desc", "one"));
+ MockDataSource.setIterator("select * from x", rows.iterator());
+
+ runFullImport(loadDataConfig("data-config-with-datasource.xml"));
+
+ assertQ(req("id:1"), "//*[@numFound='1']");
+ }
+
+ @Test
+ public void testBasic() throws Exception {
+ javax.xml.parsers.DocumentBuilder builder = DocumentBuilderFactory
+ .newInstance().newDocumentBuilder();
+ Document doc = builder.parse(new InputSource(new StringReader(xml)));
+ DataImporter di = new DataImporter();
+ DIHConfiguration dc = di.readFromXml(doc);
+ assertEquals("atrimlisting", dc.getEntities().get(0).getName());
+ }
+
+ private static final String xml = "<dataConfig>\n"
+ + "\t<document name=\"autos\" >\n"
+ + "\t\t<entity name=\"atrimlisting\" pk=\"acode\"\n"
+ + "\t\t\tquery=\"select acode,make,model,year,msrp,category,image,izmo_image_url,price_range_low,price_range_high,invoice_range_low,invoice_range_high from atrimlisting\"\n"
+ + "\t\t\tdeltaQuery=\"select acode from atrimlisting where last_modified > '${indexer.last_index_time}'\">\n"
+ +
+
+ "\t\t</entity>\n" +
+
+ "\t</document>\n" + "</dataConfig>";
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDateFormatTransformer.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDateFormatTransformer.java b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDateFormatTransformer.java
new file mode 100644
index 0000000..a1e85d7
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDateFormatTransformer.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import org.junit.Test;
+
+import java.text.SimpleDateFormat;
+import java.util.*;
+
+/**
+ * <p>
+ * Test for DateFormatTransformer
+ * </p>
+ *
+ *
+ * @since solr 1.3
+ */
+public class TestDateFormatTransformer extends AbstractDataImportHandlerTestCase {
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testTransformRow_SingleRow() throws Exception {
+ List<Map<String, String>> fields = new ArrayList<>();
+ fields.add(createMap(DataImporter.COLUMN, "lastModified"));
+ fields.add(createMap(DataImporter.COLUMN,
+ "dateAdded", RegexTransformer.SRC_COL_NAME, "lastModified",
+ DateFormatTransformer.DATE_TIME_FMT, "${xyz.myDateFormat}"));
+
+ SimpleDateFormat format = new SimpleDateFormat("MM/dd/yyyy", Locale.ROOT);
+ Date now = format.parse(format.format(new Date()));
+
+ Map<String,Object> row = createMap("lastModified", format.format(now));
+
+ VariableResolver resolver = new VariableResolver();
+ resolver.addNamespace("e", row);
+ resolver.addNamespace("xyz", createMap("myDateFormat", "MM/dd/yyyy"));
+
+ Context context = getContext(null, resolver,
+ null, Context.FULL_DUMP, fields, null);
+ new DateFormatTransformer().transformRow(row, context);
+ assertEquals(now, row.get("dateAdded"));
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testTransformRow_MultipleRows() throws Exception {
+ List<Map<String, String>> fields = new ArrayList<>();
+ fields.add(createMap(DataImporter.COLUMN, "lastModified"));
+ fields.add(createMap(DataImporter.COLUMN,
+ "dateAdded", RegexTransformer.SRC_COL_NAME, "lastModified",
+ DateFormatTransformer.DATE_TIME_FMT, "MM/dd/yyyy hh:mm:ss.SSS"));
+
+ SimpleDateFormat format = new SimpleDateFormat("MM/dd/yyyy hh:mm:ss.SSS", Locale.ROOT);
+ Date now1 = format.parse(format.format(new Date()));
+ Date now2 = format.parse(format.format(new Date()));
+
+ Map<String,Object> row = new HashMap<>();
+ List<String> list = new ArrayList<>();
+ list.add(format.format(now1));
+ list.add(format.format(now2));
+ row.put("lastModified", list);
+
+ VariableResolver resolver = new VariableResolver();
+ resolver.addNamespace("e", row);
+
+ Context context = getContext(null, resolver,
+ null, Context.FULL_DUMP, fields, null);
+ new DateFormatTransformer().transformRow(row, context);
+ List<Object> output = new ArrayList<>();
+ output.add(now1);
+ output.add(now2);
+ assertEquals(output, row.get("dateAdded"));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDocBuilder.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDocBuilder.java b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDocBuilder.java
new file mode 100644
index 0000000..7f7278e
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDocBuilder.java
@@ -0,0 +1,335 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.handler.dataimport.config.DIHConfiguration;
+import org.apache.solr.handler.dataimport.config.Entity;
+
+import org.junit.After;
+import org.junit.Test;
+
+import java.util.*;
+
+/**
+ * <p>
+ * Test for DocBuilder
+ * </p>
+ *
+ *
+ * @since solr 1.3
+ */
+// See: https://issues.apache.org/jira/browse/SOLR-12028 Tests cannot remove files on Windows machines occasionally
+public class TestDocBuilder extends AbstractDataImportHandlerTestCase {
+
+ @Override
+ @After
+ public void tearDown() throws Exception {
+ MockDataSource.clearCache();
+ MockStringDataSource.clearCache();
+ super.tearDown();
+ }
+
+ @Test
+ public void loadClass() throws Exception {
+ @SuppressWarnings("unchecked")
+ Class<Transformer> clz = DocBuilder.loadClass("RegexTransformer", null);
+ assertNotNull(clz);
+ }
+
+ @Test
+ public void singleEntityNoRows() {
+ DataImporter di = new DataImporter();
+ di.loadAndInit(dc_singleEntity);
+ DIHConfiguration cfg = di.getConfig();
+ Entity ent = cfg.getEntities().get(0);
+ MockDataSource.setIterator("select * from x", new ArrayList<Map<String, Object>>().iterator());
+ RequestInfo rp = new RequestInfo(null, createMap("command", "full-import"), null);
+ SolrWriterImpl swi = new SolrWriterImpl();
+ di.runCmd(rp, swi);
+ assertEquals(Boolean.TRUE, swi.deleteAllCalled);
+ assertEquals(Boolean.TRUE, swi.commitCalled);
+ assertEquals(Boolean.TRUE, swi.finishCalled);
+ assertEquals(0, swi.docs.size());
+ assertEquals(1, di.getDocBuilder().importStatistics.queryCount.get());
+ assertEquals(0, di.getDocBuilder().importStatistics.docCount.get());
+ assertEquals(0, di.getDocBuilder().importStatistics.rowsCount.get());
+ }
+
+ @Test
+ public void testDeltaImportNoRows_MustNotCommit() {
+ DataImporter di = new DataImporter();
+ di.loadAndInit(dc_deltaConfig);
+ redirectTempProperties(di);
+
+ DIHConfiguration cfg = di.getConfig();
+ Entity ent = cfg.getEntities().get(0);
+ MockDataSource.setIterator("select * from x", new ArrayList<Map<String, Object>>().iterator());
+ MockDataSource.setIterator("select id from x", new ArrayList<Map<String, Object>>().iterator());
+ RequestInfo rp = new RequestInfo(null, createMap("command", "delta-import"), null);
+ SolrWriterImpl swi = new SolrWriterImpl();
+ di.runCmd(rp, swi);
+ assertEquals(Boolean.FALSE, swi.deleteAllCalled);
+ assertEquals(Boolean.FALSE, swi.commitCalled);
+ assertEquals(Boolean.TRUE, swi.finishCalled);
+ assertEquals(0, swi.docs.size());
+ assertEquals(1, di.getDocBuilder().importStatistics.queryCount.get());
+ assertEquals(0, di.getDocBuilder().importStatistics.docCount.get());
+ assertEquals(0, di.getDocBuilder().importStatistics.rowsCount.get());
+ }
+
+ @Test
+ public void singleEntityOneRow() {
+ DataImporter di = new DataImporter();
+ di.loadAndInit(dc_singleEntity);
+ DIHConfiguration cfg = di.getConfig();
+ Entity ent = cfg.getEntities().get(0);
+ List<Map<String, Object>> l = new ArrayList<>();
+ l.add(createMap("id", 1, "desc", "one"));
+ MockDataSource.setIterator("select * from x", l.iterator());
+ RequestInfo rp = new RequestInfo(null, createMap("command", "full-import"), null);
+ SolrWriterImpl swi = new SolrWriterImpl();
+ di.runCmd(rp, swi);
+ assertEquals(Boolean.TRUE, swi.deleteAllCalled);
+ assertEquals(Boolean.TRUE, swi.commitCalled);
+ assertEquals(Boolean.TRUE, swi.finishCalled);
+ assertEquals(1, swi.docs.size());
+ assertEquals(1, di.getDocBuilder().importStatistics.queryCount.get());
+ assertEquals(1, di.getDocBuilder().importStatistics.docCount.get());
+ assertEquals(1, di.getDocBuilder().importStatistics.rowsCount.get());
+
+ for (int i = 0; i < l.size(); i++) {
+ Map<String, Object> map = l.get(i);
+ SolrInputDocument doc = swi.docs.get(i);
+ for (Map.Entry<String, Object> entry : map.entrySet()) {
+ assertEquals(entry.getValue(), doc.getFieldValue(entry.getKey()));
+ }
+ }
+ }
+
+ @Test
+ public void testImportCommand() {
+ DataImporter di = new DataImporter();
+ di.loadAndInit(dc_singleEntity);
+ DIHConfiguration cfg = di.getConfig();
+ Entity ent = cfg.getEntities().get(0);
+ List<Map<String, Object>> l = new ArrayList<>();
+ l.add(createMap("id", 1, "desc", "one"));
+ MockDataSource.setIterator("select * from x", l.iterator());
+ RequestInfo rp = new RequestInfo(null, createMap("command", "import"), null);
+ SolrWriterImpl swi = new SolrWriterImpl();
+ di.runCmd(rp, swi);
+ assertEquals(Boolean.FALSE, swi.deleteAllCalled);
+ assertEquals(Boolean.TRUE, swi.commitCalled);
+ assertEquals(Boolean.TRUE, swi.finishCalled);
+ assertEquals(1, swi.docs.size());
+ assertEquals(1, di.getDocBuilder().importStatistics.queryCount.get());
+ assertEquals(1, di.getDocBuilder().importStatistics.docCount.get());
+ assertEquals(1, di.getDocBuilder().importStatistics.rowsCount.get());
+
+ for (int i = 0; i < l.size(); i++) {
+ Map<String, Object> map = (Map<String, Object>) l.get(i);
+ SolrInputDocument doc = swi.docs.get(i);
+ for (Map.Entry<String, Object> entry : map.entrySet()) {
+ assertEquals(entry.getValue(), doc.getFieldValue(entry.getKey()));
+ }
+ }
+ }
+
+ @Test
+ public void singleEntityMultipleRows() {
+ DataImporter di = new DataImporter();
+ di.loadAndInit(dc_singleEntity);
+ DIHConfiguration cfg = di.getConfig();
+ Entity ent = cfg.getEntities().get(0);
+ RequestInfo rp = new RequestInfo(null, createMap("command", "full-import"), null);
+ List<Map<String, Object>> l = new ArrayList<>();
+ l.add(createMap("id", 1, "desc", "one"));
+ l.add(createMap("id", 2, "desc", "two"));
+ l.add(createMap("id", 3, "desc", "three"));
+
+ MockDataSource.setIterator("select * from x", l.iterator());
+ SolrWriterImpl swi = new SolrWriterImpl();
+ di.runCmd(rp, swi);
+ assertEquals(Boolean.TRUE, swi.deleteAllCalled);
+ assertEquals(Boolean.TRUE, swi.commitCalled);
+ assertEquals(Boolean.TRUE, swi.finishCalled);
+ assertEquals(3, swi.docs.size());
+ for (int i = 0; i < l.size(); i++) {
+ Map<String, Object> map = (Map<String, Object>) l.get(i);
+ SolrInputDocument doc = swi.docs.get(i);
+ for (Map.Entry<String, Object> entry : map.entrySet()) {
+ assertEquals(entry.getValue(), doc.getFieldValue(entry.getKey()));
+ }
+ assertEquals(map.get("desc"), doc.getFieldValue("desc_s"));
+ }
+ assertEquals(1, di.getDocBuilder().importStatistics.queryCount.get());
+ assertEquals(3, di.getDocBuilder().importStatistics.docCount.get());
+ assertEquals(3, di.getDocBuilder().importStatistics.rowsCount.get());
+ }
+
+ @Test
+ public void templateXPath() {
+ DataImporter di = new DataImporter();
+ di.loadAndInit(dc_variableXpath);
+ DIHConfiguration cfg = di.getConfig();
+
+ RequestInfo rp = new RequestInfo(null, createMap("command", "full-import"), null);
+ List<Map<String, Object>> l = new ArrayList<>();
+ l.add(createMap("id", 1, "name", "iphone", "manufacturer", "Apple"));
+ l.add(createMap("id", 2, "name", "ipad", "manufacturer", "Apple"));
+ l.add(createMap("id", 3, "name", "pixel", "manufacturer", "Google"));
+
+ MockDataSource.setIterator("select * from x", l.iterator());
+
+ List<Map<String,Object>> nestedData = new ArrayList<>();
+ nestedData.add(createMap("founded", "Cupertino, California, U.S", "year", "1976", "year2", "1976"));
+ nestedData.add(createMap("founded", "Cupertino, California, U.S", "year", "1976", "year2", "1976"));
+ nestedData.add(createMap("founded", "Menlo Park, California, U.S", "year", "1998", "year2", "1998"));
+
+ MockStringDataSource.setData("companies.xml", xml_attrVariableXpath);
+ MockStringDataSource.setData("companies2.xml", xml_variableXpath);
+ MockStringDataSource.setData("companies3.xml", xml_variableForEach);
+
+ SolrWriterImpl swi = new SolrWriterImpl();
+ di.runCmd(rp, swi);
+ assertEquals(Boolean.TRUE, swi.deleteAllCalled);
+ assertEquals(Boolean.TRUE, swi.commitCalled);
+ assertEquals(Boolean.TRUE, swi.finishCalled);
+ assertEquals(3, swi.docs.size());
+ for (int i = 0; i < l.size(); i++) {
+ SolrInputDocument doc = swi.docs.get(i);
+
+ Map<String, Object> map = l.get(i);
+ for (Map.Entry<String, Object> entry : map.entrySet()) {
+ assertEquals(entry.getValue(), doc.getFieldValue(entry.getKey()));
+ }
+
+ map = nestedData.get(i);
+ for (Map.Entry<String, Object> entry : map.entrySet()) {
+ assertEquals(entry.getValue(), doc.getFieldValue(entry.getKey()));
+ }
+ }
+ assertEquals(1, di.getDocBuilder().importStatistics.queryCount.get());
+ assertEquals(3, di.getDocBuilder().importStatistics.docCount.get());
+ }
+
+ static class SolrWriterImpl extends SolrWriter {
+ List<SolrInputDocument> docs = new ArrayList<>();
+
+ Boolean deleteAllCalled = Boolean.FALSE;
+
+ Boolean commitCalled = Boolean.FALSE;
+
+ Boolean finishCalled = Boolean.FALSE;
+
+ public SolrWriterImpl() {
+ super(null, null);
+ }
+
+ @Override
+ public boolean upload(SolrInputDocument doc) {
+ return docs.add(doc);
+ }
+
+ @Override
+ public void doDeleteAll() {
+ deleteAllCalled = Boolean.TRUE;
+ }
+
+ @Override
+ public void commit(boolean b) {
+ commitCalled = Boolean.TRUE;
+ }
+
+ @Override
+ public void close() {
+ finishCalled = Boolean.TRUE;
+ }
+ }
+
+ public static final String dc_singleEntity = "<dataConfig>\n"
+ + "<dataSource type=\"MockDataSource\"/>\n"
+ + " <document name=\"X\" >\n"
+ + " <entity name=\"x\" query=\"select * from x\">\n"
+ + " <field column=\"id\"/>\n"
+ + " <field column=\"desc\"/>\n"
+ + " <field column=\"desc\" name=\"desc_s\" />" + " </entity>\n"
+ + " </document>\n" + "</dataConfig>";
+
+ public static final String dc_deltaConfig = "<dataConfig>\n"
+ + "<dataSource type=\"MockDataSource\"/>\n"
+ + " <document name=\"X\" >\n"
+ + " <entity name=\"x\" query=\"select * from x\" deltaQuery=\"select id from x\">\n"
+ + " <field column=\"id\"/>\n"
+ + " <field column=\"desc\"/>\n"
+ + " <field column=\"desc\" name=\"desc_s\" />" + " </entity>\n"
+ + " </document>\n" + "</dataConfig>";
+
+ public static final String dc_variableXpath = "<dataConfig>\n"
+ + "<dataSource type=\"MockDataSource\"/>\n"
+ + "<dataSource name=\"xml\" type=\"MockStringDataSource\"/>\n"
+ + " <document name=\"X\" >\n"
+ + " <entity name=\"x\" query=\"select * from x\">\n"
+ + " <field column=\"id\"/>\n"
+ + " <field column=\"name\"/>\n"
+ + " <field column=\"manufacturer\"/>"
+ + " <entity name=\"c1\" url=\"companies.xml\" dataSource=\"xml\" forEach=\"/companies/company\" processor=\"XPathEntityProcessor\">"
+ + " <field column=\"year\" xpath=\"/companies/company/year[@name='p_${x.manufacturer}_s']\" />"
+ + " </entity>"
+ + " <entity name=\"c2\" url=\"companies2.xml\" dataSource=\"xml\" forEach=\"/companies/company\" processor=\"XPathEntityProcessor\">"
+ + " <field column=\"founded\" xpath=\"/companies/company/p_${x.manufacturer}_s/founded\" />"
+ + " </entity>"
+ + " <entity name=\"c3\" url=\"companies3.xml\" dataSource=\"xml\" forEach=\"/companies/${x.manufacturer}\" processor=\"XPathEntityProcessor\">"
+ + " <field column=\"year2\" xpath=\"/companies/${x.manufacturer}/year\" />"
+ + " </entity>"
+ + " </entity>\n"
+ + " </document>\n" + "</dataConfig>";
+
+
+ public static final String xml_variableForEach = "<companies>\n" +
+ "\t<Apple>\n" +
+ "\t\t<year>1976</year>\n" +
+ "\t</Apple>\n" +
+ "\t<Google>\n" +
+ "\t\t<year>1998</year>\n" +
+ "\t</Google>\n" +
+ "</companies>";
+
+ public static final String xml_variableXpath = "<companies>\n" +
+ "\t<company>\n" +
+ "\t\t<p_Apple_s>\n" +
+ "\t\t\t<founded>Cupertino, California, U.S</founded>\n" +
+ "\t\t</p_Apple_s>\t\t\n" +
+ "\t</company>\n" +
+ "\t<company>\n" +
+ "\t\t<p_Google_s>\n" +
+ "\t\t\t<founded>Menlo Park, California, U.S</founded>\n" +
+ "\t\t</p_Google_s>\n" +
+ "\t</company>\n" +
+ "</companies>";
+
+ public static final String xml_attrVariableXpath = "<companies>\n" +
+ "\t<company>\n" +
+ "\t\t<year name='p_Apple_s'>1976</year>\n" +
+ "\t</company>\n" +
+ "\t<company>\n" +
+ "\t\t<year name='p_Google_s'>1998</year>\t\t\n" +
+ "\t</company>\n" +
+ "</companies>";
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDocBuilder2.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDocBuilder2.java b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDocBuilder2.java
new file mode 100644
index 0000000..27865cd
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDocBuilder2.java
@@ -0,0 +1,433 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.solr.request.LocalSolrQueryRequest;
+import org.junit.BeforeClass;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import java.nio.charset.StandardCharsets;
+
+/**
+ * <p>
+ * Test for DocBuilder using the test harness
+ * </p>
+ *
+ *
+ * @since solr 1.3
+ */
+public class TestDocBuilder2 extends AbstractDataImportHandlerTestCase {
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ initCore("dataimport-solrconfig.xml", "dataimport-schema.xml");
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testSingleEntity() throws Exception {
+ List rows = new ArrayList();
+ rows.add(createMap("id", "1", "desc", "one"));
+ MockDataSource.setIterator("select * from x", rows.iterator());
+
+ runFullImport(loadDataConfig("single-entity-data-config.xml"));
+
+ assertQ(req("id:1"), "//*[@numFound='1']");
+
+ assertTrue("Update request processor processAdd was not called", TestUpdateRequestProcessor.processAddCalled);
+ assertTrue("Update request processor processCommit was not callled", TestUpdateRequestProcessor.processCommitCalled);
+ assertTrue("Update request processor finish was not called", TestUpdateRequestProcessor.finishCalled);
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testSingleEntity_CaseInsensitive() throws Exception {
+ List rows = new ArrayList();
+ rows.add(createMap("id", "1", "desC", "one"));
+ MockDataSource.setIterator("select * from x", rows.iterator());
+
+ runFullImport(DATA_CONFIG_WITH_CASE_INSENSITIVE_FIELDS);
+
+ assertQ(req("id:1"), "//*[@numFound='1']");
+ assertTrue("Start event listener was not called", StartEventListener.executed);
+ assertTrue("End event listener was not called", EndEventListener.executed);
+ assertTrue("Update request processor processAdd was not called", TestUpdateRequestProcessor.processAddCalled);
+ assertTrue("Update request processor finish was not called", TestUpdateRequestProcessor.finishCalled);
+ }
+
+ @Test
+ public void testErrorHandler() throws Exception {
+ List rows = new ArrayList();
+ rows.add(createMap("id", "1", "FORCE_ERROR", "true"));
+ MockDataSource.setIterator("select * from x", rows.iterator());
+
+ runFullImport(DATA_CONFIG_WITH_ERROR_HANDLER);
+
+ assertTrue("Error event listener was not called", ErrorEventListener.executed);
+ assertTrue(ErrorEventListener.lastException.getMessage().contains("ForcedException"));
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testDynamicFields() throws Exception {
+ List rows = new ArrayList();
+ rows.add(createMap("id", "1", "desc", "one"));
+ MockDataSource.setIterator("select * from x", rows.iterator());
+
+ runFullImport(DATA_CONFIG_WITH_DYNAMIC_TRANSFORMER);
+
+ assertQ(req("id:1"), "//*[@numFound='1']");
+ assertQ(req("dynamic_s:test"), "//*[@numFound='1']");
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testRequestParamsAsVariable() throws Exception {
+ List rows = new ArrayList();
+ rows.add(createMap("id", "101", "desc", "ApacheSolr"));
+ MockDataSource.setIterator("select * from books where category='search'", rows.iterator());
+
+ LocalSolrQueryRequest request = lrf.makeRequest("command", "full-import",
+ "debug", "on", "clean", "true", "commit", "true",
+ "category", "search",
+ "dataConfig", REQUEST_PARAM_AS_VARIABLE);
+ h.query("/dataimport", request);
+ assertQ(req("desc:ApacheSolr"), "//*[@numFound='1']");
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testDynamicFieldNames() throws Exception {
+ List rows = new ArrayList();
+ rows.add(createMap("mypk", "101", "text", "ApacheSolr"));
+ MockDataSource.setIterator("select * from x", rows.iterator());
+
+ LocalSolrQueryRequest request = lrf.makeRequest("command", "full-import",
+ "debug", "on", "clean", "true", "commit", "true",
+ "dataConfig", DATA_CONFIG_WITH_DYNAMIC_FIELD_NAMES);
+ h.query("/dataimport", request);
+ assertQ(req("id:101"), "//*[@numFound='1']", "//*[@name='101_s']");
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testRequestParamsAsFieldName() throws Exception {
+ List rows = new ArrayList();
+ rows.add(createMap("mypk", "101", "text", "ApacheSolr"));
+ MockDataSource.setIterator("select * from x", rows.iterator());
+
+ LocalSolrQueryRequest request = lrf.makeRequest("command", "full-import",
+ "debug", "on", "clean", "true", "commit", "true",
+ "mypk", "id", "text", "desc",
+ "dataConfig", DATA_CONFIG_WITH_TEMPLATIZED_FIELD_NAMES);
+ h.query("/dataimport", request);
+ assertQ(req("id:101"), "//*[@numFound='1']");
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testContext() throws Exception {
+ List rows = new ArrayList();
+ rows.add(createMap("id", "1", "desc", "one"));
+ MockDataSource.setIterator("select * from x", rows.iterator());
+
+ runFullImport(loadDataConfig("data-config-with-transformer.xml"));
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testSkipDoc() throws Exception {
+ List rows = new ArrayList();
+ rows.add(createMap("id", "1", "desc", "one"));
+ rows.add(createMap("id", "2", "desc", "two", DocBuilder.SKIP_DOC, "true"));
+ MockDataSource.setIterator("select * from x", rows.iterator());
+
+ runFullImport(DATA_CONFIG_WITH_DYNAMIC_TRANSFORMER);
+
+ assertQ(req("id:1"), "//*[@numFound='1']");
+ assertQ(req("id:2"), "//*[@numFound='0']");
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testSkipRow() throws Exception {
+ List rows = new ArrayList();
+ rows.add(createMap("id", "1", "desc", "one"));
+ rows.add(createMap("id", "2", "desc", "two", DocBuilder.SKIP_ROW, "true"));
+ MockDataSource.setIterator("select * from x", rows.iterator());
+
+ runFullImport(DATA_CONFIG_WITH_DYNAMIC_TRANSFORMER);
+
+ assertQ(req("id:1"), "//*[@numFound='1']");
+ assertQ(req("id:2"), "//*[@numFound='0']");
+
+ MockDataSource.clearCache();
+
+ rows = new ArrayList();
+ rows.add(createMap("id", "3", "desc", "one"));
+ rows.add(createMap("id", "4", "desc", "two"));
+ MockDataSource.setIterator("select * from x", rows.iterator());
+
+ rows = new ArrayList();
+ rows.add(createMap("name_s", "abcd"));
+ MockDataSource.setIterator("3", rows.iterator());
+
+ rows = new ArrayList();
+ rows.add(createMap("name_s", "xyz", DocBuilder.SKIP_ROW, "true"));
+ MockDataSource.setIterator("4", rows.iterator());
+
+ runFullImport(DATA_CONFIG_WITH_TWO_ENTITIES);
+ assertQ(req("id:3"), "//*[@numFound='1']");
+ assertQ(req("id:4"), "//*[@numFound='1']");
+ assertQ(req("name_s:abcd"), "//*[@numFound='1']");
+ assertQ(req("name_s:xyz"), "//*[@numFound='0']");
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testStopTransform() throws Exception {
+ List rows = new ArrayList();
+ rows.add(createMap("id", "1", "desc", "one"));
+ rows.add(createMap("id", "2", "desc", "two", "$stopTransform", "true"));
+ MockDataSource.setIterator("select * from x", rows.iterator());
+
+ runFullImport(DATA_CONFIG_FOR_SKIP_TRANSFORM);
+
+ assertQ(req("id:1"), "//*[@numFound='1']");
+ assertQ(req("id:2"), "//*[@numFound='1']");
+ assertQ(req("name_s:xyz"), "//*[@numFound='1']");
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testDeleteDocs() throws Exception {
+ List rows = new ArrayList();
+ rows.add(createMap("id", "1", "desc", "one"));
+ rows.add(createMap("id", "2", "desc", "two"));
+ rows.add(createMap("id", "3", "desc", "two", DocBuilder.DELETE_DOC_BY_ID, "2"));
+ MockDataSource.setIterator("select * from x", rows.iterator());
+
+ runFullImport(DATA_CONFIG_FOR_SKIP_TRANSFORM);
+
+ assertQ(req("id:1"), "//*[@numFound='1']");
+ assertQ(req("id:2"), "//*[@numFound='0']");
+ assertQ(req("id:3"), "//*[@numFound='1']");
+
+ assertTrue("Update request processor processDelete was not called", TestUpdateRequestProcessor.processDeleteCalled);
+ assertTrue("Update request processor finish was not called", TestUpdateRequestProcessor.finishCalled);
+
+ MockDataSource.clearCache();
+ rows = new ArrayList();
+ rows.add(createMap("id", "1", "desc", "one"));
+ rows.add(createMap("id", "2", "desc", "one"));
+ rows.add(createMap("id", "3", "desc", "two", DocBuilder.DELETE_DOC_BY_QUERY, "desc:one"));
+ MockDataSource.setIterator("select * from x", rows.iterator());
+
+ runFullImport(DATA_CONFIG_FOR_SKIP_TRANSFORM);
+
+ assertQ(req("id:1"), "//*[@numFound='0']");
+ assertQ(req("id:2"), "//*[@numFound='0']");
+ assertQ(req("id:3"), "//*[@numFound='1']");
+
+ assertTrue("Update request processor processDelete was not called", TestUpdateRequestProcessor.processDeleteCalled);
+ assertTrue("Update request processor finish was not called", TestUpdateRequestProcessor.finishCalled);
+
+ MockDataSource.clearCache();
+ rows = new ArrayList();
+ rows.add(createMap(DocBuilder.DELETE_DOC_BY_ID, "3"));
+ MockDataSource.setIterator("select * from x", rows.iterator());
+ runFullImport(DATA_CONFIG_FOR_SKIP_TRANSFORM, createMap("clean","false"));
+ assertQ(req("id:3"), "//*[@numFound='0']");
+
+ assertTrue("Update request processor processDelete was not called", TestUpdateRequestProcessor.processDeleteCalled);
+ assertTrue("Update request processor finish was not called", TestUpdateRequestProcessor.finishCalled);
+
+ }
+
+ @Test
+ @Ignore("Fix Me. See SOLR-4103.")
+ public void testFileListEntityProcessor_lastIndexTime() throws Exception {
+ File tmpdir = createTempDir().toFile();
+
+ Map<String, String> params = createMap("baseDir", tmpdir.getAbsolutePath());
+
+ createFile(tmpdir, "a.xml", "a.xml".getBytes(StandardCharsets.UTF_8), true);
+ createFile(tmpdir, "b.xml", "b.xml".getBytes(StandardCharsets.UTF_8), true);
+ createFile(tmpdir, "c.props", "c.props".getBytes(StandardCharsets.UTF_8), true);
+ runFullImport(DATA_CONFIG_FILE_LIST, params);
+ assertQ(req("*:*"), "//*[@numFound='3']");
+
+ // Add a new file after a full index is done
+ createFile(tmpdir, "t.xml", "t.xml".getBytes(StandardCharsets.UTF_8), false);
+ runFullImport(DATA_CONFIG_FILE_LIST, params);
+ // we should find only 1 because by default clean=true is passed
+ // and this particular import should find only one file t.xml
+ assertQ(req("*:*"), "//*[@numFound='1']");
+ }
+
+ public static class MockTransformer extends Transformer {
+ @Override
+ public Object transformRow(Map<String, Object> row, Context context) {
+ assertTrue("Context gave incorrect data source", context.getDataSource("mockDs") instanceof MockDataSource2);
+ return row;
+ }
+ }
+
+ public static class AddDynamicFieldTransformer extends Transformer {
+ @Override
+ public Object transformRow(Map<String, Object> row, Context context) {
+ // Add a dynamic field
+ row.put("dynamic_s", "test");
+ return row;
+ }
+ }
+
+ public static class ForcedExceptionTransformer extends Transformer {
+ @Override
+ public Object transformRow(Map<String, Object> row, Context context) {
+ throw new DataImportHandlerException(DataImportHandlerException.SEVERE, "ForcedException");
+ }
+ }
+
+ public static class MockDataSource2 extends MockDataSource {
+
+ }
+
+ public static class StartEventListener implements EventListener {
+ public static boolean executed = false;
+
+ @Override
+ public void onEvent(Context ctx) {
+ executed = true;
+ }
+ }
+
+ public static class EndEventListener implements EventListener {
+ public static boolean executed = false;
+
+ @Override
+ public void onEvent(Context ctx) {
+ executed = true;
+ }
+ }
+
+ public static class ErrorEventListener implements EventListener {
+ public static boolean executed = false;
+ public static Exception lastException = null;
+
+ @Override
+ public void onEvent(Context ctx) {
+ executed = true;
+ lastException = ((ContextImpl) ctx).getLastException();
+ }
+ }
+
+ private static final String REQUEST_PARAM_AS_VARIABLE = "<dataConfig>\n" +
+ " <dataSource type=\"MockDataSource\" />\n" +
+ " <document>\n" +
+ " <entity name=\"books\" query=\"select * from books where category='${dataimporter.request.category}'\">\n" +
+ " <field column=\"id\" />\n" +
+ " <field column=\"desc\" />\n" +
+ " </entity>\n" +
+ " </document>\n" +
+ "</dataConfig>";
+
+ private static final String DATA_CONFIG_WITH_DYNAMIC_TRANSFORMER = "<dataConfig> <dataSource type=\"MockDataSource\"/>\n" +
+ " <document>\n" +
+ " <entity name=\"books\" query=\"select * from x\"" +
+ " transformer=\"TestDocBuilder2$AddDynamicFieldTransformer\">\n" +
+ " <field column=\"id\" />\n" +
+ " <field column=\"desc\" />\n" +
+ " </entity>\n" +
+ " </document>\n" +
+ "</dataConfig>";
+
+ private static final String DATA_CONFIG_FOR_SKIP_TRANSFORM = "<dataConfig> <dataSource type=\"MockDataSource\"/>\n" +
+ " <document>\n" +
+ " <entity name=\"books\" query=\"select * from x\"" +
+ " transformer=\"TemplateTransformer\">\n" +
+ " <field column=\"id\" />\n" +
+ " <field column=\"desc\" />\n" +
+ " <field column=\"name_s\" template=\"xyz\" />\n" +
+ " </entity>\n" +
+ " </document>\n" +
+ "</dataConfig>";
+
+ private static final String DATA_CONFIG_WITH_TWO_ENTITIES = "<dataConfig><dataSource type=\"MockDataSource\"/>\n" +
+ " <document>\n" +
+ " <entity name=\"books\" query=\"select * from x\">" +
+ " <field column=\"id\" />\n" +
+ " <field column=\"desc\" />\n" +
+ " <entity name=\"authors\" query=\"${books.id}\">" +
+ " <field column=\"name_s\" />" +
+ " </entity>" +
+ " </entity>\n" +
+ " </document>\n" +
+ "</dataConfig>";
+
+ private static final String DATA_CONFIG_WITH_CASE_INSENSITIVE_FIELDS = "<dataConfig> <dataSource type=\"MockDataSource\"/>\n" +
+ " <document onImportStart=\"TestDocBuilder2$StartEventListener\" onImportEnd=\"TestDocBuilder2$EndEventListener\">\n" +
+ " <entity name=\"books\" query=\"select * from x\">\n" +
+ " <field column=\"ID\" />\n" +
+ " <field column=\"Desc\" />\n" +
+ " </entity>\n" +
+ " </document>\n" +
+ "</dataConfig>";
+
+ private static final String DATA_CONFIG_WITH_ERROR_HANDLER = "<dataConfig> <dataSource type=\"MockDataSource\"/>\n" +
+ " <document onError=\"TestDocBuilder2$ErrorEventListener\">\n" +
+ " <entity name=\"books\" query=\"select * from x\" transformer=\"TestDocBuilder2$ForcedExceptionTransformer\">\n" +
+ " <field column=\"id\" />\n" +
+ " <field column=\"FORCE_ERROR\" />\n" +
+ " </entity>\n" +
+ " </document>\n" +
+ "</dataConfig>";
+
+ private static final String DATA_CONFIG_WITH_TEMPLATIZED_FIELD_NAMES = "<dataConfig><dataSource type=\"MockDataSource\"/>\n" +
+ " <document>\n" +
+ " <entity name=\"books\" query=\"select * from x\">\n" +
+ " <field column=\"mypk\" name=\"${dih.request.mypk}\" />\n" +
+ " <field column=\"text\" name=\"${dih.request.text}\" />\n" +
+ " </entity>\n" +
+ " </document>\n" +
+ "</dataConfig>";
+
+ private static final String DATA_CONFIG_WITH_DYNAMIC_FIELD_NAMES = "<dataConfig><dataSource type=\"MockDataSource\"/>\n" +
+ " <document>\n" +
+ " <entity name=\"books\" query=\"select * from x\">\n" +
+ " <field column=\"mypk\" name=\"id\" />\n" +
+ " <field column=\"text\" name=\"${books.mypk}_s\" />\n" +
+ " </entity>\n" +
+ " </document>\n" +
+ "</dataConfig>";
+
+ private static final String DATA_CONFIG_FILE_LIST = "<dataConfig>\n" +
+ "\t<document>\n" +
+ "\t\t<entity name=\"x\" processor=\"FileListEntityProcessor\" \n" +
+ "\t\t\t\tfileName=\".*\" newerThan=\"${dih.last_index_time}\" \n" +
+ "\t\t\t\tbaseDir=\"${dih.request.baseDir}\" transformer=\"TemplateTransformer\">\n" +
+ "\t\t\t<field column=\"id\" template=\"${x.file}\" />\n" +
+ "\t\t</entity>\n" +
+ "\t</document>\n" +
+ "</dataConfig>";
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestEntityProcessorBase.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestEntityProcessorBase.java b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestEntityProcessorBase.java
new file mode 100644
index 0000000..623e49e
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestEntityProcessorBase.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * <p>
+ * Test for EntityProcessorBase
+ * </p>
+ *
+ *
+ * @since solr 1.3
+ */
+public class TestEntityProcessorBase extends AbstractDataImportHandlerTestCase {
+
+ @Test
+ public void multiTransformer() {
+ List<Map<String, String>> fields = new ArrayList<>();
+ Map<String, String> entity = new HashMap<>();
+ entity.put("transformer", T1.class.getName() + "," + T2.class.getName()
+ + "," + T3.class.getName());
+ fields.add(getField("A", null, null, null, null));
+ fields.add(getField("B", null, null, null, null));
+
+ Context context = getContext(null, null, new MockDataSource(), Context.FULL_DUMP,
+ fields, entity);
+ Map<String, Object> src = new HashMap<>();
+ src.put("A", "NA");
+ src.put("B", "NA");
+ EntityProcessorWrapper sep = new EntityProcessorWrapper(new SqlEntityProcessor(), null, null);
+ sep.init(context);
+ Map<String, Object> res = sep.applyTransformer(src);
+ assertNotNull(res.get("T1"));
+ assertNotNull(res.get("T2"));
+ assertNotNull(res.get("T3"));
+ }
+
+ static class T1 extends Transformer {
+
+ @Override
+ public Object transformRow(Map<String, Object> aRow, Context context) {
+ aRow.put("T1", "T1 called");
+ return aRow;
+
+ }
+ }
+
+ static class T2 extends Transformer {
+
+ @Override
+ public Object transformRow(Map<String, Object> aRow, Context context) {
+ aRow.put("T2", "T2 called");
+ return aRow;
+ }
+ }
+
+ static class T3 {
+
+ public Object transformRow(Map<String, Object> aRow) {
+ aRow.put("T3", "T3 called");
+ return aRow;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestEphemeralCache.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestEphemeralCache.java b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestEphemeralCache.java
new file mode 100644
index 0000000..ddccafe
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestEphemeralCache.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import java.math.BigDecimal;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.hamcrest.CoreMatchers.*;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class TestEphemeralCache extends AbstractDataImportHandlerTestCase {
+
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ initCore("dataimport-solrconfig.xml", "dataimport-schema.xml");
+ }
+
+ @Before
+ public void reset() {
+ DestroyCountCache.destroyed.clear();
+ setupMockData();
+ }
+
+ @Test
+ public void test() throws Exception {
+ assertFullImport(getDataConfigDotXml());
+ }
+
+ @SuppressWarnings("unchecked")
+ private void setupMockData() {
+ List parentRows = new ArrayList();
+ parentRows.add(createMap("id", new BigDecimal("1"), "parent_s", "one"));
+ parentRows.add(createMap("id", new BigDecimal("2"), "parent_s", "two"));
+ parentRows.add(createMap("id", new BigDecimal("3"), "parent_s", "three"));
+ parentRows.add(createMap("id", new BigDecimal("4"), "parent_s", "four"));
+ parentRows.add(createMap("id", new BigDecimal("5"), "parent_s", "five"));
+
+ List child1Rows = new ArrayList();
+ child1Rows.add(createMap("id", new BigDecimal("6"), "child1a_mult_s", "this is the number six."));
+ child1Rows.add(createMap("id", new BigDecimal("5"), "child1a_mult_s", "this is the number five."));
+ child1Rows.add(createMap("id", new BigDecimal("6"), "child1a_mult_s", "let's sing a song of six."));
+ child1Rows.add(createMap("id", new BigDecimal("3"), "child1a_mult_s", "three"));
+ child1Rows.add(createMap("id", new BigDecimal("3"), "child1a_mult_s", "III"));
+ child1Rows.add(createMap("id", new BigDecimal("3"), "child1a_mult_s", "3"));
+ child1Rows.add(createMap("id", new BigDecimal("3"), "child1a_mult_s", "|||"));
+ child1Rows.add(createMap("id", new BigDecimal("1"), "child1a_mult_s", "one"));
+ child1Rows.add(createMap("id", new BigDecimal("1"), "child1a_mult_s", "uno"));
+ child1Rows.add(createMap("id", new BigDecimal("2"), "child1b_s", "CHILD1B", "child1a_mult_s", "this is the number two."));
+
+ List child2Rows = new ArrayList();
+ child2Rows.add(createMap("id", new BigDecimal("6"), "child2a_mult_s", "Child 2 says, 'this is the number six.'"));
+ child2Rows.add(createMap("id", new BigDecimal("5"), "child2a_mult_s", "Child 2 says, 'this is the number five.'"));
+ child2Rows.add(createMap("id", new BigDecimal("6"), "child2a_mult_s", "Child 2 says, 'let's sing a song of six.'"));
+ child2Rows.add(createMap("id", new BigDecimal("3"), "child2a_mult_s", "Child 2 says, 'three'"));
+ child2Rows.add(createMap("id", new BigDecimal("3"), "child2a_mult_s", "Child 2 says, 'III'"));
+ child2Rows.add(createMap("id", new BigDecimal("3"), "child2b_s", "CHILD2B", "child2a_mult_s", "Child 2 says, '3'"));
+ child2Rows.add(createMap("id", new BigDecimal("3"), "child2a_mult_s", "Child 2 says, '|||'"));
+ child2Rows.add(createMap("id", new BigDecimal("1"), "child2a_mult_s", "Child 2 says, 'one'"));
+ child2Rows.add(createMap("id", new BigDecimal("1"), "child2a_mult_s", "Child 2 says, 'uno'"));
+ child2Rows.add(createMap("id", new BigDecimal("2"), "child2a_mult_s", "Child 2 says, 'this is the number two.'"));
+
+ MockDataSource.setIterator("SELECT * FROM PARENT", parentRows.iterator());
+ MockDataSource.setIterator("SELECT * FROM CHILD_1", child1Rows.iterator());
+ MockDataSource.setIterator("SELECT * FROM CHILD_2", child2Rows.iterator());
+
+ }
+ private String getDataConfigDotXml() {
+ return
+ "<dataConfig>" +
+ " <dataSource type=\"MockDataSource\" />" +
+ " <document>" +
+ " <entity " +
+ " name=\"PARENT\"" +
+ " processor=\"SqlEntityProcessor\"" +
+ " cacheImpl=\"org.apache.solr.handler.dataimport.DestroyCountCache\"" +
+ " cacheName=\"PARENT\"" +
+ " query=\"SELECT * FROM PARENT\" " +
+ " >" +
+ " <entity" +
+ " name=\"CHILD_1\"" +
+ " processor=\"SqlEntityProcessor\"" +
+ " cacheImpl=\"org.apache.solr.handler.dataimport.DestroyCountCache\"" +
+ " cacheName=\"CHILD\"" +
+ " cacheKey=\"id\"" +
+ " cacheLookup=\"PARENT.id\"" +
+ " fieldNames=\"id, child1a_mult_s, child1b_s\"" +
+ " fieldTypes=\"BIGDECIMAL, STRING, STRING\"" +
+ " query=\"SELECT * FROM CHILD_1\" " +
+ " />" +
+ " <entity" +
+ " name=\"CHILD_2\"" +
+ " processor=\"SqlEntityProcessor\"" +
+ " cacheImpl=\"org.apache.solr.handler.dataimport.DestroyCountCache\"" +
+ " cacheKey=\"id\"" +
+ " cacheLookup=\"PARENT.id\"" +
+ " query=\"SELECT * FROM CHILD_2\" " +
+ " />" +
+ " </entity>" +
+ " </document>" +
+ "</dataConfig>"
+ ;
+ }
+
+ private void assertFullImport(String dataConfig) throws Exception {
+ runFullImport(dataConfig);
+
+ assertQ(req("*:*"), "//*[@numFound='5']");
+ assertQ(req("id:1"), "//*[@numFound='1']");
+ assertQ(req("id:6"), "//*[@numFound='0']");
+ assertQ(req("parent_s:four"), "//*[@numFound='1']");
+ assertQ(req("child1a_mult_s:this\\ is\\ the\\ numbe*"), "//*[@numFound='2']");
+ assertQ(req("child2a_mult_s:Child\\ 2\\ say*"), "//*[@numFound='4']");
+ assertQ(req("child1b_s:CHILD1B"), "//*[@numFound='1']");
+ assertQ(req("child2b_s:CHILD2B"), "//*[@numFound='1']");
+ assertQ(req("child1a_mult_s:one"), "//*[@numFound='1']");
+ assertQ(req("child1a_mult_s:uno"), "//*[@numFound='1']");
+ assertQ(req("child1a_mult_s:(uno OR one)"), "//*[@numFound='1']");
+
+ assertThat(DestroyCountCache.destroyed.size(), is(3));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestErrorHandling.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestErrorHandling.java b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestErrorHandling.java
new file mode 100644
index 0000000..1ea1ad4
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestErrorHandling.java
@@ -0,0 +1,209 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+import java.io.Reader;
+import java.io.StringReader;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+
+/**
+ * Tests exception handling during imports in DataImportHandler
+ *
+ *
+ * @since solr 1.4
+ */
+public class TestErrorHandling extends AbstractDataImportHandlerTestCase {
+
+ //TODO: fix this test to not require FSDirectory.
+ static String savedFactory;
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ savedFactory = System.getProperty("solr.DirectoryFactory");
+ System.setProperty("solr.directoryFactory", "solr.MockFSDirectoryFactory");
+ initCore("dataimport-solrconfig.xml", "dataimport-schema.xml");
+ ignoreException("Unexpected close tag");
+ }
+
+ @AfterClass
+ public static void afterClass() {
+ if (savedFactory == null) {
+ System.clearProperty("solr.directoryFactory");
+ } else {
+ System.setProperty("solr.directoryFactory", savedFactory);
+ }
+ }
+
+ @Before @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ clearIndex();
+ assertU(commit());
+ }
+
+ public void testMalformedStreamingXml() throws Exception {
+ StringDataSource.xml = malformedXml;
+ runFullImport(dataConfigWithStreaming);
+ assertQ(req("id:1"), "//*[@numFound='1']");
+ assertQ(req("id:2"), "//*[@numFound='1']");
+ }
+
+ public void testMalformedNonStreamingXml() throws Exception {
+ StringDataSource.xml = malformedXml;
+ runFullImport(dataConfigWithoutStreaming);
+ assertQ(req("id:1"), "//*[@numFound='1']");
+ assertQ(req("id:2"), "//*[@numFound='1']");
+ }
+
+ public void testAbortOnError() throws Exception {
+ StringDataSource.xml = malformedXml;
+ runFullImport(dataConfigAbortOnError);
+ assertQ(req("*:*"), "//*[@numFound='0']");
+ }
+
+ public void testTransformerErrorContinue() throws Exception {
+ StringDataSource.xml = wellformedXml;
+ List<Map<String, Object>> rows = new ArrayList<>();
+ rows.add(createMap("id", "3", "desc", "exception-transformer"));
+ MockDataSource.setIterator("select * from foo", rows.iterator());
+ runFullImport(dataConfigWithTransformer);
+ assertQ(req("*:*"), "//*[@numFound='3']");
+ }
+
+ public void testExternalEntity() throws Exception {
+ StringDataSource.xml = wellformedXml;
+ // This should not fail as external entities are replaced by an empty string during parsing:
+ runFullImport(dataConfigWithEntity);
+ assertQ(req("*:*"), "//*[@numFound='3']");
+ }
+
+ public static class StringDataSource extends DataSource<Reader> {
+ public static String xml = "";
+
+ @Override
+ public void init(Context context, Properties initProps) {
+ }
+
+ @Override
+ public Reader getData(String query) {
+ return new StringReader(xml);
+ }
+
+ @Override
+ public void close() {
+
+ }
+ }
+
+ public static class ExceptionTransformer extends Transformer {
+ @Override
+ public Object transformRow(Map<String, Object> row, Context context) {
+ throw new RuntimeException("Test exception");
+ }
+ }
+
+ private String dataConfigWithStreaming = "<dataConfig>\n" +
+ " <dataSource name=\"str\" type=\"TestErrorHandling$StringDataSource\" />" +
+ " <document>\n" +
+ " <entity name=\"node\" dataSource=\"str\" processor=\"XPathEntityProcessor\" url=\"test\" stream=\"true\" forEach=\"/root/node\" onError=\"skip\">\n" +
+ " <field column=\"id\" xpath=\"/root/node/id\" />\n" +
+ " <field column=\"desc\" xpath=\"/root/node/desc\" />\n" +
+ " </entity>\n" +
+ " </document>\n" +
+ "</dataConfig>";
+
+ private String dataConfigWithoutStreaming = "<dataConfig>\n" +
+ " <dataSource name=\"str\" type=\"TestErrorHandling$StringDataSource\" />" +
+ " <document>\n" +
+ " <entity name=\"node\" dataSource=\"str\" processor=\"XPathEntityProcessor\" url=\"test\" forEach=\"/root/node\" onError=\"skip\">\n" +
+ " <field column=\"id\" xpath=\"/root/node/id\" />\n" +
+ " <field column=\"desc\" xpath=\"/root/node/desc\" />\n" +
+ " </entity>\n" +
+ " </document>\n" +
+ "</dataConfig>";
+
+ private String dataConfigAbortOnError = "<dataConfig>\n" +
+ " <dataSource name=\"str\" type=\"TestErrorHandling$StringDataSource\" />" +
+ " <document>\n" +
+ " <entity name=\"node\" dataSource=\"str\" processor=\"XPathEntityProcessor\" url=\"test\" forEach=\"/root/node\" onError=\"abort\">\n" +
+ " <field column=\"id\" xpath=\"/root/node/id\" />\n" +
+ " <field column=\"desc\" xpath=\"/root/node/desc\" />\n" +
+ " </entity>\n" +
+ " </document>\n" +
+ "</dataConfig>";
+
+ private String dataConfigWithTransformer = "<dataConfig>\n" +
+ " <dataSource name=\"str\" type=\"TestErrorHandling$StringDataSource\" />" +
+ "<dataSource type=\"MockDataSource\"/>" +
+ " <document>\n" +
+ " <entity name=\"node\" dataSource=\"str\" processor=\"XPathEntityProcessor\" url=\"test\" forEach=\"/root/node\">\n" +
+ " <field column=\"id\" xpath=\"/root/node/id\" />\n" +
+ " <field column=\"desc\" xpath=\"/root/node/desc\" />\n" +
+ " <entity name=\"child\" query=\"select * from foo\" transformer=\"TestErrorHandling$ExceptionTransformer\" onError=\"continue\">\n" +
+ " </entity>" +
+ " </entity>\n" +
+ " </document>\n" +
+ "</dataConfig>";
+
+ private String dataConfigWithEntity = "<!DOCTYPE dataConfig [\n" +
+ " <!ENTITY internalTerm \"node\">\n" +
+ " <!ENTITY externalTerm SYSTEM \"foo://bar.xyz/external\">\n" +
+ "]><dataConfig>\n" +
+ " <dataSource name=\"str\" type=\"TestErrorHandling$StringDataSource\" />" +
+ " <document>\n" +
+ " <entity name=\"&internalTerm;\" dataSource=\"str\" processor=\"XPathEntityProcessor\" url=\"test\" forEach=\"/root/node\" onError=\"skip\">\n" +
+ " <field column=\"id\" xpath=\"/root/node/id\">&externalTerm;</field>\n" +
+ " <field column=\"desc\" xpath=\"/root/node/desc\" />\n" +
+ " </entity>\n" +
+ " </document>\n" +
+ "</dataConfig>";
+
+ private String malformedXml = "<root>\n" +
+ " <node>\n" +
+ " <id>1</id>\n" +
+ " <desc>test1</desc>\n" +
+ " </node>\n" +
+ " <node>\n" +
+ " <id>2</id>\n" +
+ " <desc>test2</desc>\n" +
+ " </node>\n" +
+ " <node>\n" +
+ " <id/>3</id>\n" +
+ " <desc>test3</desc>\n" +
+ " </node>\n" +
+ "</root>";
+
+ private String wellformedXml = "<root>\n" +
+ " <node>\n" +
+ " <id>1</id>\n" +
+ " <desc>test1</desc>\n" +
+ " </node>\n" +
+ " <node>\n" +
+ " <id>2</id>\n" +
+ " <desc>test2</desc>\n" +
+ " </node>\n" +
+ " <node>\n" +
+ " <id>3</id>\n" +
+ " <desc>test3</desc>\n" +
+ " </node>\n" +
+ "</root>";
+}