You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by da...@apache.org on 2018/11/02 11:33:28 UTC
[07/25] lucene-solr:jira/gradle: Adding dataimporthandler-extras
module
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestVariableResolver.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestVariableResolver.java b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestVariableResolver.java
new file mode 100644
index 0000000..0028564
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestVariableResolver.java
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Properties;
+import java.util.TimeZone;
+
+import org.apache.solr.util.DateMathParser;
+import org.junit.Test;
+
+/**
+ * <p>
+ * Test for VariableResolver
+ * </p>
+ *
+ *
+ * @since solr 1.3
+ */
+public class TestVariableResolver extends AbstractDataImportHandlerTestCase {
+
+ @Test
+ public void testSimpleNamespace() {
+ VariableResolver vri = new VariableResolver();
+ Map<String,Object> ns = new HashMap<>();
+ ns.put("world", "WORLD");
+ vri.addNamespace("hello", ns);
+ assertEquals("WORLD", vri.resolve("hello.world"));
+ }
+
+ @Test
+ public void testDefaults() {
+ // System.out.println(System.setProperty(TestVariableResolver.class.getName(),"hello"));
+ System.setProperty(TestVariableResolver.class.getName(), "hello");
+ // System.out.println("s.gP()"+
+ // System.getProperty(TestVariableResolver.class.getName()));
+
+ Properties p = new Properties();
+ p.put("hello", "world");
+ VariableResolver vri = new VariableResolver(p);
+ Object val = vri.resolve(TestVariableResolver.class.getName());
+ // System.out.println("val = " + val);
+ assertEquals("hello", val);
+ assertEquals("world", vri.resolve("hello"));
+ }
+
+ @Test
+ public void testNestedNamespace() {
+ VariableResolver vri = new VariableResolver();
+ Map<String,Object> ns = new HashMap<>();
+ ns.put("world", "WORLD");
+ vri.addNamespace("hello", ns);
+ ns = new HashMap<>();
+ ns.put("world1", "WORLD1");
+ vri.addNamespace("hello.my", ns);
+ assertEquals("WORLD1", vri.resolve("hello.my.world1"));
+ }
+
+ @Test
+ public void test3LevelNestedNamespace() {
+ VariableResolver vri = new VariableResolver();
+ Map<String,Object> ns = new HashMap<>();
+ ns.put("world", "WORLD");
+ vri.addNamespace("hello", ns);
+ ns = new HashMap<>();
+ ns.put("world1", "WORLD1");
+ vri.addNamespace("hello.my.new", ns);
+ assertEquals("WORLD1", vri.resolve("hello.my.new.world1"));
+ }
+
+ @Test
+ public void dateNamespaceWithValue() {
+ VariableResolver vri = new VariableResolver();
+ vri.setEvaluators(new DataImporter().getEvaluators(Collections
+ .<Map<String,String>> emptyList()));
+ Map<String,Object> ns = new HashMap<>();
+ Date d = new Date();
+ ns.put("dt", d);
+ vri.addNamespace("A", ns);
+ assertEquals(
+ new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.ROOT).format(d),
+ vri.replaceTokens("${dataimporter.functions.formatDate(A.dt,'yyyy-MM-dd HH:mm:ss')}"));
+ }
+
+ @Test
+ public void dateNamespaceWithExpr() throws Exception {
+ VariableResolver vri = new VariableResolver();
+ vri.setEvaluators(new DataImporter().getEvaluators(Collections
+ .<Map<String,String>> emptyList()));
+ SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.ROOT);
+ format.setTimeZone(TimeZone.getTimeZone("UTC"));
+ DateMathParser dmp = new DateMathParser(TimeZone.getDefault());
+
+ String s = vri
+ .replaceTokens("${dataimporter.functions.formatDate('NOW/DAY','yyyy-MM-dd HH:mm')}");
+ assertEquals(
+ new SimpleDateFormat("yyyy-MM-dd HH:mm", Locale.ROOT).format(dmp.parseMath("/DAY")),
+ s);
+ }
+
+ @Test
+ public void testDefaultNamespace() {
+ VariableResolver vri = new VariableResolver();
+ Map<String,Object> ns = new HashMap<>();
+ ns.put("world", "WORLD");
+ vri.addNamespace(null, ns);
+ assertEquals("WORLD", vri.resolve("world"));
+ }
+
+ @Test
+ public void testDefaultNamespace1() {
+ VariableResolver vri = new VariableResolver();
+ Map<String,Object> ns = new HashMap<>();
+ ns.put("world", "WORLD");
+ vri.addNamespace(null, ns);
+ assertEquals("WORLD", vri.resolve("world"));
+ }
+
+ @Test
+ public void testFunctionNamespace1() throws Exception {
+ VariableResolver resolver = new VariableResolver();
+ final List<Map<String,String>> l = new ArrayList<>();
+ Map<String,String> m = new HashMap<>();
+ m.put("name", "test");
+ m.put("class", E.class.getName());
+ l.add(m);
+ resolver.setEvaluators(new DataImporter().getEvaluators(l));
+ ContextImpl context = new ContextImpl(null, resolver, null,
+ Context.FULL_DUMP, Collections.EMPTY_MAP, null, null);
+
+ SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.ROOT);
+ format.setTimeZone(TimeZone.getTimeZone("UTC"));
+ DateMathParser dmp = new DateMathParser(TimeZone.getDefault());
+
+ String s = resolver
+ .replaceTokens("${dataimporter.functions.formatDate('NOW/DAY','yyyy-MM-dd HH:mm')}");
+ assertEquals(
+ new SimpleDateFormat("yyyy-MM-dd HH:mm", Locale.ROOT).format(dmp.parseMath("/DAY")),
+ s);
+ assertEquals("Hello World",
+ resolver.replaceTokens("${dataimporter.functions.test('TEST')}"));
+ }
+
+ public static class E extends Evaluator {
+ @Override
+ public String evaluate(String expression, Context context) {
+ return "Hello World";
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestVariableResolverEndToEnd.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestVariableResolverEndToEnd.java b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestVariableResolverEndToEnd.java
new file mode 100644
index 0000000..8ee6878
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestVariableResolverEndToEnd.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import java.lang.invoke.MethodHandles;
+import java.sql.Connection;
+import java.sql.Statement;
+import java.util.Locale;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import junit.framework.Assert;
+
+import org.apache.solr.request.SolrQueryRequest;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class TestVariableResolverEndToEnd extends AbstractDIHJdbcTestCase {
+
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ @Test
+ public void test() throws Exception {
+ h.query("/dataimport", generateRequest());
+ SolrQueryRequest req = null;
+ try {
+ req = req("q", "*:*", "wt", "json", "indent", "true");
+ String response = h.query(req);
+ log.debug(response);
+ response = response.replaceAll("\\s","");
+ Assert.assertTrue(response.contains("\"numFound\":1"));
+ Pattern p = Pattern.compile("[\"]second1_s[\"][:][\"](.*?)[\"]");
+ Matcher m = p.matcher(response);
+ Assert.assertTrue(m.find());
+ String yearStr = m.group(1);
+ Assert.assertTrue(response.contains("\"second1_s\":\"" + yearStr + "\""));
+ Assert.assertTrue(response.contains("\"second2_s\":\"" + yearStr + "\""));
+ Assert.assertTrue(response.contains("\"second3_s\":\"" + yearStr + "\""));
+ Assert.assertTrue(response.contains("\"PORK_s\":\"GRILL\""));
+ Assert.assertTrue(response.contains("\"FISH_s\":\"FRY\""));
+ Assert.assertTrue(response.contains("\"BEEF_CUTS_mult_s\":[\"ROUND\",\"SIRLOIN\"]"));
+ } catch(Exception e) {
+ throw e;
+ } finally {
+ req.close();
+ }
+ }
+
+ @Override
+ protected String generateConfig() {
+ String thirdLocaleParam = random().nextBoolean() ? "" : (", '" + Locale.getDefault().toLanguageTag() + "'");
+ StringBuilder sb = new StringBuilder();
+ sb.append("<dataConfig> \n");
+ sb.append("<dataSource name=\"hsqldb\" driver=\"${dataimporter.request.dots.in.hsqldb.driver}\" url=\"jdbc:hsqldb:mem:.\" /> \n");
+ sb.append("<document name=\"TestEvaluators\"> \n");
+ sb.append("<entity name=\"FIRST\" processor=\"SqlEntityProcessor\" dataSource=\"hsqldb\" ");
+ sb.append(" query=\"" +
+ "select " +
+ " 1 as id, " +
+ " 'SELECT' as SELECT_KEYWORD, " +
+ " {ts '2017-02-18 12:34:56'} as FIRST_TS " +
+ "from DUAL \" >\n");
+ sb.append(" <field column=\"SELECT_KEYWORD\" name=\"select_keyword_s\" /> \n");
+ sb.append(" <entity name=\"SECOND\" processor=\"SqlEntityProcessor\" dataSource=\"hsqldb\" transformer=\"TemplateTransformer\" ");
+ sb.append(" query=\"" +
+ "${dataimporter.functions.encodeUrl(FIRST.SELECT_KEYWORD)} " +
+ " 1 as SORT, " +
+ " {ts '2017-02-18 12:34:56'} as SECOND_TS, " +
+ " '${dataimporter.functions.formatDate(FIRST.FIRST_TS, 'yyyy'" + thirdLocaleParam + ")}' as SECOND1_S, " +
+ " 'PORK' AS MEAT, " +
+ " 'GRILL' AS METHOD, " +
+ " 'ROUND' AS CUTS, " +
+ " 'BEEF_CUTS' AS WHATKIND " +
+ "from DUAL " +
+ "WHERE 1=${FIRST.ID} " +
+ "UNION " +
+ "${dataimporter.functions.encodeUrl(FIRST.SELECT_KEYWORD)} " +
+ " 2 as SORT, " +
+ " {ts '2017-02-18 12:34:56'} as SECOND_TS, " +
+ " '${dataimporter.functions.formatDate(FIRST.FIRST_TS, 'yyyy'" + thirdLocaleParam + ")}' as SECOND1_S, " +
+ " 'FISH' AS MEAT, " +
+ " 'FRY' AS METHOD, " +
+ " 'SIRLOIN' AS CUTS, " +
+ " 'BEEF_CUTS' AS WHATKIND " +
+ "from DUAL " +
+ "WHERE 1=${FIRST.ID} " +
+ "ORDER BY SORT \"" +
+ ">\n");
+ sb.append(" <field column=\"SECOND_S\" name=\"second_s\" /> \n");
+ sb.append(" <field column=\"SECOND1_S\" name=\"second1_s\" /> \n");
+ sb.append(" <field column=\"second2_s\" template=\"${dataimporter.functions.formatDate(SECOND.SECOND_TS, 'yyyy'" + thirdLocaleParam + ")}\" /> \n");
+ sb.append(" <field column=\"second3_s\" template=\"${dih.functions.formatDate(SECOND.SECOND_TS, 'yyyy'" + thirdLocaleParam + ")}\" /> \n");
+ sb.append(" <field column=\"METHOD\" name=\"${SECOND.MEAT}_s\"/>\n");
+ sb.append(" <field column=\"CUTS\" name=\"${SECOND.WHATKIND}_mult_s\"/>\n");
+ sb.append(" </entity>\n");
+ sb.append("</entity>\n");
+ sb.append("</document> \n");
+ sb.append("</dataConfig> \n");
+ String config = sb.toString();
+ log.info(config);
+ return config;
+ }
+ @Override
+ protected void populateData(Connection conn) throws Exception {
+ Statement s = null;
+ try {
+ s = conn.createStatement();
+ s.executeUpdate("create table dual(dual char(1) not null)");
+ s.executeUpdate("insert into dual values('Y')");
+ conn.commit();
+ } catch (Exception e) {
+ throw e;
+ } finally {
+ try {
+ s.close();
+ } catch (Exception ex) {}
+ try {
+ conn.close();
+ } catch (Exception ex) {}
+ }
+ }
+ @Override
+ protected Database setAllowedDatabases() {
+ return Database.HSQLDB;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestWriterImpl.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestWriterImpl.java b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestWriterImpl.java
new file mode 100644
index 0000000..e5c2a94
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestWriterImpl.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.update.processor.UpdateRequestProcessor;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.util.*;
+
+/**
+ * <p>
+ * Test for writerImpl paramater (to provide own SolrWriter)
+ * </p>
+ *
+ *
+ * @since solr 4.0
+ */
+public class TestWriterImpl extends AbstractDataImportHandlerTestCase {
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ initCore("dataimport-nodatasource-solrconfig.xml", "dataimport-schema.xml");
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testDataConfigWithDataSource() throws Exception {
+ List rows = new ArrayList();
+ rows.add(createMap("id", "1", "desc", "one"));
+ rows.add(createMap("id", "2", "desc", "two"));
+ rows.add(createMap("id", "3", "desc", "break"));
+ rows.add(createMap("id", "4", "desc", "four"));
+
+ MockDataSource.setIterator("select * from x", rows.iterator());
+
+ Map extraParams = createMap("writerImpl", TestSolrWriter.class.getName(),
+ "commit", "true");
+ runFullImport(loadDataConfig("data-config-with-datasource.xml"),
+ extraParams);
+
+ assertQ(req("id:1"), "//*[@numFound='1']");
+ assertQ(req("id:2"), "//*[@numFound='1']");
+ assertQ(req("id:3"), "//*[@numFound='0']");
+ assertQ(req("id:4"), "//*[@numFound='1']");
+ }
+
+ public static class TestSolrWriter extends SolrWriter {
+
+ public TestSolrWriter(UpdateRequestProcessor processor, SolrQueryRequest req) {
+ super(processor, req);
+ }
+
+ @Override
+ public boolean upload(SolrInputDocument doc) {
+ if (doc.getField("desc").getFirstValue().equals("break")) {
+ return false;
+ }
+ return super.upload(doc);
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestXPathEntityProcessor.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestXPathEntityProcessor.java b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestXPathEntityProcessor.java
new file mode 100644
index 0000000..72da77a
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestXPathEntityProcessor.java
@@ -0,0 +1,491 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import java.io.File;
+import java.io.Reader;
+import java.io.StringReader;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.TimeUnit;
+
+import org.junit.Test;
+
+/**
+ * <p>
+ * Test for XPathEntityProcessor
+ * </p>
+ *
+ *
+ * @since solr 1.3
+ */
+public class TestXPathEntityProcessor extends AbstractDataImportHandlerTestCase {
+ boolean simulateSlowReader;
+ boolean simulateSlowResultProcessor;
+ int rowsToRead = -1;
+
+ @Test
+ public void withFieldsAndXpath() throws Exception {
+ File tmpdir = createTempDir().toFile();
+
+ createFile(tmpdir, "x.xsl", xsl.getBytes(StandardCharsets.UTF_8), false);
+ Map entityAttrs = createMap("name", "e", "url", "cd.xml",
+ XPathEntityProcessor.FOR_EACH, "/catalog/cd");
+ List fields = new ArrayList();
+ fields.add(createMap("column", "title", "xpath", "/catalog/cd/title"));
+ fields.add(createMap("column", "artist", "xpath", "/catalog/cd/artist"));
+ fields.add(createMap("column", "year", "xpath", "/catalog/cd/year"));
+ Context c = getContext(null,
+ new VariableResolver(), getDataSource(cdData), Context.FULL_DUMP, fields, entityAttrs);
+ XPathEntityProcessor xPathEntityProcessor = new XPathEntityProcessor();
+ xPathEntityProcessor.init(c);
+ List<Map<String, Object>> result = new ArrayList<>();
+ while (true) {
+ Map<String, Object> row = xPathEntityProcessor.nextRow();
+ if (row == null)
+ break;
+ result.add(row);
+ }
+ assertEquals(3, result.size());
+ assertEquals("Empire Burlesque", result.get(0).get("title"));
+ assertEquals("Bonnie Tyler", result.get(1).get("artist"));
+ assertEquals("1982", result.get(2).get("year"));
+ }
+
+ @Test
+ public void testMultiValued() throws Exception {
+ Map entityAttrs = createMap("name", "e", "url", "testdata.xml",
+ XPathEntityProcessor.FOR_EACH, "/root");
+ List fields = new ArrayList();
+ fields.add(createMap("column", "a", "xpath", "/root/a", DataImporter.MULTI_VALUED, "true"));
+ Context c = getContext(null,
+ new VariableResolver(), getDataSource(testXml), Context.FULL_DUMP, fields, entityAttrs);
+ XPathEntityProcessor xPathEntityProcessor = new XPathEntityProcessor();
+ xPathEntityProcessor.init(c);
+ List<Map<String, Object>> result = new ArrayList<>();
+ while (true) {
+ Map<String, Object> row = xPathEntityProcessor.nextRow();
+ if (row == null)
+ break;
+ result.add(row);
+ }
+ List l = (List)result.get(0).get("a");
+ assertEquals(3, l.size());
+ assertEquals("1", l.get(0));
+ assertEquals("2", l.get(1));
+ assertEquals("ΓΌ", l.get(2));
+ }
+
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ @Test
+ public void testMultiValuedWithMultipleDocuments() throws Exception {
+ Map entityAttrs = createMap("name", "e", "url", "testdata.xml", XPathEntityProcessor.FOR_EACH, "/documents/doc");
+ List fields = new ArrayList();
+ fields.add(createMap("column", "id", "xpath", "/documents/doc/id", DataImporter.MULTI_VALUED, "false"));
+ fields.add(createMap("column", "a", "xpath", "/documents/doc/a", DataImporter.MULTI_VALUED, "true"));
+ fields.add(createMap("column", "s1dataA", "xpath", "/documents/doc/sec1/s1dataA", DataImporter.MULTI_VALUED, "true"));
+ fields.add(createMap("column", "s1dataB", "xpath", "/documents/doc/sec1/s1dataB", DataImporter.MULTI_VALUED, "true"));
+ fields.add(createMap("column", "s1dataC", "xpath", "/documents/doc/sec1/s1dataC", DataImporter.MULTI_VALUED, "true"));
+
+ Context c = getContext(null,
+ new VariableResolver(), getDataSource(textMultipleDocuments), Context.FULL_DUMP, fields, entityAttrs);
+ XPathEntityProcessor xPathEntityProcessor = new XPathEntityProcessor();
+ xPathEntityProcessor.init(c);
+ List<Map<String, Object>> result = new ArrayList<>();
+ while (true) {
+ Map<String, Object> row = xPathEntityProcessor.nextRow();
+ if (row == null)
+ break;
+ result.add(row);
+ }
+ {
+ assertEquals("1", result.get(0).get("id"));
+ List a = (List)result.get(0).get("a");
+ List s1dataA = (List)result.get(0).get("s1dataA");
+ List s1dataB = (List)result.get(0).get("s1dataB");
+ List s1dataC = (List)result.get(0).get("s1dataC");
+ assertEquals(2, a.size());
+ assertEquals("id1-a1", a.get(0));
+ assertEquals("id1-a2", a.get(1));
+ assertEquals(3, s1dataA.size());
+ assertEquals("id1-s1dataA-1", s1dataA.get(0));
+ assertNull(s1dataA.get(1));
+ assertEquals("id1-s1dataA-3", s1dataA.get(2));
+ assertEquals(3, s1dataB.size());
+ assertEquals("id1-s1dataB-1", s1dataB.get(0));
+ assertEquals("id1-s1dataB-2", s1dataB.get(1));
+ assertEquals("id1-s1dataB-3", s1dataB.get(2));
+ assertEquals(3, s1dataC.size());
+ assertNull(s1dataC.get(0));
+ assertNull(s1dataC.get(1));
+ assertNull(s1dataC.get(2));
+ }
+ {
+ assertEquals("2", result.get(1).get("id"));
+ List a = (List)result.get(1).get("a");
+ List s1dataA = (List)result.get(1).get("s1dataA");
+ List s1dataB = (List)result.get(1).get("s1dataB");
+ List s1dataC = (List)result.get(1).get("s1dataC");
+ assertTrue(a==null || a.size()==0);
+ assertEquals(1, s1dataA.size());
+ assertNull(s1dataA.get(0));
+ assertEquals(1, s1dataB.size());
+ assertEquals("id2-s1dataB-1", s1dataB.get(0));
+ assertEquals(1, s1dataC.size());
+ assertNull(s1dataC.get(0));
+ }
+ {
+ assertEquals("3", result.get(2).get("id"));
+ List a = (List)result.get(2).get("a");
+ List s1dataA = (List)result.get(2).get("s1dataA");
+ List s1dataB = (List)result.get(2).get("s1dataB");
+ List s1dataC = (List)result.get(2).get("s1dataC");
+ assertTrue(a==null || a.size()==0);
+ assertEquals(1, s1dataA.size());
+ assertEquals("id3-s1dataA-1", s1dataA.get(0));
+ assertEquals(1, s1dataB.size());
+ assertNull(s1dataB.get(0));
+ assertEquals(1, s1dataC.size());
+ assertNull(s1dataC.get(0));
+ }
+ {
+ assertEquals("4", result.get(3).get("id"));
+ List a = (List)result.get(3).get("a");
+ List s1dataA = (List)result.get(3).get("s1dataA");
+ List s1dataB = (List)result.get(3).get("s1dataB");
+ List s1dataC = (List)result.get(3).get("s1dataC");
+ assertTrue(a==null || a.size()==0);
+ assertEquals(1, s1dataA.size());
+ assertEquals("id4-s1dataA-1", s1dataA.get(0));
+ assertEquals(1, s1dataB.size());
+ assertEquals("id4-s1dataB-1", s1dataB.get(0));
+ assertEquals(1, s1dataC.size());
+ assertEquals("id4-s1dataC-1", s1dataC.get(0));
+ }
+ {
+ assertEquals("5", result.get(4).get("id"));
+ List a = (List)result.get(4).get("a");
+ List s1dataA = (List)result.get(4).get("s1dataA");
+ List s1dataB = (List)result.get(4).get("s1dataB");
+ List s1dataC = (List)result.get(4).get("s1dataC");
+ assertTrue(a==null || a.size()==0);
+ assertEquals(1, s1dataA.size());
+ assertNull(s1dataA.get(0));
+ assertEquals(1, s1dataB.size());
+ assertNull(s1dataB.get(0));
+ assertEquals(1, s1dataC.size());
+ assertEquals("id5-s1dataC-1", s1dataC.get(0));
+ }
+ {
+ assertEquals("6", result.get(5).get("id"));
+ List a = (List)result.get(5).get("a");
+ List s1dataA = (List)result.get(5).get("s1dataA");
+ List s1dataB = (List)result.get(5).get("s1dataB");
+ List s1dataC = (List)result.get(5).get("s1dataC");
+ assertTrue(a==null || a.size()==0);
+ assertEquals(3, s1dataA.size());
+ assertEquals("id6-s1dataA-1", s1dataA.get(0));
+ assertEquals("id6-s1dataA-2", s1dataA.get(1));
+ assertNull(s1dataA.get(2));
+ assertEquals(3, s1dataB.size());
+ assertEquals("id6-s1dataB-1", s1dataB.get(0));
+ assertEquals("id6-s1dataB-2", s1dataB.get(1));
+ assertEquals("id6-s1dataB-3", s1dataB.get(2));
+ assertEquals(3, s1dataC.size());
+ assertEquals("id6-s1dataC-1", s1dataC.get(0));
+ assertNull(s1dataC.get(1));
+ assertEquals("id6-s1dataC-3", s1dataC.get(2));
+ }
+ }
+
+ @Test
+ public void testMultiValuedFlatten() throws Exception {
+ Map entityAttrs = createMap("name", "e", "url", "testdata.xml",
+ XPathEntityProcessor.FOR_EACH, "/root");
+ List fields = new ArrayList();
+ fields.add(createMap("column", "a", "xpath", "/root/a" ,"flatten","true"));
+ Context c = getContext(null,
+ new VariableResolver(), getDataSource(testXmlFlatten), Context.FULL_DUMP, fields, entityAttrs);
+ XPathEntityProcessor xPathEntityProcessor = new XPathEntityProcessor();
+ xPathEntityProcessor.init(c);
+ Map<String, Object> result = null;
+ while (true) {
+ Map<String, Object> row = xPathEntityProcessor.nextRow();
+ if (row == null)
+ break;
+ result = row;
+ }
+ assertEquals("1B2", result.get("a"));
+ }
+
+ @Test
+ public void withFieldsAndXpathStream() throws Exception {
+ final Object monitor = new Object();
+ final boolean[] done = new boolean[1];
+
+ Map entityAttrs = createMap("name", "e", "url", "cd.xml",
+ XPathEntityProcessor.FOR_EACH, "/catalog/cd", "stream", "true", "batchSize","1");
+ List fields = new ArrayList();
+ fields.add(createMap("column", "title", "xpath", "/catalog/cd/title"));
+ fields.add(createMap("column", "artist", "xpath", "/catalog/cd/artist"));
+ fields.add(createMap("column", "year", "xpath", "/catalog/cd/year"));
+ Context c = getContext(null,
+ new VariableResolver(), getDataSource(cdData), Context.FULL_DUMP, fields, entityAttrs);
+ XPathEntityProcessor xPathEntityProcessor = new XPathEntityProcessor() {
+ private int count;
+
+ @Override
+ protected Map<String, Object> readRow(Map<String, Object> record,
+ String xpath) {
+ synchronized (monitor) {
+ if (simulateSlowReader && !done[0]) {
+ try {
+ monitor.wait(100);
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ return super.readRow(record, xpath);
+ }
+ };
+
+ if (simulateSlowResultProcessor) {
+ xPathEntityProcessor.blockingQueueSize = 1;
+ }
+ xPathEntityProcessor.blockingQueueTimeOut = 1;
+ xPathEntityProcessor.blockingQueueTimeOutUnits = TimeUnit.MICROSECONDS;
+
+ xPathEntityProcessor.init(c);
+ List<Map<String, Object>> result = new ArrayList<>();
+ while (true) {
+ if (rowsToRead >= 0 && result.size() >= rowsToRead) {
+ Thread.currentThread().interrupt();
+ }
+ Map<String, Object> row = xPathEntityProcessor.nextRow();
+ if (row == null)
+ break;
+ result.add(row);
+ if (simulateSlowResultProcessor) {
+ synchronized (xPathEntityProcessor.publisherThread) {
+ if (xPathEntityProcessor.publisherThread.isAlive()) {
+ xPathEntityProcessor.publisherThread.wait(1000);
+ }
+ }
+ }
+ }
+
+ synchronized (monitor) {
+ done[0] = true;
+ monitor.notify();
+ }
+
+ // confirm that publisher thread stops.
+ xPathEntityProcessor.publisherThread.join(1000);
+ assertEquals("Expected thread to stop", false, xPathEntityProcessor.publisherThread.isAlive());
+
+ assertEquals(rowsToRead < 0 ? 3 : rowsToRead, result.size());
+
+ if (rowsToRead < 0) {
+ assertEquals("Empire Burlesque", result.get(0).get("title"));
+ assertEquals("Bonnie Tyler", result.get(1).get("artist"));
+ assertEquals("1982", result.get(2).get("year"));
+ }
+ }
+
+ @Test
+ public void withFieldsAndXpathStreamContinuesOnTimeout() throws Exception {
+ simulateSlowReader = true;
+ withFieldsAndXpathStream();
+ }
+
+ @Test
+ public void streamWritesMessageAfterBlockedAttempt() throws Exception {
+ simulateSlowResultProcessor = true;
+ withFieldsAndXpathStream();
+ }
+
+ @Test
+ public void streamStopsAfterInterrupt() throws Exception {
+ simulateSlowResultProcessor = true;
+ rowsToRead = 1;
+ withFieldsAndXpathStream();
+ }
+
+ @Test
+ public void withDefaultSolrAndXsl() throws Exception {
+ File tmpdir = createTempDir().toFile();
+ AbstractDataImportHandlerTestCase.createFile(tmpdir, "x.xsl", xsl.getBytes(StandardCharsets.UTF_8),
+ false);
+
+ Map entityAttrs = createMap("name", "e",
+ XPathEntityProcessor.USE_SOLR_ADD_SCHEMA, "true", "xsl", ""
+ + new File(tmpdir, "x.xsl").toURI(), "url", "cd.xml");
+ Context c = getContext(null,
+ new VariableResolver(), getDataSource(cdData), Context.FULL_DUMP, null, entityAttrs);
+ XPathEntityProcessor xPathEntityProcessor = new XPathEntityProcessor();
+ xPathEntityProcessor.init(c);
+ List<Map<String, Object>> result = new ArrayList<>();
+ while (true) {
+ Map<String, Object> row = xPathEntityProcessor.nextRow();
+ if (row == null)
+ break;
+ result.add(row);
+ }
+ assertEquals(3, result.size());
+ assertEquals("Empire Burlesque", result.get(0).get("title"));
+ assertEquals("Bonnie Tyler", result.get(1).get("artist"));
+ assertEquals("1982", result.get(2).get("year"));
+ }
+
+ private DataSource<Reader> getDataSource(final String xml) {
+ return new DataSource<Reader>() {
+
+ @Override
+ public void init(Context context, Properties initProps) {
+ }
+
+ @Override
+ public void close() {
+ }
+
+ @Override
+ public Reader getData(String query) {
+ return new StringReader(xml);
+ }
+ };
+ }
+
+ private static final String xsl = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
+ + "<xsl:stylesheet version=\"1.0\"\n"
+ + "xmlns:xsl=\"http://www.w3.org/1999/XSL/Transform\">\n"
+ + "<xsl:output version='1.0' method='xml' encoding='UTF-8' indent='yes'/>\n"
+ + "\n"
+ + "<xsl:template match=\"/\">\n"
+ + " <add> \n"
+ + " <xsl:for-each select=\"catalog/cd\">\n"
+ + " <doc>\n"
+ + " <field name=\"title\"><xsl:value-of select=\"title\"/></field>\n"
+ + " <field name=\"artist\"><xsl:value-of select=\"artist\"/></field>\n"
+ + " <field name=\"country\"><xsl:value-of select=\"country\"/></field>\n"
+ + " <field name=\"company\"><xsl:value-of select=\"company\"/></field> \n"
+ + " <field name=\"price\"><xsl:value-of select=\"price\"/></field>\n"
+ + " <field name=\"year\"><xsl:value-of select=\"year\"/></field> \n"
+ + " </doc>\n"
+ + " </xsl:for-each>\n"
+ + " </add> \n"
+ + "</xsl:template>\n" + "</xsl:stylesheet>";
+
+ private static final String cdData = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
+ + "<?xml-stylesheet type=\"text/xsl\" href=\"solr.xsl\"?>\n"
+ + "<catalog>\n"
+ + "\t<cd>\n"
+ + "\t\t<title>Empire Burlesque</title>\n"
+ + "\t\t<artist>Bob Dylan</artist>\n"
+ + "\t\t<country>USA</country>\n"
+ + "\t\t<company>Columbia</company>\n"
+ + "\t\t<price>10.90</price>\n"
+ + "\t\t<year>1985</year>\n"
+ + "\t</cd>\n"
+ + "\t<cd>\n"
+ + "\t\t<title>Hide your heart</title>\n"
+ + "\t\t<artist>Bonnie Tyler</artist>\n"
+ + "\t\t<country>UK</country>\n"
+ + "\t\t<company>CBS Records</company>\n"
+ + "\t\t<price>9.90</price>\n"
+ + "\t\t<year>1988</year>\n"
+ + "\t</cd>\n"
+ + "\t<cd>\n"
+ + "\t\t<title>Greatest Hits</title>\n"
+ + "\t\t<artist>Dolly Parton</artist>\n"
+ + "\t\t<country>USA</country>\n"
+ + "\t\t<company>RCA</company>\n"
+ + "\t\t<price>9.90</price>\n"
+ + "\t\t<year>1982</year>\n" + "\t</cd>\n" + "</catalog>\t";
+
+ private static final String testXml = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<!DOCTYPE root [\n<!ENTITY uuml \"ü\" >\n]>\n<root><a>1</a><a>2</a><a>ü</a></root>";
+
+ private static final String testXmlFlatten = "<?xml version=\"1.0\" encoding=\"UTF-8\"?><root><a>1<b>B</b>2</a></root>";
+
+ private static final String textMultipleDocuments =
+ "<?xml version=\"1.0\" ?>" +
+ "<documents>" +
+ " <doc>" +
+ " <id>1</id>" +
+ " <a>id1-a1</a>" +
+ " <a>id1-a2</a>" +
+ " <sec1>" +
+ " <s1dataA>id1-s1dataA-1</s1dataA>" +
+ " <s1dataB>id1-s1dataB-1</s1dataB>" +
+ " </sec1>" +
+ " <sec1>" +
+ " <s1dataB>id1-s1dataB-2</s1dataB>" +
+ " </sec1>" +
+ " <sec1>" +
+ " <s1dataA>id1-s1dataA-3</s1dataA>" +
+ " <s1dataB>id1-s1dataB-3</s1dataB>" +
+ " </sec1>" +
+ " </doc>" +
+ " <doc>" +
+ " <id>2</id>" +
+ " <sec1>" +
+ " <s1dataB>id2-s1dataB-1</s1dataB>" +
+ " </sec1>" +
+ " </doc>" +
+ " <doc>" +
+ " <id>3</id>" +
+ " <sec1>" +
+ " <s1dataA>id3-s1dataA-1</s1dataA>" +
+ " </sec1>" +
+ " </doc>" +
+ " <doc>" +
+ " <id>4</id>" +
+ " <sec1>" +
+ " <s1dataA>id4-s1dataA-1</s1dataA>" +
+ " <s1dataB>id4-s1dataB-1</s1dataB>" +
+ " <s1dataC>id4-s1dataC-1</s1dataC>" +
+ " </sec1>" +
+ " </doc>" +
+ " <doc>" +
+ " <id>5</id>" +
+ " <sec1>" +
+ " <s1dataC>id5-s1dataC-1</s1dataC>" +
+ " </sec1>" +
+ " </doc>" +
+ " <doc>" +
+ " <id>6</id>" +
+ " <sec1>" +
+ " <s1dataA>id6-s1dataA-1</s1dataA>" +
+ " <s1dataB>id6-s1dataB-1</s1dataB>" +
+ " <s1dataC>id6-s1dataC-1</s1dataC>" +
+ " </sec1>" +
+ " <sec1>" +
+ " <s1dataA>id6-s1dataA-2</s1dataA>" +
+ " <s1dataB>id6-s1dataB-2</s1dataB>" +
+ " </sec1>" +
+ " <sec1>" +
+ " <s1dataB>id6-s1dataB-3</s1dataB>" +
+ " <s1dataC>id6-s1dataC-3</s1dataC>" +
+ " </sec1>" +
+ " </doc>" +
+ "</documents>"
+ ;
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestXPathRecordReader.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestXPathRecordReader.java b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestXPathRecordReader.java
new file mode 100644
index 0000000..d8e3cbe
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestXPathRecordReader.java
@@ -0,0 +1,598 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import java.io.StringReader;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.junit.Test;
+
+/**
+ * <p> Test for XPathRecordReader </p>
+ *
+ *
+ * @since solr 1.3
+ */
+public class TestXPathRecordReader extends AbstractDataImportHandlerTestCase {
+ @Test
+ public void testBasic() {
+ String xml="<root>\n"
+ + " <b><c>Hello C1</c>\n"
+ + " <c>Hello C1</c>\n"
+ + " </b>\n"
+ + " <b><c>Hello C2</c>\n"
+ + " </b>\n"
+ + "</root>";
+ XPathRecordReader rr = new XPathRecordReader("/root/b");
+ rr.addField("c", "/root/b/c", true);
+ List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
+ assertEquals(2, l.size());
+ assertEquals(2, ((List) l.get(0).get("c")).size());
+ assertEquals(1, ((List) l.get(1).get("c")).size());
+ }
+
+ @Test
+ public void testAttributes() {
+ String xml="<root>\n"
+ + " <b a=\"x0\" b=\"y0\" />\n"
+ + " <b a=\"x1\" b=\"y1\" />\n"
+ + " <b a=\"x2\" b=\"y2\" />\n"
+ + "</root>";
+ XPathRecordReader rr = new XPathRecordReader("/root/b");
+ rr.addField("a", "/root/b/@a", false);
+ rr.addField("b", "/root/b/@b", false);
+ List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
+ assertEquals(3, l.size());
+ assertEquals("x0", l.get(0).get("a"));
+ assertEquals("x1", l.get(1).get("a"));
+ assertEquals("x2", l.get(2).get("a"));
+ assertEquals("y0", l.get(0).get("b"));
+ assertEquals("y1", l.get(1).get("b"));
+ assertEquals("y2", l.get(2).get("b"));
+ }
+
+ @Test
+ public void testAttrInRoot(){
+ String xml="<r>\n" +
+ "<merchantProduct id=\"814636051\" mid=\"189973\">\n" +
+ " <in_stock type=\"stock-4\" />\n" +
+ " <condition type=\"cond-0\" />\n" +
+ " <price>301.46</price>\n" +
+ " </merchantProduct>\n" +
+ "<merchantProduct id=\"814636052\" mid=\"189974\">\n" +
+ " <in_stock type=\"stock-5\" />\n" +
+ " <condition type=\"cond-1\" />\n" +
+ " <price>302.46</price>\n" +
+ " </merchantProduct>\n" +
+ "\n" +
+ "</r>";
+ XPathRecordReader rr = new XPathRecordReader("/r/merchantProduct");
+ rr.addField("id", "/r/merchantProduct/@id", false);
+ rr.addField("mid", "/r/merchantProduct/@mid", false);
+ rr.addField("price", "/r/merchantProduct/price", false);
+ rr.addField("conditionType", "/r/merchantProduct/condition/@type", false);
+ List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
+ Map<String, Object> m = l.get(0);
+ assertEquals("814636051", m.get("id"));
+ assertEquals("189973", m.get("mid"));
+ assertEquals("301.46", m.get("price"));
+ assertEquals("cond-0", m.get("conditionType"));
+
+ m = l.get(1);
+ assertEquals("814636052", m.get("id"));
+ assertEquals("189974", m.get("mid"));
+ assertEquals("302.46", m.get("price"));
+ assertEquals("cond-1", m.get("conditionType"));
+ }
+
+ @Test
+ public void testAttributes2Level() {
+ String xml="<root>\n"
+ + "<a>\n <b a=\"x0\" b=\"y0\" />\n"
+ + " <b a=\"x1\" b=\"y1\" />\n"
+ + " <b a=\"x2\" b=\"y2\" />\n"
+ + " </a>"
+ + "</root>";
+ XPathRecordReader rr = new XPathRecordReader("/root/a/b");
+ rr.addField("a", "/root/a/b/@a", false);
+ rr.addField("b", "/root/a/b/@b", false);
+ List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
+ assertEquals(3, l.size());
+ assertEquals("x0", l.get(0).get("a"));
+ assertEquals("y1", l.get(1).get("b"));
+ }
+
+ @Test
+ public void testAttributes2LevelHetero() {
+ String xml="<root>\n"
+ + "<a>\n <b a=\"x0\" b=\"y0\" />\n"
+ + " <b a=\"x1\" b=\"y1\" />\n"
+ + " <b a=\"x2\" b=\"y2\" />\n"
+ + " </a>"
+ + "<x>\n <b a=\"x4\" b=\"y4\" />\n"
+ + " <b a=\"x5\" b=\"y5\" />\n"
+ + " <b a=\"x6\" b=\"y6\" />\n"
+ + " </x>"
+ + "</root>";
+ XPathRecordReader rr = new XPathRecordReader("/root/a | /root/x");
+ rr.addField("a", "/root/a/b/@a", false);
+ rr.addField("b", "/root/a/b/@b", false);
+ rr.addField("a", "/root/x/b/@a", false);
+ rr.addField("b", "/root/x/b/@b", false);
+
+ final List<Map<String, Object>> a = new ArrayList<>();
+ final List<Map<String, Object>> x = new ArrayList<>();
+ rr.streamRecords(new StringReader(xml), (record, xpath) -> {
+ if (record == null) return;
+ if (xpath.equals("/root/a")) a.add(record);
+ if (xpath.equals("/root/x")) x.add(record);
+ });
+
+ assertEquals(1, a.size());
+ assertEquals(1, x.size());
+ }
+
+ @Test
+ public void testAttributes2LevelMissingAttrVal() {
+ String xml="<root>\n"
+ + "<a>\n <b a=\"x0\" b=\"y0\" />\n"
+ + " <b a=\"x1\" b=\"y1\" />\n"
+ + " </a>"
+ + "<a>\n <b a=\"x3\" />\n"
+ + " <b b=\"y4\" />\n"
+ + " </a>"
+ + "</root>";
+ XPathRecordReader rr = new XPathRecordReader("/root/a");
+ rr.addField("a", "/root/a/b/@a", true);
+ rr.addField("b", "/root/a/b/@b", true);
+ List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
+ assertEquals(2, l.size());
+ assertNull(((List) l.get(1).get("a")).get(1));
+ assertNull(((List) l.get(1).get("b")).get(0));
+ }
+
+ @Test
+ public void testElems2LevelMissing() {
+ String xml="<root>\n"
+ + "\t<a>\n"
+ + "\t <b>\n\t <x>x0</x>\n"
+ + "\t <y>y0</y>\n"
+ + "\t </b>\n"
+ + "\t <b>\n\t <x>x1</x>\n"
+ + "\t <y>y1</y>\n"
+ + "\t </b>\n"
+ + "\t </a>\n"
+ + "\t<a>\n"
+ + "\t <b>\n\t <x>x3</x>\n\t </b>\n"
+ + "\t <b>\n\t <y>y4</y>\n\t </b>\n"
+ + "\t </a>\n"
+ + "</root>";
+ XPathRecordReader rr = new XPathRecordReader("/root/a");
+ rr.addField("a", "/root/a/b/x", true);
+ rr.addField("b", "/root/a/b/y", true);
+ List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
+ assertEquals(2, l.size());
+ assertNull(((List) l.get(1).get("a")).get(1));
+ assertNull(((List) l.get(1).get("b")).get(0));
+ }
+
+ @Test
+ public void testElems2LevelEmpty() {
+ String xml="<root>\n"
+ + "\t<a>\n"
+ + "\t <b>\n\t <x>x0</x>\n"
+ + "\t <y>y0</y>\n"
+ + "\t </b>\n"
+ + "\t <b>\n\t <x></x>\n" // empty
+ + "\t <y>y1</y>\n"
+ + "\t </b>\n"
+ + "\t</a>\n"
+ + "</root>";
+ XPathRecordReader rr = new XPathRecordReader("/root/a");
+ rr.addField("a", "/root/a/b/x", true);
+ rr.addField("b", "/root/a/b/y", true);
+ List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
+ assertEquals(1, l.size());
+ assertEquals("x0",((List) l.get(0).get("a")).get(0));
+ assertEquals("y0",((List) l.get(0).get("b")).get(0));
+ assertEquals("",((List) l.get(0).get("a")).get(1));
+ assertEquals("y1",((List) l.get(0).get("b")).get(1));
+ }
+
+ @Test
+ public void testMixedContent() {
+ String xml = "<xhtml:p xmlns:xhtml=\"http://xhtml.com/\" >This text is \n" +
+ " <xhtml:b>bold</xhtml:b> and this text is \n" +
+ " <xhtml:u>underlined</xhtml:u>!\n" +
+ "</xhtml:p>";
+ XPathRecordReader rr = new XPathRecordReader("/p");
+ rr.addField("p", "/p", true);
+ rr.addField("b", "/p/b", true);
+ rr.addField("u", "/p/u", true);
+ List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
+ Map<String, Object> row = l.get(0);
+
+ assertEquals("bold", ((List) row.get("b")).get(0));
+ assertEquals("underlined", ((List) row.get("u")).get(0));
+ String p = (String) ((List) row.get("p")).get(0);
+ assertTrue(p.contains("This text is"));
+ assertTrue(p.contains("and this text is"));
+ assertTrue(p.contains("!"));
+ // Should not contain content from child elements
+ assertFalse(p.contains("bold"));
+ }
+
+ @Test
+ public void testMixedContentFlattened() {
+ String xml = "<xhtml:p xmlns:xhtml=\"http://xhtml.com/\" >This text is \n" +
+ " <xhtml:b>bold</xhtml:b> and this text is \n" +
+ " <xhtml:u>underlined</xhtml:u>!\n" +
+ "</xhtml:p>";
+ XPathRecordReader rr = new XPathRecordReader("/p");
+ rr.addField("p", "/p", false, XPathRecordReader.FLATTEN);
+ List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
+ Map<String, Object> row = l.get(0);
+ assertEquals("This text is \n" +
+ " bold and this text is \n" +
+ " underlined!", ((String)row.get("p")).trim() );
+ }
+
+ @Test
+ public void testElems2LevelWithAttrib() {
+ String xml = "<root>\n\t<a>\n\t <b k=\"x\">\n"
+ + "\t <x>x0</x>\n"
+ + "\t <y></y>\n" // empty
+ + "\t </b>\n"
+ + "\t <b k=\"y\">\n"
+ + "\t <x></x>\n" // empty
+ + "\t <y>y1</y>\n"
+ + "\t </b>\n"
+ + "\t <b k=\"z\">\n"
+ + "\t <x>x2</x>\n"
+ + "\t <y>y2</y>\n"
+ + "\t </b>\n"
+ + "\t </a>\n"
+ + "\t <a>\n\t <b>\n"
+ + "\t <x>x3</x>\n"
+ + "\t </b>\n"
+ + "\t <b>\n"
+ + "\t <y>y4</y>\n"
+ + "\t </b>\n"
+ + "\t </a>\n"
+ + "</root>";
+ XPathRecordReader rr = new XPathRecordReader("/root/a");
+ rr.addField("x", "/root/a/b[@k]/x", true);
+ rr.addField("y", "/root/a/b[@k]/y", true);
+ List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
+ assertEquals(2, l.size());
+ assertEquals(3, ((List) l.get(0).get("x")).size());
+ assertEquals(3, ((List) l.get(0).get("y")).size());
+ assertEquals("x0", ((List) l.get(0).get("x")).get(0));
+ assertEquals("", ((List) l.get(0).get("y")).get(0));
+ assertEquals("", ((List) l.get(0).get("x")).get(1));
+ assertEquals("y1", ((List) l.get(0).get("y")).get(1));
+ assertEquals("x2", ((List) l.get(0).get("x")).get(2));
+ assertEquals("y2", ((List) l.get(0).get("y")).get(2));
+ assertEquals(0, l.get(1).size());
+ }
+
+ @Test
+ public void testElems2LevelWithAttribMultiple() {
+ String xml="<root>\n"
+ + "\t<a>\n\t <b k=\"x\" m=\"n\" >\n"
+ + "\t <x>x0</x>\n"
+ + "\t <y>y0</y>\n"
+ + "\t </b>\n"
+ + "\t <b k=\"y\" m=\"p\">\n"
+ + "\t <x>x1</x>\n"
+ + "\t <y>y1</y>\n"
+ + "\t </b>\n"
+ + "\t </a>\n"
+ + "\t<a>\n\t <b k=\"x\">\n"
+ + "\t <x>x3</x>\n"
+ + "\t </b>\n"
+ + "\t <b m=\"n\">\n"
+ + "\t <y>y4</y>\n"
+ + "\t </b>\n"
+ + "\t </a>\n"
+ + "</root>";
+ XPathRecordReader rr = new XPathRecordReader("/root/a");
+ rr.addField("x", "/root/a/b[@k][@m='n']/x", true);
+ rr.addField("y", "/root/a/b[@k][@m='n']/y", true);
+ List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
+ assertEquals(2, l.size());
+ assertEquals(1, ((List) l.get(0).get("x")).size());
+ assertEquals(1, ((List) l.get(0).get("y")).size());
+ assertEquals(0, l.get(1).size());
+ }
+
+ @Test
+ public void testElems2LevelWithAttribVal() {
+ String xml="<root>\n\t<a>\n <b k=\"x\">\n"
+ + "\t <x>x0</x>\n"
+ + "\t <y>y0</y>\n"
+ + "\t </b>\n"
+ + "\t <b k=\"y\">\n"
+ + "\t <x>x1</x>\n"
+ + "\t <y>y1</y>\n"
+ + "\t </b>\n"
+ + "\t </a>\n"
+ + "\t <a>\n <b><x>x3</x></b>\n"
+ + "\t <b><y>y4</y></b>\n"
+ + "\t</a>\n" + "</root>";
+ XPathRecordReader rr = new XPathRecordReader("/root/a");
+ rr.addField("x", "/root/a/b[@k='x']/x", true);
+ rr.addField("y", "/root/a/b[@k='x']/y", true);
+ List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
+ assertEquals(2, l.size());
+ assertEquals(1, ((List) l.get(0).get("x")).size());
+ assertEquals(1, ((List) l.get(0).get("y")).size());
+ assertEquals(0, l.get(1).size());
+ }
+
+ @Test
+ public void testAttribValWithSlash() {
+ String xml = "<root><b>\n" +
+ " <a x=\"a/b\" h=\"hello-A\"/> \n" +
+ "</b></root>";
+ XPathRecordReader rr = new XPathRecordReader("/root/b");
+ rr.addField("x", "/root/b/a[@x='a/b']/@h", false);
+ List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
+ assertEquals(1, l.size());
+ Map<String, Object> m = l.get(0);
+ assertEquals("hello-A", m.get("x"));
+ }
+
+ @Test
+ public void testUnsupported_Xpaths() {
+ String xml = "<root><b><a x=\"a/b\" h=\"hello-A\"/> </b></root>";
+ XPathRecordReader rr=null;
+ try {
+ rr = new XPathRecordReader("//b");
+ fail("A RuntimeException was expected: //b forEach cannot begin with '//'.");
+ }
+ catch (RuntimeException ex) { }
+ try {
+ rr.addField("bold" ,"b", false);
+ fail("A RuntimeException was expected: 'b' xpaths must begin with '/'.");
+ }
+ catch (RuntimeException ex) { }
+ }
+
+ @Test
+ public void testAny_decendent_from_root() {
+ XPathRecordReader rr = new XPathRecordReader("/anyd/contenido");
+ rr.addField("descdend", "//boo", true);
+ rr.addField("inr_descd","//boo/i", false);
+ rr.addField("cont", "/anyd/contenido", false);
+ rr.addField("id", "/anyd/contenido/@id", false);
+ rr.addField("status", "/anyd/status", false);
+ rr.addField("title", "/anyd/contenido/titulo", false,XPathRecordReader.FLATTEN);
+ rr.addField("resume", "/anyd/contenido/resumen",false);
+ rr.addField("text", "/anyd/contenido/texto", false);
+
+ String xml="<anyd>\n"
+ + " this <boo>top level</boo> is ignored because it is external to the forEach\n"
+ + " <status>as is <boo>this element</boo></status>\n"
+ + " <contenido id=\"10097\" idioma=\"cat\">\n"
+ + " This one is <boo>not ignored as it's</boo> inside a forEach\n"
+ + " <antetitulo><i> big <boo>antler</boo></i></antetitulo>\n"
+ + " <titulo> My <i>flattened <boo>title</boo></i> </titulo>\n"
+ + " <resumen> My summary <i>skip this!</i> </resumen>\n"
+ + " <texto> <boo>Within the body of</boo>My text</texto>\n"
+ + " <p>Access <boo>inner <i>sub clauses</i> as well</boo></p>\n"
+ + " </contenido>\n"
+ + "</anyd>";
+
+ List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
+ assertEquals(1, l.size());
+ Map<String, Object> m = l.get(0);
+ assertEquals("This one is inside a forEach", m.get("cont").toString().trim());
+ assertEquals("10097" ,m.get("id"));
+ assertEquals("My flattened title" ,m.get("title").toString().trim());
+ assertEquals("My summary" ,m.get("resume").toString().trim());
+ assertEquals("My text" ,m.get("text").toString().trim());
+ assertEquals("not ignored as it's",(String) ((List) m.get("descdend")).get(0) );
+ assertEquals("antler" ,(String) ((List) m.get("descdend")).get(1) );
+ assertEquals("Within the body of" ,(String) ((List) m.get("descdend")).get(2) );
+ assertEquals("inner as well" ,(String) ((List) m.get("descdend")).get(3) );
+ assertEquals("sub clauses" ,m.get("inr_descd").toString().trim());
+ }
+
+ @Test
+ public void testAny_decendent_of_a_child1() {
+ XPathRecordReader rr = new XPathRecordReader("/anycd");
+ rr.addField("descdend", "/anycd//boo", true);
+
+ // same test string as above but checking to see if *all* //boo's are collected
+ String xml="<anycd>\n"
+ + " this <boo>top level</boo> is ignored because it is external to the forEach\n"
+ + " <status>as is <boo>this element</boo></status>\n"
+ + " <contenido id=\"10097\" idioma=\"cat\">\n"
+ + " This one is <boo>not ignored as it's</boo> inside a forEach\n"
+ + " <antetitulo><i> big <boo>antler</boo></i></antetitulo>\n"
+ + " <titulo> My <i>flattened <boo>title</boo></i> </titulo>\n"
+ + " <resumen> My summary <i>skip this!</i> </resumen>\n"
+ + " <texto> <boo>Within the body of</boo>My text</texto>\n"
+ + " <p>Access <boo>inner <i>sub clauses</i> as well</boo></p>\n"
+ + " </contenido>\n"
+ + "</anycd>";
+
+ List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
+ assertEquals(1, l.size());
+ Map<String, Object> m = l.get(0);
+ assertEquals("top level" ,(String) ((List) m.get("descdend")).get(0) );
+ assertEquals("this element" ,(String) ((List) m.get("descdend")).get(1) );
+ assertEquals("not ignored as it's",(String) ((List) m.get("descdend")).get(2) );
+ assertEquals("antler" ,(String) ((List) m.get("descdend")).get(3) );
+ assertEquals("title" ,(String) ((List) m.get("descdend")).get(4) );
+ assertEquals("Within the body of" ,(String) ((List) m.get("descdend")).get(5) );
+ assertEquals("inner as well" ,(String) ((List) m.get("descdend")).get(6) );
+ }
+
+ @Test
+ public void testAny_decendent_of_a_child2() {
+ XPathRecordReader rr = new XPathRecordReader("/anycd");
+ rr.addField("descdend", "/anycd/contenido//boo", true);
+
+ // same test string as above but checking to see if *some* //boo's are collected
+ String xml="<anycd>\n"
+ + " this <boo>top level</boo> is ignored because it is external to the forEach\n"
+ + " <status>as is <boo>this element</boo></status>\n"
+ + " <contenido id=\"10097\" idioma=\"cat\">\n"
+ + " This one is <boo>not ignored as it's</boo> inside a forEach\n"
+ + " <antetitulo><i> big <boo>antler</boo></i></antetitulo>\n"
+ + " <titulo> My <i>flattened <boo>title</boo></i> </titulo>\n"
+ + " <resumen> My summary <i>skip this!</i> </resumen>\n"
+ + " <texto> <boo>Within the body of</boo>My text</texto>\n"
+ + " <p>Access <boo>inner <i>sub clauses</i> as well</boo></p>\n"
+ + " </contenido>\n"
+ + "</anycd>";
+
+ List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
+ assertEquals(1, l.size());
+ Map<String, Object> m = l.get(0);
+ assertEquals("not ignored as it's",((List) m.get("descdend")).get(0) );
+ assertEquals("antler" ,((List) m.get("descdend")).get(1) );
+ assertEquals("title" ,((List) m.get("descdend")).get(2) );
+ assertEquals("Within the body of" ,((List) m.get("descdend")).get(3) );
+ assertEquals("inner as well" ,((List) m.get("descdend")).get(4) );
+ }
+
+ @Test
+ public void testAnother() {
+ String xml="<root>\n"
+ + " <contenido id=\"10097\" idioma=\"cat\">\n"
+ + " <antetitulo></antetitulo>\n"
+ + " <titulo> This is my title </titulo>\n"
+ + " <resumen> This is my summary </resumen>\n"
+ + " <texto> This is the body of my text </texto>\n"
+ + " </contenido>\n"
+ + "</root>";
+ XPathRecordReader rr = new XPathRecordReader("/root/contenido");
+ rr.addField("id", "/root/contenido/@id", false);
+ rr.addField("title", "/root/contenido/titulo", false);
+ rr.addField("resume","/root/contenido/resumen",false);
+ rr.addField("text", "/root/contenido/texto", false);
+
+ List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
+ assertEquals(1, l.size());
+ Map<String, Object> m = l.get(0);
+ assertEquals("10097", m.get("id"));
+ assertEquals("This is my title", m.get("title").toString().trim());
+ assertEquals("This is my summary", m.get("resume").toString().trim());
+ assertEquals("This is the body of my text", m.get("text").toString()
+ .trim());
+ }
+
+ @Test
+ public void testSameForEachAndXpath(){
+ String xml="<root>\n" +
+ " <cat>\n" +
+ " <name>hello</name>\n" +
+ " </cat>\n" +
+ " <item name=\"item name\"/>\n" +
+ "</root>";
+ XPathRecordReader rr = new XPathRecordReader("/root/cat/name");
+ rr.addField("catName", "/root/cat/name",false);
+ List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
+ assertEquals("hello",l.get(0).get("catName"));
+ }
+
+ @Test
+ public void testPutNullTest(){
+ String xml = "<root>\n" +
+ " <i>\n" +
+ " <x>\n" +
+ " <a>A.1.1</a>\n" +
+ " <b>B.1.1</b>\n" +
+ " </x>\n" +
+ " <x>\n" +
+ " <b>B.1.2</b>\n" +
+ " <c>C.1.2</c>\n" +
+ " </x>\n" +
+ " </i>\n" +
+ " <i>\n" +
+ " <x>\n" +
+ " <a>A.2.1</a>\n" +
+ " <c>C.2.1</c>\n" +
+ " </x>\n" +
+ " <x>\n" +
+ " <b>B.2.2</b>\n" +
+ " <c>C.2.2</c>\n" +
+ " </x>\n" +
+ " </i>\n" +
+ "</root>";
+ XPathRecordReader rr = new XPathRecordReader("/root/i");
+ rr.addField("a", "/root/i/x/a", true);
+ rr.addField("b", "/root/i/x/b", true);
+ rr.addField("c", "/root/i/x/c", true);
+ List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
+ Map<String, Object> map = l.get(0);
+ List<String> a = (List<String>) map.get("a");
+ List<String> b = (List<String>) map.get("b");
+ List<String> c = (List<String>) map.get("c");
+
+ assertEquals("A.1.1",a.get(0));
+ assertEquals("B.1.1",b.get(0));
+ assertNull(c.get(0));
+
+ assertNull(a.get(1));
+ assertEquals("B.1.2",b.get(1));
+ assertEquals("C.1.2",c.get(1));
+
+ map = l.get(1);
+ a = (List<String>) map.get("a");
+ b = (List<String>) map.get("b");
+ c = (List<String>) map.get("c");
+ assertEquals("A.2.1",a.get(0));
+ assertNull(b.get(0));
+ assertEquals("C.2.1",c.get(0));
+
+ assertNull(a.get(1));
+ assertEquals("B.2.2",b.get(1));
+ assertEquals("C.2.2",c.get(1));
+ }
+
+
+ @Test
+ public void testError(){
+ String malformedXml = "<root>\n" +
+ " <node>\n" +
+ " <id>1</id>\n" +
+ " <desc>test1</desc>\n" +
+ " </node>\n" +
+ " <node>\n" +
+ " <id>2</id>\n" +
+ " <desc>test2</desc>\n" +
+ " </node>\n" +
+ " <node>\n" +
+ " <id/>3</id>\n" + // invalid XML
+ " <desc>test3</desc>\n" +
+ " </node>\n" +
+ "</root>";
+ XPathRecordReader rr = new XPathRecordReader("/root/node");
+ rr.addField("id", "/root/node/id", true);
+ rr.addField("desc", "/root/node/desc", true);
+ try {
+ rr.getAllRecords(new StringReader(malformedXml));
+ fail("A RuntimeException was expected: the input XML is invalid.");
+ } catch (Exception e) { }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestZKPropertiesWriter.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestZKPropertiesWriter.java b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestZKPropertiesWriter.java
new file mode 100644
index 0000000..c8727d0
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestZKPropertiesWriter.java
@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import java.lang.invoke.MethodHandles;
+
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+
+import org.apache.solr.cloud.AbstractZkTestCase;
+import org.apache.solr.cloud.ZkTestServer;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.util.SuppressForbidden;
+import org.apache.solr.core.CoreContainer;
+import org.apache.solr.request.LocalSolrQueryRequest;
+import org.apache.solr.request.SolrQueryRequest;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class TestZKPropertiesWriter extends AbstractDataImportHandlerTestCase {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ protected static ZkTestServer zkServer;
+
+ protected static String zkDir;
+
+ private static CoreContainer cc;
+
+ private String dateFormat = "yyyy-MM-dd HH:mm:ss.SSSSSS";
+
+ @BeforeClass
+ public static void dihZk_beforeClass() throws Exception {
+ zkDir = createTempDir("zkData").toFile().getAbsolutePath();
+ zkServer = new ZkTestServer(zkDir);
+ zkServer.run();
+
+ System.setProperty("solrcloud.skip.autorecovery", "true");
+ System.setProperty("zkHost", zkServer.getZkAddress());
+ System.setProperty("jetty.port", "0000");
+
+ AbstractZkTestCase.buildZooKeeper(zkServer.getZkHost(), zkServer.getZkAddress(), getFile("dih/solr"),
+ "dataimport-solrconfig.xml", "dataimport-schema.xml");
+
+ //initCore("solrconfig.xml", "schema.xml", getFile("dih/solr").getAbsolutePath());
+ cc = createDefaultCoreContainer(getFile("dih/solr").toPath());
+ }
+
+ @Before
+ public void beforeDihZKTest() throws Exception {
+
+ }
+
+ @After
+ public void afterDihZkTest() throws Exception {
+ MockDataSource.clearCache();
+ }
+
+
+ @AfterClass
+ public static void dihZk_afterClass() throws Exception {
+ cc.shutdown();
+
+ zkServer.shutdown();
+
+ zkServer = null;
+ zkDir = null;
+ cc = null;
+ }
+
+ @SuppressForbidden(reason = "Needs currentTimeMillis to construct date stamps")
+ @Test
+ public void testZKPropertiesWriter() throws Exception {
+ // test using ZooKeeper
+ assertTrue("Not using ZooKeeper", h.getCoreContainer().isZooKeeperAware());
+
+ // for the really slow/busy computer, we wait to make sure we have a leader before starting
+ h.getCoreContainer().getZkController().getZkStateReader().getLeaderUrl("collection1", "shard1", 30000);
+
+ assertQ("test query on empty index", request("qlkciyopsbgzyvkylsjhchghjrdf"),
+ "//result[@numFound='0']");
+
+ SimpleDateFormat errMsgFormat = new SimpleDateFormat(dateFormat, Locale.ROOT);
+
+ delQ("*:*");
+ commit();
+ SimpleDateFormat df = new SimpleDateFormat(dateFormat, Locale.ROOT);
+ Date oneSecondAgo = new Date(System.currentTimeMillis() - 1000);
+
+ Map<String, String> init = new HashMap<>();
+ init.put("dateFormat", dateFormat);
+ ZKPropertiesWriter spw = new ZKPropertiesWriter();
+ spw.init(new DataImporter(h.getCore(), "dataimport"), init);
+ Map<String, Object> props = new HashMap<>();
+ props.put("SomeDates.last_index_time", oneSecondAgo);
+ props.put("last_index_time", oneSecondAgo);
+ spw.persist(props);
+
+ List rows = new ArrayList();
+ rows.add(createMap("id", "1", "year_s", "2013"));
+ MockDataSource.setIterator("select " + df.format(oneSecondAgo) + " from dummy", rows.iterator());
+
+ h.query("/dataimport", lrf.makeRequest("command", "full-import", "dataConfig",
+ generateConfig(), "clean", "true", "commit", "true", "synchronous",
+ "true", "indent", "true"));
+ props = spw.readIndexerProperties();
+ Date entityDate = df.parse((String) props.get("SomeDates.last_index_time"));
+ Date docDate = df.parse((String) props.get("last_index_time"));
+
+ Assert.assertTrue("This date: " + errMsgFormat.format(oneSecondAgo) + " should be prior to the document date: " + errMsgFormat.format(docDate), docDate.getTime() - oneSecondAgo.getTime() > 0);
+ Assert.assertTrue("This date: " + errMsgFormat.format(oneSecondAgo) + " should be prior to the entity date: " + errMsgFormat.format(entityDate), entityDate.getTime() - oneSecondAgo.getTime() > 0);
+ assertQ(request("*:*"), "//*[@numFound='1']", "//doc/str[@name=\"year_s\"]=\"2013\"");
+
+ }
+
+ public SolrQueryRequest request(String... q) {
+ LocalSolrQueryRequest req = lrf.makeRequest(q);
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.add(req.getParams());
+ params.set("distrib", true);
+ req.setParams(params);
+ return req;
+ }
+
+ protected String generateConfig() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("<dataConfig> \n");
+ sb.append("<propertyWriter dateFormat=\"" + dateFormat + "\" type=\"ZKPropertiesWriter\" />\n");
+ sb.append("<dataSource name=\"mock\" type=\"MockDataSource\"/>\n");
+ sb.append("<document name=\"TestSimplePropertiesWriter\"> \n");
+ sb.append("<entity name=\"SomeDates\" processor=\"SqlEntityProcessor\" dataSource=\"mock\" ");
+ sb.append("query=\"select ${dih.last_index_time} from dummy\" >\n");
+ sb.append("<field column=\"AYEAR_S\" name=\"year_s\" /> \n");
+ sb.append("</entity>\n");
+ sb.append("</document> \n");
+ sb.append("</dataConfig> \n");
+ String config = sb.toString();
+ log.debug(config);
+ return config;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TripleThreatTransformer.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TripleThreatTransformer.java b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TripleThreatTransformer.java
new file mode 100644
index 0000000..2d0aadb
--- /dev/null
+++ b/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TripleThreatTransformer.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * This transformer does 3 things
+ * <ul>
+ * <li>It turns every row into 3 rows,
+ * modifying any "id" column to ensure duplicate entries in the index
+ * <li>The 2nd Row has 2x values for every column,
+ * with the added one being backwards of the original
+ * <li>The 3rd Row has an added static value
+ * </ul>
+ *
+ * Also, this does not extend Transformer.
+ */
+public class TripleThreatTransformer {
+ public Object transformRow(Map<String, Object> row) {
+ List<Map<String, Object>> rows = new ArrayList<>(3);
+ rows.add(row);
+ rows.add(addDuplicateBackwardsValues(row));
+ rows.add(new LinkedHashMap<>(row));
+ rows.get(2).put("AddAColumn_s", "Added");
+ modifyIdColumn(rows.get(1), 1);
+ modifyIdColumn(rows.get(2), 2);
+ return rows;
+ }
+ private LinkedHashMap<String,Object> addDuplicateBackwardsValues(Map<String, Object> row) {
+ LinkedHashMap<String,Object> n = new LinkedHashMap<>();
+ for(Map.Entry<String,Object> entry : row.entrySet()) {
+ String key = entry.getKey();
+ if(!"id".equalsIgnoreCase(key)) {
+ String[] vals = new String[2];
+ vals[0] = entry.getValue()==null ? "null" : entry.getValue().toString();
+ vals[1] = new StringBuilder(vals[0]).reverse().toString();
+ n.put(key, Arrays.asList(vals));
+ } else {
+ n.put(key, entry.getValue());
+ }
+ }
+ return n;
+ }
+
+ private void modifyIdColumn(Map<String, Object> row, int num) {
+ Object o = row.remove("ID");
+ if(o==null) {
+ o = row.remove("id");
+ }
+ if(o!=null) {
+ String id = o.toString();
+ id = "TripleThreat-" + num + "-" + id;
+ row.put("id", id);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7c03684/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractDIHCacheTestCase.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractDIHCacheTestCase.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractDIHCacheTestCase.java
deleted file mode 100644
index 9a0f3a7..0000000
--- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractDIHCacheTestCase.java
+++ /dev/null
@@ -1,235 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import java.io.Reader;
-import java.math.BigDecimal;
-import java.sql.Clob;
-import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-
-import javax.sql.rowset.serial.SerialClob;
-
-import org.apache.solr.handler.dataimport.AbstractDataImportHandlerTestCase.TestContext;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-
-public class AbstractDIHCacheTestCase {
- protected static final Date Feb21_2011 = new Date(1298268000000l);
- protected final String[] fieldTypes = { "INTEGER", "BIGDECIMAL", "STRING", "STRING", "FLOAT", "DATE", "CLOB" };
- protected final String[] fieldNames = { "a_id", "PI", "letter", "examples", "a_float", "a_date", "DESCRIPTION" };
- protected List<ControlData> data = new ArrayList<>();
- protected Clob APPLE = null;
-
- @Before
- public void setup() {
- try {
- APPLE = new SerialClob("Apples grow on trees and they are good to eat.".toCharArray());
- } catch (SQLException sqe) {
- Assert.fail("Could not Set up Test");
- }
-
- // The first row needs to have all non-null fields,
- // otherwise we would have to always send the fieldTypes & fieldNames as CacheProperties when building.
- data = new ArrayList<>();
- data.add(new ControlData(new Object[] {1, new BigDecimal(Math.PI), "A", "Apple", 1.11f, Feb21_2011, APPLE }));
- data.add(new ControlData(new Object[] {2, new BigDecimal(Math.PI), "B", "Ball", 2.22f, Feb21_2011, null }));
- data.add(new ControlData(new Object[] {4, new BigDecimal(Math.PI), "D", "Dog", 4.44f, Feb21_2011, null }));
- data.add(new ControlData(new Object[] {3, new BigDecimal(Math.PI), "C", "Cookie", 3.33f, Feb21_2011, null }));
- data.add(new ControlData(new Object[] {4, new BigDecimal(Math.PI), "D", "Daisy", 4.44f, Feb21_2011, null }));
- data.add(new ControlData(new Object[] {4, new BigDecimal(Math.PI), "D", "Drawing", 4.44f, Feb21_2011, null }));
- data.add(new ControlData(new Object[] {5, new BigDecimal(Math.PI), "E",
- Arrays.asList("Eggplant", "Ear", "Elephant", "Engine"), 5.55f, Feb21_2011, null }));
- }
-
- @After
- public void teardown() {
- APPLE = null;
- data = null;
- }
-
- //A limitation of this test class is that the primary key needs to be the first one in the list.
- //DIHCaches, however, can handle any field being the primary key.
- static class ControlData implements Comparable<ControlData>, Iterable<Object> {
- Object[] data;
-
- ControlData(Object[] data) {
- this.data = data;
- }
-
- @Override
- @SuppressWarnings("unchecked")
- public int compareTo(ControlData cd) {
- Comparable c1 = (Comparable) data[0];
- Comparable c2 = (Comparable) cd.data[0];
- return c1.compareTo(c2);
- }
-
- @Override
- public Iterator<Object> iterator() {
- return Arrays.asList(data).iterator();
- }
- }
-
- protected void loadData(DIHCache cache, List<ControlData> theData, String[] theFieldNames, boolean keepOrdered) {
- for (ControlData cd : theData) {
- cache.add(controlDataToMap(cd, theFieldNames, keepOrdered));
- }
- }
-
- protected List<ControlData> extractDataInKeyOrder(DIHCache cache, String[] theFieldNames) {
- List<Object[]> data = new ArrayList<>();
- Iterator<Map<String, Object>> cacheIter = cache.iterator();
- while (cacheIter.hasNext()) {
- data.add(mapToObjectArray(cacheIter.next(), theFieldNames));
- }
- return listToControlData(data);
- }
-
- //This method assumes that the Primary Keys are integers and that the first id=1.
- //It will look for id's sequentially until one is skipped, then will stop.
- protected List<ControlData> extractDataByKeyLookup(DIHCache cache, String[] theFieldNames) {
- int recId = 1;
- List<Object[]> data = new ArrayList<>();
- while (true) {
- Iterator<Map<String, Object>> listORecs = cache.iterator(recId);
- if (listORecs == null) {
- break;
- }
-
- while(listORecs.hasNext()) {
- data.add(mapToObjectArray(listORecs.next(), theFieldNames));
- }
- recId++;
- }
- return listToControlData(data);
- }
-
- protected List<ControlData> listToControlData(List<Object[]> data) {
- List<ControlData> returnData = new ArrayList<>(data.size());
- for (int i = 0; i < data.size(); i++) {
- returnData.add(new ControlData(data.get(i)));
- }
- return returnData;
- }
-
- protected Object[] mapToObjectArray(Map<String, Object> rec, String[] theFieldNames) {
- Object[] oos = new Object[theFieldNames.length];
- for (int i = 0; i < theFieldNames.length; i++) {
- oos[i] = rec.get(theFieldNames[i]);
- }
- return oos;
- }
-
- protected void compareData(List<ControlData> theControl, List<ControlData> test) {
- // The test data should come back primarily in Key order and secondarily in insertion order.
- List<ControlData> control = new ArrayList<>(theControl);
- Collections.sort(control);
-
- StringBuilder errors = new StringBuilder();
- if (test.size() != control.size()) {
- errors.append("-Returned data has " + test.size() + " records. expected: " + control.size() + "\n");
- }
- for (int i = 0; i < control.size() && i < test.size(); i++) {
- Object[] controlRec = control.get(i).data;
- Object[] testRec = test.get(i).data;
- if (testRec.length != controlRec.length) {
- errors.append("-Record indexAt=" + i + " has " + testRec.length + " data elements. extpected: " + controlRec.length + "\n");
- }
- for (int j = 0; j < controlRec.length && j < testRec.length; j++) {
- Object controlObj = controlRec[j];
- Object testObj = testRec[j];
- if (controlObj == null && testObj != null) {
- errors.append("-Record indexAt=" + i + ", Data Element indexAt=" + j + " is not NULL as expected.\n");
- } else if (controlObj != null && testObj == null) {
- errors.append("-Record indexAt=" + i + ", Data Element indexAt=" + j + " is NULL. Expected: " + controlObj + " (class="
- + controlObj.getClass().getName() + ")\n");
- } else if (controlObj != null && testObj != null && controlObj instanceof Clob) {
- String controlString = clobToString((Clob) controlObj);
- String testString = clobToString((Clob) testObj);
- if (!controlString.equals(testString)) {
- errors.append("-Record indexAt=" + i + ", Data Element indexAt=" + j + " has: " + testString + " (class=Clob) ... Expected: " + controlString
- + " (class=Clob)\n");
- }
- } else if (controlObj != null && !controlObj.equals(testObj)) {
- errors.append("-Record indexAt=" + i + ", Data Element indexAt=" + j + " has: " + testObj + " (class=" + testObj.getClass().getName()
- + ") ... Expected: " + controlObj + " (class=" + controlObj.getClass().getName() + ")\n");
- }
- }
- }
- if (errors.length() > 0) {
- Assert.fail(errors.toString());
- }
- }
-
- protected Map<String, Object> controlDataToMap(ControlData cd, String[] theFieldNames, boolean keepOrdered) {
- Map<String, Object> rec = null;
- if (keepOrdered) {
- rec = new LinkedHashMap<>();
- } else {
- rec = new HashMap<>();
- }
- for (int i = 0; i < cd.data.length; i++) {
- String fieldName = theFieldNames[i];
- Object data = cd.data[i];
- rec.put(fieldName, data);
- }
- return rec;
- }
-
- protected String stringArrayToCommaDelimitedList(String[] strs) {
- StringBuilder sb = new StringBuilder();
- for (String a : strs) {
- if (sb.length() > 0) {
- sb.append(",");
- }
- sb.append(a);
- }
- return sb.toString();
- }
-
- protected String clobToString(Clob cl) {
- StringBuilder sb = new StringBuilder();
- try {
- Reader in = cl.getCharacterStream();
- char[] cbuf = new char[1024];
- int numGot = -1;
- while ((numGot = in.read(cbuf)) != -1) {
- sb.append(String.valueOf(cbuf, 0, numGot));
- }
- } catch (Exception e) {
- Assert.fail(e.toString());
- }
- return sb.toString();
- }
-
- public static Context getContext(final Map<String, String> entityAttrs) {
- VariableResolver resolver = new VariableResolver();
- final Context delegate = new ContextImpl(null, resolver, null, null, new HashMap<String, Object>(), null, null);
- return new TestContext(entityAttrs, delegate, null, true);
- }
-
-}