You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jd...@apache.org on 2012/11/06 18:49:50 UTC
svn commit: r1406246 - in /lucene/dev/branches/branch_4x/solr: ./
contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/
contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/
Author: jdyer
Date: Tue Nov 6 17:49:49 2012
New Revision: 1406246
URL: http://svn.apache.org/viewvc?rev=1406246&view=rev
Log:
SOLR-3856: Improve tests for SqlEntityProcessor
Added:
lucene/dev/branches/branch_4x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AddAColumnTransformer.java
- copied unchanged from r1406231, lucene/dev/trunk/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AddAColumnTransformer.java
lucene/dev/branches/branch_4x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestNonWritablePersistFile.java
- copied unchanged from r1406231, lucene/dev/trunk/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestNonWritablePersistFile.java
lucene/dev/branches/branch_4x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TripleThreatTransformer.java
- copied unchanged from r1406231, lucene/dev/trunk/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TripleThreatTransformer.java
Removed:
lucene/dev/branches/branch_4x/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/dataimport-solrconfig-end-to-end.xml
lucene/dev/branches/branch_4x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestCachedSqlEntityProcessor.java
lucene/dev/branches/branch_4x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDIHEndToEnd.java
lucene/dev/branches/branch_4x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta2.java
lucene/dev/branches/branch_4x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta3.java
lucene/dev/branches/branch_4x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDeltaPrefixedPk.java
Modified:
lucene/dev/branches/branch_4x/solr/ (props changed)
lucene/dev/branches/branch_4x/solr/CHANGES.txt
lucene/dev/branches/branch_4x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractDIHJdbcTestCase.java
lucene/dev/branches/branch_4x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessor.java
lucene/dev/branches/branch_4x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessor2.java
lucene/dev/branches/branch_4x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta.java
Modified: lucene/dev/branches/branch_4x/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/CHANGES.txt?rev=1406246&r1=1406245&r2=1406246&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/solr/CHANGES.txt Tue Nov 6 17:49:49 2012
@@ -144,7 +144,11 @@ Other Changes
(Markus Jelsma via hossman)
* SOLR-3932: SolrCmdDistributorTest either takes 3 seconds or 3 minutes.
- (yonik, Mark Miller)
+ (yonik, Mark Miller)
+
+
+* SOLR-3856: New tests for SqlEntityProcessor/CachedSqlEntityProcessor
+ (James Dyer)
================== 4.0.0 ==================
Modified: lucene/dev/branches/branch_4x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractDIHJdbcTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractDIHJdbcTestCase.java?rev=1406246&r1=1406245&r2=1406246&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractDIHJdbcTestCase.java (original)
+++ lucene/dev/branches/branch_4x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractDIHJdbcTestCase.java Tue Nov 6 17:49:49 2012
@@ -19,88 +19,602 @@ package org.apache.solr.handler.dataimpo
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
+import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
+import java.sql.Timestamp;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import junit.framework.Assert;
+import org.apache.solr.request.LocalSolrQueryRequest;
import org.junit.AfterClass;
+import org.junit.Before;
import org.junit.BeforeClass;
/**
* This sets up an in-memory Derby Sql database with a little sample data.
- * The schema here is poorly-designed to illustrate DIH's ability to
- * overcome these kinds of challenges.
*/
public abstract class AbstractDIHJdbcTestCase extends AbstractDataImportHandlerTestCase {
+ //Start with "true" so the first test run will populate the data.
+ protected boolean underlyingDataModified = true;
+
+ protected boolean useSimpleCaches;
+ protected boolean countryEntity;
+ protected boolean countryCached;
+ protected boolean sportsEntity;
+ protected boolean sportsCached;
+ protected String rootTransformerName;
+ protected boolean countryTransformer;
+ protected boolean sportsTransformer;
+
+ private static final Pattern totalRequestsPattern = Pattern
+ .compile(".str name..Total Requests made to DataSource..(\\d+)..str.");
+
@BeforeClass
public static void beforeClassDihJdbcTest() throws Exception {
try {
Class.forName("org.hsqldb.jdbcDriver").newInstance();
} catch (Exception e) {
throw e;
+ }
+ initCore("dataimport-solrconfig.xml", "dataimport-schema.xml");
+ }
+ @Before
+ public void beforeDihJdbcTest() throws Exception {
+ useSimpleCaches = false;
+ countryEntity = false;
+ countryCached = false;
+ sportsEntity = false;
+ sportsCached = false;
+ rootTransformerName = null;
+ countryTransformer = false;
+ sportsTransformer = false;
+
+ clearIndex();
+ assertU(commit());
+
+ if(underlyingDataModified) {
+ refreshDatabase();
+ }
+ }
+ @AfterClass
+ public static void afterClassDihJdbcTest() throws Exception {
+ Connection conn = null;
+ Statement s = null;
+ try {
+ conn = DriverManager.getConnection("jdbc:hsqldb:mem:.");
+ s = conn.createStatement();
+ s.executeUpdate("shutdown");
+ } catch (SQLException e) {
+ throw e;
+ } finally {
+ try { s.close(); } catch(Exception ex) { }
+ try { conn.close(); } catch(Exception ex) { }
+ }
+ }
+ protected void singleEntity(int numToExpect) throws Exception {
+ h.query("/dataimport", generateRequest());
+ assertQ(req("*:*"), "//*[@numFound='" + totalPeople() + "']");
+ Assert.assertTrue(totalDatabaseRequests() == numToExpect);
+ }
+ protected void simpleTransform(int numToExpect) throws Exception {
+ rootTransformerName = "AddAColumnTransformer";
+ h.query("/dataimport", generateRequest());
+ assertQ(req("AddAColumn_s:Added"), "//*[@numFound='" + totalPeople() + "']");
+ Assert.assertTrue(totalDatabaseRequests() == numToExpect);
+ }
+ /**
+ * A delta update will not clean up documents added by a transformer
+ * even if the parent document that the transformer used to base the new documents
+ * were deleted
+ */
+ protected void complexTransform(int numToExpect, int numDeleted) throws Exception {
+ rootTransformerName = "TripleThreatTransformer";
+ h.query("/dataimport", generateRequest());
+ assertQ(req("q", "*:*", "rows", "" + (totalPeople() * 3), "sort", "id asc"), "//*[@numFound='" + ((totalPeople() * 3) + (numDeleted * 2)) + "']");
+ assertQ(req("id:TripleThreat-1-*"), "//*[@numFound='" + (totalPeople() + numDeleted) + "']");
+ assertQ(req("id:TripleThreat-2-*"), "//*[@numFound='" + (totalPeople() + numDeleted) + "']");
+ if(personNameExists("Michael") && countryCodeExists("NR") && countryCodeExists("RN"))
+ {
+ assertQ(
+ req("+id:TripleThreat-1-3 +NAME_mult_s:Michael +NAME_mult_s:leahciM +COUNTRY_CODES_mult_s:NR +COUNTRY_CODES_mult_s:RN"),
+ "//*[@numFound='1']");
}
+ assertQ(req("AddAColumn_s:Added"), "//*[@numFound='" + (totalPeople() + numDeleted) + "']");
+ Assert.assertTrue(totalDatabaseRequests() == numToExpect);
+ }
+ protected void withChildEntities(boolean cached, boolean checkDatabaseRequests) throws Exception {
+ rootTransformerName = random().nextBoolean() ? null : "AddAColumnTransformer";
+ int numChildren = random().nextInt(1) + 1;
+ int numDatabaseRequests = 1;
+ if(underlyingDataModified) {
+ if (countryEntity) {
+ if (cached) {
+ numDatabaseRequests++;
+ } else {
+ numDatabaseRequests += totalPeople();
+ }
+ }
+ if (sportsEntity) {
+ if (cached) {
+ numDatabaseRequests++;
+ } else {
+ numDatabaseRequests += totalPeople();
+ }
+ }
+ } else {
+ countryEntity = true;
+ sportsEntity = true;
+ if(numChildren==1) {
+ countryEntity = random().nextBoolean();
+ sportsEntity = !countryEntity;
+ }
+ if(countryEntity) {
+ countryTransformer = random().nextBoolean();
+ if(cached) {
+ numDatabaseRequests++;
+ countryCached = true;
+ } else {
+ numDatabaseRequests += totalPeople();
+ }
+ }
+ if(sportsEntity) {
+ sportsTransformer = random().nextBoolean();
+ if(cached) {
+ numDatabaseRequests++;
+ sportsCached = true;
+ } else {
+ numDatabaseRequests += totalPeople();
+ }
+ }
+ }
+ h.query("/dataimport", generateRequest());
+ assertQ(req("*:*"), "//*[@numFound='" + (totalPeople()) + "']");
+ if(!underlyingDataModified && "AddAColumnTransformer".equals(rootTransformerName)) {
+ assertQ(req("AddAColumn_s:Added"), "//*[@numFound='" + (totalPeople()) + "']");
+ }
+ if(countryEntity) {
+ if(personNameExists("Jayden"))
+ {
+ String nrName = countryNameByCode("NP");
+ if(nrName!=null && nrName.length()>0) {
+ assertQ(req("NAME_mult_s:Jayden"),
+ "//*[@numFound='1']", "//doc/str[@name='COUNTRY_NAME_s']='" + nrName + "'");
+ }
+ }
+ String nrName = countryNameByCode("NR");
+ int num = numberPeopleByCountryCode("NR");
+ if(nrName!=null && num>0) {
+ assertQ(req("COUNTRY_CODES_mult_s:NR"),
+ "//*[@numFound='" + num + "']", "//doc/str[@name='COUNTRY_NAME_s']='" + nrName + "'");
+ }
+ if(countryTransformer && !underlyingDataModified) {
+ assertQ(req("countryAdded_s:country_added"), "//*[@numFound='" + totalPeople() + "']");
+ }
+ }
+ if(sportsEntity) {
+ if(!underlyingDataModified) {
+ assertQ(req("SPORT_NAME_mult_s:Sailing"), "//*[@numFound='2']");
+ }
+ String michaelsName = personNameById(3);
+ String[] michaelsSports = sportNamesByPersonId(3);
+ if(michaelsName != null && michaelsSports.length>0) {
+ String[] xpath = new String[michaelsSports.length + 1];
+ xpath[0] = "//*[@numFound='1']";
+ int i=1;
+ for(String ms : michaelsSports) {
+ xpath[i] = "//doc/arr[@name='SPORT_NAME_mult_s']/str[" + i + "]='" + ms + "'";
+ i++;
+ }
+ assertQ(req("NAME_mult_s:" + michaelsName.replaceAll("\\W", "\\\\$0")), xpath);
+ }
+ if (!underlyingDataModified && sportsTransformer) {
+ assertQ(req("sportsAdded_s:sport_added"), "//*[@numFound='" + (totalPeople()) + "']");
+ }
+ }
+ if(checkDatabaseRequests) {
+ Assert.assertTrue("Expecting " + numDatabaseRequests
+ + " database calls, but DIH reported " + totalDatabaseRequests(),
+ totalDatabaseRequests() == numDatabaseRequests);
+ }
+ }
+ protected void simpleCacheChildEntities(boolean checkDatabaseRequests) throws Exception {
+ useSimpleCaches = true;
+ countryEntity = true;
+ sportsEntity = true;
+ countryCached=true;
+ sportsCached=true;
+ int dbRequestsMoreThan = 3;
+ int dbRequestsLessThan = totalPeople() * 2 + 1;
+ h.query("/dataimport", generateRequest());
+ assertQ(req("*:*"), "//*[@numFound='" + (totalPeople()) + "']");
+ if(!underlyingDataModified || (personNameExists("Samantha") && "Nauru".equals(countryNameByCode("NR"))))
+ {
+ assertQ(req("NAME_mult_s:Samantha"),
+ "//*[@numFound='1']", "//doc/str[@name='COUNTRY_NAME_s']='Nauru'");
+ }
+ if(!underlyingDataModified)
+ {
+ assertQ(req("COUNTRY_CODES_mult_s:NR"),
+ "//*[@numFound='2']", "//doc/str[@name='COUNTRY_NAME_s']='Nauru'");
+ assertQ(req("SPORT_NAME_mult_s:Sailing"), "//*[@numFound='2']");
+ }
+ String[] michaelsSports = sportNamesByPersonId(3);
+ if(!underlyingDataModified || michaelsSports.length>0) {
+ String[] xpath = new String[michaelsSports.length + 1];
+ xpath[0] = "//*[@numFound='1']";
+ int i=1;
+ for(String ms : michaelsSports) {
+ xpath[i] = "//doc/arr[@name='SPORT_NAME_mult_s']/str[" + i + "]='" + ms + "'";
+ i++;
+ }
+ assertQ(req("NAME_mult_s:Michael"), xpath);
+ }
+ if(checkDatabaseRequests) {
+ Assert.assertTrue("Expecting more than " + dbRequestsMoreThan
+ + " database calls, but DIH reported " + totalDatabaseRequests(),
+ totalDatabaseRequests() > dbRequestsMoreThan);
+ Assert.assertTrue("Expecting fewer than " + dbRequestsLessThan
+ + " database calls, but DIH reported " + totalDatabaseRequests(),
+ totalDatabaseRequests() < dbRequestsLessThan);
+ }
+ }
+ public void refreshDatabase() throws Exception
+ {
+ underlyingDataModified = false;
Connection conn = null;
Statement s = null;
- PreparedStatement ps = null;
+ PreparedStatement ps = null;
+ Timestamp theTime = new Timestamp(System.currentTimeMillis() - 10000); //10 seconds ago
try {
conn = DriverManager.getConnection("jdbc:hsqldb:mem:.");
s = conn.createStatement();
- s.executeUpdate("create table countries(code char(2) not null primary key, country_name varchar(50))");
- s.executeUpdate("create table people(id int not null primary key, name varchar(50), country_codes varchar(100))");
- s.executeUpdate("create table people_sports(id int not null primary key, person_id int, sport_name varchar(50))");
+ try {
+ s.executeUpdate("drop table countries");
+ s.executeUpdate("drop table people");
+ s.executeUpdate("drop table people_sports");
+ } catch(Exception e) {
+ //ignore. dbs complain when tables do no exist.
+ }
+ s.executeUpdate("create table countries(code varchar(3) not null primary key, country_name varchar(50), deleted char(1) default 'N', last_modified timestamp not null)");
+ s.executeUpdate("create table people(id int not null primary key, name varchar(50), country_code char(2), deleted char(1) default 'N', last_modified timestamp not null)");
+ s.executeUpdate("create table people_sports(id int not null primary key, person_id int, sport_name varchar(50), deleted char(1) default 'N', last_modified timestamp not null)");
- ps = conn.prepareStatement("insert into countries values (?,?)");
+ ps = conn.prepareStatement("insert into countries (code, country_name, last_modified) values (?,?,?)");
for(String[] country : countries) {
ps.setString(1, country[0]);
ps.setString(2, country[1]);
+ ps.setTimestamp(3, theTime);
Assert.assertEquals(1, ps.executeUpdate());
}
ps.close();
- ps = conn.prepareStatement("insert into people values (?,?,?)");
+ ps = conn.prepareStatement("insert into people (id, name, country_code, last_modified) values (?,?,?,?)");
for(Object[] person : people) {
ps.setInt(1, (Integer) person[0]);
ps.setString(2, (String) person[1]);
ps.setString(3, (String) person[2]);
+ ps.setTimestamp(4, theTime);
Assert.assertEquals(1, ps.executeUpdate());
}
ps.close();
- ps = conn.prepareStatement("insert into people_sports values (?,?,?)");
+ ps = conn.prepareStatement("insert into people_sports (id, person_id, sport_name, last_modified) values (?,?,?,?)");
for(Object[] sport : people_sports) {
ps.setInt(1, (Integer) sport[0]);
ps.setInt(2, (Integer) sport[1]);
ps.setString(3, (String) sport[2]);
+ ps.setTimestamp(4, theTime);
Assert.assertEquals(1, ps.executeUpdate());
}
ps.close();
+ conn.commit();
conn.close();
} catch(Exception e) {
throw e;
} finally {
- if(s!=null) { s.close(); }
- if(ps!=null) { ps.close(); }
- if(conn!=null) { conn.close(); }
+ try { ps.close(); } catch(Exception ex) { }
+ try { s.close(); } catch(Exception ex) { }
+ try { conn.close(); } catch(Exception ex) { }
}
}
+ protected abstract String deltaQueriesCountryTable() ;
+ protected abstract String deltaQueriesPersonTable() ;
- @AfterClass
- public static void afterClassDihJdbcTest() throws Exception {
+
+ private int getIntFromQuery(String query) throws Exception {
Connection conn = null;
Statement s = null;
+ ResultSet rs = null;
try {
conn = DriverManager.getConnection("jdbc:hsqldb:mem:.");
s = conn.createStatement();
- s.executeUpdate("shutdown");
+ rs = s.executeQuery(query);
+ if(rs.next()) {
+ return rs.getInt(1);
+ }
+ return 0;
+ } catch (SQLException e) {
+ throw e;
+ } finally {
+ try { rs.close(); } catch(Exception ex) { }
+ try { s.close(); } catch(Exception ex) { }
+ try { conn.close(); } catch(Exception ex) { }
+ }
+ }
+ private String[] getStringsFromQuery(String query) throws Exception {
+ Connection conn = null;
+ Statement s = null;
+ ResultSet rs = null;
+ try {
+ conn = DriverManager.getConnection("jdbc:hsqldb:mem:.");
+ s = conn.createStatement();
+ rs = s.executeQuery(query);
+ List<String> results = new ArrayList<String>();
+ while(rs.next()) {
+ results.add(rs.getString(1));
+ }
+ return results.toArray(new String[results.size()]);
+ } catch (SQLException e) {
+ throw e;
+ } finally {
+ try { rs.close(); } catch(Exception ex) { }
+ try { s.close(); } catch(Exception ex) { }
+ try { conn.close(); } catch(Exception ex) { }
+ }
+ }
+
+ public int totalCountries() throws Exception {
+ return getIntFromQuery("SELECT COUNT(1) FROM COUNTRIES WHERE DELETED != 'Y' ");
+ }
+ public int totalPeople() throws Exception {
+ return getIntFromQuery("SELECT COUNT(1) FROM PEOPLE WHERE DELETED != 'Y' ");
+ }
+ public boolean countryCodeExists(String cc) throws Exception {
+ return getIntFromQuery("SELECT COUNT(1) country_name FROM COUNTRIES WHERE DELETED != 'Y' AND CODE='" + cc + "'")>0;
+ }
+ public String countryNameByCode(String cc) throws Exception {
+ String[] s = getStringsFromQuery("SELECT country_name FROM COUNTRIES WHERE DELETED != 'Y' AND CODE='" + cc + "'");
+ return s.length==0 ? null : s[0];
+ }
+ public int numberPeopleByCountryCode(String cc) throws Exception {
+ return getIntFromQuery(
+ "Select count(1) " +
+ "from people p " +
+ "inner join countries c on p.country_code=c.code " +
+ "where p.deleted!='Y' and c.deleted!='Y' and c.code='" + cc + "'");
+ }
+ public String[] sportNamesByPersonId(int personId) throws Exception {
+ return getStringsFromQuery(
+ "SELECT ps.SPORT_NAME " +
+ "FROM people_sports ps " +
+ "INNER JOIN PEOPLE p ON p.id = ps.person_Id " +
+ "WHERE ps.DELETED != 'Y' AND p.DELETED != 'Y' " +
+ "AND ps.person_id=" + personId + " " +
+ "ORDER BY ps.id"
+ );
+ }
+ public boolean personNameExists(String pn) throws Exception {
+ return getIntFromQuery("SELECT COUNT(1) FROM PEOPLE WHERE DELETED != 'Y' AND NAME='" + pn + "'")>0;
+ }
+ public String personNameById(int id) throws Exception {
+ String[] nameArr= getStringsFromQuery("SELECT NAME FROM PEOPLE WHERE ID=" + id);
+ if(nameArr.length==0) {
+ return null;
+ }
+ return nameArr[0];
+ }
+
+ public int totalDatabaseRequests(String dihHandlerName) throws Exception {
+ LocalSolrQueryRequest request = lrf.makeRequest("indent", "true");
+ String response = h.query(dihHandlerName, request);
+ Matcher m = totalRequestsPattern.matcher(response);
+ Assert.assertTrue("The handler " + dihHandlerName
+ + " is not reporting any database requests. ", m.find()
+ && m.groupCount() == 1);
+ return Integer.parseInt(m.group(1));
+ }
+
+ public int totalDatabaseRequests() throws Exception {
+ return totalDatabaseRequests("/dataimport");
+ }
+ public IntChanges modifySomePeople() throws Exception {
+ underlyingDataModified = true;
+ int numberToChange = random().nextInt(people.length + 1);
+ Set<Integer> changeSet = new HashSet<Integer>();
+ Set<Integer> deleteSet = new HashSet<Integer>();
+ Set<Integer> addSet = new HashSet<Integer>();
+ Connection conn = null;
+ PreparedStatement change = null;
+ PreparedStatement delete = null;
+ PreparedStatement add = null;
+ //One second in the future ensures a change time after the last import (DIH uses second precision only)
+ Timestamp theTime = new Timestamp(System.currentTimeMillis() + 1000);
+ try {
+ conn = DriverManager.getConnection("jdbc:hsqldb:mem:.");
+ change = conn.prepareStatement("update people set name=?, last_modified=? where id=?");
+ delete = conn.prepareStatement("update people set deleted='Y', last_modified=? where id=?");
+ add = conn.prepareStatement("insert into people (id,name,country_code,last_modified) values (?,?,'NEW',?)");
+ for(int i=0 ; i<numberToChange ; i++) {
+ int tryIndex = random().nextInt(people.length);
+ Integer id = (Integer) people[tryIndex][0];
+ if(!changeSet.contains(id) && !deleteSet.contains(id)) {
+ boolean changeDontDelete = random().nextBoolean();
+ if(changeDontDelete) {
+ changeSet.add(id);
+ change.setString(1, "MODIFIED " + people[tryIndex][1]);
+ change.setTimestamp(2, theTime);
+ change.setInt(3, id);
+ Assert.assertEquals(1, change.executeUpdate());
+ } else {
+ deleteSet.add(id);
+ delete.setTimestamp(1, theTime);
+ delete.setInt(2, id);
+ Assert.assertEquals(1, delete.executeUpdate());
+ }
+ }
+ }
+ int numberToAdd = random().nextInt(3);
+ for(int i=0 ; i<numberToAdd ; i++) {
+ int tryIndex = random().nextInt(people.length);
+ Integer id = (Integer) people[tryIndex][0];
+ Integer newId = id+1000;
+ String newDesc = "ADDED " + people[tryIndex][1];
+ if(!addSet.contains(newId)) {
+ addSet.add(newId);
+ add.setInt(1, newId);
+ add.setString(2, newDesc);
+ add.setTimestamp(3, theTime);
+ Assert.assertEquals(1, add.executeUpdate());
+ }
+ }
+ conn.commit();
} catch (SQLException e) {
throw e;
} finally {
- if(s!=null) { s.close(); }
- if(conn!=null) { conn.close(); }
+ try { change.close(); } catch(Exception ex) { }
+ try { conn.close(); } catch(Exception ex) { }
}
+ IntChanges c = new IntChanges();
+ c.changedKeys=changeSet.toArray(new Integer[changeSet.size()]);
+ c.deletedKeys=deleteSet.toArray(new Integer[deleteSet.size()]);
+ c.addedKeys=addSet.toArray(new Integer[addSet.size()]);
+ return c;
}
+ public String[] modifySomeCountries() throws Exception {
+ underlyingDataModified = true;
+ int numberToChange = random().nextInt(countries.length + 1);
+ Set<String> changeSet = new HashSet<String>();
+ Connection conn = null;
+ PreparedStatement change = null;
+ // One second in the future ensures a change time after the last import (DIH
+ // uses second precision only)
+ Timestamp theTime = new Timestamp(System.currentTimeMillis() + 1000);
+ try {
+ conn = DriverManager.getConnection("jdbc:hsqldb:mem:.");
+ change = conn
+ .prepareStatement("update countries set country_name=?, last_modified=? where code=?");
+ for (int i = 0; i < numberToChange; i++) {
+ int tryIndex = random().nextInt(countries.length);
+ String code = countries[tryIndex][0];
+ if (!changeSet.contains(code)) {
+ changeSet.add(code);
+ change.setString(1, "MODIFIED " + countries[tryIndex][1]);
+ change.setTimestamp(2, theTime);
+ change.setString(3, code);
+ Assert.assertEquals(1, change.executeUpdate());
+
+ }
+ }
+ } catch (SQLException e) {
+ throw e;
+ } finally {
+ try {
+ change.close();
+ } catch (Exception ex) {}
+ try {
+ conn.close();
+ } catch (Exception ex) {}
+ }
+ return changeSet.toArray(new String[changeSet.size()]);
+ }
+ class IntChanges {
+ public Integer[] changedKeys;
+ public Integer[] deletedKeys;
+ public Integer[] addedKeys;
+ }
+ protected LocalSolrQueryRequest generateRequest() {
+ return lrf.makeRequest("command", "full-import", "dataConfig", generateConfig(),
+ "clean", "true", "commit", "true", "synchronous", "true", "indent", "true");
+ }
+ protected String generateConfig() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("<dataConfig> \n");
+ sb.append("<dataSource name=''hsqldb'' driver=''org.hsqldb.jdbcDriver'' url=''jdbc:hsqldb:mem:.'' /> \n");
+ sb.append("<document name=''TestSqlEntityProcessor''> \n");
+ sb.append("<entity name=''People'' ");
+ sb.append("pk=''" + (random().nextBoolean() ? "ID" : "People.ID") + "'' ");
+ sb.append("processor=''SqlEntityProcessor'' dataSource=''hsqldb'' ");
+ sb.append(rootTransformerName !=null ? "transformer=''" + rootTransformerName + "'' " : "");
+ sb.append("query=''SELECT ID, NAME, COUNTRY_CODE FROM PEOPLE WHERE DELETED != 'Y' '' ");
+ sb.append(deltaQueriesPersonTable());
+ sb.append("> \n");
+
+ sb.append("<field column=''NAME'' name=''NAME_mult_s'' /> \n");
+ sb.append("<field column=''COUNTRY_CODE'' name=''COUNTRY_CODES_mult_s'' /> \n");
+
+ if(countryEntity) {
+ sb.append("<entity name=''Countries'' ");
+ sb.append("pk=''" + (random().nextBoolean() ? "CODE" : "Countries.CODE") + "'' ");
+ sb.append("dataSource=''hsqldb'' ");
+ sb.append(countryTransformer ? "transformer=''AddAColumnTransformer'' " +
+ "newColumnName=''countryAdded_s'' newColumnValue=''country_added'' " : "");
+ if(countryCached) {
+ sb.append(random().nextBoolean() ?
+ "processor=''SqlEntityProcessor'' cacheImpl=''SortedMapBackedCache'' " :
+ "processor=''CachedSqlEntityProcessor'' "
+ );
+ if(useSimpleCaches) {
+ sb.append("query=''SELECT CODE, COUNTRY_NAME FROM COUNTRIES WHERE DELETED != 'Y' AND CODE='${People.COUNTRY_CODE}' ''>\n");
+ } else {
+ sb.append(random().nextBoolean() ?
+ "cacheKey=''CODE'' cacheLookup=''People.COUNTRY_CODE'' " :
+ "where=''CODE=People.COUNTRY_CODE'' "
+ );
+ sb.append("query=''SELECT CODE, COUNTRY_NAME FROM COUNTRIES'' ");
+ sb.append("> \n");
+ }
+ } else {
+ sb.append("processor=''SqlEntityProcessor'' query=''SELECT CODE, COUNTRY_NAME FROM COUNTRIES WHERE DELETED != 'Y' AND CODE='${People.COUNTRY_CODE}' '' ");
+ sb.append(deltaQueriesCountryTable());
+ sb.append("> \n");
+ }
+ sb.append("<field column=''CODE'' name=''COUNTRY_CODE_s'' /> \n");
+ sb.append("<field column=''COUNTRY_NAME'' name=''COUNTRY_NAME_s'' /> \n");
+ sb.append("</entity> \n");
+ }
+ if(sportsEntity) {
+ sb.append("<entity name=''Sports'' dataSource=''hsqldb'' "
+ + (sportsTransformer ? "transformer=''AddAColumnTransformer'' " +
+ "newColumnName=''sportsAdded_s'' newColumnValue=''sport_added'' " : ""));
+ if(sportsCached) {
+ sb.append(random().nextBoolean() ?
+ "processor=''SqlEntityProcessor'' cacheImpl=''SortedMapBackedCache'' " :
+ "processor=''CachedSqlEntityProcessor'' "
+ );
+ if(useSimpleCaches) {
+ sb.append("query=''SELECT ID, SPORT_NAME FROM PEOPLE_SPORTS WHERE DELETED != 'Y' AND PERSON_ID='${People.ID}' ORDER BY ID'' ");
+ } else {
+ sb.append(random().nextBoolean() ?
+ "cacheKey=''PERSON_ID'' cacheLookup=''People.ID'' " :
+ "where=''PERSON_ID=People.ID'' "
+ );
+ sb.append("query=''SELECT ID, PERSON_ID, SPORT_NAME FROM PEOPLE_SPORTS ORDER BY ID'' ");
+ }
+ } else {
+ sb.append("processor=''SqlEntityProcessor'' query=''SELECT ID, SPORT_NAME FROM PEOPLE_SPORTS WHERE DELETED != 'Y' AND PERSON_ID='${People.ID}' ORDER BY ID'' ");
+ }
+ sb.append("> \n");
+ sb.append("<field column=''SPORT_NAME'' name=''SPORT_NAME_mult_s'' /> \n");
+ sb.append("<field column=''id'' name=''SPORT_ID_mult_s'' /> \n");
+ sb.append("</entity> \n");
+ }
+
+ sb.append("</entity> \n");
+ sb.append("</document> \n");
+ sb.append("</dataConfig> \n");
+ String config = sb.toString().replaceAll("[']{2}", "\"");
+ log.debug(config);
+ return config;
+ }
+
public static final String[][] countries = {
{"NA", "Namibia"},
{"NC", "New Caledonia"},
@@ -118,7 +632,7 @@ public abstract class AbstractDIHJdbcTes
public static final Object[][] people = {
{1,"Jacob","NZ"},
- {2,"Ethan","NU,NA,NE"},
+ {2,"Ethan","NU"},
{3,"Michael","NR"},
{4,"Jayden","NP"},
{5,"William","NO"},
@@ -134,7 +648,7 @@ public abstract class AbstractDIHJdbcTes
{15,"Lily","NE"},
{16,"Madison","NC"},
{17,"Mia","NA"},
- {18,"Natalie","NP,NR,NU,NZ"},
+ {18,"Natalie","NZ"},
{19,"Olivia","NU"},
{20,"Samantha","NR"}
};
@@ -165,6 +679,5 @@ public abstract class AbstractDIHJdbcTes
{1800, 18, "White Water Rafting"},
{1900, 19, "Water skiing"},
{2000, 20, "Windsurfing"}
- };
-
+ };
}
Modified: lucene/dev/branches/branch_4x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessor.java?rev=1406246&r1=1406245&r2=1406246&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessor.java (original)
+++ lucene/dev/branches/branch_4x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessor.java Tue Nov 6 17:49:49 2012
@@ -1,3 +1,8 @@
+package org.apache.solr.handler.dataimport;
+
+import org.junit.Ignore;
+import org.junit.Test;
+
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -14,168 +19,44 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.solr.handler.dataimport;
-
-import org.junit.Test;
-import org.junit.Ignore;
-
-import java.util.*;
/**
- * <p>
- * Test for SqlEntityProcessor
- * </p>
- *
- *
- * @since solr 1.3
+ * Test with various combinations of parameters, child entites, caches, transformers.
*/
-@Ignore("FIXME: I fail so often it makes me ill!")
-public class TestSqlEntityProcessor extends AbstractDataImportHandlerTestCase {
- private static ThreadLocal<Integer> local = new ThreadLocal<Integer>();
-
+public class TestSqlEntityProcessor extends AbstractDIHJdbcTestCase {
+
@Test
- public void testSingleBatch() {
- SqlEntityProcessor sep = new SqlEntityProcessor();
- List<Map<String, Object>> rows = getRows(3);
- VariableResolverImpl vr = new VariableResolverImpl();
- HashMap<String, String> ea = new HashMap<String, String>();
- ea.put("query", "SELECT * FROM A");
- Context c = getContext(null, vr, getDs(rows), Context.FULL_DUMP, null, ea);
- sep.init(c);
- int count = 0;
- while (true) {
- Map<String, Object> r = sep.nextRow();
- if (r == null)
- break;
- count++;
- }
-
- assertEquals(3, count);
- }
-
+ public void testSingleEntity() throws Exception {
+ singleEntity(1);
+ }
@Test
- public void testTranformer() {
- EntityProcessor sep = new EntityProcessorWrapper( new SqlEntityProcessor(), null, null);
- List<Map<String, Object>> rows = getRows(2);
- VariableResolverImpl vr = new VariableResolverImpl();
- HashMap<String, String> ea = new HashMap<String, String>();
- ea.put("query", "SELECT * FROM A");
- ea.put("transformer", T.class.getName());
-
- sep.init(getContext(null, vr, getDs(rows), Context.FULL_DUMP, null, ea));
- List<Map<String, Object>> rs = new ArrayList<Map<String, Object>>();
- Map<String, Object> r = null;
- while (true) {
- r = sep.nextRow();
- if (r == null)
- break;
- rs.add(r);
-
- }
- assertEquals(2, rs.size());
- assertNotNull(rs.get(0).get("T"));
+ public void testWithSimpleTransformer() throws Exception {
+ simpleTransform(1);
}
-
@Test
- public void testTranformerWithReflection() {
- EntityProcessor sep = new EntityProcessorWrapper(new SqlEntityProcessor(), null, null);
- List<Map<String, Object>> rows = getRows(2);
- VariableResolverImpl vr = new VariableResolverImpl();
- HashMap<String, String> ea = new HashMap<String, String>();
- ea.put("query", "SELECT * FROM A");
- ea.put("transformer", T3.class.getName());
-
- sep.init(getContext(null, vr, getDs(rows), Context.FULL_DUMP, null, ea));
- List<Map<String, Object>> rs = new ArrayList<Map<String, Object>>();
- Map<String, Object> r = null;
- while (true) {
- r = sep.nextRow();
- if (r == null)
- break;
- rs.add(r);
-
- }
- assertEquals(2, rs.size());
- assertNotNull(rs.get(0).get("T3"));
+ public void testWithComplexTransformer() throws Exception {
+ complexTransform(1, 0);
}
-
@Test
- public void testTranformerList() {
- EntityProcessor sep = new EntityProcessorWrapper(new SqlEntityProcessor(),null, null);
- List<Map<String, Object>> rows = getRows(2);
- VariableResolverImpl vr = new VariableResolverImpl();
-
- HashMap<String, String> ea = new HashMap<String, String>();
- ea.put("query", "SELECT * FROM A");
- ea.put("transformer", T2.class.getName());
- sep.init(getContext(null, vr, getDs(rows), Context.FULL_DUMP, null, ea));
-
- local.set(0);
- Map<String, Object> r = null;
- int count = 0;
- while (true) {
- r = sep.nextRow();
- if (r == null)
- break;
- count++;
- }
- assertEquals(2, (int) local.get());
- assertEquals(4, count);
- }
-
- private List<Map<String, Object>> getRows(int count) {
- List<Map<String, Object>> rows = new ArrayList<Map<String, Object>>();
- for (int i = 0; i < count; i++) {
- Map<String, Object> row = new HashMap<String, Object>();
- row.put("id", i);
- row.put("value", "The value is " + i);
- rows.add(row);
- }
- return rows;
- }
-
- private static DataSource<Iterator<Map<String, Object>>> getDs(
- final List<Map<String, Object>> rows) {
- return new DataSource<Iterator<Map<String, Object>>>() {
- @Override
- public Iterator<Map<String, Object>> getData(String query) {
- return rows.iterator();
- }
-
- @Override
- public void init(Context context, Properties initProps) {
- }
-
- @Override
- public void close() {
- }
- };
+ public void testChildEntities() throws Exception {
+ withChildEntities(false, true);
}
-
- public static class T extends Transformer {
- @Override
- public Object transformRow(Map<String, Object> aRow, Context context) {
- aRow.put("T", "Class T");
- return aRow;
- }
- }
-
- public static class T3 {
- public Object transformRow(Map<String, Object> aRow) {
- aRow.put("T3", "T3 class");
- return aRow;
- }
- }
-
- public static class T2 extends Transformer {
- @Override
- public Object transformRow(Map<String, Object> aRow, Context context) {
- Integer count = local.get();
- local.set(count + 1);
- List<Map<String, Object>> l = new ArrayList<Map<String, Object>>();
- l.add(aRow);
- l.add(aRow);
- return l;
- }
+ @Test
+ public void testCachedChildEntities() throws Exception {
+ withChildEntities(true, true);
}
+ @Test
+ @Ignore("broken see SOLR-3857")
+ public void testSimpleCacheChildEntities() throws Exception {
+ simpleCacheChildEntities(true);
+ }
+
+ @Override
+ protected String deltaQueriesCountryTable() {
+ return "";
+ }
+ @Override
+ protected String deltaQueriesPersonTable() {
+ return "";
+ }
}
Modified: lucene/dev/branches/branch_4x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessor2.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessor2.java?rev=1406246&r1=1406245&r2=1406246&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessor2.java (original)
+++ lucene/dev/branches/branch_4x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessor2.java Tue Nov 6 17:49:49 2012
@@ -41,8 +41,9 @@ public class TestSqlEntityProcessor2 ext
public static void beforeClass() throws Exception {
initCore("dataimport-solrconfig.xml", "dataimport-schema.xml");
}
-
- @Before @Override
+
+ @Before
+ @Override
public void setUp() throws Exception {
super.setUp();
clearIndex();
@@ -51,140 +52,13 @@ public class TestSqlEntityProcessor2 ext
@Test
@SuppressWarnings("unchecked")
- public void testCompositePk_FullImport() throws Exception {
- List parentRow = new ArrayList();
- parentRow.add(createMap("id", "1"));
- MockDataSource.setIterator("select * from x", parentRow.iterator());
-
- List childRow = new ArrayList();
- childRow.add(createMap("desc", "hello"));
-
- MockDataSource.setIterator("select * from y where y.A=1", childRow
- .iterator());
-
- runFullImport(dataConfig);
-
- assertQ(req("id:1"), "//*[@numFound='1']");
- assertQ(req("desc:hello"), "//*[@numFound='1']");
- }
-
- @Test
- @SuppressWarnings("unchecked")
- public void testCompositePk_FullImportWithoutCommit() throws Exception {
- List parentRow = new ArrayList();
- parentRow.add(createMap("id", "10"));
- MockDataSource.setIterator("select * from x", parentRow.iterator());
-
- List childRow = new ArrayList();
- childRow.add(createMap("desc", "hello"));
-
- MockDataSource.setIterator("select * from y where y.A=10", childRow
- .iterator());
-
-
- runFullImport(dataConfig,createMap("commit","false"));
- assertQ(req("id:10"), "//*[@numFound='0']");
- }
-
- @Test
- @SuppressWarnings("unchecked")
- public void testCompositePk_DeltaImport() throws Exception {
- List deltaRow = new ArrayList();
- deltaRow.add(createMap("id", "5"));
- MockDataSource.setIterator("select id from x where last_modified > NOW",
- deltaRow.iterator());
-
- List parentRow = new ArrayList();
- parentRow.add(createMap("id", "5"));
- MockDataSource.setIterator("select * from x where id = '5'", parentRow
- .iterator());
-
- List childRow = new ArrayList();
- childRow.add(createMap("desc", "hello"));
- MockDataSource.setIterator("select * from y where y.A=5", childRow
- .iterator());
-
- runDeltaImport(dataConfig);
-
- assertQ(req("id:5"), "//*[@numFound='1']");
- assertQ(req("desc:hello"), "//*[@numFound='1']");
- }
-
- @Test
- @SuppressWarnings("unchecked")
- public void testCompositePk_DeltaImport_DeletedPkQuery() throws Exception {
- List parentRow = new ArrayList();
- parentRow.add(createMap("id", "11"));
- MockDataSource.setIterator("select * from x", parentRow.iterator());
-
- List childRow = new ArrayList();
- childRow.add(createMap("desc", "hello"));
-
- MockDataSource.setIterator("select * from y where y.A=11", childRow
- .iterator());
-
- runFullImport(dataConfig);
-
- assertQ(req("id:11"), "//*[@numFound='1']");
-
-
-
- List deltaRow = new ArrayList();
- deltaRow.add(createMap("id", "15"));
- deltaRow.add(createMap("id", "17"));
- MockDataSource.setIterator("select id from x where last_modified > NOW",
- deltaRow.iterator());
-
- List deltaDeleteRow = new ArrayList();
- deltaDeleteRow.add(createMap("id", "11"));
- deltaDeleteRow.add(createMap("id", "17"));
- MockDataSource.setIterator("select id from x where last_modified > NOW AND deleted='true'",
- deltaDeleteRow.iterator());
-
- parentRow = new ArrayList();
- parentRow.add(createMap("id", "15"));
- MockDataSource.setIterator("select * from x where id = '15'", parentRow
- .iterator());
-
- parentRow = new ArrayList();
- parentRow.add(createMap("id", "17"));
- MockDataSource.setIterator("select * from x where id = '17'", parentRow
- .iterator());
-
- runDeltaImport(dataConfig);
-
- assertQ(req("id:15"), "//*[@numFound='1']");
- assertQ(req("id:11"), "//*[@numFound='0']");
- assertQ(req("id:17"), "//*[@numFound='0']");
- }
-
- @Test
- @SuppressWarnings("unchecked")
- public void testCompositePk_DeltaImport_DeltaImportQuery() throws Exception {
- List deltaRow = new ArrayList();
- deltaRow.add(createMap("id", "5"));
- MockDataSource.setIterator("select id from x where last_modified > NOW",
- deltaRow.iterator());
-
- List parentRow = new ArrayList();
- parentRow.add(createMap("id", "5"));
- MockDataSource.setIterator("select * from x where id=5", parentRow
- .iterator());
-
- List childRow = new ArrayList();
- childRow.add(createMap("desc", "hello"));
- MockDataSource.setIterator("select * from y where y.A=5", childRow
- .iterator());
-
- runDeltaImport(dataConfig_deltaimportquery);
-
- assertQ(req("id:5"), "//*[@numFound='1']");
- assertQ(req("desc:hello"), "//*[@numFound='1']");
- }
-
- @Test
- @SuppressWarnings("unchecked")
@Ignore("Known Locale/TZ problems: see https://issues.apache.org/jira/browse/SOLR-1916")
+ /**
+ * This test is here for historical purposes only.
+ * When SOLR-1916 is fixed, it would be best to rewrite this test.
+ *
+ * @throws Exception
+ */
public void testLastIndexTime() throws Exception {
List row = new ArrayList();
row.add(createMap("id", 5));
@@ -220,22 +94,4 @@ public class TestSqlEntityProcessor2 ext
"\t\t<entity name=\"x\" query=\"select * from x where last_modified > ${dih.functions.checkDateFormat(dih.last_index_time)}\" />\n" +
"\t</document>\n" +
"</dataConfig>";
-
- private static String dataConfig = "<dataConfig><dataSource type=\"MockDataSource\"/>\n"
- + " <document>\n"
- + " <entity name=\"x\" pk=\"id\" query=\"select * from x\" deletedPkQuery=\"select id from x where last_modified > NOW AND deleted='true'\" deltaQuery=\"select id from x where last_modified > NOW\">\n"
- + " <field column=\"id\" />\n"
- + " <entity name=\"y\" query=\"select * from y where y.A=${x.id}\">\n"
- + " <field column=\"desc\" />\n"
- + " </entity>\n" + " </entity>\n"
- + " </document>\n" + "</dataConfig>\n";
-
- private static String dataConfig_deltaimportquery = "<dataConfig><dataSource type=\"MockDataSource\"/>\n"
- + " <document>\n"
- + " <entity name=\"x\" deltaImportQuery=\"select * from x where id=${dataimporter.delta.id}\" deltaQuery=\"select id from x where last_modified > NOW\">\n"
- + " <field column=\"id\" />\n"
- + " <entity name=\"y\" query=\"select * from y where y.A=${x.id}\">\n"
- + " <field column=\"desc\" />\n"
- + " </entity>\n" + " </entity>\n"
- + " </document>\n" + "</dataConfig>\n";
}
Modified: lucene/dev/branches/branch_4x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta.java?rev=1406246&r1=1406245&r2=1406246&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta.java (original)
+++ lucene/dev/branches/branch_4x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta.java Tue Nov 6 17:49:49 2012
@@ -1,3 +1,9 @@
+package org.apache.solr.handler.dataimport;
+
+import org.apache.solr.request.LocalSolrQueryRequest;
+import org.junit.Before;
+import org.junit.Test;
+
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -14,330 +20,133 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.solr.handler.dataimport;
-
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Ignore;
-import org.junit.Test;
-
-import java.io.File;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
/**
- * <p>
- * Test for SqlEntityProcessor which checks variations in primary key names and deleted ids
- * </p>
- *
- *
- *
- * @since solr 1.3
+ * Test with various combinations of parameters, child entites, transformers.
*/
-@Ignore("FIXME: I fail so often it makes me ill!")
-public class TestSqlEntityProcessorDelta extends AbstractDataImportHandlerTestCase {
- private static final String FULLIMPORT_QUERY = "select * from x";
-
- private static final String DELTA_QUERY = "select id from x where last_modified > NOW";
-
- private static final String DELETED_PK_QUERY = "select id from x where last_modified > NOW AND deleted='true'";
-
- private static final String dataConfig_delta =
- "<dataConfig>" +
- " <dataSource type=\"MockDataSource\"/>\n" +
- " <document>\n" +
- " <entity name=\"x\" transformer=\"TemplateTransformer\"" +
- " query=\"" + FULLIMPORT_QUERY + "\"" +
- " deletedPkQuery=\"" + DELETED_PK_QUERY + "\"" +
- " deltaImportQuery=\"select * from x where id='${dih.delta.id}'\"" +
- " deltaQuery=\"" + DELTA_QUERY + "\">\n" +
- " <field column=\"id\" name=\"id\"/>\n" +
- " <entity name=\"y\" query=\"select * from y where y.A='${x.id}'\">\n" +
- " <field column=\"desc\" />\n" +
- " </entity>\n" +
- " </entity>\n" +
- " </document>\n" +
- "</dataConfig>\n";
-
- //TODO: fix this test to not require FSDirectory.
- static String savedFactory;
- @BeforeClass
- public static void beforeClass() throws Exception {
- savedFactory = System.getProperty("solr.DirectoryFactory");
- System.setProperty("solr.directoryFactory", "solr.MockFSDirectoryFactory");
- initCore("dataimport-solrconfig.xml", "dataimport-schema.xml");
+public class TestSqlEntityProcessorDelta extends AbstractDIHJdbcTestCase {
+ private boolean delta = false;
+ private boolean useParentDeltaQueryParam = false;
+ private IntChanges personChanges = null;
+ private String[] countryChanges = null;
+
+ @Before
+ public void setupDeltaTest() {
+ delta = false;
+ personChanges = null;
+ countryChanges = null;
+ }
+ @Test
+ public void testSingleEntity() throws Exception {
+ singleEntity(1);
+ changeStuff();
+ int c = calculateDatabaseCalls();
+ singleEntity(c);
+ validateChanges();
+ }
+ @Test
+ public void testWithSimpleTransformer() throws Exception {
+ simpleTransform(1);
+ changeStuff();
+ simpleTransform(calculateDatabaseCalls());
+ validateChanges();
+ }
+ @Test
+ public void testWithComplexTransformer() throws Exception {
+ complexTransform(1, 0);
+ changeStuff();
+ complexTransform(calculateDatabaseCalls(), personChanges.deletedKeys.length);
+ validateChanges();
+ }
+ @Test
+ public void testChildEntities() throws Exception {
+ useParentDeltaQueryParam = random().nextBoolean();
+ withChildEntities(false, true);
+ changeStuff();
+ withChildEntities(false, false);
+ validateChanges();
}
- @AfterClass
- public static void afterClass() {
- if (savedFactory == null) {
- System.clearProperty("solr.directoryFactory");
- } else {
- System.setProperty("solr.directoryFactory", savedFactory);
+
+ private int calculateDatabaseCalls() {
+ //The main query generates 1
+ //Deletes generate 1
+ //Each add/mod generate 1
+ int c = 1;
+ if (countryChanges != null) {
+ c += countryChanges.length + 1;
}
- }
-
- @Before @Override
- public void setUp() throws Exception {
- super.setUp();
- clearIndex();
- assertU(commit());
- }
-
- @SuppressWarnings("unchecked")
- private void add1document() throws Exception {
- List parentRow = new ArrayList();
- parentRow.add(createMap("id", "1"));
- MockDataSource.setIterator(FULLIMPORT_QUERY, parentRow.iterator());
-
- List childRow = new ArrayList();
- childRow.add(createMap("desc", "hello"));
- MockDataSource.setIterator("select * from y where y.A='1'", childRow
- .iterator());
-
- runFullImport(dataConfig_delta);
-
- assertQ(req("*:* OR add1document"), "//*[@numFound='1']");
- assertQ(req("id:1"), "//*[@numFound='1']");
- assertQ(req("desc:hello"), "//*[@numFound='1']");
- }
-
- @Test
- @SuppressWarnings("unchecked")
- public void testCompositePk_FullImport() throws Exception {
- add1document();
- }
-
- @Test
- @SuppressWarnings("unchecked")
- public void testNonWritablePersistFile() throws Exception {
- // See SOLR-2551
- String configDir = h.getCore().getResourceLoader().getConfigDir();
- String filePath = configDir;
- if (configDir != null && !configDir.endsWith(File.separator))
- filePath += File.separator;
- filePath += "dataimport.properties";
- File f = new File(filePath);
-
- try {
- // execute the test only if we are able to set file to read only mode
- assumeTrue("No dataimport.properties file", f.exists() || f.createNewFile());
- assumeTrue("dataimport.proprties can't be set read only", f.setReadOnly());
- assumeFalse("dataimport.proprties is still writable even though " +
- "marked readonly - test running as superuser?", f.canWrite());
-
- ignoreException("Properties is not writable");
-
- List parentRow = new ArrayList();
- parentRow.add(createMap("id", "1"));
- MockDataSource.setIterator(FULLIMPORT_QUERY, parentRow.iterator());
-
- List childRow = new ArrayList();
- childRow.add(createMap("desc", "hello"));
- MockDataSource.setIterator("select * from y where y.A='1'",
- childRow.iterator());
-
- runFullImport(dataConfig_delta);
- assertQ(req("id:1"), "//*[@numFound='0']");
- } finally {
- f.delete();
+ if (personChanges != null) {
+ c += personChanges.addedKeys.length + personChanges.changedKeys.length + 1;
}
+ return c;
}
-
- // WORKS
-
- @Test
- @SuppressWarnings("unchecked")
- public void testCompositePk_DeltaImport_delete() throws Exception {
- add1document();
- List deletedRow = new ArrayList();
- deletedRow.add(createMap("id", "1"));
- MockDataSource.setIterator(DELETED_PK_QUERY, deletedRow.iterator());
-
- MockDataSource.setIterator(DELTA_QUERY, Collections
- .EMPTY_LIST.iterator());
-
- List childRow = new ArrayList();
- childRow.add(createMap("desc", "hello"));
- MockDataSource.setIterator("select * from y where y.A='1'", childRow
- .iterator());
-
- runDeltaImport(dataConfig_delta);
- assertQ(req("*:* OR testCompositePk_DeltaImport_delete"), "//*[@numFound='0']");
+ private void validateChanges() throws Exception
+ {
+ if(personChanges!=null) {
+ for(int id : personChanges.addedKeys) {
+ assertQ(req("id:" + id), "//*[@numFound='1']");
+ }
+ for(int id : personChanges.deletedKeys) {
+ assertQ(req("id:" + id), "//*[@numFound='0']");
+ }
+ for(int id : personChanges.changedKeys) {
+ assertQ(req("id:" + id), "//*[@numFound='1']", "substring(//doc/arr[@name='NAME_mult_s']/str[1], 1, 8)='MODIFIED'");
+ }
+ }
+ if(countryChanges!=null) {
+ for(String code : countryChanges) {
+ assertQ(req("COUNTRY_CODE_s:" + code), "//*[@numFound='" + numberPeopleByCountryCode(code) + "']", "substring(//doc/str[@name='COUNTRY_NAME_s'], 1, 8)='MODIFIED'");
+ }
+ }
}
-
- @Test
- @SuppressWarnings("unchecked")
- public void testCompositePk_DeltaImport_empty() throws Exception {
- List deltaRow = new ArrayList();
- deltaRow.add(createMap("id", "1"));
- MockDataSource.setIterator(DELTA_QUERY, deltaRow.iterator());
-
- MockDataSource.setIterator(DELETED_PK_QUERY, Collections
- .EMPTY_LIST.iterator());
-
- List parentRow = new ArrayList();
- parentRow.add(createMap("id", "1"));
- MockDataSource.setIterator("select * from x where id='1'", parentRow
- .iterator());
-
- List childRow = new ArrayList();
- childRow.add(createMap("desc", "hello"));
- MockDataSource.setIterator("select * from y where y.A='1'",
- childRow.iterator());
-
- runDeltaImport(dataConfig_delta);
-
- assertQ(req("*:* OR testCompositePk_DeltaImport_empty"), "//*[@numFound='1']");
- assertQ(req("id:1"), "//*[@numFound='1']");
- assertQ(req("desc:hello"), "//*[@numFound='1']");
+ private void changeStuff() throws Exception {
+ if(countryEntity)
+ {
+ int n = random().nextInt(2);
+ switch(n) {
+ case 0:
+ personChanges = modifySomePeople();
+ break;
+ case 1:
+ countryChanges = modifySomeCountries();
+ break;
+ case 2:
+ personChanges = modifySomePeople();
+ countryChanges = modifySomeCountries();
+ break;
+ }
+ } else {
+ personChanges = modifySomePeople();
+ }
+ delta = true;
+ }
+ protected LocalSolrQueryRequest generateRequest() {
+ return lrf.makeRequest("command", (delta ? "delta-import" : "full-import"), "dataConfig", generateConfig(),
+ "clean", (delta ? "false" : "true"), "commit", "true", "synchronous", "true", "indent", "true");
}
-
- // WORKS
- @Test
- @SuppressWarnings("unchecked")
- public void testCompositePk_DeltaImport_replace_delete() throws Exception {
- add1document();
- MockDataSource.clearCache();
-
- List deltaRow = new ArrayList();
- deltaRow.add(createMap("id", "1"));
- MockDataSource.setIterator(DELTA_QUERY,
- deltaRow.iterator());
-
- List deletedRow = new ArrayList();
- deletedRow.add(createMap("id", "1"));
- MockDataSource.setIterator(DELETED_PK_QUERY,
- deletedRow.iterator());
-
- List parentRow = new ArrayList();
- parentRow.add(createMap("id", "1"));
- MockDataSource.setIterator("select * from x where id='1'", parentRow
- .iterator());
-
- List childRow = new ArrayList();
- childRow.add(createMap("desc", "goodbye"));
- MockDataSource.setIterator("select * from y where y.A='1'", childRow
- .iterator());
-
- runDeltaImport(dataConfig_delta);
-
- assertQ(req("*:* OR testCompositePk_DeltaImport_replace_delete"), "//*[@numFound='0']");
- }
-
-
- @Test
- @SuppressWarnings("unchecked")
- public void testCompositePk_DeltaImport_replace_nodelete() throws Exception {
- add1document();
- MockDataSource.clearCache();
-
- List deltaRow = new ArrayList();
- deltaRow.add(createMap("id", "1"));
- MockDataSource.setIterator(DELTA_QUERY,
- deltaRow.iterator());
-
- MockDataSource.setIterator(DELETED_PK_QUERY, Collections
- .EMPTY_LIST.iterator());
-
- List parentRow = new ArrayList();
- parentRow.add(createMap("id", "1"));
- MockDataSource.setIterator("select * from x where id='1'", parentRow
- .iterator());
-
- List childRow = new ArrayList();
- childRow.add(createMap("desc", "goodbye"));
- MockDataSource.setIterator("select * from y where y.A='1'", childRow
- .iterator());
-
- runDeltaImport(dataConfig_delta);
-
- assertQ(req("*:* OR XtestCompositePk_DeltaImport_replace_nodelete"), "//*[@numFound='1']");
- assertQ(req("id:1"), "//*[@numFound='1']");
- assertQ(req("desc:hello OR XtestCompositePk_DeltaImport_replace_nodelete"), "//*[@numFound='0']");
- assertQ(req("desc:goodbye"), "//*[@numFound='1']");
- }
-
- @Test
- @SuppressWarnings("unchecked")
- public void testCompositePk_DeltaImport_add() throws Exception {
- add1document();
- MockDataSource.clearCache();
-
- List deltaRow = new ArrayList();
- deltaRow.add(createMap("id", "2"));
- MockDataSource.setIterator(DELTA_QUERY,
- deltaRow.iterator());
-
- List parentRow = new ArrayList();
- parentRow.add(createMap("id", "2"));
- MockDataSource.setIterator("select * from x where id='2'", parentRow
- .iterator());
-
- List childRow = new ArrayList();
- childRow.add(createMap("desc", "goodbye"));
- MockDataSource.setIterator("select * from y where y.A='2'", childRow
- .iterator());
-
- runDeltaImport(dataConfig_delta);
-
- assertQ(req("*:* OR testCompositePk_DeltaImport_add"), "//*[@numFound='2']");
- assertQ(req("id:1"), "//*[@numFound='1']");
- assertQ(req("id:2"), "//*[@numFound='1']");
- assertQ(req("desc:hello"), "//*[@numFound='1']");
- assertQ(req("desc:goodbye"), "//*[@numFound='1']");
- }
-
- @Test
- @SuppressWarnings("unchecked")
- public void testCompositePk_DeltaImport_nodelta() throws Exception {
- add1document();
- MockDataSource.clearCache();
-
- MockDataSource.setIterator(DELTA_QUERY,
- Collections.EMPTY_LIST.iterator());
-
- runDeltaImport(dataConfig_delta);
-
- assertQ(req("*:* OR testCompositePk_DeltaImport_nodelta"), "//*[@numFound='1']");
- assertQ(req("id:1 OR testCompositePk_DeltaImport_nodelta"), "//*[@numFound='1']");
- assertQ(req("desc:hello OR testCompositePk_DeltaImport_nodelta"), "//*[@numFound='1']");
- }
-
- @Test
- @SuppressWarnings("unchecked")
- public void testCompositePk_DeltaImport_add_delete() throws Exception {
- add1document();
- MockDataSource.clearCache();
-
- List deltaRow = new ArrayList();
- deltaRow.add(createMap("id", "2"));
- MockDataSource.setIterator(DELTA_QUERY,
- deltaRow.iterator());
-
- List deletedRow = new ArrayList();
- deletedRow.add(createMap("id", "1"));
- MockDataSource.setIterator(DELETED_PK_QUERY,
- deletedRow.iterator());
-
- List parentRow = new ArrayList();
- parentRow.add(createMap("id", "2"));
- MockDataSource.setIterator("select * from x where id='2'", parentRow
- .iterator());
-
- List childRow = new ArrayList();
- childRow.add(createMap("desc", "goodbye"));
- MockDataSource.setIterator("select * from y where y.A='2'", childRow
- .iterator());
-
- runDeltaImport(dataConfig_delta);
-
- assertQ(req("*:* OR XtestCompositePk_DeltaImport_add_delete"), "//*[@numFound='1']");
- assertQ(req("id:2"), "//*[@numFound='1']");
- assertQ(req("desc:hello"), "//*[@numFound='0']");
- assertQ(req("desc:goodbye"), "//*[@numFound='1']");
+ protected String deltaQueriesPersonTable() {
+ return
+ "deletedPkQuery=''SELECT ID FROM PEOPLE WHERE DELETED='Y' AND last_modified >='${dih.last_index_time}' '' " +
+ "deltaImportQuery=''SELECT ID, NAME, COUNTRY_CODE FROM PEOPLE where ID=${dih.delta.ID} '' " +
+ "deltaQuery=''" +
+ "SELECT ID FROM PEOPLE WHERE DELETED!='Y' AND last_modified >='${dih.last_index_time}' " +
+ (useParentDeltaQueryParam ? "" :
+ "UNION DISTINCT " +
+ "SELECT ID FROM PEOPLE WHERE DELETED!='Y' AND COUNTRY_CODE IN (SELECT CODE FROM COUNTRIES WHERE last_modified >='${dih.last_index_time}') "
+ ) + "'' "
+ ;
+ }
+ @Override
+ protected String deltaQueriesCountryTable() {
+ if(useParentDeltaQueryParam) {
+ return
+ "deltaQuery=''SELECT CODE FROM COUNTRIES WHERE DELETED != 'Y' AND last_modified >='${dih.last_index_time}' '' " +
+ "parentDeltaQuery=''SELECT ID FROM PEOPLE WHERE DELETED != 'Y' AND COUNTRY_CODE='${Countries.CODE}' '' "
+ ;
+
+ }
+ return "";
}
}