You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by no...@apache.org on 2014/12/15 18:12:31 UTC
svn commit: r1643101 - in /lucene/dev/branches/branch_5x: ./ solr/
solr/contrib/
solr/contrib/dataimporthandler-extras/src/java/org/apache/solr/handler/dataimport/
solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/
solr/contrib...
Author: noble
Date: Wed Dec 3 12:06:51 2014
New Revision: 1643101
URL: http://svn.apache.org/r1643101
Log:
SOLR-4799 DIH: join=zipper aka merge join for nested EntityProcessors
Added:
lucene/dev/branches/branch_5x/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/Zipper.java
- copied unchanged from r1643097, lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/Zipper.java
Modified:
lucene/dev/branches/branch_5x/ (props changed)
lucene/dev/branches/branch_5x/solr/ (props changed)
lucene/dev/branches/branch_5x/solr/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_5x/solr/contrib/ (props changed)
lucene/dev/branches/branch_5x/solr/contrib/dataimporthandler-extras/src/java/org/apache/solr/handler/dataimport/TikaEntityProcessor.java
lucene/dev/branches/branch_5x/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHCacheSupport.java
lucene/dev/branches/branch_5x/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/EntityProcessorBase.java
lucene/dev/branches/branch_5x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractSqlEntityProcessorTestCase.java
lucene/dev/branches/branch_5x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/MockSolrEntityProcessor.java
lucene/dev/branches/branch_5x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSolrEntityProcessorUnit.java
lucene/dev/branches/branch_5x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessor.java
Modified: lucene/dev/branches/branch_5x/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/CHANGES.txt?rev=1643101&r1=1643100&r2=1643101&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/CHANGES.txt (original)
+++ lucene/dev/branches/branch_5x/solr/CHANGES.txt Wed Dec 3 12:06:51 2014
@@ -185,6 +185,9 @@ New Features
* SOLR-6607: Managing requesthandlers throuh API (Noble Paul)
+* SOLR-4799: faster join using join="zipper" aka merge join for nested DIH EntityProcessors
+ (Mikhail Khludnev via Noble Paul)
+
Bug Fixes
----------------------
Modified: lucene/dev/branches/branch_5x/solr/contrib/dataimporthandler-extras/src/java/org/apache/solr/handler/dataimport/TikaEntityProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/contrib/dataimporthandler-extras/src/java/org/apache/solr/handler/dataimport/TikaEntityProcessor.java?rev=1643101&r1=1643100&r2=1643101&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/contrib/dataimporthandler-extras/src/java/org/apache/solr/handler/dataimport/TikaEntityProcessor.java (original)
+++ lucene/dev/branches/branch_5x/solr/contrib/dataimporthandler-extras/src/java/org/apache/solr/handler/dataimport/TikaEntityProcessor.java Wed Dec 3 12:06:51 2014
@@ -69,6 +69,7 @@ public class TikaEntityProcessor extends
@Override
protected void firstInit(Context context) {
+ super.firstInit(context);
try {
String tikaConfigFile = context.getResolvedEntityAttribute("tikaConfig");
if (tikaConfigFile == null) {
Modified: lucene/dev/branches/branch_5x/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHCacheSupport.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHCacheSupport.java?rev=1643101&r1=1643100&r2=1643101&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHCacheSupport.java (original)
+++ lucene/dev/branches/branch_5x/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHCacheSupport.java Wed Dec 3 12:06:51 2014
@@ -44,28 +44,11 @@ public class DIHCacheSupport {
public DIHCacheSupport(Context context, String cacheImplName) {
this.cacheImplName = cacheImplName;
- String where = context.getEntityAttribute("where");
- String cacheKey = context.getEntityAttribute(DIHCacheSupport.CACHE_PRIMARY_KEY);
- String lookupKey = context.getEntityAttribute(DIHCacheSupport.CACHE_FOREIGN_KEY);
- if (cacheKey != null && lookupKey == null) {
- throw new DataImportHandlerException(DataImportHandlerException.SEVERE,
- "'cacheKey' is specified for the entity "
- + context.getEntityAttribute("name")
- + " but 'cacheLookup' is missing");
-
- }
- if (where == null && cacheKey == null) {
- cacheDoKeyLookup = false;
- } else {
- if (where != null) {
- String[] splits = where.split("=");
- cacheKey = splits[0];
- cacheForeignKey = splits[1].trim();
- } else {
- cacheForeignKey = lookupKey;
- }
- cacheDoKeyLookup = true;
- }
+ Relation r = new Relation(context);
+ cacheDoKeyLookup = r.doKeyLookup;
+ String cacheKey = r.primaryKey;
+ cacheForeignKey = r.foreignKey;
+
context.setSessionAttribute(DIHCacheSupport.CACHE_PRIMARY_KEY, cacheKey,
Context.SCOPE_ENTITY);
context.setSessionAttribute(DIHCacheSupport.CACHE_FOREIGN_KEY, cacheForeignKey,
@@ -76,6 +59,48 @@ public class DIHCacheSupport {
Context.SCOPE_ENTITY);
}
+ static class Relation{
+ protected final boolean doKeyLookup;
+ protected final String foreignKey;
+ protected final String primaryKey;
+
+ public Relation(Context context) {
+ String where = context.getEntityAttribute("where");
+ String cacheKey = context.getEntityAttribute(DIHCacheSupport.CACHE_PRIMARY_KEY);
+ String lookupKey = context.getEntityAttribute(DIHCacheSupport.CACHE_FOREIGN_KEY);
+ if (cacheKey != null && lookupKey == null) {
+ throw new DataImportHandlerException(DataImportHandlerException.SEVERE,
+ "'cacheKey' is specified for the entity "
+ + context.getEntityAttribute("name")
+ + " but 'cacheLookup' is missing");
+
+ }
+ if (where == null && cacheKey == null) {
+ doKeyLookup = false;
+ primaryKey = null;
+ foreignKey = null;
+ } else {
+ if (where != null) {
+ String[] splits = where.split("=");
+ primaryKey = splits[0];
+ foreignKey = splits[1].trim();
+ } else {
+ primaryKey = cacheKey;
+ foreignKey = lookupKey;
+ }
+ doKeyLookup = true;
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "Relation "
+ + primaryKey + "="+foreignKey ;
+ }
+
+
+ }
+
private DIHCache instantiateCache(Context context) {
DIHCache cache = null;
try {
Modified: lucene/dev/branches/branch_5x/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/EntityProcessorBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/EntityProcessorBase.java?rev=1643101&r1=1643100&r2=1643101&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/EntityProcessorBase.java (original)
+++ lucene/dev/branches/branch_5x/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/EntityProcessorBase.java Wed Dec 3 12:06:51 2014
@@ -48,6 +48,8 @@ public class EntityProcessorBase extends
protected String onError = ABORT;
protected DIHCacheSupport cacheSupport = null;
+
+ protected Zipper zipper;
@Override
@@ -56,19 +58,30 @@ public class EntityProcessorBase extends
if (isFirstInit) {
firstInit(context);
}
- if(cacheSupport!=null) {
- cacheSupport.initNewParent(context);
- }
-
+ if(zipper!=null){
+ zipper.onNewParent(context);
+ }else{
+ if(cacheSupport!=null) {
+ cacheSupport.initNewParent(context);
+ }
+ }
}
- /**first time init call. do one-time operations here
+ /**
+ * first time init call. do one-time operations here
+ * it's necessary to call it from the overridden method,
+ * otherwise it throws NPE on accessing zipper from nextRow()
*/
protected void firstInit(Context context) {
entityName = context.getEntityAttribute("name");
String s = context.getEntityAttribute(ON_ERROR);
if (s != null) onError = s;
- initCache(context);
+
+ zipper = Zipper.createOrNull(context);
+
+ if(zipper==null){
+ initCache(context);
+ }
isFirstInit = false;
}
@@ -109,25 +122,29 @@ public class EntityProcessorBase extends
}
protected Map<String, Object> getNext() {
- if(cacheSupport==null) {
- try {
- if (rowIterator == null)
+ if(zipper!=null){
+ return zipper.supplyNextChild(rowIterator);
+ }else{
+ if(cacheSupport==null) {
+ try {
+ if (rowIterator == null)
+ return null;
+ if (rowIterator.hasNext())
+ return rowIterator.next();
+ query = null;
+ rowIterator = null;
+ return null;
+ } catch (Exception e) {
+ SolrException.log(log, "getNext() failed for query '" + query + "'", e);
+ query = null;
+ rowIterator = null;
+ wrapAndThrow(DataImportHandlerException.WARN, e);
return null;
- if (rowIterator.hasNext())
- return rowIterator.next();
- query = null;
- rowIterator = null;
- return null;
- } catch (Exception e) {
- SolrException.log(log, "getNext() failed for query '" + query + "'", e);
- query = null;
- rowIterator = null;
- wrapAndThrow(DataImportHandlerException.WARN, e);
- return null;
- }
- } else {
- return cacheSupport.getCacheData(context, query, rowIterator);
- }
+ }
+ } else {
+ return cacheSupport.getCacheData(context, query, rowIterator);
+ }
+ }
}
Modified: lucene/dev/branches/branch_5x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractSqlEntityProcessorTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractSqlEntityProcessorTestCase.java?rev=1643101&r1=1643100&r2=1643101&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractSqlEntityProcessorTestCase.java (original)
+++ lucene/dev/branches/branch_5x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractSqlEntityProcessorTestCase.java Wed Dec 3 12:06:51 2014
@@ -45,8 +45,15 @@ public abstract class AbstractSqlEntityP
protected boolean useSimpleCaches;
protected boolean countryEntity;
protected boolean countryCached;
+ protected boolean countryZipper;
protected boolean sportsEntity;
protected boolean sportsCached;
+ protected boolean sportsZipper;
+
+ protected boolean wrongPeopleOrder ;
+ protected boolean wrongSportsOrder ;
+ protected boolean wrongCountryOrder;
+
protected String rootTransformerName;
protected boolean countryTransformer;
protected boolean sportsTransformer;
@@ -65,8 +72,15 @@ public abstract class AbstractSqlEntityP
useSimpleCaches = false;
countryEntity = false;
countryCached = false;
+ countryZipper = false;
sportsEntity = false;
sportsCached = false;
+ sportsZipper = false;
+
+ wrongPeopleOrder = false;
+ wrongSportsOrder = false;
+ wrongCountryOrder= false;
+
rootTransformerName = null;
countryTransformer = false;
sportsTransformer = false;
@@ -177,9 +191,14 @@ public abstract class AbstractSqlEntityP
} else {
countryEntity = true;
sportsEntity = true;
- if (numChildren == 1) {
- countryEntity = random().nextBoolean();
- sportsEntity = !countryEntity;
+ if(countryZipper||sportsZipper){// zipper tests fully cover nums of children
+ countryEntity = countryZipper;
+ sportsEntity = sportsZipper;
+ }else{// apply default randomization on cached cases
+ if (numChildren == 1) {
+ countryEntity = random().nextBoolean();
+ sportsEntity = !countryEntity;
+ }
}
if (countryEntity) {
countryTransformer = random().nextBoolean();
@@ -212,19 +231,30 @@ public abstract class AbstractSqlEntityP
+ (totalPeople()) + "']");
}
if (countryEntity) {
- if (personNameExists("Jayden")) {
- String nrName = countryNameByCode("NP");
- if (nrName != null && nrName.length() > 0) {
- assertQ(req("NAME_mult_s:Jayden"), "//*[@numFound='1']",
+ {
+ String[] people = getStringsFromQuery("SELECT NAME FROM PEOPLE WHERE DELETED != 'Y'");
+ String man = people[random().nextInt(people.length)];
+ String[] countryNames = getStringsFromQuery("SELECT C.COUNTRY_NAME FROM PEOPLE P "
+ + "INNER JOIN COUNTRIES C ON P.COUNTRY_CODE=C.CODE "
+ + "WHERE P.DELETED!='Y' AND C.DELETED!='Y' AND P.NAME='" + man + "'");
+
+ assertQ(req("{!term f=NAME_mult_s}"+ man), "//*[@numFound='1']",
+ countryNames.length>0?
+ "//doc/str[@name='COUNTRY_NAME_s']='" + countryNames[random().nextInt(countryNames.length)] + "'"
+ :"//doc[count(*[@name='COUNTRY_NAME_s'])=0]");
+ }
+ {
+ String[] countryCodes = getStringsFromQuery("SELECT CODE FROM COUNTRIES WHERE DELETED != 'Y'");
+ String theCode = countryCodes[random().nextInt(countryCodes.length)];
+ int num = numberPeopleByCountryCode(theCode);
+ if(num>0){
+ String nrName = countryNameByCode(theCode);
+ assertQ(req("COUNTRY_CODES_mult_s:"+theCode), "//*[@numFound='" + num + "']",
"//doc/str[@name='COUNTRY_NAME_s']='" + nrName + "'");
+ }else{ // no one lives there anyway
+ assertQ(req("COUNTRY_CODES_mult_s:"+theCode), "//*[@numFound='" + num + "']");
}
}
- String nrName = countryNameByCode("NR");
- int num = numberPeopleByCountryCode("NR");
- if (nrName != null && num > 0) {
- assertQ(req("COUNTRY_CODES_mult_s:NR"), "//*[@numFound='" + num + "']",
- "//doc/str[@name='COUNTRY_NAME_s']='" + nrName + "'");
- }
if (countryTransformer && !underlyingDataModified) {
assertQ(req("countryAdded_s:country_added"), "//*[@numFound='"
+ totalPeople() + "']");
@@ -234,24 +264,27 @@ public abstract class AbstractSqlEntityP
if (!underlyingDataModified) {
assertQ(req("SPORT_NAME_mult_s:Sailing"), "//*[@numFound='2']");
}
- String michaelsName = personNameById(3);
- String[] michaelsSports = sportNamesByPersonId(3);
- if (michaelsName != null && michaelsSports.length > 0) {
- String[] xpath = new String[michaelsSports.length + 1];
- xpath[0] = "//*[@numFound='1']";
- int i = 1;
- for (String ms : michaelsSports) {
- xpath[i] = "//doc/arr[@name='SPORT_NAME_mult_s']/str[" + i + "]='"
- + ms + "'";
- i++;
- }
- assertQ(req("NAME_mult_s:" + michaelsName.replaceAll("\\W", "\\\\$0")),
- xpath);
+ String [] names = getStringsFromQuery("SELECT NAME FROM PEOPLE WHERE DELETED != 'Y'");
+ String name = names[random().nextInt(names.length)];
+ int personId = getIntFromQuery("SELECT ID FROM PEOPLE WHERE DELETED != 'Y' AND NAME='"+name+"'");
+ String[] michaelsSports = sportNamesByPersonId(personId);
+
+ String[] xpath = new String[michaelsSports.length + 1];
+ xpath[0] = "//*[@numFound='1']";
+ int i = 1;
+ for (String ms : michaelsSports) {
+ xpath[i] = "//doc/arr[@name='SPORT_NAME_mult_s']/str='"//[" + i + "]='" don't care about particular order
+ + ms + "'";
+ i++;
}
+ assertQ(req("NAME_mult_s:" + name.replaceAll("\\W", "\\\\$0")),
+ xpath);
if (!underlyingDataModified && sportsTransformer) {
assertQ(req("sportsAdded_s:sport_added"), "//*[@numFound='"
- + (totalPeople()) + "']");
+ + (totalSportsmen()) + "']");
}
+ assertQ("checking orphan sport is absent",
+ req("{!term f=SPORT_NAME_mult_s}No Fishing"), "//*[@numFound='0']");
}
if (checkDatabaseRequests) {
Assert.assertTrue("Expecting " + numDatabaseRequests
@@ -304,6 +337,7 @@ public abstract class AbstractSqlEntityP
}
}
+
private int getIntFromQuery(String query) throws Exception {
Connection conn = null;
Statement s = null;
@@ -367,6 +401,12 @@ public abstract class AbstractSqlEntityP
return getIntFromQuery("SELECT COUNT(1) FROM PEOPLE WHERE DELETED != 'Y' ");
}
+ public int totalSportsmen() throws Exception {
+ return getIntFromQuery("SELECT COUNT(*) FROM PEOPLE WHERE "
+ + "EXISTS(SELECT ID FROM PEOPLE_SPORTS WHERE PERSON_ID=PEOPLE.ID AND PEOPLE_SPORTS.DELETED != 'Y')"
+ + " AND PEOPLE.DELETED != 'Y'");
+ }
+
public boolean countryCodeExists(String cc) throws Exception {
return getIntFromQuery("SELECT COUNT(1) country_name FROM COUNTRIES WHERE DELETED != 'Y' AND CODE='"
+ cc + "'") > 0;
@@ -574,7 +614,11 @@ public abstract class AbstractSqlEntityP
sb.append("dataSource=''" + ds + "'' ");
sb.append(rootTransformerName != null ? "transformer=''"
+ rootTransformerName + "'' " : "");
- sb.append("query=''SELECT ID, NAME, COUNTRY_CODE FROM PEOPLE WHERE DELETED != 'Y' '' ");
+
+ sb.append("query=''SELECT ID, NAME, COUNTRY_CODE FROM PEOPLE WHERE DELETED != 'Y' "
+ +((sportsZipper||countryZipper?"ORDER BY ID":"")
+ +(wrongPeopleOrder? " DESC":""))+"'' ");
+
sb.append(deltaQueriesPersonTable());
sb.append("> \n");
@@ -595,9 +639,20 @@ public abstract class AbstractSqlEntityP
if (useSimpleCaches) {
sb.append("query=''SELECT CODE, COUNTRY_NAME FROM COUNTRIES WHERE DELETED != 'Y' AND CODE='${People.COUNTRY_CODE}' ''>\n");
} else {
- sb.append(random().nextBoolean() ? "cacheKey=''CODE'' cacheLookup=''People.COUNTRY_CODE'' "
- : "where=''CODE=People.COUNTRY_CODE'' ");
- sb.append("query=''SELECT CODE, COUNTRY_NAME FROM COUNTRIES'' ");
+
+ if(countryZipper){// really odd join btw. it sends duped countries
+ sb.append(random().nextBoolean() ? "cacheKey=''ID'' cacheLookup=''People.ID'' "
+ : "where=''ID=People.ID'' ");
+ sb.append("join=''zipper'' query=''SELECT PEOPLE.ID, CODE, COUNTRY_NAME FROM COUNTRIES"
+ + " JOIN PEOPLE ON COUNTRIES.CODE=PEOPLE.COUNTRY_CODE "
+ + "WHERE PEOPLE.DELETED != 'Y' ORDER BY PEOPLE.ID "+
+ (wrongCountryOrder ? " DESC":"")
+ + "'' ");
+ }else{
+ sb.append(random().nextBoolean() ? "cacheKey=''CODE'' cacheLookup=''People.COUNTRY_CODE'' "
+ : "where=''CODE=People.COUNTRY_CODE'' ");
+ sb.append("query=''SELECT CODE, COUNTRY_NAME FROM COUNTRIES'' ");
+ }
sb.append("> \n");
}
} else {
@@ -623,7 +678,14 @@ public abstract class AbstractSqlEntityP
} else {
sb.append(random().nextBoolean() ? "cacheKey=''PERSON_ID'' cacheLookup=''People.ID'' "
: "where=''PERSON_ID=People.ID'' ");
- sb.append("query=''SELECT ID, PERSON_ID, SPORT_NAME FROM PEOPLE_SPORTS ORDER BY ID'' ");
+ if(sportsZipper){
+ sb.append("join=''zipper'' query=''SELECT ID, PERSON_ID, SPORT_NAME FROM PEOPLE_SPORTS ORDER BY PERSON_ID"
+ + (wrongSportsOrder?" DESC" : "")+
+ "'' ");
+ }
+ else{
+ sb.append("query=''SELECT ID, PERSON_ID, SPORT_NAME FROM PEOPLE_SPORTS ORDER BY ID'' ");
+ }
}
} else {
sb.append("processor=''SqlEntityProcessor'' query=''SELECT ID, SPORT_NAME FROM PEOPLE_SPORTS WHERE DELETED != 'Y' AND PERSON_ID=${People.ID} ORDER BY ID'' ");
@@ -726,7 +788,9 @@ public abstract class AbstractSqlEntityP
{7,"Noah","NI"},
{8,"Daniel","NG"},
{9,"Aiden","NF"},
- {10,"Anthony","NE"},
+
+ {21,"Anthony","NE"}, // there is no ID=10 anymore
+
{11,"Emma","NL"},
{12,"Grace","NI"},
{13,"Hailey","NG"},
@@ -751,7 +815,10 @@ public abstract class AbstractSqlEntityP
{700, 7, "Boating"},
{800, 8, "Bodyboarding"},
{900, 9, "Canoeing"},
- {1000, 10, "Fishing"},
+
+ {1000, 10, "No Fishing"}, // orhpaned sport
+ //
+
{1100, 11, "Jet Ski"},
{1110, 11, "Rowing"},
{1120, 11, "Sailing"},
@@ -760,10 +827,12 @@ public abstract class AbstractSqlEntityP
{1300, 13, "Kite surfing"},
{1400, 14, "Parasailing"},
{1500, 15, "Rafting"},
- {1600, 16, "Rowing"},
+ //{1600, 16, "Rowing"}, Madison has no sport
{1700, 17, "Sailing"},
{1800, 18, "White Water Rafting"},
{1900, 19, "Water skiing"},
- {2000, 20, "Windsurfing"}
+ {2000, 20, "Windsurfing"},
+ {2100, 21, "Concrete diving"},
+ {2110, 21, "Bubble rugby"}
};
}
Modified: lucene/dev/branches/branch_5x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/MockSolrEntityProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/MockSolrEntityProcessor.java?rev=1643101&r1=1643100&r2=1643101&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/MockSolrEntityProcessor.java (original)
+++ lucene/dev/branches/branch_5x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/MockSolrEntityProcessor.java Wed Dec 3 12:06:51 2014
@@ -31,10 +31,6 @@ public class MockSolrEntityProcessor ext
private int rows;
- public MockSolrEntityProcessor(List<SolrTestCaseJ4.Doc> docsData) {
- this(docsData, ROWS_DEFAULT);
- }
-
public MockSolrEntityProcessor(List<SolrTestCaseJ4.Doc> docsData, int rows) {
this.docsData = docsData;
this.rows = rows;
Modified: lucene/dev/branches/branch_5x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSolrEntityProcessorUnit.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSolrEntityProcessorUnit.java?rev=1643101&r1=1643100&r2=1643101&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSolrEntityProcessorUnit.java (original)
+++ lucene/dev/branches/branch_5x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSolrEntityProcessorUnit.java Wed Dec 3 12:06:51 2014
@@ -31,20 +31,33 @@ public class TestSolrEntityProcessorUnit
public void testQuery() {
List<Doc> docs = generateUniqueDocs(2);
- MockSolrEntityProcessor processor = new MockSolrEntityProcessor(docs);
+ MockSolrEntityProcessor processor = createAndInit(docs);
assertExpectedDocs(docs, processor);
assertEquals(1, processor.getQueryCount());
}
+ private MockSolrEntityProcessor createAndInit(List<Doc> docs) {
+ return createAndInit(docs, SolrEntityProcessor.ROWS_DEFAULT);
+ }
+
public void testNumDocsGreaterThanRows() {
List<Doc> docs = generateUniqueDocs(44);
- MockSolrEntityProcessor processor = new MockSolrEntityProcessor(docs, 10);
+ int rowsNum = 10;
+ MockSolrEntityProcessor processor = createAndInit(docs, rowsNum);
assertExpectedDocs(docs, processor);
assertEquals(5, processor.getQueryCount());
}
+ private MockSolrEntityProcessor createAndInit(List<Doc> docs, int rowsNum) {
+ MockSolrEntityProcessor processor = new MockSolrEntityProcessor(docs, rowsNum);
+ HashMap<String,String> entityAttrs = new HashMap<String,String>(){{put(SolrEntityProcessor.SOLR_SERVER,"http://route:66/no");}};
+ processor.init(getContext(null, null, null, null, Collections.emptyList(),
+ entityAttrs));
+ return processor;
+ }
+
public void testMultiValuedFields() {
List<Doc> docs = new ArrayList<>();
List<FldType> types = new ArrayList<>();
@@ -53,7 +66,7 @@ public class TestSolrEntityProcessorUnit
Doc testDoc = createDoc(types);
docs.add(testDoc);
- MockSolrEntityProcessor processor = new MockSolrEntityProcessor(docs);
+ MockSolrEntityProcessor processor = createAndInit(docs);
Map<String, Object> next = processor.nextRow();
assertNotNull(next);
Modified: lucene/dev/branches/branch_5x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessor.java?rev=1643101&r1=1643100&r2=1643101&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessor.java (original)
+++ lucene/dev/branches/branch_5x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessor.java Wed Dec 3 12:06:51 2014
@@ -45,6 +45,60 @@ public class TestSqlEntityProcessor exte
public void testCachedChildEntities() throws Exception {
withChildEntities(true, true);
}
+
+ @Test
+ public void testSportZipperChildEntities() throws Exception {
+ sportsZipper = true;
+ withChildEntities(true, true);
+ }
+
+ @Test
+ public void testCountryZipperChildEntities() throws Exception {
+ countryZipper = true;
+ withChildEntities(true, true);
+ }
+
+ @Test
+ public void testBothZipperChildEntities() throws Exception {
+ countryZipper = true;
+ sportsZipper = true;
+ withChildEntities(true, true);
+ }
+
+ @Test(expected=RuntimeException.class /* DIH exceptions are not propagated, here we capturing assertQ exceptions */)
+ public void testSportZipperChildEntitiesWrongOrder() throws Exception {
+ if(random().nextBoolean()){
+ wrongPeopleOrder = true;
+ }else{
+ wrongSportsOrder = true;
+ }
+ testSportZipperChildEntities();
+ }
+
+ @Test(expected=RuntimeException.class )
+ public void testCountryZipperChildEntitiesWrongOrder() throws Exception {
+ if(random().nextBoolean()){
+ wrongPeopleOrder = true;
+ }else{
+ wrongCountryOrder = true;
+ }
+ testCountryZipperChildEntities();
+ }
+
+ @Test(expected=RuntimeException.class)
+ public void testBothZipperChildEntitiesWrongOrder() throws Exception {
+ if(random().nextBoolean()){
+ wrongPeopleOrder = true;
+ }else{
+ if(random().nextBoolean()){
+ wrongSportsOrder = true;
+ }else{
+ wrongCountryOrder = true;
+ }
+ }
+ testBothZipperChildEntities();
+ }
+
@Test
@Ignore("broken see SOLR-3857")
public void testSimpleCacheChildEntities() throws Exception {