You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ro...@apache.org on 2013/01/18 19:31:23 UTC
svn commit: r1435287 [26/41] - in /lucene/dev/branches/LUCENE-2878: ./
dev-tools/ dev-tools/eclipse/ dev-tools/idea/.idea/libraries/
dev-tools/idea/lucene/analysis/icu/ dev-tools/maven/
dev-tools/maven/lucene/benchmark/ dev-tools/maven/solr/ dev-tools/...
Modified: lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSimplePropertiesWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSimplePropertiesWriter.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSimplePropertiesWriter.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSimplePropertiesWriter.java Fri Jan 18 18:30:54 2013
@@ -1,6 +1,10 @@
package org.apache.solr.handler.dataimport;
import java.io.File;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
@@ -50,8 +54,12 @@ public class TestSimplePropertiesWriter
}
@After
public void spwAfter() throws Exception {
- new File(fileLocation + File.separatorChar + fileName).delete();
- new File(fileLocation).delete();
+ //If an Assume was tripped while setting up the test,
+ //the file might not ever have been created...
+ if(fileLocation!=null) {
+ new File(fileLocation + File.separatorChar + fileName).delete();
+ new File(fileLocation).delete();
+ }
}
@Test
public void testSimplePropertiesWriter() throws Exception {
@@ -91,9 +99,7 @@ public class TestSimplePropertiesWriter
props = spw.readIndexerProperties();
Date entityDate = df.parse((String) props.get("SomeDates.last_index_time"));
Date docDate= df.parse((String) props.get("last_index_time"));
- Calendar c = new GregorianCalendar(TimeZone.getTimeZone("GMT"), Locale.ROOT);
- c.setTime(docDate);
- int year = c.get(Calendar.YEAR);
+ int year = currentYearFromDatabase();
Assert.assertTrue("This date: " + errMsgFormat.format(oneSecondAgo) + " should be prior to the document date: " + errMsgFormat.format(docDate), docDate.getTime() - oneSecondAgo.getTime() > 0);
Assert.assertTrue("This date: " + errMsgFormat.format(oneSecondAgo) + " should be prior to the entity date: " + errMsgFormat.format(entityDate), entityDate.getTime() - oneSecondAgo.getTime() > 0);
@@ -101,6 +107,34 @@ public class TestSimplePropertiesWriter
}
}
+ private int currentYearFromDatabase() throws Exception {
+ Connection conn = null;
+ Statement s = null;
+ ResultSet rs = null;
+ try {
+ conn = newConnection();
+ s = conn.createStatement();
+ rs = s.executeQuery("select year(current_timestamp) from sysibm.sysdummy1");
+ if (rs.next()) {
+ return rs.getInt(1);
+ }
+ Assert.fail("We should have gotten a row from the db.");
+ } catch (SQLException e) {
+ throw e;
+ } finally {
+ try {
+ rs.close();
+ } catch (Exception ex) {}
+ try {
+ s.close();
+ } catch (Exception ex) {}
+ try {
+ conn.close();
+ } catch (Exception ex) {}
+ }
+ return 0;
+ }
+
@Override
protected Database setAllowedDatabases() {
return Database.DERBY;
Modified: lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSolrEntityProcessorEndToEnd.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSolrEntityProcessorEndToEnd.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSolrEntityProcessorEndToEnd.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSolrEntityProcessorEndToEnd.java Fri Jan 18 18:30:54 2013
@@ -30,7 +30,6 @@ import org.apache.commons.io.FileUtils;
import org.apache.http.client.HttpClient;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
-import org.apache.solr.client.solrj.impl.HttpClientUtil;
import org.apache.solr.client.solrj.impl.HttpSolrServer;
import org.apache.solr.common.SolrInputDocument;
import org.junit.After;
@@ -47,11 +46,11 @@ public class TestSolrEntityProcessorEndT
private static Logger LOG = LoggerFactory.getLogger(TestSolrEntityProcessorEndToEnd.class);
- //rivate static final String SOLR_SOURCE_URL = "http://127.0.0.1:8983/solr";
private static final String SOLR_CONFIG = "dataimport-solrconfig.xml";
private static final String SOLR_SCHEMA = "dataimport-schema.xml";
- private static final String SOLR_HOME = "dih/solr";
- private static final String CONF_DIR = "dih" + File.separator + "solr" + File.separator + "collection1" + File.separator + "conf" + File.separator;
+ private static final String SOURCE_CONF_DIR = "dih" + File.separator + "solr" + File.separator + "collection1" + File.separator + "conf" + File.separator;
+
+ private static final String DEAD_SOLR_SERVER = "http://[ff01::114]:33332/solr";
private static final List<Map<String,Object>> DB_DOCS = new ArrayList<Map<String,Object>>();
private static final List<Map<String,Object>> SOLR_DOCS = new ArrayList<Map<String,Object>>();
@@ -73,7 +72,7 @@ public class TestSolrEntityProcessorEndT
private SolrInstance instance = null;
private JettySolrRunner jetty;
- private static String getDihConfigTagsInnerEntity(int port) {
+ private String getDihConfigTagsInnerEntity() {
return "<dataConfig>\r\n"
+ " <dataSource type='MockDataSource' />\r\n"
+ " <document>\r\n"
@@ -81,21 +80,21 @@ public class TestSolrEntityProcessorEndT
+ " <field column='dbid_s' />\r\n"
+ " <field column='dbdesc_s' />\r\n"
+ " <entity name='se' processor='SolrEntityProcessor' query='id:${db.dbid_s}'\n"
- + " url='" + getSourceUrl(port) + "' fields='id,desc'>\r\n"
+ + " url='" + getSourceUrl() + "' fields='id,desc'>\r\n"
+ " <field column='id' />\r\n"
+ " <field column='desc' />\r\n" + " </entity>\r\n"
+ " </entity>\r\n" + " </document>\r\n" + "</dataConfig>\r\n";
}
- private static String generateDIHConfig(String options, int port) {
+ private String generateDIHConfig(String options, boolean useDeadServer) {
return "<dataConfig>\r\n" + " <document>\r\n"
+ " <entity name='se' processor='SolrEntityProcessor'" + " url='"
- + getSourceUrl(port) + "' " + options + " />\r\n" + " </document>\r\n"
+ + (useDeadServer ? DEAD_SOLR_SERVER : getSourceUrl()) + "' " + options + " />\r\n" + " </document>\r\n"
+ "</dataConfig>\r\n";
}
- private static String getSourceUrl(int port) {
- return "http://127.0.0.1:" + port + "/solr";
+ private String getSourceUrl() {
+ return "http://127.0.0.1:" + jetty.getLocalPort() + "/solr";
}
//TODO: fix this test to close its directories
@@ -120,7 +119,7 @@ public class TestSolrEntityProcessorEndT
public void setUp() throws Exception {
super.setUp();
// destination solr core
- initCore(SOLR_CONFIG, SOLR_SCHEMA, SOLR_HOME);
+ initCore(SOLR_CONFIG, SOLR_SCHEMA);
// data source solr instance
instance = new SolrInstance();
instance.setUp();
@@ -145,7 +144,7 @@ public class TestSolrEntityProcessorEndT
try {
addDocumentsToSolr(SOLR_DOCS);
- runFullImport(generateDIHConfig("query='*:*' rows='2' fl='id,desc' onError='skip'", jetty.getLocalPort()));
+ runFullImport(generateDIHConfig("query='*:*' rows='2' fl='id,desc' onError='skip'", false));
} catch (Exception e) {
LOG.error(e.getMessage(), e);
fail(e.getMessage());
@@ -163,7 +162,7 @@ public class TestSolrEntityProcessorEndT
addDocumentsToSolr(generateSolrDocuments(30));
Map<String,String> map = new HashMap<String,String>();
map.put("rows", "50");
- runFullImport(generateDIHConfig("query='*:*' fq='desc:Description1*,desc:Description*2' rows='2'", jetty.getLocalPort()), map);
+ runFullImport(generateDIHConfig("query='*:*' fq='desc:Description1*,desc:Description*2' rows='2'", false), map);
} catch (Exception e) {
LOG.error(e.getMessage(), e);
fail(e.getMessage());
@@ -178,7 +177,7 @@ public class TestSolrEntityProcessorEndT
try {
addDocumentsToSolr(generateSolrDocuments(7));
- runFullImport(generateDIHConfig("query='*:*' fl='id' rows='2'", jetty.getLocalPort()));
+ runFullImport(generateDIHConfig("query='*:*' fl='id' rows='2'", false));
} catch (Exception e) {
LOG.error(e.getMessage(), e);
fail(e.getMessage());
@@ -204,7 +203,7 @@ public class TestSolrEntityProcessorEndT
try {
MockDataSource.setIterator("select * from x", DB_DOCS.iterator());
addDocumentsToSolr(SOLR_DOCS);
- runFullImport(getDihConfigTagsInnerEntity(jetty.getLocalPort()));
+ runFullImport(getDihConfigTagsInnerEntity());
} catch (Exception e) {
LOG.error(e.getMessage(), e);
fail(e.getMessage());
@@ -221,17 +220,10 @@ public class TestSolrEntityProcessorEndT
}
public void testFullImportWrongSolrUrl() {
- try {
- jetty.stop();
- } catch (Exception e) {
- LOG.error("Error stopping jetty", e);
- fail(e.getMessage());
- }
-
assertQ(req("*:*"), "//result[@numFound='0']");
try {
- runFullImport(generateDIHConfig("query='*:*' rows='2' fl='id,desc' onError='skip'", jetty.getLocalPort()));
+ runFullImport(generateDIHConfig("query='*:*' rows='2' fl='id,desc' onError='skip'", true /* use dead server */));
} catch (Exception e) {
LOG.error(e.getMessage(), e);
fail(e.getMessage());
@@ -244,7 +236,7 @@ public class TestSolrEntityProcessorEndT
assertQ(req("*:*"), "//result[@numFound='0']");
try {
- runFullImport(generateDIHConfig("query='bogus:3' rows='2' fl='id,desc' onError='abort'", jetty.getLocalPort()));
+ runFullImport(generateDIHConfig("query='bogus:3' rows='2' fl='id,desc' onError='abort'", false));
} catch (Exception e) {
LOG.error(e.getMessage(), e);
fail(e.getMessage());
@@ -274,9 +266,9 @@ public class TestSolrEntityProcessorEndT
sidl.add(sd);
}
- HttpClient client = HttpClientUtil.createClient(null);
- URL url = new URL(getSourceUrl(jetty.getLocalPort()));
- HttpSolrServer solrServer = new HttpSolrServer(url.toExternalForm(), client);
+ HttpSolrServer solrServer = new HttpSolrServer(getSourceUrl());
+ solrServer.setConnectionTimeout(15000);
+ solrServer.setSoTimeout(30000);
solrServer.add(sidl);
solrServer.commit(true, true);
}
@@ -291,7 +283,7 @@ public class TestSolrEntityProcessorEndT
}
public String getSchemaFile() {
- return CONF_DIR + "dataimport-schema.xml";
+ return SOURCE_CONF_DIR + "dataimport-schema.xml";
}
public String getDataDir() {
@@ -299,7 +291,7 @@ public class TestSolrEntityProcessorEndT
}
public String getSolrConfigFile() {
- return CONF_DIR + "dataimport-solrconfig.xml";
+ return SOURCE_CONF_DIR + "dataimport-solrconfig.xml";
}
public void setUp() throws Exception {
@@ -321,7 +313,7 @@ public class TestSolrEntityProcessorEndT
FileUtils.copyFile(getFile(getSchemaFile()), f);
f = new File(confDir, "data-config.xml");
- FileUtils.copyFile(getFile(CONF_DIR + "dataconfig-contentstream.xml"), f);
+ FileUtils.copyFile(getFile(SOURCE_CONF_DIR + "dataconfig-contentstream.xml"), f);
}
public void tearDown() throws Exception {
Modified: lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta.java Fri Jan 18 18:30:54 2013
@@ -2,6 +2,7 @@ package org.apache.solr.handler.dataimpo
import org.apache.solr.request.LocalSolrQueryRequest;
import org.junit.Before;
+import org.junit.Ignore;
import org.junit.Test;
/*
@@ -38,35 +39,50 @@ public class TestSqlEntityProcessorDelta
}
@Test
public void testSingleEntity() throws Exception {
+ log.debug("testSingleEntity full-import...");
singleEntity(1);
+ logPropertiesFile();
changeStuff();
int c = calculateDatabaseCalls();
+ log.debug("testSingleEntity delta-import (" + c + " database calls expected)...");
singleEntity(c);
validateChanges();
}
@Test
public void testWithSimpleTransformer() throws Exception {
- simpleTransform(1);
+ log.debug("testWithSimpleTransformer full-import...");
+ simpleTransform(1);
+ logPropertiesFile();
changeStuff();
- simpleTransform(calculateDatabaseCalls());
+ int c = calculateDatabaseCalls();
+ simpleTransform(c);
+ log.debug("testWithSimpleTransformer delta-import (" + c + " database calls expected)...");
validateChanges();
}
@Test
public void testWithComplexTransformer() throws Exception {
+ log.debug("testWithComplexTransformer full-import...");
complexTransform(1, 0);
+ logPropertiesFile();
changeStuff();
- complexTransform(calculateDatabaseCalls(), personChanges.deletedKeys.length);
+ int c = calculateDatabaseCalls();
+ log.debug("testWithComplexTransformer delta-import (" + c + " database calls expected)...");
+ complexTransform(c, personChanges.deletedKeys.length);
validateChanges();
}
@Test
public void testChildEntities() throws Exception {
+ log.debug("testChildEntities full-import...");
useParentDeltaQueryParam = random().nextBoolean();
+ log.debug("using parent delta? " + useParentDeltaQueryParam);
withChildEntities(false, true);
+ logPropertiesFile();
changeStuff();
+ log.debug("testChildEntities delta-import...");
withChildEntities(false, false);
validateChanges();
}
-
+
private int calculateDatabaseCalls() {
//The main query generates 1
@@ -109,7 +125,7 @@ public class TestSqlEntityProcessorDelta
personChanges = modifySomePeople();
break;
case 1:
- countryChanges = modifySomeCountries();
+ countryChanges = modifySomeCountries();
break;
case 2:
personChanges = modifySomePeople();
@@ -119,8 +135,28 @@ public class TestSqlEntityProcessorDelta
} else {
personChanges = modifySomePeople();
}
+ countryChangesLog();
+ personChangesLog();
delta = true;
}
+ private void countryChangesLog()
+ {
+ if(countryChanges!=null) {
+ StringBuilder sb = new StringBuilder();
+ sb.append("country changes { ");
+ for(String s : countryChanges) {
+ sb.append(s).append(" ");
+ }
+ sb.append(" }");
+ log.debug(sb.toString());
+ }
+ }
+ private void personChangesLog()
+ {
+ if(personChanges!=null) {
+ log.debug("person changes { " + personChanges.toString() + " } ");
+ }
+ }
@Override
protected LocalSolrQueryRequest generateRequest() {
return lrf.makeRequest("command", (delta ? "delta-import" : "full-import"), "dataConfig", generateConfig(),
Modified: lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestXPathRecordReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestXPathRecordReader.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestXPathRecordReader.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestXPathRecordReader.java Fri Jan 18 18:30:54 2013
@@ -139,6 +139,7 @@ public class TestXPathRecordReader exten
final List<Map<String, Object>> a = new ArrayList<Map<String, Object>>();
final List<Map<String, Object>> x = new ArrayList<Map<String, Object>>();
rr.streamRecords(new StringReader(xml), new XPathRecordReader.Handler() {
+ @Override
public void handle(Map<String, Object> record, String xpath) {
if (record == null) return;
if (xpath.equals("/root/a")) a.add(record);
Modified: lucene/dev/branches/LUCENE-2878/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingRequestHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingRequestHandler.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingRequestHandler.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingRequestHandler.java Fri Jan 18 18:30:54 2013
@@ -64,6 +64,7 @@ public class ExtractingRequestHandler ex
super.init(args);
}
+ @Override
public void inform(SolrCore core) {
if (initArgs != null) {
//if relative,then relative to config dir, otherwise, absolute path
Modified: lucene/dev/branches/LUCENE-2878/solr/contrib/langid/src/java/org/apache/solr/update/processor/LangDetectLanguageIdentifierUpdateProcessorFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/contrib/langid/src/java/org/apache/solr/update/processor/LangDetectLanguageIdentifierUpdateProcessorFactory.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/contrib/langid/src/java/org/apache/solr/update/processor/LangDetectLanguageIdentifierUpdateProcessorFactory.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/contrib/langid/src/java/org/apache/solr/update/processor/LangDetectLanguageIdentifierUpdateProcessorFactory.java Fri Jan 18 18:30:54 2013
@@ -61,6 +61,7 @@ public class LangDetectLanguageIdentifie
protected SolrParams appends;
protected SolrParams invariants;
+ @Override
public void inform(SolrCore core) {
}
@@ -69,6 +70,7 @@ public class LangDetectLanguageIdentifie
* to a RequestHandler, with defaults, appends and invariants.
* @param args a NamedList with the configuration parameters
*/
+ @Override
@SuppressWarnings("rawtypes")
public void init( NamedList args )
{
Modified: lucene/dev/branches/LUCENE-2878/solr/contrib/langid/src/java/org/apache/solr/update/processor/TikaLanguageIdentifierUpdateProcessorFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/contrib/langid/src/java/org/apache/solr/update/processor/TikaLanguageIdentifierUpdateProcessorFactory.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/contrib/langid/src/java/org/apache/solr/update/processor/TikaLanguageIdentifierUpdateProcessorFactory.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/contrib/langid/src/java/org/apache/solr/update/processor/TikaLanguageIdentifierUpdateProcessorFactory.java Fri Jan 18 18:30:54 2013
@@ -49,6 +49,7 @@ public class TikaLanguageIdentifierUpdat
protected SolrParams appends;
protected SolrParams invariants;
+ @Override
public void inform(SolrCore core) {
}
@@ -57,6 +58,7 @@ public class TikaLanguageIdentifierUpdat
* to a RequestHandler, with defaults, appends and invariants.
* @param args a NamedList with the configuration parameters
*/
+ @Override
@SuppressWarnings("rawtypes")
public void init( NamedList args )
{
Modified: lucene/dev/branches/LUCENE-2878/solr/contrib/uima/README.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/contrib/uima/README.txt?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/contrib/uima/README.txt (original)
+++ lucene/dev/branches/LUCENE-2878/solr/contrib/uima/README.txt Fri Jan 18 18:30:54 2013
@@ -19,7 +19,7 @@ To start using Solr UIMA Metadata Extrac
<lib dir="../../contrib/uima/lib" />
<lib dir="../../contrib/uima/lucene-libs" />
- <lib dir="../../dist/" regex="apache-solr-uima-\d.*\.jar" />
+ <lib dir="../../dist/" regex="solr-uima-\d.*\.jar" />
2. modify your schema.xml adding the fields you want to be hold metadata specifying proper values for type, indexed, stored and multiValued options:
Modified: lucene/dev/branches/LUCENE-2878/solr/contrib/uima/src/java/org/apache/solr/uima/processor/FieldMappingException.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/contrib/uima/src/java/org/apache/solr/uima/processor/FieldMappingException.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/contrib/uima/src/java/org/apache/solr/uima/processor/FieldMappingException.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/contrib/uima/src/java/org/apache/solr/uima/processor/FieldMappingException.java Fri Jan 18 18:30:54 2013
@@ -20,6 +20,7 @@ package org.apache.solr.uima.processor;
/**
* Exception thrown when an error happening while mapping UIMA CAS model to Solr fields
*/
+@SuppressWarnings("serial")
public class FieldMappingException extends Exception {
public FieldMappingException(Exception e) {
super(e);
Modified: lucene/dev/branches/LUCENE-2878/solr/contrib/uima/src/test-files/uima/solr/collection1/conf/solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/contrib/uima/src/test-files/uima/solr/collection1/conf/solrconfig.xml?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/contrib/uima/src/test-files/uima/solr/collection1/conf/solrconfig.xml (original)
+++ lucene/dev/branches/LUCENE-2878/solr/contrib/uima/src/test-files/uima/solr/collection1/conf/solrconfig.xml Fri Jan 18 18:30:54 2013
@@ -44,8 +44,8 @@
in that directory which completely match the regex (anchored on both
ends) will be included.
-->
- <lib dir="../../dist/" regex="apache-solr-cell-\d.*\.jar" />
- <lib dir="../../dist/" regex="apache-solr-clustering-\d.*\.jar" />
+ <lib dir="../../dist/" regex="solr-cell-\d.*\.jar" />
+ <lib dir="../../dist/" regex="solr-clustering-\d.*\.jar" />
<!--
If a dir option (with or without a regex) is used and nothing is
found that matches, it will be ignored
Modified: lucene/dev/branches/LUCENE-2878/solr/contrib/uima/src/test-files/uima/uima-tokenizers-solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/contrib/uima/src/test-files/uima/uima-tokenizers-solrconfig.xml?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/contrib/uima/src/test-files/uima/uima-tokenizers-solrconfig.xml (original)
+++ lucene/dev/branches/LUCENE-2878/solr/contrib/uima/src/test-files/uima/uima-tokenizers-solrconfig.xml Fri Jan 18 18:30:54 2013
@@ -44,8 +44,8 @@
in that directory which completely match the regex (anchored on both
ends) will be included.
-->
- <lib dir="../../dist/" regex="apache-solr-cell-\d.*\.jar" />
- <lib dir="../../dist/" regex="apache-solr-clustering-\d.*\.jar" />
+ <lib dir="../../dist/" regex="solr-cell-\d.*\.jar" />
+ <lib dir="../../dist/" regex="solr-clustering-\d.*\.jar" />
<!--
If a dir option (with or without a regex) is used and nothing is
found that matches, it will be ignored
Modified: lucene/dev/branches/LUCENE-2878/solr/contrib/uima/src/test/org/apache/solr/uima/processor/UIMAUpdateRequestProcessorTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/contrib/uima/src/test/org/apache/solr/uima/processor/UIMAUpdateRequestProcessorTest.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/contrib/uima/src/test/org/apache/solr/uima/processor/UIMAUpdateRequestProcessorTest.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/contrib/uima/src/test/org/apache/solr/uima/processor/UIMAUpdateRequestProcessorTest.java Fri Jan 18 18:30:54 2013
@@ -17,8 +17,6 @@ package org.apache.solr.uima.processor;
* limitations under the License.
*/
-import java.net.URL;
-import java.net.URLConnection;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;
Modified: lucene/dev/branches/LUCENE-2878/solr/contrib/uima/src/test/org/apache/solr/uima/ts/DummySentimentAnnotation.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/contrib/uima/src/test/org/apache/solr/uima/ts/DummySentimentAnnotation.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/contrib/uima/src/test/org/apache/solr/uima/ts/DummySentimentAnnotation.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/contrib/uima/src/test/org/apache/solr/uima/ts/DummySentimentAnnotation.java Fri Jan 18 18:30:54 2013
@@ -24,6 +24,7 @@ public class DummySentimentAnnotation ex
*/
public final static int type = typeIndexID;
/** @generated */
+ @Override
public int getTypeIndexID() {return typeIndexID;}
/** Never called. Disable default constructor
Modified: lucene/dev/branches/LUCENE-2878/solr/contrib/uima/src/test/org/apache/solr/uima/ts/DummySentimentAnnotation_Type.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/contrib/uima/src/test/org/apache/solr/uima/ts/DummySentimentAnnotation_Type.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/contrib/uima/src/test/org/apache/solr/uima/ts/DummySentimentAnnotation_Type.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/contrib/uima/src/test/org/apache/solr/uima/ts/DummySentimentAnnotation_Type.java Fri Jan 18 18:30:54 2013
@@ -17,10 +17,12 @@ import org.apache.uima.jcas.tcas.Annotat
* @generated */
public class DummySentimentAnnotation_Type extends Annotation_Type {
/** @generated */
+ @Override
protected FSGenerator getFSGenerator() {return fsGenerator;}
/** @generated */
private final FSGenerator fsGenerator =
new FSGenerator() {
+ @Override
public FeatureStructure createFS(int addr, CASImpl cas) {
if (DummySentimentAnnotation_Type.this.useExistingInstance) {
// Return eq fs instance if already created
Modified: lucene/dev/branches/LUCENE-2878/solr/contrib/uima/src/test/org/apache/solr/uima/ts/EntityAnnotation.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/contrib/uima/src/test/org/apache/solr/uima/ts/EntityAnnotation.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/contrib/uima/src/test/org/apache/solr/uima/ts/EntityAnnotation.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/contrib/uima/src/test/org/apache/solr/uima/ts/EntityAnnotation.java Fri Jan 18 18:30:54 2013
@@ -24,6 +24,7 @@ public class EntityAnnotation extends An
*/
public final static int type = typeIndexID;
/** @generated */
+ @Override
public int getTypeIndexID() {return typeIndexID;}
/** Never called. Disable default constructor
Modified: lucene/dev/branches/LUCENE-2878/solr/contrib/uima/src/test/org/apache/solr/uima/ts/EntityAnnotation_Type.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/contrib/uima/src/test/org/apache/solr/uima/ts/EntityAnnotation_Type.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/contrib/uima/src/test/org/apache/solr/uima/ts/EntityAnnotation_Type.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/contrib/uima/src/test/org/apache/solr/uima/ts/EntityAnnotation_Type.java Fri Jan 18 18:30:54 2013
@@ -17,10 +17,12 @@ import org.apache.uima.jcas.tcas.Annotat
* @generated */
public class EntityAnnotation_Type extends Annotation_Type {
/** @generated */
+ @Override
protected FSGenerator getFSGenerator() {return fsGenerator;}
/** @generated */
private final FSGenerator fsGenerator =
new FSGenerator() {
+ @Override
public FeatureStructure createFS(int addr, CASImpl cas) {
if (EntityAnnotation_Type.this.useExistingInstance) {
// Return eq fs instance if already created
Modified: lucene/dev/branches/LUCENE-2878/solr/contrib/velocity/ivy.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/contrib/velocity/ivy.xml?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/contrib/velocity/ivy.xml (original)
+++ lucene/dev/branches/LUCENE-2878/solr/contrib/velocity/ivy.xml Fri Jan 18 18:30:54 2013
@@ -21,7 +21,7 @@
<dependencies>
<dependency org="commons-beanutils" name="commons-beanutils" rev="1.7.0" transitive="false"/>
<dependency org="commons-collections" name="commons-collections" rev="3.2.1" transitive="false"/>
- <dependency org="org.apache.velocity" name="velocity" rev="1.6.4" transitive="false"/>
+ <dependency org="org.apache.velocity" name="velocity" rev="1.7" transitive="false"/>
<dependency org="org.apache.velocity" name="velocity-tools" rev="2.0" transitive="false"/>
<exclude org="*" ext="*" matcher="regexp" type="${ivy.exclude.types}"/>
</dependencies>
Modified: lucene/dev/branches/LUCENE-2878/solr/contrib/velocity/src/java/org/apache/solr/response/VelocityResponseWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/contrib/velocity/src/java/org/apache/solr/response/VelocityResponseWriter.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/contrib/velocity/src/java/org/apache/solr/response/VelocityResponseWriter.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/contrib/velocity/src/java/org/apache/solr/response/VelocityResponseWriter.java Fri Jan 18 18:30:54 2013
@@ -30,8 +30,6 @@ import org.apache.velocity.runtime.Runti
import org.apache.velocity.tools.generic.*;
import java.io.*;
-import java.util.HashMap;
-import java.util.Map;
import java.util.Properties;
public class VelocityResponseWriter implements QueryResponseWriter {
@@ -39,6 +37,7 @@ public class VelocityResponseWriter impl
// TODO: maybe pass this Logger to the template for logging from there?
// private static final Logger log = LoggerFactory.getLogger(VelocityResponseWriter.class);
+ @Override
public void write(Writer writer, SolrQueryRequest request, SolrQueryResponse response) throws IOException {
VelocityEngine engine = getEngine(request); // TODO: have HTTP headers available for configuring engine
@@ -83,64 +82,6 @@ public class VelocityResponseWriter impl
context.put("engine", engine); // for $engine.resourceExists(...)
- // Mimetype to extension map for detecting file type and show icon
- // List of types match the icons in /solr/img/filetypes
- Map<String, String> mimeToExt = new HashMap<String, String>() {{
- put("application/x-7z-compressed", "7z");
- put("application/postscript", "ai");
- put("application/pgp-signature", "asc");
- put("application/octet-stream", "bin");
- put("application/x-bzip2", "bz2");
- put("text/x-c", "c");
- put("application/vnd.ms-htmlhelp", "chm");
- put("application/java-vm", "class");
- put("text/css", "css");
- put("text/csv", "csv");
- put("application/x-debian-package", "deb");
- put("application/msword", "doc");
- put("message/rfc822", "eml");
- put("image/gif", "gif");
- put("application/winhlp", "hlp");
- put("text/html", "html");
- put("application/java-archive", "jar");
- put("text/x-java-source", "java");
- put("image/jpeg", "jpeg");
- put("application/javascript", "js");
- put("application/vnd.oasis.opendocument.chart", "odc");
- put("application/vnd.oasis.opendocument.formula", "odf");
- put("application/vnd.oasis.opendocument.graphics", "odg");
- put("application/vnd.oasis.opendocument.image", "odi");
- put("application/vnd.oasis.opendocument.presentation", "odp");
- put("application/vnd.oasis.opendocument.spreadsheet", "ods");
- put("application/vnd.oasis.opendocument.text", "odt");
- put("application/pdf", "pdf");
- put("application/pgp-encrypted", "pgp");
- put("image/png", "png");
- put("application/vnd.ms-powerpoint", "ppt");
- put("audio/x-pn-realaudio", "ram");
- put("application/x-rar-compressed", "rar");
- put("application/vnd.rn-realmedia", "rm");
- put("application/rtf", "rtf");
- put("application/x-shockwave-flash", "swf");
- put("application/vnd.sun.xml.calc", "sxc");
- put("application/vnd.sun.xml.draw", "sxd");
- put("application/vnd.sun.xml.impress", "sxi");
- put("application/vnd.sun.xml.writer", "sxw");
- put("application/x-tar", "tar");
- put("application/x-tex", "tex");
- put("text/plain", "txt");
- put("text/x-vcard", "vcf");
- put("application/vnd.visio", "vsd");
- put("audio/x-wav", "wav");
- put("audio/x-ms-wma", "wma");
- put("video/x-ms-wmv", "wmv");
- put("application/vnd.ms-excel", "xls");
- put("application/xml", "xml");
- put("application/x-xpinstall", "xpi");
- put("application/zip", "zip");
- }};
- context.put("mimeToExt", mimeToExt);
-
String layout_template = request.getParams().get("v.layout");
String json_wrapper = request.getParams().get("v.json");
boolean wrap_response = (layout_template != null) || (json_wrapper != null);
@@ -246,6 +187,7 @@ public class VelocityResponseWriter impl
return template;
}
+ @Override
public String getContentType(SolrQueryRequest request, SolrQueryResponse response) {
return request.getParams().get("v.contentType", "text/html;charset=UTF-8");
}
@@ -260,6 +202,7 @@ public class VelocityResponseWriter impl
return "{\"result\":\"" + replaced + "\"}";
}
+ @Override
public void init(NamedList args) {
}
}
Modified: lucene/dev/branches/LUCENE-2878/solr/contrib/velocity/src/test-files/velocity/solr/collection1/conf/solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/contrib/velocity/src/test-files/velocity/solr/collection1/conf/solrconfig.xml?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/contrib/velocity/src/test-files/velocity/solr/collection1/conf/solrconfig.xml (original)
+++ lucene/dev/branches/LUCENE-2878/solr/contrib/velocity/src/test-files/velocity/solr/collection1/conf/solrconfig.xml Fri Jan 18 18:30:54 2013
@@ -24,7 +24,7 @@
<luceneMatchVersion>${tests.luceneMatchVersion:LUCENE_CURRENT}</luceneMatchVersion>
<lib dir="../../contrib/velocity/lib" />
- <lib dir="../../dist/" regex="apache-solr-velocity-\d.*\.jar" />
+ <lib dir="../../dist/" regex="solr-velocity-\d.*\.jar" />
<dataDir>${solr.data.dir:}</dataDir>
Modified: lucene/dev/branches/LUCENE-2878/solr/core/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/build.xml?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/build.xml (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/build.xml Fri Jan 18 18:30:54 2013
@@ -42,4 +42,56 @@
</target>
<target name="dist-maven" depends="dist-maven-src-java"/>
+
+
+
+
+ <target name="javacc" depends="javacc-QueryParser"/>
+ <target name="javacc-QueryParser" depends="resolve-javacc">
+ <sequential>
+ <invoke-javacc target="src/java/org/apache/solr/parser/QueryParser.jj"
+ outputDir="src/java/org/apache/solr/parser"/>
+
+ <!-- Change the incorrect public ctors for QueryParser to be protected instead -->
+ <replaceregexp file="src/java/org/apache/solr/parser/QueryParser.java"
+ byline="true"
+ match="public QueryParser\(CharStream "
+ replace="protected QueryParser(CharStream "/>
+ <replaceregexp file="src/java/org/apache/solr/parser/QueryParser.java"
+ byline="true"
+ match="public QueryParser\(QueryParserTokenManager "
+ replace="protected QueryParser(QueryParserTokenManager "/>
+
+ </sequential>
+ </target>
+ <target name="resolve-javacc" xmlns:ivy="antlib:org.apache.ivy.ant">
+ <!-- setup a "fake" JavaCC distribution folder in ${build.dir} to make JavaCC ANT task happy: -->
+ <ivy:retrieve organisation="net.java.dev.javacc" module="javacc" revision="5.0"
+ inline="true" conf="default" transitive="false" type="jar" sync="true"
+ pattern="${build.dir}/javacc/bin/lib/[artifact].[ext]"/>
+ </target>
+
+ <macrodef name="invoke-javacc">
+ <attribute name="target"/>
+ <attribute name="outputDir"/>
+ <sequential>
+ <mkdir dir="@{outputDir}"/>
+ <delete>
+ <fileset dir="@{outputDir}" includes="*.java">
+ <containsregexp expression="Generated.*By.*JavaCC"/>
+ </fileset>
+ </delete>
+ <javacc
+ target="@{target}"
+ outputDirectory="@{outputDir}"
+ javacchome="${build.dir}/javacc"
+ jdkversion="${javac.source}"
+ />
+ <fixcrlf srcdir="@{outputDir}" includes="*.java" encoding="UTF-8">
+ <containsregexp expression="Generated.*By.*JavaCC"/>
+ </fixcrlf>
+ </sequential>
+ </macrodef>
+
+
</project>
Modified: lucene/dev/branches/LUCENE-2878/solr/core/ivy.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/ivy.xml?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/ivy.xml (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/ivy.xml Fri Jan 18 18:30:54 2013
@@ -24,11 +24,12 @@
<dependency org="commons-fileupload" name="commons-fileupload" rev="1.2.1" transitive="false"/>
<dependency org="commons-cli" name="commons-cli" rev="1.2" transitive="false"/>
<dependency org="commons-lang" name="commons-lang" rev="2.6" transitive="false"/>
- <dependency org="com.google.guava" name="guava" rev="r05" transitive="false"/>
- <dependency org="org.easymock" name="easymock" rev="2.2" transitive="false"/>
+ <dependency org="com.google.guava" name="guava" rev="13.0.1" transitive="false"/>
+ <dependency org="org.easymock" name="easymock" rev="3.0" transitive="false"/>
+ <dependency org="cglib" name="cglib-nodep" rev="2.2" transitive="false"/>
+ <dependency org="org.objenesis" name="objenesis" rev="1.2" transitive="false"/>
<dependency org="com.spatial4j" name="spatial4j" rev="0.3" transitive="false"/>
<dependency org="javax.servlet" name="javax.servlet-api" rev="3.0.1" transitive="false"/>
- <dependency org="com.yammer.metrics" name="metrics-core" rev="2.1.2" transitive="false"/>
<exclude org="*" ext="*" matcher="regexp" type="${ivy.exclude.types}"/>
</dependencies>
</ivy-module>
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/SolrLogFormatter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/SolrLogFormatter.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/SolrLogFormatter.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/SolrLogFormatter.java Fri Jan 18 18:30:54 2013
@@ -196,12 +196,8 @@ sb.append("(group_name=").append(tg.getN
sb.append(" url="+info.url + " node="+zkController.getNodeName());
}
- if(info.coreProps == null) {
- info.coreProps = getCoreProps(zkController, core);
- }
-
- Map<String, Object> coreProps = getCoreProps(zkController, core);
- if(!coreProps.equals(info.coreProps)) {
+ Map<String, Object> coreProps = getReplicaProps(zkController, core);
+ if (info.coreProps == null || !coreProps.equals(info.coreProps)) {
info.coreProps = coreProps;
final String corePropsString = "coll:" + core.getCoreDescriptor().getCloudDescriptor().getCollectionName() + " core:" + core.getName() + " props:" + coreProps;
sb.append(" " + info.shortId + "_STATE=" + corePropsString);
@@ -261,11 +257,11 @@ sb.append("(group_name=").append(tg.getN
return sb.toString();
}
- private Map<String,Object> getCoreProps(ZkController zkController, SolrCore core) {
+ private Map<String,Object> getReplicaProps(ZkController zkController, SolrCore core) {
final String collection = core.getCoreDescriptor().getCloudDescriptor().getCollectionName();
- Replica props = zkController.getClusterState().getShardProps(collection, ZkStateReader.getCoreNodeName(zkController.getNodeName(), core.getName()));
- if(props!=null) {
- return props.getProperties();
+ Replica replica = zkController.getClusterState().getReplica(collection, ZkStateReader.getCoreNodeName(zkController.getNodeName(), core.getName()));
+ if(replica!=null) {
+ return replica.getProperties();
}
return Collections.EMPTY_MAP;
}
@@ -430,6 +426,7 @@ sb.append("(group_name=").append(tg.getN
public MyThreadGroup(String name) {
super(name);
}
+ @Override
public String getTag() { return "HELLO"; }
}
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/analysis/LegacyHTMLStripCharFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/analysis/LegacyHTMLStripCharFilterFactory.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/analysis/LegacyHTMLStripCharFilterFactory.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/analysis/LegacyHTMLStripCharFilterFactory.java Fri Jan 18 18:30:54 2013
@@ -54,6 +54,7 @@ import org.apache.lucene.analysis.util.C
@Deprecated
public class LegacyHTMLStripCharFilterFactory extends CharFilterFactory {
+ @Override
public LegacyHTMLStripCharFilter create(Reader input) {
return new LegacyHTMLStripCharFilter(input);
}
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/analysis/ReversedWildcardFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/analysis/ReversedWildcardFilterFactory.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/analysis/ReversedWildcardFilterFactory.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/analysis/ReversedWildcardFilterFactory.java Fri Jan 18 18:30:54 2013
@@ -82,6 +82,7 @@ public class ReversedWildcardFilterFacto
}
+ @Override
public TokenStream create(TokenStream input) {
return new ReversedWildcardFilter(input, withOriginal, markerChar);
}
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/analysis/TrieTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/analysis/TrieTokenizerFactory.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/analysis/TrieTokenizerFactory.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/analysis/TrieTokenizerFactory.java Fri Jan 18 18:30:54 2013
@@ -48,40 +48,56 @@ public class TrieTokenizerFactory extend
this.precisionStep = precisionStep;
}
+ @Override
public TrieTokenizer create(Reader input) {
- return new TrieTokenizer(input, type, precisionStep, TrieTokenizer.getNumericTokenStream(precisionStep));
+ return new TrieTokenizer(input, type, TrieTokenizer.getNumericTokenStream(precisionStep));
}
}
final class TrieTokenizer extends Tokenizer {
protected static final DateField dateField = new DateField();
- protected final int precisionStep;
protected final TrieTypes type;
protected final NumericTokenStream ts;
protected final OffsetAttribute ofsAtt = addAttribute(OffsetAttribute.class);
protected int startOfs, endOfs;
+ protected boolean hasValue;
+ protected final char[] buf = new char[32];
static NumericTokenStream getNumericTokenStream(int precisionStep) {
return new NumericTokenStream(precisionStep);
}
- public TrieTokenizer(Reader input, TrieTypes type, int precisionStep, NumericTokenStream ts) {
+ public TrieTokenizer(Reader input, TrieTypes type, NumericTokenStream ts) {
// must share the attribute source with the NumericTokenStream we delegate to
super(ts, input);
this.type = type;
- this.precisionStep = precisionStep;
this.ts = ts;
}
@Override
public void reset() {
try {
- char[] buf = new char[32];
- int len = input.read(buf);
+ int upto = 0;
+ while (upto < buf.length) {
+ final int length = input.read(buf, upto, buf.length - upto);
+ if (length == -1) break;
+ upto += length;
+ }
+ // skip remaining data if buffer was too short:
+ if (upto == buf.length) {
+ input.skip(Long.MAX_VALUE);
+ }
+
this.startOfs = correctOffset(0);
- this.endOfs = correctOffset(len);
- String v = new String(buf, 0, len);
+ this.endOfs = correctOffset(upto);
+
+ if (upto == 0) {
+ hasValue = false;
+ return;
+ }
+
+ final String v = new String(buf, 0, upto);
try {
switch (type) {
case INTEGER:
@@ -106,21 +122,24 @@ final class TrieTokenizer extends Tokeni
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
"Invalid Number: " + v);
}
+ hasValue = true;
+ ts.reset();
} catch (IOException e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unable to create TrieIndexTokenizer", e);
}
- ts.reset();
}
@Override
public void close() throws IOException {
super.close();
- ts.close();
+ if (hasValue) {
+ ts.close();
+ }
}
@Override
public boolean incrementToken() {
- if (ts.incrementToken()) {
+ if (hasValue && ts.incrementToken()) {
ofsAtt.setOffset(startOfs, endOfs);
return true;
}
@@ -129,7 +148,9 @@ final class TrieTokenizer extends Tokeni
@Override
public void end() throws IOException {
- ts.end();
+ if (hasValue) {
+ ts.end();
+ }
ofsAtt.setOffset(endOfs, endOfs);
}
}
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java Fri Jan 18 18:30:54 2013
@@ -19,13 +19,17 @@ package org.apache.solr.client.solrj.emb
import java.io.IOException;
import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
+import java.util.LinkedList;
import java.util.Random;
+import java.util.concurrent.atomic.AtomicLong;
import javax.servlet.DispatcherType;
import javax.servlet.Filter;
+import javax.servlet.FilterChain;
+import javax.servlet.FilterConfig;
+import javax.servlet.ServletException;
+import javax.servlet.ServletRequest;
+import javax.servlet.ServletResponse;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
@@ -53,6 +57,7 @@ public class JettySolrRunner {
Server server;
FilterHolder dispatchFilter;
+ FilterHolder debugFilter;
String context;
@@ -73,6 +78,53 @@ public class JettySolrRunner {
private boolean stopAtShutdown;
+ public static class DebugFilter implements Filter {
+ public int requestsToKeep = 10;
+ private AtomicLong nRequests = new AtomicLong();
+
+ public long getTotalRequests() {
+ return nRequests.get();
+
+ }
+
+ // TODO: keep track of certain number of last requests
+ private LinkedList<HttpServletRequest> requests = new LinkedList<HttpServletRequest>();
+
+
+ @Override
+ public void init(FilterConfig filterConfig) throws ServletException {
+ }
+
+ @Override
+ public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain filterChain) throws IOException, ServletException {
+ nRequests.incrementAndGet();
+
+ /***
+ HttpServletRequest req = (HttpServletRequest)servletRequest;
+ HttpServletResponse resp = (HttpServletResponse)servletResponse;
+
+ String path = req.getServletPath();
+ if( req.getPathInfo() != null ) {
+ // this lets you handle /update/commit when /update is a servlet
+ path += req.getPathInfo();
+ }
+ System.out.println("###################### FILTER request " + servletRequest);
+ System.out.println("\t\tgetServletPath="+req.getServletPath());
+ System.out.println("\t\tgetPathInfo="+req.getPathInfo());
+ ***/
+
+ filterChain.doFilter(servletRequest, servletResponse);
+ }
+
+ @Override
+ public void destroy() {
+ }
+ }
+
+
+
+
+
public JettySolrRunner(String solrHome, String context, int port) {
this.init(solrHome, context, port, true);
}
@@ -102,18 +154,29 @@ public class JettySolrRunner {
}
System.setProperty("solr.solr.home", solrHome);
if (System.getProperty("jetty.testMode") != null) {
- SelectChannelConnector connector = new SelectChannelConnector();
+ final String connectorName = System.getProperty("tests.jettyConnector", "SelectChannel");
+ final Connector connector;
+ final QueuedThreadPool threadPool;
+ if ("SelectChannel".equals(connectorName)) {
+ final SelectChannelConnector c = new SelectChannelConnector();
+ c.setReuseAddress(true);
+ c.setLowResourcesMaxIdleTime(1500);
+ connector = c;
+ threadPool = (QueuedThreadPool) c.getThreadPool();
+ } else if ("Socket".equals(connectorName)) {
+ final SocketConnector c = new SocketConnector();
+ c.setReuseAddress(true);
+ connector = c;
+ threadPool = (QueuedThreadPool) c.getThreadPool();
+ } else {
+ throw new IllegalArgumentException("Illegal value for system property 'tests.jettyConnector': " + connectorName);
+ }
connector.setPort(port);
- connector.setReuseAddress(true);
- connector.setLowResourcesMaxIdleTime(1500);
- QueuedThreadPool threadPool = (QueuedThreadPool) connector
- .getThreadPool();
+ connector.setHost("127.0.0.1");
if (threadPool != null) {
threadPool.setMaxThreads(10000);
threadPool.setMaxIdleTimeMs(5000);
- if (!stopAtShutdown) {
- threadPool.setMaxStopTimeMs(100);
- }
+ threadPool.setMaxStopTimeMs(30000);
}
server.setConnectors(new Connector[] {connector});
@@ -148,12 +211,15 @@ public class JettySolrRunner {
root.setHandler(new GzipHandler());
server.addLifeCycleListener(new LifeCycle.Listener() {
+ @Override
public void lifeCycleStopping(LifeCycle arg0) {
System.clearProperty("hostPort");
}
+ @Override
public void lifeCycleStopped(LifeCycle arg0) {}
+ @Override
public void lifeCycleStarting(LifeCycle arg0) {
synchronized (JettySolrRunner.this) {
waitOnSolr = true;
@@ -161,6 +227,7 @@ public class JettySolrRunner {
}
}
+ @Override
public void lifeCycleStarted(LifeCycle arg0) {
lastPort = getFirstConnectorPort();
System.setProperty("hostPort", Integer.toString(lastPort));
@@ -170,12 +237,14 @@ public class JettySolrRunner {
schemaFilename);
// SolrDispatchFilter filter = new SolrDispatchFilter();
// FilterHolder fh = new FilterHolder(filter);
+ debugFilter = root.addFilter(DebugFilter.class, "*", EnumSet.of(DispatcherType.REQUEST) );
dispatchFilter = root.addFilter(SolrDispatchFilter.class, "*", EnumSet.of(DispatcherType.REQUEST) );
if (solrConfigFilename != null) System.clearProperty("solrconfig");
if (schemaFilename != null) System.clearProperty("schema");
System.clearProperty("solr.solr.home");
}
+ @Override
public void lifeCycleFailure(LifeCycle arg0, Throwable arg1) {
System.clearProperty("hostPort");
}
@@ -239,24 +308,11 @@ public class JettySolrRunner {
}
public void stop() throws Exception {
- // we try and do a bunch of extra stop stuff because
- // jetty doesn't like to stop if it started
- // and ended up in a failure state (like when it cannot get the port)
- if (server.getState().equals(Server.FAILED)) {
- Connector[] connectors = server.getConnectors();
- for (Connector connector : connectors) {
- connector.stop();
- }
- }
+
Filter filter = dispatchFilter.getFilter();
- ThreadPool threadPool = server.getThreadPool();
- server.getServer().stop();
+
server.stop();
- if (threadPool instanceof QueuedThreadPool) {
- ((QueuedThreadPool) threadPool).setMaxStopTimeMs(30000);
- ((QueuedThreadPool) threadPool).stop();
- ((QueuedThreadPool) threadPool).join();
- }
+
//server.destroy();
if (server.getState().equals(Server.FAILED)) {
filter.destroy();
@@ -290,6 +346,10 @@ public class JettySolrRunner {
return lastPort;
}
+ public DebugFilter getDebugFilter() {
+ return (DebugFilter)debugFilter.getFilter();
+ }
+
// --------------------------------------------------------------
// --------------------------------------------------------------
@@ -338,17 +398,21 @@ class NoLog implements Logger {
this.name = name == null ? "" : name;
}
+ @Override
public boolean isDebugEnabled() {
return debug;
}
+ @Override
public void setDebugEnabled(boolean enabled) {
debug = enabled;
}
+ @Override
public void debug(String msg, Throwable th) {
}
+ @Override
public Logger getLogger(String name) {
if ((name == null && this.name == null)
|| (name != null && name.equals(this.name)))
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/cloud/AssignShard.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/cloud/AssignShard.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/cloud/AssignShard.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/cloud/AssignShard.java Fri Jan 18 18:30:54 2013
@@ -39,7 +39,10 @@ public class AssignShard {
numShards = 1;
}
String returnShardId = null;
- Map<String, Slice> sliceMap = state.getSlices(collection);
+ Map<String, Slice> sliceMap = state.getSlicesMap(collection);
+
+
+ // TODO: now that we create shards ahead of time, is this code needed? Esp since hash ranges aren't assigned when creating via this method?
if (sliceMap == null) {
return "shard1";
@@ -51,6 +54,8 @@ public class AssignShard {
return "shard" + (shardIdNames.size() + 1);
}
+ // TODO: don't need to sort to find shard with fewest replicas!
+
// else figure out which shard needs more replicas
final Map<String, Integer> map = new HashMap<String, Integer>();
for (String shardId : shardIdNames) {
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/cloud/DistributedQueue.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/cloud/DistributedQueue.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/cloud/DistributedQueue.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/cloud/DistributedQueue.java Fri Jan 18 18:30:54 2013
@@ -168,6 +168,7 @@ public class DistributedQueue {
public LatchChildWatcher() {}
+ @Override
public void process(WatchedEvent event) {
LOG.info("Watcher fired on path: " + event.getPath() + " state: "
+ event.getState() + " type " + event.getType());
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java Fri Jan 18 18:30:54 2013
@@ -163,7 +163,7 @@ final class ShardLeaderElectionContext e
}
// should I be leader?
- if (weAreReplacement && !shouldIBeLeader(leaderProps, core)) {
+ if (weAreReplacement && !shouldIBeLeader(leaderProps, core, weAreReplacement)) {
rejoinLeaderElection(leaderSeqPath, core);
return;
}
@@ -263,7 +263,7 @@ final class ShardLeaderElectionContext e
ZkNodeProps leaderProps, String collection, String shardId) {
ClusterState clusterState = zkController.getZkStateReader()
.getClusterState();
- Map<String,Slice> slices = clusterState.getSlices(collection);
+ Map<String,Slice> slices = clusterState.getSlicesMap(collection);
Slice slice = slices.get(shardId);
Map<String,Replica> replicasMap = slice.getReplicasMap();
for (Map.Entry<String,Replica> shard : replicasMap.entrySet()) {
@@ -293,7 +293,7 @@ final class ShardLeaderElectionContext e
final String shardsElectZkPath = electionPath + LeaderElector.ELECTION_NODE;
Slice slices = zkController.getClusterState().getSlice(collection, shardId);
-
+ int cnt = 0;
while (true && !isClosed) {
// wait for everyone to be up
if (slices != null) {
@@ -310,9 +310,11 @@ final class ShardLeaderElectionContext e
log.info("Enough replicas found to continue.");
return;
} else {
- log.info("Waiting until we see more replicas up: total="
+ if (cnt % 40 == 0) {
+ log.info("Waiting until we see more replicas up: total="
+ slices.getReplicasMap().size() + " found=" + found
+ " timeoutin=" + (timeoutAt - System.currentTimeMillis()));
+ }
}
if (System.currentTimeMillis() > timeoutAt) {
@@ -323,6 +325,8 @@ final class ShardLeaderElectionContext e
Thread.sleep(500);
slices = zkController.getClusterState().getSlice(collection, shardId);
+ // System.out.println("###### waitForReplicasToComeUp : slices=" + slices + " all=" + zkController.getClusterState().getCollectionStates() );
+ cnt++;
}
}
@@ -347,7 +351,7 @@ final class ShardLeaderElectionContext e
leaderElector.joinElection(this, true);
}
- private boolean shouldIBeLeader(ZkNodeProps leaderProps, SolrCore core) {
+ private boolean shouldIBeLeader(ZkNodeProps leaderProps, SolrCore core, boolean weAreReplacement) {
log.info("Checking if I should try and be the leader.");
if (isClosed) {
@@ -355,6 +359,12 @@ final class ShardLeaderElectionContext e
return false;
}
+ if (!weAreReplacement) {
+ // we are the first node starting in the shard - there is a configurable wait
+ // to make sure others participate in sync and leader election, we can be leader
+ return true;
+ }
+
if (core.getCoreDescriptor().getCloudDescriptor().getLastPublished()
.equals(ZkStateReader.ACTIVE)) {
log.info("My last published State was Active, it's okay to be the leader.");
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java Fri Jan 18 18:30:54 2013
@@ -62,10 +62,11 @@ public class LeaderElector {
protected SolrZkClient zkClient;
- private ZkCmdExecutor zkCmdExecutor = new ZkCmdExecutor();
+ private ZkCmdExecutor zkCmdExecutor;
public LeaderElector(SolrZkClient zkClient) {
this.zkClient = zkClient;
+ zkCmdExecutor = new ZkCmdExecutor((int) (zkClient.getZkClientTimeout()/1000.0 + 3000));
}
/**
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/cloud/Overseer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/cloud/Overseer.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/cloud/Overseer.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/java/org/apache/solr/cloud/Overseer.java Fri Jan 18 18:30:54 2013
@@ -17,22 +17,24 @@ package org.apache.solr.cloud;
* the License.
*/
+import java.util.Collection;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import org.apache.solr.common.SolrException;
-import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.ClosableThread;
-import org.apache.solr.common.cloud.HashPartitioner;
+import org.apache.solr.common.cloud.ClusterState;
+import org.apache.solr.common.cloud.DocCollection;
+import org.apache.solr.common.cloud.DocRouter;
+import org.apache.solr.common.cloud.ImplicitDocRouter;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkCoreNodeProps;
import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.cloud.ZkStateReader;
-import org.apache.solr.common.cloud.ZooKeeperException;
import org.apache.solr.handler.component.ShardHandler;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
@@ -45,7 +47,7 @@ import org.slf4j.LoggerFactory;
public class Overseer {
public static final String QUEUE_OPERATION = "operation";
- private static final int STATE_UPDATE_DELAY = 500; // delay between cloud state updates
+ private static final int STATE_UPDATE_DELAY = 1500; // delay between cloud state updates
private static Logger log = LoggerFactory.getLogger(Overseer.class);
@@ -73,44 +75,48 @@ public class Overseer {
@Override
public void run() {
- if(!this.isClosed && amILeader()) {
+ if (!this.isClosed && amILeader()) {
// see if there's something left from the previous Overseer and re
// process all events that were not persisted into cloud state
- synchronized (reader.getUpdateLock()) { //XXX this only protects against edits inside single node
- try {
- byte[] head = workQueue.peek();
+ synchronized (reader.getUpdateLock()) { // XXX this only protects
+ // against edits inside single
+ // node
+ try {
+ byte[] head = workQueue.peek();
+
+ if (head != null) {
+ reader.updateClusterState(true);
+ ClusterState clusterState = reader.getClusterState();
+ log.info("Replaying operations from work queue.");
- if (head != null) {
- reader.updateClusterState(true);
- ClusterState clusterState = reader.getClusterState();
- log.info("Replaying operations from work queue.");
+ while (head != null && amILeader()) {
+ final ZkNodeProps message = ZkNodeProps.load(head);
+ final String operation = message.getStr(QUEUE_OPERATION);
+ clusterState = processMessage(clusterState, message, operation);
+ zkClient.setData(ZkStateReader.CLUSTER_STATE,
+ ZkStateReader.toJSON(clusterState), true);
- while (head != null && amILeader()) {
- final ZkNodeProps message = ZkNodeProps.load(head);
- final String operation = message
- .getStr(QUEUE_OPERATION);
- clusterState = processMessage(clusterState, message, operation);
- zkClient.setData(ZkStateReader.CLUSTER_STATE,
- ZkStateReader.toJSON(clusterState), true);
- workQueue.remove();
- head = workQueue.peek();
- }
- }
- } catch (KeeperException e) {
- if (e.code() == KeeperException.Code.SESSIONEXPIRED
- || e.code() == KeeperException.Code.CONNECTIONLOSS) {
- log.warn("Solr cannot talk to ZK");
- return;
+ workQueue.poll();
+
+ head = workQueue.peek();
}
- SolrException.log(log, "", e);
- throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
- "", e);
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
+ }
+ } catch (KeeperException e) {
+ if (e.code() == KeeperException.Code.SESSIONEXPIRED) {
+ log.warn("Solr cannot talk to ZK, exiting Overseer work queue loop", e);
return;
}
+ log.error("Exception in Overseer work queue loop", e);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ return;
+
+ } catch (Exception e) {
+ log.error("Exception in Overseer work queue loop", e);
}
}
+
+ }
log.info("Starting to work on the main queue");
while (!this.isClosed && amILeader()) {
@@ -128,7 +134,8 @@ public class Overseer {
clusterState = processMessage(clusterState, message, operation);
workQueue.offer(head);
- stateUpdateQueue.remove();
+
+ stateUpdateQueue.poll();
head = stateUpdateQueue.peek();
}
zkClient.setData(ZkStateReader.CLUSTER_STATE,
@@ -138,17 +145,17 @@ public class Overseer {
while (workQueue.poll() != null);
} catch (KeeperException e) {
- if (e.code() == KeeperException.Code.SESSIONEXPIRED
- || e.code() == KeeperException.Code.CONNECTIONLOSS) {
- log.warn("Overseer cannot talk to ZK");
+ if (e.code() == KeeperException.Code.SESSIONEXPIRED) {
+ log.warn("Solr cannot talk to ZK, exiting Overseer main queue loop", e);
return;
}
- SolrException.log(log, "", e);
- throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
- "", e);
+ log.error("Exception in Overseer main queue loop", e);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
return;
+
+ } catch (Exception e) {
+ log.error("Exception in Overseer main queue loop", e);
}
}
@@ -229,7 +236,7 @@ public class Overseer {
//request new shardId
if (collectionExists) {
// use existing numShards
- numShards = state.getCollectionStates().get(collection).size();
+ numShards = state.getCollectionStates().get(collection).getSlices().size();
log.info("Collection already exists with " + ZkStateReader.NUM_SHARDS_PROP + "=" + numShards);
}
sliceName = AssignShard.assignShard(collection, state, numShards);
@@ -274,16 +281,23 @@ public class Overseer {
return newClusterState;
}
+ private Map<String,Object> defaultCollectionProps() {
+ HashMap<String,Object> props = new HashMap<String, Object>(2);
+ props.put(DocCollection.DOC_ROUTER, DocRouter.DEFAULT_NAME);
+ return props;
+ }
+
private ClusterState createCollection(ClusterState state, String collectionName, int numShards) {
log.info("Create collection {} with numShards {}", collectionName, numShards);
-
- HashPartitioner hp = new HashPartitioner();
- List<HashPartitioner.Range> ranges = hp.partitionRange(numShards, hp.fullRange());
+
+ DocRouter router = DocRouter.DEFAULT;
+ List<DocRouter.Range> ranges = router.partitionRange(numShards, router.fullRange());
+
+ Map<String, DocCollection> newCollections = new LinkedHashMap<String,DocCollection>();
- Map<String, Map<String, Slice>> newStates = new LinkedHashMap<String,Map<String, Slice>>();
Map<String, Slice> newSlices = new LinkedHashMap<String,Slice>();
- newStates.putAll(state.getCollectionStates());
+ newCollections.putAll(state.getCollectionStates());
for (int i = 0; i < numShards; i++) {
final String sliceName = "shard" + (i+1);
@@ -292,20 +306,26 @@ public class Overseer {
newSlices.put(sliceName, new Slice(sliceName, null, sliceProps));
}
- newStates.put(collectionName, newSlices);
- ClusterState newClusterState = new ClusterState(state.getLiveNodes(), newStates);
+
+ // TODO: fill in with collection properties read from the /collections/<collectionName> node
+ Map<String,Object> collectionProps = defaultCollectionProps();
+
+ DocCollection newCollection = new DocCollection(collectionName, newSlices, collectionProps, router);
+
+ newCollections.put(collectionName, newCollection);
+ ClusterState newClusterState = new ClusterState(state.getLiveNodes(), newCollections);
return newClusterState;
}
-
+
/*
* Return an already assigned id or null if not assigned
*/
private String getAssignedId(final ClusterState state, final String nodeName,
final ZkNodeProps coreState) {
final String key = coreState.getStr(ZkStateReader.NODE_NAME_PROP) + "_" + coreState.getStr(ZkStateReader.CORE_NAME_PROP);
- Map<String, Slice> slices = state.getSlices(coreState.getStr(ZkStateReader.COLLECTION_PROP));
+ Collection<Slice> slices = state.getSlices(coreState.getStr(ZkStateReader.COLLECTION_PROP));
if (slices != null) {
- for (Slice slice : slices.values()) {
+ for (Slice slice : slices) {
if (slice.getReplicasMap().get(key) != null) {
return slice.getName();
}
@@ -314,40 +334,49 @@ public class Overseer {
return null;
}
- private ClusterState updateSlice(ClusterState state, String collection, Slice slice) {
+ private ClusterState updateSlice(ClusterState state, String collectionName, Slice slice) {
// System.out.println("###!!!### OLD CLUSTERSTATE: " + JSONUtil.toJSON(state.getCollectionStates()));
// System.out.println("Updating slice:" + slice);
- Map<String, Map<String, Slice>> newCollections = new LinkedHashMap<String,Map<String,Slice>>(state.getCollectionStates()); // make a shallow copy
- Map<String, Slice> slices = newCollections.get(collection);
- if (slices == null) {
+ Map<String, DocCollection> newCollections = new LinkedHashMap<String,DocCollection>(state.getCollectionStates()); // make a shallow copy
+ DocCollection coll = newCollections.get(collectionName);
+ Map<String,Slice> slices;
+ Map<String,Object> props;
+ DocRouter router;
+
+ if (coll == null) {
+ // when updateSlice is called on a collection that doesn't exist, it's currently when a core is publishing itself
+ // without explicitly creating a collection. In this current case, we assume custom sharding with an "implicit" router.
slices = new HashMap<String, Slice>(1);
+ props = new HashMap<String,Object>(1);
+ props.put(DocCollection.DOC_ROUTER, ImplicitDocRouter.NAME);
+ router = new ImplicitDocRouter();
} else {
- slices = new LinkedHashMap<String, Slice>(slices); // make a shallow copy
- }
- slices.put(slice.getName(), slice);
- newCollections.put(collection, slices);
+ props = coll.getProperties();
+ router = coll.getRouter();
+ slices = new LinkedHashMap<String, Slice>(coll.getSlicesMap()); // make a shallow copy
+ }
+ slices.put(slice.getName(), slice);
+ DocCollection newCollection = new DocCollection(collectionName, slices, props, router);
+ newCollections.put(collectionName, newCollection);
// System.out.println("###!!!### NEW CLUSTERSTATE: " + JSONUtil.toJSON(newCollections));
return new ClusterState(state.getLiveNodes(), newCollections);
}
- private ClusterState setShardLeader(ClusterState state, String collection, String sliceName, String leaderUrl) {
+ private ClusterState setShardLeader(ClusterState state, String collectionName, String sliceName, String leaderUrl) {
- final Map<String, Map<String, Slice>> newStates = new LinkedHashMap<String,Map<String,Slice>>(state.getCollectionStates());
-
- Map<String, Slice> slices = newStates.get(collection);
-
- if(slices==null) {
- log.error("Could not mark shard leader for non existing collection:" + collection);
+ final Map<String, DocCollection> newCollections = new LinkedHashMap<String,DocCollection>(state.getCollectionStates());
+ DocCollection coll = newCollections.get(collectionName);
+ if(coll == null) {
+ log.error("Could not mark shard leader for non existing collection:" + collectionName);
return state;
}
+ Map<String, Slice> slices = coll.getSlicesMap();
// make a shallow copy and add it to the new collection
slices = new LinkedHashMap<String,Slice>(slices);
- newStates.put(collection, slices);
-
Slice slice = slices.get(sliceName);
if (slice == null) {
@@ -383,7 +412,11 @@ public class Overseer {
Slice newSlice = new Slice(slice.getName(), newReplicas, slice.getProperties());
slices.put(newSlice.getName(), newSlice);
}
- return new ClusterState(state.getLiveNodes(), newStates);
+
+
+ DocCollection newCollection = new DocCollection(coll.getName(), slices, coll.getProperties(), coll.getRouter());
+ newCollections.put(collectionName, newCollection);
+ return new ClusterState(state.getLiveNodes(), newCollections);
}
/*
@@ -394,51 +427,57 @@ public class Overseer {
final String coreNodeName = message.getStr(ZkStateReader.NODE_NAME_PROP) + "_" + message.getStr(ZkStateReader.CORE_NAME_PROP);
final String collection = message.getStr(ZkStateReader.COLLECTION_PROP);
- final LinkedHashMap<String, Map<String, Slice>> newStates = new LinkedHashMap<String,Map<String,Slice>>();
- for(String collectionName: clusterState.getCollections()) {
- if(collection.equals(collectionName)) {
- Map<String, Slice> slices = clusterState.getSlices(collection);
- LinkedHashMap<String, Slice> newSlices = new LinkedHashMap<String, Slice>();
- for(Slice slice: slices.values()) {
- if(slice.getReplicasMap().containsKey(coreNodeName)) {
- Map<String, Replica> newReplicas = slice.getReplicasCopy();
- newReplicas.remove(coreNodeName);
- if (newReplicas.size() != 0) {
- Slice newSlice = new Slice(slice.getName(), newReplicas,
- slice.getProperties());
- newSlices.put(slice.getName(), newSlice);
- }
- } else {
- newSlices.put(slice.getName(), slice);
- }
- }
- int cnt = 0;
- for (Slice slice : newSlices.values()) {
- cnt+=slice.getReplicasMap().size();
- }
- // TODO: if no nodes are left after this unload
- // remove from zk - do we have a race where Overseer
- // see's registered nodes and publishes though?
- if (cnt > 0) {
- newStates.put(collectionName, newSlices);
+ final Map<String, DocCollection> newCollections = new LinkedHashMap<String,DocCollection>(clusterState.getCollectionStates()); // shallow copy
+ DocCollection coll = newCollections.get(collection);
+ if (coll == null) {
+ // TODO: log/error that we didn't find it?
+ return clusterState;
+ }
+
+ Map<String, Slice> newSlices = new LinkedHashMap<String, Slice>();
+ for (Slice slice : coll.getSlices()) {
+ Replica replica = slice.getReplica(coreNodeName);
+ if (replica != null) {
+ Map<String, Replica> newReplicas = slice.getReplicasCopy();
+ newReplicas.remove(coreNodeName);
+ // TODO TODO TODO!!! if there are no replicas left for the slice, and the slice has no hash range, remove it
+ // if (newReplicas.size() == 0 && slice.getRange() == null) {
+ // if there are no replicas left for the slice remove it
+ if (newReplicas.size() == 0) {
+ slice = null;
} else {
- // TODO: it might be better logically to have this in ZkController
- // but for tests (it's easier) it seems better for the moment to leave CoreContainer and/or
- // ZkController out of the Overseer.
- try {
- zkClient.clean("/collections/" + collectionName);
- } catch (InterruptedException e) {
- SolrException.log(log, "Cleaning up collection in zk was interrupted:" + collectionName, e);
- Thread.currentThread().interrupt();
- } catch (KeeperException e) {
- SolrException.log(log, "Problem cleaning up collection in zk:" + collectionName, e);
- }
+ slice = new Slice(slice.getName(), newReplicas, slice.getProperties());
}
- } else {
- newStates.put(collectionName, clusterState.getSlices(collectionName));
}
+
+ if (slice != null) {
+ newSlices.put(slice.getName(), slice);
+ }
+ }
+
+ // if there are no slices left in the collection, remove it?
+ if (newSlices.size() == 0) {
+ newCollections.remove(coll.getName());
+
+ // TODO: it might be better logically to have this in ZkController
+ // but for tests (it's easier) it seems better for the moment to leave CoreContainer and/or
+ // ZkController out of the Overseer.
+ try {
+ zkClient.clean("/collections/" + collection);
+ } catch (InterruptedException e) {
+ SolrException.log(log, "Cleaning up collection in zk was interrupted:" + collection, e);
+ Thread.currentThread().interrupt();
+ } catch (KeeperException e) {
+ SolrException.log(log, "Problem cleaning up collection in zk:" + collection, e);
+ }
+
+
+ } else {
+ DocCollection newCollection = new DocCollection(coll.getName(), newSlices, coll.getProperties(), coll.getRouter());
+ newCollections.put(newCollection.getName(), newCollection);
}
- ClusterState newState = new ClusterState(clusterState.getLiveNodes(), newStates);
+
+ ClusterState newState = new ClusterState(clusterState.getLiveNodes(), newCollections);
return newState;
}