You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2014/12/07 12:37:35 UTC
svn commit: r1643662 [5/6] - in /lucene/dev/branches/lucene6005: ./ lucene/
lucene/codecs/ lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/
lucene/core/ lucene/core/src/java/org/apache/lucene/codecs/
lucene/core/src/java/org/apache/lucene/co...
Modified: lucene/dev/branches/lucene6005/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSolrEntityProcessorUnit.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSolrEntityProcessorUnit.java?rev=1643662&r1=1643661&r2=1643662&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSolrEntityProcessorUnit.java (original)
+++ lucene/dev/branches/lucene6005/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSolrEntityProcessorUnit.java Sun Dec 7 11:37:32 2014
@@ -31,20 +31,33 @@ public class TestSolrEntityProcessorUnit
public void testQuery() {
List<Doc> docs = generateUniqueDocs(2);
- MockSolrEntityProcessor processor = new MockSolrEntityProcessor(docs);
+ MockSolrEntityProcessor processor = createAndInit(docs);
assertExpectedDocs(docs, processor);
assertEquals(1, processor.getQueryCount());
}
+ private MockSolrEntityProcessor createAndInit(List<Doc> docs) {
+ return createAndInit(docs, SolrEntityProcessor.ROWS_DEFAULT);
+ }
+
public void testNumDocsGreaterThanRows() {
List<Doc> docs = generateUniqueDocs(44);
- MockSolrEntityProcessor processor = new MockSolrEntityProcessor(docs, 10);
+ int rowsNum = 10;
+ MockSolrEntityProcessor processor = createAndInit(docs, rowsNum);
assertExpectedDocs(docs, processor);
assertEquals(5, processor.getQueryCount());
}
+ private MockSolrEntityProcessor createAndInit(List<Doc> docs, int rowsNum) {
+ MockSolrEntityProcessor processor = new MockSolrEntityProcessor(docs, rowsNum);
+ HashMap<String,String> entityAttrs = new HashMap<String,String>(){{put(SolrEntityProcessor.SOLR_SERVER,"http://route:66/no");}};
+ processor.init(getContext(null, null, null, null, Collections.emptyList(),
+ entityAttrs));
+ return processor;
+ }
+
public void testMultiValuedFields() {
List<Doc> docs = new ArrayList<>();
List<FldType> types = new ArrayList<>();
@@ -53,7 +66,7 @@ public class TestSolrEntityProcessorUnit
Doc testDoc = createDoc(types);
docs.add(testDoc);
- MockSolrEntityProcessor processor = new MockSolrEntityProcessor(docs);
+ MockSolrEntityProcessor processor = createAndInit(docs);
Map<String, Object> next = processor.nextRow();
assertNotNull(next);
Modified: lucene/dev/branches/lucene6005/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessor.java?rev=1643662&r1=1643661&r2=1643662&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessor.java (original)
+++ lucene/dev/branches/lucene6005/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessor.java Sun Dec 7 11:37:32 2014
@@ -45,6 +45,60 @@ public class TestSqlEntityProcessor exte
public void testCachedChildEntities() throws Exception {
withChildEntities(true, true);
}
+
+ @Test
+ public void testSportZipperChildEntities() throws Exception {
+ sportsZipper = true;
+ withChildEntities(true, true);
+ }
+
+ @Test
+ public void testCountryZipperChildEntities() throws Exception {
+ countryZipper = true;
+ withChildEntities(true, true);
+ }
+
+ @Test
+ public void testBothZipperChildEntities() throws Exception {
+ countryZipper = true;
+ sportsZipper = true;
+ withChildEntities(true, true);
+ }
+
+ @Test(expected=RuntimeException.class /* DIH exceptions are not propagated, here we capturing assertQ exceptions */)
+ public void testSportZipperChildEntitiesWrongOrder() throws Exception {
+ if(random().nextBoolean()){
+ wrongPeopleOrder = true;
+ }else{
+ wrongSportsOrder = true;
+ }
+ testSportZipperChildEntities();
+ }
+
+ @Test(expected=RuntimeException.class )
+ public void testCountryZipperChildEntitiesWrongOrder() throws Exception {
+ if(random().nextBoolean()){
+ wrongPeopleOrder = true;
+ }else{
+ wrongCountryOrder = true;
+ }
+ testCountryZipperChildEntities();
+ }
+
+ @Test(expected=RuntimeException.class)
+ public void testBothZipperChildEntitiesWrongOrder() throws Exception {
+ if(random().nextBoolean()){
+ wrongPeopleOrder = true;
+ }else{
+ if(random().nextBoolean()){
+ wrongSportsOrder = true;
+ }else{
+ wrongCountryOrder = true;
+ }
+ }
+ testBothZipperChildEntities();
+ }
+
@Test
@Ignore("broken see SOLR-3857")
public void testSimpleCacheChildEntities() throws Exception {
Modified: lucene/dev/branches/lucene6005/solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/solrconfig.xml?rev=1643662&r1=1643661&r2=1643662&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/solrconfig.xml (original)
+++ lucene/dev/branches/lucene6005/solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/solrconfig.xml Sun Dec 7 11:37:32 2014
@@ -194,6 +194,18 @@
<requestHandler name="/update/extract" class="org.apache.solr.handler.extraction.ExtractingRequestHandler"/>
+ <requestHandler name="/update/extract/lit-def" class="org.apache.solr.handler.extraction.ExtractingRequestHandler">
+ <lst name="defaults">
+ <str name="literal.foo_s">x</str>
+ </lst>
+ <lst name="appends">
+ <str name="literal.bar_s">y</str>
+ </lst>
+ <lst name="invariants">
+ <str name="literal.zot_s">z</str>
+ <str name="uprefix">ignored_</str>
+ </lst>
+ </requestHandler>
<highlighting>
<!-- Configure the standard fragmenter -->
Modified: lucene/dev/branches/lucene6005/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTest.java?rev=1643662&r1=1643661&r2=1643662&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTest.java (original)
+++ lucene/dev/branches/lucene6005/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTest.java Sun Dec 7 11:37:32 2014
@@ -288,6 +288,74 @@ public class ExtractingRequestHandlerTes
}
+ public void testLiteralDefaults() throws Exception {
+
+ // sanity check config
+ loadLocalFromHandler("/update/extract/lit-def",
+ "extraction/simple.html",
+ "literal.id", "lit-def-simple");
+ assertU(commit());
+ assertQ(req("q", "id:lit-def-simple")
+ , "//*[@numFound='1']"
+ , "count(//arr[@name='foo_s']/str)=1"
+ , "//arr[@name='foo_s']/str[.='x']"
+ , "count(//arr[@name='bar_s']/str)=1"
+ , "//arr[@name='bar_s']/str[.='y']"
+ , "count(//arr[@name='zot_s']/str)=1"
+ , "//arr[@name='zot_s']/str[.='z']"
+ );
+
+ // override the default foo_s
+ loadLocalFromHandler("/update/extract/lit-def",
+ "extraction/simple.html",
+ "literal.foo_s", "1111",
+ "literal.id", "lit-def-simple");
+ assertU(commit());
+ assertQ(req("q", "id:lit-def-simple")
+ , "//*[@numFound='1']"
+ , "count(//arr[@name='foo_s']/str)=1"
+ , "//arr[@name='foo_s']/str[.='1111']"
+ , "count(//arr[@name='bar_s']/str)=1"
+ , "//arr[@name='bar_s']/str[.='y']"
+ , "count(//arr[@name='zot_s']/str)=1"
+ , "//arr[@name='zot_s']/str[.='z']"
+ );
+
+ // pre-pend the bar_s
+ loadLocalFromHandler("/update/extract/lit-def",
+ "extraction/simple.html",
+ "literal.bar_s", "2222",
+ "literal.id", "lit-def-simple");
+ assertU(commit());
+ assertQ(req("q", "id:lit-def-simple")
+ , "//*[@numFound='1']"
+ , "count(//arr[@name='foo_s']/str)=1"
+ , "//arr[@name='foo_s']/str[.='x']"
+ , "count(//arr[@name='bar_s']/str)=2"
+ , "//arr[@name='bar_s']/str[.='2222']"
+ , "//arr[@name='bar_s']/str[.='y']"
+ , "count(//arr[@name='zot_s']/str)=1"
+ , "//arr[@name='zot_s']/str[.='z']"
+ );
+
+ // invariant zot_s can not be changed
+ loadLocalFromHandler("/update/extract/lit-def",
+ "extraction/simple.html",
+ "literal.zot_s", "3333",
+ "literal.id", "lit-def-simple");
+ assertU(commit());
+ assertQ(req("q", "id:lit-def-simple")
+ , "//*[@numFound='1']"
+ , "count(//arr[@name='foo_s']/str)=1"
+ , "//arr[@name='foo_s']/str[.='x']"
+ , "count(//arr[@name='bar_s']/str)=1"
+ , "//arr[@name='bar_s']/str[.='y']"
+ , "count(//arr[@name='zot_s']/str)=1"
+ , "//arr[@name='zot_s']/str[.='z']"
+ );
+
+ }
+
@Test
public void testPlainTextSpecifyingMimeType() throws Exception {
ExtractingRequestHandler handler = (ExtractingRequestHandler) h.getCore().getRequestHandler("/update/extract");
@@ -612,7 +680,9 @@ public class ExtractingRequestHandlerTes
assertQ(req("wdf_nocase:\"Test password protected word doc\""), "//*[@numFound='2']");
}
- SolrQueryResponse loadLocal(String filename, String... args) throws Exception {
+ SolrQueryResponse loadLocalFromHandler(String handler, String filename,
+ String... args) throws Exception {
+
LocalSolrQueryRequest req = (LocalSolrQueryRequest) req(args);
try {
// TODO: stop using locally defined streams once stream.file and
@@ -620,11 +690,15 @@ public class ExtractingRequestHandlerTes
List<ContentStream> cs = new ArrayList<>();
cs.add(new ContentStreamBase.FileStream(getFile(filename)));
req.setContentStreams(cs);
- return h.queryAndResponse("/update/extract", req);
+ return h.queryAndResponse(handler, req);
} finally {
req.close();
}
}
+ SolrQueryResponse loadLocal(String filename, String... args) throws Exception {
+ return loadLocalFromHandler("/update/extract", filename, args);
+ }
+
}
Modified: lucene/dev/branches/lucene6005/solr/contrib/langid/src/java/org/apache/solr/update/processor/LangDetectLanguageIdentifierUpdateProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/contrib/langid/src/java/org/apache/solr/update/processor/LangDetectLanguageIdentifierUpdateProcessor.java?rev=1643662&r1=1643661&r2=1643662&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/contrib/langid/src/java/org/apache/solr/update/processor/LangDetectLanguageIdentifierUpdateProcessor.java (original)
+++ lucene/dev/branches/lucene6005/solr/contrib/langid/src/java/org/apache/solr/update/processor/LangDetectLanguageIdentifierUpdateProcessor.java Sun Dec 7 11:37:32 2014
@@ -18,6 +18,7 @@ package org.apache.solr.update.processor
*/
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
import java.util.List;
@@ -28,6 +29,7 @@ import com.cybozu.labs.langdetect.Detect
import com.cybozu.labs.langdetect.DetectorFactory;
import com.cybozu.labs.langdetect.LangDetectException;
import com.cybozu.labs.langdetect.Language;
+import org.apache.solr.common.SolrInputDocument;
/**
* Identifies the language of a set of input fields using http://code.google.com/p/language-detection
@@ -43,15 +45,32 @@ public class LangDetectLanguageIdentifie
}
@Override
- protected List<DetectedLanguage> detectLanguage(String content) {
- if (content.trim().length() == 0) { // to be consistent with the tika impl?
- log.debug("No input text to detect language from, returning empty list");
- return Collections.emptyList();
- }
-
+ protected List<DetectedLanguage> detectLanguage(SolrInputDocument doc) {
try {
Detector detector = DetectorFactory.create();
- detector.append(content);
+ detector.setMaxTextLength(maxTotalChars);
+
+ for (String fieldName : inputFields) {
+ log.debug("Appending field " + fieldName);
+ if (doc.containsKey(fieldName)) {
+ Collection<Object> fieldValues = doc.getFieldValues(fieldName);
+ if (fieldValues != null) {
+ for (Object content : fieldValues) {
+ if (content instanceof String) {
+ String stringContent = (String) content;
+ if (stringContent.length() > maxFieldValueChars) {
+ detector.append(stringContent.substring(0, maxFieldValueChars));
+ } else {
+ detector.append(stringContent);
+ }
+ detector.append(" ");
+ } else {
+ log.warn("Field " + fieldName + " not a String value, not including in detection");
+ }
+ }
+ }
+ }
+ }
ArrayList<Language> langlist = detector.getProbabilities();
ArrayList<DetectedLanguage> solrLangList = new ArrayList<>();
for (Language l: langlist) {
Modified: lucene/dev/branches/lucene6005/solr/contrib/langid/src/java/org/apache/solr/update/processor/LangIdParams.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/contrib/langid/src/java/org/apache/solr/update/processor/LangIdParams.java?rev=1643662&r1=1643661&r2=1643662&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/contrib/langid/src/java/org/apache/solr/update/processor/LangIdParams.java (original)
+++ lucene/dev/branches/lucene6005/solr/contrib/langid/src/java/org/apache/solr/update/processor/LangIdParams.java Sun Dec 7 11:37:32 2014
@@ -41,12 +41,16 @@ public interface LangIdParams {
String MAP_LCMAP = LANGUAGE_ID + ".map.lcmap"; // Enables mapping multiple langs to same output field
String MAP_PATTERN = LANGUAGE_ID + ".map.pattern"; // RegEx pattern to match field name
String MAP_REPLACE = LANGUAGE_ID + ".map.replace"; // Replace pattern
+ String MAX_FIELD_VALUE_CHARS = LANGUAGE_ID + ".maxFieldValueChars"; // Maximum number of characters to use per field for language detection
+ String MAX_TOTAL_CHARS = LANGUAGE_ID + ".maxTotalChars"; // Maximum number of characters to use per all concatenated fields for language detection
String DOCID_FIELD_DEFAULT = "id";
String DOCID_LANGFIELD_DEFAULT = null;
String DOCID_LANGSFIELD_DEFAULT = null;
String MAP_PATTERN_DEFAULT = "(.*)";
String MAP_REPLACE_DEFAULT = "$1_{lang}";
+ int MAX_FIELD_VALUE_CHARS_DEFAULT = 10000;
+ int MAX_TOTAL_CHARS_DEFAULT = 20000;
// TODO: This default threshold accepts even "uncertain" detections.
// Increase &langid.threshold above 0.5 to return only certain detections
Modified: lucene/dev/branches/lucene6005/solr/contrib/langid/src/java/org/apache/solr/update/processor/LanguageIdentifierUpdateProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/contrib/langid/src/java/org/apache/solr/update/processor/LanguageIdentifierUpdateProcessor.java?rev=1643662&r1=1643661&r2=1643662&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/contrib/langid/src/java/org/apache/solr/update/processor/LanguageIdentifierUpdateProcessor.java (original)
+++ lucene/dev/branches/lucene6005/solr/contrib/langid/src/java/org/apache/solr/update/processor/LanguageIdentifierUpdateProcessor.java Sun Dec 7 11:37:32 2014
@@ -78,6 +78,8 @@ public abstract class LanguageIdentifier
protected HashMap<String,String> lcMap;
protected HashMap<String,String> mapLcMap;
protected IndexSchema schema;
+ protected int maxFieldValueChars;
+ protected int maxTotalChars;
// Regex patterns
protected final Pattern tikaSimilarityPattern = Pattern.compile(".*\\((.*?)\\)");
@@ -169,8 +171,21 @@ public abstract class LanguageIdentifier
mapPattern = Pattern.compile(params.get(MAP_PATTERN, MAP_PATTERN_DEFAULT));
mapReplaceStr = params.get(MAP_REPLACE, MAP_REPLACE_DEFAULT);
-
-
+ maxFieldValueChars = params.getInt(MAX_FIELD_VALUE_CHARS, MAX_FIELD_VALUE_CHARS_DEFAULT);
+ maxTotalChars = params.getInt(MAX_TOTAL_CHARS, MAX_TOTAL_CHARS_DEFAULT);
+ if (maxFieldValueChars > maxTotalChars) {
+ if (maxTotalChars == MAX_TOTAL_CHARS_DEFAULT) {
+ // If the user specified only maxFieldValueChars, make maxTotalChars the same as it
+ log.warn(MAX_FIELD_VALUE_CHARS + " (" + maxFieldValueChars + ") is less than " + MAX_TOTAL_CHARS + " ("
+ + maxTotalChars + "). Setting " + MAX_TOTAL_CHARS + " to " + maxFieldValueChars + ".");
+ maxTotalChars = maxFieldValueChars;
+ } else {
+ // If the user specified maxTotalChars, make maxFieldValueChars the same as it
+ log.warn(MAX_FIELD_VALUE_CHARS + " (" + maxFieldValueChars + ") is less than " + MAX_TOTAL_CHARS + " ("
+ + maxTotalChars + "). Setting " + MAX_FIELD_VALUE_CHARS + " to " + maxTotalChars + ".");
+ maxFieldValueChars = maxTotalChars;
+ }
+ }
}
log.debug("LangId configured");
@@ -203,11 +218,10 @@ public abstract class LanguageIdentifier
String fallbackLang = getFallbackLang(doc, fallbackFields, fallbackValue);
if(langField == null || !doc.containsKey(langField) || (doc.containsKey(langField) && overwrite)) {
- String allText = concatFields(doc, inputFields);
- List<DetectedLanguage> languagelist = detectLanguage(allText);
+ List<DetectedLanguage> languagelist = detectLanguage(doc);
docLang = resolveLanguage(languagelist, fallbackLang);
docLangs.add(docLang);
- log.debug("Detected main document language from fields "+inputFields+": "+docLang);
+ log.debug("Detected main document language from fields "+ Arrays.toString(inputFields) +": "+docLang);
if(doc.containsKey(langField) && overwrite) {
log.debug("Overwritten old value "+doc.getFieldValue(langField));
@@ -227,8 +241,7 @@ public abstract class LanguageIdentifier
if(doc.containsKey(fieldName)) {
String fieldLang;
if(mapIndividual && mapIndividualFieldsSet.contains(fieldName)) {
- String text = (String) doc.getFieldValue(fieldName);
- List<DetectedLanguage> languagelist = detectLanguage(text);
+ List<DetectedLanguage> languagelist = detectLanguage(doc);
fieldLang = resolveLanguage(languagelist, docLang);
docLangs.add(fieldLang);
log.debug("Mapping field "+fieldName+" using individually detected language "+fieldLang);
@@ -284,37 +297,13 @@ public abstract class LanguageIdentifier
return lang;
}
- /*
- * Concatenates content from multiple fields
- */
- protected String concatFields(SolrInputDocument doc, String[] fields) {
- StringBuilder sb = new StringBuilder();
- for (String fieldName : inputFields) {
- log.debug("Appending field "+fieldName);
- if (doc.containsKey(fieldName)) {
- Collection<Object> fieldValues = doc.getFieldValues(fieldName);
- if (fieldValues != null) {
- for (Object content : fieldValues) {
- if (content instanceof String) {
- sb.append((String) content);
- sb.append(" ");
- } else {
- log.warn("Field " + fieldName + " not a String value, not including in detection");
- }
- }
- }
- }
- }
- return sb.toString();
- }
-
/**
* Detects language(s) from a string.
* Classes wishing to implement their own language detection module should override this method.
* @param content The content to identify
* @return List of detected language(s) according to RFC-3066
*/
- protected abstract List<DetectedLanguage> detectLanguage(String content);
+ protected abstract List<DetectedLanguage> detectLanguage(SolrInputDocument content);
/**
* Chooses a language based on the list of candidates detected
Modified: lucene/dev/branches/lucene6005/solr/contrib/langid/src/java/org/apache/solr/update/processor/TikaLanguageIdentifierUpdateProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/contrib/langid/src/java/org/apache/solr/update/processor/TikaLanguageIdentifierUpdateProcessor.java?rev=1643662&r1=1643661&r2=1643662&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/contrib/langid/src/java/org/apache/solr/update/processor/TikaLanguageIdentifierUpdateProcessor.java (original)
+++ lucene/dev/branches/lucene6005/solr/contrib/langid/src/java/org/apache/solr/update/processor/TikaLanguageIdentifierUpdateProcessor.java Sun Dec 7 11:37:32 2014
@@ -24,6 +24,9 @@ import org.apache.solr.request.SolrQuery
import org.apache.solr.response.SolrQueryResponse;
import org.apache.tika.language.LanguageIdentifier;
+import org.apache.solr.common.SolrInputDocument;
+import java.util.Collection;
+
/**
* Identifies the language of a set of input fields using Tika's
* LanguageIdentifier.
@@ -40,9 +43,10 @@ public class TikaLanguageIdentifierUpdat
}
@Override
- protected List<DetectedLanguage> detectLanguage(String content) {
+ protected List<DetectedLanguage> detectLanguage(SolrInputDocument doc) {
List<DetectedLanguage> languages = new ArrayList<>();
- if(content.trim().length() != 0) {
+ String content = concatFields(doc);
+ if (content.length() != 0) {
LanguageIdentifier identifier = new LanguageIdentifier(content);
// FIXME: Hack - we get the distance from toString and calculate our own certainty score
Double distance = Double.parseDouble(tikaSimilarityPattern.matcher(identifier.toString()).replaceFirst("$1"));
@@ -57,4 +61,59 @@ public class TikaLanguageIdentifierUpdat
}
return languages;
}
+
+
+ /**
+ * Concatenates content from multiple fields
+ */
+ protected String concatFields(SolrInputDocument doc) {
+ StringBuilder sb = new StringBuilder(getExpectedSize(doc, inputFields));
+ for (String fieldName : inputFields) {
+ log.debug("Appending field " + fieldName);
+ if (doc.containsKey(fieldName)) {
+ Collection<Object> fieldValues = doc.getFieldValues(fieldName);
+ if (fieldValues != null) {
+ for (Object content : fieldValues) {
+ if (content instanceof String) {
+ String stringContent = (String) content;
+ if (stringContent.length() > maxFieldValueChars) {
+ sb.append(stringContent.substring(0, maxFieldValueChars));
+ } else {
+ sb.append(stringContent);
+}
+ sb.append(" ");
+ if (sb.length() > maxTotalChars) {
+ sb.setLength(maxTotalChars);
+ break;
+ }
+ } else {
+ log.warn("Field " + fieldName + " not a String value, not including in detection");
+ }
+ }
+ }
+ }
+ }
+ return sb.toString();
+ }
+
+ /**
+ * Calculate expected string size.
+ *
+ * @param doc solr input document
+ * @param fields fields to select
+ * @return expected size of string value
+ */
+ private int getExpectedSize(SolrInputDocument doc, String[] fields) {
+ int docSize = 0;
+ for (String field : fields) {
+ Collection<Object> contents = doc.getFieldValues(field);
+ for (Object content : contents) {
+ if (content instanceof String) {
+ docSize += Math.min(((String) content).length(), maxFieldValueChars);
+ }
+ }
+ docSize = Math.min(docSize, maxTotalChars);
+ }
+ return docSize;
+ }
}
Modified: lucene/dev/branches/lucene6005/solr/contrib/langid/src/test/org/apache/solr/update/processor/TikaLanguageIdentifierUpdateProcessorFactoryTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/contrib/langid/src/test/org/apache/solr/update/processor/TikaLanguageIdentifierUpdateProcessorFactoryTest.java?rev=1643662&r1=1643661&r2=1643662&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/contrib/langid/src/test/org/apache/solr/update/processor/TikaLanguageIdentifierUpdateProcessorFactoryTest.java (original)
+++ lucene/dev/branches/lucene6005/solr/contrib/langid/src/test/org/apache/solr/update/processor/TikaLanguageIdentifierUpdateProcessorFactoryTest.java Sun Dec 7 11:37:32 2014
@@ -17,11 +17,166 @@ package org.apache.solr.update.processor
* limitations under the License.
*/
+import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.params.ModifiableSolrParams;
+import org.junit.Test;
public class TikaLanguageIdentifierUpdateProcessorFactoryTest extends LanguageIdentifierUpdateProcessorFactoryTestCase {
@Override
protected LanguageIdentifierUpdateProcessor createLangIdProcessor(ModifiableSolrParams parameters) throws Exception {
return new TikaLanguageIdentifierUpdateProcessor(_parser.buildRequestFrom(h.getCore(), parameters, null), resp, null);
}
+
+
+ @Test
+ public void testMaxFieldValueChars() throws Exception {
+ SolrInputDocument doc = new SolrInputDocument();
+ String valueF1 = "Apache Lucene is a free/open source information retrieval software library, originally created in Java by Doug Cutting. It is supported by the Apache Software Foundation and is released under the Apache Software License.";
+ String valueF2 = "An open-source search server based on the Lucene Java search library. News, documentation, resources, and download.";
+ doc.addField("foo_s", valueF1);
+
+ ModifiableSolrParams parameters = new ModifiableSolrParams();
+ parameters.add("langid.fl", "foo_s");
+ parameters.add("langid.langField", "language");
+ parameters.add("langid.enforceSchema", "false");
+ TikaLanguageIdentifierUpdateProcessor p = (TikaLanguageIdentifierUpdateProcessor) createLangIdProcessor(parameters);
+ assertEquals(valueF1, p.concatFields(doc).trim());
+
+ parameters = new ModifiableSolrParams();
+ parameters.add("langid.fl", "foo_s");
+ parameters.add("langid.langField", "language");
+ parameters.add("langid.enforceSchema", "false");
+ parameters.add("langid.maxFieldValueChars", "6");
+ p = (TikaLanguageIdentifierUpdateProcessor) createLangIdProcessor(parameters);
+ assertEquals("Apache", p.concatFields(doc).trim());
+
+ doc.addField("bar_s", valueF2);
+
+ parameters = new ModifiableSolrParams();
+ parameters.add("langid.fl", "foo_s,bar_s");
+ parameters.add("langid.langField", "language");
+ parameters.add("langid.enforceSchema", "false");
+ p = (TikaLanguageIdentifierUpdateProcessor) createLangIdProcessor(parameters);
+ assertEquals(valueF1 + " " + valueF2, p.concatFields(doc).trim());
+
+ parameters = new ModifiableSolrParams();
+ parameters.add("langid.fl", "foo_s,bar_s");
+ parameters.add("langid.langField", "language");
+ parameters.add("langid.enforceSchema", "false");
+ parameters.add("langid.maxFieldValueChars", "6");
+ p = (TikaLanguageIdentifierUpdateProcessor) createLangIdProcessor(parameters);
+ assertEquals("Apache" + " " + "An ope", p.concatFields(doc).trim());
+
+ parameters = new ModifiableSolrParams();
+ parameters.add("langid.fl", "foo_s,bar_s");
+ parameters.add("langid.langField", "language");
+ parameters.add("langid.enforceSchema", "false");
+ parameters.add("langid.maxFieldValueChars", "100000");
+ p = (TikaLanguageIdentifierUpdateProcessor) createLangIdProcessor(parameters);
+ assertEquals(valueF1 + " " + valueF2, p.concatFields(doc).trim());
+
+}
+
+ @Test
+ public void testMaxTotalChars() throws Exception {
+ SolrInputDocument doc = new SolrInputDocument();
+ String valueF1 = "Apache Lucene is a free/open source information retrieval software library, originally created in Java by Doug Cutting. It is supported by the Apache Software Foundation and is released under the Apache Software License.";
+ String valueF2 = "An open-source search server based on the Lucene Java search library. News, documentation, resources, and download.";
+ doc.addField("foo_s", valueF1);
+
+ ModifiableSolrParams parameters = new ModifiableSolrParams();
+ parameters.add("langid.fl", "foo_s");
+ parameters.add("langid.langField", "language");
+ parameters.add("langid.enforceSchema", "false");
+ TikaLanguageIdentifierUpdateProcessor p = (TikaLanguageIdentifierUpdateProcessor) createLangIdProcessor(parameters);
+ assertEquals(valueF1, p.concatFields(doc).trim());
+
+ parameters = new ModifiableSolrParams();
+ parameters.add("langid.fl", "foo_s");
+ parameters.add("langid.langField", "language");
+ parameters.add("langid.enforceSchema", "false");
+ parameters.add("langid.maxTotalChars", "6");
+ p = (TikaLanguageIdentifierUpdateProcessor) createLangIdProcessor(parameters);
+ assertEquals("Apache", p.concatFields(doc).trim());
+
+ doc.addField("bar_s", valueF2);
+
+ parameters = new ModifiableSolrParams();
+ parameters.add("langid.fl", "foo_s,bar_s");
+ parameters.add("langid.langField", "language");
+ parameters.add("langid.enforceSchema", "false");
+ p = (TikaLanguageIdentifierUpdateProcessor) createLangIdProcessor(parameters);
+ assertEquals(valueF1 + " " + valueF2, p.concatFields(doc).trim());
+
+ parameters = new ModifiableSolrParams();
+ parameters.add("langid.fl", "foo_s,bar_s");
+ parameters.add("langid.langField", "language");
+ parameters.add("langid.enforceSchema", "false");
+ parameters.add("langid.maxTotalChars", "6");
+ p = (TikaLanguageIdentifierUpdateProcessor) createLangIdProcessor(parameters);
+ assertEquals("Apache", p.concatFields(doc).trim());
+
+ parameters = new ModifiableSolrParams();
+ parameters.add("langid.fl", "foo_s,bar_s");
+ parameters.add("langid.langField", "language");
+ parameters.add("langid.enforceSchema", "false");
+ parameters.add("langid.maxTotalChars", "100000");
+ p = (TikaLanguageIdentifierUpdateProcessor) createLangIdProcessor(parameters);
+ assertEquals(valueF1 + " " + valueF2, p.concatFields(doc).trim());
+
+ }
+
+
+ @Test
+ public void testMaxFieldValueCharsAndMaxTotalChars() throws Exception {
+ SolrInputDocument doc = new SolrInputDocument();
+ String valueF1 = "Apache Lucene is a free/open source information retrieval software library, originally created in Java by Doug Cutting. It is supported by the Apache Software Foundation and is released under the Apache Software License.";
+ String valueF2 = "An open-source search server based on the Lucene Java search library. News, documentation, resources, and download.";
+ doc.addField("foo_s", valueF1);
+
+ ModifiableSolrParams parameters = new ModifiableSolrParams();
+ parameters.add("langid.fl", "foo_s");
+ parameters.add("langid.langField", "language");
+ parameters.add("langid.enforceSchema", "false");
+ TikaLanguageIdentifierUpdateProcessor p = (TikaLanguageIdentifierUpdateProcessor) createLangIdProcessor(parameters);
+ assertEquals(valueF1, p.concatFields(doc).trim());
+
+ parameters = new ModifiableSolrParams();
+ parameters.add("langid.fl", "foo_s");
+ parameters.add("langid.langField", "language");
+ parameters.add("langid.enforceSchema", "false");
+ parameters.add("langid.maxFieldValueChars", "8");
+ parameters.add("langid.maxTotalChars", "6");
+ p = (TikaLanguageIdentifierUpdateProcessor) createLangIdProcessor(parameters);
+ assertEquals("Apache", p.concatFields(doc).trim());
+
+ doc.addField("bar_s", valueF2);
+
+ parameters = new ModifiableSolrParams();
+ parameters.add("langid.fl", "foo_s,bar_s");
+ parameters.add("langid.langField", "language");
+ parameters.add("langid.enforceSchema", "false");
+ p = (TikaLanguageIdentifierUpdateProcessor) createLangIdProcessor(parameters);
+ assertEquals(valueF1 + " " + valueF2, p.concatFields(doc).trim());
+
+ parameters = new ModifiableSolrParams();
+ parameters.add("langid.fl", "foo_s,bar_s");
+ parameters.add("langid.langField", "language");
+ parameters.add("langid.enforceSchema", "false");
+ parameters.add("langid.maxFieldValueChars", "3");
+ parameters.add("langid.maxTotalChars", "8");
+ p = (TikaLanguageIdentifierUpdateProcessor) createLangIdProcessor(parameters);
+ assertEquals("Apa An", p.concatFields(doc).trim());
+
+ parameters = new ModifiableSolrParams();
+ parameters.add("langid.fl", "foo_s,bar_s");
+ parameters.add("langid.langField", "language");
+ parameters.add("langid.enforceSchema", "false");
+ parameters.add("langid.maxFieldValueChars", "10000");
+ parameters.add("langid.maxTotalChars", "100000");
+ p = (TikaLanguageIdentifierUpdateProcessor) createLangIdProcessor(parameters);
+ assertEquals(valueF1 + " " + valueF2, p.concatFields(doc).trim());
+
+ }
+
}
Modified: lucene/dev/branches/lucene6005/solr/contrib/morphlines-core/src/test-files/solr/collection1/conf/solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/contrib/morphlines-core/src/test-files/solr/collection1/conf/solrconfig.xml?rev=1643662&r1=1643661&r2=1643662&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/contrib/morphlines-core/src/test-files/solr/collection1/conf/solrconfig.xml (original)
+++ lucene/dev/branches/lucene6005/solr/contrib/morphlines-core/src/test-files/solr/collection1/conf/solrconfig.xml Sun Dec 7 11:37:32 2014
@@ -233,11 +233,6 @@
<unlockOnStartup>false</unlockOnStartup>
-->
- <!-- Expert: Controls how often Lucene loads terms into memory
- Default is 128 and is likely good for most everyone.
- -->
- <!-- <termIndexInterval>128</termIndexInterval> -->
-
<!-- If true, IndexReaders will be reopened (often more efficient)
instead of closed and then opened. Default: true
-->
Modified: lucene/dev/branches/lucene6005/solr/contrib/morphlines-core/src/test-files/solr/minimr/conf/solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/contrib/morphlines-core/src/test-files/solr/minimr/conf/solrconfig.xml?rev=1643662&r1=1643661&r2=1643662&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/contrib/morphlines-core/src/test-files/solr/minimr/conf/solrconfig.xml (original)
+++ lucene/dev/branches/lucene6005/solr/contrib/morphlines-core/src/test-files/solr/minimr/conf/solrconfig.xml Sun Dec 7 11:37:32 2014
@@ -248,11 +248,6 @@
<!--
<unlockOnStartup>false</unlockOnStartup>
-->
-
- <!-- Expert: Controls how often Lucene loads terms into memory
- Default is 128 and is likely good for most everyone.
- -->
- <!-- <termIndexInterval>128</termIndexInterval> -->
<!-- If true, IndexReaders will be reopened (often more efficient)
instead of closed and then opened. Default: true
Modified: lucene/dev/branches/lucene6005/solr/contrib/morphlines-core/src/test-files/solr/mrunit/conf/solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/contrib/morphlines-core/src/test-files/solr/mrunit/conf/solrconfig.xml?rev=1643662&r1=1643661&r2=1643662&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/contrib/morphlines-core/src/test-files/solr/mrunit/conf/solrconfig.xml (original)
+++ lucene/dev/branches/lucene6005/solr/contrib/morphlines-core/src/test-files/solr/mrunit/conf/solrconfig.xml Sun Dec 7 11:37:32 2014
@@ -251,11 +251,6 @@
<unlockOnStartup>false</unlockOnStartup>
-->
- <!-- Expert: Controls how often Lucene loads terms into memory
- Default is 128 and is likely good for most everyone.
- -->
- <!-- <termIndexInterval>128</termIndexInterval> -->
-
<!-- If true, IndexReaders will be reopened (often more efficient)
instead of closed and then opened. Default: true
-->
Modified: lucene/dev/branches/lucene6005/solr/contrib/morphlines-core/src/test-files/solr/solrcelltest/collection1/conf/solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/contrib/morphlines-core/src/test-files/solr/solrcelltest/collection1/conf/solrconfig.xml?rev=1643662&r1=1643661&r2=1643662&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/contrib/morphlines-core/src/test-files/solr/solrcelltest/collection1/conf/solrconfig.xml (original)
+++ lucene/dev/branches/lucene6005/solr/contrib/morphlines-core/src/test-files/solr/solrcelltest/collection1/conf/solrconfig.xml Sun Dec 7 11:37:32 2014
@@ -232,11 +232,6 @@
<!--
<unlockOnStartup>false</unlockOnStartup>
-->
-
- <!-- Expert: Controls how often Lucene loads terms into memory
- Default is 128 and is likely good for most everyone.
- -->
- <!-- <termIndexInterval>128</termIndexInterval> -->
<!-- If true, IndexReaders will be reopened (often more efficient)
instead of closed and then opened. Default: true
Modified: lucene/dev/branches/lucene6005/solr/contrib/morphlines-core/src/test-files/solr/solrcloud/conf/solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/contrib/morphlines-core/src/test-files/solr/solrcloud/conf/solrconfig.xml?rev=1643662&r1=1643661&r2=1643662&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/contrib/morphlines-core/src/test-files/solr/solrcloud/conf/solrconfig.xml (original)
+++ lucene/dev/branches/lucene6005/solr/contrib/morphlines-core/src/test-files/solr/solrcloud/conf/solrconfig.xml Sun Dec 7 11:37:32 2014
@@ -252,11 +252,6 @@
<unlockOnStartup>false</unlockOnStartup>
-->
- <!-- Expert: Controls how often Lucene loads terms into memory
- Default is 128 and is likely good for most everyone.
- -->
- <!-- <termIndexInterval>128</termIndexInterval> -->
-
<!-- If true, IndexReaders will be reopened (often more efficient)
instead of closed and then opened. Default: true
-->
Modified: lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/cloud/Overseer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/cloud/Overseer.java?rev=1643662&r1=1643661&r2=1643662&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/cloud/Overseer.java (original)
+++ lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/cloud/Overseer.java Sun Dec 7 11:37:32 2014
@@ -17,32 +17,26 @@ package org.apache.solr.cloud;
* the License.
*/
-import static java.util.Collections.singletonMap;
import static org.apache.solr.cloud.OverseerCollectionProcessor.SHARD_UNIQUE;
import static org.apache.solr.cloud.OverseerCollectionProcessor.ONLY_ACTIVE_NODES;
-import static org.apache.solr.cloud.OverseerCollectionProcessor.COLL_PROP_PREFIX;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.BALANCESHARDUNIQUE;
import java.io.Closeable;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
-import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.ListIterator;
import java.util.Locale;
import java.util.Map;
-import java.util.Map.Entry;
import java.util.Random;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
-import com.google.common.collect.ImmutableSet;
import org.apache.commons.lang.StringUtils;
import org.apache.solr.client.solrj.SolrResponse;
import org.apache.solr.cloud.overseer.ClusterStateMutator;
@@ -55,7 +49,6 @@ import org.apache.solr.cloud.overseer.Zk
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.DocCollection;
-import org.apache.solr.common.cloud.ImplicitDocRouter;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.SolrZkClient;
@@ -75,7 +68,8 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
- * Cluster leader. Responsible node assignments, cluster state file?
+ * Cluster leader. Responsible for processing state updates, node assignments, creating/deleting
+ * collections, shards, replicas and setting various properties.
*/
public class Overseer implements Closeable {
public static final String QUEUE_OPERATION = "operation";
@@ -98,8 +92,6 @@ public class Overseer implements Closeab
static enum LeaderStatus {DONT_KNOW, NO, YES}
- private long lastUpdatedTime = 0;
-
private class ClusterStateUpdater implements Runnable, Closeable {
private final ZkStateReader reader;
@@ -122,10 +114,6 @@ public class Overseer implements Closeab
private Map clusterProps;
private boolean isClosed = false;
- private final Map<String, Object> updateNodes = new LinkedHashMap<>();
- private boolean isClusterStateModified = false;
-
-
public ClusterStateUpdater(final ZkStateReader reader, final String myId, Stats zkStats) {
this.zkClient = reader.getZkClient();
this.zkStats = zkStats;
@@ -178,35 +166,18 @@ public class Overseer implements Closeab
}
else if (LeaderStatus.YES == isLeader) {
final ZkNodeProps message = ZkNodeProps.load(head);
- final String operation = message.getStr(QUEUE_OPERATION);
- final TimerContext timerContext = stats.time(operation);
- try {
- ZkWriteCommand zkWriteCommand = processMessage(clusterState, message, operation, workQueue.getStats().getQueueLength());
- clusterState = zkStateWriter.enqueueUpdate(clusterState, zkWriteCommand);
- stats.success(operation);
- } catch (Exception e) {
- // generally there is nothing we can do - in most cases, we have
- // an issue that will fail again on retry or we cannot communicate with a
- // ZooKeeper in which case another Overseer should take over
- // TODO: if ordering for the message is not important, we could
- // track retries and put it back on the end of the queue
- log.error("Overseer could not process the current clusterstate state update message, skipping the message.", e);
- stats.error(operation);
- } finally {
- timerContext.stop();
- }
- if (zkStateWriter.hasPendingUpdates()) {
- clusterState = zkStateWriter.writePendingUpdates();
- }
+ log.info("processMessage: queueSize: {}, message = {}", workQueue.getStats().getQueueLength(), message);
+ clusterState = processQueueItem(message, clusterState, zkStateWriter);
workQueue.poll(); // poll-ing removes the element we got by peek-ing
}
else {
log.info("am_i_leader unclear {}", isLeader);
// re-peek below in case our 'head' value is out-of-date by now
}
-
head = workQueue.peek();
}
+ // force flush at the end of the loop
+ clusterState = zkStateWriter.writePendingUpdates();
}
} catch (KeeperException e) {
if (e.code() == KeeperException.Code.SESSIONEXPIRED) {
@@ -226,8 +197,6 @@ public class Overseer implements Closeab
}
log.info("Starting to work on the main queue");
- int lastStateFormat = -1; // sentinel
- String lastCollectionName = null;
try {
ZkStateWriter zkStateWriter = new ZkStateWriter(reader, stats);
ClusterState clusterState = null;
@@ -269,90 +238,37 @@ public class Overseer implements Closeab
// the state queue, items would have been left in the
// work queue so let's process those first
byte[] data = workQueue.peek();
+ boolean hadWorkItems = data != null;
while (data != null) {
final ZkNodeProps message = ZkNodeProps.load(data);
- final String operation = message.getStr(QUEUE_OPERATION);
- final TimerContext timerContext = stats.time(operation);
- try {
- ZkWriteCommand zkWriteCommand = processMessage(clusterState, message, operation, workQueue.getStats().getQueueLength());
- clusterState = zkStateWriter.enqueueUpdate(clusterState, zkWriteCommand);
- stats.success(operation);
- } catch (Exception e) {
- // generally there is nothing we can do - in most cases, we have
- // an issue that will fail again on retry or we cannot communicate with a
- // ZooKeeper in which case another Overseer should take over
- // TODO: if ordering for the message is not important, we could
- // track retries and put it back on the end of the queue
- log.error("Overseer could not process the current clusterstate state update message, skipping the message.", e);
- stats.error(operation);
- } finally {
- timerContext.stop();
- }
- if (zkStateWriter.hasPendingUpdates()) {
- clusterState = zkStateWriter.writePendingUpdates();
- }
+ log.info("processMessage: queueSize: {}, message = {}", workQueue.getStats().getQueueLength(), message);
+ clusterState = processQueueItem(message, clusterState, zkStateWriter);
workQueue.poll(); // poll-ing removes the element we got by peek-ing
data = workQueue.peek();
}
+ // force flush at the end of the loop
+ if (hadWorkItems) {
+ clusterState = zkStateWriter.writePendingUpdates();
+ }
}
while (head != null) {
final ZkNodeProps message = ZkNodeProps.load(head.getBytes());
- final String operation = message.getStr(QUEUE_OPERATION);
-
- // we batch updates for the main cluster state together (stateFormat=1)
- // but if we encounter a message for a collection with a stateFormat different than the last
- // then we stop batching at that point
- String collection = message.getStr(ZkStateReader.COLLECTION_PROP);
- if (collection == null) collection = message.getStr("name");
- if (collection != null) {
- DocCollection docCollection = clusterState.getCollectionOrNull(collection);
- if (lastStateFormat != -1 && docCollection != null && docCollection.getStateFormat() != lastStateFormat) {
- lastStateFormat = docCollection.getStateFormat();
- // we don't want to mix batching of different state formats together because that makes
- // it harder to guarantee atomicity of ZK writes
- break;
- }
- if (docCollection != null) {
- lastStateFormat = docCollection.getStateFormat();
- }
- }
-
- final TimerContext timerContext = stats.time(operation);
- try {
- ZkWriteCommand zkWriteCommand = processMessage(clusterState, message, operation, stateUpdateQueue.getStats().getQueueLength());
- clusterState = zkStateWriter.enqueueUpdate(clusterState, zkWriteCommand);
- stats.success(operation);
- } catch (Exception e) {
- // generally there is nothing we can do - in most cases, we have
- // an issue that will fail again on retry or we cannot communicate with
- // ZooKeeper in which case another Overseer should take over
- // TODO: if ordering for the message is not important, we could
- // track retries and put it back on the end of the queue
- log.error("Overseer could not process the current clusterstate state update message, skipping the message.", e);
- stats.error(operation);
- } finally {
- timerContext.stop();
- }
+ log.info("processMessage: queueSize: {}, message = {} current state version: {}", stateUpdateQueue.getStats().getQueueLength(), message, clusterState.getZkClusterStateVersion());
+ clusterState = processQueueItem(message, clusterState, zkStateWriter);
workQueue.offer(head.getBytes());
stateUpdateQueue.poll();
- if (isClosed || System.nanoTime() - lastUpdatedTime > TimeUnit.NANOSECONDS.convert(STATE_UPDATE_DELAY, TimeUnit.MILLISECONDS)) break;
- if (!updateNodes.isEmpty() && !collection.equals(lastCollectionName)) {
- lastCollectionName = collection;
- break;
- }
- lastCollectionName = collection;
+ if (isClosed) break;
// if an event comes in the next 100ms batch it together
head = stateUpdateQueue.peek(100);
}
- if (zkStateWriter.hasPendingUpdates()) {
- clusterState = zkStateWriter.writePendingUpdates();
- lastUpdatedTime = zkStateWriter.getLastUpdatedTime();
- }
+ // we should force write all pending updates because the next iteration might sleep until there
+ // are more items in the main queue
+ clusterState = zkStateWriter.writePendingUpdates();
// clean work queue
- while (workQueue.poll() != null) ;
+ while (workQueue.poll() != null);
} catch (KeeperException e) {
if (e.code() == KeeperException.Code.SESSIONEXPIRED) {
@@ -383,6 +299,30 @@ public class Overseer implements Closeab
}
}
+ private ClusterState processQueueItem(ZkNodeProps message, ClusterState clusterState, ZkStateWriter zkStateWriter) throws KeeperException, InterruptedException {
+ final String operation = message.getStr(QUEUE_OPERATION);
+ ZkWriteCommand zkWriteCommand = null;
+ final TimerContext timerContext = stats.time(operation);
+ try {
+ zkWriteCommand = processMessage(clusterState, message, operation);
+ stats.success(operation);
+ } catch (Exception e) {
+ // generally there is nothing we can do - in most cases, we have
+ // an issue that will fail again on retry or we cannot communicate with a
+ // ZooKeeper in which case another Overseer should take over
+ // TODO: if ordering for the message is not important, we could
+ // track retries and put it back on the end of the queue
+ log.error("Overseer could not process the current clusterstate state update message, skipping the message.", e);
+ stats.error(operation);
+ } finally {
+ timerContext.stop();
+ }
+ if (zkWriteCommand != null) {
+ clusterState = zkStateWriter.enqueueUpdate(clusterState, zkWriteCommand);
+ }
+ return clusterState;
+ }
+
private void checkIfIamStillLeader() {
if (zkController != null && zkController.getCoreContainer().isShutDown()) return;//shutting down no need to go further
org.apache.zookeeper.data.Stat stat = new org.apache.zookeeper.data.Stat();
@@ -423,8 +363,7 @@ public class Overseer implements Closeab
}
private ZkWriteCommand processMessage(ClusterState clusterState,
- final ZkNodeProps message, final String operation, int queueSize) {
- log.info("processMessage: queueSize: {}, message = {}", queueSize, message);
+ final ZkNodeProps message, final String operation) {
CollectionParams.CollectionAction collectionAction = CollectionParams.CollectionAction.get(operation);
if (collectionAction != null) {
switch (collectionAction) {
@@ -445,7 +384,7 @@ public class Overseer implements Closeab
case DELETEREPLICAPROP:
return new ReplicaMutator(getZkStateReader()).removeReplicaProperty(clusterState, message);
case BALANCESHARDUNIQUE:
- ExclusiveSliceProperty dProp = new ExclusiveSliceProperty(this, clusterState, message);
+ ExclusiveSliceProperty dProp = new ExclusiveSliceProperty(clusterState, message);
if (dProp.balanceProperty()) {
String collName = message.getStr(ZkStateReader.COLLECTION_PROP);
return new ZkWriteCommand(collName, dProp.getDocCollection());
@@ -557,55 +496,6 @@ public class Overseer implements Closeab
return LeaderStatus.NO;
}
-
-
-
- private ClusterState updateSlice(ClusterState state, String collectionName, Slice slice) {
- DocCollection newCollection = null;
- DocCollection coll = state.getCollectionOrNull(collectionName) ;
- Map<String,Slice> slices;
-
- if (coll == null) {
- // when updateSlice is called on a collection that doesn't exist, it's currently when a core is publishing itself
- // without explicitly creating a collection. In this current case, we assume custom sharding with an "implicit" router.
- slices = new LinkedHashMap<>(1);
- slices.put(slice.getName(), slice);
- Map<String,Object> props = new HashMap<>(1);
- props.put(DocCollection.DOC_ROUTER, ZkNodeProps.makeMap("name",ImplicitDocRouter.NAME));
- newCollection = new DocCollection(collectionName, slices, props, new ImplicitDocRouter());
- } else {
- slices = new LinkedHashMap<>(coll.getSlicesMap()); // make a shallow copy
- slices.put(slice.getName(), slice);
- newCollection = coll.copyWithSlices(slices);
- }
-
- // System.out.println("###!!!### NEW CLUSTERSTATE: " + JSONUtil.toJSON(newCollections));
-
- return newState(state, singletonMap(collectionName, newCollection));
- }
-
- private ClusterState newState(ClusterState state, Map<String, DocCollection> colls) {
- for (Entry<String, DocCollection> e : colls.entrySet()) {
- DocCollection c = e.getValue();
- if (c == null) {
- isClusterStateModified = true;
- state = state.copyWith(e.getKey(), null);
- updateNodes.put(ZkStateReader.getCollectionPath(e.getKey()) ,null);
- continue;
- }
-
- if (c.getStateFormat() > 1) {
- updateNodes.put(ZkStateReader.getCollectionPath(c.getName()),
- new ClusterState(-1, Collections.<String>emptySet(), singletonMap(c.getName(), c)));
- } else {
- isClusterStateModified = true;
- }
- state = state.copyWith(e.getKey(), c);
-
- }
- return state;
- }
-
@Override
public void close() {
this.isClosed = true;
@@ -614,7 +504,6 @@ public class Overseer implements Closeab
}
// Class to encapsulate processing replica properties that have at most one replica hosting a property per slice.
private class ExclusiveSliceProperty {
- private ClusterStateUpdater updater;
private ClusterState clusterState;
private final boolean onlyActiveNodes;
private final String property;
@@ -636,8 +525,7 @@ public class Overseer implements Closeab
private int assigned = 0;
- ExclusiveSliceProperty(ClusterStateUpdater updater, ClusterState clusterState, ZkNodeProps message) {
- this.updater = updater;
+ ExclusiveSliceProperty(ClusterState clusterState, ZkNodeProps message) {
this.clusterState = clusterState;
String tmp = message.getStr(ZkStateReader.PROPERTY_PROP);
if (StringUtils.startsWith(tmp, OverseerCollectionProcessor.COLL_PROP_PREFIX) == false) {
@@ -898,7 +786,8 @@ public class Overseer implements Closeab
balanceUnassignedReplicas();
for (Slice newSlice : changedSlices.values()) {
- clusterState = updater.updateSlice(clusterState, collectionName, newSlice);
+ DocCollection docCollection = CollectionMutator.updateSlice(collectionName, clusterState.getCollection(collectionName), newSlice);
+ clusterState = ClusterStateMutator.newState(clusterState, collectionName, docCollection);
}
return true;
}
Modified: lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/cloud/ZkController.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/cloud/ZkController.java?rev=1643662&r1=1643661&r2=1643662&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/cloud/ZkController.java (original)
+++ lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/cloud/ZkController.java Sun Dec 7 11:37:32 2014
@@ -24,6 +24,7 @@ import java.net.InetAddress;
import java.net.NetworkInterface;
import java.net.URLEncoder;
import java.net.UnknownHostException;
+import java.nio.charset.StandardCharsets;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Collection;
@@ -2010,15 +2011,17 @@ public final class ZkController {
Map<String,Object> stateObj = null;
if (stateData != null && stateData.length > 0) {
- Object parsedJson = ZkStateReader.fromJSON(stateData);
- if (parsedJson instanceof Map) {
- stateObj = (Map<String,Object>)parsedJson;
- } else if (parsedJson instanceof String) {
- // old format still in ZK
- stateObj = new LinkedHashMap<>();
- stateObj.put("state", (String)parsedJson);
+ // TODO: Remove later ... this is for upgrading from 4.8.x to 4.10.3 (see: SOLR-6732)
+ if (stateData[0] == (byte)'{') {
+ Object parsedJson = ZkStateReader.fromJSON(stateData);
+ if (parsedJson instanceof Map) {
+ stateObj = (Map<String,Object>)parsedJson;
+ } else {
+ throw new SolrException(ErrorCode.SERVER_ERROR, "Leader-initiated recovery state data is invalid! "+parsedJson);
+ }
} else {
- throw new SolrException(ErrorCode.SERVER_ERROR, "Leader-initiated recovery state data is invalid! "+parsedJson);
+ // old format still in ZK
+ stateObj = ZkNodeProps.makeMap("state", new String(stateData, StandardCharsets.UTF_8));
}
}
@@ -2051,7 +2054,7 @@ public final class ZkController {
log.warn(exc.getMessage(), exc);
}
if (stateObj == null)
- stateObj = new LinkedHashMap<>();
+ stateObj = ZkNodeProps.makeMap();
stateObj.put("state", state);
// only update the createdBy value if its not set
@@ -2123,7 +2126,7 @@ public final class ZkController {
final ZkController zkController = zkLoader.getZkController();
final SolrZkClient zkClient = zkController.getZkClient();
final String resourceLocation = zkLoader.getConfigSetZkPath() + "/" + resourceName;
- String errMsg = "Failed to persist resource at {0} - version mismatch {1}";
+ String errMsg = "Failed to persist resource at {0} - old {1}";
try {
try {
zkClient.setData(resourceLocation , content,znodeVersion, true);
@@ -2136,7 +2139,7 @@ public final class ZkController {
} catch (KeeperException.NodeExistsException nee) {
try {
Stat stat = zkClient.exists(resourceLocation, null, true);
- log.info("failed to set data version in zk is {} and expected version is {} ", stat.getVersion(),znodeVersion);
+ log.info("failed to set data version in zk is {0} and expected version is {1} ", stat.getVersion(),znodeVersion);
} catch (Exception e1) {
log.warn("could not get stat");
}
@@ -2148,7 +2151,15 @@ public final class ZkController {
}
} catch (KeeperException.BadVersionException bve){
- log.info(MessageFormat.format(errMsg,resourceLocation));
+ int v = -1;
+ try {
+ Stat stat = zkClient.exists(resourceLocation, null, true);
+ v = stat.getVersion();
+ } catch (Exception e) {
+ log.error(e.getMessage());
+
+ }
+ log.info(MessageFormat.format(errMsg+ " zkVersion= "+v,resourceLocation,znodeVersion));
throw new ResourceModifiedInZkException(ErrorCode.CONFLICT, MessageFormat.format(errMsg,resourceLocation,znodeVersion) + ", retry.");
}catch (ResourceModifiedInZkException e){
throw e;
Modified: lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java?rev=1643662&r1=1643661&r2=1643662&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java (original)
+++ lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java Sun Dec 7 11:37:32 2014
@@ -21,6 +21,7 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
+import java.util.concurrent.TimeUnit;
import org.apache.solr.cloud.Overseer;
import org.apache.solr.common.cloud.ClusterState;
@@ -37,7 +38,7 @@ import static java.util.Collections.sing
public class ZkStateWriter {
private static Logger log = LoggerFactory.getLogger(ZkStateWriter.class);
- public static ZkWriteCommand NO_OP = new ZkWriteCommand();
+ public static ZkWriteCommand NO_OP = ZkWriteCommand.noop();
protected final ZkStateReader reader;
protected final Overseer.Stats stats;
@@ -45,7 +46,11 @@ public class ZkStateWriter {
protected Map<String, DocCollection> updates = new HashMap<>();
protected ClusterState clusterState = null;
protected boolean isClusterStateModified = false;
- protected long lastUpdatedTime = -1;
+ protected long lastUpdatedTime = 0;
+
+ // state information which helps us batch writes
+ protected int lastStateFormat = -1; // sentinel value
+ protected String lastCollectionName = null;
public ZkStateWriter(ZkStateReader zkStateReader, Overseer.Stats stats) {
assert zkStateReader != null;
@@ -54,12 +59,17 @@ public class ZkStateWriter {
this.stats = stats;
}
- public ClusterState enqueueUpdate(ClusterState prevState, ZkWriteCommand cmd) {
+ public ClusterState enqueueUpdate(ClusterState prevState, ZkWriteCommand cmd) throws KeeperException, InterruptedException {
if (cmd == NO_OP) return prevState;
+ if (maybeFlushBefore(cmd)) {
+ // we must update the prev state to the new one
+ prevState = clusterState = writePendingUpdates();
+ }
+
if (cmd.collection == null) {
isClusterStateModified = true;
- clusterState = prevState.copyWith(cmd.name, (DocCollection) null);
+ clusterState = prevState.copyWith(cmd.name, null);
updates.put(cmd.name, null);
} else {
if (cmd.collection.getStateFormat() > 1) {
@@ -69,15 +79,54 @@ public class ZkStateWriter {
}
clusterState = prevState.copyWith(cmd.name, cmd.collection);
}
+
+ if (maybeFlushAfter(cmd)) {
+ return writePendingUpdates();
+ }
+
return clusterState;
}
+ /**
+ * Logic to decide a flush before processing a ZkWriteCommand
+ *
+ * @param cmd the ZkWriteCommand instance
+ * @return true if a flush is required, false otherwise
+ */
+ protected boolean maybeFlushBefore(ZkWriteCommand cmd) {
+ if (lastUpdatedTime == 0) {
+ // first update, make sure we go through
+ return false;
+ }
+ if (cmd.collection == null) {
+ return false;
+ }
+ if (cmd.collection.getStateFormat() != lastStateFormat) {
+ return true;
+ }
+ return cmd.collection.getStateFormat() > 1 && !cmd.name.equals(lastCollectionName);
+ }
+
+ /**
+ * Logic to decide a flush after processing a ZkWriteCommand
+ *
+ * @param cmd the ZkWriteCommand instance
+ * @return true if a flush to ZK is required, false otherwise
+ */
+ protected boolean maybeFlushAfter(ZkWriteCommand cmd) {
+ if (cmd.collection == null)
+ return false;
+ lastCollectionName = cmd.name;
+ lastStateFormat = cmd.collection.getStateFormat();
+ return System.nanoTime() - lastUpdatedTime > TimeUnit.NANOSECONDS.convert(Overseer.STATE_UPDATE_DELAY, TimeUnit.MILLISECONDS);
+ }
+
public boolean hasPendingUpdates() {
return !updates.isEmpty() || isClusterStateModified;
}
public ClusterState writePendingUpdates() throws KeeperException, InterruptedException {
- if (!hasPendingUpdates()) throw new IllegalStateException("No queued updates to execute");
+ if (!hasPendingUpdates()) return clusterState;
TimerContext timerContext = stats.time("update_state");
boolean success = false;
try {
@@ -94,7 +143,7 @@ public class ZkStateWriter {
byte[] data = ZkStateReader.toJSON(new ClusterState(-1, Collections.<String>emptySet(), singletonMap(c.getName(), c)));
if (reader.getZkClient().exists(path, true)) {
assert c.getZNodeVersion() >= 0;
- log.info("going to update_collection {}", path);
+ log.info("going to update_collection {} version: {}", path, c.getZNodeVersion());
Stat stat = reader.getZkClient().setData(path, data, c.getZNodeVersion(), true);
DocCollection newCollection = new DocCollection(name, c.getSlicesMap(), c.getProperties(), c.getRouter(), stat.getVersion(), path);
clusterState = clusterState.copyWith(name, newCollection);
@@ -140,10 +189,17 @@ public class ZkStateWriter {
return clusterState;
}
+ /**
+ * @return time returned by System.nanoTime at which the main cluster state was last written to ZK or 0 if
+ * never
+ */
public long getLastUpdatedTime() {
return lastUpdatedTime;
}
+ /**
+ * @return the most up-to-date cluster state until the last enqueueUpdate operation
+ */
public ClusterState getClusterState() {
return clusterState;
}
Modified: lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/cloud/overseer/ZkWriteCommand.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/cloud/overseer/ZkWriteCommand.java?rev=1643662&r1=1643661&r2=1643662&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/cloud/overseer/ZkWriteCommand.java (original)
+++ lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/cloud/overseer/ZkWriteCommand.java Sun Dec 7 11:37:32 2014
@@ -17,33 +17,30 @@ package org.apache.solr.cloud.overseer;
* limitations under the License.
*/
-import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.DocCollection;
-/**
-* Created by shalin on 29/10/14.
-*/
public class ZkWriteCommand {
public final String name;
public final DocCollection collection;
-// public final ClusterState state;
public final boolean noop;
public ZkWriteCommand(String name, DocCollection collection) {
this.name = name;
this.collection = collection;
-// this.state = state;
this.noop = false;
}
/**
* Returns a no-op
*/
- public ZkWriteCommand() {
+ protected ZkWriteCommand() {
this.noop = true;
this.name = null;
this.collection = null;
-// this.state = null;
+ }
+
+ public static ZkWriteCommand noop() {
+ return new ZkWriteCommand();
}
}
Modified: lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java?rev=1643662&r1=1643661&r2=1643662&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java (original)
+++ lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java Sun Dec 7 11:37:32 2014
@@ -209,7 +209,7 @@ public class HdfsDirectoryFactory extend
private BlockCache createBlockCache(int numberOfBlocksPerBank, int blockSize,
int bankCount, boolean directAllocation, int slabSize, int bufferSize,
int bufferCount) {
- BufferStore.initNewBuffer(bufferSize, bufferCount);
+ BufferStore.initNewBuffer(bufferSize, bufferCount, metrics);
long totalMemory = (long) bankCount * (long) numberOfBlocksPerBank
* (long) blockSize;
Modified: lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/PluginsRegistry.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/PluginsRegistry.java?rev=1643662&r1=1643661&r2=1643662&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/PluginsRegistry.java (original)
+++ lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/PluginsRegistry.java Sun Dec 7 11:37:32 2014
@@ -27,6 +27,7 @@ import org.apache.solr.common.util.Named
import org.apache.solr.handler.PingRequestHandler;
import org.apache.solr.handler.RealTimeGetHandler;
import org.apache.solr.handler.ReplicationHandler;
+import org.apache.solr.handler.SchemaHandler;
import org.apache.solr.handler.SolrConfigHandler;
import org.apache.solr.handler.UpdateRequestHandler;
import org.apache.solr.handler.admin.LoggingHandler;
@@ -57,7 +58,8 @@ public class PluginsRegistry {
//solrconfighandler
implicits.add(getReqHandlerInfo("/config", SolrConfigHandler.class, null));
-
+ //schemahandler
+ implicits.add(getReqHandlerInfo("/schema", SchemaHandler.class, null));
//register replicationhandler always for SolrCloud
implicits.add(getReqHandlerInfo("/replication", ReplicationHandler.class,null));
Modified: lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java?rev=1643662&r1=1643661&r2=1643662&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java (original)
+++ lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java Sun Dec 7 11:37:32 2014
@@ -52,6 +52,7 @@ import org.apache.solr.request.SolrQuery
import org.apache.solr.request.SolrRequestHandler;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.schema.FieldType;
+import org.apache.solr.schema.ManagedIndexSchema;
import org.apache.solr.schema.SchemaManager;
import org.apache.solr.util.CommandOperation;
import org.apache.solr.util.plugin.SolrCoreAware;
@@ -104,22 +105,33 @@ public class SolrConfigHandler extends R
private static Runnable getListener(SolrCore core, ZkSolrResourceLoader zkSolrResourceLoader) {
final String coreName = core.getName();
final CoreContainer cc = core.getCoreDescriptor().getCoreContainer();
- final String overlayPath = (zkSolrResourceLoader).getConfigSetZkPath() + "/" + ConfigOverlay.RESOURCE_NAME;
- final String solrConfigPath = (zkSolrResourceLoader).getConfigSetZkPath() + "/" + core.getSolrConfig().getName();
+ final String overlayPath = zkSolrResourceLoader.getConfigSetZkPath() + "/" + ConfigOverlay.RESOURCE_NAME;
+ final String solrConfigPath = zkSolrResourceLoader.getConfigSetZkPath() + "/" + core.getSolrConfig().getName();
+ String schemaRes = null;
+ if(core.getLatestSchema().isMutable() && core.getLatestSchema() instanceof ManagedIndexSchema){
+ ManagedIndexSchema mis = (ManagedIndexSchema) core.getLatestSchema();
+ schemaRes = mis.getResourceName();
+ }
+ final String managedSchmaResourcePath = schemaRes ==null ? null: zkSolrResourceLoader.getConfigSetZkPath() + "/" + schemaRes;
return new Runnable() {
@Override
public void run() {
log.info("config update listener called for core {}", coreName);
SolrZkClient zkClient = cc.getZkController().getZkClient();
- int solrConfigversion,overlayVersion;
+ int solrConfigversion,overlayVersion, managedSchemaVersion=0;
try (SolrCore core = cc.getCore(coreName)) {
if (core.isClosed()) return;
solrConfigversion = core.getSolrConfig().getOverlay().getZnodeVersion();
overlayVersion = core.getSolrConfig().getZnodeVersion();
+ if(managedSchmaResourcePath != null){
+ managedSchemaVersion = ((ManagedIndexSchema)core.getLatestSchema()).getSchemaZkVersion();
+ }
+
}
if (checkStale(zkClient, overlayPath, solrConfigversion) ||
- checkStale(zkClient, solrConfigPath, overlayVersion)) {
+ checkStale(zkClient, solrConfigPath, overlayVersion) ||
+ checkStale(zkClient, managedSchmaResourcePath,managedSchemaVersion)) {
log.info("core reload {}",coreName);
cc.reload(coreName);
}
@@ -128,6 +140,7 @@ public class SolrConfigHandler extends R
}
private static boolean checkStale(SolrZkClient zkClient, String zkPath, int currentVersion) {
+ if(zkPath == null) return false;
try {
Stat stat = zkClient.exists(zkPath, null, true);
if(stat == null){
Modified: lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java?rev=1643662&r1=1643661&r2=1643662&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java (original)
+++ lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java Sun Dec 7 11:37:32 2014
@@ -30,7 +30,6 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
-import java.util.regex.Pattern;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.IndexReaderContext;
@@ -839,51 +838,33 @@ public class QueryComponent extends Sear
boolean shardQueryIncludeScore = (rb.getFieldFlags() & SolrIndexSearcher.GET_SCORES) != 0 || rb.getSortSpec().includesScore();
if (distribSinglePass) {
- String fl = rb.req.getParams().get(CommonParams.FL);
- if (fl == null) {
- if (fields.getRequestedFieldNames() == null && fields.wantsAllFields()) {
- fl = "*";
- } else {
- fl = "";
- for (String s : fields.getRequestedFieldNames()) {
- fl += s + ",";
- }
- }
- }
- if (!fields.wantsField(keyFieldName)) {
- // the user has not requested the unique key but
- // we still need to add it otherwise mergeIds can't work
- if (fl.endsWith(",")) {
- fl += keyFieldName;
- } else {
- fl += "," + keyFieldName;
- }
- }
- sreq.params.set(CommonParams.FL, updateFl(fl, shardQueryIncludeScore));
- } else {
- // in this first phase, request only the unique key field and any fields needed for merging.
- if (shardQueryIncludeScore) {
- sreq.params.set(CommonParams.FL, keyFieldName + ",score");
+ String[] fls = rb.req.getParams().getParams(CommonParams.FL);
+ if (fls != null && fls.length > 0 && (fls.length != 1 || !fls[0].isEmpty())) {
+ // If the outer request contains actual FL's use them...
+ sreq.params.set(CommonParams.FL, fls);
} else {
- sreq.params.set(CommonParams.FL, keyFieldName);
+ // ... else we need to explicitly ask for all fields, because we are going to add
+ // additional fields below
+ sreq.params.set(CommonParams.FL, "*");
}
}
+ StringBuilder additionalFL = new StringBuilder();
+ boolean additionalAdded = false;
+ if (!distribSinglePass || !fields.wantsField(keyFieldName))
+ additionalAdded = addFL(additionalFL, keyFieldName, additionalAdded);
+ if ((!distribSinglePass || !fields.wantsScore()) && shardQueryIncludeScore)
+ additionalAdded = addFL(additionalFL, "score", additionalAdded);
+ if (additionalAdded) sreq.params.add(CommonParams.FL, additionalFL.toString());
rb.addRequest(this, sreq);
}
-
-
- String updateFl(String originalFields, boolean includeScoreIfMissing) {
- if (includeScoreIfMissing && !scorePattern.matcher(originalFields).find()) {
- return originalFields + ",score";
- } else {
- return originalFields;
- }
+
+ private boolean addFL(StringBuilder fl, String field, boolean additionalAdded) {
+ if (additionalAdded) fl.append(",");
+ fl.append(field);
+ return true;
}
- private static final Pattern scorePattern = Pattern.compile("\\bscore\\b");
-
-
private void mergeIds(ResponseBuilder rb, ShardRequest sreq) {
List<MergeStrategy> mergeStrategies = rb.getMergeStrategies();
if(mergeStrategies != null) {
@@ -1257,6 +1238,10 @@ public class QueryComponent extends Sear
if (sdoc != null) {
if (returnScores) {
doc.setField("score", sdoc.score);
+ } else {
+ // Score might have been added (in createMainQuery) to shard-requests (and therefore in shard-response-docs)
+ // Remove score if the outer request did not ask for it returned
+ doc.remove("score");
}
if (removeKeyField) {
doc.removeFields(keyFieldName);
Modified: lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/rest/BaseSolrResource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/rest/BaseSolrResource.java?rev=1643662&r1=1643661&r2=1643662&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/rest/BaseSolrResource.java (original)
+++ lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/rest/BaseSolrResource.java Sun Dec 7 11:37:32 2014
@@ -116,8 +116,7 @@ public abstract class BaseSolrResource e
responseWriter = solrCore.getQueryResponseWriter(responseWriterName);
contentType = responseWriter.getContentType(solrRequest, solrResponse);
final String path = getRequest().getRootRef().getPath();
- if ( ! RestManager.SCHEMA_BASE_PATH.equals(path)
- && ! RestManager.CONFIG_BASE_PATH.equals(path)) {
+ if ( ! RestManager.SCHEMA_BASE_PATH.equals(path)) {
// don't set webapp property on the request when context and core/collection are excluded
final int cutoffPoint = path.indexOf("/", 1);
final String firstPathElement = -1 == cutoffPoint ? path : path.substring(0, cutoffPoint);
Modified: lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/rest/RestManager.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/rest/RestManager.java?rev=1643662&r1=1643661&r2=1643662&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/rest/RestManager.java (original)
+++ lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/rest/RestManager.java Sun Dec 7 11:37:32 2014
@@ -60,7 +60,6 @@ public class RestManager {
public static final Logger log = LoggerFactory.getLogger(RestManager.class);
public static final String SCHEMA_BASE_PATH = "/schema";
- public static final String CONFIG_BASE_PATH = "/config";
public static final String MANAGED_ENDPOINT = "/managed";
// used for validating resourceIds provided during registration
@@ -118,15 +117,12 @@ public class RestManager {
private final Pattern reservedEndpointsPattern;
public Registry() {
- reservedEndpoints.add(CONFIG_BASE_PATH + MANAGED_ENDPOINT);
reservedEndpoints.add(SCHEMA_BASE_PATH + MANAGED_ENDPOINT);
for (String reservedEndpoint : SolrSchemaRestApi.getReservedEndpoints()) {
reservedEndpoints.add(reservedEndpoint);
}
- for (String reservedEndpoint : SolrConfigRestApi.getReservedEndpoints()) {
- reservedEndpoints.add(reservedEndpoint);
- }
+
reservedEndpointsPattern = getReservedEndpointsPattern();
}
@@ -192,8 +188,8 @@ public class RestManager {
Matcher resourceIdValidator = resourceIdRegex.matcher(resourceId);
if (!resourceIdValidator.matches()) {
String errMsg = String.format(Locale.ROOT,
- "Invalid resourceId '%s'; must start with %s or %s.",
- resourceId, CONFIG_BASE_PATH, SCHEMA_BASE_PATH);
+ "Invalid resourceId '%s'; must start with %s.",
+ resourceId, SCHEMA_BASE_PATH);
throw new SolrException(ErrorCode.SERVER_ERROR, errMsg);
}
@@ -603,7 +599,6 @@ public class RestManager {
endpoint = new RestManagerManagedResource(this);
endpoint.loadManagedDataAndNotify(null); // no observers for my endpoint
// responds to requests to /config/managed and /schema/managed
- managed.put(CONFIG_BASE_PATH+MANAGED_ENDPOINT, endpoint);
managed.put(SCHEMA_BASE_PATH+MANAGED_ENDPOINT, endpoint);
// init registered managed resources
@@ -734,10 +729,7 @@ public class RestManager {
* @param router - Restlet Router
*/
public synchronized void attachManagedResources(String routerPath, Router router) {
-
- if (CONFIG_BASE_PATH.equals(routerPath)) {
- this.configRouter = router;
- } else if (SCHEMA_BASE_PATH.equals(routerPath)) {
+ if (SCHEMA_BASE_PATH.equals(routerPath)) {
this.schemaRouter = router;
} else {
throw new SolrException(ErrorCode.SERVER_ERROR,