You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2012/07/31 22:59:01 UTC
svn commit: r1367777 [13/14] - in /lucene/dev/branches/pforcodec_3892: ./
dev-tools/ dev-tools/eclipse/ dev-tools/maven/ dev-tools/scripts/ lucene/
lucene/analysis/ lucene/analysis/common/
lucene/analysis/common/src/java/org/apache/lucene/analysis/ar/ ...
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java Tue Jul 31 20:58:32 2012
@@ -19,6 +19,7 @@ package org.apache.solr.handler.admin;
import java.io.File;
import java.io.IOException;
+import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
@@ -567,13 +568,19 @@ public class CoreAdminHandler extends Re
String cname = params.get(CoreAdminParams.CORE);
boolean doPersist = false;
NamedList<Object> status = new SimpleOrderedMap<Object>();
+ Map<String,Exception> allFailures = coreContainer.getCoreInitFailures();
try {
if (cname == null) {
rsp.add("defaultCoreName", coreContainer.getDefaultCoreName());
for (String name : coreContainer.getCoreNames()) {
status.add(name, getCoreStatus(coreContainer, name));
}
+ rsp.add("initFailures", allFailures);
} else {
+ Map failures = allFailures.containsKey(cname)
+ ? Collections.singletonMap(cname, allFailures.get(cname))
+ : Collections.emptyMap();
+ rsp.add("initFailures", failures);
status.add(cname, getCoreStatus(coreContainer, cname));
}
rsp.add("status", status);
@@ -658,7 +665,7 @@ public class CoreAdminHandler extends Re
protected void handleRequestRecoveryAction(SolrQueryRequest req,
SolrQueryResponse rsp) throws IOException {
final SolrParams params = req.getParams();
- log.info("The leader requested that we recover");
+ log.info("It has been requested that we recover");
String cname = params.get(CoreAdminParams.CORE);
if (cname == null) {
cname = "";
@@ -667,6 +674,15 @@ public class CoreAdminHandler extends Re
try {
core = coreContainer.getCore(cname);
if (core != null) {
+ // try to publish as recovering right away
+ try {
+ coreContainer.getZkController().publish(core.getCoreDescriptor(), ZkStateReader.RECOVERING);
+ } catch (KeeperException e) {
+ SolrException.log(log, "", e);
+ } catch (InterruptedException e) {
+ SolrException.log(log, "", e);
+ }
+
core.getUpdateHandler().getSolrCoreState().doRecovery(coreContainer, cname);
} else {
SolrException.log(log, "Cound not find core to call recovery:" + cname);
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java Tue Jul 31 20:58:32 2012
@@ -390,7 +390,7 @@ public class LukeRequestHandler extends
docsEnum = reader.termDocsEnum(reader.getLiveDocs(),
term.field(),
new BytesRef(term.text()),
- false);
+ 0);
if (docsEnum != null) {
int docId;
if ((docId = docsEnum.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/handler/admin/ShowFileRequestHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/handler/admin/ShowFileRequestHandler.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/handler/admin/ShowFileRequestHandler.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/handler/admin/ShowFileRequestHandler.java Tue Jul 31 20:58:32 2012
@@ -144,6 +144,9 @@ public class ShowFileRequestHandler exte
if (fname.indexOf("..") >= 0) {
throw new SolrException(ErrorCode.FORBIDDEN, "Invalid path: " + fname);
}
+ if (fname.startsWith("/")) { // Only files relative to conf are valid
+ fname = fname.substring(1);
+ }
adminFile = confPath + "/" + fname;
}
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/handler/component/QueryElevationComponent.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/handler/component/QueryElevationComponent.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/handler/component/QueryElevationComponent.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/handler/component/QueryElevationComponent.java Tue Jul 31 20:58:32 2012
@@ -543,7 +543,7 @@ public class QueryElevationComponent ext
for (String id : elevations.ids) {
term.copyChars(id);
if (seen.contains(id) == false && termsEnum.seekExact(term, false)) {
- docsEnum = termsEnum.docs(liveDocs, docsEnum, false);
+ docsEnum = termsEnum.docs(liveDocs, docsEnum, 0);
if (docsEnum != null) {
int docId = docsEnum.nextDoc();
if (docId == DocIdSetIterator.NO_MORE_DOCS ) continue; // must have been deleted
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/handler/component/TermVectorComponent.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/handler/component/TermVectorComponent.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/handler/component/TermVectorComponent.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/handler/component/TermVectorComponent.java Tue Jul 31 20:58:32 2012
@@ -329,28 +329,19 @@ public class TermVectorComponent extends
termInfo.add("tf", freq);
}
- dpEnum = termsEnum.docsAndPositions(null, dpEnum, fieldOptions.offsets);
- boolean useOffsets = fieldOptions.offsets;
- if (dpEnum == null) {
- useOffsets = false;
- dpEnum = termsEnum.docsAndPositions(null, dpEnum, false);
- }
-
+ dpEnum = termsEnum.docsAndPositions(null, dpEnum);
+ boolean useOffsets = false;
boolean usePositions = false;
if (dpEnum != null) {
dpEnum.nextDoc();
usePositions = fieldOptions.positions;
- }
-
- NamedList<Number> theOffsets = null;
- if (useOffsets) {
- theOffsets = new NamedList<Number>();
- termInfo.add("offsets", theOffsets);
+ useOffsets = fieldOptions.offsets;
}
NamedList<Integer> positionsNL = null;
+ NamedList<Number> theOffsets = null;
- if (usePositions || theOffsets != null) {
+ if (usePositions || useOffsets) {
for (int i = 0; i < freq; i++) {
final int pos = dpEnum.nextPosition();
if (usePositions && pos >= 0) {
@@ -361,6 +352,15 @@ public class TermVectorComponent extends
positionsNL.add("position", pos);
}
+ if (useOffsets && theOffsets == null) {
+ if (dpEnum.startOffset() == -1) {
+ useOffsets = false;
+ } else {
+ theOffsets = new NamedList<Number>();
+ termInfo.add("offsets", theOffsets);
+ }
+ }
+
if (theOffsets != null) {
theOffsets.add("start", dpEnum.startOffset());
theOffsets.add("end", dpEnum.endOffset());
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/request/SimpleFacets.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/request/SimpleFacets.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/request/SimpleFacets.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/request/SimpleFacets.java Tue Jul 31 20:58:32 2012
@@ -756,7 +756,7 @@ public class SimpleFacets {
// TODO: specialize when base docset is a bitset or hash set (skipDocs)? or does it matter for this?
// TODO: do this per-segment for better efficiency (MultiDocsEnum just uses base class impl)
// TODO: would passing deleted docs lead to better efficiency over checking the fastForRandomSet?
- docsEnum = termsEnum.docs(null, docsEnum, false);
+ docsEnum = termsEnum.docs(null, docsEnum, 0);
c=0;
if (docsEnum instanceof MultiDocsEnum) {
@@ -924,6 +924,11 @@ public class SimpleFacets {
(SolrException.ErrorCode.BAD_REQUEST,
"date facet infinite loop (is gap negative?)");
}
+ if (high.equals(low)) {
+ throw new SolrException
+ (SolrException.ErrorCode.BAD_REQUEST,
+ "date facet infinite loop: gap is effectively zero");
+ }
final boolean includeLower =
(include.contains(FacetRangeInclude.LOWER) ||
(include.contains(FacetRangeInclude.EDGE) && low.equals(start)));
@@ -1113,6 +1118,11 @@ public class SimpleFacets {
(SolrException.ErrorCode.BAD_REQUEST,
"range facet infinite loop (is gap negative? did the math overflow?)");
}
+ if (high.compareTo(low) == 0) {
+ throw new SolrException
+ (SolrException.ErrorCode.BAD_REQUEST,
+ "range facet infinite loop: gap is either zero, or too small relative start/end and caused underflow: " + low + " + " + gap + " = " + high );
+ }
final boolean includeLower =
(include.contains(FacetRangeInclude.LOWER) ||
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/schema/FieldTypePluginLoader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/schema/FieldTypePluginLoader.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/schema/FieldTypePluginLoader.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/schema/FieldTypePluginLoader.java Tue Jul 31 20:58:32 2012
@@ -19,9 +19,9 @@ package org.apache.solr.schema;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.core.KeywordAnalyzer;
+import org.apache.lucene.analysis.core.KeywordTokenizerFactory;
import org.apache.lucene.analysis.util.*;
import org.apache.lucene.util.Version;
-import org.apache.solr.analysis.KeywordTokenizerFactory;
import org.apache.solr.analysis.TokenizerChain;
import org.apache.solr.common.SolrException;
import org.apache.solr.util.DOMUtil;
@@ -73,7 +73,7 @@ public final class FieldTypePluginLoader
@Override
- protected FieldType create( ResourceLoader loader,
+ protected FieldType create( SolrResourceLoader loader,
String name,
String className,
Node node ) throws Exception {
@@ -225,7 +225,30 @@ public final class FieldTypePluginLoader
if (node == null) return null;
NamedNodeMap attrs = node.getAttributes();
String analyzerName = DOMUtil.getAttr(attrs,"class");
+
+ // check for all of these up front, so we can error if used in
+ // conjunction with an explicit analyzer class.
+ NodeList charFilterNodes = (NodeList)xpath.evaluate
+ ("./charFilter", node, XPathConstants.NODESET);
+ NodeList tokenizerNodes = (NodeList)xpath.evaluate
+ ("./tokenizer", node, XPathConstants.NODESET);
+ NodeList tokenFilterNodes = (NodeList)xpath.evaluate
+ ("./filter", node, XPathConstants.NODESET);
+
if (analyzerName != null) {
+
+ // explicitly check for child analysis factories instead of
+ // just any child nodes, because the user might have their
+ // own custom nodes (ie: <description> or something like that)
+ if (0 != charFilterNodes.getLength() ||
+ 0 != tokenizerNodes.getLength() ||
+ 0 != tokenFilterNodes.getLength()) {
+ throw new SolrException
+ ( SolrException.ErrorCode.SERVER_ERROR,
+ "Configuration Error: Analyzer class='" + analyzerName +
+ "' can not be combined with nested analysis factories");
+ }
+
try {
// No need to be core-aware as Analyzers are not in the core-aware list
final Class<? extends Analyzer> clazz = loader.findClass(analyzerName, Analyzer.class);
@@ -286,8 +309,7 @@ public final class FieldTypePluginLoader
}
};
- charFilterLoader.load( loader, (NodeList)xpath.evaluate("./charFilter", node, XPathConstants.NODESET) );
-
+ charFilterLoader.load( loader, charFilterNodes );
// Load the Tokenizer
// Although an analyzer only allows a single Tokenizer, we load a list to make sure
@@ -319,13 +341,12 @@ public final class FieldTypePluginLoader
}
};
- tokenizerLoader.load( loader, (NodeList)xpath.evaluate("./tokenizer", node, XPathConstants.NODESET) );
+ tokenizerLoader.load( loader, tokenizerNodes );
// Make sure something was loaded
if( tokenizers.isEmpty() ) {
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,"analyzer without class or tokenizer & filter list");
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,"analyzer without class or tokenizer");
}
-
// Load the Filters
@@ -353,7 +374,7 @@ public final class FieldTypePluginLoader
return null; // used for map registration
}
};
- filterLoader.load( loader, (NodeList)xpath.evaluate("./filter", node, XPathConstants.NODESET) );
+ filterLoader.load( loader, tokenFilterNodes );
return new TokenizerChain(charFilters.toArray(new CharFilterFactory[charFilters.size()]),
tokenizers.get(0), filters.toArray(new TokenFilterFactory[filters.size()]));
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/schema/IndexSchema.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/schema/IndexSchema.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/schema/IndexSchema.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/schema/IndexSchema.java Tue Jul 31 20:58:32 2012
@@ -40,6 +40,7 @@ import javax.xml.xpath.XPathConstants;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -104,12 +105,16 @@ public final class IndexSchema {
name = DEFAULT_SCHEMA_FILE;
this.resourceName = name;
loader = solrConfig.getResourceLoader();
- if (is == null) {
- is = new InputSource(loader.openSchema(name));
- is.setSystemId(SystemIdResolver.createSystemIdFromResourceName(name));
+ try {
+ if (is == null) {
+ is = new InputSource(loader.openSchema(name));
+ is.setSystemId(SystemIdResolver.createSystemIdFromResourceName(name));
+ }
+ readSchema(is);
+ loader.inform( loader );
+ } catch (IOException e) {
+ throw new RuntimeException(e);
}
- readSchema(is);
- loader.inform( loader );
}
/**
@@ -494,7 +499,10 @@ public final class IndexSchema {
log.error("uniqueKey is not stored - distributed search will not work");
}
if (uniqueKeyField.multiValued()) {
- log.error("uniqueKey should not be multivalued");
+ String msg = "uniqueKey field ("+uniqueKeyFieldName+
+ ") can not be configured to be multivalued";
+ log.error(msg);
+ throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, msg );
}
uniqueKeyFieldName=uniqueKeyField.getName();
uniqueKeyFieldType=uniqueKeyField.getType();
@@ -693,7 +701,7 @@ public final class IndexSchema {
return newArr;
}
- static SimilarityFactory readSimilarity(ResourceLoader loader, Node node) {
+ static SimilarityFactory readSimilarity(SolrResourceLoader loader, Node node) {
if (node==null) {
return null;
} else {
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/search/ExtendedDismaxQParserPlugin.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/search/ExtendedDismaxQParserPlugin.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/search/ExtendedDismaxQParserPlugin.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/search/ExtendedDismaxQParserPlugin.java Tue Jul 31 20:58:32 2012
@@ -27,6 +27,7 @@ import java.util.Map;
import java.util.Set;
import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.core.StopFilterFactory;
import org.apache.lucene.analysis.util.TokenFilterFactory;
import org.apache.lucene.queries.function.BoostedQuery;
import org.apache.lucene.queries.function.FunctionQuery;
@@ -36,7 +37,6 @@ import org.apache.lucene.queries.functio
import org.apache.lucene.queryparser.classic.ParseException;
import org.apache.lucene.queryparser.classic.QueryParser;
import org.apache.lucene.search.*;
-import org.apache.solr.analysis.StopFilterFactory;
import org.apache.solr.analysis.TokenizerChain;
import org.apache.solr.search.SolrQueryParser.MagicFieldName;
import org.apache.solr.common.params.DisMaxParams;
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java Tue Jul 31 20:58:32 2012
@@ -341,7 +341,7 @@ class JoinQuery extends Query {
if (freq < minDocFreqFrom) {
fromTermDirectCount++;
// OK to skip liveDocs, since we check for intersection with docs matching query
- fromDeState.docsEnum = fromDeState.termsEnum.docs(null, fromDeState.docsEnum, false);
+ fromDeState.docsEnum = fromDeState.termsEnum.docs(null, fromDeState.docsEnum, 0);
DocsEnum docsEnum = fromDeState.docsEnum;
if (docsEnum instanceof MultiDocsEnum) {
@@ -406,7 +406,7 @@ class JoinQuery extends Query {
toTermDirectCount++;
// need to use liveDocs here so we don't map to any deleted ones
- toDeState.docsEnum = toDeState.termsEnum.docs(toDeState.liveDocs, toDeState.docsEnum, false);
+ toDeState.docsEnum = toDeState.termsEnum.docs(toDeState.liveDocs, toDeState.docsEnum, 0);
DocsEnum docsEnum = toDeState.docsEnum;
if (docsEnum instanceof MultiDocsEnum) {
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java Tue Jul 31 20:58:32 2012
@@ -598,7 +598,7 @@ public class SolrIndexSearcher extends I
if (!termsEnum.seekExact(termBytes, false)) {
return -1;
}
- DocsEnum docs = termsEnum.docs(atomicReader.getLiveDocs(), null, false);
+ DocsEnum docs = termsEnum.docs(atomicReader.getLiveDocs(), null, 0);
if (docs == null) return -1;
int id = docs.nextDoc();
return id == DocIdSetIterator.NO_MORE_DOCS ? -1 : id;
@@ -620,7 +620,7 @@ public class SolrIndexSearcher extends I
final Bits liveDocs = reader.getLiveDocs();
- final DocsEnum docs = reader.termDocsEnum(liveDocs, field, idBytes, false);
+ final DocsEnum docs = reader.termDocsEnum(liveDocs, field, idBytes, 0);
if (docs == null) continue;
int id = docs.nextDoc();
@@ -925,7 +925,7 @@ public class SolrIndexSearcher extends I
int bitsSet = 0;
OpenBitSet obs = null;
- DocsEnum docsEnum = deState.termsEnum.docs(deState.liveDocs, deState.docsEnum, false);
+ DocsEnum docsEnum = deState.termsEnum.docs(deState.liveDocs, deState.docsEnum, 0);
if (deState.docsEnum == null) {
deState.docsEnum = docsEnum;
}
@@ -1003,7 +1003,7 @@ public class SolrIndexSearcher extends I
if (terms != null) {
final TermsEnum termsEnum = terms.iterator(null);
if (termsEnum.seekExact(termBytes, false)) {
- docsEnum = termsEnum.docs(liveDocs, null, false);
+ docsEnum = termsEnum.docs(liveDocs, null, 0);
}
}
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/search/function/FileFloatSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/search/function/FileFloatSource.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/search/function/FileFloatSource.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/search/function/FileFloatSource.java Tue Jul 31 20:58:32 2012
@@ -278,7 +278,7 @@ public class FileFloatSource extends Val
continue;
}
- docsEnum = termsEnum.docs(null, docsEnum, false);
+ docsEnum = termsEnum.docs(null, docsEnum, 0);
int doc;
while ((doc = docsEnum.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
vals[doc] = fval;
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/spelling/ConjunctionSolrSpellChecker.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/spelling/ConjunctionSolrSpellChecker.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/spelling/ConjunctionSolrSpellChecker.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/spelling/ConjunctionSolrSpellChecker.java Tue Jul 31 20:58:32 2012
@@ -106,7 +106,7 @@ public class ConjunctionSolrSpellChecker
}
@Override
- public void build(SolrCore core, SolrIndexSearcher searcher) {
+ public void build(SolrCore core, SolrIndexSearcher searcher) throws IOException {
for (SolrSpellChecker c : checkers) {
c.build(core, searcher);
}
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/spelling/DirectSolrSpellChecker.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/spelling/DirectSolrSpellChecker.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/spelling/DirectSolrSpellChecker.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/spelling/DirectSolrSpellChecker.java Tue Jul 31 20:58:32 2012
@@ -168,11 +168,10 @@ public class DirectSolrSpellChecker exte
}
@Override
- public void reload(SolrCore core, SolrIndexSearcher searcher)
- throws IOException {}
+ public void reload(SolrCore core, SolrIndexSearcher searcher) throws IOException {}
@Override
- public void build(SolrCore core, SolrIndexSearcher searcher) {}
+ public void build(SolrCore core, SolrIndexSearcher searcher) throws IOException {}
@Override
public SpellingResult getSuggestions(SpellingOptions options)
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/spelling/FileBasedSpellChecker.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/spelling/FileBasedSpellChecker.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/spelling/FileBasedSpellChecker.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/spelling/FileBasedSpellChecker.java Tue Jul 31 20:58:32 2012
@@ -59,17 +59,13 @@ public class FileBasedSpellChecker exten
}
@Override
- public void build(SolrCore core, SolrIndexSearcher searcher) {
- try {
- loadExternalFileDictionary(core);
- spellChecker.clearIndex();
- // TODO: you should be able to specify the IWC params?
- // TODO: if we enable this, codec gets angry since field won't exist in the schema
- // config.setCodec(core.getCodec());
- spellChecker.indexDictionary(dictionary, new IndexWriterConfig(core.getSolrConfig().luceneMatchVersion, null), false);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
+ public void build(SolrCore core, SolrIndexSearcher searcher) throws IOException {
+ loadExternalFileDictionary(core);
+ spellChecker.clearIndex();
+ // TODO: you should be able to specify the IWC params?
+ // TODO: if we enable this, codec gets angry since field won't exist in the schema
+ // config.setCodec(core.getCodec());
+ spellChecker.indexDictionary(dictionary, new IndexWriterConfig(core.getSolrConfig().luceneMatchVersion, null), false);
}
/**
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/spelling/IndexBasedSpellChecker.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/spelling/IndexBasedSpellChecker.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/spelling/IndexBasedSpellChecker.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/spelling/IndexBasedSpellChecker.java Tue Jul 31 20:58:32 2012
@@ -73,31 +73,27 @@ public class IndexBasedSpellChecker exte
}
@Override
- public void build(SolrCore core, SolrIndexSearcher searcher) {
+ public void build(SolrCore core, SolrIndexSearcher searcher) throws IOException {
IndexReader reader = null;
- try {
- if (sourceLocation == null) {
- // Load from Solr's index
- reader = searcher.getIndexReader();
- } else {
- // Load from Lucene index at given sourceLocation
- reader = this.reader;
- }
-
- // Create the dictionary
- dictionary = new HighFrequencyDictionary(reader, field,
- threshold);
- // TODO: maybe whether or not to clear the index should be configurable?
- // an incremental update is faster (just adds new terms), but if you 'expunged'
- // old terms I think they might hang around.
- spellChecker.clearIndex();
- // TODO: you should be able to specify the IWC params?
- // TODO: if we enable this, codec gets angry since field won't exist in the schema
- // config.setCodec(core.getCodec());
- spellChecker.indexDictionary(dictionary, new IndexWriterConfig(core.getSolrConfig().luceneMatchVersion, null), false);
- } catch (IOException e) {
- throw new RuntimeException(e);
+ if (sourceLocation == null) {
+ // Load from Solr's index
+ reader = searcher.getIndexReader();
+ } else {
+ // Load from Lucene index at given sourceLocation
+ reader = this.reader;
}
+
+ // Create the dictionary
+ dictionary = new HighFrequencyDictionary(reader, field,
+ threshold);
+ // TODO: maybe whether or not to clear the index should be configurable?
+ // an incremental update is faster (just adds new terms), but if you 'expunged'
+ // old terms I think they might hang around.
+ spellChecker.clearIndex();
+ // TODO: you should be able to specify the IWC params?
+ // TODO: if we enable this, codec gets angry since field won't exist in the schema
+ // config.setCodec(core.getCodec());
+ spellChecker.indexDictionary(dictionary, new IndexWriterConfig(core.getSolrConfig().luceneMatchVersion, null), false);
}
@Override
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/spelling/SolrSpellChecker.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/spelling/SolrSpellChecker.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/spelling/SolrSpellChecker.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/spelling/SolrSpellChecker.java Tue Jul 31 20:58:32 2012
@@ -167,7 +167,7 @@ public abstract class SolrSpellChecker {
/**
* (re)Builds the spelling index. May be a NOOP if the implementation doesn't require building, or can't be rebuilt.
*/
- public abstract void build(SolrCore core, SolrIndexSearcher searcher);
+ public abstract void build(SolrCore core, SolrIndexSearcher searcher) throws IOException;
/**
* Get the value of {@link SpellingParams#SPELLCHECK_ACCURACY} if supported.
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/spelling/suggest/Suggester.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/spelling/suggest/Suggester.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/spelling/suggest/Suggester.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/spelling/suggest/Suggester.java Tue Jul 31 20:58:32 2012
@@ -35,6 +35,7 @@ import org.apache.lucene.search.suggest.
import org.apache.lucene.search.suggest.Lookup;
import org.apache.lucene.search.suggest.Lookup.LookupResult;
import org.apache.lucene.util.CharsRef;
+import org.apache.lucene.util.IOUtils;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.SolrCore;
@@ -121,7 +122,7 @@ public class Suggester extends SolrSpell
}
@Override
- public void build(SolrCore core, SolrIndexSearcher searcher) {
+ public void build(SolrCore core, SolrIndexSearcher searcher) throws IOException {
LOG.info("build()");
if (sourceLocation == null) {
reader = searcher.getIndexReader();
@@ -129,30 +130,26 @@ public class Suggester extends SolrSpell
} else {
try {
dictionary = new FileDictionary(new InputStreamReader(
- core.getResourceLoader().openResource(sourceLocation), "UTF-8"));
+ core.getResourceLoader().openResource(sourceLocation), IOUtils.CHARSET_UTF_8));
} catch (UnsupportedEncodingException e) {
// should not happen
LOG.error("should not happen", e);
}
}
- try {
- lookup.build(dictionary);
- if (storeDir != null) {
- File target = new File(storeDir, factory.storeFileName());
- if(!lookup.store(new FileOutputStream(target))) {
- if (sourceLocation == null) {
- assert reader != null && field != null;
- LOG.error("Store Lookup build from index on field: " + field + " failed reader has: " + reader.maxDoc() + " docs");
- } else {
- LOG.error("Store Lookup build from sourceloaction: " + sourceLocation + " failed");
- }
+
+ lookup.build(dictionary);
+ if (storeDir != null) {
+ File target = new File(storeDir, factory.storeFileName());
+ if(!lookup.store(new FileOutputStream(target))) {
+ if (sourceLocation == null) {
+ assert reader != null && field != null;
+ LOG.error("Store Lookup build from index on field: " + field + " failed reader has: " + reader.maxDoc() + " docs");
} else {
- LOG.info("Stored suggest data to: " + target.getAbsolutePath());
+ LOG.error("Store Lookup build from sourceloaction: " + sourceLocation + " failed");
}
+ } else {
+ LOG.info("Stored suggest data to: " + target.getAbsolutePath());
}
-
- } catch (Exception e) {
- LOG.error("Error while building or storing Suggester data", e);
}
}
@@ -161,8 +158,13 @@ public class Suggester extends SolrSpell
LOG.info("reload()");
if (dictionary == null && storeDir != null) {
// this may be a firstSearcher event, try loading it
- if (lookup.load(new FileInputStream(new File(storeDir, factory.storeFileName())))) {
- return; // loaded ok
+ FileInputStream is = new FileInputStream(new File(storeDir, factory.storeFileName()));
+ try {
+ if (lookup.load(is)) {
+ return; // loaded ok
+ }
+ } finally {
+ IOUtils.closeWhileHandlingException(is);
}
LOG.debug("load failed, need to build Lookup again");
}
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/CommitTracker.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/CommitTracker.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/CommitTracker.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/CommitTracker.java Tue Jul 31 20:58:32 2012
@@ -39,8 +39,10 @@ import org.slf4j.LoggerFactory;
* definitely change in the future, so the interface should not be relied-upon
*
* Note: all access must be synchronized.
+ *
+ * Public for tests.
*/
-final class CommitTracker implements Runnable {
+public final class CommitTracker implements Runnable {
protected final static Logger log = LoggerFactory.getLogger(CommitTracker.class);
// scheduler delay for maxDoc-triggered autocommits
@@ -248,7 +250,8 @@ final class CommitTracker implements Run
this.docsUpperBound = docsUpperBound;
}
- void setTimeUpperBound(long timeUpperBound) {
+ // only for testing - not thread safe
+ public void setTimeUpperBound(long timeUpperBound) {
this.timeUpperBound = timeUpperBound;
}
}
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java Tue Jul 31 20:58:32 2012
@@ -35,6 +35,10 @@ public final class DefaultSolrCoreState
private final boolean SKIP_AUTO_RECOVERY = Boolean.getBoolean("solrcloud.skip.autorecovery");
private final Object recoveryLock = new Object();
+
+ // protects pauseWriter and writerFree
+ private final Object writerPauseLock = new Object();
+
private int refCnt = 1;
private SolrIndexWriter indexWriter = null;
private DirectoryFactory directoryFactory;
@@ -55,68 +59,74 @@ public final class DefaultSolrCoreState
@Override
public synchronized RefCounted<IndexWriter> getIndexWriter(SolrCore core)
throws IOException {
-
- if (core == null) {
- // core == null is a signal to just return the current writer, or null if none.
- if (refCntWriter != null) refCntWriter.incref();
- return refCntWriter;
- }
-
- while (pauseWriter) {
- try {
- wait();
- } catch (InterruptedException e) {}
- }
-
- if (indexWriter == null) {
- indexWriter = createMainIndexWriter(core, "DirectUpdateHandler2", false,
- false);
- }
- if (refCntWriter == null) {
- refCntWriter = new RefCounted<IndexWriter>(indexWriter) {
- @Override
- public void close() {
- synchronized (DefaultSolrCoreState.this) {
- writerFree = true;
- DefaultSolrCoreState.this.notifyAll();
+ synchronized (writerPauseLock) {
+ if (core == null) {
+ // core == null is a signal to just return the current writer, or null
+ // if none.
+ if (refCntWriter != null) refCntWriter.incref();
+ return refCntWriter;
+ }
+
+ while (pauseWriter) {
+ try {
+ writerPauseLock.wait();
+ } catch (InterruptedException e) {}
+ }
+
+ if (indexWriter == null) {
+ indexWriter = createMainIndexWriter(core, "DirectUpdateHandler2",
+ false, false);
+ }
+ if (refCntWriter == null) {
+ refCntWriter = new RefCounted<IndexWriter>(indexWriter) {
+ @Override
+ public void close() {
+ synchronized (writerPauseLock) {
+ writerFree = true;
+ writerPauseLock.notifyAll();
+ }
}
- }
- };
+ };
+ }
+ writerFree = false;
+ writerPauseLock.notifyAll();
+ refCntWriter.incref();
+ return refCntWriter;
}
- writerFree = false;
- notifyAll();
- refCntWriter.incref();
- return refCntWriter;
}
@Override
- public synchronized void newIndexWriter(SolrCore core) throws IOException {
- // we need to wait for the Writer to fall out of use
- // first lets stop it from being lent out
- pauseWriter = true;
- // then lets wait until its out of use
- while(!writerFree) {
- try {
- wait();
- } catch (InterruptedException e) {}
- }
+ public synchronized void newIndexWriter(SolrCore core, boolean rollback) throws IOException {
- try {
- if (indexWriter != null) {
+ synchronized (writerPauseLock) {
+ // we need to wait for the Writer to fall out of use
+ // first lets stop it from being lent out
+ pauseWriter = true;
+ // then lets wait until its out of use
+ while (!writerFree) {
try {
- indexWriter.close();
- } catch (Throwable t) {
- SolrException.log(log, "Error closing old IndexWriter", t);
- }
+ writerPauseLock.wait();
+ } catch (InterruptedException e) {}
}
- indexWriter = createMainIndexWriter(core, "DirectUpdateHandler2", false,
- true);
- // we need to null this so it picks up the new writer next get call
- refCntWriter = null;
- } finally {
- pauseWriter = false;
- notifyAll();
+ try {
+ if (indexWriter != null) {
+ try {
+ indexWriter.close();
+ } catch (Throwable t) {
+ SolrException.log(log, "Error closing old IndexWriter", t);
+ }
+ }
+
+ indexWriter = createMainIndexWriter(core, "DirectUpdateHandler2",
+ false, true);
+ // we need to null this so it picks up the new writer next get call
+ refCntWriter = null;
+ } finally {
+
+ pauseWriter = false;
+ writerPauseLock.notifyAll();
+ }
}
}
@@ -161,7 +171,7 @@ public final class DefaultSolrCoreState
@Override
public synchronized void rollbackIndexWriter(SolrCore core) throws IOException {
indexWriter.rollback();
- newIndexWriter(core);
+ newIndexWriter(core, true);
}
protected SolrIndexWriter createMainIndexWriter(SolrCore core, String name,
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java Tue Jul 31 20:58:32 2012
@@ -343,7 +343,7 @@ public class DirectUpdateHandler2 extend
// currently for testing purposes. Do a delete of complete index w/o worrying about versions, don't log, clean up most state in update log, etc
if (delAll && cmd.getVersion() == -Long.MAX_VALUE) {
- synchronized (solrCoreState) {
+ synchronized (solrCoreState.getUpdateLock()) {
deleteAll();
ulog.deleteAll();
return;
@@ -356,7 +356,7 @@ public class DirectUpdateHandler2 extend
// a realtime view of the index. When a new searcher is opened after a DBQ, that
// flag can be cleared. If those thing happen concurrently, it's not thread safe.
//
- synchronized (solrCoreState) {
+ synchronized (solrCoreState.getUpdateLock()) {
if (delAll) {
deleteAll();
} else {
@@ -392,7 +392,7 @@ public class DirectUpdateHandler2 extend
Term idTerm = new Term(idField.getName(), cmd.getIndexedId());
// see comment in deleteByQuery
- synchronized (solrCoreState) {
+ synchronized (solrCoreState.getUpdateLock()) {
RefCounted<IndexWriter> iw = solrCoreState.getIndexWriter(core);
try {
IndexWriter writer = iw.get();
@@ -518,7 +518,7 @@ public class DirectUpdateHandler2 extend
}
if (!cmd.softCommit) {
- synchronized (solrCoreState) { // sync is currently needed to prevent preCommit
+ synchronized (solrCoreState.getUpdateLock()) { // sync is currently needed to prevent preCommit
// from being called between preSoft and
// postSoft... see postSoft comments.
if (ulog != null) ulog.preCommit(cmd);
@@ -547,14 +547,14 @@ public class DirectUpdateHandler2 extend
if (cmd.softCommit) {
// ulog.preSoftCommit();
- synchronized (solrCoreState) {
+ synchronized (solrCoreState.getUpdateLock()) {
if (ulog != null) ulog.preSoftCommit(cmd);
core.getSearcher(true, false, waitSearcher, true);
if (ulog != null) ulog.postSoftCommit(cmd);
}
// ulog.postSoftCommit();
} else {
- synchronized (solrCoreState) {
+ synchronized (solrCoreState.getUpdateLock()) {
if (ulog != null) ulog.preSoftCommit(cmd);
if (cmd.openSearcher) {
core.getSearcher(true, false, waitSearcher);
@@ -606,8 +606,8 @@ public class DirectUpdateHandler2 extend
}
@Override
- public void newIndexWriter() throws IOException {
- solrCoreState.newIndexWriter(core);
+ public void newIndexWriter(boolean rollback) throws IOException {
+ solrCoreState.newIndexWriter(core, rollback);
}
/**
@@ -705,7 +705,7 @@ public class DirectUpdateHandler2 extend
// TODO: keep other commit callbacks from being called?
// this.commit(cmd); // too many test failures using this method... is it because of callbacks?
- synchronized (solrCoreState) {
+ synchronized (solrCoreState.getUpdateLock()) {
ulog.preCommit(cmd);
}
@@ -714,7 +714,7 @@ public class DirectUpdateHandler2 extend
commitData.put(SolrIndexWriter.COMMIT_TIME_MSEC_KEY, String.valueOf(System.currentTimeMillis()));
writer.commit(commitData);
- synchronized (solrCoreState) {
+ synchronized (solrCoreState.getUpdateLock()) {
ulog.postCommit(cmd);
}
}
@@ -823,4 +823,14 @@ public class DirectUpdateHandler2 extend
public void incref() {
solrCoreState.incref();
}
+
+ // allow access for tests
+ public CommitTracker getCommitTracker() {
+ return commitTracker;
+ }
+
+ // allow access for tests
+ public CommitTracker getSoftCommitTracker() {
+ return softCommitTracker;
+ }
}
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/SolrCoreState.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/SolrCoreState.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/SolrCoreState.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/SolrCoreState.java Tue Jul 31 20:58:32 2012
@@ -31,15 +31,21 @@ import org.apache.solr.util.RefCounted;
*
*/
public abstract class SolrCoreState {
+ private final Object deleteLock = new Object();
+
+ public Object getUpdateLock() {
+ return deleteLock;
+ }
/**
* Force the creation of a new IndexWriter using the settings from the given
* SolrCore.
*
* @param core
+ * @param rollback close IndexWriter if false, else rollback
* @throws IOException
*/
- public abstract void newIndexWriter(SolrCore core) throws IOException;
+ public abstract void newIndexWriter(SolrCore core, boolean rollback) throws IOException;
/**
* Get the current IndexWriter. If a new IndexWriter must be created, use the
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/UpdateCommand.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/UpdateCommand.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/UpdateCommand.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/UpdateCommand.java Tue Jul 31 20:58:32 2012
@@ -44,7 +44,7 @@ public abstract class UpdateCommand impl
@Override
public String toString() {
- return name() + "{flags="+flags+",version="+version;
+ return name() + "{flags="+flags+",_version_="+version;
}
public long getVersion() {
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/UpdateHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/UpdateHandler.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/UpdateHandler.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/UpdateHandler.java Tue Jul 31 20:58:32 2012
@@ -127,9 +127,11 @@ public abstract class UpdateHandler impl
* Called when the Writer should be opened again - eg when replication replaces
* all of the index files.
*
+ * @param rollback IndexWriter if true else close
+ *
* @throws IOException
*/
- public abstract void newIndexWriter() throws IOException;
+ public abstract void newIndexWriter(boolean rollback) throws IOException;
public abstract SolrCoreState getSolrCoreState();
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java Tue Jul 31 20:58:32 2012
@@ -748,7 +748,10 @@ public class DistributedUpdateProcessor
if (zkEnabled && DistribPhase.TOLEADER == phase) {
// This core should be a leader
+ isLeader = true;
replicas = setupRequest();
+ } else if (DistribPhase.FROMLEADER == phase) {
+ isLeader = false;
}
if (vinfo == null) {
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessorFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessorFactory.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessorFactory.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessorFactory.java Tue Jul 31 20:58:32 2012
@@ -21,6 +21,11 @@ import org.apache.solr.common.util.Named
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
+/**
+ * Factory for {@link DistributedUpdateProcessor}.
+ *
+ * @see DistributedUpdateProcessor
+ */
public class DistributedUpdateProcessorFactory
extends UpdateRequestProcessorFactory
implements DistributingUpdateProcessorFactory {
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/processor/DistributingUpdateProcessorFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/processor/DistributingUpdateProcessorFactory.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/processor/DistributingUpdateProcessorFactory.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/processor/DistributingUpdateProcessorFactory.java Tue Jul 31 20:58:32 2012
@@ -23,7 +23,8 @@ import org.apache.solr.common.SolrExcept
* A marker interface for denoting that a factory is responsible for handling
* distributed communication of updates across a SolrCloud cluster.
*
- * @see UpdateRequestProcessorChain
+ * @see UpdateRequestProcessorChain#init
+ * @see UpdateRequestProcessorChain#createProcessor
*/
public interface DistributingUpdateProcessorFactory {
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/processor/LogUpdateProcessorFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/processor/LogUpdateProcessorFactory.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/processor/LogUpdateProcessorFactory.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/processor/LogUpdateProcessorFactory.java Tue Jul 31 20:58:32 2012
@@ -35,12 +35,15 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
+ * <p>
* A logging processor. This keeps track of all commands that have passed through
* the chain and prints them on finish(). At the Debug (FINE) level, a message
* will be logged for each command prior to the next stage in the chain.
- *
- * If the Log level is not >= INFO the processor will not be created or added to the chain.
- *
+ * </p>
+ * <p>
+ * If the Log level is not >= INFO the processor will not be created or added to the chain.
+ * </p>
+ *
* @since solr 1.3
*/
public class LogUpdateProcessorFactory extends UpdateRequestProcessorFactory {
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/processor/RunUpdateProcessorFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/processor/RunUpdateProcessorFactory.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/processor/RunUpdateProcessorFactory.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/processor/RunUpdateProcessorFactory.java Tue Jul 31 20:58:32 2012
@@ -25,9 +25,14 @@ import org.apache.solr.update.*;
/**
- * Pass the command to the UpdateHandler without any modifications
+ * Executes the update commands using the underlying UpdateHandler.
+ * Allmost all processor chains should end with an instance of
+ * <code>RunUpdateProcessorFactory</code> unless the user is explicitly
+ * executing the update commands in an alternative custom
+ * <code>UpdateRequestProcessorFactory</code>
*
* @since solr 1.3
+ * @see DistributingUpdateProcessorFactory
*/
public class RunUpdateProcessorFactory extends UpdateRequestProcessorFactory
{
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/processor/StatelessScriptUpdateProcessorFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/processor/StatelessScriptUpdateProcessorFactory.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/processor/StatelessScriptUpdateProcessorFactory.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/processor/StatelessScriptUpdateProcessorFactory.java Tue Jul 31 20:58:32 2012
@@ -305,16 +305,22 @@ public class StatelessScriptUpdateProces
}
scriptEngines.add(new EngineInfo((Invocable)engine, scriptFile));
- Reader scriptSrc = scriptFile.openReader(resourceLoader);
-
try {
- engine.eval(scriptSrc);
- } catch (ScriptException e) {
+ Reader scriptSrc = scriptFile.openReader(resourceLoader);
+
+ try {
+ engine.eval(scriptSrc);
+ } catch (ScriptException e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+ "Unable to evaluate script: " +
+ scriptFile.getFileName(), e);
+ } finally {
+ IOUtils.closeQuietly(scriptSrc);
+ }
+ } catch (IOException ioe) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
- "Unable to evaluate script: " +
- scriptFile.getFileName(), e);
- } finally {
- IOUtils.closeQuietly(scriptSrc);
+ "Unable to evaluate script: " +
+ scriptFile.getFileName(), ioe);
}
}
return scriptEngines;
@@ -485,7 +491,7 @@ public class StatelessScriptUpdateProces
return extension;
}
- public Reader openReader(SolrResourceLoader resourceLoader) {
+ public Reader openReader(SolrResourceLoader resourceLoader) throws IOException {
InputStream input = resourceLoader.openResource(fileName);
return org.apache.lucene.util.IOUtils.getDecodingReader
(input, org.apache.lucene.util.IOUtils.CHARSET_UTF_8);
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/processor/UpdateRequestProcessorChain.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/processor/UpdateRequestProcessorChain.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/processor/UpdateRequestProcessorChain.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/update/processor/UpdateRequestProcessorChain.java Tue Jul 31 20:58:32 2012
@@ -36,7 +36,7 @@ import java.util.ArrayList;
* <p>
* Chain can be configured via solrconfig.xml:
* </p>
- * <pre>
+ * <pre class="prettyprint">
* <updateRequestProcessors name="key" default="true">
* <processor class="PathToClass1" />
* <processor class="PathToClass2" />
@@ -46,9 +46,16 @@ import java.util.ArrayList;
* <processor class="solr.RunUpdateProcessorFactory" />
* </updateRequestProcessors>
* </pre>
+ * <p>
+ * Allmost all processor chains should end with an instance of
+ * {@link RunUpdateProcessorFactory} unless the user is explicitly
+ * executing the update commands in an alternative custom
+ * <code>UpdateRequestProcessorFactory</code>.
+ * </p>
*
* @see UpdateRequestProcessorFactory
* @see #init
+ * @see #createProcessor
* @since solr 1.3
*/
public final class UpdateRequestProcessorChain implements PluginInfoInitialized
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/util/FileUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/util/FileUtils.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/util/FileUtils.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/util/FileUtils.java Tue Jul 31 20:58:32 2012
@@ -54,7 +54,7 @@ public class FileUtils {
}
/**
- * Copied from Lucene's FSDirectory.fsync(String) <!-- protected -->
+ * Copied from Lucene's FSDirectory.fsync(String)
*
* @param fullFile the File to be synced to disk
* @throws IOException if the file could not be synced
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/util/SimplePostTool.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/util/SimplePostTool.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/util/SimplePostTool.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/util/SimplePostTool.java Tue Jul 31 20:58:32 2012
@@ -18,123 +18,170 @@ package org.apache.solr.util;
*/
import java.io.File;
+import java.io.FileFilter;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.ByteArrayInputStream;
import java.io.OutputStream;
import java.io.UnsupportedEncodingException;
+import java.util.Locale;
+import java.util.HashMap;
import java.util.Set;
import java.util.HashSet;
+import java.util.regex.Pattern;
+import java.util.regex.PatternSyntaxException;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.ProtocolException;
import java.net.URL;
+import java.net.URLEncoder;
/**
* A simple utility class for posting raw updates to a Solr server,
* has a main method so it can be run on the command line.
- *
*/
public class SimplePostTool {
public static final String DEFAULT_POST_URL = "http://localhost:8983/solr/update";
- public static final String VERSION_OF_THIS_TOOL = "1.4";
+ public static final String VERSION_OF_THIS_TOOL = "1.5";
private static final String DEFAULT_COMMIT = "yes";
private static final String DEFAULT_OPTIMIZE = "no";
private static final String DEFAULT_OUT = "no";
+ private static final String DEFAULT_AUTO = "no";
+ private static final String DEFAULT_RECURSIVE = "no";
- public static final String DEFAULT_DATA_TYPE = "application/xml";
+ private static final String DEFAULT_CONTENT_TYPE = "application/xml";
+ private static final String DEFAULT_FILE_TYPES = "xml,json,csv,pdf,doc,docx,ppt,pptx,xls,xlsx,odt,odp,ods,ott,otp,ots,rtf,htm,html,txt,log";
private static final String DATA_MODE_FILES = "files";
private static final String DATA_MODE_ARGS = "args";
private static final String DATA_MODE_STDIN = "stdin";
private static final String DEFAULT_DATA_MODE = DATA_MODE_FILES;
+ private static final String TRUE_STRINGS = "true,on,yes,1";
+
+ private boolean auto = false;
+ private boolean recursive = false;
+ private String fileTypes;
+
+ private static HashMap<String,String> mimeMap;
+ private GlobFileFilter globFileFilter;
+
private static final Set<String> DATA_MODES = new HashSet<String>();
+ private static final String USAGE_STRING_SHORT =
+ "Usage: java [SystemProperties] -jar post.jar [-h|-] [<file|folder|arg> [<file|folder|arg>...]]";
+
static {
DATA_MODES.add(DATA_MODE_FILES);
DATA_MODES.add(DATA_MODE_ARGS);
DATA_MODES.add(DATA_MODE_STDIN);
+
+ mimeMap = new HashMap<String,String>();
+ mimeMap.put("xml", "text/xml");
+ mimeMap.put("csv", "text/csv");
+ mimeMap.put("json", "application/json");
+ mimeMap.put("pdf", "application/pdf");
+ mimeMap.put("rtf", "text/rtf");
+ mimeMap.put("html", "text/html");
+ mimeMap.put("htm", "text/html");
+ mimeMap.put("doc", "application/msword");
+ mimeMap.put("docx", "application/vnd.openxmlformats-officedocument.wordprocessingml.document");
+ mimeMap.put("ppt", "application/vnd.ms-powerpoint");
+ mimeMap.put("pptx", "application/vnd.openxmlformats-officedocument.presentationml.presentation");
+ mimeMap.put("xls", "application/vnd.ms-excel");
+ mimeMap.put("xlsx", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet");
+ mimeMap.put("odt", "application/vnd.oasis.opendocument.text");
+ mimeMap.put("ott", "application/vnd.oasis.opendocument.text");
+ mimeMap.put("odp", "application/vnd.oasis.opendocument.presentation");
+ mimeMap.put("otp", "application/vnd.oasis.opendocument.presentation");
+ mimeMap.put("ods", "application/vnd.oasis.opendocument.spreadsheet");
+ mimeMap.put("ots", "application/vnd.oasis.opendocument.spreadsheet");
+ mimeMap.put("txt", "text/plain");
+ mimeMap.put("log", "text/plain");
}
protected URL solrUrl;
-
+
public static void main(String[] args) {
- info("version " + VERSION_OF_THIS_TOOL);
+ info("SimplePostTool version " + VERSION_OF_THIS_TOOL);
if (0 < args.length && ("-help".equals(args[0]) || "--help".equals(args[0]) || "-h".equals(args[0]))) {
- System.out.println
- ("This is a simple command line tool for POSTing raw data to a Solr\n"+
- "port. Data can be read from files specified as commandline args,\n"+
- "as raw commandline arg strings, or via STDIN.\n"+
- "Examples:\n"+
- " java -jar post.jar *.xml\n"+
- " java -Ddata=args -jar post.jar '<delete><id>42</id></delete>'\n"+
- " java -Ddata=stdin -jar post.jar < hd.xml\n"+
- " java -Dtype=text/csv -jar post.jar *.csv\n"+
- " java -Dtype=application/json -jar post.jar *.json\n"+
- " java -Durl=http://localhost:8983/solr/update/extract?literal.id=a -Dtype=application/pdf -jar post.jar a.pdf\n"+
- "Other options controlled by System Properties include the Solr\n"+
- "URL to POST to, the Content-Type of the data, whether a commit\n"+
- "or optimize should be executed, and whether the response should\n"+
- "be written to STDOUT. These are the defaults for all System Properties:\n"+
- " -Ddata=" + DEFAULT_DATA_MODE + "\n"+
- " -Dtype=" + DEFAULT_DATA_TYPE + "\n"+
- " -Durl=" + DEFAULT_POST_URL + "\n"+
- " -Dcommit=" + DEFAULT_COMMIT + "\n"+
- " -Doptimize=" + DEFAULT_OPTIMIZE + "\n"+
- " -Dout=" + DEFAULT_OUT + "\n");
+ usage();
return;
}
-
+
OutputStream out = null;
- final String type = System.getProperty("type", DEFAULT_DATA_TYPE);
+ final String type = System.getProperty("type");
+
+ final String params = System.getProperty("params", "");
URL u = null;
try {
- u = new URL(System.getProperty("url", DEFAULT_POST_URL));
+ u = new URL(System.getProperty("url", SimplePostTool.appendParam(DEFAULT_POST_URL, params)));
} catch (MalformedURLException e) {
fatal("System Property 'url' is not a valid URL: " + u);
}
final SimplePostTool t = new SimplePostTool(u);
+ if (isOn(System.getProperty("auto", DEFAULT_AUTO))) {
+ t.setAuto(true);
+ }
+
+ if (isOn(System.getProperty("recursive", DEFAULT_RECURSIVE))) {
+ t.setRecursive(true);
+ }
+
final String mode = System.getProperty("data", DEFAULT_DATA_MODE);
if (! DATA_MODES.contains(mode)) {
fatal("System Property 'data' is not valid for this tool: " + mode);
}
- if ("yes".equals(System.getProperty("out", DEFAULT_OUT))) {
+ if (isOn(System.getProperty("out", DEFAULT_OUT))) {
out = System.out;
}
+
+ t.setFileTypes(System.getProperty("filetypes", DEFAULT_FILE_TYPES));
+ int numFilesPosted = 0;
+
try {
if (DATA_MODE_FILES.equals(mode)) {
if (0 < args.length) {
- info("POSTing files to " + u + "..");
- t.postFiles(args, 0, out, type);
+ // Skip posting files if special param "-" given
+ if (!args[0].equals("-")) {
+ info("Posting files to base url " + u + (!t.auto?" using content-type "+(type==null?DEFAULT_CONTENT_TYPE:type):"")+"..");
+ if(t.auto)
+ info("Entering auto mode. File endings considered are "+t.getFileTypes());
+ if(t.recursive)
+ info("Entering recursive mode");
+ numFilesPosted = t.postFiles(args, 0, out, type);
+ info(numFilesPosted + " files indexed.");
+ }
} else {
- info("No files specified. (Use -h for help)");
+ usageShort();
+ return;
}
-
} else if (DATA_MODE_ARGS.equals(mode)) {
if (0 < args.length) {
info("POSTing args to " + u + "..");
for (String a : args) {
t.postData(SimplePostTool.stringToStream(a), null, out, type);
}
+ } else {
+ usageShort();
+ return;
}
-
} else if (DATA_MODE_STDIN.equals(mode)) {
info("POSTing stdin to " + u + "..");
t.postData(System.in, null, out, type);
}
- if ("yes".equals(System.getProperty("commit",DEFAULT_COMMIT))) {
- info("COMMITting Solr index changes..");
+ if (isOn(System.getProperty("commit",DEFAULT_COMMIT))) {
+ info("COMMITting Solr index changes to " + u + "..");
t.commit();
}
- if ("yes".equals(System.getProperty("optimize",DEFAULT_OPTIMIZE))) {
- info("Performing an OPTIMIZE..");
+ if (isOn(System.getProperty("optimize",DEFAULT_OPTIMIZE))) {
+ info("Performing an OPTIMIZE to " + u + "..");
t.optimize();
}
@@ -144,37 +191,115 @@ public class SimplePostTool {
}
}
- /**
- * @deprecated use {@link #postData(InputStream, Integer, OutputStream, String)} instead
- */
- @Deprecated
- int postFiles(String [] args,int startIndexInArgs, OutputStream out) {
- final String type = System.getProperty("type", DEFAULT_DATA_TYPE);
- return postFiles(args, startIndexInArgs, out, type);
- }
-
- /** Post all filenames provided in args, return the number of files posted*/
+ private static void usageShort() {
+ System.out.println(USAGE_STRING_SHORT+"\n"+
+ " Please invoke with -h option for extended usage help.");
+ }
+
+ private static void usage() {
+ System.out.println
+ (USAGE_STRING_SHORT+"\n\n" +
+ "Supported System Properties and their defaults:\n"+
+ " -Ddata=files|args|stdin (default=" + DEFAULT_DATA_MODE + ")\n"+
+ " -Dtype=<content-type> (default=" + DEFAULT_CONTENT_TYPE + ")\n"+
+ " -Durl=<solr-update-url> (default=" + DEFAULT_POST_URL + ")\n"+
+ " -Dauto=yes|no (default=" + DEFAULT_AUTO + ")\n"+
+ " -Drecursive=yes|no (default=" + DEFAULT_RECURSIVE + ")\n"+
+ " -Dfiletypes=<type>[,<type>,...] (default=" + DEFAULT_FILE_TYPES + ")\n"+
+ " -Dparams=\"<key>=<value>[&<key>=<value>...]\" (values must be URL-encoded)\n"+
+ " -Dcommit=yes|no (default=" + DEFAULT_COMMIT + ")\n"+
+ " -Doptimize=yes|no (default=" + DEFAULT_OPTIMIZE + ")\n"+
+ " -Dout=yes|no (default=" + DEFAULT_OUT + ")\n\n"+
+ "This is a simple command line tool for POSTing raw data to a Solr\n"+
+ "port. Data can be read from files specified as commandline args,\n"+
+ "as raw commandline arg strings, or via STDIN.\n"+
+ "Examples:\n"+
+ " java -jar post.jar *.xml\n"+
+ " java -Ddata=args -jar post.jar '<delete><id>42</id></delete>'\n"+
+ " java -Ddata=stdin -jar post.jar < hd.xml\n"+
+ " java -Dtype=text/csv -jar post.jar *.csv\n"+
+ " java -Dtype=application/json -jar post.jar *.json\n"+
+ " java -Durl=http://localhost:8983/solr/update/extract -Dparams=literal.id=a -Dtype=application/pdf -jar post.jar a.pdf\n"+
+ " java -Dauto -jar post.jar *\n"+
+ " java -Dauto -Drecursive -jar post.jar afolder\n"+
+ " java -Dauto -Dfiletypes=ppt,html -jar post.jar afolder\n"+
+ "The options controlled by System Properties include the Solr\n"+
+ "URL to POST to, the Content-Type of the data, whether a commit\n"+
+ "or optimize should be executed, and whether the response should\n"+
+ "be written to STDOUT. If auto=yes the tool will try to set type\n"+
+ "and url automatically from file name. When posting rich documents\n"+
+ "the file name will be propagated as \"resource.name\" and also used as \"literal.id\".\n" +
+ "You may override these or any other request parameter through the -Dparams property.\n"+
+ "If you want to do a commit only, use \"-\" as argument.");
+ }
+
+ private static boolean isOn(String property) {
+ return(TRUE_STRINGS.indexOf(property) >= 0);
+ }
+
+ /** Post all filenames provided in args
+ * @param args array of file names
+ * @param startIndexInArgs offset to start
+ * @param out output stream to post data to
+ * @param type default content-type to use when posting (may be overridden in auto mode)
+ * @return number of files posted
+ * */
int postFiles(String [] args,int startIndexInArgs, OutputStream out, String type) {
int filesPosted = 0;
for (int j = startIndexInArgs; j < args.length; j++) {
File srcFile = new File(args[j]);
- if (srcFile.canRead()) {
- info("POSTing file " + srcFile.getName());
- postFile(srcFile, out, type);
- filesPosted++;
+ if(srcFile.isDirectory() && srcFile.canRead()) {
+ filesPosted += postDirectory(srcFile, out, type);
+ } else if (srcFile.isFile() && srcFile.canRead()) {
+ filesPosted += postFiles(new File[] {srcFile}, out, type);
} else {
- warn("Cannot read input file: " + srcFile);
+ File parent = srcFile.getParentFile();
+ if(parent == null) parent = new File(".");
+ String fileGlob = srcFile.getName();
+ GlobFileFilter ff = new GlobFileFilter(fileGlob, false);
+ File[] files = parent.listFiles(ff);
+ if(files.length == 0) {
+ warn("No files or directories matching "+srcFile);
+ continue;
+ }
+ filesPosted += postFiles(parent.listFiles(ff), out, type);
}
}
return filesPosted;
}
+ private int postDirectory(File dir, OutputStream out, String type) {
+ if(dir.isHidden() && !dir.getName().equals("."))
+ return(0);
+ info("Indexing directory "+dir.getPath());
+ int posted = 0;
+ posted += postFiles(dir.listFiles(globFileFilter), out, type);
+ if(recursive) {
+ for(File d : dir.listFiles()) {
+ if(d.isDirectory())
+ posted += postDirectory(d, out, type);
+ }
+ }
+ return posted;
+ }
+
+ int postFiles(File[] files, OutputStream out, String type) {
+ int filesPosted = 0;
+ for(File srcFile : files) {
+ if(!srcFile.isFile() || srcFile.isHidden())
+ continue;
+ postFile(srcFile, out, type);
+ filesPosted++;
+ }
+ return filesPosted;
+ }
+
static void warn(String msg) {
System.err.println("SimplePostTool: WARNING: " + msg);
}
static void info(String msg) {
- System.out.println("SimplePostTool: " + msg);
+ System.out.println(msg);
}
static void fatal(String msg) {
@@ -204,30 +329,58 @@ public class SimplePostTool {
doGet(appendParam(solrUrl.toString(), "optimize=true"));
}
- private String appendParam(String url, String param) {
- return url + (url.indexOf('?')>0 ? "&" : "?") + param;
+ public static String appendParam(String url, String param) {
+ String[] pa = param.split("&");
+ for(String p : pa) {
+ if(p.trim().length() == 0) continue;
+ String[] kv = p.split("=");
+ if(kv.length == 2) {
+ url = url + (url.indexOf('?')>0 ? "&" : "?") + kv[0] +"="+ kv[1];
+ } else {
+ warn("Skipping param "+p+" which is not on form key=value");
+ }
+ }
+ return url;
}
/**
- * @deprecated use {@link #postFile(File, OutputStream, String)} instead
- */
- public void postFile(File file, OutputStream output) {
- final String type = System.getProperty("type", DEFAULT_DATA_TYPE);
- postFile(file, output, type);
- }
-
- /**
* Opens the file and posts it's contents to the solrUrl,
* writes to response to output.
*/
public void postFile(File file, OutputStream output, String type) {
-
InputStream is = null;
try {
+ URL url = solrUrl;
+ if(auto) {
+ if(type == null) {
+ type = guessType(file);
+ }
+ if(type != null) {
+ if(type.equals("text/xml") || type.equals("text/csv") || type.equals("application/json")) {
+ // Default handler
+ } else {
+ // SolrCell
+ String urlStr = url.getProtocol() + "://" + url.getAuthority() + url.getPath() + "/extract" + (url.getQuery() != null ? "?"+url.getQuery() : "");
+ if(urlStr.indexOf("resource.name")==-1)
+ urlStr = appendParam(urlStr, "resource.name=" + URLEncoder.encode(file.getAbsolutePath(), "UTF-8"));
+ if(urlStr.indexOf("literal.id")==-1)
+ urlStr = appendParam(urlStr, "literal.id=" + URLEncoder.encode(file.getAbsolutePath(), "UTF-8"));
+ url = new URL(urlStr);
+// info("Indexing to ExtractingRequestHandler with URL "+url);
+ }
+ } else {
+ warn("Skipping "+file.getName()+". Unsupported file type for auto mode.");
+ return;
+ }
+ } else {
+ if(type == null) type = DEFAULT_CONTENT_TYPE;
+ }
+ info("POSTing file " + file.getName() + (auto?" ("+type+")":""));
is = new FileInputStream(file);
- postData(is, (int)file.length(), output, type);
+ postData(is, (int)file.length(), output, type, url);
} catch (IOException e) {
- fatal("Can't open/read file: " + file);
+ e.printStackTrace();
+ warn("Can't open/read file: " + file);
} finally {
try {
if(is!=null) is.close();
@@ -237,53 +390,53 @@ public class SimplePostTool {
}
}
+ private String guessType(File file) {
+ String name = file.getName();
+ String suffix = name.substring(name.lastIndexOf(".")+1);
+ return mimeMap.get(suffix.toLowerCase(Locale.ROOT));
+ }
+
/**
* Performs a simple get on the given URL
- * @param url
*/
public static void doGet(String url) {
try {
doGet(new URL(url));
} catch (MalformedURLException e) {
- fatal("The specified URL "+url+" is not a valid URL. Please check");
+ warn("The specified URL "+url+" is not a valid URL. Please check");
}
}
/**
* Performs a simple get on the given URL
- * @param url
*/
public static void doGet(URL url) {
try {
HttpURLConnection urlc = (HttpURLConnection) url.openConnection();
if (HttpURLConnection.HTTP_OK != urlc.getResponseCode()) {
- fatal("Solr returned an error #" + urlc.getResponseCode() +
- " " + urlc.getResponseMessage());
+ warn("Solr returned an error #" + urlc.getResponseCode() +
+ " " + urlc.getResponseMessage() + " for url "+url);
}
} catch (IOException e) {
- fatal("An error occured posting data to "+url+". Please check that Solr is running.");
+ warn("An error occured posting data to "+url+". Please check that Solr is running.");
}
}
- /**
- * @deprecated use {@link #postData(InputStream, Integer, OutputStream, String)} instead
- */
- @Deprecated
- public void postData(InputStream data, Integer length, OutputStream output) {
- final String type = System.getProperty("type", DEFAULT_DATA_TYPE);
- postData(data, length, output, type);
+ public void postData(InputStream data, Integer length, OutputStream output, String type) {
+ postData(data, length, output, type, solrUrl);
}
-
+
/**
* Reads data from the data stream and posts it to solr,
* writes to the response to output
*/
- public void postData(InputStream data, Integer length, OutputStream output, String type) {
-
+ public void postData(InputStream data, Integer length, OutputStream output, String type, URL url) {
+ if(type == null)
+ type = DEFAULT_CONTENT_TYPE;
HttpURLConnection urlc = null;
try {
try {
- urlc = (HttpURLConnection) solrUrl.openConnection();
+ urlc = (HttpURLConnection) url.openConnection();
try {
urlc.setRequestMethod("POST");
} catch (ProtocolException e) {
@@ -315,14 +468,14 @@ public class SimplePostTool {
InputStream in = null;
try {
if (HttpURLConnection.HTTP_OK != urlc.getResponseCode()) {
- fatal("Solr returned an error #" + urlc.getResponseCode() +
+ warn("Solr returned an error #" + urlc.getResponseCode() +
" " + urlc.getResponseMessage());
}
in = urlc.getInputStream();
pipe(in, output);
} catch (IOException e) {
- fatal("IOException while reading response: " + e);
+ warn("IOException while reading response: " + e);
} finally {
try { if(in!=null) in.close(); } catch (IOException x) { /*NOOP*/ }
}
@@ -354,4 +507,68 @@ public class SimplePostTool {
}
if (null != dest) dest.flush();
}
+
+ public boolean isAuto() {
+ return auto;
+ }
+
+ public void setAuto(boolean auto) {
+ this.auto = auto;
+ }
+
+ public boolean isRecursive() {
+ return recursive;
+ }
+
+ public void setRecursive(boolean recursive) {
+ this.recursive = recursive;
+ }
+
+ public String getFileTypes() {
+ return fileTypes;
+ }
+
+ public void setFileTypes(String fileTypes) {
+ this.fileTypes = fileTypes;
+ String glob;
+ if(fileTypes.equals("*"))
+ glob = ".*";
+ else
+ glob = "^.*\\.(" + fileTypes.replace(",", "|") + ")$";
+ this.globFileFilter = new GlobFileFilter(glob, true);
+ }
+
+ class GlobFileFilter implements FileFilter
+ {
+ private String _pattern;
+ private Pattern p;
+
+ public GlobFileFilter(String pattern, boolean isRegex)
+ {
+ _pattern = pattern;
+ if(!isRegex) {
+ _pattern = _pattern
+ .replace("^", "\\^")
+ .replace("$", "\\$")
+ .replace(".", "\\.")
+ .replace("(", "\\(")
+ .replace(")", "\\)")
+ .replace("+", "\\+")
+ .replace("*", ".*")
+ .replace("?", ".");
+ _pattern = "^" + _pattern + "$";
+ }
+
+ try {
+ p = Pattern.compile(_pattern,Pattern.CASE_INSENSITIVE);
+ } catch(PatternSyntaxException e) {
+ fatal("Invalid type list "+pattern+". "+e.getDescription());
+ }
+ }
+
+ public boolean accept(File file)
+ {
+ return p.matcher(file.getName()).find();
+ }
+ }
}
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/util/plugin/AbstractPluginLoader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/util/plugin/AbstractPluginLoader.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/util/plugin/AbstractPluginLoader.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/util/plugin/AbstractPluginLoader.java Tue Jul 31 20:58:32 2012
@@ -22,11 +22,11 @@ import java.util.List;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.lucene.analysis.util.ResourceLoader;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.util.DOMUtil;
import org.apache.solr.core.SolrConfig;
+import org.apache.solr.core.SolrResourceLoader;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
@@ -81,7 +81,7 @@ public abstract class AbstractPluginLoad
* @param node - the XML node defining this plugin
*/
@SuppressWarnings("unchecked")
- protected T create( ResourceLoader loader, String name, String className, Node node ) throws Exception
+ protected T create( SolrResourceLoader loader, String name, String className, Node node ) throws Exception
{
return loader.newInstance(className, pluginClassType, getDefaultPackages());
}
@@ -124,7 +124,7 @@ public abstract class AbstractPluginLoad
* If a default element is defined, it will be returned from this function.
*
*/
- public T load( ResourceLoader loader, NodeList nodes )
+ public T load( SolrResourceLoader loader, NodeList nodes )
{
List<PluginInitInfo> info = new ArrayList<PluginInitInfo>();
T defaultPlugin = null;
@@ -204,7 +204,7 @@ public abstract class AbstractPluginLoad
* The created class for the plugin will be returned from this function.
*
*/
- public T loadSingle(ResourceLoader loader, Node node) {
+ public T loadSingle(SolrResourceLoader loader, Node node) {
List<PluginInitInfo> info = new ArrayList<PluginInitInfo>();
T plugin = null;
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/test-files/solr/collection1/conf/schema.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/test-files/solr/collection1/conf/schema.xml?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/test-files/solr/collection1/conf/schema.xml (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/test-files/solr/collection1/conf/schema.xml Tue Jul 31 20:58:32 2012
@@ -414,7 +414,32 @@
<fieldType name="location" class="solr.LatLonType" subFieldSuffix="_coordinate"/>
- </types>
+ <!--
+ Example of using PathHierarchyTokenizerFactory at index time, so
+ queries for paths match documents at that path, or in descendent paths
+ -->
+ <fieldType name="path" class="solr.TextField">
+ <analyzer type="index">
+ <tokenizer class="solr.PathHierarchyTokenizerFactory" delimiter="/" />
+ </analyzer>
+ <analyzer type="query">
+ <tokenizer class="solr.KeywordTokenizerFactory" />
+ </analyzer>
+ </fieldType>
+ <!--
+ Example of using PathHierarchyTokenizerFactory at query time, so
+ queries for paths match documents at that path, or in ancestor paths
+ -->
+ <fieldType name="ancestor_path" class="solr.TextField">
+ <analyzer type="index">
+ <tokenizer class="solr.KeywordTokenizerFactory" />
+ </analyzer>
+ <analyzer type="query">
+ <tokenizer class="solr.PathHierarchyTokenizerFactory" delimiter="/" />
+ </analyzer>
+ </fieldType>
+
+</types>
<fields>
@@ -640,6 +665,10 @@
<!-- Type used to index the lat and lon components for the "location" FieldType -->
<dynamicField name="*_coordinate" type="tdouble" indexed="true" stored="false" omitNorms="true" />
+
+ <dynamicField name="*_path" type="path" indexed="true" stored="true" omitNorms="true" multiValued="true" />
+ <dynamicField name="*_ancestor" type="ancestor_path" indexed="true" stored="true" omitNorms="true" multiValued="true" />
+
</fields>
<defaultSearchField>text</defaultSearchField>
@@ -673,6 +702,7 @@
<!-- dynamic destination -->
<copyField source="*_dynamic" dest="dynamic_*"/>
+ <copyField source="*_path" dest="*_ancestor"/>
<!-- example of a custom similarity -->
<similarity class="solr.CustomSimilarityFactory">
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/test-files/solr/solr.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/test-files/solr/solr.xml?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/test-files/solr/solr.xml (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/test-files/solr/solr.xml Tue Jul 31 20:58:32 2012
@@ -28,7 +28,7 @@
adminPath: RequestHandler path to manage cores.
If 'null' (or absent), cores will not be manageable via request handler
-->
- <cores adminPath="/admin/cores" defaultCoreName="collection1" host="127.0.0.1" hostPort="${hostPort:8983}" hostContext="solr" zkClientTimeout="12000" numShards="${numShards:3}">
+ <cores adminPath="/admin/cores" defaultCoreName="collection1" host="127.0.0.1" hostPort="${hostPort:8983}" hostContext="solr" zkClientTimeout="8000" numShards="${numShards:3}">
<core name="collection1" instanceDir="collection1" shard="${shard:}" collection="${collection:collection1}" config="${solrconfig:solrconfig.xml}" schema="${schema:schema.xml}"/>
</cores>
</solr>