You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by us...@apache.org on 2012/08/09 12:21:07 UTC
svn commit: r1371142 [30/32] - in /lucene/dev/branches/lucene3312: ./
dev-tools/ dev-tools/eclipse/ dev-tools/maven/ dev-tools/maven/lucene/
dev-tools/maven/lucene/analysis/common/
dev-tools/maven/lucene/analysis/icu/ dev-tools/maven/lucene/analysis/ku...
Modified: lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/schema/FieldTypePluginLoader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/schema/FieldTypePluginLoader.java?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/schema/FieldTypePluginLoader.java (original)
+++ lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/schema/FieldTypePluginLoader.java Thu Aug 9 10:20:53 2012
@@ -225,7 +225,30 @@ public final class FieldTypePluginLoader
if (node == null) return null;
NamedNodeMap attrs = node.getAttributes();
String analyzerName = DOMUtil.getAttr(attrs,"class");
+
+ // check for all of these up front, so we can error if used in
+ // conjunction with an explicit analyzer class.
+ NodeList charFilterNodes = (NodeList)xpath.evaluate
+ ("./charFilter", node, XPathConstants.NODESET);
+ NodeList tokenizerNodes = (NodeList)xpath.evaluate
+ ("./tokenizer", node, XPathConstants.NODESET);
+ NodeList tokenFilterNodes = (NodeList)xpath.evaluate
+ ("./filter", node, XPathConstants.NODESET);
+
if (analyzerName != null) {
+
+ // explicitly check for child analysis factories instead of
+ // just any child nodes, because the user might have their
+ // own custom nodes (ie: <description> or something like that)
+ if (0 != charFilterNodes.getLength() ||
+ 0 != tokenizerNodes.getLength() ||
+ 0 != tokenFilterNodes.getLength()) {
+ throw new SolrException
+ ( SolrException.ErrorCode.SERVER_ERROR,
+ "Configuration Error: Analyzer class='" + analyzerName +
+ "' can not be combined with nested analysis factories");
+ }
+
try {
// No need to be core-aware as Analyzers are not in the core-aware list
final Class<? extends Analyzer> clazz = loader.findClass(analyzerName, Analyzer.class);
@@ -286,8 +309,7 @@ public final class FieldTypePluginLoader
}
};
- charFilterLoader.load( loader, (NodeList)xpath.evaluate("./charFilter", node, XPathConstants.NODESET) );
-
+ charFilterLoader.load( loader, charFilterNodes );
// Load the Tokenizer
// Although an analyzer only allows a single Tokenizer, we load a list to make sure
@@ -319,13 +341,12 @@ public final class FieldTypePluginLoader
}
};
- tokenizerLoader.load( loader, (NodeList)xpath.evaluate("./tokenizer", node, XPathConstants.NODESET) );
+ tokenizerLoader.load( loader, tokenizerNodes );
// Make sure something was loaded
if( tokenizers.isEmpty() ) {
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,"analyzer without class or tokenizer & filter list");
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,"analyzer without class or tokenizer");
}
-
// Load the Filters
@@ -353,7 +374,7 @@ public final class FieldTypePluginLoader
return null; // used for map registration
}
};
- filterLoader.load( loader, (NodeList)xpath.evaluate("./filter", node, XPathConstants.NODESET) );
+ filterLoader.load( loader, tokenFilterNodes );
return new TokenizerChain(charFilters.toArray(new CharFilterFactory[charFilters.size()]),
tokenizers.get(0), filters.toArray(new TokenFilterFactory[filters.size()]));
Modified: lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/schema/IndexSchema.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/schema/IndexSchema.java?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/schema/IndexSchema.java (original)
+++ lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/schema/IndexSchema.java Thu Aug 9 10:20:53 2012
@@ -501,7 +501,10 @@ public final class IndexSchema {
log.error("uniqueKey is not stored - distributed search will not work");
}
if (uniqueKeyField.multiValued()) {
- log.error("uniqueKey should not be multivalued");
+ String msg = "uniqueKey field ("+uniqueKeyFieldName+
+ ") can not be configured to be multivalued";
+ log.error(msg);
+ throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, msg );
}
uniqueKeyFieldName=uniqueKeyField.getName();
uniqueKeyFieldType=uniqueKeyField.getType();
Modified: lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/schema/LatLonType.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/schema/LatLonType.java?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/schema/LatLonType.java (original)
+++ lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/schema/LatLonType.java Thu Aug 9 10:20:53 2012
@@ -75,18 +75,18 @@ public class LatLonType extends Abstract
}
//latitude
SchemaField lat = subField(field, i);
- f[i] = lat.createField(String.valueOf(latLon[LAT]), lat.omitNorms() ? 1F : boost);
+ f[i] = lat.createField(String.valueOf(latLon[LAT]), lat.indexed() && !lat.omitNorms() ? boost : 1f);
i++;
//longitude
SchemaField lon = subField(field, i);
- f[i] = lon.createField(String.valueOf(latLon[LON]), lon.omitNorms() ? 1F : boost);
+ f[i] = lon.createField(String.valueOf(latLon[LON]), lon.indexed() && !lon.omitNorms() ? boost : 1f);
}
if (field.stored()) {
FieldType customType = new FieldType();
customType.setStored(true);
- f[f.length - 1] = createField(field.getName(), externalVal, customType, boost);
+ f[f.length - 1] = createField(field.getName(), externalVal, customType, 1f);
}
return f;
}
Modified: lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/schema/PointType.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/schema/PointType.java?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/schema/PointType.java (original)
+++ lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/schema/PointType.java Thu Aug 9 10:20:53 2012
@@ -84,7 +84,8 @@ public class PointType extends Coordinat
if (field.indexed()) {
for (int i=0; i<dimension; i++) {
- f[i] = subField(field, i).createField(point[i], boost);
+ SchemaField sf = subField(field, i);
+ f[i] = sf.createField(point[i], sf.indexed() && !sf.omitNorms() ? boost : 1f);
}
}
@@ -92,7 +93,7 @@ public class PointType extends Coordinat
String storedVal = externalVal; // normalize or not?
FieldType customType = new FieldType();
customType.setStored(true);
- f[f.length - 1] = createField(field.getName(), storedVal, customType, boost);
+ f[f.length - 1] = createField(field.getName(), storedVal, customType, 1f);
}
return f;
Modified: lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/search/Grouping.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/search/Grouping.java?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/search/Grouping.java (original)
+++ lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/search/Grouping.java Thu Aug 9 10:20:53 2012
@@ -787,7 +787,7 @@ public class Grouping {
SchemaField schemaField = searcher.getSchema().getField(groupBy);
FieldType fieldType = schemaField.getType();
String readableValue = fieldType.indexedToReadable(group.groupValue.utf8ToString());
- IndexableField field = schemaField.createField(readableValue, 0.0f);
+ IndexableField field = schemaField.createField(readableValue, 1.0f);
nl.add("groupValue", fieldType.toObject(field));
} else {
nl.add("groupValue", null);
Modified: lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java (original)
+++ lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java Thu Aug 9 10:20:53 2012
@@ -341,7 +341,7 @@ class JoinQuery extends Query {
if (freq < minDocFreqFrom) {
fromTermDirectCount++;
// OK to skip liveDocs, since we check for intersection with docs matching query
- fromDeState.docsEnum = fromDeState.termsEnum.docs(null, fromDeState.docsEnum, false);
+ fromDeState.docsEnum = fromDeState.termsEnum.docs(null, fromDeState.docsEnum, 0);
DocsEnum docsEnum = fromDeState.docsEnum;
if (docsEnum instanceof MultiDocsEnum) {
@@ -406,7 +406,7 @@ class JoinQuery extends Query {
toTermDirectCount++;
// need to use liveDocs here so we don't map to any deleted ones
- toDeState.docsEnum = toDeState.termsEnum.docs(toDeState.liveDocs, toDeState.docsEnum, false);
+ toDeState.docsEnum = toDeState.termsEnum.docs(toDeState.liveDocs, toDeState.docsEnum, 0);
DocsEnum docsEnum = toDeState.docsEnum;
if (docsEnum instanceof MultiDocsEnum) {
Modified: lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java (original)
+++ lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java Thu Aug 9 10:20:53 2012
@@ -599,7 +599,7 @@ public class SolrIndexSearcher extends I
if (!termsEnum.seekExact(termBytes, false)) {
return -1;
}
- DocsEnum docs = termsEnum.docs(atomicReader.getLiveDocs(), null, false);
+ DocsEnum docs = termsEnum.docs(atomicReader.getLiveDocs(), null, 0);
if (docs == null) return -1;
int id = docs.nextDoc();
return id == DocIdSetIterator.NO_MORE_DOCS ? -1 : id;
@@ -621,7 +621,7 @@ public class SolrIndexSearcher extends I
final Bits liveDocs = reader.getLiveDocs();
- final DocsEnum docs = reader.termDocsEnum(liveDocs, field, idBytes, false);
+ final DocsEnum docs = reader.termDocsEnum(liveDocs, field, idBytes, 0);
if (docs == null) continue;
int id = docs.nextDoc();
@@ -926,7 +926,7 @@ public class SolrIndexSearcher extends I
int bitsSet = 0;
OpenBitSet obs = null;
- DocsEnum docsEnum = deState.termsEnum.docs(deState.liveDocs, deState.docsEnum, false);
+ DocsEnum docsEnum = deState.termsEnum.docs(deState.liveDocs, deState.docsEnum, 0);
if (deState.docsEnum == null) {
deState.docsEnum = docsEnum;
}
@@ -1004,7 +1004,7 @@ public class SolrIndexSearcher extends I
if (terms != null) {
final TermsEnum termsEnum = terms.iterator(null);
if (termsEnum.seekExact(termBytes, false)) {
- docsEnum = termsEnum.docs(liveDocs, null, false);
+ docsEnum = termsEnum.docs(liveDocs, null, 0);
}
}
Modified: lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/search/ValueSourceParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/search/ValueSourceParser.java?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/search/ValueSourceParser.java (original)
+++ lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/search/ValueSourceParser.java Thu Aug 9 10:20:53 2012
@@ -107,6 +107,25 @@ public abstract class ValueSourceParser
return new LiteralValueSource(fp.parseArg());
}
});
+ addParser("threadid", new ValueSourceParser() {
+ @Override
+ public ValueSource parse(FunctionQParser fp) throws ParseException {
+ return new LongConstValueSource(Thread.currentThread().getId());
+ }
+ });
+ addParser("sleep", new ValueSourceParser() {
+ @Override
+ public ValueSource parse(FunctionQParser fp) throws ParseException {
+ int ms = fp.parseInt();
+ ValueSource source = fp.parseValueSource();
+ try {
+ Thread.sleep(ms);
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ return source;
+ }
+ });
addParser("rord", new ValueSourceParser() {
@Override
public ValueSource parse(FunctionQParser fp) throws ParseException {
Modified: lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/search/function/FileFloatSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/search/function/FileFloatSource.java?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/search/function/FileFloatSource.java (original)
+++ lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/search/function/FileFloatSource.java Thu Aug 9 10:20:53 2012
@@ -278,7 +278,7 @@ public class FileFloatSource extends Val
continue;
}
- docsEnum = termsEnum.docs(null, docsEnum, false);
+ docsEnum = termsEnum.docs(null, docsEnum, 0);
int doc;
while ((doc = docsEnum.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
vals[doc] = fval;
Modified: lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/search/grouping/distributed/shardresultserializer/SearchGroupsResultTransformer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/search/grouping/distributed/shardresultserializer/SearchGroupsResultTransformer.java?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/search/grouping/distributed/shardresultserializer/SearchGroupsResultTransformer.java (original)
+++ lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/search/grouping/distributed/shardresultserializer/SearchGroupsResultTransformer.java Thu Aug 9 10:20:53 2012
@@ -110,9 +110,9 @@ public class SearchGroupsResultTransform
if (sortValue instanceof BytesRef) {
UnicodeUtil.UTF8toUTF16((BytesRef)sortValue, spare);
String indexedValue = spare.toString();
- sortValue = (Comparable) fieldType.toObject(field.createField(fieldType.indexedToReadable(indexedValue), 0.0f));
+ sortValue = (Comparable) fieldType.toObject(field.createField(fieldType.indexedToReadable(indexedValue), 1.0f));
} else if (sortValue instanceof String) {
- sortValue = (Comparable) fieldType.toObject(field.createField(fieldType.indexedToReadable((String) sortValue), 0.0f));
+ sortValue = (Comparable) fieldType.toObject(field.createField(fieldType.indexedToReadable((String) sortValue), 1.0f));
}
}
convertedSortValues[i] = sortValue;
Modified: lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/search/grouping/distributed/shardresultserializer/TopGroupsResultTransformer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/search/grouping/distributed/shardresultserializer/TopGroupsResultTransformer.java?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/search/grouping/distributed/shardresultserializer/TopGroupsResultTransformer.java (original)
+++ lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/search/grouping/distributed/shardresultserializer/TopGroupsResultTransformer.java Thu Aug 9 10:20:53 2012
@@ -200,9 +200,9 @@ public class TopGroupsResultTransformer
if (sortValue instanceof BytesRef) {
UnicodeUtil.UTF8toUTF16((BytesRef)sortValue, spare);
String indexedValue = spare.toString();
- sortValue = fieldType.toObject(field.createField(fieldType.indexedToReadable(indexedValue), 0.0f));
+ sortValue = fieldType.toObject(field.createField(fieldType.indexedToReadable(indexedValue), 1.0f));
} else if (sortValue instanceof String) {
- sortValue = fieldType.toObject(field.createField(fieldType.indexedToReadable((String) sortValue), 0.0f));
+ sortValue = fieldType.toObject(field.createField(fieldType.indexedToReadable((String) sortValue), 1.0f));
}
}
convertedSortValues[j] = sortValue;
@@ -253,9 +253,9 @@ public class TopGroupsResultTransformer
if (sortValue instanceof BytesRef) {
UnicodeUtil.UTF8toUTF16((BytesRef)sortValue, spare);
String indexedValue = spare.toString();
- sortValue = fieldType.toObject(field.createField(fieldType.indexedToReadable(indexedValue), 0.0f));
+ sortValue = fieldType.toObject(field.createField(fieldType.indexedToReadable(indexedValue), 1.0f));
} else if (sortValue instanceof String) {
- sortValue = fieldType.toObject(field.createField(fieldType.indexedToReadable((String) sortValue), 0.0f));
+ sortValue = fieldType.toObject(field.createField(fieldType.indexedToReadable((String) sortValue), 1.0f));
}
}
convertedSortValues[j] = sortValue;
Modified: lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/search/grouping/endresulttransformer/GroupedEndResultTransformer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/search/grouping/endresulttransformer/GroupedEndResultTransformer.java?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/search/grouping/endresulttransformer/GroupedEndResultTransformer.java (original)
+++ lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/search/grouping/endresulttransformer/GroupedEndResultTransformer.java Thu Aug 9 10:20:53 2012
@@ -69,7 +69,7 @@ public class GroupedEndResultTransformer
SimpleOrderedMap<Object> groupResult = new SimpleOrderedMap<Object>();
if (group.groupValue != null) {
groupResult.add(
- "groupValue", groupFieldType.toObject(groupField.createField(group.groupValue.utf8ToString(), 0.0f))
+ "groupValue", groupFieldType.toObject(groupField.createField(group.groupValue.utf8ToString(), 1.0f))
);
} else {
groupResult.add("groupValue", null);
Modified: lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java (original)
+++ lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java Thu Aug 9 10:20:53 2012
@@ -39,7 +39,7 @@ import javax.servlet.http.HttpServletReq
import javax.servlet.http.HttpServletResponse;
import org.apache.solr.common.SolrException;
-import org.apache.solr.common.cloud.CloudState;
+import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.cloud.ZkStateReader;
@@ -315,8 +315,8 @@ public class SolrDispatchFilter implemen
String collection = corename;
ZkStateReader zkStateReader = cores.getZkController().getZkStateReader();
- CloudState cloudState = zkStateReader.getCloudState();
- Map<String,Slice> slices = cloudState.getSlices(collection);
+ ClusterState clusterState = zkStateReader.getClusterState();
+ Map<String,Slice> slices = clusterState.getSlices(collection);
if (slices == null) {
return null;
}
@@ -326,7 +326,7 @@ public class SolrDispatchFilter implemen
done:
for (Entry<String,Slice> entry : entries) {
// first see if we have the leader
- ZkNodeProps leaderProps = cloudState.getLeader(collection, entry.getKey());
+ ZkNodeProps leaderProps = clusterState.getLeader(collection, entry.getKey());
if (leaderProps != null) {
core = checkProps(cores, path, leaderProps);
}
Modified: lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/update/CommitTracker.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/update/CommitTracker.java?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/update/CommitTracker.java (original)
+++ lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/update/CommitTracker.java Thu Aug 9 10:20:53 2012
@@ -39,8 +39,10 @@ import org.slf4j.LoggerFactory;
* definitely change in the future, so the interface should not be relied-upon
*
* Note: all access must be synchronized.
+ *
+ * Public for tests.
*/
-final class CommitTracker implements Runnable {
+public final class CommitTracker implements Runnable {
protected final static Logger log = LoggerFactory.getLogger(CommitTracker.class);
// scheduler delay for maxDoc-triggered autocommits
@@ -248,7 +250,8 @@ final class CommitTracker implements Run
this.docsUpperBound = docsUpperBound;
}
- void setTimeUpperBound(long timeUpperBound) {
+ // only for testing - not thread safe
+ public void setTimeUpperBound(long timeUpperBound) {
this.timeUpperBound = timeUpperBound;
}
}
Modified: lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java (original)
+++ lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java Thu Aug 9 10:20:53 2012
@@ -35,6 +35,10 @@ public final class DefaultSolrCoreState
private final boolean SKIP_AUTO_RECOVERY = Boolean.getBoolean("solrcloud.skip.autorecovery");
private final Object recoveryLock = new Object();
+
+ // protects pauseWriter and writerFree
+ private final Object writerPauseLock = new Object();
+
private int refCnt = 1;
private SolrIndexWriter indexWriter = null;
private DirectoryFactory directoryFactory;
@@ -55,68 +59,74 @@ public final class DefaultSolrCoreState
@Override
public synchronized RefCounted<IndexWriter> getIndexWriter(SolrCore core)
throws IOException {
-
- if (core == null) {
- // core == null is a signal to just return the current writer, or null if none.
- if (refCntWriter != null) refCntWriter.incref();
- return refCntWriter;
- }
-
- while (pauseWriter) {
- try {
- wait();
- } catch (InterruptedException e) {}
- }
-
- if (indexWriter == null) {
- indexWriter = createMainIndexWriter(core, "DirectUpdateHandler2", false,
- false);
- }
- if (refCntWriter == null) {
- refCntWriter = new RefCounted<IndexWriter>(indexWriter) {
- @Override
- public void close() {
- synchronized (DefaultSolrCoreState.this) {
- writerFree = true;
- DefaultSolrCoreState.this.notifyAll();
+ synchronized (writerPauseLock) {
+ if (core == null) {
+ // core == null is a signal to just return the current writer, or null
+ // if none.
+ if (refCntWriter != null) refCntWriter.incref();
+ return refCntWriter;
+ }
+
+ while (pauseWriter) {
+ try {
+ writerPauseLock.wait();
+ } catch (InterruptedException e) {}
+ }
+
+ if (indexWriter == null) {
+ indexWriter = createMainIndexWriter(core, "DirectUpdateHandler2",
+ false, false);
+ }
+ if (refCntWriter == null) {
+ refCntWriter = new RefCounted<IndexWriter>(indexWriter) {
+ @Override
+ public void close() {
+ synchronized (writerPauseLock) {
+ writerFree = true;
+ writerPauseLock.notifyAll();
+ }
}
- }
- };
+ };
+ }
+ writerFree = false;
+ writerPauseLock.notifyAll();
+ refCntWriter.incref();
+ return refCntWriter;
}
- writerFree = false;
- notifyAll();
- refCntWriter.incref();
- return refCntWriter;
}
@Override
- public synchronized void newIndexWriter(SolrCore core) throws IOException {
- // we need to wait for the Writer to fall out of use
- // first lets stop it from being lent out
- pauseWriter = true;
- // then lets wait until its out of use
- while(!writerFree) {
- try {
- wait();
- } catch (InterruptedException e) {}
- }
+ public synchronized void newIndexWriter(SolrCore core, boolean rollback) throws IOException {
- try {
- if (indexWriter != null) {
+ synchronized (writerPauseLock) {
+ // we need to wait for the Writer to fall out of use
+ // first lets stop it from being lent out
+ pauseWriter = true;
+ // then lets wait until its out of use
+ while (!writerFree) {
try {
- indexWriter.close();
- } catch (Throwable t) {
- SolrException.log(log, "Error closing old IndexWriter", t);
- }
+ writerPauseLock.wait();
+ } catch (InterruptedException e) {}
}
- indexWriter = createMainIndexWriter(core, "DirectUpdateHandler2", false,
- true);
- // we need to null this so it picks up the new writer next get call
- refCntWriter = null;
- } finally {
- pauseWriter = false;
- notifyAll();
+ try {
+ if (indexWriter != null) {
+ try {
+ indexWriter.close();
+ } catch (Throwable t) {
+ SolrException.log(log, "Error closing old IndexWriter", t);
+ }
+ }
+
+ indexWriter = createMainIndexWriter(core, "DirectUpdateHandler2",
+ false, true);
+ // we need to null this so it picks up the new writer next get call
+ refCntWriter = null;
+ } finally {
+
+ pauseWriter = false;
+ writerPauseLock.notifyAll();
+ }
}
}
@@ -161,7 +171,7 @@ public final class DefaultSolrCoreState
@Override
public synchronized void rollbackIndexWriter(SolrCore core) throws IOException {
indexWriter.rollback();
- newIndexWriter(core);
+ newIndexWriter(core, true);
}
protected SolrIndexWriter createMainIndexWriter(SolrCore core, String name,
@@ -188,8 +198,9 @@ public final class DefaultSolrCoreState
return;
}
- cancelRecovery();
synchronized (recoveryLock) {
+ cancelRecovery();
+
while (recoveryRunning) {
try {
recoveryLock.wait(1000);
Modified: lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java (original)
+++ lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java Thu Aug 9 10:20:53 2012
@@ -343,7 +343,7 @@ public class DirectUpdateHandler2 extend
// currently for testing purposes. Do a delete of complete index w/o worrying about versions, don't log, clean up most state in update log, etc
if (delAll && cmd.getVersion() == -Long.MAX_VALUE) {
- synchronized (solrCoreState) {
+ synchronized (solrCoreState.getUpdateLock()) {
deleteAll();
ulog.deleteAll();
return;
@@ -356,7 +356,7 @@ public class DirectUpdateHandler2 extend
// a realtime view of the index. When a new searcher is opened after a DBQ, that
// flag can be cleared. If those thing happen concurrently, it's not thread safe.
//
- synchronized (solrCoreState) {
+ synchronized (solrCoreState.getUpdateLock()) {
if (delAll) {
deleteAll();
} else {
@@ -392,7 +392,7 @@ public class DirectUpdateHandler2 extend
Term idTerm = new Term(idField.getName(), cmd.getIndexedId());
// see comment in deleteByQuery
- synchronized (solrCoreState) {
+ synchronized (solrCoreState.getUpdateLock()) {
RefCounted<IndexWriter> iw = solrCoreState.getIndexWriter(core);
try {
IndexWriter writer = iw.get();
@@ -518,7 +518,7 @@ public class DirectUpdateHandler2 extend
}
if (!cmd.softCommit) {
- synchronized (solrCoreState) { // sync is currently needed to prevent preCommit
+ synchronized (solrCoreState.getUpdateLock()) { // sync is currently needed to prevent preCommit
// from being called between preSoft and
// postSoft... see postSoft comments.
if (ulog != null) ulog.preCommit(cmd);
@@ -547,14 +547,14 @@ public class DirectUpdateHandler2 extend
if (cmd.softCommit) {
// ulog.preSoftCommit();
- synchronized (solrCoreState) {
+ synchronized (solrCoreState.getUpdateLock()) {
if (ulog != null) ulog.preSoftCommit(cmd);
core.getSearcher(true, false, waitSearcher, true);
if (ulog != null) ulog.postSoftCommit(cmd);
}
// ulog.postSoftCommit();
} else {
- synchronized (solrCoreState) {
+ synchronized (solrCoreState.getUpdateLock()) {
if (ulog != null) ulog.preSoftCommit(cmd);
if (cmd.openSearcher) {
core.getSearcher(true, false, waitSearcher);
@@ -606,8 +606,8 @@ public class DirectUpdateHandler2 extend
}
@Override
- public void newIndexWriter() throws IOException {
- solrCoreState.newIndexWriter(core);
+ public void newIndexWriter(boolean rollback) throws IOException {
+ solrCoreState.newIndexWriter(core, rollback);
}
/**
@@ -705,7 +705,7 @@ public class DirectUpdateHandler2 extend
// TODO: keep other commit callbacks from being called?
// this.commit(cmd); // too many test failures using this method... is it because of callbacks?
- synchronized (solrCoreState) {
+ synchronized (solrCoreState.getUpdateLock()) {
ulog.preCommit(cmd);
}
@@ -714,7 +714,7 @@ public class DirectUpdateHandler2 extend
commitData.put(SolrIndexWriter.COMMIT_TIME_MSEC_KEY, String.valueOf(System.currentTimeMillis()));
writer.commit(commitData);
- synchronized (solrCoreState) {
+ synchronized (solrCoreState.getUpdateLock()) {
ulog.postCommit(cmd);
}
}
@@ -823,4 +823,14 @@ public class DirectUpdateHandler2 extend
public void incref() {
solrCoreState.incref();
}
+
+ // allow access for tests
+ public CommitTracker getCommitTracker() {
+ return commitTracker;
+ }
+
+ // allow access for tests
+ public CommitTracker getSoftCommitTracker() {
+ return softCommitTracker;
+ }
}
Modified: lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/update/DocumentBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/update/DocumentBuilder.java?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/update/DocumentBuilder.java (original)
+++ lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/update/DocumentBuilder.java Thu Aug 9 10:20:53 2012
@@ -235,7 +235,7 @@ public class DocumentBuilder {
SchemaField sfield = schema.getFieldOrNull(name);
boolean used = false;
float boost = field.getBoost();
- boolean omitNorms = sfield != null && sfield.omitNorms();
+ boolean applyBoost = sfield != null && sfield.indexed() && !sfield.omitNorms();
// Make sure it has the correct number
if( sfield!=null && !sfield.multiValued() && field.getValueCount() > 1 ) {
@@ -244,9 +244,9 @@ public class DocumentBuilder {
sfield.getName() + ": " +field.getValue() );
}
- if (omitNorms && boost != 1.0F) {
+ if (applyBoost == false && boost != 1.0F) {
throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,
- "ERROR: "+getID(doc, schema)+"cannot set an index-time boost, norms are omitted for field " +
+ "ERROR: "+getID(doc, schema)+"cannot set an index-time boost, unindexed or norms are omitted for field " +
sfield.getName() + ": " +field.getValue() );
}
@@ -260,7 +260,7 @@ public class DocumentBuilder {
hasField = true;
if (sfield != null) {
used = true;
- addField(out, sfield, v, omitNorms ? 1F : docBoost*boost);
+ addField(out, sfield, v, applyBoost ? docBoost*boost : 1f);
}
// Check if we should copy this field to any other fields.
@@ -282,7 +282,7 @@ public class DocumentBuilder {
if( val instanceof String && cf.getMaxChars() > 0 ) {
val = cf.getLimitedValue((String)val);
}
- addField(out, destinationField, val, destinationField.omitNorms() ? 1F : docBoost*boost);
+ addField(out, destinationField, val, destinationField.indexed() && !destinationField.omitNorms() ? docBoost*boost : 1F);
}
// In lucene, the boost for a given field is the product of the
Modified: lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java (original)
+++ lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java Thu Aug 9 10:20:53 2012
@@ -24,16 +24,12 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.BlockingQueue;
import java.util.concurrent.Callable;
import java.util.concurrent.CompletionService;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorCompletionService;
import java.util.concurrent.Future;
-import java.util.concurrent.RejectedExecutionException;
-import java.util.concurrent.Semaphore;
-import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
@@ -48,6 +44,7 @@ import org.apache.solr.common.cloud.ZkCo
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.SolrCore;
+import org.apache.solr.util.AdjustableSemaphore;
import org.apache.solr.util.DefaultSolrThreadFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -58,15 +55,17 @@ public class SolrCmdDistributor {
public static Logger log = LoggerFactory.getLogger(SolrCmdDistributor.class);
// TODO: shut this thing down
- // TODO: this cannot be per instance...
- static BoundedExecutor commExecutor;
+ static ThreadPoolExecutor commExecutor = new ThreadPoolExecutor(0, Integer.MAX_VALUE, 5,
+ TimeUnit.SECONDS, new SynchronousQueue<Runnable>(),
+ new DefaultSolrThreadFactory("cmdDistribExecutor"));;
static final HttpClient client;
+ static AdjustableSemaphore semaphore = new AdjustableSemaphore(8);
static {
ModifiableSolrParams params = new ModifiableSolrParams();
- params.set(HttpClientUtil.PROP_MAX_CONNECTIONS, 200);
- params.set(HttpClientUtil.PROP_MAX_CONNECTIONS_PER_HOST, 8);
+ params.set(HttpClientUtil.PROP_MAX_CONNECTIONS, 500);
+ params.set(HttpClientUtil.PROP_MAX_CONNECTIONS_PER_HOST, 16);
client = HttpClientUtil.createClient(params);
}
@@ -92,20 +91,14 @@ public class SolrCmdDistributor {
}
public SolrCmdDistributor(int numHosts) {
-
- BoundedExecutor executor = null;
- synchronized (SolrCmdDistributor.class) {
- if (commExecutor == null || commExecutor.getMaximumPoolSize() != numHosts) {
- // we don't shutdown the previous because all it's threads will die
- int maxPoolSize = Math.max(8, (numHosts-1) * 8);
- commExecutor = new BoundedExecutor(0, maxPoolSize, 5,
- TimeUnit.SECONDS, new ArrayBlockingQueue<Runnable>(maxPoolSize * 2),
- new DefaultSolrThreadFactory("cmdDistribExecutor"));
- }
- executor = commExecutor;
- }
+ int maxPermits = Math.max(8, (numHosts - 1) * 8);
- completionService = new ExecutorCompletionService<Request>(executor);
+ // limits how many tasks can actually execute at once
+ if (maxPermits != semaphore.getMaxPermits()) {
+ semaphore.setMaxPermits(maxPermits);
+ }
+
+ completionService = new ExecutorCompletionService<Request>(commExecutor);
pending = new HashSet<Future<Request>>();
}
@@ -221,7 +214,7 @@ public class SolrCmdDistributor {
for (Node node : nodes) {
List<AddRequest> alist = adds.get(node);
- if (alist == null || alist.size() < limit) return false;
+ if (alist == null || alist.size() < limit) continue;
UpdateRequestExt ureq = new UpdateRequestExt();
@@ -256,7 +249,7 @@ public class SolrCmdDistributor {
Set<Node> nodes = deletes.keySet();
for (Node node : nodes) {
List<DeleteRequest> dlist = deletes.get(node);
- if (dlist == null || dlist.size() < limit) return false;
+ if (dlist == null || dlist.size() < limit) continue;
UpdateRequestExt ureq = new UpdateRequestExt();
ModifiableSolrParams combinedParams = new ModifiableSolrParams();
@@ -343,11 +336,17 @@ public class SolrCmdDistributor {
} else {
clonedRequest.rspCode = -1;
}
+ } finally {
+ semaphore.release();
}
return clonedRequest;
}
};
-
+ try {
+ semaphore.acquire();
+ } catch (InterruptedException e) {
+ throw new RuntimeException();
+ }
pending.add(completionService.submit(task));
}
@@ -514,39 +513,6 @@ public class SolrCmdDistributor {
}
}
- public class BoundedExecutor extends ThreadPoolExecutor {
- private final Semaphore semaphore;
-
- public BoundedExecutor(int corePoolSize,
- int maximumPoolSize, long keepAliveTime, TimeUnit unit,
- BlockingQueue<Runnable> workQueue, ThreadFactory threadFactory) {
- super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, threadFactory);
- this.semaphore = new Semaphore(maximumPoolSize);
- }
-
- @Override
- public void execute(final Runnable command) {
- try {
- semaphore.acquire();
- } catch (InterruptedException e1) {
- throw new RuntimeException();
- }
- try {
- super.execute(new Runnable() {
- public void run() {
- try {
- command.run();
- } finally {
- semaphore.release();
- }
- }
- });
- } catch (RejectedExecutionException e) {
- semaphore.release();
- throw e;
- }
- }
-}
}
Modified: lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/update/SolrCoreState.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/update/SolrCoreState.java?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/update/SolrCoreState.java (original)
+++ lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/update/SolrCoreState.java Thu Aug 9 10:20:53 2012
@@ -31,15 +31,21 @@ import org.apache.solr.util.RefCounted;
*
*/
public abstract class SolrCoreState {
+ private final Object deleteLock = new Object();
+
+ public Object getUpdateLock() {
+ return deleteLock;
+ }
/**
* Force the creation of a new IndexWriter using the settings from the given
* SolrCore.
*
* @param core
+ * @param rollback close IndexWriter if false, else rollback
* @throws IOException
*/
- public abstract void newIndexWriter(SolrCore core) throws IOException;
+ public abstract void newIndexWriter(SolrCore core, boolean rollback) throws IOException;
/**
* Get the current IndexWriter. If a new IndexWriter must be created, use the
Modified: lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/update/UpdateHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/update/UpdateHandler.java?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/update/UpdateHandler.java (original)
+++ lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/update/UpdateHandler.java Thu Aug 9 10:20:53 2012
@@ -127,9 +127,11 @@ public abstract class UpdateHandler impl
* Called when the Writer should be opened again - eg when replication replaces
* all of the index files.
*
+ * @param rollback IndexWriter if true else close
+ *
* @throws IOException
*/
- public abstract void newIndexWriter() throws IOException;
+ public abstract void newIndexWriter(boolean rollback) throws IOException;
public abstract SolrCoreState getSolrCoreState();
Modified: lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/update/UpdateLog.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/update/UpdateLog.java?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/update/UpdateLog.java (original)
+++ lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/update/UpdateLog.java Thu Aug 9 10:20:53 2012
@@ -1048,6 +1048,7 @@ public class UpdateLog implements Plugin
try {
cancelApplyBufferUpdate = false;
if (state != State.BUFFERING) return null;
+ operationFlags &= ~FLAG_GAP;
// handle case when no log was even created because no updates
// were received.
@@ -1057,7 +1058,6 @@ public class UpdateLog implements Plugin
}
tlog.incref();
state = State.APPLYING_BUFFERED;
- operationFlags &= ~FLAG_GAP;
} finally {
versionInfo.unblockUpdates();
}
Modified: lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java (original)
+++ lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java Thu Aug 9 10:20:53 2012
@@ -36,7 +36,7 @@ import org.apache.solr.common.SolrExcept
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.SolrInputField;
-import org.apache.solr.common.cloud.CloudState;
+import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.ZkCoreNodeProps;
import org.apache.solr.common.cloud.ZkNodeProps;
@@ -158,17 +158,20 @@ public class DistributedUpdateProcessor
CoreDescriptor coreDesc = req.getCore().getCoreDescriptor();
this.zkEnabled = coreDesc.getCoreContainer().isZooKeeperAware();
+ zkController = req.getCore().getCoreDescriptor().getCoreContainer().getZkController();
+ if (zkEnabled) {
+ numNodes = zkController.getZkStateReader().getClusterState().getLiveNodes().size();
+ }
//this.rsp = reqInfo != null ? reqInfo.getRsp() : null;
-
- zkController = req.getCore().getCoreDescriptor().getCoreContainer().getZkController();
+
cloudDesc = coreDesc.getCloudDescriptor();
if (cloudDesc != null) {
collection = cloudDesc.getCollectionName();
}
-
+
cmdDistrib = new SolrCmdDistributor(numNodes);
}
@@ -178,13 +181,13 @@ public class DistributedUpdateProcessor
// if we are in zk mode...
if (zkEnabled) {
// set num nodes
- numNodes = zkController.getCloudState().getLiveNodes().size();
+ numNodes = zkController.getClusterState().getLiveNodes().size();
// the leader is...
// TODO: if there is no leader, wait and look again
// TODO: we are reading the leader from zk every time - we should cache
// this and watch for changes?? Just pull it from ZkController cluster state probably?
- String shardId = getShard(hash, collection, zkController.getCloudState()); // get the right shard based on the hash...
+ String shardId = getShard(hash, collection, zkController.getClusterState()); // get the right shard based on the hash...
try {
// TODO: if we find out we cannot talk to zk anymore, we should probably realize we are not
@@ -249,11 +252,11 @@ public class DistributedUpdateProcessor
}
- private String getShard(int hash, String collection, CloudState cloudState) {
+ private String getShard(int hash, String collection, ClusterState clusterState) {
// ranges should be part of the cloud state and eventually gotten from zk
// get the shard names
- return cloudState.getShard(hash, collection);
+ return clusterState.getShard(hash, collection);
}
// used for deleteByQuery to get the list of nodes this leader should forward to
@@ -695,11 +698,11 @@ public class DistributedUpdateProcessor
if (zkEnabled && DistribPhase.NONE == phase) {
boolean leaderForAnyShard = false; // start off by assuming we are not a leader for any shard
- Map<String,Slice> slices = zkController.getCloudState().getSlices(collection);
+ Map<String,Slice> slices = zkController.getClusterState().getSlices(collection);
if (slices == null) {
throw new SolrException(ErrorCode.BAD_REQUEST,
"Cannot find collection:" + collection + " in "
- + zkController.getCloudState().getCollections());
+ + zkController.getClusterState().getCollections());
}
ModifiableSolrParams params = new ModifiableSolrParams(req.getParams());
@@ -994,13 +997,13 @@ public class DistributedUpdateProcessor
private List<Node> getCollectionUrls(SolrQueryRequest req, String collection, String shardZkNodeName) {
- CloudState cloudState = req.getCore().getCoreDescriptor()
- .getCoreContainer().getZkController().getCloudState();
+ ClusterState clusterState = req.getCore().getCoreDescriptor()
+ .getCoreContainer().getZkController().getClusterState();
List<Node> urls = new ArrayList<Node>();
- Map<String,Slice> slices = cloudState.getSlices(collection);
+ Map<String,Slice> slices = clusterState.getSlices(collection);
if (slices == null) {
throw new ZooKeeperException(ErrorCode.BAD_REQUEST,
- "Could not find collection in zk: " + cloudState);
+ "Could not find collection in zk: " + clusterState);
}
for (Map.Entry<String,Slice> sliceEntry : slices.entrySet()) {
Slice replicas = slices.get(sliceEntry.getKey());
@@ -1009,7 +1012,7 @@ public class DistributedUpdateProcessor
for (Entry<String,ZkNodeProps> entry : shardMap.entrySet()) {
ZkCoreNodeProps nodeProps = new ZkCoreNodeProps(entry.getValue());
- if (cloudState.liveNodesContain(nodeProps.getNodeName()) && !entry.getKey().equals(shardZkNodeName)) {
+ if (clusterState.liveNodesContain(nodeProps.getNodeName()) && !entry.getKey().equals(shardZkNodeName)) {
urls.add(new StdNode(nodeProps));
}
}
Modified: lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/util/SimplePostTool.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/util/SimplePostTool.java?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/util/SimplePostTool.java (original)
+++ lucene/dev/branches/lucene3312/solr/core/src/java/org/apache/solr/util/SimplePostTool.java Thu Aug 9 10:20:53 2012
@@ -18,123 +18,170 @@ package org.apache.solr.util;
*/
import java.io.File;
+import java.io.FileFilter;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.ByteArrayInputStream;
import java.io.OutputStream;
import java.io.UnsupportedEncodingException;
+import java.util.Locale;
+import java.util.HashMap;
import java.util.Set;
import java.util.HashSet;
+import java.util.regex.Pattern;
+import java.util.regex.PatternSyntaxException;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.ProtocolException;
import java.net.URL;
+import java.net.URLEncoder;
/**
* A simple utility class for posting raw updates to a Solr server,
* has a main method so it can be run on the command line.
- *
*/
public class SimplePostTool {
public static final String DEFAULT_POST_URL = "http://localhost:8983/solr/update";
- public static final String VERSION_OF_THIS_TOOL = "1.4";
+ public static final String VERSION_OF_THIS_TOOL = "1.5";
private static final String DEFAULT_COMMIT = "yes";
private static final String DEFAULT_OPTIMIZE = "no";
private static final String DEFAULT_OUT = "no";
+ private static final String DEFAULT_AUTO = "no";
+ private static final String DEFAULT_RECURSIVE = "no";
- public static final String DEFAULT_DATA_TYPE = "application/xml";
+ private static final String DEFAULT_CONTENT_TYPE = "application/xml";
+ private static final String DEFAULT_FILE_TYPES = "xml,json,csv,pdf,doc,docx,ppt,pptx,xls,xlsx,odt,odp,ods,ott,otp,ots,rtf,htm,html,txt,log";
private static final String DATA_MODE_FILES = "files";
private static final String DATA_MODE_ARGS = "args";
private static final String DATA_MODE_STDIN = "stdin";
private static final String DEFAULT_DATA_MODE = DATA_MODE_FILES;
+ private static final String TRUE_STRINGS = "true,on,yes,1";
+
+ private boolean auto = false;
+ private boolean recursive = false;
+ private String fileTypes;
+
+ private static HashMap<String,String> mimeMap;
+ private GlobFileFilter globFileFilter;
+
private static final Set<String> DATA_MODES = new HashSet<String>();
+ private static final String USAGE_STRING_SHORT =
+ "Usage: java [SystemProperties] -jar post.jar [-h|-] [<file|folder|arg> [<file|folder|arg>...]]";
+
static {
DATA_MODES.add(DATA_MODE_FILES);
DATA_MODES.add(DATA_MODE_ARGS);
DATA_MODES.add(DATA_MODE_STDIN);
+
+ mimeMap = new HashMap<String,String>();
+ mimeMap.put("xml", "text/xml");
+ mimeMap.put("csv", "text/csv");
+ mimeMap.put("json", "application/json");
+ mimeMap.put("pdf", "application/pdf");
+ mimeMap.put("rtf", "text/rtf");
+ mimeMap.put("html", "text/html");
+ mimeMap.put("htm", "text/html");
+ mimeMap.put("doc", "application/msword");
+ mimeMap.put("docx", "application/vnd.openxmlformats-officedocument.wordprocessingml.document");
+ mimeMap.put("ppt", "application/vnd.ms-powerpoint");
+ mimeMap.put("pptx", "application/vnd.openxmlformats-officedocument.presentationml.presentation");
+ mimeMap.put("xls", "application/vnd.ms-excel");
+ mimeMap.put("xlsx", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet");
+ mimeMap.put("odt", "application/vnd.oasis.opendocument.text");
+ mimeMap.put("ott", "application/vnd.oasis.opendocument.text");
+ mimeMap.put("odp", "application/vnd.oasis.opendocument.presentation");
+ mimeMap.put("otp", "application/vnd.oasis.opendocument.presentation");
+ mimeMap.put("ods", "application/vnd.oasis.opendocument.spreadsheet");
+ mimeMap.put("ots", "application/vnd.oasis.opendocument.spreadsheet");
+ mimeMap.put("txt", "text/plain");
+ mimeMap.put("log", "text/plain");
}
protected URL solrUrl;
-
+
public static void main(String[] args) {
- info("version " + VERSION_OF_THIS_TOOL);
+ info("SimplePostTool version " + VERSION_OF_THIS_TOOL);
if (0 < args.length && ("-help".equals(args[0]) || "--help".equals(args[0]) || "-h".equals(args[0]))) {
- System.out.println
- ("This is a simple command line tool for POSTing raw data to a Solr\n"+
- "port. Data can be read from files specified as commandline args,\n"+
- "as raw commandline arg strings, or via STDIN.\n"+
- "Examples:\n"+
- " java -jar post.jar *.xml\n"+
- " java -Ddata=args -jar post.jar '<delete><id>42</id></delete>'\n"+
- " java -Ddata=stdin -jar post.jar < hd.xml\n"+
- " java -Dtype=text/csv -jar post.jar *.csv\n"+
- " java -Dtype=application/json -jar post.jar *.json\n"+
- " java -Durl=http://localhost:8983/solr/update/extract?literal.id=a -Dtype=application/pdf -jar post.jar a.pdf\n"+
- "Other options controlled by System Properties include the Solr\n"+
- "URL to POST to, the Content-Type of the data, whether a commit\n"+
- "or optimize should be executed, and whether the response should\n"+
- "be written to STDOUT. These are the defaults for all System Properties:\n"+
- " -Ddata=" + DEFAULT_DATA_MODE + "\n"+
- " -Dtype=" + DEFAULT_DATA_TYPE + "\n"+
- " -Durl=" + DEFAULT_POST_URL + "\n"+
- " -Dcommit=" + DEFAULT_COMMIT + "\n"+
- " -Doptimize=" + DEFAULT_OPTIMIZE + "\n"+
- " -Dout=" + DEFAULT_OUT + "\n");
+ usage();
return;
}
-
+
OutputStream out = null;
- final String type = System.getProperty("type", DEFAULT_DATA_TYPE);
+ final String type = System.getProperty("type");
+
+ final String params = System.getProperty("params", "");
URL u = null;
try {
- u = new URL(System.getProperty("url", DEFAULT_POST_URL));
+ u = new URL(System.getProperty("url", SimplePostTool.appendParam(DEFAULT_POST_URL, params)));
} catch (MalformedURLException e) {
fatal("System Property 'url' is not a valid URL: " + u);
}
final SimplePostTool t = new SimplePostTool(u);
+ if (isOn(System.getProperty("auto", DEFAULT_AUTO))) {
+ t.setAuto(true);
+ }
+
+ if (isOn(System.getProperty("recursive", DEFAULT_RECURSIVE))) {
+ t.setRecursive(true);
+ }
+
final String mode = System.getProperty("data", DEFAULT_DATA_MODE);
if (! DATA_MODES.contains(mode)) {
fatal("System Property 'data' is not valid for this tool: " + mode);
}
- if ("yes".equals(System.getProperty("out", DEFAULT_OUT))) {
+ if (isOn(System.getProperty("out", DEFAULT_OUT))) {
out = System.out;
}
+
+ t.setFileTypes(System.getProperty("filetypes", DEFAULT_FILE_TYPES));
+ int numFilesPosted = 0;
+
try {
if (DATA_MODE_FILES.equals(mode)) {
if (0 < args.length) {
- info("POSTing files to " + u + "..");
- t.postFiles(args, 0, out, type);
+ // Skip posting files if special param "-" given
+ if (!args[0].equals("-")) {
+ info("Posting files to base url " + u + (!t.auto?" using content-type "+(type==null?DEFAULT_CONTENT_TYPE:type):"")+"..");
+ if(t.auto)
+ info("Entering auto mode. File endings considered are "+t.getFileTypes());
+ if(t.recursive)
+ info("Entering recursive mode");
+ numFilesPosted = t.postFiles(args, 0, out, type);
+ info(numFilesPosted + " files indexed.");
+ }
} else {
- info("No files specified. (Use -h for help)");
+ usageShort();
+ return;
}
-
} else if (DATA_MODE_ARGS.equals(mode)) {
if (0 < args.length) {
info("POSTing args to " + u + "..");
for (String a : args) {
t.postData(SimplePostTool.stringToStream(a), null, out, type);
}
+ } else {
+ usageShort();
+ return;
}
-
} else if (DATA_MODE_STDIN.equals(mode)) {
info("POSTing stdin to " + u + "..");
t.postData(System.in, null, out, type);
}
- if ("yes".equals(System.getProperty("commit",DEFAULT_COMMIT))) {
- info("COMMITting Solr index changes..");
+ if (isOn(System.getProperty("commit",DEFAULT_COMMIT))) {
+ info("COMMITting Solr index changes to " + u + "..");
t.commit();
}
- if ("yes".equals(System.getProperty("optimize",DEFAULT_OPTIMIZE))) {
- info("Performing an OPTIMIZE..");
+ if (isOn(System.getProperty("optimize",DEFAULT_OPTIMIZE))) {
+ info("Performing an OPTIMIZE to " + u + "..");
t.optimize();
}
@@ -144,37 +191,115 @@ public class SimplePostTool {
}
}
- /**
- * @deprecated use {@link #postData(InputStream, Integer, OutputStream, String)} instead
- */
- @Deprecated
- int postFiles(String [] args,int startIndexInArgs, OutputStream out) {
- final String type = System.getProperty("type", DEFAULT_DATA_TYPE);
- return postFiles(args, startIndexInArgs, out, type);
- }
-
- /** Post all filenames provided in args, return the number of files posted*/
+ private static void usageShort() {
+ System.out.println(USAGE_STRING_SHORT+"\n"+
+ " Please invoke with -h option for extended usage help.");
+ }
+
+ private static void usage() {
+ System.out.println
+ (USAGE_STRING_SHORT+"\n\n" +
+ "Supported System Properties and their defaults:\n"+
+ " -Ddata=files|args|stdin (default=" + DEFAULT_DATA_MODE + ")\n"+
+ " -Dtype=<content-type> (default=" + DEFAULT_CONTENT_TYPE + ")\n"+
+ " -Durl=<solr-update-url> (default=" + DEFAULT_POST_URL + ")\n"+
+ " -Dauto=yes|no (default=" + DEFAULT_AUTO + ")\n"+
+ " -Drecursive=yes|no (default=" + DEFAULT_RECURSIVE + ")\n"+
+ " -Dfiletypes=<type>[,<type>,...] (default=" + DEFAULT_FILE_TYPES + ")\n"+
+ " -Dparams=\"<key>=<value>[&<key>=<value>...]\" (values must be URL-encoded)\n"+
+ " -Dcommit=yes|no (default=" + DEFAULT_COMMIT + ")\n"+
+ " -Doptimize=yes|no (default=" + DEFAULT_OPTIMIZE + ")\n"+
+ " -Dout=yes|no (default=" + DEFAULT_OUT + ")\n\n"+
+ "This is a simple command line tool for POSTing raw data to a Solr\n"+
+ "port. Data can be read from files specified as commandline args,\n"+
+ "as raw commandline arg strings, or via STDIN.\n"+
+ "Examples:\n"+
+ " java -jar post.jar *.xml\n"+
+ " java -Ddata=args -jar post.jar '<delete><id>42</id></delete>'\n"+
+ " java -Ddata=stdin -jar post.jar < hd.xml\n"+
+ " java -Dtype=text/csv -jar post.jar *.csv\n"+
+ " java -Dtype=application/json -jar post.jar *.json\n"+
+ " java -Durl=http://localhost:8983/solr/update/extract -Dparams=literal.id=a -Dtype=application/pdf -jar post.jar a.pdf\n"+
+ " java -Dauto -jar post.jar *\n"+
+ " java -Dauto -Drecursive -jar post.jar afolder\n"+
+ " java -Dauto -Dfiletypes=ppt,html -jar post.jar afolder\n"+
+ "The options controlled by System Properties include the Solr\n"+
+ "URL to POST to, the Content-Type of the data, whether a commit\n"+
+ "or optimize should be executed, and whether the response should\n"+
+ "be written to STDOUT. If auto=yes the tool will try to set type\n"+
+ "and url automatically from file name. When posting rich documents\n"+
+ "the file name will be propagated as \"resource.name\" and also used as \"literal.id\".\n" +
+ "You may override these or any other request parameter through the -Dparams property.\n"+
+ "If you want to do a commit only, use \"-\" as argument.");
+ }
+
+ private static boolean isOn(String property) {
+ return(TRUE_STRINGS.indexOf(property) >= 0);
+ }
+
+ /** Post all filenames provided in args
+ * @param args array of file names
+ * @param startIndexInArgs offset to start
+ * @param out output stream to post data to
+ * @param type default content-type to use when posting (may be overridden in auto mode)
+ * @return number of files posted
+ * */
int postFiles(String [] args,int startIndexInArgs, OutputStream out, String type) {
int filesPosted = 0;
for (int j = startIndexInArgs; j < args.length; j++) {
File srcFile = new File(args[j]);
- if (srcFile.canRead()) {
- info("POSTing file " + srcFile.getName());
- postFile(srcFile, out, type);
- filesPosted++;
+ if(srcFile.isDirectory() && srcFile.canRead()) {
+ filesPosted += postDirectory(srcFile, out, type);
+ } else if (srcFile.isFile() && srcFile.canRead()) {
+ filesPosted += postFiles(new File[] {srcFile}, out, type);
} else {
- warn("Cannot read input file: " + srcFile);
+ File parent = srcFile.getParentFile();
+ if(parent == null) parent = new File(".");
+ String fileGlob = srcFile.getName();
+ GlobFileFilter ff = new GlobFileFilter(fileGlob, false);
+ File[] files = parent.listFiles(ff);
+ if(files.length == 0) {
+ warn("No files or directories matching "+srcFile);
+ continue;
+ }
+ filesPosted += postFiles(parent.listFiles(ff), out, type);
}
}
return filesPosted;
}
+ private int postDirectory(File dir, OutputStream out, String type) {
+ if(dir.isHidden() && !dir.getName().equals("."))
+ return(0);
+ info("Indexing directory "+dir.getPath());
+ int posted = 0;
+ posted += postFiles(dir.listFiles(globFileFilter), out, type);
+ if(recursive) {
+ for(File d : dir.listFiles()) {
+ if(d.isDirectory())
+ posted += postDirectory(d, out, type);
+ }
+ }
+ return posted;
+ }
+
+ int postFiles(File[] files, OutputStream out, String type) {
+ int filesPosted = 0;
+ for(File srcFile : files) {
+ if(!srcFile.isFile() || srcFile.isHidden())
+ continue;
+ postFile(srcFile, out, type);
+ filesPosted++;
+ }
+ return filesPosted;
+ }
+
static void warn(String msg) {
System.err.println("SimplePostTool: WARNING: " + msg);
}
static void info(String msg) {
- System.out.println("SimplePostTool: " + msg);
+ System.out.println(msg);
}
static void fatal(String msg) {
@@ -204,30 +329,58 @@ public class SimplePostTool {
doGet(appendParam(solrUrl.toString(), "optimize=true"));
}
- private String appendParam(String url, String param) {
- return url + (url.indexOf('?')>0 ? "&" : "?") + param;
+ public static String appendParam(String url, String param) {
+ String[] pa = param.split("&");
+ for(String p : pa) {
+ if(p.trim().length() == 0) continue;
+ String[] kv = p.split("=");
+ if(kv.length == 2) {
+ url = url + (url.indexOf('?')>0 ? "&" : "?") + kv[0] +"="+ kv[1];
+ } else {
+ warn("Skipping param "+p+" which is not on form key=value");
+ }
+ }
+ return url;
}
/**
- * @deprecated use {@link #postFile(File, OutputStream, String)} instead
- */
- public void postFile(File file, OutputStream output) {
- final String type = System.getProperty("type", DEFAULT_DATA_TYPE);
- postFile(file, output, type);
- }
-
- /**
* Opens the file and posts it's contents to the solrUrl,
* writes to response to output.
*/
public void postFile(File file, OutputStream output, String type) {
-
InputStream is = null;
try {
+ URL url = solrUrl;
+ if(auto) {
+ if(type == null) {
+ type = guessType(file);
+ }
+ if(type != null) {
+ if(type.equals("text/xml") || type.equals("text/csv") || type.equals("application/json")) {
+ // Default handler
+ } else {
+ // SolrCell
+ String urlStr = url.getProtocol() + "://" + url.getAuthority() + url.getPath() + "/extract" + (url.getQuery() != null ? "?"+url.getQuery() : "");
+ if(urlStr.indexOf("resource.name")==-1)
+ urlStr = appendParam(urlStr, "resource.name=" + URLEncoder.encode(file.getAbsolutePath(), "UTF-8"));
+ if(urlStr.indexOf("literal.id")==-1)
+ urlStr = appendParam(urlStr, "literal.id=" + URLEncoder.encode(file.getAbsolutePath(), "UTF-8"));
+ url = new URL(urlStr);
+// info("Indexing to ExtractingRequestHandler with URL "+url);
+ }
+ } else {
+ warn("Skipping "+file.getName()+". Unsupported file type for auto mode.");
+ return;
+ }
+ } else {
+ if(type == null) type = DEFAULT_CONTENT_TYPE;
+ }
+ info("POSTing file " + file.getName() + (auto?" ("+type+")":""));
is = new FileInputStream(file);
- postData(is, (int)file.length(), output, type);
+ postData(is, (int)file.length(), output, type, url);
} catch (IOException e) {
- fatal("Can't open/read file: " + file);
+ e.printStackTrace();
+ warn("Can't open/read file: " + file);
} finally {
try {
if(is!=null) is.close();
@@ -237,53 +390,53 @@ public class SimplePostTool {
}
}
+ private String guessType(File file) {
+ String name = file.getName();
+ String suffix = name.substring(name.lastIndexOf(".")+1);
+ return mimeMap.get(suffix.toLowerCase(Locale.ROOT));
+ }
+
/**
* Performs a simple get on the given URL
- * @param url
*/
public static void doGet(String url) {
try {
doGet(new URL(url));
} catch (MalformedURLException e) {
- fatal("The specified URL "+url+" is not a valid URL. Please check");
+ warn("The specified URL "+url+" is not a valid URL. Please check");
}
}
/**
* Performs a simple get on the given URL
- * @param url
*/
public static void doGet(URL url) {
try {
HttpURLConnection urlc = (HttpURLConnection) url.openConnection();
if (HttpURLConnection.HTTP_OK != urlc.getResponseCode()) {
- fatal("Solr returned an error #" + urlc.getResponseCode() +
- " " + urlc.getResponseMessage());
+ warn("Solr returned an error #" + urlc.getResponseCode() +
+ " " + urlc.getResponseMessage() + " for url "+url);
}
} catch (IOException e) {
- fatal("An error occured posting data to "+url+". Please check that Solr is running.");
+ warn("An error occured posting data to "+url+". Please check that Solr is running.");
}
}
- /**
- * @deprecated use {@link #postData(InputStream, Integer, OutputStream, String)} instead
- */
- @Deprecated
- public void postData(InputStream data, Integer length, OutputStream output) {
- final String type = System.getProperty("type", DEFAULT_DATA_TYPE);
- postData(data, length, output, type);
+ public void postData(InputStream data, Integer length, OutputStream output, String type) {
+ postData(data, length, output, type, solrUrl);
}
-
+
/**
* Reads data from the data stream and posts it to solr,
* writes to the response to output
*/
- public void postData(InputStream data, Integer length, OutputStream output, String type) {
-
+ public void postData(InputStream data, Integer length, OutputStream output, String type, URL url) {
+ if(type == null)
+ type = DEFAULT_CONTENT_TYPE;
HttpURLConnection urlc = null;
try {
try {
- urlc = (HttpURLConnection) solrUrl.openConnection();
+ urlc = (HttpURLConnection) url.openConnection();
try {
urlc.setRequestMethod("POST");
} catch (ProtocolException e) {
@@ -315,14 +468,14 @@ public class SimplePostTool {
InputStream in = null;
try {
if (HttpURLConnection.HTTP_OK != urlc.getResponseCode()) {
- fatal("Solr returned an error #" + urlc.getResponseCode() +
+ warn("Solr returned an error #" + urlc.getResponseCode() +
" " + urlc.getResponseMessage());
}
in = urlc.getInputStream();
pipe(in, output);
} catch (IOException e) {
- fatal("IOException while reading response: " + e);
+ warn("IOException while reading response: " + e);
} finally {
try { if(in!=null) in.close(); } catch (IOException x) { /*NOOP*/ }
}
@@ -354,4 +507,68 @@ public class SimplePostTool {
}
if (null != dest) dest.flush();
}
+
+ public boolean isAuto() {
+ return auto;
+ }
+
+ public void setAuto(boolean auto) {
+ this.auto = auto;
+ }
+
+ public boolean isRecursive() {
+ return recursive;
+ }
+
+ public void setRecursive(boolean recursive) {
+ this.recursive = recursive;
+ }
+
+ public String getFileTypes() {
+ return fileTypes;
+ }
+
+ public void setFileTypes(String fileTypes) {
+ this.fileTypes = fileTypes;
+ String glob;
+ if(fileTypes.equals("*"))
+ glob = ".*";
+ else
+ glob = "^.*\\.(" + fileTypes.replace(",", "|") + ")$";
+ this.globFileFilter = new GlobFileFilter(glob, true);
+ }
+
+ class GlobFileFilter implements FileFilter
+ {
+ private String _pattern;
+ private Pattern p;
+
+ public GlobFileFilter(String pattern, boolean isRegex)
+ {
+ _pattern = pattern;
+ if(!isRegex) {
+ _pattern = _pattern
+ .replace("^", "\\^")
+ .replace("$", "\\$")
+ .replace(".", "\\.")
+ .replace("(", "\\(")
+ .replace(")", "\\)")
+ .replace("+", "\\+")
+ .replace("*", ".*")
+ .replace("?", ".");
+ _pattern = "^" + _pattern + "$";
+ }
+
+ try {
+ p = Pattern.compile(_pattern,Pattern.CASE_INSENSITIVE);
+ } catch(PatternSyntaxException e) {
+ fatal("Invalid type list "+pattern+". "+e.getDescription());
+ }
+ }
+
+ public boolean accept(File file)
+ {
+ return p.matcher(file.getName()).find();
+ }
+ }
}
Modified: lucene/dev/branches/lucene3312/solr/core/src/test-files/solr/collection1/conf/schema.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/core/src/test-files/solr/collection1/conf/schema.xml?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/core/src/test-files/solr/collection1/conf/schema.xml (original)
+++ lucene/dev/branches/lucene3312/solr/core/src/test-files/solr/collection1/conf/schema.xml Thu Aug 9 10:20:53 2012
@@ -414,7 +414,32 @@
<fieldType name="location" class="solr.LatLonType" subFieldSuffix="_coordinate"/>
- </types>
+ <!--
+ Example of using PathHierarchyTokenizerFactory at index time, so
+ queries for paths match documents at that path, or in descendent paths
+ -->
+ <fieldType name="path" class="solr.TextField">
+ <analyzer type="index">
+ <tokenizer class="solr.PathHierarchyTokenizerFactory" delimiter="/" />
+ </analyzer>
+ <analyzer type="query">
+ <tokenizer class="solr.KeywordTokenizerFactory" />
+ </analyzer>
+ </fieldType>
+ <!--
+ Example of using PathHierarchyTokenizerFactory at query time, so
+ queries for paths match documents at that path, or in ancestor paths
+ -->
+ <fieldType name="ancestor_path" class="solr.TextField">
+ <analyzer type="index">
+ <tokenizer class="solr.KeywordTokenizerFactory" />
+ </analyzer>
+ <analyzer type="query">
+ <tokenizer class="solr.PathHierarchyTokenizerFactory" delimiter="/" />
+ </analyzer>
+ </fieldType>
+
+</types>
<fields>
@@ -640,6 +665,10 @@
<!-- Type used to index the lat and lon components for the "location" FieldType -->
<dynamicField name="*_coordinate" type="tdouble" indexed="true" stored="false" omitNorms="true" />
+
+ <dynamicField name="*_path" type="path" indexed="true" stored="true" omitNorms="true" multiValued="true" />
+ <dynamicField name="*_ancestor" type="ancestor_path" indexed="true" stored="true" omitNorms="true" multiValued="true" />
+
</fields>
<defaultSearchField>text</defaultSearchField>
@@ -673,6 +702,7 @@
<!-- dynamic destination -->
<copyField source="*_dynamic" dest="dynamic_*"/>
+ <copyField source="*_path" dest="*_ancestor"/>
<!-- example of a custom similarity -->
<similarity class="solr.CustomSimilarityFactory">
Modified: lucene/dev/branches/lucene3312/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml (original)
+++ lucene/dev/branches/lucene3312/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml Thu Aug 9 10:20:53 2012
@@ -39,4 +39,6 @@
</updateLog>
</updateHandler>
+ <requestHandler name="/admin/" class="org.apache.solr.handler.admin.AdminHandlers" />
+
</config>
Modified: lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/TestDistributedSearch.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/TestDistributedSearch.java?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/TestDistributedSearch.java (original)
+++ lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/TestDistributedSearch.java Thu Aug 9 10:20:53 2012
@@ -310,7 +310,8 @@ public class TestDistributedSearch exten
unIgnoreException("isShard is only acceptable");
// test debugging
- handle.put("explain", UNORDERED);
+ // handle.put("explain", UNORDERED);
+ handle.put("explain", SKIPVAL); // internal docids differ, idf differs w/o global idf
handle.put("debug", UNORDERED);
handle.put("time", SKIPVAL);
query("q","now their fox sat had put","fl","*,score",CommonParams.DEBUG_QUERY, "true");
Modified: lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java (original)
+++ lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java Thu Aug 9 10:20:53 2012
@@ -57,7 +57,7 @@ import org.apache.solr.common.SolrDocume
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.SolrInputDocument;
-import org.apache.solr.common.cloud.CloudState;
+import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.ZkCoreNodeProps;
import org.apache.solr.common.cloud.ZkNodeProps;
@@ -71,10 +71,11 @@ import org.apache.solr.update.SolrCmdDis
import org.apache.solr.util.DefaultSolrThreadFactory;
/**
- *
+ * This test simply does a bunch of basic things in solrcloud mode and asserts things
+ * work as expected.
*/
@Slow
-public class BasicDistributedZkTest extends AbstractDistributedZkTestCase {
+public class BasicDistributedZkTest extends AbstractDistribZkTestBase {
private static final String DEFAULT_COLLECTION = "collection1";
private static final boolean DEBUG = false;
@@ -281,7 +282,7 @@ public class BasicDistributedZkTest exte
}
// test debugging
- handle.put("explain", UNORDERED);
+ handle.put("explain", SKIPVAL);
handle.put("debug", UNORDERED);
handle.put("time", SKIPVAL);
query("q","now their fox sat had put","fl","*,score",CommonParams.DEBUG_QUERY, "true");
@@ -355,7 +356,7 @@ public class BasicDistributedZkTest exte
HttpSolrServer collectionClient = new HttpSolrServer(url);
// poll for a second - it can take a moment before we are ready to serve
- waitForNon404(collectionClient);
+ waitForNon404or503(collectionClient);
}
List<String> collectionNameList = new ArrayList<String>();
@@ -451,7 +452,7 @@ public class BasicDistributedZkTest exte
private void collectStartTimes(String collectionName,
Map<String,Long> urlToTime) throws SolrServerException, IOException {
Map<String,Map<String,Slice>> collections = solrj.getZkStateReader()
- .getCloudState().getCollectionStates();
+ .getClusterState().getCollectionStates();
if (collections.containsKey(collectionName)) {
Map<String,Slice> slices = collections.get(collectionName);
@@ -478,8 +479,8 @@ public class BasicDistributedZkTest exte
}
private String getUrlFromZk(String collection) {
- CloudState cloudState = solrj.getZkStateReader().getCloudState();
- Map<String,Slice> slices = cloudState.getCollectionStates().get(collection);
+ ClusterState clusterState = solrj.getZkStateReader().getClusterState();
+ Map<String,Slice> slices = clusterState.getCollectionStates().get(collection);
if (slices == null) {
throw new SolrException(ErrorCode.BAD_REQUEST, "Could not find collection:" + collection);
@@ -491,7 +492,7 @@ public class BasicDistributedZkTest exte
Set<Map.Entry<String,ZkNodeProps>> shardEntries = shards.entrySet();
for (Map.Entry<String,ZkNodeProps> shardEntry : shardEntries) {
final ZkNodeProps node = shardEntry.getValue();
- if (cloudState.liveNodesContain(node.get(ZkStateReader.NODE_NAME_PROP))) {
+ if (clusterState.liveNodesContain(node.get(ZkStateReader.NODE_NAME_PROP))) {
return new ZkCoreNodeProps(node).getCoreUrl();
}
}
@@ -500,20 +501,21 @@ public class BasicDistributedZkTest exte
throw new RuntimeException("Could not find a live node for collection:" + collection);
}
- private void waitForNon404(HttpSolrServer collectionClient)
+ private void waitForNon404or503(HttpSolrServer collectionClient)
throws Exception {
-
+ SolrException exp = null;
long timeoutAt = System.currentTimeMillis() + 30000;
while (System.currentTimeMillis() < timeoutAt) {
boolean missing = false;
+
try {
collectionClient.query(new SolrQuery("*:*"));
} catch (SolrException e) {
- // How do I get the response code!?
- if (!e.getMessage().contains("(404)")) {
+ if (!(e.code() == 403 || e.code() == 503)) {
throw e;
}
+ exp = e;
missing = true;
}
if (!missing) {
@@ -522,7 +524,7 @@ public class BasicDistributedZkTest exte
Thread.sleep(50);
}
printLayout();
- fail("Could not find the new collection - 404 : " + collectionClient.getBaseURL());
+ fail("Could not find the new collection - " + exp.code() + " : " + collectionClient.getBaseURL());
}
private void checkForCollection(String collectionName, int expectedSlices)
@@ -532,9 +534,9 @@ public class BasicDistributedZkTest exte
boolean found = false;
boolean sliceMatch = false;
while (System.currentTimeMillis() < timeoutAt) {
- solrj.getZkStateReader().updateCloudState(true);
- CloudState cloudState = solrj.getZkStateReader().getCloudState();
- Map<String,Map<String,Slice>> collections = cloudState
+ solrj.getZkStateReader().updateClusterState(true);
+ ClusterState clusterState = solrj.getZkStateReader().getClusterState();
+ Map<String,Map<String,Slice>> collections = clusterState
.getCollectionStates();
if (collections.containsKey(collectionName)) {
Map<String,Slice> slices = collections.get(collectionName);
@@ -580,9 +582,9 @@ public class BasicDistributedZkTest exte
long timeoutAt = System.currentTimeMillis() + 15000;
boolean found = true;
while (System.currentTimeMillis() < timeoutAt) {
- solrj.getZkStateReader().updateCloudState(true);
- CloudState cloudState = solrj.getZkStateReader().getCloudState();
- Map<String,Map<String,Slice>> collections = cloudState
+ solrj.getZkStateReader().updateClusterState(true);
+ ClusterState clusterState = solrj.getZkStateReader().getClusterState();
+ Map<String,Map<String,Slice>> collections = clusterState
.getCollectionStates();
if (!collections.containsKey(collectionName)) {
found = false;
@@ -772,8 +774,8 @@ public class BasicDistributedZkTest exte
// we added a role of none on these creates - check for it
ZkStateReader zkStateReader = solrj.getZkStateReader();
- zkStateReader.updateCloudState(true);
- Map<String,Slice> slices = zkStateReader.getCloudState().getSlices(oneInstanceCollection2);
+ zkStateReader.updateClusterState(true);
+ Map<String,Slice> slices = zkStateReader.getClusterState().getSlices(oneInstanceCollection2);
assertNotNull(slices);
String roles = slices.get("slice1").getShards().values().iterator().next().get(ZkStateReader.ROLES_PROP);
assertEquals("none", roles);
Modified: lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/BasicZkTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/BasicZkTest.java?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/BasicZkTest.java (original)
+++ lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/BasicZkTest.java Thu Aug 9 10:20:53 2012
@@ -126,8 +126,8 @@ public class BasicZkTest extends Abstrac
// ensure zk still thinks node is up
assertTrue(
- zkController.getCloudState().getLiveNodes().toString(),
- zkController.getCloudState().liveNodesContain(
+ zkController.getClusterState().getLiveNodes().toString(),
+ zkController.getClusterState().liveNodesContain(
zkController.getNodeName()));
// test maxint
Modified: lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java (original)
+++ lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java Thu Aug 9 10:20:53 2012
@@ -39,7 +39,7 @@ import org.slf4j.LoggerFactory;
@Slow
@Ignore("ignore while investigating jenkins fails")
-public class ChaosMonkeyNothingIsSafeTest extends FullSolrCloudTest {
+public class ChaosMonkeyNothingIsSafeTest extends AbstractFullDistribZkTestBase {
public static Logger log = LoggerFactory.getLogger(ChaosMonkeyNothingIsSafeTest.class);
private static final int BASE_RUN_LENGTH = 180000;
@@ -148,8 +148,8 @@ public class ChaosMonkeyNothingIsSafeTes
// TODO: assert we didnt kill everyone
- zkStateReader.updateCloudState(true);
- assertTrue(zkStateReader.getCloudState().getLiveNodes().size() > 0);
+ zkStateReader.updateClusterState(true);
+ assertTrue(zkStateReader.getClusterState().getLiveNodes().size() > 0);
checkShardConsistency(false, true);