You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ja...@apache.org on 2013/05/30 09:53:46 UTC
svn commit: r1487777 [39/50] - in /lucene/dev/branches/security: ./
dev-tools/ dev-tools/eclipse/dot.settings/ dev-tools/idea/.idea/
dev-tools/idea/.idea/libraries/ dev-tools/idea/lucene/replicator/
dev-tools/maven/ dev-tools/maven/lucene/ dev-tools/ma...
Modified: lucene/dev/branches/security/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java (original)
+++ lucene/dev/branches/security/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java Thu May 30 07:53:18 2013
@@ -28,8 +28,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Properties;
-
-import javax.xml.parsers.ParserConfigurationException;
+import java.util.concurrent.Future;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringUtils;
@@ -42,6 +41,7 @@ import org.apache.solr.cloud.ZkControlle
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.cloud.ClusterState;
+import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.DocRouter;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.ZkNodeProps;
@@ -66,6 +66,7 @@ import org.apache.solr.response.SolrQuer
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.update.MergeIndexesCommand;
import org.apache.solr.update.SplitIndexCommand;
+import org.apache.solr.update.UpdateLog;
import org.apache.solr.update.processor.UpdateRequestProcessor;
import org.apache.solr.update.processor.UpdateRequestProcessorChain;
import org.apache.solr.util.NumberUtils;
@@ -73,7 +74,6 @@ import org.apache.solr.util.RefCounted;
import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.xml.sax.SAXException;
/**
*
@@ -201,6 +201,12 @@ public class CoreAdminHandler extends Re
break;
}
+ // todo : Can this be done by the regular RecoveryStrategy route?
+ case REQUESTAPPLYUPDATES: {
+ this.handleRequestApplyUpdatesAction(req, rsp);
+ break;
+ }
+
default: {
doPersist = this.handleCustomAction(req, rsp);
break;
@@ -217,20 +223,26 @@ public class CoreAdminHandler extends Re
rsp.setHttpCaching(false);
}
+
+ /**
+ * Handle the core admin SPLIT action.
+ * @return true if a modification has resulted that requires persistence
+ * of the CoreContainer configuration.
+ */
protected boolean handleSplitAction(SolrQueryRequest adminReq, SolrQueryResponse rsp) throws IOException {
SolrParams params = adminReq.getParams();
- // partitions=N (split into N partitions, leaving it up to solr what the ranges are and where to put them)
- // path - multiValued param, or comma separated param? Only creates indexes, not cores
-
List<DocRouter.Range> ranges = null;
- // boolean closeDirectories = true;
- // DirectoryFactory dirFactory = null;
String[] pathsArr = params.getParams("path");
String rangesStr = params.get("ranges"); // ranges=a-b,c-d,e-f
String[] newCoreNames = params.getParams("targetCore");
-
String cname = params.get(CoreAdminParams.CORE, "");
+
+ if ((pathsArr == null || pathsArr.length == 0) && (newCoreNames == null || newCoreNames.length == 0)) {
+ throw new SolrException(ErrorCode.BAD_REQUEST, "Either path or targetCore param must be specified");
+ }
+
+ log.info("Invoked split action for core: " + cname);
SolrCore core = coreContainer.getCore(cname);
SolrQueryRequest req = new LocalSolrQueryRequest(core, params);
List<SolrCore> newCores = null;
@@ -238,17 +250,19 @@ public class CoreAdminHandler extends Re
try {
// TODO: allow use of rangesStr in the future
List<String> paths = null;
- int partitions = pathsArr != null ? pathsArr.length : params.getInt("partitions", 2);
+ int partitions = pathsArr != null ? pathsArr.length : newCoreNames.length;
-
- // TODO: if we don't know the real range of the current core, we should just
- // split on every other doc rather than hash.
-
- // TODO (cloud): get from the current core
- DocRouter.Range currentRange = new DocRouter.Range(Integer.MIN_VALUE, Integer.MAX_VALUE);
-
- DocRouter hp = DocRouter.DEFAULT; // TODO: get actual doc router for collection if available
- ranges = hp.partitionRange(partitions, currentRange);
+ DocRouter router = null;
+ if (coreContainer.isZooKeeperAware()) {
+ ClusterState clusterState = coreContainer.getZkController().getClusterState();
+ String collectionName = req.getCore().getCoreDescriptor().getCloudDescriptor().getCollectionName();
+ DocCollection collection = clusterState.getCollection(collectionName);
+ String sliceName = req.getCore().getCoreDescriptor().getCloudDescriptor().getShardId();
+ Slice slice = clusterState.getSlice(collectionName, sliceName);
+ DocRouter.Range currentRange = slice.getRange();
+ router = collection.getRouter() != null ? collection.getRouter() : DocRouter.DEFAULT;
+ ranges = currentRange != null ? router.partitionRange(partitions, currentRange) : null;
+ }
if (pathsArr == null) {
newCores = new ArrayList<SolrCore>(partitions);
@@ -260,15 +274,12 @@ public class CoreAdminHandler extends Re
throw new SolrException(ErrorCode.BAD_REQUEST, "Core with core name " + newCoreName + " expected but doesn't exist.");
}
}
- // TODO (cloud): cores should be registered, should be in recovery / buffering-updates mode, and the shard
- // leader should be forwarding updates to the new shards *before* we split the current shard
- // into the new shards.
} else {
paths = Arrays.asList(pathsArr);
}
- SplitIndexCommand cmd = new SplitIndexCommand(req, paths, newCores, ranges);
+ SplitIndexCommand cmd = new SplitIndexCommand(req, paths, newCores, ranges, router);
core.getUpdateHandler().split(cmd);
// After the split has completed, someone (here?) should start the process of replaying the buffered updates.
@@ -403,17 +414,13 @@ public class CoreAdminHandler extends Re
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
"Core name is mandatory to CREATE a SolrCore");
}
+ CoreDescriptor dcore = null;
try {
- //for now, do not allow creating new core with same name when in cloud mode
- //XXX perhaps it should just be unregistered from cloud before readding it?,
- //XXX perhaps we should also check that cores are of same type before adding new core to collection?
- if (coreContainer.getZkController() != null) {
- if (coreContainer.getCore(name) != null) {
- log.info("Re-creating a core with existing name is not allowed in cloud mode");
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
- "Core with name '" + name + "' already exists.");
- }
+ if (coreContainer.getAllCoreNames().contains(name)) {
+ log.warn("Creating a core with existing name is not allowed");
+ throw new SolrException(ErrorCode.SERVER_ERROR,
+ "Core with name '" + name + "' already exists.");
}
String instanceDir = params.get(CoreAdminParams.INSTANCE_DIR);
@@ -422,7 +429,7 @@ public class CoreAdminHandler extends Re
instanceDir = name; // bare name is already relative to solr home
}
- CoreDescriptor dcore = new CoreDescriptor(coreContainer, name, instanceDir);
+ dcore = new CoreDescriptor(coreContainer, name, instanceDir);
// fillup optional parameters
String opts = params.get(CoreAdminParams.CONFIG);
@@ -436,11 +443,23 @@ public class CoreAdminHandler extends Re
opts = params.get(CoreAdminParams.DATA_DIR);
if (opts != null)
dcore.setDataDir(opts);
-
+
opts = params.get(CoreAdminParams.ULOG_DIR);
if (opts != null)
dcore.setUlogDir(opts);
+ opts = params.get(CoreAdminParams.LOAD_ON_STARTUP);
+ if (opts != null){
+ Boolean value = Boolean.valueOf(opts);
+ dcore.setLoadOnStartup(value);
+ }
+
+ opts = params.get(CoreAdminParams.TRANSIENT);
+ if (opts != null){
+ Boolean value = Boolean.valueOf(opts);
+ dcore.setTransient(value);
+ }
+
CloudDescriptor cd = dcore.getCloudDescriptor();
if (cd != null) {
cd.setParams(req.getParams());
@@ -453,6 +472,14 @@ public class CoreAdminHandler extends Re
if (opts != null)
cd.setShardId(opts);
+ opts = params.get(CoreAdminParams.SHARD_RANGE);
+ if (opts != null)
+ cd.setShardRange(opts);
+
+ opts = params.get(CoreAdminParams.SHARD_STATE);
+ if (opts != null)
+ cd.setShardState(opts);
+
opts = params.get(CoreAdminParams.ROLES);
if (opts != null)
cd.setRoles(opts);
@@ -460,7 +487,7 @@ public class CoreAdminHandler extends Re
opts = params.get(CoreAdminParams.CORE_NODE_NAME);
if (opts != null)
cd.setCoreNodeName(opts);
-
+
Integer numShards = params.getInt(ZkStateReader.NUM_SHARDS_PROP);
if (numShards != null)
cd.setNumShards(numShards);
@@ -480,10 +507,21 @@ public class CoreAdminHandler extends Re
dcore.setCoreProperties(coreProperties);
SolrCore core = coreContainer.create(dcore);
+
coreContainer.register(name, core, false);
rsp.add("core", core.getName());
return coreContainer.isPersistent();
} catch (Exception ex) {
+ if (coreContainer.isZooKeeperAware() && dcore != null) {
+ try {
+ coreContainer.getZkController().unregister(name, dcore);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ SolrException.log(log, null, e);
+ } catch (KeeperException e) {
+ SolrException.log(log, null, e);
+ }
+ }
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
"Error CREATEing SolrCore '" + name + "': " +
ex.getMessage(), ex);
@@ -568,43 +606,24 @@ public class CoreAdminHandler extends Re
+ e.getMessage(), e);
}
}
- }
- if (params.getBool(CoreAdminParams.DELETE_INDEX, false)) {
- core.addCloseHook(new CloseHook() {
- private String indexDir;
-
- @Override
- public void preClose(SolrCore core) {
- indexDir = core.getIndexDir();
- }
-
- @Override
- public void postClose(SolrCore core) {
- try {
- core.getDirectoryFactory().remove(indexDir);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
+
+ if (params.getBool(CoreAdminParams.DELETE_INDEX, false)) {
+ try {
+ core.getDirectoryFactory().remove(core.getIndexDir());
+ } catch (Exception e) {
+ SolrException.log(log, "Failed to flag index dir for removal for core:"
+ + core.getName() + " dir:" + core.getIndexDir());
}
- });
+ }
}
-
+
if (params.getBool(CoreAdminParams.DELETE_DATA_DIR, false)) {
- core.addCloseHook(new CloseHook() {
- @Override
- public void preClose(SolrCore core) {}
-
- @Override
- public void postClose(SolrCore core) {
- File dataDir = new File(core.getDataDir());
- try {
- FileUtils.deleteDirectory(dataDir);
- } catch (IOException e) {
- SolrException.log(log, "Failed to delete data dir for core:"
- + core.getName() + " dir:" + dataDir.getAbsolutePath());
- }
- }
- });
+ try {
+ core.getDirectoryFactory().remove(core.getDataDir(), true);
+ } catch (Exception e) {
+ SolrException.log(log, "Failed to flag data dir for removal for core:"
+ + core.getName() + " dir:" + core.getDataDir());
+ }
}
if (params.getBool(CoreAdminParams.DELETE_INSTANCE_DIR, false)) {
@@ -858,6 +877,9 @@ public class CoreAdminHandler extends Re
Boolean checkLive = params.getBool("checkLive");
Boolean onlyIfLeader = params.getBool("onlyIfLeader");
+ log.info("Going to wait for coreNodeName: " + coreNodeName + ", state: " + waitForState
+ + ", checkLive: " + checkLive + ", onlyIfLeader: " + onlyIfLeader);
+
String state = null;
boolean live = false;
int retry = 0;
@@ -952,6 +974,47 @@ public class CoreAdminHandler extends Re
Thread.sleep(1000);
}
+ log.info("Waited coreNodeName: " + coreNodeName + ", state: " + waitForState
+ + ", checkLive: " + checkLive + ", onlyIfLeader: " + onlyIfLeader + " for: " + retry + " seconds.");
+ }
+
+ private void handleRequestApplyUpdatesAction(SolrQueryRequest req, SolrQueryResponse rsp) {
+ SolrParams params = req.getParams();
+ String cname = params.get(CoreAdminParams.NAME, "");
+ SolrCore core = coreContainer.getCore(cname);
+ try {
+ UpdateLog updateLog = core.getUpdateHandler().getUpdateLog();
+ if (updateLog.getState() != UpdateLog.State.BUFFERING) {
+ throw new SolrException(ErrorCode.SERVER_ERROR, "Core " + cname + " not in buffering state");
+ }
+ Future<UpdateLog.RecoveryInfo> future = updateLog.applyBufferedUpdates();
+ if (future == null) {
+ log.info("No buffered updates available. core=" + cname);
+ rsp.add("core", cname);
+ rsp.add("status", "EMPTY_BUFFER");
+ return;
+ }
+ UpdateLog.RecoveryInfo report = future.get();
+ if (report.failed) {
+ SolrException.log(log, "Replay failed");
+ throw new SolrException(ErrorCode.SERVER_ERROR, "Replay failed");
+ }
+ coreContainer.getZkController().publish(core.getCoreDescriptor(), ZkStateReader.ACTIVE);
+ rsp.add("core", cname);
+ rsp.add("status", "BUFFER_APPLIED");
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ log.warn("Recovery was interrupted", e);
+ } catch (Throwable e) {
+ if (e instanceof SolrException)
+ throw (SolrException)e;
+ else
+ throw new SolrException(ErrorCode.SERVER_ERROR, "Could not apply buffered updates", e);
+ } finally {
+ if (req != null) req.close();
+ if (core != null)
+ core.close();
+ }
}
Modified: lucene/dev/branches/security/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java (original)
+++ lucene/dev/branches/security/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java Thu May 30 07:53:18 2013
@@ -476,7 +476,7 @@ public class LukeRequestHandler extends
Map<String, Object> tok = new HashMap<String, Object>();
String className = cfiltfac.getClass().getName();
tok.put("className", className);
- tok.put("args", cfiltfac.getArgs());
+ tok.put("args", cfiltfac.getOriginalArgs());
cfilters.add(className.substring(className.lastIndexOf('.')+1), tok);
}
if (cfilters.size() > 0) {
@@ -486,7 +486,7 @@ public class LukeRequestHandler extends
SimpleOrderedMap<Object> tokenizer = new SimpleOrderedMap<Object>();
TokenizerFactory tfac = tchain.getTokenizerFactory();
tokenizer.add("className", tfac.getClass().getName());
- tokenizer.add("args", tfac.getArgs());
+ tokenizer.add("args", tfac.getOriginalArgs());
aninfo.add("tokenizer", tokenizer);
TokenFilterFactory[] filtfacs = tchain.getTokenFilterFactories();
@@ -495,7 +495,7 @@ public class LukeRequestHandler extends
Map<String, Object> tok = new HashMap<String, Object>();
String className = filtfac.getClass().getName();
tok.put("className", className);
- tok.put("args", filtfac.getArgs());
+ tok.put("args", filtfac.getOriginalArgs());
filters.add(className.substring(className.lastIndexOf('.')+1), tok);
}
if (filters.size() > 0) {
@@ -525,7 +525,7 @@ public class LukeRequestHandler extends
field.add("positionIncrementGap", ft.getAnalyzer().getPositionIncrementGap(f.getName()));
}
field.add("copyDests", toListOfStringDests(schema.getCopyFieldsList(f.getName())));
- field.add("copySources", toListOfStrings(schema.getCopySources(f.getName())));
+ field.add("copySources", schema.getCopySources(f.getName()));
fields.put( f.getName(), field );
Modified: lucene/dev/branches/security/solr/core/src/java/org/apache/solr/handler/admin/ShowFileRequestHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/solr/core/src/java/org/apache/solr/handler/admin/ShowFileRequestHandler.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/solr/core/src/java/org/apache/solr/handler/admin/ShowFileRequestHandler.java (original)
+++ lucene/dev/branches/security/solr/core/src/java/org/apache/solr/handler/admin/ShowFileRequestHandler.java Thu May 30 07:53:18 2013
@@ -191,9 +191,7 @@ public class ShowFileRequestHandler exte
ModifiableSolrParams params = new ModifiableSolrParams(req.getParams());
params.set(CommonParams.WT, "raw");
req.setParams(params);
-
- ContentStreamBase content = new ContentStreamBase.StringStream(
- new String(zkClient.getData(adminFile, null, null, true), "UTF-8"));
+ ContentStreamBase content = new ContentStreamBase.ByteArrayStream(zkClient.getData(adminFile, null, null, true), adminFile);
content.setContentType(req.getParams().get(USE_CONTENT_TYPE));
rsp.add(RawResponseWriter.CONTENT, content);
Modified: lucene/dev/branches/security/solr/core/src/java/org/apache/solr/handler/admin/SystemInfoHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/solr/core/src/java/org/apache/solr/handler/admin/SystemInfoHandler.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/solr/core/src/java/org/apache/solr/handler/admin/SystemInfoHandler.java (original)
+++ lucene/dev/branches/security/solr/core/src/java/org/apache/solr/handler/admin/SystemInfoHandler.java Thu May 30 07:53:18 2013
@@ -75,7 +75,7 @@ public class SystemInfoHandler extends R
@Override
public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception
{
- rsp.add( "core", getCoreInfo( req.getCore() ) );
+ rsp.add( "core", getCoreInfo( req.getCore(), req.getSchema() ) );
boolean solrCloudMode = req.getCore().getCoreDescriptor().getCoreContainer().isZooKeeperAware();
rsp.add( "mode", solrCloudMode ? "solrcloud" : "std");
rsp.add( "lucene", getLuceneInfo() );
@@ -87,10 +87,9 @@ public class SystemInfoHandler extends R
/**
* Get system info
*/
- private SimpleOrderedMap<Object> getCoreInfo( SolrCore core ) {
+ private SimpleOrderedMap<Object> getCoreInfo( SolrCore core, IndexSchema schema ) {
SimpleOrderedMap<Object> info = new SimpleOrderedMap<Object>();
- IndexSchema schema = core.getSchema();
info.add( "schema", schema != null ? schema.getSchemaName():"no schema!" );
// Host
@@ -106,7 +105,12 @@ public class SystemInfoHandler extends R
SimpleOrderedMap<Object> dirs = new SimpleOrderedMap<Object>();
dirs.add( "cwd" , new File( System.getProperty("user.dir")).getAbsolutePath() );
dirs.add( "instance", new File( core.getResourceLoader().getInstanceDir() ).getAbsolutePath() );
- dirs.add( "data", new File( core.getDataDir() ).getAbsolutePath() );
+ try {
+ dirs.add( "data", core.getDirectoryFactory().normalize(core.getDataDir()));
+ } catch (IOException e) {
+ log.warn("Problem getting the normalized data directory path", e);
+ dirs.add( "data", "N/A" );
+ }
dirs.add( "dirimpl", core.getDirectoryFactory().getClass().getName());
try {
dirs.add( "index", core.getDirectoryFactory().normalize(core.getIndexDir()) );
Modified: lucene/dev/branches/security/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java (original)
+++ lucene/dev/branches/security/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java Thu May 30 07:53:18 2013
@@ -18,7 +18,6 @@ package org.apache.solr.handler.componen
import java.net.ConnectException;
import java.util.Collection;
-import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -27,7 +26,6 @@ import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.CompletionService;
import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorCompletionService;
import java.util.concurrent.Future;
import org.apache.http.client.HttpClient;
@@ -61,21 +59,20 @@ public class HttpShardHandler extends Sh
private HttpShardHandlerFactory httpShardHandlerFactory;
private CompletionService<ShardResponse> completionService;
- private Set<Future<ShardResponse>> pending;
+ private Set<Future<ShardResponse>> pending;
private Map<String,List<String>> shardToURLs;
private HttpClient httpClient;
-
public HttpShardHandler(HttpShardHandlerFactory httpShardHandlerFactory, HttpClient httpClient) {
this.httpClient = httpClient;
this.httpShardHandlerFactory = httpShardHandlerFactory;
- completionService = new ExecutorCompletionService<ShardResponse>(httpShardHandlerFactory.commExecutor);
+ completionService = httpShardHandlerFactory.newCompletionService();
pending = new HashSet<Future<ShardResponse>>();
// maps "localhost:8983|localhost:7574" to a shuffled List("http://localhost:8983","http://localhost:7574")
- // This is primarily to keep track of what order we should use to query the replicas of a shard
- // so that we use the same replica for all phases of a distributed request.
+ // This is primarily to keep track of what order we should use to query the replicas of a shard
+ // so that we use the same replica for all phases of a distributed request.
shardToURLs = new HashMap<String,List<String>>();
}
@@ -106,21 +103,8 @@ public class HttpShardHandler extends Sh
// Don't modify the returned URL list.
private List<String> getURLs(String shard) {
List<String> urls = shardToURLs.get(shard);
- if (urls==null) {
- urls = StrUtils.splitSmart(shard, "|", true);
-
- // convert shard to URL
- for (int i=0; i<urls.size(); i++) {
- urls.set(i, httpShardHandlerFactory.scheme + urls.get(i));
- }
-
- //
- // Shuffle the list instead of use round-robin by default.
- // This prevents accidental synchronization where multiple shards could get in sync
- // and query the same replica at the same time.
- //
- if (urls.size() > 1)
- Collections.shuffle(urls, httpShardHandlerFactory.r);
+ if (urls == null) {
+ urls = httpShardHandlerFactory.makeURLList(shard);
shardToURLs.put(shard, urls);
}
return urls;
@@ -137,6 +121,9 @@ public class HttpShardHandler extends Sh
public ShardResponse call() throws Exception {
ShardResponse srsp = new ShardResponse();
+ if (sreq.nodeName != null) {
+ srsp.setNodeName(sreq.nodeName);
+ }
srsp.setShardRequest(sreq);
srsp.setShard(shard);
SimpleSolrResponse ssr = new SimpleSolrResponse();
@@ -168,7 +155,7 @@ public class HttpShardHandler extends Sh
SolrServer server = new HttpSolrServer(url, httpClient);
ssr.nl = server.request(req);
} else {
- LBHttpSolrServer.Rsp rsp = httpShardHandlerFactory.loadbalancer.request(new LBHttpSolrServer.Req(req, urls));
+ LBHttpSolrServer.Rsp rsp = httpShardHandlerFactory.makeLoadBalancedRequest(req, urls);
ssr.nl = rsp.getResponse();
srsp.setShardAddress(rsp.getServer());
}
@@ -337,7 +324,7 @@ public class HttpShardHandler extends Sh
// and make it a non-distributed request.
String ourSlice = cloudDescriptor.getShardId();
String ourCollection = cloudDescriptor.getCollectionName();
- if (rb.slices.length == 1
+ if (rb.slices.length == 1 && rb.slices[0] != null
&& ( rb.slices[0].equals(ourSlice) || rb.slices[0].equals(ourCollection + "_" + ourSlice) ) // handle the <collection>_<slice> format
&& ZkStateReader.ACTIVE.equals(cloudDescriptor.getLastPublished()) )
{
@@ -418,4 +405,4 @@ public class HttpShardHandler extends Sh
-}
\ No newline at end of file
+}
Modified: lucene/dev/branches/security/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java (original)
+++ lucene/dev/branches/security/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java Thu May 30 07:53:18 2013
@@ -16,24 +16,36 @@ package org.apache.solr.handler.componen
* limitations under the License.
*/
+import java.io.IOException;
import java.net.MalformedURLException;
+import java.util.Collections;
+import java.util.List;
import java.util.Random;
-import java.util.concurrent.*;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.CompletionService;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
import org.apache.http.client.HttpClient;
+import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.HttpClientUtil;
import org.apache.solr.client.solrj.impl.LBHttpSolrServer;
+import org.apache.solr.client.solrj.request.QueryRequest;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.ExecutorUtil;
import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.StrUtils;
import org.apache.solr.core.PluginInfo;
import org.apache.solr.util.DefaultSolrThreadFactory;
-import org.apache.solr.util.plugin.PluginInfoInitialized;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-public class HttpShardHandlerFactory extends ShardHandlerFactory implements PluginInfoInitialized {
+
+public class HttpShardHandlerFactory extends ShardHandlerFactory implements org.apache.solr.util.plugin.PluginInfoInitialized {
protected static Logger log = LoggerFactory.getLogger(HttpShardHandlerFactory.class);
// We want an executor that doesn't take up any resources if
@@ -42,7 +54,7 @@ public class HttpShardHandlerFactory ext
//
// Consider CallerRuns policy and a lower max threads to throttle
// requests at some point (or should we simply return failure?)
- ThreadPoolExecutor commExecutor = new ThreadPoolExecutor(
+ private ThreadPoolExecutor commExecutor = new ThreadPoolExecutor(
0,
Integer.MAX_VALUE,
5, TimeUnit.SECONDS, // terminate idle threads after 5 sec
@@ -51,7 +63,7 @@ public class HttpShardHandlerFactory ext
);
private HttpClient defaultClient;
- LBHttpSolrServer loadbalancer;
+ private LBHttpSolrServer loadbalancer;
//default values:
int soTimeout = 0;
int connectionTimeout = 0;
@@ -62,9 +74,9 @@ public class HttpShardHandlerFactory ext
int queueSize = -1;
boolean accessPolicy = false;
- public String scheme = "http://"; //current default values
+ private String scheme = "http://"; //current default values
- final Random r = new Random();
+ private final Random r = new Random();
// URL scheme to be used in distributed search.
static final String INIT_URL_SCHEME = "urlScheme";
@@ -112,6 +124,12 @@ public class HttpShardHandlerFactory ext
this.keepAliveTime = getParameter(args, MAX_THREAD_IDLE_TIME, keepAliveTime);
this.queueSize = getParameter(args, INIT_SIZE_OF_QUEUE, queueSize);
this.accessPolicy = getParameter(args, INIT_FAIRNESS_POLICY, accessPolicy);
+
+ // magic sysprop to make tests reproducible: set by SolrTestCaseJ4.
+ String v = System.getProperty("tests.shardhandler.randomSeed");
+ if (v != null) {
+ r.setSeed(Long.parseLong(v));
+ }
BlockingQueue<Runnable> blockingQueue = (this.queueSize == -1) ?
new SynchronousQueue<Runnable>(this.accessPolicy) :
@@ -132,17 +150,23 @@ public class HttpShardHandlerFactory ext
clientParams.set(HttpClientUtil.PROP_CONNECTION_TIMEOUT, connectionTimeout);
clientParams.set(HttpClientUtil.PROP_USE_RETRY, false);
this.defaultClient = HttpClientUtil.createClient(clientParams);
+ this.loadbalancer = createLoadbalancer(defaultClient);
+ }
+
+ protected ThreadPoolExecutor getThreadPoolExecutor(){
+ return this.commExecutor;
+ }
+ protected LBHttpSolrServer createLoadbalancer(HttpClient httpClient){
try {
- loadbalancer = new LBHttpSolrServer(defaultClient);
+ return new LBHttpSolrServer(httpClient);
} catch (MalformedURLException e) {
// should be impossible since we're not passing any URLs here
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
}
-
}
- private <T> T getParameter(NamedList initArgs, String configKey, T defaultValue) {
+ protected <T> T getParameter(NamedList initArgs, String configKey, T defaultValue) {
T toReturn = defaultValue;
if (initArgs != null) {
T temp = (T) initArgs.get(configKey);
@@ -177,4 +201,48 @@ public class HttpShardHandlerFactory ext
}
}
+
+ /**
+ * Makes a request to one or more of the given urls, using the configured load balancer.
+ *
+ * @param req The solr search request that should be sent through the load balancer
+ * @param urls The list of solr server urls to load balance across
+ * @return The response from the request
+ */
+ public LBHttpSolrServer.Rsp makeLoadBalancedRequest(final QueryRequest req, List<String> urls)
+ throws SolrServerException, IOException {
+ return loadbalancer.request(new LBHttpSolrServer.Req(req, urls));
+ }
+
+ /**
+ * Creates a randomized list of urls for the given shard.
+ *
+ * @param shard the urls for the shard (minus "http://"), separated by '|'
+ * @return A list of valid urls (including protocol) that are replicas for the shard
+ */
+ public List<String> makeURLList(String shard) {
+ List<String> urls = StrUtils.splitSmart(shard, "|", true);
+
+ // convert shard to URL
+ for (int i=0; i<urls.size(); i++) {
+ urls.set(i, scheme + urls.get(i));
+ }
+
+ //
+ // Shuffle the list instead of use round-robin by default.
+ // This prevents accidental synchronization where multiple shards could get in sync
+ // and query the same replica at the same time.
+ //
+ if (urls.size() > 1)
+ Collections.shuffle(urls, r);
+
+ return urls;
+ }
+
+ /**
+ * Creates a new completion service for use by a single set of distributed requests.
+ */
+ public CompletionService newCompletionService() {
+ return new ExecutorCompletionService<ShardResponse>(commExecutor);
+ }
}
Modified: lucene/dev/branches/security/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java (original)
+++ lucene/dev/branches/security/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java Thu May 30 07:53:18 2013
@@ -23,7 +23,12 @@ import org.apache.lucene.index.AtomicRea
import org.apache.lucene.index.IndexReaderContext;
import org.apache.lucene.index.ReaderUtil;
import org.apache.lucene.index.Term;
-import org.apache.lucene.search.*;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.FieldComparator;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.Sort;
+import org.apache.lucene.search.SortField;
import org.apache.lucene.search.grouping.GroupDocs;
import org.apache.lucene.search.grouping.SearchGroup;
import org.apache.lucene.search.grouping.TopGroups;
@@ -43,7 +48,19 @@ import org.apache.solr.response.ResultCo
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.schema.FieldType;
import org.apache.solr.schema.SchemaField;
-import org.apache.solr.search.*;
+import org.apache.solr.search.DocIterator;
+import org.apache.solr.search.DocList;
+import org.apache.solr.search.DocListAndSet;
+import org.apache.solr.search.DocSlice;
+import org.apache.solr.search.Grouping;
+import org.apache.solr.search.QParser;
+import org.apache.solr.search.QParserPlugin;
+import org.apache.solr.search.QueryParsing;
+import org.apache.solr.search.ReturnFields;
+import org.apache.solr.search.SolrIndexSearcher;
+import org.apache.solr.search.SolrReturnFields;
+import org.apache.solr.search.SortSpec;
+import org.apache.solr.search.SyntaxError;
import org.apache.solr.search.grouping.CommandHandler;
import org.apache.solr.search.grouping.GroupingSpecification;
import org.apache.solr.search.grouping.distributed.ShardRequestFactory;
@@ -69,7 +86,15 @@ import java.io.IOException;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.net.URL;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
/**
* TODO!
@@ -101,7 +126,7 @@ public class QueryComponent extends Sear
}
rb.setFieldFlags( flags );
- String defType = params.get(QueryParsing.DEFTYPE,QParserPlugin.DEFAULT_QTYPE);
+ String defType = params.get(QueryParsing.DEFTYPE, QParserPlugin.DEFAULT_QTYPE);
// get it from the response builder to give a different component a chance
// to set it.
@@ -225,7 +250,7 @@ public class QueryComponent extends Sear
// too if desired).
String ids = params.get(ShardParams.IDS);
if (ids != null) {
- SchemaField idField = req.getSchema().getUniqueKeyField();
+ SchemaField idField = searcher.getSchema().getUniqueKeyField();
List<String> idArr = StrUtils.splitSmart(ids, ",", true);
int[] luceneIds = new int[idArr.size()];
int docs = 0;
@@ -463,7 +488,7 @@ public class QueryComponent extends Sear
FieldComparator comparator = null;
String fieldname = sortField.getField();
- FieldType ft = fieldname==null ? null : req.getSchema().getFieldTypeNoEx(fieldname);
+ FieldType ft = fieldname==null ? null : searcher.getSchema().getFieldTypeNoEx(fieldname);
Object[] vals = new Object[nDocs];
Modified: lucene/dev/branches/security/solr/core/src/java/org/apache/solr/handler/component/QueryElevationComponent.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/solr/core/src/java/org/apache/solr/handler/component/QueryElevationComponent.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/solr/core/src/java/org/apache/solr/handler/component/QueryElevationComponent.java (original)
+++ lucene/dev/branches/security/solr/core/src/java/org/apache/solr/handler/component/QueryElevationComponent.java Thu May 30 07:53:18 2013
@@ -20,8 +20,22 @@ package org.apache.solr.handler.componen
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
-import org.apache.lucene.index.*;
-import org.apache.lucene.search.*;
+import org.apache.lucene.index.AtomicReaderContext;
+import org.apache.lucene.index.DocsEnum;
+import org.apache.lucene.index.Fields;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.index.Terms;
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.FieldComparator;
+import org.apache.lucene.search.FieldComparatorSource;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.Sort;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.TermQuery;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.SentinelIntSet;
@@ -29,6 +43,7 @@ import org.apache.solr.cloud.ZkControlle
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.QueryElevationParams;
import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.schema.IndexSchema;
import org.apache.solr.util.DOMUtil;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
@@ -60,7 +75,14 @@ import java.io.InputStream;
import java.io.StringReader;
import java.net.MalformedURLException;
import java.net.URL;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.WeakHashMap;
/**
* A component to elevate some documents to the top of the result set.
@@ -139,9 +161,10 @@ public class QueryElevationComponent ext
@Override
public void inform(SolrCore core) {
+ IndexSchema schema = core.getLatestSchema();
String a = initArgs.get(FIELD_TYPE);
if (a != null) {
- FieldType ft = core.getSchema().getFieldTypes().get(a);
+ FieldType ft = schema.getFieldTypes().get(a);
if (ft == null) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
"Unknown FieldType: '" + a + "' used in QueryElevationComponent");
@@ -149,7 +172,7 @@ public class QueryElevationComponent ext
analyzer = ft.getQueryAnalyzer();
}
- SchemaField sf = core.getSchema().getUniqueKeyField();
+ SchemaField sf = schema.getUniqueKeyField();
if( sf == null) {
throw new SolrException( SolrException.ErrorCode.SERVER_ERROR,
"QueryElevationComponent requires the schema to have a uniqueKeyField." );
Modified: lucene/dev/branches/security/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java (original)
+++ lucene/dev/branches/security/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java Thu May 30 07:53:18 2013
@@ -43,7 +43,6 @@ import org.apache.solr.common.cloud.Slic
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.ShardParams;
import org.apache.solr.common.params.SolrParams;
-import org.apache.solr.common.util.Hash;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.core.SolrCore;
@@ -121,11 +120,12 @@ public class RealTimeGetComponent extend
allIds = lst.toArray(new String[lst.size()]);
}
- SchemaField idField = req.getSchema().getUniqueKeyField();
+ SolrCore core = req.getCore();
+ SchemaField idField = core.getLatestSchema().getUniqueKeyField();
FieldType fieldType = idField.getType();
SolrDocumentList docList = new SolrDocumentList();
- UpdateLog ulog = req.getCore().getUpdateHandler().getUpdateLog();
+ UpdateLog ulog = core.getUpdateHandler().getUpdateLog();
RefCounted<SolrIndexSearcher> searcherHolder = null;
@@ -150,7 +150,7 @@ public class RealTimeGetComponent extend
int oper = (Integer)entry.get(0) & UpdateLog.OPERATION_MASK;
switch (oper) {
case UpdateLog.ADD:
- SolrDocument doc = toSolrDoc((SolrInputDocument)entry.get(entry.size()-1), req.getSchema());
+ SolrDocument doc = toSolrDoc((SolrInputDocument)entry.get(entry.size()-1), core.getLatestSchema());
if(transformer!=null) {
transformer.transform(doc, -1); // unknown docID
}
@@ -167,7 +167,7 @@ public class RealTimeGetComponent extend
// didn't find it in the update log, so it should be in the newest searcher opened
if (searcher == null) {
- searcherHolder = req.getCore().getRealtimeSearcher();
+ searcherHolder = core.getRealtimeSearcher();
searcher = searcherHolder.get();
}
@@ -176,7 +176,7 @@ public class RealTimeGetComponent extend
int docid = searcher.getFirstMatch(new Term(idField.getName(), idBytes));
if (docid < 0) continue;
StoredDocument luceneDocument = searcher.doc(docid);
- SolrDocument doc = toSolrDoc(luceneDocument, req.getSchema());
+ SolrDocument doc = toSolrDoc(luceneDocument, core.getLatestSchema());
if( transformer != null ) {
transformer.transform(doc, docid);
}
@@ -238,12 +238,12 @@ public class RealTimeGetComponent extend
}
// SolrCore.verbose("RealTimeGet using searcher ", searcher);
- SchemaField idField = core.getSchema().getUniqueKeyField();
+ SchemaField idField = core.getLatestSchema().getUniqueKeyField();
int docid = searcher.getFirstMatch(new Term(idField.getName(), idBytes));
if (docid < 0) return null;
StoredDocument luceneDocument = searcher.doc(docid);
- sid = toSolrInputDocument(luceneDocument, core.getSchema());
+ sid = toSolrInputDocument(luceneDocument, core.getLatestSchema());
}
} finally {
if (searcherHolder != null) {
@@ -554,13 +554,13 @@ public class RealTimeGetComponent extend
List<String> versions = StrUtils.splitSmart(versionsStr, ",", true);
- // TODO: get this from cache instead of rebuilding?
- UpdateLog.RecentUpdates recentUpdates = ulog.getRecentUpdates();
List<Object> updates = new ArrayList<Object>(versions.size());
long minVersion = Long.MAX_VALUE;
-
+
+ // TODO: get this from cache instead of rebuilding?
+ UpdateLog.RecentUpdates recentUpdates = ulog.getRecentUpdates();
try {
for (String versionStr : versions) {
long version = Long.parseLong(versionStr);
Modified: lucene/dev/branches/security/solr/core/src/java/org/apache/solr/handler/component/ResponseLogComponent.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/solr/core/src/java/org/apache/solr/handler/component/ResponseLogComponent.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/solr/core/src/java/org/apache/solr/handler/component/ResponseLogComponent.java (original)
+++ lucene/dev/branches/security/solr/core/src/java/org/apache/solr/handler/component/ResponseLogComponent.java Thu May 30 07:53:18 2013
@@ -20,7 +20,6 @@ import java.io.IOException;
import java.util.Collections;
import java.util.Set;
-import org.apache.lucene.index.StoredFieldVisitor;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.response.ResultContext;
import org.apache.solr.schema.IndexSchema;
@@ -67,11 +66,11 @@ public class ResponseLogComponent extend
SolrParams params = rb.req.getParams();
if (!params.getBool(COMPONENT_NAME, false)) return;
- IndexSchema schema = rb.req.getSchema();
+ SolrIndexSearcher searcher = rb.req.getSearcher();
+ IndexSchema schema = searcher.getSchema();
if (schema.getUniqueKeyField() == null) return;
ResultContext rc = (ResultContext) rb.rsp.getValues().get("response");
- SolrIndexSearcher searcher = rb.req.getSearcher();
if (rc.docs.hasScores()) {
processScores(rb, rc.docs, schema, searcher);
Modified: lucene/dev/branches/security/solr/core/src/java/org/apache/solr/handler/component/ShardRequest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/solr/core/src/java/org/apache/solr/handler/component/ShardRequest.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/solr/core/src/java/org/apache/solr/handler/component/ShardRequest.java (original)
+++ lucene/dev/branches/security/solr/core/src/java/org/apache/solr/handler/component/ShardRequest.java Thu May 30 07:53:18 2013
@@ -53,6 +53,9 @@ public class ShardRequest {
/** actual shards to send the request to, filled out by framework */
public String[] actualShards;
+ /** may be null */
+ public String nodeName;
+
// TODO: one could store a list of numbers to correlate where returned docs
// go in the top-level response rather than looking up by id...
// this would work well if we ever transitioned to using internal ids and
Modified: lucene/dev/branches/security/solr/core/src/java/org/apache/solr/handler/component/ShardResponse.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/solr/core/src/java/org/apache/solr/handler/component/ShardResponse.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/solr/core/src/java/org/apache/solr/handler/component/ShardResponse.java (original)
+++ lucene/dev/branches/security/solr/core/src/java/org/apache/solr/handler/component/ShardResponse.java Thu May 30 07:53:18 2013
@@ -22,6 +22,7 @@ import org.apache.solr.common.SolrExcept
public final class ShardResponse {
private ShardRequest req;
private String shard;
+ private String nodeName;
private String shardAddress; // the specific shard that this response was received from
private int rspCode;
private Throwable exception;
@@ -56,6 +57,11 @@ public final class ShardResponse {
return shard;
}
+ public String getNodeName()
+ {
+ return nodeName;
+ }
+
public void setShardRequest(ShardRequest rsp)
{
this.req = rsp;
@@ -80,9 +86,15 @@ public final class ShardResponse {
{
this.rspCode = rspCode;
}
+
+ void setNodeName(String nodeName)
+ {
+ this.nodeName = nodeName;
+ }
/** What was the shard address that returned this response. Example: "http://localhost:8983/solr" */
public String getShardAddress() { return this.shardAddress; }
void setShardAddress(String addr) { this.shardAddress = addr; }
+
}
Modified: lucene/dev/branches/security/solr/core/src/java/org/apache/solr/handler/component/SpellCheckComponent.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/solr/core/src/java/org/apache/solr/handler/component/SpellCheckComponent.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/solr/core/src/java/org/apache/solr/handler/component/SpellCheckComponent.java (original)
+++ lucene/dev/branches/security/solr/core/src/java/org/apache/solr/handler/component/SpellCheckComponent.java Thu May 30 07:53:18 2013
@@ -19,29 +19,35 @@ package org.apache.solr.handler.componen
import java.io.IOException;
import java.io.StringReader;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
-import org.apache.lucene.search.spell.SuggestMode;
-import org.apache.lucene.search.spell.SuggestWord;
-import org.apache.solr.client.solrj.response.SpellCheckResponse;
-import org.apache.solr.common.params.ModifiableSolrParams;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.FlagsAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.search.spell.SuggestMode;
+import org.apache.lucene.search.spell.SuggestWord;
+import org.apache.solr.client.solrj.response.SpellCheckResponse;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.ShardParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.params.SpellingParams;
@@ -53,8 +59,19 @@ import org.apache.solr.core.SolrResource
import org.apache.solr.schema.FieldType;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.search.SolrIndexSearcher;
-import org.apache.solr.spelling.*;
+import org.apache.solr.spelling.AbstractLuceneSpellChecker;
+import org.apache.solr.spelling.ConjunctionSolrSpellChecker;
+import org.apache.solr.spelling.IndexBasedSpellChecker;
+import org.apache.solr.spelling.QueryConverter;
+import org.apache.solr.spelling.SolrSpellChecker;
+import org.apache.solr.spelling.SpellCheckCollation;
+import org.apache.solr.spelling.SpellCheckCollator;
+import org.apache.solr.spelling.SpellingOptions;
+import org.apache.solr.spelling.SpellingQueryConverter;
+import org.apache.solr.spelling.SpellingResult;
import org.apache.solr.util.plugin.SolrCoreAware;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* A SearchComponent implementation which provides support for spell checking
@@ -197,10 +214,20 @@ public class SpellCheckComponent extends
int maxCollationTries = params.getInt(SPELLCHECK_MAX_COLLATION_TRIES, 0);
int maxCollationEvaluations = params.getInt(SPELLCHECK_MAX_COLLATION_EVALUATIONS, 10000);
boolean collationExtendedResults = params.getBool(SPELLCHECK_COLLATE_EXTENDED_RESULTS, false);
+ int maxCollationCollectDocs = params.getInt(SPELLCHECK_COLLATE_MAX_COLLECT_DOCS, 0);
+ // If not reporting hits counts, don't bother collecting more than 1 document per try.
+ if (!collationExtendedResults) {
+ maxCollationCollectDocs = 1;
+ }
boolean shard = params.getBool(ShardParams.IS_SHARD, false);
-
- SpellCheckCollator collator = new SpellCheckCollator();
- List<SpellCheckCollation> collations = collator.collate(spellingResult, q, rb, maxCollations, maxCollationTries, maxCollationEvaluations, suggestionsMayOverlap);
+ SpellCheckCollator collator = new SpellCheckCollator()
+ .setMaxCollations(maxCollations)
+ .setMaxCollationTries(maxCollationTries)
+ .setMaxCollationEvaluations(maxCollationEvaluations)
+ .setSuggestionsMayOverlap(suggestionsMayOverlap)
+ .setDocCollectionLimit(maxCollationCollectDocs)
+ ;
+ List<SpellCheckCollation> collations = collator.collate(spellingResult, q, rb);
//by sorting here we guarantee a non-distributed request returns all
//results in the same order as a distributed request would,
//even in cases when the internal rank is the same.
@@ -250,9 +277,9 @@ public class SpellCheckComponent extends
@Override
public void modifyRequest(ResponseBuilder rb, SearchComponent who, ShardRequest sreq) {
SolrParams params = rb.req.getParams();
- // Turn on spellcheck only only when retrieving fields
if (!params.getBool(COMPONENT_NAME, false)) return;
- if ((sreq.purpose & ShardRequest.PURPOSE_GET_TOP_IDS) != 0) {
+ int purpose = rb.grouping() ? ShardRequest.PURPOSE_GET_TOP_GROUPS : ShardRequest.PURPOSE_GET_TOP_IDS;
+ if ((sreq.purpose & purpose) != 0) {
// fetch at least 5 suggestions from each shard
int count = sreq.params.getInt(SPELLCHECK_COUNT, 1);
if (count < 5) count = 5;
@@ -287,7 +314,7 @@ public class SpellCheckComponent extends
}
}
- long hits = rb.getNumberDocumentsFound();
+ long hits = rb.grouping() ? rb.totalHitCount : rb.getNumberDocumentsFound();
boolean isCorrectlySpelled = hits > (maxResultsForSuggest==null ? 0 : maxResultsForSuggest);
SpellCheckMergeData mergeData = new SpellCheckMergeData();
@@ -637,7 +664,7 @@ public class SpellCheckComponent extends
//there should only be one
if (queryConverters.size() == 1) {
queryConverter = queryConverters.values().iterator().next();
- IndexSchema schema = core.getSchema();
+ IndexSchema schema = core.getLatestSchema();
String fieldTypeName = (String) initParams.get("queryAnalyzerFieldType");
FieldType fieldType = schema.getFieldTypes().get(fieldTypeName);
Analyzer analyzer = fieldType == null ? new WhitespaceAnalyzer(core.getSolrConfig().luceneMatchVersion)
Modified: lucene/dev/branches/security/solr/core/src/java/org/apache/solr/handler/component/StatsComponent.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/solr/core/src/java/org/apache/solr/handler/component/StatsComponent.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/solr/core/src/java/org/apache/solr/handler/component/StatsComponent.java (original)
+++ lucene/dev/branches/security/solr/core/src/java/org/apache/solr/handler/component/StatsComponent.java Thu May 30 07:53:18 2013
@@ -34,6 +34,7 @@ import org.apache.solr.common.util.Simpl
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.UnInvertedField;
import org.apache.solr.schema.FieldType;
+import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.search.DocIterator;
import org.apache.solr.search.DocSet;
@@ -204,12 +205,13 @@ class SimpleStats {
String[] statsFs = params.getParams(StatsParams.STATS_FIELD);
boolean isShard = params.getBool(ShardParams.IS_SHARD, false);
if (null != statsFs) {
+ final IndexSchema schema = searcher.getSchema();
for (String f : statsFs) {
String[] facets = params.getFieldParams(f, StatsParams.STATS_FACET);
if (facets == null) {
facets = new String[0]; // make sure it is something...
}
- SchemaField sf = searcher.getSchema().getField(f);
+ SchemaField sf = schema.getField(f);
FieldType ft = sf.getType();
NamedList<?> stv;
@@ -231,13 +233,14 @@ class SimpleStats {
}
public NamedList<?> getFieldCacheStats(String fieldName, String[] facet) throws IOException {
- final SchemaField sf = searcher.getSchema().getField(fieldName);
+ IndexSchema schema = searcher.getSchema();
+ final SchemaField sf = schema.getField(fieldName);
final StatsValues allstats = StatsValuesFactory.createStatsValues(sf);
List<FieldFacetStats> facetStats = new ArrayList<FieldFacetStats>();
for( String facetField : facet ) {
- SchemaField fsf = searcher.getSchema().getField(facetField);
+ SchemaField fsf = schema.getField(facetField);
if ( fsf.multiValued()) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
Modified: lucene/dev/branches/security/solr/core/src/java/org/apache/solr/handler/loader/JsonLoader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/solr/core/src/java/org/apache/solr/handler/loader/JsonLoader.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/solr/core/src/java/org/apache/solr/handler/loader/JsonLoader.java (original)
+++ lucene/dev/branches/security/solr/core/src/java/org/apache/solr/handler/loader/JsonLoader.java Thu May 30 07:53:18 2013
@@ -22,8 +22,8 @@ import java.io.StringReader;
import java.util.*;
import org.apache.commons.io.IOUtils;
-import org.apache.noggit.JSONParser;
-import org.apache.noggit.ObjectBuilder;
+import org.noggit.JSONParser;
+import org.noggit.ObjectBuilder;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.SolrInputField;
Modified: lucene/dev/branches/security/solr/core/src/java/org/apache/solr/handler/loader/XMLLoader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/solr/core/src/java/org/apache/solr/handler/loader/XMLLoader.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/solr/core/src/java/org/apache/solr/handler/loader/XMLLoader.java (original)
+++ lucene/dev/branches/security/solr/core/src/java/org/apache/solr/handler/loader/XMLLoader.java Thu May 30 07:53:18 2013
@@ -438,6 +438,7 @@ public class XMLLoader extends ContentSt
}
boost = 1.0f;
update = null;
+ isNull = false;
String attrVal = "";
for (int i = 0; i < parser.getAttributeCount(); i++) {
attrName = parser.getAttributeLocalName(i);
Modified: lucene/dev/branches/security/solr/core/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/solr/core/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/solr/core/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java (original)
+++ lucene/dev/branches/security/solr/core/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java Thu May 30 07:53:18 2013
@@ -69,6 +69,30 @@ public class DefaultSolrHighlighter exte
this.solrCore = solrCore;
}
+ // Thread safe registry
+ protected final Map<String,SolrFormatter> formatters =
+ new HashMap<String, SolrFormatter>();
+
+ // Thread safe registry
+ protected final Map<String,SolrEncoder> encoders =
+ new HashMap<String, SolrEncoder>();
+
+ // Thread safe registry
+ protected final Map<String,SolrFragmenter> fragmenters =
+ new HashMap<String, SolrFragmenter>() ;
+
+ // Thread safe registry
+ protected final Map<String, SolrFragListBuilder> fragListBuilders =
+ new HashMap<String, SolrFragListBuilder>() ;
+
+ // Thread safe registry
+ protected final Map<String, SolrFragmentsBuilder> fragmentsBuilders =
+ new HashMap<String, SolrFragmentsBuilder>() ;
+
+ // Thread safe registry
+ protected final Map<String, SolrBoundaryScanner> boundaryScanners =
+ new HashMap<String, SolrBoundaryScanner>() ;
+
@Override
public void init(PluginInfo info) {
formatters.clear();
@@ -429,21 +453,15 @@ public class DefaultSolrHighlighter exte
)) return;
// END: Hack
- SolrParams params = req.getParams();
- StorableField[] docFields = doc.getFields(fieldName);
- List<String> listFields = new ArrayList<String>();
- for (StorableField field : docFields) {
- listFields.add(field.stringValue());
- }
+ SolrParams params = req.getParams();
// preserve order of values in a multiValued list
boolean preserveMulti = params.getFieldBool(fieldName, HighlightParams.PRESERVE_MULTI, false);
- String[] docTexts = (String[]) listFields.toArray(new String[listFields.size()]);
-
- // according to Document javadoc, doc.getValues() never returns null. check empty instead of null
- if (docTexts.length == 0) return;
-
+ List<StorableField> allFields = doc.getFields();
+ if (allFields != null && allFields.size() == 0) return; // No explicit contract that getFields returns != null,
+ // although currently it can't.
+
TokenStream tstream = null;
int numFragments = getMaxSnippets(fieldName, params);
boolean mergeContiguousFragments = isMergeContiguousFragments(fieldName, params);
@@ -456,15 +474,25 @@ public class DefaultSolrHighlighter exte
if (tvStream != null) {
tots = new TermOffsetsTokenStream(tvStream);
}
+ int mvToExamine = Integer.parseInt(req.getParams().get(HighlightParams.MAX_MULTIVALUED_TO_EXAMINE,
+ Integer.toString(Integer.MAX_VALUE)));
+ int mvToMatch = Integer.parseInt(req.getParams().get(HighlightParams.MAX_MULTIVALUED_TO_MATCH,
+ Integer.toString(Integer.MAX_VALUE)));
+
+ for (StorableField thisField : allFields) {
+ if (mvToExamine <= 0 || mvToMatch <= 0) break;
+
+ if (! thisField.name().equals(fieldName)) continue; // Is there a better way to do this?
- for (int j = 0; j < docTexts.length; j++) {
+ --mvToExamine;
+ String thisText = thisField.stringValue();
if( tots != null ) {
// if we're using TermOffsets optimization, then get the next
// field value's TokenStream (i.e. get field j's TokenStream) from tots:
- tstream = tots.getMultiValuedTokenStream( docTexts[j].length() );
+ tstream = tots.getMultiValuedTokenStream( thisText.length() );
} else {
// fall back to analyzer
- tstream = createAnalyzerTStream(schema, fieldName, docTexts[j]);
+ tstream = createAnalyzerTStream(schema, fieldName, thisText);
}
int maxCharsToAnalyze = params.getFieldInt(fieldName,
@@ -491,21 +519,23 @@ public class DefaultSolrHighlighter exte
}
if (maxCharsToAnalyze < 0) {
- highlighter.setMaxDocCharsToAnalyze(docTexts[j].length());
+ highlighter.setMaxDocCharsToAnalyze(thisText.length());
} else {
highlighter.setMaxDocCharsToAnalyze(maxCharsToAnalyze);
}
try {
- TextFragment[] bestTextFragments = highlighter.getBestTextFragments(tstream, docTexts[j], mergeContiguousFragments, numFragments);
+ TextFragment[] bestTextFragments = highlighter.getBestTextFragments(tstream, thisText, mergeContiguousFragments, numFragments);
for (int k = 0; k < bestTextFragments.length; k++) {
if (preserveMulti) {
if (bestTextFragments[k] != null) {
frags.add(bestTextFragments[k]);
+ --mvToMatch;
}
} else {
if ((bestTextFragments[k] != null) && (bestTextFragments[k].getScore() > 0)) {
frags.add(bestTextFragments[k]);
+ --mvToMatch;
}
}
}
Modified: lucene/dev/branches/security/solr/core/src/java/org/apache/solr/highlight/PostingsSolrHighlighter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/solr/core/src/java/org/apache/solr/highlight/PostingsSolrHighlighter.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/solr/core/src/java/org/apache/solr/highlight/PostingsSolrHighlighter.java (original)
+++ lucene/dev/branches/security/solr/core/src/java/org/apache/solr/highlight/PostingsSolrHighlighter.java Thu May 30 07:53:18 2013
@@ -26,11 +26,12 @@ import java.util.Set;
import org.apache.lucene.index.StoredDocument;
import org.apache.lucene.search.Query;
-import org.apache.lucene.search.ScoreDoc;
-import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.search.postingshighlight.DefaultPassageFormatter;
+import org.apache.lucene.search.postingshighlight.Passage;
import org.apache.lucene.search.postingshighlight.PassageFormatter;
import org.apache.lucene.search.postingshighlight.PassageScorer;
import org.apache.lucene.search.postingshighlight.PostingsHighlighter;
+import org.apache.lucene.search.postingshighlight.WholeBreakIterator;
import org.apache.solr.common.params.HighlightParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
@@ -50,76 +51,125 @@ import org.apache.solr.util.plugin.Plugi
* <p>
* Example configuration:
* <pre class="prettyprint">
+ * <requestHandler name="standard" class="solr.StandardRequestHandler">
+ * <lst name="defaults">
+ * <int name="hl.snippets">1</int>
+ * <str name="hl.tag.pre">&lt;em&gt;</str>
+ * <str name="hl.tag.post">&lt;/em&gt;</str>
+ * <str name="hl.tag.ellipsis">... </str>
+ * <bool name="hl.defaultSummary">true</bool>
+ * <str name="hl.encoder">simple</str>
+ * <float name="hl.score.k1">1.2</float>
+ * <float name="hl.score.b">0.75</float>
+ * <float name="hl.score.pivot">87</float>
+ * <str name="hl.bs.language"></str>
+ * <str name="hl.bs.country"></str>
+ * <str name="hl.bs.variant"></str>
+ * <str name="hl.bs.type">SENTENCE</str>
+ * <int name="hl.maxAnalyzedChars">10000</int>
+ * </lst>
+ * </requestHandler>
+ * </pre>
+ * ...
+ * <pre class="prettyprint">
* <searchComponent class="solr.HighlightComponent" name="highlight">
- * <highlighting class="org.apache.solr.highlight.PostingsSolrHighlighter"
- * preTag="&lt;em&gt;"
- * postTag="&lt;/em&gt;"
- * ellipsis="... "
- * maxLength=10000/>
+ * <highlighting class="org.apache.solr.highlight.PostingsSolrHighlighter"/>
* </searchComponent>
* </pre>
* <p>
* Notes:
* <ul>
* <li>fields to highlight must be configured with storeOffsetsWithPositions="true"
- * <li>hl.fl specifies the field list.
- * <li>hl.snippets specifies how many underlying sentence fragments form the resulting snippet.
+ * <li>hl.q (string) can specify the query
+ * <li>hl.fl (string) specifies the field list.
+ * <li>hl.snippets (int) specifies how many underlying passages form the resulting snippet.
+ * <li>hl.tag.pre (string) specifies text which appears before a highlighted term.
+ * <li>hl.tag.post (string) specifies text which appears after a highlighted term.
+ * <li>hl.tag.ellipsis (string) specifies text which joins non-adjacent passages.
+ * <li>hl.defaultSummary (bool) specifies if a field should have a default summary.
+ * <li>hl.encoder (string) can be 'html' (html escapes content) or 'simple' (no escaping).
+ * <li>hl.score.k1 (float) specifies bm25 scoring parameter 'k1'
+ * <li>hl.score.b (float) specifies bm25 scoring parameter 'b'
+ * <li>hl.score.pivot (float) specifies bm25 scoring parameter 'avgdl'
+ * <li>hl.bs.type (string) specifies how to divide text into passages: [SENTENCE, LINE, WORD, CHAR, WHOLE]
+ * <li>hl.bs.language (string) specifies language code for BreakIterator. default is empty string (root locale)
+ * <li>hl.bs.country (string) specifies country code for BreakIterator. default is empty string (root locale)
+ * <li>hl.bs.variant (string) specifies country code for BreakIterator. default is empty string (root locale)
+ * <li>hl.maxAnalyzedChars specifies how many characters at most will be processed in a document.
+ * NOTE: currently hl.maxAnalyzedChars cannot yet be specified per-field
* </ul>
*
* @lucene.experimental
*/
public class PostingsSolrHighlighter extends SolrHighlighter implements PluginInfoInitialized {
- protected PostingsHighlighter highlighter;
@Override
public void initalize(SolrConfig config) {}
@Override
- public void init(PluginInfo info) {
- Map<String,String> attributes = info.attributes;
- BreakIterator breakIterator = BreakIterator.getSentenceInstance(Locale.ROOT);
- PassageScorer scorer = new PassageScorer();
-
- // formatter parameters: preTag/postTag/ellipsis
- String preTag = attributes.get("preTag");
- if (preTag == null) {
- preTag = "<em>";
- }
- String postTag = attributes.get("postTag");
- if (postTag == null) {
- postTag = "</em>";
- }
- String ellipsis = attributes.get("ellipsis");
- if (ellipsis == null) {
- ellipsis = "... ";
- }
- PassageFormatter formatter = new PassageFormatter(preTag, postTag, ellipsis);
-
- // maximum content size to process
- int maxLength = PostingsHighlighter.DEFAULT_MAX_LENGTH;
- if (attributes.containsKey("maxLength")) {
- maxLength = Integer.parseInt(attributes.get("maxLength"));
- }
- highlighter = new PostingsHighlighter(maxLength, breakIterator, scorer, formatter);
- }
+ public void init(PluginInfo info) {}
@Override
public NamedList<Object> doHighlighting(DocList docs, Query query, SolrQueryRequest req, String[] defaultFields) throws IOException {
- SolrParams params = req.getParams();
+ final SolrParams params = req.getParams();
// if highlighting isnt enabled, then why call doHighlighting?
if (isHighlightingEnabled(params)) {
SolrIndexSearcher searcher = req.getSearcher();
- TopDocs topDocs = toTopDocs(docs);
+ int[] docIDs = toDocIDs(docs);
// fetch the unique keys
- String[] keys = getUniqueKeys(searcher, topDocs);
+ String[] keys = getUniqueKeys(searcher, docIDs);
// query-time parameters
+ int maxLength = params.getInt(HighlightParams.MAX_CHARS, PostingsHighlighter.DEFAULT_MAX_LENGTH);
String[] fieldNames = getHighlightFields(query, req, defaultFields);
- int numSnippets = params.getInt(HighlightParams.SNIPPETS, 1);
- Map<String,String[]> snippets = highlighter.highlightFields(fieldNames, query, searcher, topDocs, numSnippets);
+ int maxPassages[] = new int[fieldNames.length];
+ for (int i = 0; i < fieldNames.length; i++) {
+ maxPassages[i] = params.getFieldInt(fieldNames[i], HighlightParams.SNIPPETS, 1);
+ }
+
+ PostingsHighlighter highlighter = new PostingsHighlighter(maxLength) {
+ @Override
+ protected Passage[] getEmptyHighlight(String fieldName, BreakIterator bi, int maxPassages) {
+ boolean defaultSummary = params.getFieldBool(fieldName, HighlightParams.DEFAULT_SUMMARY, true);
+ if (defaultSummary) {
+ return super.getEmptyHighlight(fieldName, bi, maxPassages);
+ } else {
+ return new Passage[0];
+ }
+ }
+
+ @Override
+ protected PassageFormatter getFormatter(String fieldName) {
+ String preTag = params.getFieldParam(fieldName, HighlightParams.TAG_PRE, "<em>");
+ String postTag = params.getFieldParam(fieldName, HighlightParams.TAG_POST, "</em>");
+ String ellipsis = params.getFieldParam(fieldName, HighlightParams.TAG_ELLIPSIS, "... ");
+ String encoder = params.getFieldParam(fieldName, HighlightParams.ENCODER, "simple");
+ return new DefaultPassageFormatter(preTag, postTag, ellipsis, "html".equals(encoder));
+ }
+
+ @Override
+ protected PassageScorer getScorer(String fieldName) {
+ float k1 = params.getFieldFloat(fieldName, HighlightParams.SCORE_K1, 1.2f);
+ float b = params.getFieldFloat(fieldName, HighlightParams.SCORE_B, 0.75f);
+ float pivot = params.getFieldFloat(fieldName, HighlightParams.SCORE_PIVOT, 87f);
+ return new PassageScorer(k1, b, pivot);
+ }
+
+ @Override
+ protected BreakIterator getBreakIterator(String field) {
+ String language = params.getFieldParam(field, HighlightParams.BS_LANGUAGE);
+ String country = params.getFieldParam(field, HighlightParams.BS_COUNTRY);
+ String variant = params.getFieldParam(field, HighlightParams.BS_VARIANT);
+ Locale locale = parseLocale(language, country, variant);
+ String type = params.getFieldParam(field, HighlightParams.BS_TYPE);
+ return parseBreakIterator(type, locale);
+ }
+ };
+
+ Map<String,String[]> snippets = highlighter.highlightFields(fieldNames, query, searcher, docIDs, maxPassages);
return encodeSnippets(keys, fieldNames, snippets);
} else {
return null;
@@ -152,38 +202,70 @@ public class PostingsSolrHighlighter ext
return list;
}
- /** Converts solr's DocList to a lucene TopDocs */
- protected TopDocs toTopDocs(DocList docs) {
- ScoreDoc[] scoreDocs = new ScoreDoc[docs.size()];
+ /** Converts solr's DocList to the int[] docIDs */
+ protected int[] toDocIDs(DocList docs) {
+ int[] docIDs = new int[docs.size()];
DocIterator iterator = docs.iterator();
- for (int i = 0; i < scoreDocs.length; i++) {
+ for (int i = 0; i < docIDs.length; i++) {
if (!iterator.hasNext()) {
throw new AssertionError();
}
- scoreDocs[i] = new ScoreDoc(iterator.nextDoc(), Float.NaN);
+ docIDs[i] = iterator.nextDoc();
}
if (iterator.hasNext()) {
throw new AssertionError();
}
- return new TopDocs(docs.matches(), scoreDocs, Float.NaN);
+ return docIDs;
}
/** Retrieves the unique keys for the topdocs to key the results */
- protected String[] getUniqueKeys(SolrIndexSearcher searcher, TopDocs topDocs) throws IOException {
+ protected String[] getUniqueKeys(SolrIndexSearcher searcher, int[] docIDs) throws IOException {
IndexSchema schema = searcher.getSchema();
SchemaField keyField = schema.getUniqueKeyField();
if (keyField != null) {
Set<String> selector = Collections.singleton(keyField.getName());
- String uniqueKeys[] = new String[topDocs.scoreDocs.length];
- for (int i = 0; i < topDocs.scoreDocs.length; i++) {
- int docid = topDocs.scoreDocs[i].doc;
+ String uniqueKeys[] = new String[docIDs.length];
+ for (int i = 0; i < docIDs.length; i++) {
+ int docid = docIDs[i];
StoredDocument doc = searcher.doc(docid, selector);
String id = schema.printableUniqueKey(doc);
uniqueKeys[i] = id;
}
return uniqueKeys;
} else {
- return new String[topDocs.scoreDocs.length];
+ return new String[docIDs.length];
+ }
+ }
+
+ /** parse a break iterator type for the specified locale */
+ protected BreakIterator parseBreakIterator(String type, Locale locale) {
+ if (type == null || "SENTENCE".equals(type)) {
+ return BreakIterator.getSentenceInstance(locale);
+ } else if ("LINE".equals(type)) {
+ return BreakIterator.getLineInstance(locale);
+ } else if ("WORD".equals(type)) {
+ return BreakIterator.getWordInstance(locale);
+ } else if ("CHARACTER".equals(type)) {
+ return BreakIterator.getCharacterInstance(locale);
+ } else if ("WHOLE".equals(type)) {
+ return new WholeBreakIterator();
+ } else {
+ throw new IllegalArgumentException("Unknown " + HighlightParams.BS_TYPE + ": " + type);
+ }
+ }
+
+ /** parse a locale from a language+country+variant spec */
+ protected Locale parseLocale(String language, String country, String variant) {
+ if (language == null && country == null && variant == null) {
+ return Locale.ROOT;
+ } else if (language != null && country == null && variant != null) {
+ throw new IllegalArgumentException("To specify variant, country is required");
+ } else if (language != null && country != null && variant != null) {
+ return new Locale(language, country, variant);
+ } else if (language != null && country != null) {
+ return new Locale(language, country);
+ } else {
+ return new Locale(language);
}
}
}
Modified: lucene/dev/branches/security/solr/core/src/java/org/apache/solr/highlight/SolrHighlighter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/solr/core/src/java/org/apache/solr/highlight/SolrHighlighter.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/solr/core/src/java/org/apache/solr/highlight/SolrHighlighter.java (original)
+++ lucene/dev/branches/security/solr/core/src/java/org/apache/solr/highlight/SolrHighlighter.java Thu May 30 07:53:18 2013
@@ -19,9 +19,7 @@ package org.apache.solr.highlight;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
-import java.util.HashMap;
import java.util.List;
-import java.util.Map;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -38,34 +36,9 @@ public abstract class SolrHighlighter
{
public static Logger log = LoggerFactory.getLogger(SolrHighlighter.class);
- // Thread safe registry
- protected final Map<String,SolrFormatter> formatters =
- new HashMap<String, SolrFormatter>();
-
- // Thread safe registry
- protected final Map<String,SolrEncoder> encoders =
- new HashMap<String, SolrEncoder>();
-
- // Thread safe registry
- protected final Map<String,SolrFragmenter> fragmenters =
- new HashMap<String, SolrFragmenter>() ;
-
- // Thread safe registry
- protected final Map<String, SolrFragListBuilder> fragListBuilders =
- new HashMap<String, SolrFragListBuilder>() ;
-
- // Thread safe registry
- protected final Map<String, SolrFragmentsBuilder> fragmentsBuilders =
- new HashMap<String, SolrFragmentsBuilder>() ;
-
- // Thread safe registry
- protected final Map<String, SolrBoundaryScanner> boundaryScanners =
- new HashMap<String, SolrBoundaryScanner>() ;
-
@Deprecated
public abstract void initalize( SolrConfig config );
-
/**
* Check whether Highlighting is enabled for this request.
* @param params The params controlling Highlighting
Modified: lucene/dev/branches/security/solr/core/src/java/org/apache/solr/logging/LogWatcher.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/solr/core/src/java/org/apache/solr/logging/LogWatcher.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/solr/core/src/java/org/apache/solr/logging/LogWatcher.java (original)
+++ lucene/dev/branches/security/solr/core/src/java/org/apache/solr/logging/LogWatcher.java Thu May 30 07:53:18 2013
@@ -17,21 +17,29 @@
package org.apache.solr.logging;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.core.ConfigSolr;
+import org.apache.solr.core.SolrResourceLoader;
+import org.apache.solr.logging.jul.JulWatcher;
+import org.apache.solr.logging.log4j.Log4jWatcher;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.slf4j.impl.StaticLoggerBinder;
+
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
-import org.apache.solr.common.SolrDocument;
-import org.apache.solr.common.SolrDocumentList;
-import org.apache.solr.core.CoreContainer;
-
/**
* A Class to monitor Logging events and hold N events in memory
*
* This is abstract so we can support both JUL and Log4j (and other logging platforms)
*/
public abstract class LogWatcher<E> {
+
+ private static final Logger log = LoggerFactory.getLogger(LogWatcher.class);
protected CircularList<E> history;
protected long last = -1;
@@ -98,10 +106,82 @@ public abstract class LogWatcher<E> {
public abstract long getTimestamp(E event);
public abstract SolrDocument toSolrDocument(E event);
- public abstract void registerListener(ListenerConfig cfg, CoreContainer container);
+ public abstract void registerListener(ListenerConfig cfg);
public void reset() {
history.clear();
last = -1;
}
+
+ /**
+ * Create and register a LogWatcher.
+ *
+ * JUL and Log4j watchers are supported out-of-the-box. You can register your own
+ * LogWatcher implementation via the plugins architecture
+ *
+ * @param config the CoreContainer's config, with logging configuration details
+ * @param loader a SolrResourceLoader, to be used to load plugin LogWatcher implementations.
+ * Can be null if
+ *
+ * @return a LogWatcher configured for the container's logging framework
+ */
+ public static LogWatcher newRegisteredLogWatcher(ConfigSolr config, SolrResourceLoader loader) {
+
+ if (!config.getBool(ConfigSolr.CfgProp.SOLR_LOGGING_ENABLED, true))
+ return null;
+
+ LogWatcher logWatcher = createWatcher(config, loader);
+
+ if (logWatcher != null) {
+ ListenerConfig v = new ListenerConfig();
+ v.size = config.getInt(ConfigSolr.CfgProp.SOLR_LOGGING_WATCHER_SIZE, 50);
+ v.threshold = config.get(ConfigSolr.CfgProp.SOLR_LOGGING_WATCHER_THRESHOLD, null);
+ if (v.size > 0) {
+ log.info("Registering Log Listener");
+ logWatcher.registerListener(v);
+ }
+ }
+
+ return logWatcher;
+ }
+
+ private static LogWatcher createWatcher(ConfigSolr config, SolrResourceLoader loader) {
+
+ String fname = config.get(ConfigSolr.CfgProp.SOLR_LOGGING_CLASS, null);
+ String slf4jImpl;
+
+ try {
+ slf4jImpl = StaticLoggerBinder.getSingleton().getLoggerFactoryClassStr();
+ if (fname == null) {
+ if (slf4jImpl.indexOf("Log4j") > 0) {
+ fname = "Log4j";
+ } else if (slf4jImpl.indexOf("JDK") > 0) {
+ fname = "JUL";
+ }
+ }
+ }
+ catch (Throwable e) {
+ log.warn("Unable to read SLF4J version. LogWatcher will be disabled: " + e);
+ return null;
+ }
+
+ if (fname == null) {
+ log.info("No LogWatcher configured");
+ return null;
+ }
+
+ if ("JUL".equalsIgnoreCase(fname))
+ return new JulWatcher(slf4jImpl);
+ if ("Log4j".equals(fname))
+ return new Log4jWatcher(slf4jImpl);
+
+ try {
+ return loader != null ? loader.newInstance(fname, LogWatcher.class) : null;
+ }
+ catch (Throwable e) {
+ log.warn("Unable to load LogWatcher {}: {}", fname, e);
+ }
+
+ return null;
+ }
}
\ No newline at end of file
Modified: lucene/dev/branches/security/solr/core/src/java/org/apache/solr/logging/jul/JulWatcher.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/solr/core/src/java/org/apache/solr/logging/jul/JulWatcher.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/solr/core/src/java/org/apache/solr/logging/jul/JulWatcher.java (original)
+++ lucene/dev/branches/security/solr/core/src/java/org/apache/solr/logging/jul/JulWatcher.java Thu May 30 07:53:18 2013
@@ -16,6 +16,13 @@
*/
package org.apache.solr.logging.jul;
+import com.google.common.base.Throwables;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.logging.CircularList;
+import org.apache.solr.logging.ListenerConfig;
+import org.apache.solr.logging.LogWatcher;
+import org.apache.solr.logging.LoggerInfo;
+
import java.util.Arrays;
import java.util.Collection;
import java.util.Date;
@@ -28,15 +35,6 @@ import java.util.logging.LogManager;
import java.util.logging.LogRecord;
import java.util.logging.Logger;
-import org.apache.solr.common.SolrDocument;
-import org.apache.solr.core.CoreContainer;
-import org.apache.solr.logging.CircularList;
-import org.apache.solr.logging.ListenerConfig;
-import org.apache.solr.logging.LoggerInfo;
-import org.apache.solr.logging.LogWatcher;
-
-import com.google.common.base.Throwables;
-
public class JulWatcher extends LogWatcher<LogRecord> {
final String name;
@@ -131,7 +129,7 @@ public class JulWatcher extends LogWatch
}
@Override
- public void registerListener(ListenerConfig cfg, CoreContainer container) {
+ public void registerListener(ListenerConfig cfg) {
if(history!=null) {
throw new IllegalStateException("History already registered");
}
Modified: lucene/dev/branches/security/solr/core/src/java/org/apache/solr/parser/SolrQueryParserBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/solr/core/src/java/org/apache/solr/parser/SolrQueryParserBase.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/solr/core/src/java/org/apache/solr/parser/SolrQueryParserBase.java (original)
+++ lucene/dev/branches/security/solr/core/src/java/org/apache/solr/parser/SolrQueryParserBase.java Thu May 30 07:53:18 2013
@@ -573,11 +573,17 @@ public abstract class SolrQueryParserBas
throws SyntaxError {
Query query = getFieldQuery(field, queryText, true);
- if (query instanceof PhraseQuery) {
- ((PhraseQuery) query).setSlop(slop);
- }
- if (query instanceof MultiPhraseQuery) {
- ((MultiPhraseQuery) query).setSlop(slop);
+ // only set slop of the phrase query was a result of this parser
+ // and not a sub-parser.
+ if (subQParser == null) {
+
+ if (query instanceof PhraseQuery) {
+ ((PhraseQuery) query).setSlop(slop);
+ }
+ if (query instanceof MultiPhraseQuery) {
+ ((MultiPhraseQuery) query).setSlop(slop);
+ }
+
}
return query;
@@ -779,7 +785,7 @@ public abstract class SolrQueryParserBas
float boostVal = Float.parseFloat(boost.image);
// avoid boosting null queries, such as those caused by stop words
if (q != null) {
- q.setBoost(boostVal);
+ q.setBoost(q.getBoost() * boostVal);
}
}
return q;
@@ -922,7 +928,8 @@ public abstract class SolrQueryParserBas
}
protected String analyzeIfMultitermTermText(String field, String part, FieldType fieldType) {
- if (part == null) return part;
+
+ if (part == null || ! (fieldType instanceof TextField) || ((TextField)fieldType).getMultiTermAnalyzer() == null) return part;
SchemaField sf = schema.getFieldOrNull((field));
if (sf == null || ! (fieldType instanceof TextField)) return part;
@@ -931,7 +938,8 @@ public abstract class SolrQueryParserBas
}
- // called from parser
+ private QParser subQParser = null;
+
protected Query getFieldQuery(String field, String queryText, boolean quoted) throws SyntaxError {
checkNullField(field);
// intercept magic field name of "_" to use as a hook for our
@@ -939,8 +947,8 @@ public abstract class SolrQueryParserBas
if (field.charAt(0) == '_' && parser != null) {
MagicFieldName magic = MagicFieldName.get(field);
if (null != magic) {
- QParser nested = parser.subQuery(queryText, magic.subParser);
- return nested.getQuery();
+ subQParser = parser.subQuery(queryText, magic.subParser);
+ return subQParser.getQuery();
}
}
SchemaField sf = schema.getFieldOrNull(field);