You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ro...@apache.org on 2014/12/01 18:25:47 UTC
svn commit: r1642718 [9/12] - in /lucene/dev/branches/lucene2878: ./
dev-tools/ dev-tools/eclipse/dot.settings/ dev-tools/idea/.idea/
dev-tools/idea/lucene/benchmark/src/ dev-tools/idea/lucene/highlighter/
dev-tools/maven/ dev-tools/maven/solr/webapp/ ...
Modified: lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/cloud/ZkController.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/cloud/ZkController.java?rev=1642718&r1=1642717&r2=1642718&view=diff
==============================================================================
--- lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/cloud/ZkController.java (original)
+++ lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/cloud/ZkController.java Mon Dec 1 17:25:39 2014
@@ -41,12 +41,13 @@ import java.util.Set;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
-import java.util.concurrent.atomic.AtomicInteger;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.solr.client.solrj.impl.HttpSolrServer;
import org.apache.solr.client.solrj.request.CoreAdminRequest.WaitForState;
+import org.apache.solr.cloud.overseer.OverseerAction;
+import org.apache.solr.cloud.overseer.SliceMutator;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.cloud.BeforeReconnect;
@@ -839,7 +840,7 @@ public final class ZkController {
boolean joinAtHead = false;
Replica replica = zkStateReader.getClusterState().getReplica(desc.getCloudDescriptor().getCollectionName(), coreZkNodeName);
if (replica != null) {
- joinAtHead = replica.getBool(Overseer.preferredLeaderProp, false);
+ joinAtHead = replica.getBool(SliceMutator.PREFERRED_LEADER_PROP, false);
}
joinElection(desc, afterExpiration, joinAtHead);
} catch (InterruptedException e) {
@@ -1080,7 +1081,7 @@ public final class ZkController {
}
public void publish(final CoreDescriptor cd, final String state, boolean updateLastState) throws KeeperException, InterruptedException {
- publish(cd, state, true, false);
+ publish(cd, state, updateLastState, false);
}
/**
@@ -1215,16 +1216,16 @@ public final class ZkController {
}
if (removeWatch) zkStateReader.removeZKWatch(collection);
ZkNodeProps m = new ZkNodeProps(Overseer.QUEUE_OPERATION,
- Overseer.OverseerAction.DELETECORE.toLower(), ZkStateReader.CORE_NAME_PROP, coreName,
+ OverseerAction.DELETECORE.toLower(), ZkStateReader.CORE_NAME_PROP, coreName,
ZkStateReader.NODE_NAME_PROP, getNodeName(),
ZkStateReader.COLLECTION_PROP, cloudDescriptor.getCollectionName(),
ZkStateReader.CORE_NODE_NAME_PROP, coreNodeName);
overseerJobQueue.offer(ZkStateReader.toJSON(m));
if(configLocation != null) {
- synchronized (confDirectoryWatchers) {
+ synchronized (confDirectoryListeners) {
log.info("This conf directory is no more watched {0}",configLocation);
- confDirectoryWatchers.remove(configLocation);
+ confDirectoryListeners.remove(configLocation);
}
}
}
@@ -2115,29 +2116,50 @@ public final class ZkController {
*
* @return true on success
*/
- public static boolean persistConfigResourceToZooKeeper( SolrResourceLoader loader, int znodeVersion , String resourceName, byte[] content, boolean createIfNotExists) {
+ public static boolean persistConfigResourceToZooKeeper( SolrResourceLoader loader, int znodeVersion ,
+ String resourceName, byte[] content,
+ boolean createIfNotExists) {
final ZkSolrResourceLoader zkLoader = (ZkSolrResourceLoader)loader;
final ZkController zkController = zkLoader.getZkController();
final SolrZkClient zkClient = zkController.getZkClient();
final String resourceLocation = zkLoader.getConfigSetZkPath() + "/" + resourceName;
- String errMsg = "Failed to persist resource at {0} - version mismatch";
+ String errMsg = "Failed to persist resource at {0} - old {1}";
try {
try {
zkClient.setData(resourceLocation , content,znodeVersion, true);
+ zkClient.setData(zkLoader.getConfigSetZkPath(),new byte[]{0},true);
} catch (NoNodeException e) {
if(createIfNotExists){
try {
zkClient.create(resourceLocation,content, CreateMode.PERSISTENT,true);
+ zkClient.setData(zkLoader.getConfigSetZkPath(), new byte[]{0}, true);
} catch (KeeperException.NodeExistsException nee) {
- log.info(MessageFormat.format(errMsg,resourceLocation));
- throw new ResourceModifiedInZkException(ErrorCode.CONFLICT, MessageFormat.format(errMsg,resourceLocation) + ", retry.");
+ try {
+ Stat stat = zkClient.exists(resourceLocation, null, true);
+ log.info("failed to set data version in zk is {0} and expected version is {1} ", stat.getVersion(),znodeVersion);
+ } catch (Exception e1) {
+ log.warn("could not get stat");
+ }
+
+ log.info(MessageFormat.format(errMsg,resourceLocation,znodeVersion));
+ throw new ResourceModifiedInZkException(ErrorCode.CONFLICT, MessageFormat.format(errMsg,resourceLocation,znodeVersion) + ", retry.");
}
}
}
} catch (KeeperException.BadVersionException bve){
- log.info(MessageFormat.format(errMsg,resourceLocation));
- throw new ResourceModifiedInZkException(ErrorCode.CONFLICT, MessageFormat.format(errMsg,resourceLocation) + ", retry.");
+ int v = -1;
+ try {
+ Stat stat = zkClient.exists(resourceLocation, null, true);
+ v = stat.getVersion();
+ } catch (Exception e) {
+ log.error(e.getMessage());
+
+ }
+ log.info(MessageFormat.format(errMsg+ " zkVersion= "+v,resourceLocation,znodeVersion));
+ throw new ResourceModifiedInZkException(ErrorCode.CONFLICT, MessageFormat.format(errMsg,resourceLocation,znodeVersion) + ", retry.");
+ }catch (ResourceModifiedInZkException e){
+ throw e;
} catch (Exception e) {
if (e instanceof InterruptedException) {
Thread.currentThread().interrupt(); // Restore the interrupted status
@@ -2157,9 +2179,13 @@ public final class ZkController {
public void unRegisterConfListener(Runnable listener) {
if(listener == null) return;
- synchronized (confDirectoryWatchers){
- for (Set<Runnable> runnables : confDirectoryWatchers.values()) {
- if(runnables != null) runnables.remove(listener);
+ synchronized (confDirectoryListeners){
+ for (Set<Runnable> listeners : confDirectoryListeners.values()) {
+ if(listeners != null) {
+ if(listeners.remove(listener)) {
+ log.info(" a listener was removed because of core close");
+ }
+ }
}
}
@@ -2172,9 +2198,9 @@ public final class ZkController {
*/
public void registerConfListenerForCore(String confDir,SolrCore core, final Runnable listener){
if(listener==null) throw new NullPointerException("listener cannot be null");
- synchronized (confDirectoryWatchers){
- if(confDirectoryWatchers.containsKey(confDir)){
- confDirectoryWatchers.get(confDir).add(listener);
+ synchronized (confDirectoryListeners){
+ if(confDirectoryListeners.containsKey(confDir)){
+ confDirectoryListeners.get(confDir).add(listener);
core.addCloseHook(new CloseHook() {
@Override
public void preClose(SolrCore core) {
@@ -2184,69 +2210,78 @@ public final class ZkController {
@Override
public void postClose(SolrCore core) { }
});
-
-
} else {
throw new SolrException(ErrorCode.SERVER_ERROR,"This conf directory is not valid");
}
}
}
- private Map<String , Set<Runnable>> confDirectoryWatchers = new HashMap<>();
- void watchZKConfDir(final String zkDir) {
+ private final Map<String , Set<Runnable>> confDirectoryListeners = new HashMap<>();
- if(!confDirectoryWatchers.containsKey(zkDir)){
- confDirectoryWatchers.put(zkDir,new HashSet<Runnable>());
- }else{
- //it's already watched
- return;
- }
+ void watchZKConfDir(final String zkDir) {
+ log.info("watch zkdir " + zkDir);
+ if (!confDirectoryListeners.containsKey(zkDir)) {
+ confDirectoryListeners.put(zkDir, new HashSet<Runnable>());
+ setConfWatcher(zkDir, new WatcherImpl(zkDir));
- Watcher watcher = new Watcher() {
- @Override
- public void process(WatchedEvent event) {
- try {
- synchronized (confDirectoryWatchers) {
- // if this is not among directories to be watched then don't set the watcher anymore
- if(!confDirectoryWatchers.containsKey(zkDir)) return;
- }
+ }
- if (event.getType() == Event.EventType.NodeChildrenChanged) {
- synchronized (confDirectoryWatchers) {
- final Set<Runnable> listeners = confDirectoryWatchers.get(zkDir);
- if (listeners != null) {
- new Thread() {
- @Override
- public synchronized void run() {
- //running in a separate thread so that the zk event thread is not
- // unnecessarily held up
- for (Runnable listener : listeners) listener.run();
- }
- }.start();
- }
- }
- }
- } finally {
- if (Event.EventType.None.equals(event.getType())) {
+ }
+ private class WatcherImpl implements Watcher{
+ private final String zkDir ;
+
+ private WatcherImpl(String dir) {
+ this.zkDir = dir;
+ }
+
+ @Override
+ public void process(WatchedEvent event) {
+ try {
+
+ synchronized (confDirectoryListeners) {
+ // if this is not among directories to be watched then don't set the watcher anymore
+ if( !confDirectoryListeners.containsKey(zkDir)) {
+ log.info("Watcher on {} is removed ", zkDir);
return;
- } else {
- setConfWatcher(zkDir,this);
}
+ Set<Runnable> listeners = confDirectoryListeners.get(zkDir);
+ if (listeners != null && !listeners.isEmpty()) {
+ final Set<Runnable> listenersCopy = new HashSet<>(listeners);
+ new Thread() {
+ //run these in a separate thread because this can be long running
+ public void run() {
+ for (final Runnable listener : listenersCopy) {
+ try {
+ listener.run();
+ } catch (Exception e) {
+ log.warn("listener throws error", e);
+ }
+ }
+ }
+ }.start();
+ }
+
}
- }
- };
- setConfWatcher(zkDir,watcher);
+ } finally {
+ if (Event.EventType.None.equals(event.getType())) {
+ log.info("A node got unwatched for {}", zkDir);
+ return;
+ } else {
+ setConfWatcher(zkDir,this);
+ }
+ }
+ }
}
private void setConfWatcher(String zkDir, Watcher watcher) {
try {
- zkClient.getChildren(zkDir,watcher,true);
+ zkClient.exists(zkDir,watcher,true);
} catch (KeeperException e) {
log.error("failed to set watcher for conf dir {} ", zkDir);
} catch (InterruptedException e) {
- Thread.interrupted();
+ Thread.currentThread().interrupt();
log.error("failed to set watcher for conf dir {} ", zkDir);
}
}
@@ -2255,11 +2290,10 @@ public final class ZkController {
return new OnReconnect() {
@Override
public void command() {
- synchronized (confDirectoryWatchers){
- for (String s : confDirectoryWatchers.keySet()) {
+ synchronized (confDirectoryListeners){
+ for (String s : confDirectoryListeners.keySet()) {
watchZKConfDir(s);
}
-
}
}
};
Modified: lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/cloud/ZkSolrResourceLoader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/cloud/ZkSolrResourceLoader.java?rev=1642718&r1=1642717&r2=1642718&view=diff
==============================================================================
--- lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/cloud/ZkSolrResourceLoader.java (original)
+++ lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/cloud/ZkSolrResourceLoader.java Mon Dec 1 17:25:39 2014
@@ -111,10 +111,6 @@ public class ZkSolrResourceLoader extend
}
- public ZkByteArrayInputStream(byte[] buf, int offset, int length, Stat stat) {
- super(buf, offset, length);
- this.stat = stat;
- }
public Stat getStat(){
return stat;
}
Modified: lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/core/Config.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/core/Config.java?rev=1642718&r1=1642717&r2=1642718&view=diff
==============================================================================
--- lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/core/Config.java (original)
+++ lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/core/Config.java Mon Dec 1 17:25:39 2014
@@ -18,6 +18,7 @@
package org.apache.solr.core;
import org.apache.lucene.util.Version;
+import org.apache.solr.cloud.ZkSolrResourceLoader;
import org.apache.solr.common.SolrException;
import org.apache.solr.util.DOMUtil;
import org.apache.solr.util.SystemIdResolver;
@@ -48,6 +49,7 @@ import javax.xml.xpath.XPathExpressionEx
import javax.xml.xpath.XPathFactory;
import java.io.IOException;
+import java.io.InputStream;
import java.text.ParseException;
import java.util.Arrays;
import java.util.HashSet;
@@ -74,6 +76,7 @@ public class Config {
private final String prefix;
private final String name;
private final SolrResourceLoader loader;
+ private int zkVersion = -1;
/**
* Builds a config from a resource name with no xpath prefix.
@@ -113,9 +116,14 @@ public class Config {
this.prefix = (prefix != null && !prefix.endsWith("/"))? prefix + '/' : prefix;
try {
javax.xml.parsers.DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
-
+
if (is == null) {
- is = new InputSource(loader.openConfig(name));
+ InputStream in = loader.openConfig(name);
+ if (in instanceof ZkSolrResourceLoader.ZkByteArrayInputStream) {
+ zkVersion = ((ZkSolrResourceLoader.ZkByteArrayInputStream) in).getStat().getVersion();
+ log.info("loaded config {} with version {} ",name,zkVersion);
+ }
+ is = new InputSource(in);
is.setSystemId(SystemIdResolver.createSystemIdFromResourceName(name));
}
@@ -464,6 +472,12 @@ public class Config {
return version;
}
+ /**If this config is loaded from zk the version is relevant other wise -1 is returned
+ */
+ public int getZnodeVersion(){
+ return zkVersion;
+ }
+
public Config getOriginalConfig() {
return new Config(loader, null, origDoc);
}
Modified: lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/core/ConfigOverlay.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/core/ConfigOverlay.java?rev=1642718&r1=1642717&r2=1642718&view=diff
==============================================================================
--- lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/core/ConfigOverlay.java (original)
+++ lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/core/ConfigOverlay.java Mon Dec 1 17:25:39 2014
@@ -28,17 +28,22 @@ import java.util.Map;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.common.params.CoreAdminParams;
import org.apache.solr.common.util.StrUtils;
+import org.apache.solr.request.SolrRequestHandler;
import org.noggit.CharArr;
import org.noggit.JSONParser;
import org.noggit.JSONWriter;
import org.noggit.ObjectBuilder;
-public class ConfigOverlay {
+import static org.apache.solr.common.params.CoreAdminParams.NAME;
+
+public class ConfigOverlay implements MapSerializable{
private final int znodeVersion ;
private Map<String, Object> data;
private Map<String,Object> props;
private Map<String,Object> userProps;
+ private Map<String, Map> reqHandlers;
public ConfigOverlay(Map<String,Object> jsonObj, int znodeVersion){
if(jsonObj == null) jsonObj= Collections.EMPTY_MAP;
@@ -48,6 +53,8 @@ public class ConfigOverlay {
if(props == null) props= Collections.EMPTY_MAP;
userProps = (Map<String, Object>) data.get("userProps");
if(userProps == null) userProps= Collections.EMPTY_MAP;
+ reqHandlers = (Map<String, Map>) data.get(SolrRequestHandler.TYPE);
+ if(reqHandlers == null) reqHandlers = new LinkedHashMap<>();
}
public Object getXPathProperty(String xpath){
@@ -177,13 +184,6 @@ public class ConfigOverlay {
return out.toString();
}
- public Map toOutputFormat() {
- Map result = new LinkedHashMap();
- result.put("version",znodeVersion);
- result.putAll(data);
- return result;
- }
-
public static final String RESOURCE_NAME = "configoverlay.json";
@@ -254,4 +254,36 @@ public class ConfigOverlay {
public Map<String, Object> getUserProps() {
return userProps;
}
+
+ @Override
+ public Map<String, Object> toMap() {
+ Map result = new LinkedHashMap();
+ result.put("znodeVersion",znodeVersion);
+ result.putAll(data);
+ return result;
+ }
+
+ public Map<String, Map> getReqHandlers() {
+ return Collections.unmodifiableMap(reqHandlers);
+ }
+
+ public ConfigOverlay addReqHandler(Map<String, Object> info) {
+ ConfigOverlay copy = copyOverLayWithReqHandler();
+ copy.reqHandlers.put((String)info.get(NAME) , info);
+ return copy;
+ }
+
+ private ConfigOverlay copyOverLayWithReqHandler() {
+ LinkedHashMap<String, Object> newmap = new LinkedHashMap<>(data);
+ ConfigOverlay copy = new ConfigOverlay(newmap, znodeVersion);
+ newmap.put(SolrRequestHandler.TYPE, copy.reqHandlers = new LinkedHashMap<>(reqHandlers));
+ return copy;
+ }
+
+ public ConfigOverlay deleteHandler(String name) {
+ ConfigOverlay copy = copyOverLayWithReqHandler();
+ copy.reqHandlers.remove(name);
+ return copy;
+
+ }
}
Modified: lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/core/CoreContainer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/core/CoreContainer.java?rev=1642718&r1=1642717&r2=1642718&view=diff
==============================================================================
--- lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/core/CoreContainer.java (original)
+++ lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/core/CoreContainer.java Mon Dec 1 17:25:39 2014
@@ -22,7 +22,6 @@ import static com.google.common.base.Pre
import java.io.File;
import java.util.ArrayList;
import java.util.Collection;
-import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
@@ -602,7 +601,7 @@ public class CoreContainer {
solrCores.waitAddPendingCoreOps(name);
ConfigSet coreConfig = coreConfigService.getConfig(cd);
log.info("Reloading SolrCore '{}' using configuration from {}", cd.getName(), coreConfig.getName());
- SolrCore newCore = core.reload(coreConfig, core);
+ SolrCore newCore = core.reload(coreConfig);
registerCore(name, newCore, false);
}
catch (Exception e) {
Modified: lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java?rev=1642718&r1=1642717&r2=1642718&view=diff
==============================================================================
--- lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java (original)
+++ lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java Mon Dec 1 17:25:39 2014
@@ -209,7 +209,7 @@ public class HdfsDirectoryFactory extend
private BlockCache createBlockCache(int numberOfBlocksPerBank, int blockSize,
int bankCount, boolean directAllocation, int slabSize, int bufferSize,
int bufferCount) {
- BufferStore.initNewBuffer(bufferSize, bufferCount);
+ BufferStore.initNewBuffer(bufferSize, bufferCount, metrics);
long totalMemory = (long) bankCount * (long) numberOfBlocksPerBank
* (long) blockSize;
Modified: lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/core/InitParams.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/core/InitParams.java?rev=1642718&r1=1642717&r2=1642718&view=diff
==============================================================================
--- lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/core/InitParams.java (original)
+++ lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/core/InitParams.java Mon Dec 1 17:25:39 2014
@@ -78,10 +78,16 @@ public class InitParams {
}
- public void apply(NamedList initArgs) {
- merge( (NamedList) initArgs.get(PluginInfo.DEFAULTS), defaults,initArgs, PluginInfo.DEFAULTS, false);
- merge((NamedList) initArgs.get(PluginInfo.INVARIANTS), invariants, initArgs, PluginInfo.INVARIANTS, false);
- merge((NamedList) initArgs.get(PluginInfo.APPENDS), appends, initArgs, PluginInfo.APPENDS, true);
+ public void apply(PluginInfo info) {
+ if (info.isFromSolrConfig) {
+ //if this is a component implicitly defined in code it should be overridden by initPrams
+ merge(defaults, (NamedList) info.initArgs.get(PluginInfo.DEFAULTS) ,info.initArgs, PluginInfo.DEFAULTS, false);
+ } else {
+ //if the args is initialized from solrconfig.xml inside the requesthHandler it should be taking precedence over initParams
+ merge( (NamedList) info.initArgs.get(PluginInfo.DEFAULTS), defaults,info.initArgs, PluginInfo.DEFAULTS, false);
+ }
+ merge((NamedList) info.initArgs.get(PluginInfo.INVARIANTS), invariants, info.initArgs, PluginInfo.INVARIANTS, false);
+ merge((NamedList) info.initArgs.get(PluginInfo.APPENDS), appends, info.initArgs, PluginInfo.APPENDS, true);
}
private static void merge(NamedList first, NamedList second, NamedList sink, String name, boolean appends) {
Modified: lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/core/JmxMonitoredMap.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/core/JmxMonitoredMap.java?rev=1642718&r1=1642717&r2=1642718&view=diff
==============================================================================
--- lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/core/JmxMonitoredMap.java (original)
+++ lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/core/JmxMonitoredMap.java Mon Dec 1 17:25:39 2014
@@ -53,6 +53,10 @@ public class JmxMonitoredMap<K, V> exten
private static final Logger LOG = LoggerFactory.getLogger(JmxMonitoredMap.class
.getName());
+ // set to true to use cached statistics NamedLists between getMBeanInfo calls to work
+ // around over calling getStatistics on MBeanInfos when iterating over all attributes (SOLR-6586)
+ private boolean useCachedStatsBetweenGetMBeanInfoCalls = Boolean.getBoolean("useCachedStatsBetweenGetMBeanInfoCalls");
+
private MBeanServer server = null;
private String jmxRootName;
@@ -144,7 +148,7 @@ public class JmxMonitoredMap<K, V> exten
ObjectName name = getObjectName(key, infoBean);
if (server.isRegistered(name))
server.unregisterMBean(name);
- SolrDynamicMBean mbean = new SolrDynamicMBean(coreHashCode, infoBean);
+ SolrDynamicMBean mbean = new SolrDynamicMBean(coreHashCode, infoBean, useCachedStatsBetweenGetMBeanInfoCalls);
server.registerMBean(mbean, name);
} catch (Exception e) {
LOG.warn( "Failed to register info bean: " + key, e);
@@ -213,8 +217,17 @@ public class JmxMonitoredMap<K, V> exten
private HashSet<String> staticStats;
private String coreHashCode;
-
+
+ private volatile NamedList cachedDynamicStats;
+
+ private boolean useCachedStatsBetweenGetMBeanInfoCalls;
+
public SolrDynamicMBean(String coreHashCode, SolrInfoMBean managedResource) {
+ this(coreHashCode, managedResource, false);
+ }
+
+ public SolrDynamicMBean(String coreHashCode, SolrInfoMBean managedResource, boolean useCachedStatsBetweenGetMBeanInfoCalls) {
+ this.useCachedStatsBetweenGetMBeanInfoCalls = useCachedStatsBetweenGetMBeanInfoCalls;
this.infoBean = managedResource;
staticStats = new HashSet<>();
@@ -242,6 +255,11 @@ public class JmxMonitoredMap<K, V> exten
try {
NamedList dynamicStats = infoBean.getStatistics();
+
+ if (useCachedStatsBetweenGetMBeanInfoCalls) {
+ cachedDynamicStats = dynamicStats;
+ }
+
if (dynamicStats != null) {
for (int i = 0; i < dynamicStats.size(); i++) {
String name = dynamicStats.getName(i);
@@ -303,8 +321,17 @@ public class JmxMonitoredMap<K, V> exten
throw new AttributeNotFoundException(attribute);
}
} else {
- NamedList list = infoBean.getStatistics();
- val = list.get(attribute);
+ NamedList stats = null;
+ if (useCachedStatsBetweenGetMBeanInfoCalls) {
+ NamedList cachedStats = this.cachedDynamicStats;
+ if (cachedStats != null) {
+ stats = cachedStats;
+ }
+ }
+ if (stats == null) {
+ stats = infoBean.getStatistics();
+ }
+ val = stats.get(attribute);
}
if (val != null) {
Modified: lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/core/PluginInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/core/PluginInfo.java?rev=1642718&r1=1642717&r2=1642718&view=diff
==============================================================================
--- lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/core/PluginInfo.java (original)
+++ lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/core/PluginInfo.java Mon Dec 1 17:25:39 2014
@@ -22,8 +22,12 @@ import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import java.util.*;
+
+import static java.util.Arrays.asList;
import static java.util.Collections.unmodifiableList;
import static java.util.Collections.unmodifiableMap;
+import static org.apache.solr.common.params.CoreAdminParams.NAME;
+import static org.apache.solr.schema.FieldType.CLASS_NAME;
/**
* An Object which represents a Plugin of any type
@@ -34,24 +38,40 @@ public class PluginInfo implements MapSe
public final NamedList initArgs;
public final Map<String, String> attributes;
public final List<PluginInfo> children;
+ public final boolean isFromSolrConfig;
public PluginInfo(String type, Map<String, String> attrs ,NamedList initArgs, List<PluginInfo> children) {
this.type = type;
- this.name = attrs.get("name");
- this.className = attrs.get("class");
+ this.name = attrs.get(NAME);
+ this.className = attrs.get(CLASS_NAME);
this.initArgs = initArgs;
attributes = unmodifiableMap(attrs);
this.children = children == null ? Collections.<PluginInfo>emptyList(): unmodifiableList(children);
+ isFromSolrConfig = false;
}
public PluginInfo(Node node, String err, boolean requireName, boolean requireClass) {
type = node.getNodeName();
- name = DOMUtil.getAttr(node, "name", requireName ? err : null);
- className = DOMUtil.getAttr(node, "class", requireClass ? err : null);
+ name = DOMUtil.getAttr(node, NAME, requireName ? err : null);
+ className = DOMUtil.getAttr(node, CLASS_NAME, requireClass ? err : null);
initArgs = DOMUtil.childNodesToNamedList(node);
attributes = unmodifiableMap(DOMUtil.toMap(node.getAttributes()));
children = loadSubPlugins(node);
+ isFromSolrConfig = true;
+ }
+
+ public PluginInfo(String type, Map<String,Object> map) {
+ LinkedHashMap m = new LinkedHashMap<>(map);
+ NamedList nl = new NamedList();
+ for (String s : asList(DEFAULTS, APPENDS, INVARIANTS)) if (m.get(s) != null) nl.add(s, map.remove(s));
+ this.type = type;
+ this.name = (String) m.get(NAME);
+ this.className = (String) m.get(CLASS_NAME);
+ this.initArgs = nl;
+ attributes = unmodifiableMap(m);
+ this.children = Collections.<PluginInfo>emptyList();
+ isFromSolrConfig = true;
}
private List<PluginInfo> loadSubPlugins(Node node) {
@@ -128,11 +148,11 @@ public class PluginInfo implements MapSe
public static final PluginInfo EMPTY_INFO = new PluginInfo("",Collections.<String,String>emptyMap(), new NamedList(),Collections.<PluginInfo>emptyList());
private static final HashSet<String> NL_TAGS = new HashSet<>
- (Arrays.asList("lst", "arr",
- "bool",
- "str",
- "int","long",
- "float","double"));
+ (asList("lst", "arr",
+ "bool",
+ "str",
+ "int", "long",
+ "float", "double"));
public static final String DEFAULTS = "defaults";
public static final String APPENDS = "appends";
public static final String INVARIANTS = "invariants";
Copied: lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/core/PluginsRegistry.java (from r1642144, lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/PluginsRegistry.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/core/PluginsRegistry.java?p2=lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/core/PluginsRegistry.java&p1=lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/PluginsRegistry.java&r1=1642144&r2=1642718&rev=1642718&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/PluginsRegistry.java (original)
+++ lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/core/PluginsRegistry.java Mon Dec 1 17:25:39 2014
@@ -27,6 +27,7 @@ import org.apache.solr.common.util.Named
import org.apache.solr.handler.PingRequestHandler;
import org.apache.solr.handler.RealTimeGetHandler;
import org.apache.solr.handler.ReplicationHandler;
+import org.apache.solr.handler.SchemaHandler;
import org.apache.solr.handler.SolrConfigHandler;
import org.apache.solr.handler.UpdateRequestHandler;
import org.apache.solr.handler.admin.LoggingHandler;
@@ -57,7 +58,8 @@ public class PluginsRegistry {
//solrconfighandler
implicits.add(getReqHandlerInfo("/config", SolrConfigHandler.class, null));
-
+ //schemahandler
+ implicits.add(getReqHandlerInfo("/schema", SchemaHandler.class, null));
//register replicationhandler always for SolrCloud
implicits.add(getReqHandlerInfo("/replication", ReplicationHandler.class,null));
Modified: lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/core/RequestHandlers.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/core/RequestHandlers.java?rev=1642718&r1=1642717&r2=1642718&view=diff
==============================================================================
--- lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/core/RequestHandlers.java (original)
+++ lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/core/RequestHandlers.java Mon Dec 1 17:25:39 2014
@@ -139,15 +139,19 @@ public final class RequestHandlers {
* Handlers will be registered and initialized in the order they appear in solrconfig.xml
*/
- void initHandlersFromConfig(SolrConfig config, List<PluginInfo> implicits){
+ void initHandlersFromConfig(SolrConfig config){
+ List<PluginInfo> implicits = PluginsRegistry.getHandlers(core);
// use link map so we iterate in the same order
Map<PluginInfo,SolrRequestHandler> handlers = new LinkedHashMap<>();
- Map<String, PluginInfo> implicitInfoMap= new HashMap<>();
+ Map<String, PluginInfo> infoMap= new LinkedHashMap<>();
//deduping implicit and explicit requesthandlers
- for (PluginInfo info : implicits) implicitInfoMap.put(info.name,info);
+ for (PluginInfo info : implicits) infoMap.put(info.name,info);
for (PluginInfo info : config.getPluginInfos(SolrRequestHandler.class.getName()))
- if(implicitInfoMap.containsKey(info.name)) implicitInfoMap.remove(info.name);
- ArrayList<PluginInfo> infos = new ArrayList<>(implicitInfoMap.values());
+ if(infoMap.containsKey(info.name)) infoMap.remove(info.name);
+ for (Map.Entry e : core.getSolrConfig().getOverlay().getReqHandlers().entrySet())
+ infoMap.put((String)e.getKey(), new PluginInfo(SolrRequestHandler.TYPE, (Map)e.getValue()));
+
+ ArrayList<PluginInfo> infos = new ArrayList<>(infoMap.values());
infos.addAll(config.getPluginInfos(SolrRequestHandler.class.getName()));
for (PluginInfo info : infos) {
try {
@@ -212,7 +216,7 @@ public final class RequestHandlers {
if(!ags.isEmpty()){
info = new PluginInfo(info.type, info.attributes, info.initArgs.clone(), info.children);
for (InitParams initParam : ags) {
- initParam.apply(info.initArgs);
+ initParam.apply(info);
}
}
return info;
Modified: lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/core/SolrConfig.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/core/SolrConfig.java?rev=1642718&r1=1642717&r2=1642718&view=diff
==============================================================================
--- lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/core/SolrConfig.java (original)
+++ lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/core/SolrConfig.java Mon Dec 1 17:25:39 2014
@@ -287,7 +287,7 @@ public class SolrConfig extends Config i
}
public static List<SolrPluginInfo> plugins = ImmutableList.<SolrPluginInfo>builder()
- .add(new SolrPluginInfo(SolrRequestHandler.class, "requestHandler", REQUIRE_NAME, REQUIRE_CLASS, MULTI_OK))
+ .add(new SolrPluginInfo(SolrRequestHandler.class, SolrRequestHandler.TYPE, REQUIRE_NAME, REQUIRE_CLASS, MULTI_OK))
.add(new SolrPluginInfo(QParserPlugin.class, "queryParser", REQUIRE_NAME, REQUIRE_CLASS, MULTI_OK))
.add(new SolrPluginInfo(QueryResponseWriter.class, "queryResponseWriter", REQUIRE_NAME, REQUIRE_CLASS, MULTI_OK))
.add(new SolrPluginInfo(ValueSourceParser.class, "valueSourceParser", REQUIRE_NAME, REQUIRE_CLASS, MULTI_OK))
@@ -328,19 +328,20 @@ public class SolrConfig extends Config i
}
}
- private static ConfigOverlay getConfigOverlay(SolrResourceLoader loader) {
+ public static ConfigOverlay getConfigOverlay(SolrResourceLoader loader) {
InputStream in = null;
try {
in = loader.openResource(ConfigOverlay.RESOURCE_NAME);
} catch (IOException e) {
//no problem no overlay.json file
- return new ConfigOverlay(Collections.EMPTY_MAP,0);
+ return new ConfigOverlay(Collections.EMPTY_MAP,-1);
}
try {
int version = 0; //will be always 0 for file based resourceloader
if (in instanceof ZkSolrResourceLoader.ZkByteArrayInputStream) {
version = ((ZkSolrResourceLoader.ZkByteArrayInputStream) in).getStat().getVersion();
+ log.info("config overlay loaded . version : {} ", version);
}
Map m = (Map) ObjectBuilder.getVal(new JSONParser(new InputStreamReader(in, StandardCharsets.UTF_8)));
return new ConfigOverlay(m,version);
@@ -712,6 +713,7 @@ public class SolrConfig extends Config i
@Override
public Map<String, Object> toMap() {
LinkedHashMap result = new LinkedHashMap();
+ if(getZnodeVersion() > -1) result.put("znodeVersion",getZnodeVersion());
result.put("luceneMatchVersion",luceneMatchVersion);
result.put("updateHandler", getUpdateHandlerInfo().toMap());
Map m = new LinkedHashMap();
@@ -730,6 +732,9 @@ public class SolrConfig extends Config i
if(plugin.options.contains(PluginOpts.REQUIRE_NAME)){
LinkedHashMap items = new LinkedHashMap();
for (PluginInfo info : infos) items.put(info.name, info.toMap());
+ if(tag.equals(SolrRequestHandler.TYPE)){
+ for (Map.Entry e : overlay.getReqHandlers().entrySet()) items.put(e.getKey(),e.getValue());
+ }
result.put(tag,items);
} else {
if(plugin.options.contains(MULTI_OK)){
Modified: lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/core/SolrCore.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/core/SolrCore.java?rev=1642718&r1=1642717&r2=1642718&view=diff
==============================================================================
--- lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/core/SolrCore.java (original)
+++ lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/core/SolrCore.java Mon Dec 1 17:25:39 2014
@@ -17,6 +17,43 @@
package org.apache.solr.core;
+import java.io.Closeable;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.Writer;
+import java.lang.reflect.Constructor;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.NoSuchFileException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.IdentityHashMap;
+import java.util.LinkedHashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+import java.util.StringTokenizer;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.ReentrantLock;
+
import org.apache.commons.io.FileUtils;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.index.DirectoryReader;
@@ -104,43 +141,6 @@ import org.xml.sax.SAXException;
import javax.xml.parsers.ParserConfigurationException;
-import java.io.Closeable;
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.Writer;
-import java.lang.reflect.Constructor;
-import java.net.URL;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.NoSuchFileException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.IdentityHashMap;
-import java.util.LinkedHashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Set;
-import java.util.StringTokenizer;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.locks.ReentrantLock;
-
/**
*
*/
@@ -423,18 +423,21 @@ public final class SolrCore implements S
return responseWriters.put(name, responseWriter);
}
- public SolrCore reload(ConfigSet coreConfig, SolrCore prev) throws IOException,
+ public SolrCore reload(ConfigSet coreConfig) throws IOException,
ParserConfigurationException, SAXException {
solrCoreState.increfSolrCoreState();
+ SolrCore currentCore;
boolean indexDirChange = !getNewIndexDir().equals(getIndexDir());
if (indexDirChange || !coreConfig.getSolrConfig().nrtMode) {
// the directory is changing, don't pass on state
- prev = null;
+ currentCore = null;
+ } else {
+ currentCore = this;
}
SolrCore core = new SolrCore(getName(), getDataDir(), coreConfig.getSolrConfig(),
- coreConfig.getIndexSchema(), coreDescriptor, updateHandler, this.solrDelPolicy, prev);
+ coreConfig.getIndexSchema(), coreDescriptor, updateHandler, this.solrDelPolicy, currentCore);
core.solrDelPolicy = this.solrDelPolicy;
@@ -507,7 +510,7 @@ public final class SolrCore implements S
logid
+ "WARNING: Solr index directory '{}' is locked. Unlocking...",
indexDir);
- IndexWriter.unlock(dir);
+ dir.makeLock(IndexWriter.WRITE_LOCK_NAME).close();
} else {
log.error(logid
+ "Solr index directory '{}' is locked. Throwing exception",
@@ -807,11 +810,7 @@ public final class SolrCore implements S
// Processors initialized before the handlers
updateProcessorChains = loadUpdateProcessorChains();
reqHandlers = new RequestHandlers(this);
- List<PluginInfo> implicitReqHandlerInfo = new ArrayList<>();
- UpdateRequestHandler.addImplicits(implicitReqHandlerInfo);
- SolrConfigHandler.addImplicits(implicitReqHandlerInfo);
-
- reqHandlers.initHandlersFromConfig(solrConfig, implicitReqHandlerInfo);
+ reqHandlers.initHandlersFromConfig(solrConfig);
// Handle things that should eventually go away
initDeprecatedSupport();
Modified: lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java?rev=1642718&r1=1642717&r2=1642718&view=diff
==============================================================================
--- lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java (original)
+++ lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java Mon Dec 1 17:25:39 2014
@@ -695,10 +695,14 @@ public class SolrResourceLoader implemen
for (SolrInfoMBean bean : arr) {
- try {
- infoRegistry.put(bean.getName(), bean);
- } catch (Exception e) {
- log.warn("could not register MBean '" + bean.getName() + "'.", e);
+ // Too slow? I suspect not, but we may need
+ // to start tracking this in a Set.
+ if (!infoRegistry.containsValue(bean)) {
+ try {
+ infoRegistry.put(bean.getName(), bean);
+ } catch (Exception e) {
+ log.warn("could not register MBean '" + bean.getName() + "'.", e);
+ }
}
}
}
Modified: lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java?rev=1642718&r1=1642717&r2=1642718&view=diff
==============================================================================
--- lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java (original)
+++ lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java Mon Dec 1 17:25:39 2014
@@ -19,8 +19,8 @@ package org.apache.solr.handler;
import java.io.IOException;
-import java.net.URL;
import java.text.MessageFormat;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
@@ -28,13 +28,10 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
-import org.apache.lucene.analysis.util.ResourceLoader;
-import org.apache.lucene.analysis.util.ResourceLoaderAware;
import org.apache.solr.cloud.ZkController;
import org.apache.solr.cloud.ZkSolrResourceLoader;
import org.apache.solr.common.SolrException;
+import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.params.CollectionParams;
import org.apache.solr.common.params.CommonParams;
@@ -44,17 +41,18 @@ import org.apache.solr.common.params.Sol
import org.apache.solr.common.util.ContentStream;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.StrUtils;
-import org.apache.solr.core.CloseHook;
import org.apache.solr.core.ConfigOverlay;
+import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.PluginInfo;
import org.apache.solr.core.SolrConfig;
import org.apache.solr.core.SolrCore;
-import org.apache.solr.core.SolrInfoMBean;
import org.apache.solr.core.SolrResourceLoader;
import org.apache.solr.request.LocalSolrQueryRequest;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrRequestHandler;
import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.schema.FieldType;
+import org.apache.solr.schema.ManagedIndexSchema;
import org.apache.solr.schema.SchemaManager;
import org.apache.solr.util.CommandOperation;
import org.apache.solr.util.plugin.SolrCoreAware;
@@ -67,8 +65,10 @@ import static java.text.MessageFormat.fo
import static java.util.Collections.singletonList;
import static java.util.Collections.singletonMap;
import static org.apache.solr.common.cloud.ZkNodeProps.makeMap;
+import static org.apache.solr.common.params.CoreAdminParams.NAME;
import static org.apache.solr.core.ConfigOverlay.NOT_EDITABLE;
import static org.apache.solr.core.PluginInfo.DEFAULTS;
+import static org.apache.solr.schema.FieldType.CLASS_NAME;
public class SolrConfigHandler extends RequestHandlerBase implements SolrCoreAware{
public static final Logger log = LoggerFactory.getLogger(SolrConfigHandler.class);
@@ -94,30 +94,72 @@ public class SolrConfigHandler extends R
public void inform(final SolrCore core) {
if( ! (core.getResourceLoader() instanceof ZkSolrResourceLoader)) return;
final ZkSolrResourceLoader zkSolrResourceLoader = (ZkSolrResourceLoader) core.getResourceLoader();
- if(zkSolrResourceLoader != null){
- Runnable listener = new Runnable() {
- @Override
- public void run() {
- try {
- if(core.isClosed()) return;
- Stat stat = zkSolrResourceLoader.getZkController().getZkClient().exists((zkSolrResourceLoader).getConfigSetZkPath() + "/" + ConfigOverlay.RESOURCE_NAME, null, true);
- if(stat == null) return;
- if (stat.getVersion() > core.getSolrConfig().getOverlay().getZnodeVersion()) {
- core.getCoreDescriptor().getCoreContainer().reload(core.getName());
+ if(zkSolrResourceLoader != null)
+ zkSolrResourceLoader.getZkController().registerConfListenerForCore(
+ zkSolrResourceLoader.getConfigSetZkPath(),
+ core,
+ getListener(core, zkSolrResourceLoader));
+
+ }
+
+ private static Runnable getListener(SolrCore core, ZkSolrResourceLoader zkSolrResourceLoader) {
+ final String coreName = core.getName();
+ final CoreContainer cc = core.getCoreDescriptor().getCoreContainer();
+ final String overlayPath = zkSolrResourceLoader.getConfigSetZkPath() + "/" + ConfigOverlay.RESOURCE_NAME;
+ final String solrConfigPath = zkSolrResourceLoader.getConfigSetZkPath() + "/" + core.getSolrConfig().getName();
+ String schemaRes = null;
+ if(core.getLatestSchema().isMutable() && core.getLatestSchema() instanceof ManagedIndexSchema){
+ ManagedIndexSchema mis = (ManagedIndexSchema) core.getLatestSchema();
+ schemaRes = mis.getResourceName();
+ }
+ final String managedSchmaResourcePath = schemaRes ==null ? null: zkSolrResourceLoader.getConfigSetZkPath() + "/" + schemaRes;
+ return new Runnable() {
+ @Override
+ public void run() {
+ log.info("config update listener called for core {}", coreName);
+ SolrZkClient zkClient = cc.getZkController().getZkClient();
+ int solrConfigversion,overlayVersion, managedSchemaVersion=0;
+ try (SolrCore core = cc.getCore(coreName)) {
+ if (core.isClosed()) return;
+ solrConfigversion = core.getSolrConfig().getOverlay().getZnodeVersion();
+ overlayVersion = core.getSolrConfig().getZnodeVersion();
+ if(managedSchmaResourcePath != null){
+ managedSchemaVersion = ((ManagedIndexSchema)core.getLatestSchema()).getSchemaZkVersion();
+ }
+
+ }
+
+ if (checkStale(zkClient, overlayPath, solrConfigversion) ||
+ checkStale(zkClient, solrConfigPath, overlayVersion) ||
+ checkStale(zkClient, managedSchmaResourcePath,managedSchemaVersion)) {
+ log.info("core reload {}",coreName);
+ cc.reload(coreName);
}
- } catch (KeeperException.NoNodeException nne){
- //no problem
- } catch (KeeperException e) {
- log.error("error refreshing solrconfig ", e);
- } catch (InterruptedException e) {
- Thread.currentThread().isInterrupted();
}
- }
- };
+ };
+ }
- zkSolrResourceLoader.getZkController().registerConfListenerForCore(zkSolrResourceLoader.getConfigSetZkPath(), core,listener);
+ private static boolean checkStale(SolrZkClient zkClient, String zkPath, int currentVersion) {
+ if(zkPath == null) return false;
+ try {
+ Stat stat = zkClient.exists(zkPath, null, true);
+ if(stat == null){
+ if(currentVersion > -1) return true;
+ return false;
+ }
+ if (stat.getVersion() > currentVersion) {
+ log.info(zkPath+" is stale will need an update from {} to {}", currentVersion,stat.getVersion());
+ return true;
+ }
+ return false;
+ } catch (KeeperException.NoNodeException nne){
+ //no problem
+ } catch (KeeperException e) {
+ log.error("error refreshing solrconfig ", e);
+ } catch (InterruptedException e) {
+ Thread.currentThread().isInterrupted();
}
-
+ return false;
}
@@ -136,8 +178,7 @@ public class SolrConfigHandler extends R
String path = (String) req.getContext().get("path");
if(path == null) path="/config";
if("/config/overlay".equals(path)){
- resp.add("overlay", req.getCore().getSolrConfig().getOverlay().toOutputFormat());
- return;
+ resp.add("overlay", req.getCore().getSolrConfig().getOverlay().toMap());
} else {
List<String> parts =StrUtils.splitSmart(path, '/');
if(parts.get(0).isEmpty()) parts.remove(0);
@@ -152,13 +193,32 @@ public class SolrConfigHandler extends R
private void handlePOST() throws IOException {
- Iterable<ContentStream> streams = req.getContentStreams();
- if(streams == null ){
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "missing content stream");
- }
+ Iterable<ContentStream> streams = req.getContentStreams();
+ if (streams == null) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "missing content stream");
+ }
+ ArrayList<CommandOperation> ops = new ArrayList<>();
+
+ for (ContentStream stream : streams)
+ ops.addAll(CommandOperation.parse(stream.getReader()));
+ List<Map> errList = CommandOperation.captureErrors(ops);
+ if(!errList.isEmpty()) {
+ resp.add(CommandOperation.ERR_MSGS,errList);
+ return;
+ }
+
try {
- for (ContentStream stream : streams) {
- runCommandsTillSuccess(stream);
+ for (;;) {
+ ArrayList<CommandOperation> opsCopy = new ArrayList<>(ops.size());
+ ConfigOverlay overlay = SolrConfig.getConfigOverlay(req.getCore().getResourceLoader());
+ for (CommandOperation op : ops) opsCopy.add(op.getCopy());
+ try {
+ handleCommands(opsCopy, overlay);
+ break;//succeeded . so no need to go over the loop again
+ } catch (ZkController.ResourceModifiedInZkException e) {
+ //retry
+ log.info("Race condition, the node is modified in ZK by someone else " +e.getMessage());
+ }
}
} catch (Exception e) {
resp.setException(e);
@@ -167,30 +227,28 @@ public class SolrConfigHandler extends R
}
- private void runCommandsTillSuccess(ContentStream stream) throws IOException {
- for (;;) {
- try {
- handleCommands(stream);
- break;
- } catch (ZkController.ResourceModifiedInZkException e) {
- log.info(e.getMessage());
-
- }
- }
- }
-
- private void handleCommands( ContentStream stream) throws IOException {
- ConfigOverlay overlay = req.getCore().getSolrConfig().getOverlay();
- List<CommandOperation> ops = CommandOperation.parse(stream.getReader());
+ private void handleCommands(List<CommandOperation> ops, ConfigOverlay overlay ) throws IOException {
for (CommandOperation op : ops) {
- if(SET_PROPERTY.equals( op.name) ){
- overlay = applySetProp(op, overlay);
- }else if(UNSET_PROPERTY.equals(op.name)){
- overlay = applyUnset(op,overlay);
- }else if(SET_USER_PROPERTY.equals(op.name)){
- overlay = applySetUserProp(op ,overlay);
- }else if(UNSET_USER_PROPERTY.equals(op.name)){
- overlay = applyUnsetUserProp(op, overlay);
+ switch (op.name) {
+ case SET_PROPERTY:
+ overlay = applySetProp(op, overlay);
+ break;
+ case UNSET_PROPERTY:
+ overlay = applyUnset(op, overlay);
+ break;
+ case SET_USER_PROPERTY:
+ overlay = applySetUserProp(op, overlay);
+ break;
+ case UNSET_USER_PROPERTY:
+ overlay = applyUnsetUserProp(op, overlay);
+ break;
+ case UPDATE_REQHANDLER:
+ case CREATE_REQHANDLER:
+ overlay = applyRequestHandler(op, overlay);
+ break;
+ case DELETE_REQHANDLER:
+ overlay = applyDeleteHandler(op,overlay);
+ break;
}
}
List errs = CommandOperation.captureErrors(ops);
@@ -204,21 +262,6 @@ public class SolrConfigHandler extends R
ZkController.persistConfigResourceToZooKeeper(loader,overlay.getZnodeVersion(),
ConfigOverlay.RESOURCE_NAME,overlay.toByteArray(),true);
- String collectionName = req.getCore().getCoreDescriptor().getCloudDescriptor().getCollectionName();
- Map map = ZkNodeProps.makeMap(CoreAdminParams.ACTION, CollectionParams.CollectionAction.RELOAD.toString() ,
- CollectionParams.NAME, collectionName);
-
- SolrQueryRequest solrQueryRequest = new LocalSolrQueryRequest(req.getCore(), new MapSolrParams(map));
- SolrQueryResponse tmpResp = new SolrQueryResponse();
- try {
- //doing a collection reload
- req.getCore().getCoreDescriptor().getCoreContainer().getCollectionsHandler().handleRequestBody(solrQueryRequest,tmpResp);
- } catch (Exception e) {
- String msg = MessageFormat.format("Unable to reload collection {0}", collectionName);
- log.error(msg);
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, msg);
- }
-
} else {
SolrResourceLoader.persistConfLocally(loader, ConfigOverlay.RESOURCE_NAME, overlay.toByteArray());
req.getCore().getCoreDescriptor().getCoreContainer().reload(req.getCore().getName());
@@ -226,6 +269,50 @@ public class SolrConfigHandler extends R
}
+ private ConfigOverlay applyDeleteHandler(CommandOperation op, ConfigOverlay overlay) {
+ String name = op.getStr(CommandOperation.ROOT_OBJ);
+ if(op.hasError()) return overlay;
+ if(overlay.getReqHandlers().containsKey(name)){
+ return overlay.deleteHandler(name);
+ } else {
+ op.addError(MessageFormat.format("NO such requestHandler ''{0}'' ",name));
+ return overlay;
+ }
+
+ }
+
+ private ConfigOverlay applyRequestHandler(CommandOperation op, ConfigOverlay overlay) {
+ String name=op.getStr(NAME);
+ op.getStr(CLASS_NAME);
+ op.getMap(PluginInfo.DEFAULTS, null);
+ op.getMap(PluginInfo.INVARIANTS,null);
+ op.getMap(PluginInfo.APPENDS,null);
+ if(op.hasError()) return overlay;
+
+
+ if(CREATE_REQHANDLER.equals(op.name)) {
+ if (overlay.getReqHandlers().containsKey(name)) {
+ op.addError(MessageFormat.format(" ''{0}'' already exists . Do an ''{1}'' , if you want to change it ", name, UPDATE_REQHANDLER));
+ return overlay;
+ } else {
+ return overlay.addReqHandler(op.getDataMap());
+ }
+ } else if(UPDATE_REQHANDLER.equals(op.name)){
+ if (!overlay.getReqHandlers().containsKey(name)) {
+ op.addError(MessageFormat.format(" ''{0}'' does not exist . Do an ''{1}'' , if you want to create it ", name, CREATE_REQHANDLER));
+ return overlay;
+ } else {
+ return overlay.addReqHandler(op.getDataMap());
+
+ }
+ }
+
+ return overlay;
+
+
+
+ }
+
private ConfigOverlay applySetUserProp(CommandOperation op, ConfigOverlay overlay) {
Map<String, Object> m = op.getDataMap();
if(op.hasError()) return overlay;
@@ -292,14 +379,6 @@ public class SolrConfigHandler extends R
req.setParams(SolrParams.wrapDefaults(params, new MapSolrParams(map)));
}
-
- public static void addImplicits(List<PluginInfo> infoList){
- Map m = makeMap("name", "/config", "class", SolrConfigHandler.class.getName());
- infoList.add(new PluginInfo(SolrRequestHandler.TYPE, m, new NamedList<>(singletonMap(DEFAULTS, new NamedList())), null));
- }
-
-
-
@Override
public SolrRequestHandler getSubHandler(String path) {
if(subPaths.contains(path)) return this;
@@ -339,7 +418,8 @@ public class SolrConfigHandler extends R
public static final String UNSET_PROPERTY = "unset-property";
public static final String SET_USER_PROPERTY = "set-user-property";
public static final String UNSET_USER_PROPERTY = "unset-user-property";
-
-
+ public static final String CREATE_REQHANDLER = "create-requesthandler";
+ public static final String DELETE_REQHANDLER = "delete-requesthandler";
+ public static final String UPDATE_REQHANDLER = "update-requesthandler";
}
Modified: lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/handler/UpdateRequestHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/handler/UpdateRequestHandler.java?rev=1642718&r1=1642717&r2=1642718&view=diff
==============================================================================
--- lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/handler/UpdateRequestHandler.java (original)
+++ lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/handler/UpdateRequestHandler.java Mon Dec 1 17:25:39 2014
@@ -172,17 +172,6 @@ public class UpdateRequestHandler extend
return "Add documents using XML (with XSLT), CSV, JSON, or javabin";
}
- public static void addImplicits(List<PluginInfo> implicits) {
- implicits.add(getPluginInfo("/update",Collections.emptyMap()));
- implicits.add(getPluginInfo(JSON_PATH, singletonMap("update.contentType", "application/json")));
- implicits.add(getPluginInfo(CSV_PATH, singletonMap("update.contentType", "application/csv")));
- implicits.add(getPluginInfo(DOC_PATH, makeMap("update.contentType", "application/json", "json.command","false")));
- }
-
- static PluginInfo getPluginInfo(String name, Map defaults){
- Map m = makeMap("name", name, "class", UpdateRequestHandler.class.getName());
- return new PluginInfo("requestHandler", m, new NamedList<>( singletonMap("defaults", new NamedList(defaults))) ,null);
- }
public static final String DOC_PATH = "/update/json/docs";
public static final String JSON_PATH = "/update/json";
public static final String CSV_PATH = "/update/csv";
Modified: lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/handler/admin/AdminHandlers.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/handler/admin/AdminHandlers.java?rev=1642718&r1=1642717&r2=1642718&view=diff
==============================================================================
--- lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/handler/admin/AdminHandlers.java (original)
+++ lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/handler/admin/AdminHandlers.java Mon Dec 1 17:25:39 2014
@@ -27,14 +27,18 @@ import org.apache.solr.request.SolrQuery
import org.apache.solr.request.SolrRequestHandler;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.util.plugin.SolrCoreAware;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* A special Handler that registers all standard admin handlers
*
* @since solr 1.3
*/
+@Deprecated
public class AdminHandlers implements SolrCoreAware, SolrRequestHandler
{
+ public static Logger log = LoggerFactory.getLogger(AdminHandlers.class);
NamedList initArgs = null;
private static class StandardHandler {
@@ -100,6 +104,7 @@ public class AdminHandlers implements So
}
}
}
+ log.warn("<requestHandler name=\"/admin/\" \n class=\"solr.admin.AdminHandlers\" /> is deprecated . It is not required anymore");
}
Modified: lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java?rev=1642718&r1=1642717&r2=1642718&view=diff
==============================================================================
--- lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java (original)
+++ lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java Mon Dec 1 17:25:39 2014
@@ -83,6 +83,7 @@ import org.apache.solr.cloud.Distributed
import org.apache.solr.cloud.Overseer;
import org.apache.solr.cloud.OverseerCollectionProcessor;
import org.apache.solr.cloud.OverseerSolrResponse;
+import org.apache.solr.cloud.overseer.SliceMutator;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.cloud.ClusterState;
@@ -301,7 +302,7 @@ public class CollectionsHandler extends
for (Slice slice : dc.getSlices()) {
for (Replica replica : slice.getReplicas()) {
// Tell the replica to become the leader if we're the preferred leader AND active AND not the leader already
- if (replica.getBool(Overseer.preferredLeaderProp, false) == false) {
+ if (replica.getBool(SliceMutator.PREFERRED_LEADER_PROP, false) == false) {
continue;
}
if (StringUtils.equalsIgnoreCase(replica.getStr(STATE_PROP), ACTIVE) == false) {
@@ -442,7 +443,7 @@ public class CollectionsHandler extends
// Check if we're trying to set a property with parameters that allow us to set the property on multiple replicas
// in a slice on properties that are known to only be one-per-slice and error out if so.
if (StringUtils.isNotBlank((String)map.get(SHARD_UNIQUE)) &&
- Overseer.sliceUniqueBooleanProperties.contains(property.toLowerCase(Locale.ROOT)) &&
+ SliceMutator.SLICE_UNIQUE_BOOLEAN_PROPERTIES.contains(property.toLowerCase(Locale.ROOT)) &&
uniquePerSlice == false) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
"Overseer replica property command received for property " + property +
@@ -472,7 +473,7 @@ public class CollectionsHandler extends
}
if (shardUnique == false &&
- Overseer.sliceUniqueBooleanProperties.contains(prop) == false) {
+ SliceMutator.SLICE_UNIQUE_BOOLEAN_PROPERTIES.contains(prop) == false) {
throw new SolrException(ErrorCode.BAD_REQUEST, "Balancing properties amongst replicas in a slice requires that"
+ " the property be pre-defined as a unique property (e.g. 'preferredLeader') or that 'shardUnique' be set to 'true'. " +
" Property: " + prop + " shardUnique: " + Boolean.toString(shardUnique));
Modified: lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java?rev=1642718&r1=1642717&r2=1642718&view=diff
==============================================================================
--- lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java (original)
+++ lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java Mon Dec 1 17:25:39 2014
@@ -596,14 +596,8 @@ public class LukeRequestHandler extends
final CharsRefBuilder spare = new CharsRefBuilder();
- Fields fields = MultiFields.getFields(req.getSearcher().getIndexReader());
-
- if (fields == null) { // No indexed fields
- return;
- }
-
- Terms terms = fields.terms(field);
- if (terms == null) { // No terms in the field.
+ Terms terms = MultiFields.getTerms(req.getSearcher().getIndexReader(), field);
+ if (terms == null) { // field does not exist
return;
}
TermsEnum termsEnum = terms.iterator(null);
Modified: lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/handler/admin/SystemInfoHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/handler/admin/SystemInfoHandler.java?rev=1642718&r1=1642717&r2=1642718&view=diff
==============================================================================
--- lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/handler/admin/SystemInfoHandler.java (original)
+++ lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/handler/admin/SystemInfoHandler.java Mon Dec 1 17:25:39 2014
@@ -95,6 +95,8 @@ public class SystemInfoHandler extends R
if (solrCloudMode) {
rsp.add("zkHost", getCoreContainer(req, core).getZkController().getZkServerAddress());
}
+ if (cc != null)
+ rsp.add( "solr_home", cc.getSolrHome());
rsp.add( "lucene", getLuceneInfo() );
rsp.add( "jvm", getJvmInfo() );
rsp.add( "system", getSystemInfo() );
Modified: lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/handler/component/TermsComponent.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/handler/component/TermsComponent.java?rev=1642718&r1=1642717&r2=1642718&view=diff
==============================================================================
--- lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/handler/component/TermsComponent.java (original)
+++ lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/handler/component/TermsComponent.java Mon Dec 1 17:25:39 2014
@@ -124,9 +124,9 @@ public class TermsComponent extends Sear
NamedList<Integer> fieldTerms = new NamedList<>();
termsResult.add(field, fieldTerms);
- Terms terms = lfields == null ? null : lfields.terms(field);
+ Terms terms = lfields.terms(field);
if (terms == null) {
- // no terms for this field
+ // field does not exist
continue;
}
Modified: lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java?rev=1642718&r1=1642717&r2=1642718&view=diff
==============================================================================
--- lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java (original)
+++ lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java Mon Dec 1 17:25:39 2014
@@ -176,14 +176,13 @@ public class DefaultSolrHighlighter exte
* @param query The current Query
* @param fieldName The name of the field
* @param request The current SolrQueryRequest
- * @param tokenStream document text CachingTokenStream
+ * @param tokenStream document text tokenStream that implements reset() efficiently (e.g. CachingTokenFilter).
+ * If it's used, call reset() first.
* @throws IOException If there is a low-level I/O error.
*/
- protected Highlighter getPhraseHighlighter(Query query, String fieldName, SolrQueryRequest request, CachingTokenFilter tokenStream) throws IOException {
+ protected Highlighter getPhraseHighlighter(Query query, String fieldName, SolrQueryRequest request, TokenStream tokenStream) throws IOException {
SolrParams params = request.getParams();
- Highlighter highlighter = null;
-
- highlighter = new Highlighter(
+ Highlighter highlighter = new Highlighter(
getFormatter(fieldName, params),
getEncoder(fieldName, params),
getSpanQueryScorer(query, fieldName, tokenStream, request));
@@ -212,16 +211,14 @@ public class DefaultSolrHighlighter exte
/**
* Return a {@link org.apache.lucene.search.highlight.QueryScorer} suitable for this Query and field.
* @param query The current query
- * @param tokenStream document text CachingTokenStream
+ * @param tokenStream document text tokenStream that implements reset() efficiently (e.g. CachingTokenFilter).
+ * If it's used, call reset() first.
* @param fieldName The name of the field
* @param request The SolrQueryRequest
*/
private QueryScorer getSpanQueryScorer(Query query, String fieldName, TokenStream tokenStream, SolrQueryRequest request) {
boolean reqFieldMatch = request.getParams().getFieldBool(fieldName, HighlightParams.FIELD_MATCH, false);
- Boolean highlightMultiTerm = request.getParams().getBool(HighlightParams.HIGHLIGHT_MULTI_TERM, true);
- if(highlightMultiTerm == null) {
- highlightMultiTerm = false;
- }
+ boolean highlightMultiTerm = request.getParams().getBool(HighlightParams.HIGHLIGHT_MULTI_TERM, true);
QueryScorer scorer;
if (reqFieldMatch) {
scorer = new QueryScorer(query, fieldName);
@@ -435,6 +432,7 @@ public class DefaultSolrHighlighter exte
return termPosOff;
}
+ @SuppressWarnings("unchecked")
private void doHighlightingByHighlighter( Query query, SolrQueryRequest req, NamedList docSummaries,
int docId, StoredDocument doc, String fieldName ) throws IOException {
final SolrIndexSearcher searcher = req.getSearcher();
@@ -444,10 +442,7 @@ public class DefaultSolrHighlighter exte
// so we disable them until fixed (see LUCENE-3080)!
// BEGIN: Hack
final SchemaField schemaField = schema.getFieldOrNull(fieldName);
- if (schemaField != null && (
- (schemaField.getType() instanceof org.apache.solr.schema.TrieField) ||
- (schemaField.getType() instanceof org.apache.solr.schema.TrieDateField)
- )) return;
+ if (schemaField != null && schemaField.getType() instanceof org.apache.solr.schema.TrieField) return;
// END: Hack
SolrParams params = req.getParams();
@@ -456,19 +451,18 @@ public class DefaultSolrHighlighter exte
boolean preserveMulti = params.getFieldBool(fieldName, HighlightParams.PRESERVE_MULTI, false);
List<StorableField> allFields = doc.getFields();
- if (allFields != null && allFields.size() == 0) return; // No explicit contract that getFields returns != null,
+ if (allFields == null || allFields.isEmpty()) return; // No explicit contract that getFields returns != null,
// although currently it can't.
- TokenStream tstream = null;
int numFragments = getMaxSnippets(fieldName, params);
boolean mergeContiguousFragments = isMergeContiguousFragments(fieldName, params);
String[] summaries = null;
List<TextFragment> frags = new ArrayList<>();
- TermOffsetsTokenStream tots = null; // to be non-null iff we're using TermOffsets optimization
+ TermOffsetsTokenStream tots = null; // to be non-null iff we're using TermOffsets optimization (multi-valued)
TokenStream tvStream = TokenSources.getTokenStreamWithOffsets(searcher.getIndexReader(), docId, fieldName);
- if (tvStream != null) {
+ if (tvStream != null && schemaField.multiValued() && isActuallyMultiValued(allFields, fieldName)) {
tots = new TermOffsetsTokenStream(tvStream);
}
int mvToExamine = Integer.parseInt(req.getParams().get(HighlightParams.MAX_MULTIVALUED_TO_EXAMINE,
@@ -483,10 +477,13 @@ public class DefaultSolrHighlighter exte
--mvToExamine;
String thisText = thisField.stringValue();
- if( tots != null ) {
- // if we're using TermOffsets optimization, then get the next
+ TokenStream tstream;
+ if (tots != null) {
+ // if we're using TermOffsets optimization (multi-valued field with term vectors), then get the next
// field value's TokenStream (i.e. get field j's TokenStream) from tots:
- tstream = tots.getMultiValuedTokenStream( thisText.length() );
+ tstream = tots.getMultiValuedTokenStream(thisText.length());
+ } else if (tvStream != null) {
+ tstream = tvStream; // single-valued with term vectors
} else {
// fall back to analyzer
tstream = createAnalyzerTStream(schema, fieldName, thisText);
@@ -498,17 +495,30 @@ public class DefaultSolrHighlighter exte
Highlighter highlighter;
if (Boolean.valueOf(req.getParams().get(HighlightParams.USE_PHRASE_HIGHLIGHTER, "true"))) {
- if (maxCharsToAnalyze < 0) {
- tstream = new CachingTokenFilter(tstream);
+ // We're going to call getPhraseHighlighter and it might consume the tokenStream. If it does, the tokenStream
+ // needs to implement reset() efficiently.
+
+ //If the tokenStream is right from the term vectors, then CachingTokenFilter is unnecessary.
+ // It should be okay if OffsetLimit won't get applied in this case.
+ final TokenStream tempTokenStream;
+ if (tstream != tvStream) {
+ if (maxCharsToAnalyze < 0) {
+ tempTokenStream = new CachingTokenFilter(tstream);
+ } else {
+ tempTokenStream = new CachingTokenFilter(new OffsetLimitTokenFilter(tstream, maxCharsToAnalyze));
+ }
} else {
- tstream = new CachingTokenFilter(new OffsetLimitTokenFilter(tstream, maxCharsToAnalyze));
+ tempTokenStream = tstream;
}
-
+
// get highlighter
- highlighter = getPhraseHighlighter(query, fieldName, req, (CachingTokenFilter) tstream);
+ highlighter = getPhraseHighlighter(query, fieldName, req, tempTokenStream);
- // after highlighter initialization, reset tstream since construction of highlighter already used it
- tstream.reset();
+ // if the CachingTokenFilter was consumed then use it going forward.
+ if (tempTokenStream instanceof CachingTokenFilter && ((CachingTokenFilter)tempTokenStream).isCached()) {
+ tstream = tempTokenStream;
+ }
+ //tstream.reset(); not needed; getBestTextFragments will reset it.
}
else {
// use "the old way"
@@ -523,15 +533,15 @@ public class DefaultSolrHighlighter exte
try {
TextFragment[] bestTextFragments = highlighter.getBestTextFragments(tstream, thisText, mergeContiguousFragments, numFragments);
- for (int k = 0; k < bestTextFragments.length; k++) {
+ for (TextFragment bestTextFragment : bestTextFragments) {
if (preserveMulti) {
- if (bestTextFragments[k] != null) {
- frags.add(bestTextFragments[k]);
+ if (bestTextFragment != null) {
+ frags.add(bestTextFragment);
--mvToMatch;
}
} else {
- if ((bestTextFragments[k] != null) && (bestTextFragments[k].getScore() > 0)) {
- frags.add(bestTextFragments[k]);
+ if ((bestTextFragment != null) && (bestTextFragment.getScore() > 0)) {
+ frags.add(bestTextFragment);
--mvToMatch;
}
}
@@ -539,19 +549,20 @@ public class DefaultSolrHighlighter exte
} catch (InvalidTokenOffsetsException e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
}
- }
+ }//end field value loop
+
// sort such that the fragments with the highest score come first
- if(!preserveMulti){
- Collections.sort(frags, new Comparator<TextFragment>() {
- @Override
- public int compare(TextFragment arg0, TextFragment arg1) {
- return Math.round(arg1.getScore() - arg0.getScore());
+ if (!preserveMulti) {
+ Collections.sort(frags, new Comparator<TextFragment>() {
+ @Override
+ public int compare(TextFragment arg0, TextFragment arg1) {
+ return Math.round(arg1.getScore() - arg0.getScore());
}
- });
- }
+ });
+ }
- // convert fragments back into text
- // TODO: we can include score and position information in output as snippet attributes
+ // convert fragments back into text
+ // TODO: we can include score and position information in output as snippet attributes
if (frags.size() > 0) {
ArrayList<String> fragTexts = new ArrayList<>();
for (TextFragment fragment: frags) {
@@ -577,6 +588,22 @@ public class DefaultSolrHighlighter exte
}
}
+ /** Is this field *actually* multi-valued for this document's fields? */
+ private boolean isActuallyMultiValued(List<StorableField> allFields, String fieldName) {
+ boolean foundFirst = false;
+ for (StorableField field : allFields) {
+ if (field.name().equals(fieldName)) {
+ if (foundFirst) {
+ return true;//we found another
+ } else {
+ foundFirst = true;
+ }
+ }
+ }
+ return false;//0 or 1 value
+ }
+
+ @SuppressWarnings("unchecked")
private void doHighlightingByFastVectorHighlighter( FastVectorHighlighter highlighter, FieldQuery fieldQuery,
SolrQueryRequest req, NamedList docSummaries, int docId, StoredDocument doc,
String fieldName ) throws IOException {
@@ -596,6 +623,7 @@ public class DefaultSolrHighlighter exte
alternateField( docSummaries, params, doc, fieldName );
}
+ @SuppressWarnings("unchecked")
private void alternateField( NamedList docSummaries, SolrParams params, StoredDocument doc, String fieldName ){
String alternateField = params.getFieldParam(fieldName, HighlightParams.ALTERNATE_FIELD);
if (alternateField != null && alternateField.length() > 0) {
@@ -612,7 +640,7 @@ public class DefaultSolrHighlighter exte
String[] altTexts = listFields.toArray(new String[listFields.size()]);
- if (altTexts != null && altTexts.length > 0){
+ if (altTexts.length > 0){
Encoder encoder = getEncoder(fieldName, params);
int alternateFieldLen = params.getFieldInt(fieldName, HighlightParams.ALTERNATE_FIELD_LENGTH,0);
List<String> altList = new ArrayList<>();
@@ -622,6 +650,7 @@ public class DefaultSolrHighlighter exte
altList.add(encoder.encodeText(altText));
}
else{
+ //note: seemingly redundant new String(...) releases memory to the larger text
altList.add( len + altText.length() > alternateFieldLen ?
encoder.encodeText(new String(altText.substring( 0, alternateFieldLen - len ))) :
encoder.encodeText(altText) );
@@ -635,12 +664,7 @@ public class DefaultSolrHighlighter exte
}
private TokenStream createAnalyzerTStream(IndexSchema schema, String fieldName, String docText) throws IOException {
-
- TokenStream tstream;
- TokenStream ts = schema.getIndexAnalyzer().tokenStream(fieldName, docText);
- ts.reset();
- tstream = new TokenOrderingFilter(ts, 10);
- return tstream;
+ return new TokenOrderingFilter(schema.getIndexAnalyzer().tokenStream(fieldName, docText), 10);
}
}
@@ -651,7 +675,7 @@ public class DefaultSolrHighlighter exte
*/
final class TokenOrderingFilter extends TokenFilter {
private final int windowSize;
- private final LinkedList<OrderedToken> queue = new LinkedList<>();
+ private final LinkedList<OrderedToken> queue = new LinkedList<>(); //TODO replace with Deque, Array impl
private boolean done=false;
private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
@@ -694,10 +718,6 @@ final class TokenOrderingFilter extends
}
}
- @Override
- public void reset() throws IOException {
- // this looks wrong: but its correct.
- }
}
// for TokenOrderingFilter, so it can easily sort by startOffset
@@ -708,19 +728,18 @@ class OrderedToken {
class TermOffsetsTokenStream {
- TokenStream bufferedTokenStream = null;
- OffsetAttribute bufferedOffsetAtt;
+ final TokenStream bufferedTokenStream;
+ final OffsetAttribute bufferedOffsetAtt;
State bufferedToken;
int bufferedStartOffset;
int bufferedEndOffset;
- int startOffset;
+ int startOffset = 0;
int endOffset;
+ boolean bufferedTokenStreamWasReset = false;
public TermOffsetsTokenStream( TokenStream tstream ){
bufferedTokenStream = tstream;
bufferedOffsetAtt = bufferedTokenStream.addAttribute(OffsetAttribute.class);
- startOffset = 0;
- bufferedToken = null;
}
public TokenStream getMultiValuedTokenStream( final int length ){
@@ -730,38 +749,53 @@ class TermOffsetsTokenStream {
final class MultiValuedStream extends TokenStream {
private final int length;
+ private boolean incrementTokenWasCalled = false;
OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
- MultiValuedStream(int length) {
- super(bufferedTokenStream.cloneAttributes());
- this.length = length;
- }
-
- @Override
- public boolean incrementToken() throws IOException {
- while( true ){
- if( bufferedToken == null ) {
- if (!bufferedTokenStream.incrementToken())
- return false;
- bufferedToken = bufferedTokenStream.captureState();
- bufferedStartOffset = bufferedOffsetAtt.startOffset();
- bufferedEndOffset = bufferedOffsetAtt.endOffset();
- }
-
- if( startOffset <= bufferedStartOffset &&
- bufferedEndOffset <= endOffset ){
- restoreState(bufferedToken);
- bufferedToken = null;
- offsetAtt.setOffset( offsetAtt.startOffset() - startOffset, offsetAtt.endOffset() - startOffset );
- return true;
- }
- else if( bufferedEndOffset > endOffset ){
- startOffset += length + 1;
+ MultiValuedStream(int length) {
+ super(bufferedTokenStream.cloneAttributes());//clone so we don't manipulate the buffered offsets
+ this.length = length;
+ }
+
+ @Override
+ public void reset() throws IOException {
+ //this flag allows reset() to be called multiple times up-front without a problem
+ if (incrementTokenWasCalled) {
+ throw new IllegalStateException("This TokenStream does not support being subsequently reset()");
+ }
+ if (!bufferedTokenStreamWasReset) {
+ bufferedTokenStream.reset();
+ bufferedTokenStreamWasReset = true;
+ }
+ super.reset();
+ }
+
+ @Override
+ public boolean incrementToken() throws IOException {
+ incrementTokenWasCalled = true;
+ while( true ){
+ if( bufferedToken == null ) {
+ if (!bufferedTokenStream.incrementToken())
return false;
- }
+ bufferedToken = bufferedTokenStream.captureState();
+ bufferedStartOffset = bufferedOffsetAtt.startOffset();
+ bufferedEndOffset = bufferedOffsetAtt.endOffset();
+ }
+
+ if( startOffset <= bufferedStartOffset &&
+ bufferedEndOffset <= endOffset ){
+ restoreState(bufferedToken);
bufferedToken = null;
+ offsetAtt.setOffset( offsetAtt.startOffset() - startOffset, offsetAtt.endOffset() - startOffset );
+ return true;
}
+ else if( bufferedEndOffset > endOffset ){
+ startOffset += length + 1;
+ return false;
+ }
+ bufferedToken = null;
}
+ }
- };
-};
+ }
+}
Modified: lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/rest/BaseSolrResource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/rest/BaseSolrResource.java?rev=1642718&r1=1642717&r2=1642718&view=diff
==============================================================================
--- lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/rest/BaseSolrResource.java (original)
+++ lucene/dev/branches/lucene2878/solr/core/src/java/org/apache/solr/rest/BaseSolrResource.java Mon Dec 1 17:25:39 2014
@@ -116,8 +116,7 @@ public abstract class BaseSolrResource e
responseWriter = solrCore.getQueryResponseWriter(responseWriterName);
contentType = responseWriter.getContentType(solrRequest, solrResponse);
final String path = getRequest().getRootRef().getPath();
- if ( ! RestManager.SCHEMA_BASE_PATH.equals(path)
- && ! RestManager.CONFIG_BASE_PATH.equals(path)) {
+ if ( ! RestManager.SCHEMA_BASE_PATH.equals(path)) {
// don't set webapp property on the request when context and core/collection are excluded
final int cutoffPoint = path.indexOf("/", 1);
final String firstPathElement = -1 == cutoffPoint ? path : path.substring(0, cutoffPoint);