You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2014/11/08 12:32:23 UTC
svn commit: r1637544 [4/6] - in /lucene/dev/branches/lucene6005: ./ lucene/
lucene/core/ lucene/core/src/java/org/apache/lucene/codecs/perfield/
lucene/core/src/java/org/apache/lucene/document/
lucene/core/src/java/org/apache/lucene/index/ lucene/core/...
Modified: lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java?rev=1637544&r1=1637543&r2=1637544&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java (original)
+++ lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java Sat Nov 8 11:32:18 2014
@@ -28,7 +28,7 @@ import static org.apache.solr.common.clo
import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICA;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICAPROP;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDROLE;
-import static org.apache.solr.common.params.CollectionParams.CollectionAction.BALANCESLICEUNIQUE;
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.BALANCESHARDUNIQUE;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.CLUSTERSTATUS;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.CREATE;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.CREATESHARD;
@@ -155,7 +155,7 @@ public class OverseerCollectionProcessor
public static final String ONLY_IF_DOWN = "onlyIfDown";
- public static final String SLICE_UNIQUE = "sliceUnique";
+ public static final String SHARD_UNIQUE = "shardUnique";
public static final String ONLY_ACTIVE_NODES = "onlyactivenodes";
@@ -650,7 +650,7 @@ public class OverseerCollectionProcessor
case DELETEREPLICAPROP:
processReplicaDeletePropertyCommand(message);
break;
- case BALANCESLICEUNIQUE:
+ case BALANCESHARDUNIQUE:
balanceProperty(message);
break;
case REBALANCELEADERS:
@@ -749,12 +749,12 @@ public class OverseerCollectionProcessor
if (StringUtils.isBlank(message.getStr(COLLECTION_PROP)) || StringUtils.isBlank(message.getStr(PROPERTY_PROP))) {
throw new SolrException(ErrorCode.BAD_REQUEST,
"The '" + COLLECTION_PROP + "' and '" + PROPERTY_PROP +
- "' parameters are required for the BALANCESLICEUNIQUE operation, no action taken");
+ "' parameters are required for the BALANCESHARDUNIQUE operation, no action taken");
}
SolrZkClient zkClient = zkStateReader.getZkClient();
DistributedQueue inQueue = Overseer.getInQueue(zkClient);
Map<String, Object> propMap = new HashMap<>();
- propMap.put(Overseer.QUEUE_OPERATION, BALANCESLICEUNIQUE.toLower());
+ propMap.put(Overseer.QUEUE_OPERATION, BALANCESHARDUNIQUE.toLower());
propMap.putAll(message.getProperties());
inQueue.offer(ZkStateReader.toJSON(new ZkNodeProps(propMap)));
}
Modified: lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/cloud/ZkController.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/cloud/ZkController.java?rev=1637544&r1=1637543&r2=1637544&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/cloud/ZkController.java (original)
+++ lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/cloud/ZkController.java Sat Nov 8 11:32:18 2014
@@ -20,18 +20,17 @@ package org.apache.solr.cloud;
import java.io.File;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
-import java.net.ConnectException;
import java.net.InetAddress;
import java.net.NetworkInterface;
-import java.net.SocketException;
-import java.net.SocketTimeoutException;
import java.net.URLEncoder;
import java.net.UnknownHostException;
+import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Enumeration;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
@@ -42,11 +41,10 @@ import java.util.Set;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicInteger;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringUtils;
-import org.apache.http.NoHttpResponseException;
-import org.apache.http.conn.ConnectTimeoutException;
import org.apache.solr.client.solrj.impl.HttpSolrServer;
import org.apache.solr.client.solrj.request.CoreAdminRequest.WaitForState;
import org.apache.solr.common.SolrException;
@@ -72,9 +70,11 @@ import org.apache.solr.common.cloud.ZooK
import org.apache.solr.common.params.CollectionParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.URLUtil;
+import org.apache.solr.core.CloseHook;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.CoreDescriptor;
import org.apache.solr.core.SolrCore;
+import org.apache.solr.core.SolrResourceLoader;
import org.apache.solr.handler.component.ShardHandler;
import org.apache.solr.update.UpdateLog;
import org.apache.solr.update.UpdateShardHandler;
@@ -83,9 +83,9 @@ import org.apache.zookeeper.KeeperExcept
import org.apache.zookeeper.KeeperException.ConnectionLossException;
import org.apache.zookeeper.KeeperException.NoNodeException;
import org.apache.zookeeper.KeeperException.SessionExpiredException;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
import org.apache.zookeeper.data.Stat;
-import org.noggit.JSONParser;
-import org.noggit.ObjectBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -200,11 +200,10 @@ public final class ZkController {
// keeps track of a list of objects that need to know a new ZooKeeper session was created after expiration occurred
private List<OnReconnect> reconnectListeners = new ArrayList<OnReconnect>();
-
+
public ZkController(final CoreContainer cc, String zkServerAddress, int zkClientTimeout, int zkClientConnectTimeout, String localHost, String locaHostPort,
- String localHostContext, int leaderVoteWait, int leaderConflictResolveWait, boolean genericCoreNodeNames, final CurrentCoreDescriptorProvider registerOnReconnect)
- throws InterruptedException, TimeoutException, IOException
- {
+ String localHostContext, int leaderVoteWait, int leaderConflictResolveWait, boolean genericCoreNodeNames, final CurrentCoreDescriptorProvider registerOnReconnect)
+ throws InterruptedException, TimeoutException, IOException {
if (cc == null) throw new IllegalArgumentException("CoreContainer cannot be null.");
this.cc = cc;
@@ -214,40 +213,41 @@ public final class ZkController {
// solr.xml to indicate the root context, instead of hostContext=""
// which means the default of "solr"
localHostContext = trimLeadingAndTrailingSlashes(localHostContext);
-
+
this.zkServerAddress = zkServerAddress;
this.localHostPort = locaHostPort;
this.localHostContext = localHostContext;
this.hostName = normalizeHostName(localHost);
- this.nodeName = generateNodeName(this.hostName,
- this.localHostPort,
- this.localHostContext);
+ this.nodeName = generateNodeName(this.hostName,
+ this.localHostPort,
+ this.localHostContext);
this.leaderVoteWait = leaderVoteWait;
this.leaderConflictResolveWait = leaderConflictResolveWait;
-
+
this.clientTimeout = zkClientTimeout;
DefaultConnectionStrategy strat = new DefaultConnectionStrategy();
String zkACLProviderClass = cc.getConfig().getZkACLProviderClass();
ZkACLProvider zkACLProvider = null;
if (zkACLProviderClass != null && zkACLProviderClass.trim().length() > 0) {
- zkACLProvider = cc.getResourceLoader().newInstance(zkACLProviderClass, ZkACLProvider.class);
+ zkACLProvider = cc.getResourceLoader().newInstance(zkACLProviderClass, ZkACLProvider.class);
} else {
zkACLProvider = new DefaultZkACLProvider();
}
-
- String zkCredentialProviderClass = cc.getConfig().getZkCredentialProviderClass();
- if (zkCredentialProviderClass != null && zkCredentialProviderClass.trim().length() > 0) {
- strat.setZkCredentialsToAddAutomatically(cc.getResourceLoader().newInstance(zkCredentialProviderClass, ZkCredentialsProvider.class));
+
+ String zkCredentialsProviderClass = cc.getConfig().getZkCredentialsProviderClass();
+ if (zkCredentialsProviderClass != null && zkCredentialsProviderClass.trim().length() > 0) {
+ strat.setZkCredentialsToAddAutomatically(cc.getResourceLoader().newInstance(zkCredentialsProviderClass, ZkCredentialsProvider.class));
} else {
strat.setZkCredentialsToAddAutomatically(new DefaultZkCredentialsProvider());
}
-
+ addOnReconnectListener(getConfigDirListener());
+
zkClient = new SolrZkClient(zkServerAddress, zkClientTimeout,
zkClientConnectTimeout, strat,
// on reconnect, reload cloud info
new OnReconnect() {
-
+
@Override
public void command() {
log.info("ZooKeeper session re-connected ... refreshing core states after session expiration.");
@@ -260,32 +260,32 @@ public final class ZkController {
// he is involved in the sync, and he certainly may not be
// ExecutorUtil.shutdownAndAwaitTermination(cc.getCmdDistribExecutor());
// we need to create all of our lost watches
-
+
// seems we dont need to do this again...
// Overseer.createClientNodes(zkClient, getNodeName());
-
+
cc.cancelCoreRecoveries();
-
+
registerAllCoresAsDown(registerOnReconnect, false);
-
+
if (!zkRunOnly) {
ElectionContext context = new OverseerElectionContext(zkClient,
overseer, getNodeName());
-
+
ElectionContext prevContext = overseerElector.getContext();
if (prevContext != null) {
prevContext.cancelElection();
}
-
+
overseerElector.setup(context);
overseerElector.joinElection(context, true);
}
-
+
zkStateReader.createClusterStateWatchersAndUpdate();
-
+
// we have to register as live first to pick up docs in the buffer
createEphemeralLiveNode();
-
+
List<CoreDescriptor> descriptors = registerOnReconnect
.getCurrentDescriptors();
// re register all descriptors
@@ -314,7 +314,7 @@ public final class ZkController {
listener.command();
} catch (Exception exc) {
// not much we can do here other than warn in the log
- log.warn("Error when notifying OnReconnect listener "+listener+" after session re-connected.", exc);
+ log.warn("Error when notifying OnReconnect listener " + listener + " after session re-connected.", exc);
}
}
}
@@ -330,20 +330,20 @@ public final class ZkController {
SolrException.ErrorCode.SERVER_ERROR, "", e);
}
}
-
+
}, new BeforeReconnect() {
-
- @Override
- public void command() {
- try {
- ZkController.this.overseer.close();
- } catch (Exception e) {
- log.error("Error trying to stop any Overseer threads", e);
- }
- markAllAsNotLeader(registerOnReconnect);
- }
- }, zkACLProvider);
-
+
+ @Override
+ public void command() {
+ try {
+ ZkController.this.overseer.close();
+ } catch (Exception e) {
+ log.error("Error trying to stop any Overseer threads", e);
+ }
+ markAllAsNotLeader(registerOnReconnect);
+ }
+ }, zkACLProvider);
+
this.overseerJobQueue = Overseer.getInQueue(zkClient);
this.overseerCollectionQueue = Overseer.getCollectionQueue(zkClient);
this.overseerRunningMap = Overseer.getRunningMap(zkClient);
@@ -352,9 +352,9 @@ public final class ZkController {
cmdExecutor = new ZkCmdExecutor(zkClientTimeout);
leaderElector = new LeaderElector(zkClient);
zkStateReader = new ZkStateReader(zkClient);
-
+
this.baseURL = zkStateReader.getBaseUrlForNodeName(this.nodeName);
-
+
init(registerOnReconnect);
}
@@ -1179,7 +1179,7 @@ public final class ZkController {
return true;
}
- public void unregister(String coreName, CoreDescriptor cd)
+ public void unregister(String coreName, CoreDescriptor cd, String configLocation)
throws InterruptedException, KeeperException {
final String coreNodeName = cd.getCloudDescriptor().getCoreNodeName();
final String collection = cd.getCloudDescriptor().getCollectionName();
@@ -1200,6 +1200,10 @@ public final class ZkController {
boolean removeWatch = true;
// if there is no SolrCore which is a member of this collection, remove the watch
for (SolrCore solrCore : cc.getCores()) {
+ if (((ZkSolrResourceLoader)solrCore.getResourceLoader()).getConfigSetZkPath().equals(configLocation))
+ configLocation = null; //if a core uses this config dir , then set it to null
+
+
CloudDescriptor cloudDesc = solrCore.getCoreDescriptor()
.getCloudDescriptor();
if (cloudDesc != null
@@ -1216,6 +1220,13 @@ public final class ZkController {
ZkStateReader.COLLECTION_PROP, cloudDescriptor.getCollectionName(),
ZkStateReader.CORE_NODE_NAME_PROP, coreNodeName);
overseerJobQueue.offer(ZkStateReader.toJSON(m));
+
+ if(configLocation != null) {
+ synchronized (confDirectoryWatchers) {
+ log.info("This conf directory is no more watched {0}",configLocation);
+ confDirectoryWatchers.remove(configLocation);
+ }
+ }
}
public void createCollection(String collection) throws KeeperException,
@@ -2098,4 +2109,159 @@ public final class ZkController {
}
}
}
+
+ /**
+ * Persists a config file to ZooKeeper using optimistic concurrency.
+ *
+ * @return true on success
+ */
+ public static boolean persistConfigResourceToZooKeeper( SolrResourceLoader loader, int znodeVersion , String resourceName, byte[] content, boolean createIfNotExists) {
+ final ZkSolrResourceLoader zkLoader = (ZkSolrResourceLoader)loader;
+ final ZkController zkController = zkLoader.getZkController();
+ final SolrZkClient zkClient = zkController.getZkClient();
+ final String resourceLocation = zkLoader.getConfigSetZkPath() + "/" + resourceName;
+ String errMsg = "Failed to persist resource at {0} - version mismatch";
+ try {
+ try {
+ zkClient.setData(resourceLocation , content,znodeVersion, true);
+ } catch (NoNodeException e) {
+ if(createIfNotExists){
+ try {
+ zkClient.create(resourceLocation,content, CreateMode.PERSISTENT,true);
+ } catch (KeeperException.NodeExistsException nee) {
+ log.info(MessageFormat.format(errMsg,resourceLocation));
+ throw new ResourceModifiedInZkException(ErrorCode.CONFLICT, MessageFormat.format(errMsg,resourceLocation) + ", retry.");
+ }
+ }
+ }
+
+ } catch (KeeperException.BadVersionException bve){
+ log.info(MessageFormat.format(errMsg,resourceLocation));
+ throw new ResourceModifiedInZkException(ErrorCode.CONFLICT, MessageFormat.format(errMsg,resourceLocation) + ", retry.");
+ } catch (Exception e) {
+ if (e instanceof InterruptedException) {
+ Thread.currentThread().interrupt(); // Restore the interrupted status
+ }
+ final String msg = "Error persisting resource at " + resourceLocation;
+ log.error(msg, e);
+ throw new SolrException(ErrorCode.SERVER_ERROR, msg, e);
+ }
+ return true;
+ }
+
+ public static class ResourceModifiedInZkException extends SolrException {
+ public ResourceModifiedInZkException(ErrorCode code, String msg) {
+ super(code, msg);
+ }
+ }
+
+ public void unRegisterConfListener(Runnable listener) {
+ if(listener == null) return;
+ synchronized (confDirectoryWatchers){
+ for (Set<Runnable> runnables : confDirectoryWatchers.values()) {
+ if(runnables != null) runnables.remove(listener);
+ }
+ }
+
+ }
+
+ /**This will give a callback to the listener whenever a child is modified in the
+ * conf directory. It is the responsibility of the listener to check if the individual
+ * item of interest has been modified. When the last core which was interested in
+ * this conf directory is gone the listeners will be removed automatically.
+ */
+ public void registerConfListenerForCore(String confDir,SolrCore core, final Runnable listener){
+ if(listener==null) throw new NullPointerException("listener cannot be null");
+ synchronized (confDirectoryWatchers){
+ if(confDirectoryWatchers.containsKey(confDir)){
+ confDirectoryWatchers.get(confDir).add(listener);
+ core.addCloseHook(new CloseHook() {
+ @Override
+ public void preClose(SolrCore core) {
+ unRegisterConfListener(listener);
+ }
+
+ @Override
+ public void postClose(SolrCore core) { }
+ });
+
+
+ } else {
+ throw new SolrException(ErrorCode.SERVER_ERROR,"This conf directory is not valid");
+ }
+ }
+ }
+
+ private Map<String , Set<Runnable>> confDirectoryWatchers = new HashMap<>();
+ void watchZKConfDir(final String zkDir) {
+
+ if(!confDirectoryWatchers.containsKey(zkDir)){
+ confDirectoryWatchers.put(zkDir,new HashSet<Runnable>());
+ }else{
+ //it's already watched
+ return;
+ }
+
+ Watcher watcher = new Watcher() {
+ @Override
+ public void process(WatchedEvent event) {
+ try {
+ synchronized (confDirectoryWatchers) {
+ // if this is not among directories to be watched then don't set the watcher anymore
+ if(!confDirectoryWatchers.containsKey(zkDir)) return;
+ }
+
+ if (event.getType() == Event.EventType.NodeChildrenChanged) {
+ synchronized (confDirectoryWatchers) {
+ final Set<Runnable> listeners = confDirectoryWatchers.get(zkDir);
+ if (listeners != null) {
+ new Thread() {
+ @Override
+ public synchronized void run() {
+ //running in a separate thread so that the zk event thread is not
+ // unnecessarily held up
+ for (Runnable listener : listeners) listener.run();
+ }
+ }.start();
+ }
+ }
+
+ }
+ } finally {
+ if (Event.EventType.None.equals(event.getType())) {
+ return;
+ } else {
+ setConfWatcher(zkDir,this);
+ }
+ }
+ }
+ };
+
+ setConfWatcher(zkDir,watcher);
+ }
+
+ private void setConfWatcher(String zkDir, Watcher watcher) {
+ try {
+ zkClient.getChildren(zkDir,watcher,true);
+ } catch (KeeperException e) {
+ log.error("failed to set watcher for conf dir {} ", zkDir);
+ } catch (InterruptedException e) {
+ Thread.interrupted();
+ log.error("failed to set watcher for conf dir {} ", zkDir);
+ }
+ }
+
+ public OnReconnect getConfigDirListener() {
+ return new OnReconnect() {
+ @Override
+ public void command() {
+ synchronized (confDirectoryWatchers){
+ for (String s : confDirectoryWatchers.keySet()) {
+ watchZKConfDir(s);
+ }
+
+ }
+ }
+ };
+ }
}
Modified: lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/cloud/ZkSolrResourceLoader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/cloud/ZkSolrResourceLoader.java?rev=1637544&r1=1637543&r2=1637544&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/cloud/ZkSolrResourceLoader.java (original)
+++ lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/cloud/ZkSolrResourceLoader.java Sat Nov 8 11:32:18 2014
@@ -30,6 +30,7 @@ import org.apache.solr.common.cloud.ZooK
import org.apache.solr.core.SolrResourceLoader;
import org.apache.solr.schema.ZkIndexSchemaReader;
import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.Stat;
/**
* ResourceLoader that works with ZooKeeper.
@@ -37,15 +38,16 @@ import org.apache.zookeeper.KeeperExcept
*/
public class ZkSolrResourceLoader extends SolrResourceLoader {
- private final String collectionZkPath;
+ private final String configSetZkPath;
private ZkController zkController;
private ZkIndexSchemaReader zkIndexSchemaReader;
- public ZkSolrResourceLoader(String instanceDir, String collection,
+ public ZkSolrResourceLoader(String instanceDir, String configSet,
ZkController zooKeeperController) {
super(instanceDir);
this.zkController = zooKeeperController;
- collectionZkPath = ZkController.CONFIGS_ZKNODE + "/" + collection;
+ configSetZkPath = ZkController.CONFIGS_ZKNODE + "/" + configSet;
+ zkController.watchZKConfDir(configSetZkPath);
}
/**
@@ -56,11 +58,12 @@ public class ZkSolrResourceLoader extend
* the "lib/" directory in the specified instance directory.
* <p>
*/
- public ZkSolrResourceLoader(String instanceDir, String collection, ClassLoader parent,
+ public ZkSolrResourceLoader(String instanceDir, String configSet, ClassLoader parent,
Properties coreProperties, ZkController zooKeeperController) {
super(instanceDir, parent, coreProperties);
this.zkController = zooKeeperController;
- collectionZkPath = ZkController.CONFIGS_ZKNODE + "/" + collection;
+ configSetZkPath = ZkController.CONFIGS_ZKNODE + "/" + configSet;
+ zkController.watchZKConfDir(configSetZkPath);
}
/**
@@ -75,11 +78,12 @@ public class ZkSolrResourceLoader extend
@Override
public InputStream openResource(String resource) throws IOException {
InputStream is = null;
- String file = collectionZkPath + "/" + resource;
+ String file = configSetZkPath + "/" + resource;
try {
if (zkController.pathExists(file)) {
- byte[] bytes = zkController.getZkClient().getData(file, null, null, true);
- return new ByteArrayInputStream(bytes);
+ Stat stat = new Stat();
+ byte[] bytes = zkController.getZkClient().getData(file, null, stat, true);
+ return new ZkByteArrayInputStream(bytes, stat);
}
} catch (Exception e) {
throw new IOException("Error opening " + file, e);
@@ -92,12 +96,30 @@ public class ZkSolrResourceLoader extend
}
if (is == null) {
throw new IOException("Can't find resource '" + resource
- + "' in classpath or '" + collectionZkPath + "', cwd="
+ + "' in classpath or '" + configSetZkPath + "', cwd="
+ System.getProperty("user.dir"));
}
return is;
}
+ public static class ZkByteArrayInputStream extends ByteArrayInputStream{
+
+ private final Stat stat;
+ public ZkByteArrayInputStream(byte[] buf, Stat stat) {
+ super(buf);
+ this.stat = stat;
+
+ }
+
+ public ZkByteArrayInputStream(byte[] buf, int offset, int length, Stat stat) {
+ super(buf, offset, length);
+ this.stat = stat;
+ }
+ public Stat getStat(){
+ return stat;
+ }
+ }
+
@Override
public String getConfigDir() {
throw new ZooKeeperException(
@@ -109,7 +131,7 @@ public class ZkSolrResourceLoader extend
public String[] listConfigDir() {
List<String> list;
try {
- list = zkController.getZkClient().getChildren(collectionZkPath, null, true);
+ list = zkController.getZkClient().getChildren(configSetZkPath, null, true);
} catch (InterruptedException e) {
// Restore the interrupted status
Thread.currentThread().interrupt();
@@ -124,8 +146,8 @@ public class ZkSolrResourceLoader extend
return list.toArray(new String[0]);
}
- public String getCollectionZkPath() {
- return collectionZkPath;
+ public String getConfigSetZkPath() {
+ return configSetZkPath;
}
public ZkController getZkController() {
Modified: lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/Config.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/Config.java?rev=1637544&r1=1637543&r2=1637544&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/Config.java (original)
+++ lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/Config.java Sat Nov 8 11:32:18 2014
@@ -52,6 +52,7 @@ import java.text.ParseException;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Map;
+import java.util.Properties;
import java.util.Set;
import java.util.SortedMap;
import java.util.SortedSet;
@@ -102,7 +103,7 @@ public class Config {
* @param is the resource as a SAX InputSource
* @param prefix an optional prefix that will be preprended to all non-absolute xpath expressions
*/
- public Config(SolrResourceLoader loader, String name, InputSource is, String prefix, boolean subProps) throws ParserConfigurationException, IOException, SAXException
+ public Config(SolrResourceLoader loader, String name, InputSource is, String prefix, boolean substituteProps) throws ParserConfigurationException, IOException, SAXException
{
if( loader == null ) {
loader = new SolrResourceLoader( null );
@@ -138,8 +139,8 @@ public class Config {
// some XML parsers are broken and don't close the byte stream (but they should according to spec)
IOUtils.closeQuietly(is.getByteStream());
}
- if (subProps) {
- DOMUtil.substituteProperties(doc, loader.getCoreProperties());
+ if (substituteProps) {
+ DOMUtil.substituteProperties(doc, getSubstituteProperties());
}
} catch (ParserConfigurationException e) {
SolrException.log(log, "Exception during parsing file: " + name, e);
@@ -152,7 +153,11 @@ public class Config {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
}
}
-
+
+ protected Properties getSubstituteProperties() {
+ return loader.getCoreProperties();
+ }
+
public Config(SolrResourceLoader loader, String name, Document doc) {
this.prefix = null;
this.doc = doc;
@@ -207,7 +212,7 @@ public class Config {
}
public void substituteProperties() {
- DOMUtil.substituteProperties(doc, loader.getCoreProperties());
+ DOMUtil.substituteProperties(doc, getSubstituteProperties());
}
Modified: lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/ConfigSolr.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/ConfigSolr.java?rev=1637544&r1=1637543&r2=1637544&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/ConfigSolr.java (original)
+++ lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/ConfigSolr.java Sat Nov 8 11:32:18 2014
@@ -215,8 +215,8 @@ public abstract class ConfigSolr {
return get(CfgProp.SOLR_ADMINHANDLER, "org.apache.solr.handler.admin.CoreAdminHandler");
}
- public String getZkCredentialProviderClass() {
- return get(CfgProp.SOLR_ZKCREDENTIALPROVIDER, null);
+ public String getZkCredentialsProviderClass() {
+ return get(CfgProp.SOLR_ZKCREDENTIALSPROVIDER, null);
}
public String getZkACLProviderClass() {
@@ -299,7 +299,7 @@ public abstract class ConfigSolr {
SOLR_AUTOREPLICAFAILOVERWORKLOOPDELAY,
SOLR_AUTOREPLICAFAILOVERBADNODEEXPIRATION,
- SOLR_ZKCREDENTIALPROVIDER,
+ SOLR_ZKCREDENTIALSPROVIDER,
SOLR_ZKACLPROVIDER,
//TODO: Remove all of these elements for 5.0
Modified: lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/ConfigSolrXml.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/ConfigSolrXml.java?rev=1637544&r1=1637543&r2=1637544&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/ConfigSolrXml.java (original)
+++ lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/ConfigSolrXml.java Sat Nov 8 11:32:18 2014
@@ -165,7 +165,7 @@ public class ConfigSolrXml extends Confi
storeConfigPropertyAsBoolean(s, nl, CfgProp.SOLR_GENERICCORENODENAMES, "genericCoreNodeNames");
storeConfigPropertyAsString(s, nl, CfgProp.SOLR_ZKACLPROVIDER, "zkACLProvider");
- storeConfigPropertyAsString(s, nl, CfgProp.SOLR_ZKCREDENTIALPROVIDER, "zkCredentialProvider");
+ storeConfigPropertyAsString(s, nl, CfgProp.SOLR_ZKCREDENTIALSPROVIDER, "zkCredentialsProvider");
errorOnLeftOvers(s, nl);
}
Modified: lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/ConfigSolrXmlOld.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/ConfigSolrXmlOld.java?rev=1637544&r1=1637543&r2=1637544&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/ConfigSolrXmlOld.java (original)
+++ lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/ConfigSolrXmlOld.java Sat Nov 8 11:32:18 2014
@@ -146,7 +146,7 @@ public class ConfigSolrXmlOld extends Co
storeConfigPropertyAsBoolean(CfgProp.SOLR_AUTOREPLICAFAILOVERWAITAFTEREXPIRATION, "solr/cores/@autoReplicaFailoverWaitAfterExpiration");
storeConfigPropertyAsBoolean(CfgProp.SOLR_AUTOREPLICAFAILOVERWORKLOOPDELAY, "solr/cores/@autoReplicaFailoverWorkLoopDelay");
storeConfigPropertyAsString(CfgProp.SOLR_ZKACLPROVIDER, "solr/cores/@zkACLProvider");
- storeConfigPropertyAsString(CfgProp.SOLR_ZKCREDENTIALPROVIDER, "solr/cores/@zkCredentialProvider");
+ storeConfigPropertyAsString(CfgProp.SOLR_ZKCREDENTIALSPROVIDER, "solr/cores/@zkCredentialsProvider");
storeConfigPropertyAsString(CfgProp.SOLR_MANAGEMENTPATH, "solr/cores/@managementPath");
storeConfigPropertyAsBoolean(CfgProp.SOLR_SHARESCHEMA, "solr/cores/@shareSchema");
storeConfigPropertyAsInt(CfgProp.SOLR_TRANSIENTCACHESIZE, "solr/cores/@transientCacheSize");
Modified: lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/CoreContainer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/CoreContainer.java?rev=1637544&r1=1637543&r2=1637544&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/CoreContainer.java (original)
+++ lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/CoreContainer.java Sat Nov 8 11:32:18 2014
@@ -34,6 +34,7 @@ import java.util.concurrent.ExecutorServ
import java.util.concurrent.Executors;
import org.apache.solr.cloud.ZkController;
+import org.apache.solr.cloud.ZkSolrResourceLoader;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.util.ExecutorUtil;
@@ -683,6 +684,7 @@ public class CoreContainer {
// cancel recovery in cloud mode
core.getSolrCoreState().cancelRecovery();
}
+ String configSetZkPath = core.getResourceLoader() instanceof ZkSolrResourceLoader ? ((ZkSolrResourceLoader)core.getResourceLoader()).getConfigSetZkPath() : null;
core.unloadOnClose(deleteIndexDir, deleteDataDir, deleteInstanceDir);
if (close)
@@ -690,7 +692,7 @@ public class CoreContainer {
if (zkSys.getZkController() != null) {
try {
- zkSys.getZkController().unregister(name, cd);
+ zkSys.getZkController().unregister(name, cd, configSetZkPath);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new SolrException(ErrorCode.SERVER_ERROR, "Interrupted while unregistering core [" + name + "] from cloud state");
Modified: lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/InitParams.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/InitParams.java?rev=1637544&r1=1637543&r2=1637544&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/InitParams.java (original)
+++ lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/InitParams.java Sat Nov 8 11:32:18 2014
@@ -79,7 +79,7 @@ public class InitParams {
}
public void apply(NamedList initArgs) {
- merge(defaults, (NamedList) initArgs.get(PluginInfo.DEFAULTS), initArgs, PluginInfo.DEFAULTS, false);
+ merge( (NamedList) initArgs.get(PluginInfo.DEFAULTS), defaults,initArgs, PluginInfo.DEFAULTS, false);
merge((NamedList) initArgs.get(PluginInfo.INVARIANTS), invariants, initArgs, PluginInfo.INVARIANTS, false);
merge((NamedList) initArgs.get(PluginInfo.APPENDS), appends, initArgs, PluginInfo.APPENDS, true);
}
Modified: lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/JmxMonitoredMap.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/JmxMonitoredMap.java?rev=1637544&r1=1637543&r2=1637544&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/JmxMonitoredMap.java (original)
+++ lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/JmxMonitoredMap.java Sat Nov 8 11:32:18 2014
@@ -112,8 +112,16 @@ public class JmxMonitoredMap<K, V> exten
@Override
public void clear() {
if (server != null) {
- for (Map.Entry<String, SolrInfoMBean> entry : entrySet()) {
- unregister(entry.getKey(), entry.getValue());
+ QueryExp exp = Query.eq(Query.attr("coreHashCode"), Query.value(coreHashCode));
+ Set<ObjectName> objectNames = server.queryNames(null, exp);
+ if (objectNames != null) {
+ for (ObjectName name : objectNames) {
+ try {
+ server.unregisterMBean(name);
+ } catch (Exception e) {
+ LOG.error("Exception un-registering mbean {}", name, e);
+ }
+ }
}
}
Modified: lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/PluginInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/PluginInfo.java?rev=1637544&r1=1637543&r2=1637544&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/PluginInfo.java (original)
+++ lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/PluginInfo.java Sat Nov 8 11:32:18 2014
@@ -29,7 +29,7 @@ import static java.util.Collections.unmo
* An Object which represents a Plugin of any type
*
*/
-public class PluginInfo {
+public class PluginInfo implements MapSerializable{
public final String name, className, type;
public final NamedList initArgs;
public final Map<String, String> attributes;
@@ -92,6 +92,28 @@ public class PluginInfo {
List<PluginInfo> l = getChildren(type);
return l.isEmpty() ? null:l.get(0);
}
+ public Map<String,Object> toMap(){
+ LinkedHashMap m = new LinkedHashMap(attributes);
+ if(initArgs!=null ) m.putAll(initArgs.asMap(3));
+ if(children != null){
+ for (PluginInfo child : children) {
+ Object old = m.get(child.name);
+ if(old == null){
+ m.put(child.name, child.toMap());
+ } else if (old instanceof List) {
+ List list = (List) old;
+ list.add(child.toMap());
+ } else {
+ ArrayList l = new ArrayList();
+ l.add(old);
+ l.add(child.toMap());
+ m.put(child.name,l);
+ }
+ }
+
+ }
+ return m;
+ }
/**Filter children by type
* @param type The type name. must not be null
Modified: lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/RequestHandlers.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/RequestHandlers.java?rev=1637544&r1=1637543&r2=1637544&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/RequestHandlers.java (original)
+++ lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/RequestHandlers.java Sat Nov 8 11:32:18 2014
@@ -184,7 +184,7 @@ public final class RequestHandlers {
for (Map.Entry<PluginInfo,SolrRequestHandler> entry : handlers.entrySet()) {
PluginInfo info = entry.getKey();
SolrRequestHandler requestHandler = entry.getValue();
- info = applyParamSet(config, info);
+ info = applyInitParams(config, info);
if (requestHandler instanceof PluginInfoInitialized) {
((PluginInfoInitialized) requestHandler).init(info);
} else{
@@ -198,7 +198,7 @@ public final class RequestHandlers {
log.warn("no default request handler is registered (either '/select' or 'standard')");
}
- private PluginInfo applyParamSet(SolrConfig config, PluginInfo info) {
+ private PluginInfo applyInitParams(SolrConfig config, PluginInfo info) {
List<InitParams> ags = new ArrayList<>();
String p = info.attributes.get(InitParams.TYPE);
if(p!=null) {
@@ -207,13 +207,12 @@ public final class RequestHandlers {
else log.warn("INVALID paramSet {} in requestHandler {}", arg, info.toString());
}
}
- for (InitParams args : config.getInitParams().values()) {
+ for (InitParams args : config.getInitParams().values())
if(args.matchPath(info.name)) ags.add(args);
- }
if(!ags.isEmpty()){
info = new PluginInfo(info.type, info.attributes, info.initArgs.clone(), info.children);
- for (InitParams args : ags) {
- args.apply(info.initArgs);
+ for (InitParams initParam : ags) {
+ initParam.apply(info.initArgs);
}
}
return info;
Modified: lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/SolrConfig.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/SolrConfig.java?rev=1637544&r1=1637543&r2=1637544&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/SolrConfig.java (original)
+++ lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/SolrConfig.java Sat Nov 8 11:32:18 2014
@@ -18,12 +18,14 @@
package org.apache.solr.core;
+import com.google.common.collect.ImmutableList;
import org.apache.lucene.index.IndexDeletionPolicy;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.util.Version;
import org.apache.solr.cloud.ZkSolrResourceLoader;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
+import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.handler.component.SearchComponent;
import org.apache.solr.request.SolrRequestHandler;
import org.apache.solr.response.QueryResponseWriter;
@@ -42,6 +44,8 @@ import org.apache.solr.update.processor.
import org.apache.solr.util.DOMUtil;
import org.apache.solr.util.FileUtils;
import org.apache.solr.util.RegexFileFilter;
+import org.noggit.JSONParser;
+import org.noggit.ObjectBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.Node;
@@ -54,6 +58,9 @@ import javax.xml.xpath.XPathConstants;
import java.io.File;
import java.io.FileFilter;
import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
import java.util.EnumSet;
@@ -62,6 +69,8 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -76,7 +85,7 @@ import static org.apache.solr.core.SolrC
* configuration data for a a Solr instance -- typically found in
* "solrconfig.xml".
*/
-public class SolrConfig extends Config {
+public class SolrConfig extends Config implements MapSerializable{
public static final Logger log = LoggerFactory.getLogger(SolrConfig.class);
@@ -165,6 +174,7 @@ public class SolrConfig extends Config {
public SolrConfig(SolrResourceLoader loader, String name, InputSource is)
throws ParserConfigurationException, IOException, SAXException {
super(loader, name, is, "/config/");
+ getOverlay();//just in case it is not initialized
initLibs();
luceneMatchVersion = getLuceneVersion("luceneMatchVersion");
String indexConfigPrefix;
@@ -242,48 +252,7 @@ public class SolrConfig extends Config {
}
maxWarmingSearchers = getInt("query/maxWarmingSearchers",Integer.MAX_VALUE);
slowQueryThresholdMillis = getInt("query/slowQueryThresholdMillis", -1);
-
- loadPluginInfo(SolrRequestHandler.class,"requestHandler",
- REQUIRE_NAME, REQUIRE_CLASS, MULTI_OK);
- loadPluginInfo(QParserPlugin.class,"queryParser",
- REQUIRE_NAME, REQUIRE_CLASS, MULTI_OK);
- loadPluginInfo(QueryResponseWriter.class,"queryResponseWriter",
- REQUIRE_NAME, REQUIRE_CLASS, MULTI_OK);
- loadPluginInfo(ValueSourceParser.class,"valueSourceParser",
- REQUIRE_NAME, REQUIRE_CLASS, MULTI_OK);
- loadPluginInfo(TransformerFactory.class,"transformer",
- REQUIRE_NAME, REQUIRE_CLASS, MULTI_OK);
- loadPluginInfo(SearchComponent.class,"searchComponent",
- REQUIRE_NAME, REQUIRE_CLASS, MULTI_OK);
-
- // TODO: WTF is up with queryConverter???
- // it aparently *only* works as a singleton? - SOLR-4304
- // and even then -- only if there is a single SpellCheckComponent
- // because of queryConverter.setIndexAnalyzer
- loadPluginInfo(QueryConverter.class,"queryConverter",
- REQUIRE_NAME, REQUIRE_CLASS);
-
- // this is hackish, since it picks up all SolrEventListeners,
- // regardless of when/how/why they are used (or even if they are
- // declared outside of the appropriate context) but there's no nice
- // way around that in the PluginInfo framework
- loadPluginInfo(SolrEventListener.class, "//listener",
- REQUIRE_CLASS, MULTI_OK);
-
- loadPluginInfo(DirectoryFactory.class,"directoryFactory",
- REQUIRE_CLASS);
- loadPluginInfo(IndexDeletionPolicy.class,indexConfigPrefix+"/deletionPolicy",
- REQUIRE_CLASS);
- loadPluginInfo(CodecFactory.class,"codecFactory",
- REQUIRE_CLASS);
- loadPluginInfo(IndexReaderFactory.class,"indexReaderFactory",
- REQUIRE_CLASS);
- loadPluginInfo(UpdateRequestProcessorChain.class,"updateRequestProcessorChain",
- MULTI_OK);
- loadPluginInfo(UpdateLog.class,"updateHandler/updateLog");
- loadPluginInfo(IndexSchemaFactory.class,"schemaFactory",
- REQUIRE_CLASS);
- loadPluginInfo(RestManager.class, "restManager");
+ for (SolrPluginInfo plugin : plugins) loadPluginInfo(plugin);
updateHandlerInfo = loadUpdatehandlerInfo();
multipartUploadLimitKB = getInt(
@@ -302,7 +271,6 @@ public class SolrConfig extends Config {
addHttpRequestToContext = getBool(
"requestDispatcher/requestParsers/@addHttpRequestToContext", false );
- loadPluginInfo(InitParams.class, InitParams.TYPE, MULTI_OK);
List<PluginInfo> argsInfos = pluginStore.get(InitParams.class.getName()) ;
if(argsInfos!=null){
Map<String,InitParams> argsMap = new HashMap<>();
@@ -317,6 +285,71 @@ public class SolrConfig extends Config {
solrRequestParsers = new SolrRequestParsers(this);
Config.log.info("Loaded SolrConfig: " + name);
}
+
+ public static List<SolrPluginInfo> plugins = ImmutableList.<SolrPluginInfo>builder()
+ .add(new SolrPluginInfo(SolrRequestHandler.class, "requestHandler", REQUIRE_NAME, REQUIRE_CLASS, MULTI_OK))
+ .add(new SolrPluginInfo(QParserPlugin.class, "queryParser", REQUIRE_NAME, REQUIRE_CLASS, MULTI_OK))
+ .add(new SolrPluginInfo(QueryResponseWriter.class, "queryResponseWriter", REQUIRE_NAME, REQUIRE_CLASS, MULTI_OK))
+ .add(new SolrPluginInfo(ValueSourceParser.class, "valueSourceParser", REQUIRE_NAME, REQUIRE_CLASS, MULTI_OK))
+ .add(new SolrPluginInfo(TransformerFactory.class, "transformer", REQUIRE_NAME, REQUIRE_CLASS, MULTI_OK))
+ .add(new SolrPluginInfo(SearchComponent.class, "searchComponent", REQUIRE_NAME, REQUIRE_CLASS, MULTI_OK))
+ // TODO: WTF is up with queryConverter???
+ // it aparently *only* works as a singleton? - SOLR-4304
+ // and even then -- only if there is a single SpellCheckComponent
+ // because of queryConverter.setIndexAnalyzer
+ .add(new SolrPluginInfo(QueryConverter.class, "queryConverter", REQUIRE_NAME, REQUIRE_CLASS))
+ // this is hackish, since it picks up all SolrEventListeners,
+ // regardless of when/how/why they are used (or even if they are
+ // declared outside of the appropriate context) but there's no nice
+ // way around that in the PluginInfo framework
+ .add(new SolrPluginInfo(SolrEventListener.class, "//listener", REQUIRE_CLASS, MULTI_OK))
+ .add(new SolrPluginInfo(DirectoryFactory.class, "directoryFactory", REQUIRE_CLASS))
+ .add(new SolrPluginInfo(IndexDeletionPolicy.class, "indexConfig/deletionPolicy", REQUIRE_CLASS))
+ .add(new SolrPluginInfo(CodecFactory.class, "codecFactory", REQUIRE_CLASS))
+ .add(new SolrPluginInfo(IndexReaderFactory.class, "indexReaderFactory", REQUIRE_CLASS))
+ .add(new SolrPluginInfo(UpdateRequestProcessorChain.class,"updateRequestProcessorChain", MULTI_OK))
+ .add(new SolrPluginInfo(UpdateLog.class,"updateHandler/updateLog"))
+ .add(new SolrPluginInfo(IndexSchemaFactory.class, "schemaFactory", REQUIRE_CLASS))
+ .add(new SolrPluginInfo(RestManager.class, "restManager"))
+ .add(new SolrPluginInfo(InitParams.class, InitParams.TYPE, MULTI_OK))
+ .build();
+
+ public static class SolrPluginInfo{
+
+ public final Class clazz;
+ public final String tag;
+ public final Set<PluginOpts> options;
+
+
+ private SolrPluginInfo(Class clz, String tag, PluginOpts... opts) {
+ this.clazz = clz;
+ this.tag = tag;
+ this.options= opts == null? Collections.EMPTY_SET : EnumSet.of(NOOP, opts);
+ }
+ }
+
+ private static ConfigOverlay getConfigOverlay(SolrResourceLoader loader) {
+ InputStream in = null;
+ try {
+ in = loader.openResource(ConfigOverlay.RESOURCE_NAME);
+ } catch (IOException e) {
+ //no problem no overlay.json file
+ return new ConfigOverlay(Collections.EMPTY_MAP,0);
+ }
+
+ try {
+ int version = 0; //will be always 0 for file based resourceloader
+ if (in instanceof ZkSolrResourceLoader.ZkByteArrayInputStream) {
+ version = ((ZkSolrResourceLoader.ZkByteArrayInputStream) in).getStat().getVersion();
+ }
+ Map m = (Map) ObjectBuilder.getVal(new JSONParser(new InputStreamReader(in, StandardCharsets.UTF_8)));
+ return new ConfigOverlay(m,version);
+ } catch (Exception e) {
+ throw new SolrException(ErrorCode.SERVER_ERROR,"Error reading config overlay",e);
+ }
+
+ }
+
private Map<String,InitParams> initParams = Collections.emptyMap();
public Map<String, InitParams> getInitParams() {
return initParams;
@@ -333,20 +366,19 @@ public class SolrConfig extends Config {
getBool("updateHandler/commitWithin/softCommit",true));
}
- private void loadPluginInfo(Class clazz, String tag, PluginOpts... opts) {
- EnumSet<PluginOpts> options = EnumSet.<PluginOpts>of(NOOP, opts);
- boolean requireName = options.contains(REQUIRE_NAME);
- boolean requireClass = options.contains(REQUIRE_CLASS);
+ private void loadPluginInfo(SolrPluginInfo pluginInfo) {
+ boolean requireName = pluginInfo.options.contains(REQUIRE_NAME);
+ boolean requireClass = pluginInfo.options.contains(REQUIRE_CLASS);
- List<PluginInfo> result = readPluginInfos(tag, requireName, requireClass);
+ List<PluginInfo> result = readPluginInfos(pluginInfo.tag, requireName, requireClass);
- if (1 < result.size() && ! options.contains(MULTI_OK)) {
+ if (1 < result.size() && ! pluginInfo.options.contains(MULTI_OK)) {
throw new SolrException
(SolrException.ErrorCode.SERVER_ERROR,
"Found " + result.size() + " configuration sections when at most "
- + "1 is allowed matching expression: " + tag);
+ + "1 is allowed matching expression: " + pluginInfo.tag);
}
- if(!result.isEmpty()) pluginStore.put(clazz.getName(),result);
+ if(!result.isEmpty()) pluginStore.put(pluginInfo.clazz.getName(),result);
}
public List<PluginInfo> readPluginInfos(String tag, boolean requireName, boolean requireClass) {
@@ -411,7 +443,7 @@ public class SolrConfig extends Config {
return httpCachingConfig;
}
- public static class JmxConfiguration {
+ public static class JmxConfiguration implements MapSerializable{
public boolean enabled = false;
public String agentId;
public String serviceUrl;
@@ -434,9 +466,18 @@ public class SolrConfig extends Config {
}
}
+
+ @Override
+ public Map<String, Object> toMap() {
+ LinkedHashMap map = new LinkedHashMap();
+ map.put("agentId",agentId);
+ map.put("serviceUrl",serviceUrl);
+ map.put("rootName",rootName);
+ return map;
+ }
}
- public static class HttpCachingConfig {
+ public static class HttpCachingConfig implements MapSerializable{
/** config xpath prefix for getting HTTP Caching options */
private final static String CACHE_PRE
@@ -445,7 +486,15 @@ public class SolrConfig extends Config {
/** For extracting Expires "ttl" from <cacheControl> config */
private final static Pattern MAX_AGE
= Pattern.compile("\\bmax-age=(\\d+)");
-
+
+ @Override
+ public Map<String, Object> toMap() {
+ return ZkNodeProps.makeMap("never304",never304,
+ "etagSeed",etagSeed,
+ "lastModFrom",lastModFrom.name().toLowerCase(Locale.ROOT),
+ "cacheControl",cacheControlHeader);
+ }
+
public static enum LastModFrom {
OPENTIME, DIRLASTMOD, BOGUS;
@@ -505,7 +554,7 @@ public class SolrConfig extends Config {
public LastModFrom getLastModFrom() { return lastModFrom; }
}
- public static class UpdateHandlerInfo{
+ public static class UpdateHandlerInfo implements MapSerializable{
public final String className;
public final int autoCommmitMaxDocs,autoCommmitMaxTime,commitIntervalLowerBound,
autoSoftCommmitMaxDocs,autoSoftCommmitMaxTime;
@@ -531,7 +580,29 @@ public class SolrConfig extends Config {
this.autoSoftCommmitMaxTime = autoSoftCommmitMaxTime;
this.commitWithinSoftCommit = commitWithinSoftCommit;
- }
+ }
+
+
+
+ @Override
+ public Map<String, Object> toMap() {
+ LinkedHashMap result = new LinkedHashMap();
+ result.put("class",className);
+ result.put("autoCommmitMaxDocs",autoCommmitMaxDocs);
+ result.put("indexWriterCloseWaitsForMerges",indexWriterCloseWaitsForMerges);
+ result.put("openSearcher",openSearcher);
+ result.put("commitIntervalLowerBound",commitIntervalLowerBound);
+ result.put("commitWithinSoftCommit",commitWithinSoftCommit);
+ result.put("autoCommit", ZkNodeProps.makeMap(
+ "maxDocs", autoCommmitMaxDocs,
+ "maxTime",autoCommmitMaxTime,
+ "commitIntervalLowerBound", commitIntervalLowerBound
+ ));
+ result.put("autoSoftCommit" ,
+ ZkNodeProps.makeMap("maxDocs", autoSoftCommmitMaxDocs,
+ "maxTime",autoSoftCommmitMaxTime));
+ return result;
+ }
}
// public Map<String, List<PluginInfo>> getUpdateProcessorChainInfo() { return updateProcessorChainInfo; }
@@ -619,5 +690,99 @@ public class SolrConfig extends Config {
return enableRemoteStreams;
}
+ @Override
+ public int getInt(String path) {
+ return getInt(path, 0);
+ }
+
+ @Override
+ public int getInt(String path, int def) {
+ Object v = overlay.getXPathProperty(path);
+
+ Object val = overlay.getXPathProperty(path);
+ if (val != null) return Integer.parseInt(val.toString());
+ return super.getInt(path, def);
+ }
+ @Override
+ public boolean getBool(String path, boolean def) {
+ Object val = overlay.getXPathProperty(path);
+ if (val != null) return Boolean.parseBoolean(val.toString());
+ return super.getBool(path, def);
+ }
+ @Override
+ public Map<String, Object> toMap() {
+ LinkedHashMap result = new LinkedHashMap();
+ result.put("luceneMatchVersion",luceneMatchVersion);
+ result.put("updateHandler", getUpdateHandlerInfo().toMap());
+ Map m = new LinkedHashMap();
+ result.put("query", m);
+ m.put("useFilterForSortedQuery", useFilterForSortedQuery);
+ m.put("queryResultWindowSize", queryResultWindowSize);
+ m.put("queryResultMaxDocsCached", queryResultMaxDocsCached);
+ m.put("enableLazyFieldLoading", enableLazyFieldLoading);
+ m.put("maxBooleanClauses", booleanQueryMaxClauseCount);
+
+ for (SolrPluginInfo plugin : plugins) {
+ List<PluginInfo> infos = getPluginInfos(plugin.clazz.getName());
+ if(infos == null || infos.isEmpty()) continue;
+ String tag = plugin.tag;
+ tag = tag.replace("/","");
+ if(plugin.options.contains(PluginOpts.REQUIRE_NAME)){
+ LinkedHashMap items = new LinkedHashMap();
+ for (PluginInfo info : infos) items.put(info.name, info.toMap());
+ result.put(tag,items);
+ } else {
+ if(plugin.options.contains(MULTI_OK)){
+ ArrayList<Map> l = new ArrayList<>();
+ for (PluginInfo info : infos) l.add(info.toMap());
+ result.put(tag,l);
+ } else {
+ result.put(tag, infos.get(0).toMap());
+ }
+
+ }
+
+ }
+
+
+ addCacheConfig(m,filterCacheConfig,queryResultCacheConfig,documentCacheConfig,fieldValueCacheConfig);
+ if(jmxConfig != null) result.put("jmx",jmxConfig.toMap());
+ m = new LinkedHashMap();
+ result.put("requestDispatcher", m);
+ m.put("handleSelect",handleSelect);
+ if(httpCachingConfig!=null) m.put("httpCaching", httpCachingConfig.toMap());
+ m.put("requestParsers", ZkNodeProps.makeMap("multipartUploadLimitKB",multipartUploadLimitKB,
+ "formUploadLimitKB",formUploadLimitKB,
+ "addHttpRequestToContext",addHttpRequestToContext));
+ if(indexConfig != null) result.put("indexConfig",indexConfig.toMap());
+
+ //TODO there is more to add
+
+ return result;
+ }
+
+ private void addCacheConfig(Map queryMap, CacheConfig... cache) {
+ if(cache==null)return;
+ for (CacheConfig config : cache) if(config !=null) queryMap.put(config.getNodeName(),config.toMap());
+
+ }
+
+ @Override
+ protected Properties getSubstituteProperties() {
+ Map<String, Object> p = getOverlay().getUserProps();
+ if(p==null || p.isEmpty()) return super.getSubstituteProperties();
+ Properties result = new Properties(super.getSubstituteProperties());
+ result.putAll(p);
+ return result;
+ }
+ private ConfigOverlay overlay;
+
+ public ConfigOverlay getOverlay() {
+ if(overlay ==null) {
+ overlay = getConfigOverlay(getResourceLoader());
+ }
+ return overlay;
+ }
+
}
Modified: lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/SolrCore.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/SolrCore.java?rev=1637544&r1=1637543&r2=1637544&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/SolrCore.java (original)
+++ lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/SolrCore.java Sat Nov 8 11:32:18 2014
@@ -40,6 +40,7 @@ import org.apache.solr.common.util.Simpl
import org.apache.solr.core.DirectoryFactory.DirContext;
import org.apache.solr.handler.RequestHandlerBase;
import org.apache.solr.handler.SnapPuller;
+import org.apache.solr.handler.SolrConfigHandler;
import org.apache.solr.handler.UpdateRequestHandler;
import org.apache.solr.handler.admin.ShowFileRequestHandler;
import org.apache.solr.handler.component.DebugComponent;
@@ -808,6 +809,8 @@ public final class SolrCore implements S
reqHandlers = new RequestHandlers(this);
List<PluginInfo> implicitReqHandlerInfo = new ArrayList<>();
UpdateRequestHandler.addImplicits(implicitReqHandlerInfo);
+ SolrConfigHandler.addImplicits(implicitReqHandlerInfo);
+
reqHandlers.initHandlersFromConfig(solrConfig, implicitReqHandlerInfo);
// Handle things that should eventually go away
@@ -1065,14 +1068,6 @@ public final class SolrCore implements S
}
- try {
- infoRegistry.clear();
- } catch (Throwable e) {
- SolrException.log(log, e);
- if (e instanceof Error) {
- throw (Error) e;
- }
- }
try {
if (null != updateHandler) {
@@ -1125,6 +1120,15 @@ public final class SolrCore implements S
throw (Error) e;
}
}
+
+ try {
+ infoRegistry.clear();
+ } catch (Throwable e) {
+ SolrException.log(log, e);
+ if (e instanceof Error) {
+ throw (Error) e;
+ }
+ }
if (coreStateClosed) {
Modified: lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java?rev=1637544&r1=1637543&r2=1637544&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java (original)
+++ lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java Sat Nov 8 11:32:18 2014
@@ -53,8 +53,10 @@ import java.io.Closeable;
import java.io.File;
import java.io.FileFilter;
import java.io.FileInputStream;
+import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.io.OutputStreamWriter;
import java.lang.reflect.Constructor;
import java.net.MalformedURLException;
import java.net.URI;
@@ -819,4 +821,36 @@ public class SolrResourceLoader implemen
public List<SolrInfoMBean> getInfoMBeans(){
return Collections.unmodifiableList(infoMBeans);
}
+
+
+ public static void persistConfLocally(SolrResourceLoader loader, String resourceName, byte[] content) {
+ // Persist locally
+ File managedSchemaFile = new File(loader.getConfigDir(), resourceName);
+ OutputStreamWriter writer = null;
+ try {
+ File parentDir = managedSchemaFile.getParentFile();
+ if ( ! parentDir.isDirectory()) {
+ if ( ! parentDir.mkdirs()) {
+ final String msg = "Can't create managed schema directory " + parentDir.getAbsolutePath();
+ log.error(msg);
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, msg);
+ }
+ }
+ final FileOutputStream out = new FileOutputStream(managedSchemaFile);
+ out.write(content);
+ log.info("Upgraded to managed schema at " + managedSchemaFile.getPath());
+ } catch (IOException e) {
+ final String msg = "Error persisting managed schema " + managedSchemaFile;
+ log.error(msg, e);
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, msg, e);
+ } finally {
+ org.apache.commons.io.IOUtils.closeQuietly(writer);
+ try {
+ FileUtils.sync(managedSchemaFile);
+ } catch (IOException e) {
+ final String msg = "Error syncing the managed schema file " + managedSchemaFile;
+ log.error(msg, e);
+ }
+ }
+ }
}
Modified: lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java?rev=1637544&r1=1637543&r2=1637544&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java (original)
+++ lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java Sat Nov 8 11:32:18 2014
@@ -21,7 +21,7 @@ import static org.apache.solr.cloud.Over
import static org.apache.solr.cloud.OverseerCollectionProcessor.ASYNC;
import static org.apache.solr.cloud.OverseerCollectionProcessor.COLL_CONF;
import static org.apache.solr.cloud.OverseerCollectionProcessor.CREATE_NODE_SET;
-import static org.apache.solr.cloud.OverseerCollectionProcessor.SLICE_UNIQUE;
+import static org.apache.solr.cloud.OverseerCollectionProcessor.SHARD_UNIQUE;
import static org.apache.solr.cloud.OverseerCollectionProcessor.NUM_SLICES;
import static org.apache.solr.cloud.OverseerCollectionProcessor.ONLY_ACTIVE_NODES;
import static org.apache.solr.cloud.OverseerCollectionProcessor.ONLY_IF_DOWN;
@@ -46,7 +46,7 @@ import static org.apache.solr.common.clo
import static org.apache.solr.common.cloud.ZkStateReader.STATE_PROP;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDROLE;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICAPROP;
-import static org.apache.solr.common.params.CollectionParams.CollectionAction.BALANCESLICEUNIQUE;
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.BALANCESHARDUNIQUE;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.CLUSTERPROP;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.CREATE;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.CREATEALIAS;
@@ -258,8 +258,8 @@ public class CollectionsHandler extends
this.handleDeleteReplicaProp(req, rsp);
break;
}
- case BALANCESLICEUNIQUE: {
- this.handleBalanceSliceUnique(req, rsp);
+ case BALANCESHARDUNIQUE: {
+ this.handleBalanceShardUnique(req, rsp);
break;
}
case REBALANCELEADERS: {
@@ -430,23 +430,23 @@ public class CollectionsHandler extends
Map<String, Object> map = ZkNodeProps.makeMap(Overseer.QUEUE_OPERATION, ADDREPLICAPROP.toLower());
copyIfNotNull(req.getParams(), map, COLLECTION_PROP, SHARD_ID_PROP, REPLICA_PROP, PROPERTY_PROP,
- SLICE_UNIQUE, PROPERTY_VALUE_PROP);
+ SHARD_UNIQUE, PROPERTY_VALUE_PROP);
String property = (String) map.get(PROPERTY_PROP);
if (property.startsWith(OverseerCollectionProcessor.COLL_PROP_PREFIX) == false) {
property = OverseerCollectionProcessor.COLL_PROP_PREFIX + property;
}
- boolean uniquePerSlice = Boolean.parseBoolean((String) map.get(SLICE_UNIQUE));
+ boolean uniquePerSlice = Boolean.parseBoolean((String) map.get(SHARD_UNIQUE));
// Check if we're trying to set a property with parameters that allow us to set the property on multiple replicas
// in a slice on properties that are known to only be one-per-slice and error out if so.
- if (StringUtils.isNotBlank((String)map.get(SLICE_UNIQUE)) &&
+ if (StringUtils.isNotBlank((String)map.get(SHARD_UNIQUE)) &&
Overseer.sliceUniqueBooleanProperties.contains(property.toLowerCase(Locale.ROOT)) &&
uniquePerSlice == false) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
"Overseer replica property command received for property " + property +
- " with the " + SLICE_UNIQUE +
+ " with the " + SHARD_UNIQUE +
" parameter set to something other than 'true'. No action taken.");
}
handleResponse(ADDREPLICAPROP.toLower(), new ZkNodeProps(map), rsp);
@@ -463,25 +463,25 @@ public class CollectionsHandler extends
- private void handleBalanceSliceUnique(SolrQueryRequest req, SolrQueryResponse rsp) throws KeeperException, InterruptedException {
+ private void handleBalanceShardUnique(SolrQueryRequest req, SolrQueryResponse rsp) throws KeeperException, InterruptedException {
req.getParams().required().check(COLLECTION_PROP, PROPERTY_PROP);
- Boolean sliceUnique = Boolean.parseBoolean(req.getParams().get(SLICE_UNIQUE));
+ Boolean shardUnique = Boolean.parseBoolean(req.getParams().get(SHARD_UNIQUE));
String prop = req.getParams().get(PROPERTY_PROP).toLowerCase(Locale.ROOT);
if (StringUtils.startsWith(prop, OverseerCollectionProcessor.COLL_PROP_PREFIX) == false) {
prop = OverseerCollectionProcessor.COLL_PROP_PREFIX + prop;
}
- if (sliceUnique == false &&
+ if (shardUnique == false &&
Overseer.sliceUniqueBooleanProperties.contains(prop) == false) {
throw new SolrException(ErrorCode.BAD_REQUEST, "Balancing properties amongst replicas in a slice requires that"
- + " the property be pre-defined as a unique property (e.g. 'preferredLeader') or that 'sliceUnique' be set to 'true'. " +
- " Property: " + prop + " sliceUnique: " + Boolean.toString(sliceUnique));
+ + " the property be pre-defined as a unique property (e.g. 'preferredLeader') or that 'shardUnique' be set to 'true'. " +
+ " Property: " + prop + " shardUnique: " + Boolean.toString(shardUnique));
}
- Map<String, Object> map = ZkNodeProps.makeMap(Overseer.QUEUE_OPERATION, BALANCESLICEUNIQUE.toLower());
- copyIfNotNull(req.getParams(), map, COLLECTION_PROP, PROPERTY_PROP, ONLY_ACTIVE_NODES, SLICE_UNIQUE);
+ Map<String, Object> map = ZkNodeProps.makeMap(Overseer.QUEUE_OPERATION, BALANCESHARDUNIQUE.toLower());
+ copyIfNotNull(req.getParams(), map, COLLECTION_PROP, PROPERTY_PROP, ONLY_ACTIVE_NODES, SHARD_UNIQUE);
- handleResponse(BALANCESLICEUNIQUE.toLower(), new ZkNodeProps(map), rsp);
+ handleResponse(BALANCESHARDUNIQUE.toLower(), new ZkNodeProps(map), rsp);
}
private void handleOverseerStatus(SolrQueryRequest req, SolrQueryResponse rsp) throws KeeperException, InterruptedException {
Modified: lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java?rev=1637544&r1=1637543&r2=1637544&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java (original)
+++ lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java Sat Nov 8 11:32:18 2014
@@ -587,7 +587,7 @@ public class CoreAdminHandler extends Re
catch (Exception ex) {
if (coreContainer.isZooKeeperAware() && dcore != null && !preExisitingZkEntry) {
try {
- coreContainer.getZkController().unregister(dcore.getName(), dcore);
+ coreContainer.getZkController().unregister(dcore.getName(), dcore,null);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
SolrException.log(log, null, e);
Modified: lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/handler/admin/EditFileRequestHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/handler/admin/EditFileRequestHandler.java?rev=1637544&r1=1637543&r2=1637544&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/handler/admin/EditFileRequestHandler.java (original)
+++ lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/handler/admin/EditFileRequestHandler.java Sat Nov 8 11:32:18 2014
@@ -282,7 +282,7 @@ public class EditFileRequestHandler exte
if (coreContainer.isZooKeeperAware()) {
try {
- String confPath = ((ZkSolrResourceLoader) core.getResourceLoader()).getCollectionZkPath();
+ String confPath = ((ZkSolrResourceLoader) core.getResourceLoader()).getConfigSetZkPath();
ZkController.downloadConfigDir(coreContainer.getZkController().getZkClient(), confPath,
new File(coll, "conf"));
Modified: lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java?rev=1637544&r1=1637543&r2=1637544&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java (original)
+++ lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java Sat Nov 8 11:32:18 2014
@@ -180,7 +180,7 @@ public class LukeRequestHandler extends
flags.append( (f != null && f.fieldType().indexOptions() != IndexOptions.NONE) ? FieldFlag.INDEXED.getAbbreviation() : '-' );
flags.append( (f != null && f.fieldType().tokenized()) ? FieldFlag.TOKENIZED.getAbbreviation() : '-' );
flags.append( (f != null && f.fieldType().stored()) ? FieldFlag.STORED.getAbbreviation() : '-' );
- flags.append( (f != null && f.fieldType().docValueType() != DocValuesType.NONE) ? FieldFlag.DOC_VALUES.getAbbreviation() : "-" );
+ flags.append( (f != null && f.fieldType().docValuesType() != DocValuesType.NONE) ? FieldFlag.DOC_VALUES.getAbbreviation() : "-" );
flags.append( (false) ? FieldFlag.MULTI_VALUED.getAbbreviation() : '-' ); // SchemaField Specific
flags.append( (f != null && f.fieldType().storeTermVectors()) ? FieldFlag.TERM_VECTOR_STORED.getAbbreviation() : '-' );
flags.append( (f != null && f.fieldType().storeTermVectorOffsets()) ? FieldFlag.TERM_VECTOR_OFFSET.getAbbreviation() : '-' );
Modified: lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/handler/admin/ShowFileRequestHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/handler/admin/ShowFileRequestHandler.java?rev=1637544&r1=1637543&r2=1637544&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/handler/admin/ShowFileRequestHandler.java (original)
+++ lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/handler/admin/ShowFileRequestHandler.java Sat Nov 8 11:32:18 2014
@@ -304,7 +304,7 @@ public class ShowFileRequestHandler exte
final ZkSolrResourceLoader loader = (ZkSolrResourceLoader) core
.getResourceLoader();
- String confPath = loader.getCollectionZkPath();
+ String confPath = loader.getConfigSetZkPath();
String fname = req.getParams().get("file", null);
if (fname == null) {
Modified: lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/handler/component/PivotFacetHelper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/handler/component/PivotFacetHelper.java?rev=1637544&r1=1637543&r2=1637544&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/handler/component/PivotFacetHelper.java (original)
+++ lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/handler/component/PivotFacetHelper.java Sat Nov 8 11:32:18 2014
@@ -18,15 +18,21 @@
package org.apache.solr.handler.component;
import org.apache.solr.util.PivotListEntry;
+import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.params.FacetParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.common.util.StrUtils;
import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Collections;
+import java.util.Map;
+import java.util.Map.Entry;
public class PivotFacetHelper {
@@ -91,31 +97,63 @@ public class PivotFacetHelper {
/** @see PivotListEntry#VALUE */
public static Comparable getValue(NamedList<Object> pivotList) {
- return (Comparable) PivotFacetHelper.retrieve(PivotListEntry.VALUE,
- pivotList);
+ return (Comparable) PivotListEntry.VALUE.extract(pivotList);
}
/** @see PivotListEntry#FIELD */
public static String getField(NamedList<Object> pivotList) {
- return (String) PivotFacetHelper.retrieve(PivotListEntry.FIELD, pivotList);
+ return (String) PivotListEntry.FIELD.extract(pivotList);
}
/** @see PivotListEntry#COUNT */
public static Integer getCount(NamedList<Object> pivotList) {
- return (Integer) PivotFacetHelper.retrieve(PivotListEntry.COUNT, pivotList);
+ return (Integer) PivotListEntry.COUNT.extract(pivotList);
}
/** @see PivotListEntry#PIVOT */
public static List<NamedList<Object>> getPivots(NamedList<Object> pivotList) {
- int pivotIdx = pivotList.indexOf(PivotListEntry.PIVOT.getName(), 0);
- if (pivotIdx > -1) {
- return (List<NamedList<Object>>) pivotList.getVal(pivotIdx);
- }
- return null;
+ return (List<NamedList<Object>>) PivotListEntry.PIVOT.extract(pivotList);
}
- private static Object retrieve(PivotListEntry entryToGet, NamedList<Object> pivotList) {
- return pivotList.get(entryToGet.getName(), entryToGet.getIndex());
+ /** @see PivotListEntry#STATS */
+ public static NamedList<NamedList<NamedList<?>>> getStats(NamedList<Object> pivotList) {
+ return (NamedList<NamedList<NamedList<?>>>) PivotListEntry.STATS.extract(pivotList);
+ }
+
+ /**
+ * Given a mapping of keys to {@link StatsValues} representing the currently
+ * known "merged" stats (which may be null if none exist yet), and a
+ * {@link NamedList} containing the "stats" response block returned by an individual
+ * shard, this method accumulates the stasts for each {@link StatsField} found in
+ * the shard response with the existing mergeStats
+ *
+ * @return the original <code>merged</code> Map after modifying, or a new Map if the <code>merged</code> param was originally null.
+ * @see StatsInfo#getStatsField
+ * @see StatsValuesFactory#createStatsValues
+ * @see StatsValues#accumulate(NamedList)
+ */
+ public static Map<String,StatsValues> mergeStats
+ (Map<String,StatsValues> merged,
+ NamedList<NamedList<NamedList<?>>> remoteWrapper,
+ StatsInfo statsInfo) {
+
+ if (null == merged) merged = new LinkedHashMap<String,StatsValues>();
+
+ NamedList<NamedList<?>> remoteStats = StatsComponent.unwrapStats(remoteWrapper);
+
+ for (Entry<String,NamedList<?>> entry : remoteStats) {
+ StatsValues receivingStatsValues = merged.get(entry.getKey());
+ if (receivingStatsValues == null) {
+ StatsField recievingStatsField = statsInfo.getStatsField(entry.getKey());
+ if (null == recievingStatsField) {
+ throw new SolrException(ErrorCode.SERVER_ERROR , "No stats.field found corrisponding to pivot stats recieved from shard: "+entry.getKey());
+ }
+ receivingStatsValues = StatsValuesFactory.createStatsValues(recievingStatsField);
+ merged.put(entry.getKey(), receivingStatsValues);
+ }
+ receivingStatsValues.accumulate(entry.getValue());
+ }
+ return merged;
}
}
Modified: lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/handler/component/PivotFacetProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/handler/component/PivotFacetProcessor.java?rev=1637544&r1=1637543&r2=1637544&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/handler/component/PivotFacetProcessor.java (original)
+++ lucene/dev/branches/lucene6005/solr/core/src/java/org/apache/solr/handler/component/PivotFacetProcessor.java Sat Nov 8 11:32:18 2014
@@ -23,20 +23,26 @@ import org.apache.solr.schema.FieldType;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.search.DocSet;
import org.apache.solr.search.SyntaxError;
+import org.apache.solr.util.PivotListEntry;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.params.ShardParams;
import org.apache.solr.common.params.FacetParams;
+import org.apache.solr.common.params.StatsParams;
import org.apache.solr.request.SimpleFacets;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.lucene.search.Query;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.Deque;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
@@ -63,9 +69,15 @@ public class PivotFacetProcessor extends
if (!rb.doFacets || pivots == null)
return null;
+ // rb._statsInfo may be null if stats=false, ie: refine requests
+ // if that's the case, but we need to refine w/stats, then we'll lazy init our
+ // own instance of StatsInfo
+ StatsInfo statsInfo = rb._statsInfo;
+
SimpleOrderedMap<List<NamedList<Object>>> pivotResponse = new SimpleOrderedMap<>();
for (String pivotList : pivots) {
try {
+ // NOTE: this sets localParams (SimpleFacets is stateful)
this.parseParams(FacetParams.FACET_PIVOT, pivotList);
} catch (SyntaxError e) {
throw new SolrException(ErrorCode.BAD_REQUEST, e);
@@ -84,15 +96,37 @@ public class PivotFacetProcessor extends
}
}
- //REFINEMENT
- String fieldValueKey = localParams == null ? null : localParams.get(PivotFacet.REFINE_PARAM);
- if(fieldValueKey != null ){
- String[] refinementValuesByField = params.getParams(PivotFacet.REFINE_PARAM+fieldValueKey);
+ // start by assuing no local params...
+
+ String refineKey = null; // no local => no refinement
+ List<StatsField> statsFields = Collections.emptyList(); // no local => no stats
+
+ if (null != localParams) {
+ // we might be refining..
+ refineKey = localParams.get(PivotFacet.REFINE_PARAM);
+
+ String statsLocalParam = localParams.get(StatsParams.STATS);
+ if (null != refineKey
+ && null != statsLocalParam
+ && null == statsInfo) {
+ // we are refining and need to compute stats,
+ // but stats component hasn't inited StatsInfo (because we
+ // don't need/want top level stats when refining) so we lazy init
+ // our own copy of StatsInfo
+ statsInfo = new StatsInfo(rb);
+ }
+ statsFields = getTaggedStatsFields(statsInfo, statsLocalParam);
+ }
+
+ if (null != refineKey) {
+ String[] refinementValuesByField
+ = params.getParams(PivotFacet.REFINE_PARAM + refineKey);
+
for(String refinements : refinementValuesByField){
- pivotResponse.addAll(processSingle(pivotFields, refinements));
+ pivotResponse.addAll(processSingle(pivotFields, refinements, statsFields));
}
} else{
- pivotResponse.addAll(processSingle(pivotFields, null));
+ pivotResponse.addAll(processSingle(pivotFields, null, statsFields));
}
}
return pivotResponse;
@@ -102,9 +136,13 @@ public class PivotFacetProcessor extends
* Process a single branch of refinement values for a specific pivot
* @param pivotFields the ordered list of fields in this pivot
* @param refinements the comma seperate list of refinement values corrisponding to each field in the pivot, or null if there are no refinements
+ * @param statsFields List of {@link StatsField} instances to compute for each pivot value
*/
- private SimpleOrderedMap<List<NamedList<Object>>> processSingle(List<String> pivotFields,
- String refinements) throws IOException {
+ private SimpleOrderedMap<List<NamedList<Object>>> processSingle
+ (List<String> pivotFields,
+ String refinements,
+ List<StatsField> statsFields) throws IOException {
+
SolrIndexSearcher searcher = rb.req.getSearcher();
SimpleOrderedMap<List<NamedList<Object>>> pivotResponse = new SimpleOrderedMap<>();
@@ -141,18 +179,54 @@ public class PivotFacetProcessor extends
if(pivotFields.size() > 1) {
String subField = pivotFields.get(1);
pivotResponse.add(key,
- doPivots(facetCounts, field, subField, fnames, vnames, this.docs));
+ doPivots(facetCounts, field, subField, fnames, vnames, this.docs, statsFields));
} else {
- pivotResponse.add(key, doPivots(facetCounts, field, null, fnames, vnames, this.docs));
+ pivotResponse.add(key, doPivots(facetCounts, field, null, fnames, vnames, this.docs, statsFields));
}
return pivotResponse;
}
/**
+ * returns the {@link StatsField} instances that should be computed for a pivot
+ * based on the 'stats' local params used.
+ *
+ * @return A list of StatsFields to comput for this pivot, or the empty list if none
+ */
+ private static List<StatsField> getTaggedStatsFields(StatsInfo statsInfo,
+ String statsLocalParam) {
+ if (null == statsLocalParam || null == statsInfo) {
+ return Collections.emptyList();
+ }
+
+ List<StatsField> fields = new ArrayList<>(7);
+ List<String> statsAr = StrUtils.splitSmart(statsLocalParam, ',');
+
+ // TODO: for now, we only support a single tag name - we reserve using
+ // ',' as a possible delimeter for logic related to only computing stats
+ // at certain levels -- see SOLR-6663
+ if (1 < statsAr.size()) {
+ String msg = StatsParams.STATS + " local param of " + FacetParams.FACET_PIVOT +
+ "may not include tags separated by a comma - please use a common tag on all " +
+ StatsParams.STATS_FIELD + " params you wish to compute under this pivot";
+ throw new SolrException(ErrorCode.BAD_REQUEST, msg);
+ }
+
+ for(String stat : statsAr) {
+ fields.addAll(statsInfo.getStatsFieldsByTag(stat));
+ }
+ return fields;
+ }
+
+ /**
* Recursive function to compute all the pivot counts for the values under teh specified field
*/
protected List<NamedList<Object>> doPivots(NamedList<Integer> superFacets,
- String field, String subField, Deque<String> fnames,Deque<String> vnames,DocSet docs) throws IOException {
+ String field, String subField,
+ Deque<String> fnames, Deque<String> vnames,
+ DocSet docs, List<StatsField> statsFields)
+ throws IOException {
+
+ boolean isShard = rb.req.getParams().getBool(ShardParams.IS_SHARD, false);
SolrIndexSearcher searcher = rb.req.getSearcher();
// TODO: optimize to avoid converting to an external string and then having to convert back to internal below
@@ -169,6 +243,7 @@ public class PivotFacetProcessor extends
// Only sub-facet if parent facet has positive count - still may not be any values for the sub-field though
if (kv.getValue() >= getMinCountForField(field)) {
final String fieldValue = kv.getKey();
+ final int pivotCount = kv.getValue();
SimpleOrderedMap<Object> pivot = new SimpleOrderedMap<>();
pivot.add( "field", field );
@@ -178,7 +253,7 @@ public class PivotFacetProcessor extends
ftype.readableToIndexed(fieldValue, termval);
pivot.add( "value", ftype.toObject(sfield, termval.get()) );
}
- pivot.add( "count", kv.getValue() );
+ pivot.add( "count", pivotCount );
DocSet subset = getSubset(docs, sfield, fieldValue);
@@ -195,8 +270,16 @@ public class PivotFacetProcessor extends
}
if (facetCounts.size() >= 1) {
- pivot.add( "pivot", doPivots( facetCounts, subField, nextField, fnames, vnames, subset) );
+ pivot.add( "pivot", doPivots( facetCounts, subField, nextField, fnames, vnames, subset, statsFields ) );
+ }
+ }
+ if ((isShard || 0 < pivotCount) && ! statsFields.isEmpty()) {
+ Map<String, StatsValues> stv = new LinkedHashMap<>();
+ for (StatsField statsField : statsFields) {
+ stv.put(statsField.getOutputKey(), statsField.computeLocalStatsValues(subset));
}
+ // for pivots, we *always* include requested stats - even if 'empty'
+ pivot.add("stats", StatsComponent.convertToResponse(true, stv));
}
values.add( pivot );
}