You are viewing a plain text version of this content. The canonical link for it is here.
Posted to solr-commits@lucene.apache.org by ma...@apache.org on 2010/02/04 02:08:05 UTC
svn commit: r906316 - in /lucene/solr/branches/cloud/src:
java/org/apache/solr/cloud/ java/org/apache/solr/core/
java/org/apache/solr/handler/component/ test/org/apache/solr/cloud/
webapp/web/admin/
Author: markrmiller
Date: Thu Feb 4 01:08:04 2010
New Revision: 906316
URL: http://svn.apache.org/viewvc?rev=906316&view=rev
Log:
remove non common dependencies from zkController for possible sharing with solrj
Modified:
lucene/solr/branches/cloud/src/java/org/apache/solr/cloud/ZkController.java
lucene/solr/branches/cloud/src/java/org/apache/solr/core/CoreContainer.java
lucene/solr/branches/cloud/src/java/org/apache/solr/core/CoreDescriptor.java
lucene/solr/branches/cloud/src/java/org/apache/solr/handler/component/QueryComponent.java
lucene/solr/branches/cloud/src/java/org/apache/solr/handler/component/QueryElevationComponent.java
lucene/solr/branches/cloud/src/test/org/apache/solr/cloud/BasicZkTest.java
lucene/solr/branches/cloud/src/test/org/apache/solr/cloud/CloudStateUpdateTest.java
lucene/solr/branches/cloud/src/test/org/apache/solr/cloud/DistributedZkFailuresTest.java
lucene/solr/branches/cloud/src/webapp/web/admin/zookeeper.jsp
Modified: lucene/solr/branches/cloud/src/java/org/apache/solr/cloud/ZkController.java
URL: http://svn.apache.org/viewvc/lucene/solr/branches/cloud/src/java/org/apache/solr/cloud/ZkController.java?rev=906316&r1=906315&r2=906316&view=diff
==============================================================================
--- lucene/solr/branches/cloud/src/java/org/apache/solr/cloud/ZkController.java (original)
+++ lucene/solr/branches/cloud/src/java/org/apache/solr/cloud/ZkController.java Thu Feb 4 01:08:04 2010
@@ -17,10 +17,8 @@
* limitations under the License.
*/
-import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
-import java.io.InputStream;
import java.net.InetAddress;
import java.util.HashMap;
import java.util.HashSet;
@@ -34,14 +32,8 @@
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import javax.xml.parsers.ParserConfigurationException;
-
import org.apache.solr.cloud.SolrZkClient.OnReconnect;
import org.apache.solr.common.SolrException;
-import org.apache.solr.core.SolrConfig;
-import org.apache.solr.core.SolrCore;
-import org.apache.solr.core.SolrResourceLoader;
-import org.apache.solr.schema.IndexSchema;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.WatchedEvent;
@@ -50,7 +42,6 @@
import org.apache.zookeeper.data.Stat;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.xml.sax.SAXException;
/**
* Handle ZooKeeper interactions.
@@ -219,35 +210,6 @@
}
/**
- * Load SolrConfig from ZooKeeper.
- *
- * TODO: consider *many* cores firing up at once and loading the same files
- * from ZooKeeper
- *
- * @param resourceLoader
- * @param solrConfigFileName
- * @return
- * @throws IOException
- * @throws ParserConfigurationException
- * @throws SAXException
- * @throws InterruptedException
- * @throws KeeperException
- */
- public SolrConfig getConfig(String zkConfigName, String solrConfigFileName,
- SolrResourceLoader resourceLoader) throws IOException,
- ParserConfigurationException, SAXException, KeeperException,
- InterruptedException {
- byte[] config = zkClient.getData(CONFIGS_ZKNODE + "/" + zkConfigName + "/"
- + solrConfigFileName, null, null);
- InputStream is = new ByteArrayInputStream(config);
- SolrConfig cfg = solrConfigFileName == null ? new SolrConfig(
- resourceLoader, SolrConfig.DEFAULT_CONF_FILE, is) : new SolrConfig(
- resourceLoader, solrConfigFileName, is);
-
- return cfg;
- }
-
- /**
* @param zkConfigName
* @param fileName
* @return
@@ -256,7 +218,7 @@
*/
public byte[] getConfigFileData(String zkConfigName, String fileName)
throws KeeperException, InterruptedException {
- return zkClient.getData(CONFIGS_ZKNODE + "/" + zkConfigName, null, null);
+ return zkClient.getData(CONFIGS_ZKNODE + "/" + zkConfigName + "/" + fileName, null, null);
}
// TODO: consider how this is done
@@ -281,29 +243,6 @@
return hostName;
}
- /**
- * Load IndexSchema from ZooKeeper.
- *
- * TODO: consider *many* cores firing up at once and loading the same files
- * from ZooKeeper
- *
- * @param resourceLoader
- * @param schemaName
- * @param config
- * @return
- * @throws InterruptedException
- * @throws KeeperException
- */
- public IndexSchema getSchema(String zkConfigName, String schemaName,
- SolrConfig config, SolrResourceLoader resourceLoader)
- throws KeeperException, InterruptedException {
- byte[] configBytes = zkClient.getData(CONFIGS_ZKNODE + "/" + zkConfigName
- + "/" + schemaName, null, null);
- InputStream is = new ByteArrayInputStream(configBytes);
- IndexSchema schema = new IndexSchema(config, schemaName, is);
- return schema;
- }
-
public SolrZkClient getZkClient() {
return zkClient;
}
@@ -528,23 +467,20 @@
}
/**
- * Register shard. A SolrCore calls this on startup to register with
- * ZooKeeper.
+ * Register shard with ZooKeeper.
*
- * @param core SolrCore to register as a shard
+ * @param coreName
+ * @param cloudDesc
* @param forcePropsUpdate update solr.xml core props even if the shard is already registered
- *
* @throws IOException
* @throws KeeperException
* @throws InterruptedException
*/
- public void register(SolrCore core, boolean forcePropsUpdate) throws IOException,
+ public void register(String coreName, CloudDescriptor cloudDesc, boolean forcePropsUpdate) throws IOException,
KeeperException, InterruptedException {
- String coreName = core.getCoreDescriptor().getName();
String shardUrl = localHostName + ":" + localHostPort + "/" + localHostContext
+ "/" + coreName;
- CloudDescriptor cloudDesc = core.getCoreDescriptor().getCloudDescriptor();
String collection = cloudDesc.getCollectionName();
String shardsZkPath = COLLECTIONS_ZKNODE + "/" + collection + SHARDS_ZKNODE + "/" + cloudDesc.getShardId();
@@ -556,7 +492,7 @@
}
if (log.isInfoEnabled()) {
- log.info("Register shard - core:" + core.getName() + " address:"
+ log.info("Register shard - core:" + coreName + " address:"
+ shardUrl);
}
@@ -596,9 +532,10 @@
}
/**
- * @param core
+ * @param coreName
+ * @param cloudDesc
*/
- public void unregister(SolrCore core) {
+ public void unregister(String coreName, CloudDescriptor cloudDesc) {
// TODO : perhaps mark the core down in zk?
}
Modified: lucene/solr/branches/cloud/src/java/org/apache/solr/core/CoreContainer.java
URL: http://svn.apache.org/viewvc/lucene/solr/branches/cloud/src/java/org/apache/solr/core/CoreContainer.java?rev=906316&r1=906315&r2=906316&view=diff
==============================================================================
--- lucene/solr/branches/cloud/src/java/org/apache/solr/core/CoreContainer.java (original)
+++ lucene/solr/branches/cloud/src/java/org/apache/solr/core/CoreContainer.java Thu Feb 4 01:08:04 2010
@@ -79,7 +79,7 @@
protected String solrDataDirOverride;
protected String zkPortOverride;
private String testShardIdOverride;
- private ZkController zooKeeperController;
+ private ZkController zkController;
private SolrZkServer zkServer;
private String zkHost;
@@ -124,7 +124,7 @@
if (zkRun != null && zkServer.getServers().size() > 1)
zkClientConnectTimeout = 24 * 60 * 60 * 1000; // 1 day for embedded ensemble
- zooKeeperController = new ZkController(zookeeperHost, zkClientTimeout, zkClientConnectTimeout, host, hostPort, hostContext);
+ zkController = new ZkController(zookeeperHost, zkClientTimeout, zkClientConnectTimeout, host, hostPort, hostContext);
String confDir = System.getProperty("bootstrap_confdir");
if(confDir != null) {
@@ -133,7 +133,7 @@
throw new IllegalArgumentException("bootstrap_confdir must be a directory of configuration files");
}
String confName = System.getProperty("bootstrap_confname", "configuration1");
- zooKeeperController.uploadConfigDir(dir, confName);
+ zkController.uploadConfigDir(dir, confName);
}
} catch (InterruptedException e) {
// Restore the interrupted status
@@ -385,7 +385,7 @@
if (opt != null) {
p.setSchemaName(opt);
}
- if (zooKeeperController != null) {
+ if (zkController != null) {
opt = DOMUtil.getAttr(node, "shardId", null);
if (testShardIdOverride != null
&& name.equals("")) {
@@ -428,11 +428,11 @@
}
- if(zooKeeperController != null) {
+ if(zkController != null) {
try {
- synchronized (zooKeeperController) {
- zooKeeperController.addShardZkNodeWatches();
- zooKeeperController.updateCloudState(true);
+ synchronized (zkController) {
+ zkController.addShardZkNodeWatches();
+ zkController.updateCloudState(true);
}
} catch (InterruptedException e) {
// Restore the interrupted status
@@ -474,8 +474,8 @@
}
cores.clear();
} finally {
- if(zooKeeperController != null) {
- zooKeeperController.close();
+ if(zkController != null) {
+ zkController.close();
}
if (zkServer != null) {
zkServer.stop();
@@ -519,9 +519,9 @@
core.setName(name);
}
- if (zooKeeperController != null) {
+ if (zkController != null) {
try {
- zooKeeperController.register(core, true);
+ zkController.register(core.getName(), core.getCoreDescriptor().getCloudDescriptor(), true);
} catch (InterruptedException e) {
// Restore the interrupted status
Thread.currentThread().interrupt();
@@ -583,21 +583,21 @@
SolrConfig config = null;
String zkConfigName = null;
- if(zooKeeperController == null) {
+ if(zkController == null) {
solrLoader = new SolrResourceLoader(instanceDir, libLoader, getCoreProps(instanceDir, dcore.getPropertiesName(),dcore.getCoreProperties()));
config = new SolrConfig(solrLoader, dcore.getConfigName(), null);
} else {
try {
String collection = dcore.getCloudDescriptor().getCollectionName();
- zooKeeperController.createCollectionZkNode(collection);
- zkConfigName = zooKeeperController.readConfigName(collection);
+ zkController.createCollectionZkNode(collection);
+ zkConfigName = zkController.readConfigName(collection);
if (zkConfigName == null) {
log.error("Could not find config name for collection:" + collection);
throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
"Could not find config name for collection:" + collection);
}
- solrLoader = new ZkSolrResourceLoader(instanceDir, zkConfigName, libLoader, getCoreProps(instanceDir, dcore.getPropertiesName(),dcore.getCoreProperties()), zooKeeperController);
- config = zooKeeperController.getConfig(zkConfigName, dcore.getConfigName(), solrLoader);
+ solrLoader = new ZkSolrResourceLoader(instanceDir, zkConfigName, libLoader, getCoreProps(instanceDir, dcore.getPropertiesName(),dcore.getCoreProperties()), zkController);
+ config = getSolrConfigFromZk(zkConfigName, dcore.getConfigName(), solrLoader);
} catch (KeeperException e) {
log.error("", e);
throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
@@ -612,7 +612,7 @@
}
IndexSchema schema = null;
if (indexSchemaCache != null) {
- if (zooKeeperController != null) {
+ if (zkController != null) {
File schemaFile = new File(dcore.getSchemaName());
if (!schemaFile.isAbsolute()) {
schemaFile = new File(solrLoader.getInstanceDir() + "conf"
@@ -638,9 +638,9 @@
}
}
if(schema == null){
- if(zooKeeperController != null) {
+ if(zkController != null) {
try {
- schema = zooKeeperController.getSchema(zkConfigName, dcore.getSchemaName(), config, solrLoader);
+ schema = getSchemaFromZk(zkConfigName, dcore.getSchemaName(), config, solrLoader);
} catch (KeeperException e) {
log.error("", e);
throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
@@ -1006,10 +1006,41 @@
throw xforward;
}
}
-
+
public String getSolrHome() {
return solrHome;
}
+
+ public boolean isZooKeeperAware() {
+ return zkController != null;
+ }
+
+ public ZkController getZkController() {
+ return zkController;
+ }
+
+ private SolrConfig getSolrConfigFromZk(String zkConfigName, String solrConfigFileName,
+ SolrResourceLoader resourceLoader) throws IOException,
+ ParserConfigurationException, SAXException, KeeperException,
+ InterruptedException {
+ byte[] config = zkController.getConfigFileData(zkConfigName, solrConfigFileName);
+ InputStream is = new ByteArrayInputStream(config);
+ SolrConfig cfg = solrConfigFileName == null ? new SolrConfig(
+ resourceLoader, SolrConfig.DEFAULT_CONF_FILE, is) : new SolrConfig(
+ resourceLoader, solrConfigFileName, is);
+
+ return cfg;
+ }
+
+ private IndexSchema getSchemaFromZk(String zkConfigName, String schemaName,
+ SolrConfig config, SolrResourceLoader resourceLoader)
+ throws KeeperException, InterruptedException {
+ byte[] configBytes = zkController.getConfigFileData(zkConfigName, schemaName);
+ InputStream is = new ByteArrayInputStream(configBytes);
+ IndexSchema schema = new IndexSchema(config, schemaName, is);
+ return schema;
+ }
+
private static final String DEF_SOLR_XML ="<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n" +
"<solr persistent=\"false\">\n" +
" <cores adminPath=\"/admin/cores\">\n" +
@@ -1017,12 +1048,5 @@
" </cores>\n" +
"</solr>";
- public boolean isZooKeeperAware() {
- return zooKeeperController != null;
- }
-
- public ZkController getZooKeeperController() {
- return zooKeeperController;
- }
}
Modified: lucene/solr/branches/cloud/src/java/org/apache/solr/core/CoreDescriptor.java
URL: http://svn.apache.org/viewvc/lucene/solr/branches/cloud/src/java/org/apache/solr/core/CoreDescriptor.java?rev=906316&r1=906315&r2=906316&view=diff
==============================================================================
--- lucene/solr/branches/cloud/src/java/org/apache/solr/core/CoreDescriptor.java (original)
+++ lucene/solr/branches/cloud/src/java/org/apache/solr/core/CoreDescriptor.java Thu Feb 4 01:08:04 2010
@@ -43,11 +43,11 @@
this.coreContainer = coreContainer;
this.name = name;
- if(coreContainer.getZooKeeperController() != null) {
+ if(coreContainer.getZkController() != null) {
this.cloudDesc = new CloudDescriptor();
// cloud collection defaults to core name
cloudDesc.setCollectionName(name == "" ? coreContainer.getDefaultCoreName() : name);
- this.cloudDesc.setShardId(coreContainer.getZooKeeperController().getNodeName() + "_" + name);
+ this.cloudDesc.setShardId(coreContainer.getZkController().getNodeName() + "_" + name);
}
if (name == null) {
Modified: lucene/solr/branches/cloud/src/java/org/apache/solr/handler/component/QueryComponent.java
URL: http://svn.apache.org/viewvc/lucene/solr/branches/cloud/src/java/org/apache/solr/handler/component/QueryComponent.java?rev=906316&r1=906315&r2=906316&view=diff
==============================================================================
--- lucene/solr/branches/cloud/src/java/org/apache/solr/handler/component/QueryComponent.java (original)
+++ lucene/solr/branches/cloud/src/java/org/apache/solr/handler/component/QueryComponent.java Thu Feb 4 01:08:04 2010
@@ -132,7 +132,7 @@
Map<String,Slice> slices = null;
CoreDescriptor coreDescriptor = req.getCore().getCoreDescriptor();
CloudDescriptor cloudDescriptor = coreDescriptor.getCloudDescriptor();
- ZkController zkController = coreDescriptor.getCoreContainer().getZooKeeperController();
+ ZkController zkController = coreDescriptor.getCoreContainer().getZkController();
if (shards != null) {
Modified: lucene/solr/branches/cloud/src/java/org/apache/solr/handler/component/QueryElevationComponent.java
URL: http://svn.apache.org/viewvc/lucene/solr/branches/cloud/src/java/org/apache/solr/handler/component/QueryElevationComponent.java?rev=906316&r1=906315&r2=906316&view=diff
==============================================================================
--- lucene/solr/branches/cloud/src/java/org/apache/solr/handler/component/QueryElevationComponent.java (original)
+++ lucene/solr/branches/cloud/src/java/org/apache/solr/handler/component/QueryElevationComponent.java Thu Feb 4 01:08:04 2010
@@ -173,7 +173,7 @@
boolean exists = false;
//nocommit: double check this how we want to handle this
// check if using ZooKeeper
- ZkController zooKeeperController = core.getCoreDescriptor().getCoreContainer().getZooKeeperController();
+ ZkController zooKeeperController = core.getCoreDescriptor().getCoreContainer().getZkController();
if(zooKeeperController != null) {
// TODO : shouldn't have to keep reading the config name when it has been read before
exists = zooKeeperController.configFileExists(zooKeeperController.readConfigName(core.getCoreDescriptor().getCloudDescriptor().getCollectionName()), f);
Modified: lucene/solr/branches/cloud/src/test/org/apache/solr/cloud/BasicZkTest.java
URL: http://svn.apache.org/viewvc/lucene/solr/branches/cloud/src/test/org/apache/solr/cloud/BasicZkTest.java?rev=906316&r1=906315&r2=906316&view=diff
==============================================================================
--- lucene/solr/branches/cloud/src/test/org/apache/solr/cloud/BasicZkTest.java (original)
+++ lucene/solr/branches/cloud/src/test/org/apache/solr/cloud/BasicZkTest.java Thu Feb 4 01:08:04 2010
@@ -29,7 +29,7 @@
// test using ZooKeeper
assertTrue("Not using ZooKeeper", h.getCoreContainer().isZooKeeperAware());
- ZkController zkController = h.getCoreContainer().getZooKeeperController();
+ ZkController zkController = h.getCoreContainer().getZkController();
// test merge factor picked up
SolrCore core = h.getCore();
Modified: lucene/solr/branches/cloud/src/test/org/apache/solr/cloud/CloudStateUpdateTest.java
URL: http://svn.apache.org/viewvc/lucene/solr/branches/cloud/src/test/org/apache/solr/cloud/CloudStateUpdateTest.java?rev=906316&r1=906315&r2=906316&view=diff
==============================================================================
--- lucene/solr/branches/cloud/src/test/org/apache/solr/cloud/CloudStateUpdateTest.java (original)
+++ lucene/solr/branches/cloud/src/test/org/apache/solr/cloud/CloudStateUpdateTest.java Thu Feb 4 01:08:04 2010
@@ -122,7 +122,7 @@
// slight pause - TODO: takes an oddly long amount of time to schedule tasks with almost no delay ...
Thread.sleep(5000);
- ZkController zkController2 = container2.getZooKeeperController();
+ ZkController zkController2 = container2.getZkController();
String host = zkController2.getHostName();
Modified: lucene/solr/branches/cloud/src/test/org/apache/solr/cloud/DistributedZkFailuresTest.java
URL: http://svn.apache.org/viewvc/lucene/solr/branches/cloud/src/test/org/apache/solr/cloud/DistributedZkFailuresTest.java?rev=906316&r1=906315&r2=906316&view=diff
==============================================================================
--- lucene/solr/branches/cloud/src/test/org/apache/solr/cloud/DistributedZkFailuresTest.java (original)
+++ lucene/solr/branches/cloud/src/test/org/apache/solr/cloud/DistributedZkFailuresTest.java Thu Feb 4 01:08:04 2010
@@ -101,7 +101,7 @@
@Override
public void doTest() throws Exception {
- ZkController zkController = h.getCoreContainer().getZooKeeperController();
+ ZkController zkController = h.getCoreContainer().getZkController();
del("*:*");
// nocommit : test too long
Modified: lucene/solr/branches/cloud/src/webapp/web/admin/zookeeper.jsp
URL: http://svn.apache.org/viewvc/lucene/solr/branches/cloud/src/webapp/web/admin/zookeeper.jsp?rev=906316&r1=906315&r2=906316&view=diff
==============================================================================
--- lucene/solr/branches/cloud/src/webapp/web/admin/zookeeper.jsp (original)
+++ lucene/solr/branches/cloud/src/webapp/web/admin/zookeeper.jsp Thu Feb 4 01:08:04 2010
@@ -101,7 +101,7 @@
this.addr = addr;
if (addr == null) {
- ZkController controller = core.getCoreDescriptor().getCoreContainer().getZooKeeperController();
+ ZkController controller = core.getCoreDescriptor().getCoreContainer().getZkController();
if (controller != null) {
// this core is zk enabled
keeperAddr = controller.getZkServerAddress();