You are viewing a plain text version of this content. The canonical link for it is here.
Posted to solr-commits@lucene.apache.org by ma...@apache.org on 2010/01/19 14:49:47 UTC
svn commit: r900779 - in /lucene/solr/branches/cloud: example/solr/
src/java/org/apache/solr/cloud/ src/java/org/apache/solr/core/
src/test/org/apache/solr/ src/test/org/apache/solr/cloud/
src/test/test-files/solr/
Author: markrmiller
Date: Tue Jan 19 13:49:47 2010
New Revision: 900779
URL: http://svn.apache.org/viewvc?rev=900779&view=rev
Log:
updates
Added:
lucene/solr/branches/cloud/src/java/org/apache/solr/cloud/NodesWatcher.java
lucene/solr/branches/cloud/src/java/org/apache/solr/cloud/Slice.java
lucene/solr/branches/cloud/src/java/org/apache/solr/cloud/ZkNodeProps.java
lucene/solr/branches/cloud/src/java/org/apache/solr/cloud/ZooKeeperException.java
lucene/solr/branches/cloud/src/test/org/apache/solr/cloud/ZkNodePropsTest.java
Removed:
lucene/solr/branches/cloud/src/java/org/apache/solr/cloud/CollectionState.java
lucene/solr/branches/cloud/src/java/org/apache/solr/cloud/ShardInfo.java
lucene/solr/branches/cloud/src/java/org/apache/solr/cloud/ShardInfoList.java
Modified:
lucene/solr/branches/cloud/example/solr/solr.xml
lucene/solr/branches/cloud/src/java/org/apache/solr/cloud/CloudDescriptor.java
lucene/solr/branches/cloud/src/java/org/apache/solr/cloud/CloudState.java
lucene/solr/branches/cloud/src/java/org/apache/solr/cloud/ConnectionManager.java
lucene/solr/branches/cloud/src/java/org/apache/solr/cloud/DefaultConnectionStrategy.java
lucene/solr/branches/cloud/src/java/org/apache/solr/cloud/ShardsWatcher.java
lucene/solr/branches/cloud/src/java/org/apache/solr/cloud/ZkController.java
lucene/solr/branches/cloud/src/java/org/apache/solr/core/CoreContainer.java
lucene/solr/branches/cloud/src/java/org/apache/solr/core/CoreDescriptor.java
lucene/solr/branches/cloud/src/test/org/apache/solr/BaseDistributedSearchTestCase.java
lucene/solr/branches/cloud/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java
lucene/solr/branches/cloud/src/test/org/apache/solr/cloud/ZkControllerTest.java
lucene/solr/branches/cloud/src/test/test-files/solr/solr.lowZkTimeout.xml
lucene/solr/branches/cloud/src/test/test-files/solr/solr.xml
Modified: lucene/solr/branches/cloud/example/solr/solr.xml
URL: http://svn.apache.org/viewvc/lucene/solr/branches/cloud/example/solr/solr.xml?rev=900779&r1=900778&r2=900779&view=diff
==============================================================================
--- lucene/solr/branches/cloud/example/solr/solr.xml (original)
+++ lucene/solr/branches/cloud/example/solr/solr.xml Tue Jan 19 13:49:47 2010
@@ -28,7 +28,7 @@
adminPath: RequestHandler path to manage cores.
If 'null' (or absent), cores will not be manageable via request handler
-->
- <cores adminPath="/admin/cores">
- <core name="DEFAULT_CORE" instanceDir="." />
+ <cores adminPath="/admin/cores" defaultCoreName="collection1">
+ <core name="collection1" instanceDir="." />
</cores>
</solr>
Modified: lucene/solr/branches/cloud/src/java/org/apache/solr/cloud/CloudDescriptor.java
URL: http://svn.apache.org/viewvc/lucene/solr/branches/cloud/src/java/org/apache/solr/cloud/CloudDescriptor.java?rev=900779&r1=900778&r2=900779&view=diff
==============================================================================
--- lucene/solr/branches/cloud/src/java/org/apache/solr/cloud/CloudDescriptor.java (original)
+++ lucene/solr/branches/cloud/src/java/org/apache/solr/cloud/CloudDescriptor.java Tue Jan 19 13:49:47 2010
@@ -18,8 +18,8 @@
*/
public class CloudDescriptor {
- // nocommit : zk
- private String shardList;
+
+ private String slice;
private String collectionName;
private String role = "none";
@@ -31,13 +31,12 @@
this.role = role;
}
- public void setShardList(String shardList) {
- this.shardList = shardList;
+ public void setSlice(String slice) {
+ this.slice = slice;
}
- //nocommit: may be null
- public String getShardList() {
- return shardList;
+ public String getSlice() {
+ return slice;
}
public String getCollectionName() {
Modified: lucene/solr/branches/cloud/src/java/org/apache/solr/cloud/CloudState.java
URL: http://svn.apache.org/viewvc/lucene/solr/branches/cloud/src/java/org/apache/solr/cloud/CloudState.java?rev=900779&r1=900778&r2=900779&view=diff
==============================================================================
--- lucene/solr/branches/cloud/src/java/org/apache/solr/cloud/CloudState.java (original)
+++ lucene/solr/branches/cloud/src/java/org/apache/solr/cloud/CloudState.java Tue Jan 19 13:49:47 2010
@@ -17,18 +17,27 @@
* limitations under the License.
*/
+import java.util.Collections;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
+// effectively immutable
public class CloudState {
- private Map<String,CollectionState> collectionStates = new HashMap<String,CollectionState>();
+ private Map<String,Slice> collectionStates = new HashMap<String,Slice>();
+ private final List<String> nodes = null;
- //nocommit
- public void addCollectionInfo(String collection, CollectionState collectionInfo) {
- collectionStates.put(collection, collectionInfo);
+ // nocommit
+ public void addSlice(String collection, Slice slice) {
+ collectionStates.put(collection, slice);
}
- public CollectionState getCollectionInfo(String collection) {
+ // nocommit
+ public Slice getSlice(String collection) {
return collectionStates.get(collection);
}
+
+ public List<String> getNodes() {
+ return Collections.unmodifiableList(nodes);
+ }
}
Modified: lucene/solr/branches/cloud/src/java/org/apache/solr/cloud/ConnectionManager.java
URL: http://svn.apache.org/viewvc/lucene/solr/branches/cloud/src/java/org/apache/solr/cloud/ConnectionManager.java?rev=900779&r1=900778&r2=900779&view=diff
==============================================================================
--- lucene/solr/branches/cloud/src/java/org/apache/solr/cloud/ConnectionManager.java (original)
+++ lucene/solr/branches/cloud/src/java/org/apache/solr/cloud/ConnectionManager.java Tue Jan 19 13:49:47 2010
@@ -102,7 +102,7 @@
log.info("Connected:" + connected);
} else if (state == KeeperState.Disconnected) {
// nocommit : not sure we have to reconnect on disconnect
- // ZooKeeper will recover when it can
+ // ZooKeeper will recover when it can it seems ??
connected = false;
} else {
Modified: lucene/solr/branches/cloud/src/java/org/apache/solr/cloud/DefaultConnectionStrategy.java
URL: http://svn.apache.org/viewvc/lucene/solr/branches/cloud/src/java/org/apache/solr/cloud/DefaultConnectionStrategy.java?rev=900779&r1=900778&r2=900779&view=diff
==============================================================================
--- lucene/solr/branches/cloud/src/java/org/apache/solr/cloud/DefaultConnectionStrategy.java (original)
+++ lucene/solr/branches/cloud/src/java/org/apache/solr/cloud/DefaultConnectionStrategy.java Tue Jan 19 13:49:47 2010
@@ -45,22 +45,30 @@
public void reconnect(final String serverAddress, final int zkClientTimeout,
final Watcher watcher, final ZkUpdate updater) throws IOException {
log.info("Starting reconnect to ZooKeeper attempts ...");
- executor.scheduleAtFixedRate(new Runnable() {
+ executor.schedule(new Runnable() {
+ private int delay = 1000;
public void run() {
log.info("Attempting the connect...");
+ boolean connected = false;
try {
updater.update(new ZooKeeper(serverAddress, zkClientTimeout, watcher));
// nocommit
log.info("Reconnected to ZooKeeper");
+ connected = true;
} catch (Exception e) {
// nocommit
e.printStackTrace();
log.info("Reconnect to ZooKeeper failed");
}
- executor.shutdownNow();
+ if(connected) {
+ executor.shutdownNow();
+ } else {
+ delay = delay * 2; // nocommit : back off retry that levels off
+ executor.schedule(this, delay, TimeUnit.MILLISECONDS);
+ }
}
- }, 0, 1000, TimeUnit.MILLISECONDS); // nocommit : we actually want to do backoff retry
+ }, 1000, TimeUnit.MILLISECONDS);
}
}
Added: lucene/solr/branches/cloud/src/java/org/apache/solr/cloud/NodesWatcher.java
URL: http://svn.apache.org/viewvc/lucene/solr/branches/cloud/src/java/org/apache/solr/cloud/NodesWatcher.java?rev=900779&view=auto
==============================================================================
--- lucene/solr/branches/cloud/src/java/org/apache/solr/cloud/NodesWatcher.java (added)
+++ lucene/solr/branches/cloud/src/java/org/apache/solr/cloud/NodesWatcher.java Tue Jan 19 13:49:47 2010
@@ -0,0 +1,64 @@
+package org.apache.solr.cloud;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+import java.io.IOException;
+
+import org.apache.solr.common.SolrException;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.Watcher.Event.EventType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class NodesWatcher implements Watcher {
+ private static Logger log = LoggerFactory.getLogger(NodesWatcher.class);
+
+ private ZkController controller;
+
+ public NodesWatcher(ZkController controller) {
+ this.controller = controller;
+ }
+
+ public void process(WatchedEvent event) {
+ if (event.getType() == EventType.NodeChildrenChanged) {
+
+ try {
+
+ // nocommit : rewatch
+ controller.getZkClient().exists(event.getPath(), this);
+
+ // nocommit : update node list
+
+ } catch (KeeperException e) {
+ log.error("", e);
+ throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
+ "", e);
+ } catch (InterruptedException e) {
+ // Restore the interrupted status
+ Thread.currentThread().interrupt();
+ log.error("", e);
+ throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
+ "", e);
+ }
+ }
+
+ }
+
+}
Modified: lucene/solr/branches/cloud/src/java/org/apache/solr/cloud/ShardsWatcher.java
URL: http://svn.apache.org/viewvc/lucene/solr/branches/cloud/src/java/org/apache/solr/cloud/ShardsWatcher.java?rev=900779&r1=900778&r2=900779&view=diff
==============================================================================
--- lucene/solr/branches/cloud/src/java/org/apache/solr/cloud/ShardsWatcher.java (original)
+++ lucene/solr/branches/cloud/src/java/org/apache/solr/cloud/ShardsWatcher.java Tue Jan 19 13:49:47 2010
@@ -23,6 +23,7 @@
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.WatchedEvent;
import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.Watcher.Event.EventType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -42,30 +43,31 @@
public void process(WatchedEvent event) {
// nocommit : this will be called too often as shards register themselves?
System.out.println("shard node changed");
-
+ if (event.getType() == EventType.NodeChildrenChanged) {
- try {
- // nocommit:
- controller.printLayoutToStdOut();
- // nocommit : refresh watcher
-
- // nocommit : rewatch
- controller.getZkClient().exists(event.getPath(), this);
-
- // nocommit : just see what has changed
- controller.readCloudInfo();
-
- } catch (KeeperException e) {
- log.error("", e);
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
- "ZooKeeper Exception", e);
- } catch (InterruptedException e) {
- // Restore the interrupted status
- Thread.currentThread().interrupt();
- } catch (IOException e) {
- log.error("", e);
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
- "IOException", e);
+ try {
+
+ // nocommit : rewatch
+ controller.getZkClient().exists(event.getPath(), this);
+
+ // nocommit : just see what has changed
+ controller.updateCloudState();
+
+ } catch (KeeperException e) {
+ log.error("", e);
+ throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
+ "", e);
+ } catch (InterruptedException e) {
+ // Restore the interrupted status
+ Thread.currentThread().interrupt();
+ log.error("", e);
+ throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
+ "", e);
+ } catch (IOException e) {
+ log.error("", e);
+ throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
+ "", e);
+ }
}
}
Added: lucene/solr/branches/cloud/src/java/org/apache/solr/cloud/Slice.java
URL: http://svn.apache.org/viewvc/lucene/solr/branches/cloud/src/java/org/apache/solr/cloud/Slice.java?rev=900779&view=auto
==============================================================================
--- lucene/solr/branches/cloud/src/java/org/apache/solr/cloud/Slice.java (added)
+++ lucene/solr/branches/cloud/src/java/org/apache/solr/cloud/Slice.java Tue Jan 19 13:49:47 2010
@@ -0,0 +1,34 @@
+package org.apache.solr.cloud;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.Collections;
+import java.util.Map;
+
+// effectively immutable
+public class Slice {
+ private final Map<String,ZkNodeProps> shards;
+
+ public Slice(Map<String,ZkNodeProps> shards) {
+ this.shards = shards;
+ }
+
+ public Map<String,ZkNodeProps> getShards() {
+ return Collections.unmodifiableMap(shards);
+ }
+}
Modified: lucene/solr/branches/cloud/src/java/org/apache/solr/cloud/ZkController.java
URL: http://svn.apache.org/viewvc/lucene/solr/branches/cloud/src/java/org/apache/solr/cloud/ZkController.java?rev=900779&r1=900778&r2=900779&view=diff
==============================================================================
--- lucene/solr/branches/cloud/src/java/org/apache/solr/cloud/ZkController.java (original)
+++ lucene/solr/branches/cloud/src/java/org/apache/solr/cloud/ZkController.java Tue Jan 19 13:49:47 2010
@@ -19,17 +19,16 @@
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.net.InetAddress;
import java.util.Collections;
import java.util.HashMap;
-import java.util.HashSet;
import java.util.List;
import java.util.Map;
-import java.util.Properties;
-import java.util.Set;
import java.util.concurrent.TimeoutException;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -69,13 +68,14 @@
private final static Pattern URL_PREFIX = Pattern.compile("(https?://).*");
// package private for tests
- static final String CORE_ZKPREFIX = "/core";
- static final String SHARDS_ZKNODE = "/shards";
+ static final String SLICES_ZKNODE = "/slices";
// nocommit : ok to be public? for corecontainer access
public static final String CONFIGS_ZKNODE = "/configs";
static final String COLLECTIONS_ZKNODE = "/collections";
+ static final String NODES_ZKNODE = "/nodes";
- static final String PROPS_DESC = "CoreDesc";
+ static final String URL_PROP = "url";
+ static final String ROLE_PROP = "role";
final ShardsWatcher shardWatcher = new ShardsWatcher(this);
@@ -92,6 +92,7 @@
private String hostName;
+
/**
*
* @param zkServerAddress ZooKeeper server host address
@@ -118,21 +119,24 @@
public void command() {
try {
-
- // nocommit : re-register ephemeral nodes, wait a while
+ // nocommit : re-register ephemeral nodes, (possibly) wait a while
// for others to do the same, then load
- readCloudInfo();
+ createEphemeralNode();
+ updateCloudState();
} catch (KeeperException e) {
- log.error("ZooKeeper Exception", e);
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
- "ZooKeeper Exception", e);
+ log.error("", e);
+ throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
+ "", e);
} catch (InterruptedException e) {
// Restore the interrupted status
Thread.currentThread().interrupt();
+ log.error("", e);
+ throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
+ "", e);
} catch (IOException e) {
log.error("", e);
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "",
- e);
+ throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
+ "", e);
}
}
@@ -144,10 +148,12 @@
/**
* nocommit: adds nodes if they don't exist, eg /shards/ node. consider race
* conditions
+ * @param collection2
*/
- private void addZkCoresNode(String collection) throws IOException {
+ private void addZkShardsNode(String slice, String collection) throws IOException {
- String shardsZkPath = COLLECTIONS_ZKNODE + "/" + collection + SHARDS_ZKNODE;
+ String shardsZkPath = COLLECTIONS_ZKNODE + "/" + collection + SLICES_ZKNODE + "/" + slice;
+
try {
// shards node
if (!zkClient.exists(shardsZkPath)) {
@@ -157,31 +163,22 @@
// makes shards zkNode if it doesn't exist
zkClient.makePath(shardsZkPath, CreateMode.PERSISTENT, null);
- // ping that there is a new collection
+ // ping that there is a new collection (nocommit : or now possibly a new slice)
zkClient.setData(COLLECTIONS_ZKNODE, (byte[])null);
}
} catch (KeeperException e) {
// its okay if another beats us creating the node
if (e.code() != KeeperException.Code.NODEEXISTS) {
- log.error("ZooKeeper Exception", e);
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
- "ZooKeeper Exception", e);
+ log.error("", e);
+ throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
+ "", e);
}
} catch (InterruptedException e) {
// Restore the interrupted status
Thread.currentThread().interrupt();
- }
-
- // now watch the shards node
- try {
- zkClient.getChildren(shardsZkPath, shardWatcher);
- } catch (KeeperException e) {
- log.error("ZooKeeper Exception", e);
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
- "ZooKeeper Exception", e);
- } catch (InterruptedException e) {
- // Restore the interrupted status
- Thread.currentThread().interrupt();
+ log.error("", e);
+ throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
+ "", e);
}
}
@@ -194,6 +191,9 @@
} catch (InterruptedException e) {
// Restore the interrupted status
Thread.currentThread().interrupt();
+ log.error("", e);
+ throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
+ "", e);
}
}
@@ -287,6 +287,10 @@
return localHost;
}
+
+ public String getHostName() {
+ return hostName;
+ }
/**
* Load IndexSchema from ZooKeeper.
@@ -311,26 +315,6 @@
return schema;
}
- // nocommit - testing
-// public String getSearchNodes(String collection) {
-// StringBuilder nodeString = new StringBuilder();
-// boolean first = true;
-// List<String> nodes;
-//
-// nodes = cloudInfo.getCollectionInfo(collection).getSearchShards();
-// // nocommit
-// System.out.println("there are " + nodes.size() + " node(s)");
-// for (String node : nodes) {
-// nodeString.append(node);
-// if (first) {
-// first = false;
-// } else {
-// nodeString.append(',');
-// }
-// }
-// return nodeString.toString();
-// }
-
SolrZkClient getZkClient() {
return zkClient;
}
@@ -360,22 +344,20 @@
// makes nodes node
try {
- zkClient.makePath("/nodes");
+ zkClient.makePath(NODES_ZKNODE);
} catch (KeeperException e) {
// its okay if another beats us creating the node
if (e.code() != KeeperException.Code.NODEEXISTS) {
- log.error("ZooKeeper Exception", e);
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
- "ZooKeeper Exception", e);
+ log.error("", e);
+ throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
+ "", e);
}
}
- String nodeName = hostName + ":" + localHostPort + "_"+ localHostContext;
- zkClient.makePath("/nodes/" + nodeName, CreateMode.EPHEMERAL);
+ createEphemeralNode();
// nocommit
setUpCollectionsNode();
-
} catch (IOException e) {
log.error("", e);
@@ -384,18 +366,29 @@
} catch (InterruptedException e) {
// Restore the interrupted status
Thread.currentThread().interrupt();
+ log.error("", e);
+ throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
+ "", e);
} catch (KeeperException e) {
- log.error("KeeperException", e);
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "", e);
+ log.error("", e);
+ throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
+ "", e);
}
}
+ private void createEphemeralNode() throws KeeperException,
+ InterruptedException {
+ String nodeName = hostName + ":" + localHostPort + "_"+ localHostContext;
+ zkClient.makePath(NODES_ZKNODE + "/" + nodeName, CreateMode.EPHEMERAL);
+ }
+
// load and publish a new CollectionInfo
- public void readCloudInfo() throws KeeperException, InterruptedException,
+ public synchronized void updateCloudState() throws KeeperException, InterruptedException,
IOException {
- // nocommit : not thread safe anymore ?
+ // nocommit - incremental update rather than reread everything
+
log.info("Updating cloud state from ZooKeeper... :" + zkClient.keeper);
// build immutable CloudInfo
@@ -403,14 +396,14 @@
List<String> collections = getCollectionNames();
// nocommit : load all collection info
for (String collection : collections) {
- String shardsZkPath = COLLECTIONS_ZKNODE + "/" + collection
- + SHARDS_ZKNODE;
+ String slicePaths = COLLECTIONS_ZKNODE + "/" + collection + SLICES_ZKNODE;
+ List<String> sliceNames = zkClient.getChildren(slicePaths, null);
+ for(String sliceZkPath : sliceNames) {
+ Map<String,ZkNodeProps> shards = readShards(slicePaths + "/" + sliceZkPath);
+ Slice slice = new Slice(shards);
+ cloudInfo.addSlice(collection, slice);
+ }
- List<String> nodes = zkClient.getChildren(shardsZkPath, null);
- Map<String,Properties> shards = readShardsInfo(collection, shardsZkPath, nodes);
-
- CollectionState collectionInfo = new CollectionState(shards, nodes);
- cloudInfo.addCollectionInfo(collection, collectionInfo);
}
// update volatile
@@ -491,37 +484,35 @@
}
/**
- * Read info on the available Shards and Nodes.
- *
- * @param path to the shards zkNode
- * @param shardsZkPath
- * @param nodeList
- * @return Map from shard name to a {@link ShardInfoList}
- * @throws InterruptedException
+ * @param zkClient
+ * @param shardsZkPath
+ * @return
* @throws KeeperException
+ * @throws InterruptedException
* @throws IOException
*/
- public Map<String,Properties> readShardsInfo(String collection, String path, List<String> nodes)
+ private Map<String,ZkNodeProps> readShards(String shardsZkPath)
throws KeeperException, InterruptedException, IOException {
- Set<String> nodesSet = new HashSet<String>(nodes.size());
- nodesSet.addAll(nodes);
- if(cloudInfo != null) {
- List<String> oldNodes = cloudInfo.getCollectionInfo(collection).getNodes();
- }
-
- Map<String,Properties> cores = new HashMap<String,Properties>();
-
- for (String zkNodeName : nodes) {
- byte[] data = zkClient.getData(path + "/" + zkNodeName, null, null);
- Properties props = new Properties();
- props.load(new ByteArrayInputStream(data));
+ Map<String,ZkNodeProps> shardNameToProps = new HashMap<String,ZkNodeProps>();
- cores.put(zkNodeName, props);
+ if (zkClient.exists(shardsZkPath, null) == null) {
+ throw new IllegalStateException("Cannot find zk shards node that should exist:"
+ + shardsZkPath);
+ }
+ List<String> shardZkPaths = zkClient.getChildren(shardsZkPath, null);
+
+ for(String shardPath : shardZkPaths) {
+ byte[] data = zkClient.getData(shardsZkPath + "/" + shardPath, null,
+ null);
+
+ ZkNodeProps props = new ZkNodeProps();
+ props.load(new DataInputStream(new ByteArrayInputStream(data)));
+ shardNameToProps.put(shardPath, props);
}
- return Collections.unmodifiableMap(cores);
+ return Collections.unmodifiableMap(shardNameToProps);
}
/**
@@ -550,30 +541,25 @@
String collection = cloudDesc.getCollectionName();
String nodePath = null;
- String shardsZkPath = COLLECTIONS_ZKNODE + "/" + collection + SHARDS_ZKNODE;
+ String shardsZkPath = COLLECTIONS_ZKNODE + "/" + collection + SLICES_ZKNODE + "/" + cloudDesc.getSlice();
// build layout if not exists
// nocommit : consider how we watch shards on all collections
- addZkCoresNode(collection);
+ addZkShardsNode(cloudDesc.getSlice(), collection);
// create node
ByteArrayOutputStream baos = new ByteArrayOutputStream();
// nocommit: could do xml
- Properties props = new Properties();
- props.put(CollectionState.URL_PROP, shardUrl);
+ ZkNodeProps props = new ZkNodeProps();
+ props.put(URL_PROP, shardUrl);
- String shardList = cloudDesc.getShardList();
+ props.put(ROLE_PROP, cloudDesc.getRole());
- props.put(CollectionState.SHARD_LIST_PROP, shardList == null ? ""
- : shardList);
+ props.store(new DataOutputStream(baos));
- props.put(CollectionState.ROLE_PROP, cloudDesc.getRole());
-
- props.store(baos, PROPS_DESC);
-
- String nodeName = hostName + ":" + localHostPort + "_"+ localHostContext + (coreName.length() == 0 ? "" : "_" + coreName);
+ String shardZkNodeName = hostName + ":" + localHostPort + "_"+ localHostContext + (coreName.length() == 0 ? "" : "_" + coreName);
try {
- nodePath = zkClient.create(shardsZkPath + "/" + nodeName,
+ nodePath = zkClient.create(shardsZkPath + "/" + shardZkNodeName,
baos.toByteArray(), CreateMode.PERSISTENT);
} catch (KeeperException e) {
// its okay if the node already exists
@@ -583,6 +569,7 @@
}
// signal that the shards node has changed
+ // nocommit
zkClient.setData(shardsZkPath, (byte[])null);
return nodePath;
@@ -593,23 +580,7 @@
* @param zkNodePath
*/
public void unregister(SolrCore core, String zkNodePath) {
- // nocommit : version?
- try {
- zkClient.delete(zkNodePath, -1);
- } catch (InterruptedException e) {
- // Restore the interrupted status
- Thread.currentThread().interrupt();
- } catch (KeeperException.NoNodeException e) {
- // nocommit - this is okay - for some reason the node is already gone
- log.warn("Unregistering core: " + core.getName()
- + " but core's ZooKeeper node has already been removed");
- } catch (KeeperException e) {
- log.error("ZooKeeper Exception", e);
- // we can't get through to ZooKeeper, so log error
- // and allow close process to continue -
- // if ZooKeeper is down, our ephemeral node
- // should be removed anyway
- }
+ // nocommit : perhaps mark the core down in zk?
}
/**
@@ -650,7 +621,9 @@
collections = zkClient.getChildren(COLLECTIONS_ZKNODE, null);
for(String collection : collections) {
- zkClient.exists(COLLECTIONS_ZKNODE + "/" + collection + "/shards", shardWatcher);
+ for(String slice : zkClient.getChildren(COLLECTIONS_ZKNODE + "/" + collection + SLICES_ZKNODE, null)) {
+ zkClient.getChildren(COLLECTIONS_ZKNODE + "/" + collection + SLICES_ZKNODE + "/" + slice, shardWatcher);
+ }
}
}
@@ -666,13 +639,16 @@
} catch (KeeperException e) {
// its okay if another beats us creating the node
if (e.code() != KeeperException.Code.NODEEXISTS) {
- log.error("ZooKeeper Exception", e);
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
- "ZooKeeper Exception", e);
+ log.error("", e);
+ throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
+ "", e);
}
} catch (InterruptedException e) {
// Restore the interrupted status
Thread.currentThread().interrupt();
+ log.error("", e);
+ throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
+ "", e);
}
log.info("Start watching collections node for changes");
@@ -686,14 +662,17 @@
// re-watch
try {
- zkClient.exists(COLLECTIONS_ZKNODE, this);
+ zkClient.exists(event.getPath(), this);
} catch (KeeperException e) {
- log.error("ZooKeeper Exception", e);
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
- "ZooKeeper Exception", e);
+ log.error("", e);
+ throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
+ "", e);
} catch (InterruptedException e) {
// Restore the interrupted status
Thread.currentThread().interrupt();
+ log.error("", e);
+ throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
+ "", e);
}
}
Added: lucene/solr/branches/cloud/src/java/org/apache/solr/cloud/ZkNodeProps.java
URL: http://svn.apache.org/viewvc/lucene/solr/branches/cloud/src/java/org/apache/solr/cloud/ZkNodeProps.java?rev=900779&view=auto
==============================================================================
--- lucene/solr/branches/cloud/src/java/org/apache/solr/cloud/ZkNodeProps.java (added)
+++ lucene/solr/branches/cloud/src/java/org/apache/solr/cloud/ZkNodeProps.java Tue Jan 19 13:49:47 2010
@@ -0,0 +1,33 @@
+package org.apache.solr.cloud;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Set;
+
+public class ZkNodeProps extends HashMap<String,String> {
+
+ private static final long serialVersionUID = 1L;
+
+ public void load(DataInputStream in) throws IOException {
+ String stringRep = in.readUTF();
+ String[] lines = stringRep.split("\n");
+ for(String line : lines) {
+ int sepIndex = line.indexOf('=');
+ String key = line.substring(0, sepIndex);
+ String value = line.substring(sepIndex + 1, line.length());
+ put(key, value);
+ }
+ }
+
+ public void store(DataOutputStream out) throws IOException {
+ StringBuilder sb = new StringBuilder();
+ Set<String> keys = keySet();
+ for(String key : keys) {
+ String value = get(key);
+ sb.append(key + "=" + value + "\n");
+ }
+ out.writeUTF(sb.toString());
+ }
+}
Added: lucene/solr/branches/cloud/src/java/org/apache/solr/cloud/ZooKeeperException.java
URL: http://svn.apache.org/viewvc/lucene/solr/branches/cloud/src/java/org/apache/solr/cloud/ZooKeeperException.java?rev=900779&view=auto
==============================================================================
--- lucene/solr/branches/cloud/src/java/org/apache/solr/cloud/ZooKeeperException.java (added)
+++ lucene/solr/branches/cloud/src/java/org/apache/solr/cloud/ZooKeeperException.java Tue Jan 19 13:49:47 2010
@@ -0,0 +1,30 @@
+package org.apache.solr.cloud;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+import org.apache.solr.common.SolrException;
+
+public class ZooKeeperException extends SolrException {
+
+ public ZooKeeperException(ErrorCode code, String msg, Throwable th) {
+ super(code, msg, th);
+
+ }
+
+}
Modified: lucene/solr/branches/cloud/src/java/org/apache/solr/core/CoreContainer.java
URL: http://svn.apache.org/viewvc/lucene/solr/branches/cloud/src/java/org/apache/solr/core/CoreContainer.java?rev=900779&r1=900778&r2=900779&view=diff
==============================================================================
--- lucene/solr/branches/cloud/src/java/org/apache/solr/core/CoreContainer.java (original)
+++ lucene/solr/branches/cloud/src/java/org/apache/solr/core/CoreContainer.java Tue Jan 19 13:49:47 2010
@@ -34,6 +34,7 @@
import org.apache.solr.cloud.ZkSolrResourceLoader;
import org.apache.solr.cloud.ZkController;
+import org.apache.solr.cloud.ZooKeeperException;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.CoreAdminParams;
import org.apache.solr.common.util.DOMUtil;
@@ -54,7 +55,7 @@
*/
public class CoreContainer
{
- static final String DEFAULT_CORE_NAME = "DEFAULT_CORE";
+ private static final String DEFAULT_DEFAULT_CORE_NAME = "collection1";
protected static Logger log = LoggerFactory.getLogger(CoreContainer.class);
@@ -79,10 +80,12 @@
protected String solrConfigFilenameOverride;
protected String solrDataDirOverride;
protected String zkPortOverride;
- private String testShardsListOverride;
+ private String testSliceOverride;
private ZkController zooKeeperController;
private String zkHost;
+
+ private String defaultCoreName = DEFAULT_DEFAULT_CORE_NAME;
public CoreContainer() {
@@ -105,31 +108,30 @@
try {
zooKeeperController = new ZkController(zookeeperHost, zkClientTimeout, host, hostPort, hostContext);
- String confDir = System.getProperty("bootstrapConfDir");
+ String confDir = System.getProperty("bootstrap_confdir");
if(confDir != null) {
File dir = new File(confDir);
if(!dir.isDirectory()) {
- throw new IllegalArgumentException("bootstrap conf dir must be directory");
+ throw new IllegalArgumentException("bootstrap_confdir must be a directory of configuration files");
}
- String confName = System.getProperty("bootstrapConfName", "conf1");
+ String confName = System.getProperty("bootstrap_confname", "configuration1");
zooKeeperController.uploadDirToCloud(dir, ZkController.CONFIGS_ZKNODE + confName);
-
}
-
} catch (InterruptedException e) {
// Restore the interrupted status
Thread.currentThread().interrupt();
} catch (TimeoutException e) {
log.error("Could not connect to ZooKeeper", e);
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
- "Could not connect to ZooKeeper", e);
+ log.error("Could not connect to ZooKeeper", e);
+ throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
+ "", e);
} catch (IOException e) {
log.error("", e);
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+ throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
"", e);
} catch (KeeperException e) {
log.error("", e);
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+ throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
"", e);
}
}
@@ -183,7 +185,7 @@
cores = new CoreContainer();
cores.solrDataDirOverride = dataDir;
cores.zkPortOverride = zkPortOverride;
- cores.testShardsListOverride = testShardListOverride;
+ cores.testSliceOverride = testShardListOverride;
if (fconf.exists())
cores.load(solrHome, fconf);
else {
@@ -288,7 +290,10 @@
solrHome = loader.getInstanceDir();
try {
Config cfg = new Config(loader, null, cfgis, null);
-
+ String dcoreName = cfg.get("solr/@defaultCoreName", null);
+ if(dcoreName != null) {
+ defaultCoreName = dcoreName;
+ }
persistent = cfg.getBool("solr/@persistent", false);
libDir = cfg.get("solr/@sharedLib", null);
zkHost = cfg.get("solr/@zkHost" , null);
@@ -303,9 +308,6 @@
hostContext = cfg.get("solr/cores/@hostContext", "solr");
host = cfg.get("solr/cores/@host", null);
- // nocommit read from core
- //collection = cfg.get("solr/cores/@collection", "collection1"); //nocommit: default collection
-
if(shareSchema){
indexSchemaCache = new ConcurrentHashMap<String ,IndexSchema>();
}
@@ -341,7 +343,7 @@
Node node = nodes.item(i);
try {
String name = DOMUtil.getAttr(node, "name", null);
- if(name.equals(DEFAULT_CORE_NAME)){
+ if(name.equals(defaultCoreName)){
if(defaultCoreFound) throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,"Only one 'DEFAULT_CORE' is allowed ");
defaultCoreFound = true;
name="";
@@ -360,11 +362,15 @@
if (opt != null) {
p.setSchemaName(opt);
}
- opt = DOMUtil.getAttr(node, "shardList", null);
- if(testShardsListOverride != null && name.equals("")) {
- p.getCloudDescriptor().setShardList(testShardsListOverride);
- } else if(opt != null) {
- p.getCloudDescriptor().setShardList(opt);
+ // nocommit : default shard list to SHARD: + host:port + context + core
+ opt = DOMUtil.getAttr(node, "slice", null);
+ if(testSliceOverride != null && name.equals("")) {
+ p.getCloudDescriptor().setSlice(testSliceOverride);
+ } else if(zooKeeperController != null) {
+ if(opt == null) {
+ opt = "SLICE:" + zooKeeperController.getHostName() + ":" + hostPort + "_" + hostContext + "_" + (name.length() == 0 ? "" : "_" + name);
+ }
+ p.getCloudDescriptor().setSlice(opt);
}
opt = DOMUtil.getAttr(node, "role", null);
if(opt != null) {
@@ -403,20 +409,23 @@
if(zooKeeperController != null) {
// nocommit : exceptions
try {
- zooKeeperController.readCloudInfo();
+ zooKeeperController.updateCloudState();
// nocommit : set shards node watches
zooKeeperController.watchShards();
} catch (InterruptedException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
+ // Restore the interrupted status
+ Thread.currentThread().interrupt();
+ log.error("", e);
+ throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
+ "", e);
} catch (KeeperException e) {
- log.error("ZooKeeper Exception", e);
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
- "ZooKeeper Exception", e);
+ log.error("", e);
+ throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
+ "", e);
} catch (IOException e) {
log.error("", e);
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+ throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
"", e);
}
}
@@ -541,13 +550,15 @@
solrLoader = new ZkSolrResourceLoader(instanceDir, zkConfigName, libLoader, getCoreProps(instanceDir, dcore.getPropertiesName(),dcore.getCoreProperties()), zooKeeperController);
config = zooKeeperController.getConfig(zkConfigName, dcore.getConfigName(), solrLoader);
} catch (KeeperException e) {
- log.error("ZooKeeper Exception", e);
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
- "ZooKeeper Exception", e);
+ log.error("", e);
+ throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
+ "", e);
} catch (InterruptedException e) {
// Restore the interrupted status
Thread.currentThread().interrupt();
- //nocommit: not good - we can't continue
+ log.error("", e);
+ throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
+ "", e);
}
}
IndexSchema schema = null;
@@ -582,16 +593,17 @@
if(schema == null){
if(zooKeeperController != null) {
try {
- System.out.println("config:" + zkConfigName);
schema = zooKeeperController.getSchema(zkConfigName, dcore.getSchemaName(), config, solrLoader);
} catch (KeeperException e) {
- log.error("ZooKeeper Exception", e);
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
- "ZooKeeper Exception", e);
+ log.error("", e);
+ throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
+ "", e);
} catch (InterruptedException e) {
// Restore the interrupted status
Thread.currentThread().interrupt();
- //nocommit: we may not know have the schema - now what
+ log.error("", e);
+ throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
+ "", e);
}
} else {
schema = new IndexSchema(config, dcore.getSchemaName(), null);
@@ -669,7 +681,7 @@
}
private String checkDefault(String name) {
- return name.length() == 0 || DEFAULT_CORE_NAME.equals(name) || name.trim().length() == 0 ? "" : name;
+ return name.length() == 0 || defaultCoreName.equals(name) || name.trim().length() == 0 ? "" : name;
}
/**
@@ -784,6 +796,10 @@
this.managementPath = path;
}
+ public String getDefaultCoreName() {
+ return defaultCoreName;
+ }
+
public File getConfigFile() {
return configFile;
}
@@ -950,7 +966,7 @@
private static final String DEF_SOLR_XML ="<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n" +
"<solr persistent=\"false\">\n" +
" <cores adminPath=\"/admin/cores\">\n" +
- " <core name=\""+ DEFAULT_CORE_NAME + "\" instanceDir=\".\" />\n" +
+ " <core name=\""+ DEFAULT_DEFAULT_CORE_NAME + "\" instanceDir=\".\" />\n" +
" </cores>\n" +
"</solr>";
Modified: lucene/solr/branches/cloud/src/java/org/apache/solr/core/CoreDescriptor.java
URL: http://svn.apache.org/viewvc/lucene/solr/branches/cloud/src/java/org/apache/solr/core/CoreDescriptor.java?rev=900779&r1=900778&r2=900779&view=diff
==============================================================================
--- lucene/solr/branches/cloud/src/java/org/apache/solr/core/CoreDescriptor.java (original)
+++ lucene/solr/branches/cloud/src/java/org/apache/solr/core/CoreDescriptor.java Tue Jan 19 13:49:47 2010
@@ -45,7 +45,7 @@
this.name = name;
// cloud collection defaults to core name
- this.cloudDesc.setCollectionName(name == "" ? CoreContainer.DEFAULT_CORE_NAME : name);
+ this.cloudDesc.setCollectionName(name == "" ? coreContainer.getDefaultCoreName() : name);
if (name == null) {
throw new RuntimeException("Core needs a name");
}
Modified: lucene/solr/branches/cloud/src/test/org/apache/solr/BaseDistributedSearchTestCase.java
URL: http://svn.apache.org/viewvc/lucene/solr/branches/cloud/src/test/org/apache/solr/BaseDistributedSearchTestCase.java?rev=900779&r1=900778&r2=900779&view=diff
==============================================================================
--- lucene/solr/branches/cloud/src/test/org/apache/solr/BaseDistributedSearchTestCase.java (original)
+++ lucene/solr/branches/cloud/src/test/org/apache/solr/BaseDistributedSearchTestCase.java Tue Jan 19 13:49:47 2010
@@ -192,8 +192,8 @@
return createJetty(baseDir, dataDirName, null, null);
}
- public JettySolrRunner createJetty(File baseDir, String dataDirName, String shardList) throws Exception {
- return createJetty(baseDir, dataDirName, shardList, null);
+ public JettySolrRunner createJetty(File baseDir, String dataDirName, String slice) throws Exception {
+ return createJetty(baseDir, dataDirName, slice, null);
}
public JettySolrRunner createJetty(File baseDir, String dataDirName, String shardList, String solrConfigOverride) throws Exception {
Modified: lucene/solr/branches/cloud/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java
URL: http://svn.apache.org/viewvc/lucene/solr/branches/cloud/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java?rev=900779&r1=900778&r2=900779&view=diff
==============================================================================
--- lucene/solr/branches/cloud/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java (original)
+++ lucene/solr/branches/cloud/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java Tue Jan 19 13:49:47 2010
@@ -57,13 +57,13 @@
String invalidField="invalid_field_not_in_schema";
protected void createServers(int numShards) throws Exception {
- controlJetty = createJetty(testDir, "control", "shard2");
+ controlJetty = createJetty(testDir, "control", "control_slice");
controlClient = createNewSolrServer(controlJetty.getLocalPort());
StringBuilder sb = new StringBuilder();
for (int i = 1; i <= numShards; i++) {
if (sb.length() > 0) sb.append(',');
- JettySolrRunner j = createJetty(testDir, "shard" + i, "shard" + (i + 2));
+ JettySolrRunner j = createJetty(testDir, "jetty" + i, "slice" + (i + 2));
jettys.add(j);
clients.add(createNewSolrServer(j.getLocalPort()));
sb.append("localhost:").append(j.getLocalPort()).append(context);
@@ -129,7 +129,7 @@
query("q","*:*", "sort",f+" asc");
}
- h.getCoreContainer().getCore("DEFAULT_CORE").close();
+ h.getCoreContainer().getCore(h.getCoreContainer().getDefaultCoreName()).close();
CoreDescriptor dcore= new CoreDescriptor( h.getCoreContainer(), "testcore", "testcore");
h.getCoreContainer().create(dcore);
Modified: lucene/solr/branches/cloud/src/test/org/apache/solr/cloud/ZkControllerTest.java
URL: http://svn.apache.org/viewvc/lucene/solr/branches/cloud/src/test/org/apache/solr/cloud/ZkControllerTest.java?rev=900779&r1=900778&r2=900779&view=diff
==============================================================================
--- lucene/solr/branches/cloud/src/test/org/apache/solr/cloud/ZkControllerTest.java (original)
+++ lucene/solr/branches/cloud/src/test/org/apache/solr/cloud/ZkControllerTest.java Tue Jan 19 13:49:47 2010
@@ -18,27 +18,18 @@
*/
import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
import java.io.File;
import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Set;
-import java.util.Map.Entry;
import junit.framework.TestCase;
-import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
public class ZkControllerTest extends TestCase {
private static final String COLLECTION_NAME = "collection1";
- private static final String SHARD2 = "shard2";
-
- private static final String SHARD1 = "shard1";
-
static final String ZOO_KEEPER_ADDRESS = "localhost:2181/solr";
static final String ZOO_KEEPER_HOST = "localhost:2181";
@@ -66,16 +57,16 @@
AbstractZkTestCase.makeSolrZkNode();
zkClient = new SolrZkClient(ZOO_KEEPER_ADDRESS, TIMEOUT);
- String shardsPath = "/collections/collection1/shards";
+ String shardsPath = "/collections/collection1/slices/slice1";
zkClient.makePath(shardsPath);
zkClient.makePath("collections/collection1/config=collection1");
- addShardToZk(zkClient, shardsPath, URL1, SHARD1 + "," + SHARD2);
+ addShardToZk(zkClient, shardsPath, URL1, "slave");
addShardToZk(zkClient, shardsPath, "http://localhost:3123/solr/core1",
- SHARD1);
+ "master");
addShardToZk(zkClient, shardsPath, "http://localhost:3133/solr/core1",
- SHARD1);
+ "slave");
if (DEBUG) {
zkClient.printLayoutToStdOut();
@@ -83,15 +74,15 @@
zkController = new ZkController(ZOO_KEEPER_ADDRESS, TIMEOUT,
"localhost", "8983", "/solr");
- zkController.readCloudInfo();
+ zkController.updateCloudState();
CloudState cloudInfo = zkController.getCloudInfo();
- CollectionState collectionInfo = cloudInfo.getCollectionInfo("collection1");
- assertNotNull(collectionInfo);
+ Slice slice = cloudInfo.getSlice("collection1");
+ assertNotNull(slice);
if (DEBUG) {
- for (String node : collectionInfo.getNodes()) {
- System.out.println("shard:" + node);
+ for (String shard : slice.getShards().keySet()) {
+ System.out.println("shard:" + shard);
}
}
@@ -169,17 +160,18 @@
}
private void addShardToZk(SolrZkClient zkClient, String shardsPath,
- String url, String shardList) throws IOException, KeeperException,
+ String url, String role) throws IOException, KeeperException,
InterruptedException {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
- // nocommit: could do xml
- Properties props = new Properties();
- props.put(CollectionState.URL_PROP, url);
- props.put(CollectionState.SHARD_LIST_PROP, shardList);
- props.store(baos, ZkController.PROPS_DESC);
- zkClient.create(shardsPath + ZkController.CORE_ZKPREFIX,
- baos.toByteArray(), CreateMode.EPHEMERAL_SEQUENTIAL);
+ ZkNodeProps props = new ZkNodeProps();
+ props.put(ZkController.URL_PROP, url);
+ props.put(ZkController.ROLE_PROP, role);
+ props.store(new DataOutputStream(baos));
+
+ //nocommit : fix
+// zkClient.create(shardsPath + ZkController.CORE_ZKPREFIX,
+// baos.toByteArray(), CreateMode.EPHEMERAL_SEQUENTIAL);
}
Added: lucene/solr/branches/cloud/src/test/org/apache/solr/cloud/ZkNodePropsTest.java
URL: http://svn.apache.org/viewvc/lucene/solr/branches/cloud/src/test/org/apache/solr/cloud/ZkNodePropsTest.java?rev=900779&view=auto
==============================================================================
--- lucene/solr/branches/cloud/src/test/org/apache/solr/cloud/ZkNodePropsTest.java (added)
+++ lucene/solr/branches/cloud/src/test/org/apache/solr/cloud/ZkNodePropsTest.java Tue Jan 19 13:49:47 2010
@@ -0,0 +1,46 @@
+package org.apache.solr.cloud;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+
+import junit.framework.TestCase;
+
+
+public class ZkNodePropsTest extends TestCase {
+
+ public void testBasic() throws IOException {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+
+ ZkNodeProps props = new ZkNodeProps();
+ props.put("prop1", "value1");
+ props.put("prop2", "value2");
+ props.put("prop3", "value3");
+ props.store(new DataOutputStream(baos));
+
+ ZkNodeProps props2 = new ZkNodeProps();
+ props2.load(new DataInputStream(new ByteArrayInputStream(baos.toByteArray())));
+ assertEquals("value1", props2.get("prop1"));
+ assertEquals("value2", props2.get("prop2"));
+ assertEquals("value3", props2.get("prop3"));
+ }
+}
Modified: lucene/solr/branches/cloud/src/test/test-files/solr/solr.lowZkTimeout.xml
URL: http://svn.apache.org/viewvc/lucene/solr/branches/cloud/src/test/test-files/solr/solr.lowZkTimeout.xml?rev=900779&r1=900778&r2=900779&view=diff
==============================================================================
--- lucene/solr/branches/cloud/src/test/test-files/solr/solr.lowZkTimeout.xml (original)
+++ lucene/solr/branches/cloud/src/test/test-files/solr/solr.lowZkTimeout.xml Tue Jan 19 13:49:47 2010
@@ -28,7 +28,7 @@
adminPath: RequestHandler path to manage cores.
If 'null' (or absent), cores will not be manageable via request handler
-->
- <cores adminPath="/admin/cores" hostContext="solr" zkClientTimeout="30">
- <core name="DEFAULT_CORE" instanceDir="." shardList="shard1"/>
+ <cores adminPath="/admin/cores" defaultCoreName="collection1" hostContext="solr" zkClientTimeout="30">
+ <core name="collection1" instanceDir="." slice="slice1"/>
</cores>
</solr>
Modified: lucene/solr/branches/cloud/src/test/test-files/solr/solr.xml
URL: http://svn.apache.org/viewvc/lucene/solr/branches/cloud/src/test/test-files/solr/solr.xml?rev=900779&r1=900778&r2=900779&view=diff
==============================================================================
--- lucene/solr/branches/cloud/src/test/test-files/solr/solr.xml (original)
+++ lucene/solr/branches/cloud/src/test/test-files/solr/solr.xml Tue Jan 19 13:49:47 2010
@@ -28,7 +28,7 @@
adminPath: RequestHandler path to manage cores.
If 'null' (or absent), cores will not be manageable via request handler
-->
- <cores adminPath="/admin/cores" hostPort="8983" hostContext="solr" zkClientTimeout="8000">
- <core name="DEFAULT_CORE" instanceDir="." shardList="shard1"/>
+ <cores adminPath="/admin/cores" defaultCoreName="collection1" hostPort="8983" hostContext="solr" zkClientTimeout="8000">
+ <core name="collection1" instanceDir="."/>
</cores>
</solr>