You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2014/08/06 18:48:46 UTC
svn commit: r1616278 - in /lucene/dev/branches/branch_4x: ./ solr/
solr/cloud-dev/ solr/core/ solr/core/src/java/org/apache/solr/cloud/
Author: markrmiller
Date: Wed Aug 6 16:48:46 2014
New Revision: 1616278
URL: http://svn.apache.org/r1616278
Log:
SOLR-6313: Improve SolrCloud cloud-dev scripts.
Modified:
lucene/dev/branches/branch_4x/ (props changed)
lucene/dev/branches/branch_4x/solr/ (props changed)
lucene/dev/branches/branch_4x/solr/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_4x/solr/cloud-dev/ (props changed)
lucene/dev/branches/branch_4x/solr/cloud-dev/control.sh
lucene/dev/branches/branch_4x/solr/cloud-dev/functions.sh
lucene/dev/branches/branch_4x/solr/cloud-dev/solrcloud-start-existing.sh
lucene/dev/branches/branch_4x/solr/cloud-dev/solrcloud-start.sh
lucene/dev/branches/branch_4x/solr/cloud-dev/stop.sh
lucene/dev/branches/branch_4x/solr/core/ (props changed)
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/ZkController.java
Modified: lucene/dev/branches/branch_4x/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/CHANGES.txt?rev=1616278&r1=1616277&r2=1616278&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/solr/CHANGES.txt Wed Aug 6 16:48:46 2014
@@ -261,6 +261,8 @@ Other Changes
* SOLR-5664: Added support for multi-valued field highlighting in /browse UI. (ehatcher)
+* SOLR-6313: Improve SolrCloud cloud-dev scripts. (Mark Miller, Vamsee Yarlagadda)
+
================== 4.9.0 ==================
Versions of Major Components
Modified: lucene/dev/branches/branch_4x/solr/cloud-dev/control.sh
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/cloud-dev/control.sh?rev=1616278&r1=1616277&r2=1616278&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/cloud-dev/control.sh (original)
+++ lucene/dev/branches/branch_4x/solr/cloud-dev/control.sh Wed Aug 6 16:48:46 2014
@@ -4,7 +4,7 @@ source ./functions.sh
case "$1" in
start)
- start $2 $3
+ start $2 $3 "$4"
;;
stop)
stop $2
Modified: lucene/dev/branches/branch_4x/solr/cloud-dev/functions.sh
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/cloud-dev/functions.sh?rev=1616278&r1=1616277&r2=1616278&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/cloud-dev/functions.sh (original)
+++ lucene/dev/branches/branch_4x/solr/cloud-dev/functions.sh Wed Aug 6 16:48:46 2014
@@ -1,7 +1,7 @@
-JAVA_OPTS="-server -Xms256M -Xmx256M"
-BASE_PORT=7572
-BASE_STOP_PORT=6572
-ZK_PORT="9983"
+INT_JAVA_OPTS="-server -Xms256M -Xmx256M"
+BASE_PORT=8900
+BASE_STOP_PORT=9900
+ZK_PORT="2414"
rebuild() {
echo "Rebuilding"
@@ -14,13 +14,8 @@ rebuild() {
}
setports() {
- if [ "1" = "$1" ]; then
- PORT="8983"
- STOP_PORT="7983"
- else
- PORT="$(( $BASE_PORT + $1 ))"
- STOP_PORT="$(( $BASE_STOP_PORT + $1 ))"
- fi
+ PORT="$(( $BASE_PORT + $1 ))"
+ STOP_PORT="$(( $BASE_STOP_PORT + $1 ))"
}
reinstall() {
Modified: lucene/dev/branches/branch_4x/solr/cloud-dev/solrcloud-start-existing.sh
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/cloud-dev/solrcloud-start-existing.sh?rev=1616278&r1=1616277&r2=1616278&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/cloud-dev/solrcloud-start-existing.sh (original)
+++ lucene/dev/branches/branch_4x/solr/cloud-dev/solrcloud-start-existing.sh Wed Aug 6 16:48:46 2014
@@ -1,29 +1,34 @@
#!/bin/bash
numServers=$1
-baseJettyPort=7572
-baseStopPort=6572
+
+baseJettyPort=8900
+baseStopPort=9900
die () {
echo >&2 "$@"
exit 1
}
-[ "$#" -eq 1 ] || die "1 argument required, $# provided, usage: solrcloud-start-exisiting.sh {numServers}"
+[ "$#" -eq 1 ] || die "1 argument required, $# provided, usage: solrcloud-start-exisiting.sh [numServers]"
cd ..
-cd example1
-echo "starting example1"
-java -DzkRun -DSTOP.PORT=7983 -DSTOP.KEY=key -jar start.jar 1>example1.log 2>&1 &
+cd examplezk
+stopPort=1313
+jettyPort=8900
+exec -a jettyzk java -Xmx512m $JAVA_OPTS -Djetty.port=$jettyPort -DhostPort=$jettyPort -DzkRun -DzkRunOnly=true -DSTOP.PORT=$stopPort -DSTOP.KEY=key -jar start.jar 1>examplezk.log 2>&1 &
+# TODO: we could also remove the default core
+cd ..
+cd example
-for (( i=2; i <= $numServers; i++ ))
+for (( i=1; i <= $numServers; i++ ))
do
echo "starting example$i"
cd ../example$i
stopPort=`expr $baseStopPort + $i`
jettyPort=`expr $baseJettyPort + $i`
- java -Xmx1g -Djetty.port=$jettyPort -DzkHost=localhost:9983 -DnumShards=1 -DSTOP.PORT=$stopPort -DSTOP.KEY=key -jar start.jar 1>example$i.log 2>&1 &
+ exec -a jetty java -Xmx1g $JAVA_OPTS -Djetty.port=$jettyPort -DzkHost=localhost:9900 -DSTOP.PORT=$stopPort -DSTOP.KEY=key -jar start.jar 1>example$i.log 2>&1 &
done
Modified: lucene/dev/branches/branch_4x/solr/cloud-dev/solrcloud-start.sh
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/cloud-dev/solrcloud-start.sh?rev=1616278&r1=1616277&r2=1616278&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/cloud-dev/solrcloud-start.sh (original)
+++ lucene/dev/branches/branch_4x/solr/cloud-dev/solrcloud-start.sh Wed Aug 6 16:48:46 2014
@@ -1,19 +1,22 @@
#!/bin/bash
+# To run on hdfs, try something along the lines of:
+# export JAVA_OPTS="-Dsolr.directoryFactory=solr.HdfsDirectoryFactory -Dsolr.lock.type=hdfs -Dsolr.hdfs.home=hdfs://localhost:8020/solr -Dsolr.hdfs.confdir=/etc/hadoop_conf/conf"
+
numServers=$1
numShards=$2
-baseJettyPort=7572
-baseStopPort=6572
+baseJettyPort=8900
+baseStopPort=9900
-zkaddress = localhost:2181/solr
+zkAddress=localhost:9900
die () {
echo >&2 "$@"
exit 1
}
-[ "$#" -eq 2 ] || die "2 arguments required, $# provided, usage: solrcloud-start.sh {numServers} {numShards}"
+[ "$#" -eq 2 ] || die "2 arguments required, $# provided, usage: solrcloud-start.sh [numServers] [numShards]"
cd ..
@@ -29,6 +32,7 @@ rm -r -f example/solr/zoo_data
rm -r -f example/solr/collection1/data
rm -f example/example.log
+ant -f ../build.xml clean
ant example dist
rm -r example/solr-webapp/*
@@ -40,23 +44,28 @@ do
cp -r -f example example$i
done
+rm -r -f examplezk
+cp -r -f example examplezk
+cp core/src/test-files/solr/solr-no-core.xml examplezk/solr/solr.xml
+rm -r -f examplezk/solr/collection1/core.properties
+cd examplezk
+stopPort=1313
+jettyPort=8900
+exec -a jettyzk java -Xmx512m $JAVA_OPTS -Djetty.port=$jettyPort -DhostPort=$jettyPort -DzkRun -DzkRunOnly=true -DSTOP.PORT=$stopPort -DSTOP.KEY=key -jar start.jar 1>examplezk.log 2>&1 &
+cd ..
-java -classpath "example1/solr-webapp/webapp/WEB-INF/lib/*:example/lib/ext/*" org.apache.solr.cloud.ZkCLI -cmd bootstrap -zkhost 127.0.0.1:9983 -solrhome example1/solr -runzk 8983
-
-echo "starting example1"
-
-cd example1
-java -Xmx1g -DzkRun -DnumShards=$numShards -DSTOP.PORT=7983 -DSTOP.KEY=key -jar start.jar 1>example1.log 2>&1 &
-
-
+# upload config files
+java -classpath "example1/solr-webapp/webapp/WEB-INF/lib/*:example/lib/ext/*" org.apache.solr.cloud.ZkCLI -cmd bootstrap -zkhost $zkAddress -solrhome example1/solr
+
+cd example
-for (( i=2; i <= $numServers; i++ ))
+for (( i=1; i <= $numServers; i++ ))
do
echo "starting example$i"
cd ../example$i
stopPort=`expr $baseStopPort + $i`
jettyPort=`expr $baseJettyPort + $i`
- java -Xmx1g -Djetty.port=$jettyPort -DzkHost=localhost:9983 -DnumShards=1 -DSTOP.PORT=$stopPort -DSTOP.KEY=key -jar start.jar 1>example$i.log 2>&1 &
+ exec -a jetty java -Xmx1g $JAVA_OPTS -DnumShards=$numShards -Djetty.port=$jettyPort -DzkHost=$zkAddress -DSTOP.PORT=$stopPort -DSTOP.KEY=key -jar start.jar 1>example$i.log 2>&1 &
done
Modified: lucene/dev/branches/branch_4x/solr/cloud-dev/stop.sh
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/cloud-dev/stop.sh?rev=1616278&r1=1616277&r2=1616278&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/cloud-dev/stop.sh (original)
+++ lucene/dev/branches/branch_4x/solr/cloud-dev/stop.sh Wed Aug 6 16:48:46 2014
@@ -1,7 +1,8 @@
#!/bin/bash
numServers=$1
-baseStopPort=6572
+baseJettyPort=8900
+baseStopPort=9900
die () {
echo >&2 "$@"
@@ -12,13 +13,52 @@ die () {
cd ../example
-java -DSTOP.PORT=7983 -DSTOP.KEY=key -jar start.jar --stop
-
-
-for (( i=2; i <= $numServers; i++ ))
+for (( i=1; i <= $numServers; i++ ))
do
- echo "stopping example$i"
- cd ../example$i
stopPort=`expr $baseStopPort + $i`
+ echo "stopping example$i, stop port is $stopPort"
+ cd ../example$i
java -DSTOP.PORT=$stopPort -DSTOP.KEY=key -jar start.jar --stop
done
+
+
+mkdir ../example-lastlogs
+
+for (( i=1; i <= $numServers; i++ ))
+do
+ cd ../example$i
+
+ jettyPort=`expr $baseJettyPort + $i`
+ echo "Make sure jetty stops and wait for it: $jettyPort"
+
+ pid=`lsof -i:$jettyPort -sTCP:LISTEN -t`
+ echo "pid:$pid"
+ #kill $pid
+ #wait $pid
+ if [ ! -z "$pid" ]
+ then
+ while [ -e /proc/$pid ]; do sleep 1; done
+ fi
+
+ # save the last shutdown logs
+ echo "copy example$i.log to lastlogs"
+ cp -r -f example$i.log ../example-lastlogs/example-last$i.log
+done
+
+# stop zk runner
+java -DSTOP.PORT=1313 -DSTOP.KEY=key -jar start.jar --stop
+
+echo "wait for port to be available: $baseJettyPort"
+
+pid=`lsof -i:$baseJettyPort -sTCP:LISTEN -t`
+echo "pid:$pid"
+#kill $pid
+#wait $pid
+if [ ! -z "$pid" ]
+then
+ while [ -e /proc/$pid ]; do sleep 0.1; done
+fi
+nc -w 30 127.0.0.1 $baseJettyPort
+
+sleep 5
+
\ No newline at end of file
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/ZkController.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/ZkController.java?rev=1616278&r1=1616277&r2=1616278&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/ZkController.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/ZkController.java Wed Aug 6 16:48:46 2014
@@ -178,6 +178,12 @@ public final class ZkController {
// keeps track of replicas that have been asked to recover by leaders running on this node
private Map<String,String> replicasInLeaderInitiatedRecovery = new HashMap<String,String>();
+ // This is an expert and unsupported development mode that does not create
+ // an Overseer or register a /live node. This let's you monitor the cluster
+ // and interact with zookeeper via the Solr admin UI on a node outside the cluster,
+ // and so one that will not be killed or stopped when testing. See developer cloud-scripts.
+ private boolean zkRunOnly = Boolean.getBoolean("zkRunOnly"); // expert
+
public ZkController(final CoreContainer cc, String zkServerAddress, int zkClientTimeout, int zkClientConnectTimeout, String localHost, String locaHostPort,
String localHostContext, int leaderVoteWait, int leaderConflictResolveWait, boolean genericCoreNodeNames, final CurrentCoreDescriptorProvider registerOnReconnect)
throws InterruptedException, TimeoutException, IOException
@@ -228,16 +234,19 @@ public final class ZkController {
registerAllCoresAsDown(registerOnReconnect, false);
- ElectionContext context = new OverseerElectionContext(zkClient,
- overseer, getNodeName());
-
- ElectionContext prevContext = overseerElector.getContext();
- if (prevContext != null) {
- prevContext.cancelElection();
+ if (!zkRunOnly) {
+ ElectionContext context = new OverseerElectionContext(zkClient,
+ overseer, getNodeName());
+
+ ElectionContext prevContext = overseerElector.getContext();
+ if (prevContext != null) {
+ prevContext.cancelElection();
+ }
+
+ overseerElector.setup(context);
+ overseerElector.joinElection(context, true);
}
-
- overseerElector.setup(context);
- overseerElector.joinElection(context, true);
+
zkStateReader.createClusterStateWatchersAndUpdate();
// we have to register as live first to pick up docs in the buffer
@@ -686,6 +695,9 @@ public final class ZkController {
private void createEphemeralLiveNode() throws KeeperException,
InterruptedException {
+ if (zkRunOnly) {
+ return;
+ }
String nodeName = getNodeName();
String nodePath = ZkStateReader.LIVE_NODES_ZKNODE + "/" + nodeName;
log.info("Register node as live in ZooKeeper:" + nodePath);