You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2021/01/20 03:57:39 UTC
[lucene-solr] branch reference_impl_dev updated (1e99ed8 -> 53ec024)
This is an automated email from the ASF dual-hosted git repository.
markrmiller pushed a change to branch reference_impl_dev
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git.
from 1e99ed8 @1254 Rename folder.
new c281ec0 @1255 Remove lock.
new 53ec024 @1256 Fix up cloud-dev script.
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
solr/cloud-dev/cloud.sh | 62 +++++------------
.../org/apache/solr/cloud/ZkCollectionTerms.java | 77 +++++++---------------
.../java/org/apache/solr/cloud/ZkController.java | 16 ++---
3 files changed, 48 insertions(+), 107 deletions(-)
[lucene-solr] 01/02: @1255 Remove lock.
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
markrmiller pushed a commit to branch reference_impl_dev
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit c281ec08052a0eccf337b7469769bb27241b1c40
Author: markrmiller@gmail.com <ma...@gmail.com>
AuthorDate: Tue Jan 19 21:56:35 2021 -0600
@1255 Remove lock.
---
.../org/apache/solr/cloud/ZkCollectionTerms.java | 77 +++++++---------------
.../java/org/apache/solr/cloud/ZkController.java | 16 ++---
2 files changed, 31 insertions(+), 62 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkCollectionTerms.java b/solr/core/src/java/org/apache/solr/cloud/ZkCollectionTerms.java
index 08ebef4..9e6f71e 100644
--- a/solr/core/src/java/org/apache/solr/cloud/ZkCollectionTerms.java
+++ b/solr/core/src/java/org/apache/solr/cloud/ZkCollectionTerms.java
@@ -26,7 +26,6 @@ import org.apache.zookeeper.KeeperException;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.locks.ReentrantLock;
/**
* Used to manage all ZkShardTerms of a collection
@@ -34,9 +33,6 @@ import java.util.concurrent.locks.ReentrantLock;
class ZkCollectionTerms implements AutoCloseable {
private final String collection;
private final Map<String,ZkShardTerms> terms;
-
- private final ReentrantLock collectionToTermsLock = new ReentrantLock(true);
-
private final SolrZkClient zkClient;
private volatile boolean closed;
@@ -48,34 +44,23 @@ class ZkCollectionTerms implements AutoCloseable {
}
ZkShardTerms getShard(String shardId) throws Exception {
- collectionToTermsLock.lock();
- try {
- ZkShardTerms zkterms = terms.get(shardId);
- if (zkterms == null) {
- if (closed) {
- throw new AlreadyClosedException();
- }
- zkterms = new ZkShardTerms(collection, shardId, zkClient);
- IOUtils.closeQuietly(terms.put(shardId, zkterms));
- if (closed) {
- IOUtils.closeQuietly(zkterms);
- throw new AlreadyClosedException();
- }
+ ZkShardTerms zkterms = terms.get(shardId);
+ if (zkterms == null) {
+ zkterms = new ZkShardTerms(collection, shardId, zkClient);
+ ZkShardTerms returned = terms.putIfAbsent(shardId, zkterms);
+ if (returned == null) {
+ return zkterms;
+ } else {
+ IOUtils.closeQuietly(zkterms);
+ return returned;
}
- return zkterms;
- } finally {
- collectionToTermsLock.unlock();
}
+ return zkterms;
}
public ZkShardTerms getShardOrNull(String shardId) {
- collectionToTermsLock.lock();
- try {
- if (!terms.containsKey(shardId)) return null;
- return terms.get(shardId);
- } finally {
- collectionToTermsLock.unlock();
- }
+ if (!terms.containsKey(shardId)) return null;
+ return terms.get(shardId);
}
public void register(String shardId, String coreNodeName) throws Exception {
@@ -84,43 +69,27 @@ class ZkCollectionTerms implements AutoCloseable {
}
public void remove(String shardId, CoreDescriptor coreDescriptor) throws KeeperException, InterruptedException {
- collectionToTermsLock.lock();
- try {
- ZkShardTerms zterms = getShardOrNull(shardId);
- if (zterms != null) {
- if (zterms.removeTerm(coreDescriptor)) {
- IOUtils.closeQuietly(terms.remove(shardId));
- }
+ ZkShardTerms zterms = getShardOrNull(shardId);
+ if (zterms != null) {
+ if (zterms.removeTerm(coreDescriptor)) {
+ IOUtils.closeQuietly(terms.remove(shardId));
}
- } finally {
- collectionToTermsLock.unlock();
}
}
public void close() {
- collectionToTermsLock.lock();
- try {
- closed = true;
- terms.values().forEach(ZkShardTerms::close);
-
- terms.clear();
- } finally {
- collectionToTermsLock.unlock();
- }
+ closed = true;
+ terms.values().forEach(ZkShardTerms::close);
+ terms.clear();
assert ObjectReleaseTracker.release(this);
}
public boolean cleanUp() {
- collectionToTermsLock.lock();
- try {
- for (ZkShardTerms zkShardTerms : terms.values()) {
- if (zkShardTerms.getTerms().size() > 0) {
- return false;
- }
+ for (ZkShardTerms zkShardTerms : terms.values()) {
+ if (zkShardTerms.getTerms().size() > 0) {
+ return false;
}
- return true;
- } finally {
- collectionToTermsLock.unlock();
}
+ return true;
}
}
diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkController.java b/solr/core/src/java/org/apache/solr/cloud/ZkController.java
index e1e9193..99be781 100644
--- a/solr/core/src/java/org/apache/solr/cloud/ZkController.java
+++ b/solr/core/src/java/org/apache/solr/cloud/ZkController.java
@@ -1383,6 +1383,7 @@ public class ZkController implements Closeable, Runnable {
//ZkCollectionTerms ct = createCollectionTerms(collection);
shardTerms = getShardTerms(collection, cloudDesc.getShardId());
+ log.info("Create leader elector for replica {}", coreName);
leaderElector = leaderElectors.get(replica.getName());
if (leaderElector == null) {
ContextKey contextKey = new ContextKey(collection, coreName);
@@ -1783,9 +1784,6 @@ public class ZkController implements Closeable, Runnable {
public ZkShardTerms getShardTerms(String collection, String shardId) throws Exception {
ZkCollectionTerms ct = getCollectionTerms(collection);
if (ct == null) {
- if (getCoreContainer().isShutDown()) {
- throw new AlreadyClosedException();
- }
ct = createCollectionTerms(collection);
}
return ct.getShard(shardId);
@@ -1808,12 +1806,14 @@ public class ZkController implements Closeable, Runnable {
}
public ZkCollectionTerms createCollectionTerms(String collection) {
-// if (isClosed || dcCalled) {
-// throw new AlreadyClosedException();
-// }
ZkCollectionTerms ct = new ZkCollectionTerms(collection, zkClient);
- IOUtils.closeQuietly(collectionToTerms.put(collection, ct));
- return ct;
+ ZkCollectionTerms returned = collectionToTerms.putIfAbsent(collection, ct);
+ if (returned == null) {
+ return ct;
+ } else {
+ IOUtils.closeQuietly(ct);
+ }
+ return returned;
}
public void clearZkCollectionTerms() {
[lucene-solr] 02/02: @1256 Fix up cloud-dev script.
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
markrmiller pushed a commit to branch reference_impl_dev
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 53ec024e86f822869b572bde30f1aa99be6ad6b1
Author: markrmiller@gmail.com <ma...@gmail.com>
AuthorDate: Tue Jan 19 21:56:57 2021 -0600
@1256 Fix up cloud-dev script.
---
solr/cloud-dev/cloud.sh | 62 ++++++++++++++-----------------------------------
1 file changed, 17 insertions(+), 45 deletions(-)
diff --git a/solr/cloud-dev/cloud.sh b/solr/cloud-dev/cloud.sh
index fd731e0..25d98c3 100755
--- a/solr/cloud-dev/cloud.sh
+++ b/solr/cloud-dev/cloud.sh
@@ -90,7 +90,7 @@
#
##################################################################################
-DEFAULT_VCS_WORKSPACE='../../'
+DEFAULT_VCS_WORKSPACE="../../"
############## Normally no need to edit below this line ##############
@@ -106,7 +106,7 @@ MEMORY=1g # default
JVM_ARGS='' # default
RECOMPILE=false # default
NUM_NODES=0 # need to detect if not specified
-VCS_WORK=/data2/lucene-solr
+VCS_WORK=$DEFAULT_VCS_WORKSPACE
ZK_PORT=2181
while getopts ":crm:a:n:w:z:" opt; do
@@ -222,7 +222,7 @@ findSolr() {
SOLR=${CLUSTER_WD}/$(find . -maxdepth 1 -name 'solr*' -type d -print0 | xargs -0 ls -1 -td | sed -E 's/\.\/(solr.*)/\1/' | head -n1)
popd
- #echo "Found solr at $SOLR"
+ echo "Found solr at $SOLR"
SAFE_DEST="${CLUSTER_WD_FULL//\//_}";
}
@@ -254,7 +254,6 @@ cleanIfReq() {
recompileIfReq() {
if [[ "$RECOMPILE" = true ]]; then
pushd "$VCS_WORK"
- ./gradlew clean
./gradlew -p solr distTar
if [[ "$?" -ne 0 ]]; then
echo "BUILD FAIL - cloud.sh stopping, see above output for details"; popd; exit 7;
@@ -330,48 +329,16 @@ start(){
echo "Final NUM_NODES is $NUM_NODES"
for i in `seq 1 $NUM_NODES`; do
mkdir -p "${CLUSTER_WD}/n${i}"
- argsArray=(-c -s $CLUSTER_WD_FULL/n${i} -z localhost:${ZK_PORT}/solr_${SAFE_DEST} -f -p 898${i} -m $MEMORY \
- -a "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=500${i} \
- -Dsolr.solrxml.location=zookeeper -Dsolr.log.dir=$CLUSTER_WD_FULL/n${i} $JVM_ARGS")
+ argsArray=(-c -s $CLUSTER_WD_FULL/n${i} -z localhost:${ZK_PORT}/solr_${SAFE_DEST} -p 898${i} -m $MEMORY \
+ -a "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=500${i}" \
+ -Dsolr.solrxml.location=zookeeper -Dsolr.log.dir=$CLUSTER_WD_FULL/n${i} "$JVM_ARGS")
FINAL_COMMAND="${SOLR}/bin/solr ${argsArray[@]}"
echo ${FINAL_COMMAND}
${SOLR}/bin/solr "${argsArray[@]}"
done
- SOLR_STOP_WAIT=180
- SOLR_LOGS_DIR=logs
- for i in `seq 1 $NUM_NODES`; do
- if lsof -v 2>&1 | grep -q revision; then
- echo -n "Waiting up to $SOLR_STOP_WAIT seconds to see Solr running on port $SOLR_PORT"
- SOLR_PORT="898${i}"
- (loops=0
- while true
- do
- running=$(lsof -t -PniTCP:$SOLR_PORT -sTCP:LISTEN)
- if [ -z "$running" ]; then
- slept=$((loops / 2))
- if [ $slept -lt $SOLR_STOP_WAIT ]; then
- sleep 0.5
- loops=$[$loops+1]
- else
- echo -e "Still not seeing Solr listening on $SOLR_PORT after $SOLR_STOP_WAIT seconds!"
- tail -30 "$SOLR_LOGS_DIR/solr.log"
- exit # subshell!
- fi
- else
- SOLR_PID=`ps auxww | grep start\.jar | grep -w "\-Djetty\.port=$SOLR_PORT" | grep -v grep | awk '{print $2}' | sort -r`
- echo -e "\nStarted Solr server on port $SOLR_PORT (pid=$SOLR_PID). Happy searching!\n"
- exit # subshell!
- fi
- done) &
- else
- echo -e "NOTE: Please install lsof as this script needs it to determine if Solr is listening on port $SOLR_PORT."
- sleep 10
- SOLR_PID=`ps auxww | grep start\.jar | grep -w "\-Djetty\.port=$SOLR_PORT" | grep -v grep | awk '{print $2}' | sort -r`
- echo -e "\nStarted Solr server on port $SOLR_PORT (pid=$SOLR_PID). Happy searching!\n"
- return;
- fi
- done
+ echo "/solr_${SAFE_DEST}" > ${CLUSTER_WD}/chroot.txt
+
touch ${CLUSTER_WD} # make this the most recently updated dir for ls -t
@@ -379,11 +346,16 @@ start(){
stop() {
echo "Stopping servers"
- pushd ${CLUSTER_WD}
- SOLR=${CLUSTER_WD}/$(find . -maxdepth 1 -name 'solr*' -type d -print0 | xargs -0 ls -1 -td | sed -E 's/\.\/(solr.*)/\1/' | head -n1)
- popd
- "${SOLR}/bin/solr" stop -all
+ findSolr
+
+ echo "Final NUM_NODES is $NUM_NODES"
+ for i in `seq 1 $NUM_NODES`; do
+ argsArray=(-s $CLUSTER_WD_FULL/n${i} -p "898${i}")
+ FINAL_COMMAND="${SOLR}/bin/solr ${argsArray[@]}"
+ echo ${FINAL_COMMAND}
+ ${SOLR}/bin/solr stop "${argsArray[@]}"
+ done
}
########################