You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2021/02/10 01:43:53 UTC
[lucene-solr] 09/09: @1336 Startup and logging tweaks.
This is an automated email from the ASF dual-hosted git repository.
markrmiller pushed a commit to branch reference_impl_dev
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 1ea3336da0973cec82d95abce4712af4a33e1e11
Author: markrmiller@gmail.com <ma...@gmail.com>
AuthorDate: Tue Feb 9 19:36:15 2021 -0600
@1336 Startup and logging tweaks.
---
solr/bin/jetty.sh | 8 +--
solr/bin/solr | 105 ++++++++++++++++++++++++--------------
solr/docker/tests/shared.sh | 2 +-
solr/server/etc/jetty-started.xml | 2 +-
solr/server/resources/log4j2.xml | 32 +++++++++---
5 files changed, 97 insertions(+), 52 deletions(-)
diff --git a/solr/bin/jetty.sh b/solr/bin/jetty.sh
index edcb03a..ad377dc 100644
--- a/solr/bin/jetty.sh
+++ b/solr/bin/jetty.sh
@@ -91,7 +91,7 @@ NAME=$(echo $(basename $0) | sed -e 's/^[SK][0-9]*//' -e 's/\.sh$//')
# have shell access, e.g. /bin/false
#
# JETTY_START_TIMEOUT
-# Time spent waiting to see if startup was successful/failed. Defaults to 60 seconds
+# Time spent waiting to see if startup was successful/failed. Defaults to 10 seconds
#
usage()
@@ -387,7 +387,7 @@ fi
#####################################################
if [ -z "$JETTY_START_TIMEOUT" ]
then
- JETTY_START_TIMEOUT=60
+ JETTY_START_TIMEOUT=10
fi
#####################################################
@@ -480,11 +480,11 @@ case "$ACTION" in
# FIXME: Broken solution: wordsplitting, pathname expansion, arbitrary command execution, etc.
su - "$JETTY_USER" $SU_SHELL -c "
cd \"$JETTY_BASE\"
- exec ${RUN_CMD[*]} start-log-file=\"$JETTY_START_LOG\" > /dev/null &
+ exec ("${RUN_CMD[@]}") start-log-file=\"$JETTY_START_LOG\" > /dev/null &
disown \$!
echo \$! > \"$JETTY_PID\""
else
- "${RUN_CMD[@]}" > /dev/null &
+ ("${RUN_CMD[@]}") > /dev/null &
disown $!
echo $! > "$JETTY_PID"
fi
diff --git a/solr/bin/solr b/solr/bin/solr
index a3d2452..de30d8a 100755
--- a/solr/bin/solr
+++ b/solr/bin/solr
@@ -46,7 +46,7 @@
# ensuring that no include files exist in the aforementioned search list.
SOLR_SCRIPT="$0"
-verbose=false
+verbose=true
THIS_OS=`uname -s`
# What version of Java is required to run this version of Solr.
@@ -97,10 +97,6 @@ elif [ -r "$SOLR_INCLUDE" ]; then
. "$SOLR_INCLUDE"
fi
-if [ -z "$SOLR_PID_DIR" ]; then
- SOLR_PID_DIR="$SOLR_TIP/bin"
-fi
-
if [ -n "$SOLR_JAVA_HOME" ]; then
JAVA="$SOLR_JAVA_HOME/bin/java"
JSTACK="$SOLR_JAVA_HOME/bin/jstack"
@@ -706,7 +702,7 @@ function solr_pid_by_port() {
# extract the value of the -Djetty.port parameter from a running Solr process
function jetty_port() {
SOLR_PID="$1"
- SOLR_PROC=`ps auxww | grep -w $SOLR_PID | grep start\.jar | grep jetty\.port`
+ SOLR_PROC=$(ps auxww | grep -w $SOLR_PID | grep start\.jar | grep jetty\.port)
IFS=' ' read -a proc_args <<< "$SOLR_PROC"
for arg in "${proc_args[@]}"
do
@@ -879,7 +875,8 @@ function stop_solr() {
done
rm -f "$SOLR_PID_DIR/$JETTY_PID"
- rm -f "${SOLR_HOME}/jetty.state"
+ JETTY_STATE="${SOLR_PID_DIR}/jetty.state"
+ rm -f "${JETTY_STATE}"
echo OK
} # end stop_solr
@@ -1138,7 +1135,8 @@ if [[ "$SCRIPT_CMD" == "create" || "$SCRIPT_CMD" == "create_core" || "$SCRIPT_CM
fi
if [ -z "$CREATE_PORT" ]; then
- for ID in `ps auxww | grep java | grep start\.jar | awk '{print $2}' | sort -r`
+ running=$(ps auxww | grep java | grep start\.jar | awk '{print $2}' | sort -r)
+ for ID in $running
do
port=`jetty_port "$ID"`
if [ "$port" != "" ]; then
@@ -1541,10 +1539,13 @@ if [[ "$SCRIPT_CMD" == "auth" ]]; then
SOLR_HOME="$(pwd)/$SOLR_HOME"
elif [[ -d "$SOLR_SERVER_DIR/$SOLR_HOME" ]]; then
SOLR_HOME="$SOLR_SERVER_DIR/$SOLR_HOME"
- SOLR_PID_DIR="$SOLR_HOME"
fi
fi
+ if [ -z "$SOLR_PID_DIR" ]; then
+ SOLR_PID_DIR="$SOLR_HOME"
+ fi
+
if [ -z "$AUTH_OP" ]; then
print_usage "$SCRIPT_CMD"
exit 0
@@ -1626,7 +1627,7 @@ fi
FG="false"
FORCE=false
noprompt=false
-SOLR_OPTS=("$SOLR_OPTS")
+SOLR_OPTS=($SOLR_OPTS)
PASS_TO_RUN_EXAMPLE=
if [ $# -gt 0 ]; then
@@ -1873,14 +1874,17 @@ fi
if [ -z "$SOLR_HOME" ]; then
SOLR_HOME="$SOLR_SERVER_DIR/solr"
elif [[ $SOLR_HOME != /* ]]; then
- if [[ -d "`pwd`/$SOLR_HOME" ]]; then
- SOLR_HOME="$(pwd)/$SOLR_HOME"
+ if [[ -d "`pwd`/$SOLR_HOME" ]]; then
+ SOLR_HOME="$(pwd)/$SOLR_HOME"
elif [[ -d "$SOLR_SERVER_DIR/$SOLR_HOME" ]]; then
SOLR_HOME="$SOLR_SERVER_DIR/$SOLR_HOME"
- SOLR_PID_DIR="$SOLR_HOME"
fi
fi
+if [ -z "$SOLR_PID_DIR" ]; then
+ SOLR_PID_DIR="$SOLR_HOME"
+fi
+
# Set the default configset dir to be bootstrapped as _default
if [ -z "$DEFAULT_CONFDIR" ]; then
DEFAULT_CONFDIR="$SOLR_SERVER_DIR/solr/configsets/_default/conf"
@@ -1899,7 +1903,11 @@ fi
LOG4J_CONFIG=()
if [ -n "$LOG4J_PROPS" ]; then
- LOG4J_CONFIG+=("-Dlog4j.configurationFile=$LOG4J_PROPS" '-Dlog4j2.is.webapp' '-Dlog4j2.garbagefreeThreadContextMap=true' '-Dlog4j2.enableDirectEncoders=true' '-Dlog4j2.enable.threadlocals=true')
+ LOG4J_CONFIG+=("-Dlog4j.configurationFile=$LOG4J_PROPS")
+ LOG4J_CONFIG+=('-Dlog4j2.is.webapp')
+ LOG4J_CONFIG+=('-Dlog4j2.garbagefreeThreadContextMap=true')
+ LOG4J_CONFIG+=('-Dlog4j2.enableDirectEncoders=true')
+ LOG4J_CONFIG+=('-Dlog4j2.enable.threadlocals=true')
fi
if [ "$SCRIPT_CMD" == "stop" ]; then
@@ -2108,7 +2116,7 @@ function start_solr() {
'-XX:MaxTenuringThreshold=16' \
'-XX:+ExplicitGCInvokesConcurrent')
else
- GC_TUNE=($GC_TUNE)
+ GC_TUNE=${GC_TUNE[@]}
fi
if [ -n "$SOLR_WAIT_FOR_ZK" ]; then
@@ -2145,17 +2153,17 @@ function start_solr() {
echo -e " SOLR_HOST = $SOLR_HOST"
echo -e " SOLR_PORT = $SOLR_PORT"
echo -e " STOP_PORT = $STOP_PORT"
- echo -e " JAVA_MEM_OPTS = " "${JAVA_MEM_OPTS[@]}"
- echo -e " GC_TUNE = " "${GC_TUNE[@]}"
- echo -e " GC_LOG_OPTS = " "${GC_LOG_OPTS[@]}"
+ echo -e " JAVA_MEM_OPTS = ${JAVA_MEM_OPTS[@]}"
+ echo -e " GC_TUNE = ${GC_TUNE[@]}"
+ echo -e " GC_LOG_OPTS = ${GC_LOG_OPTS[@]}"
echo -e " SOLR_TIMEZONE = $SOLR_TIMEZONE"
if [ "$SOLR_MODE" == "solrcloud" ]; then
- echo -e " CLOUD_MODE_OPTS = " "${CLOUD_MODE_OPTS[@]}"
+ echo -e " CLOUD_MODE_OPTS = ${CLOUD_MODE_OPTS[@]}"
fi
if [ "$SOLR_OPTS" != "" ]; then
- echo -e " SOLR_OPTS = " "${SOLR_OPTS[@]}"
+ echo -e " SOLR_OPTS = ${SOLR_OPTS[@]}"
fi
if [ "$SOLR_ADDL_ARGS" != "" ]; then
@@ -2164,7 +2172,7 @@ function start_solr() {
if [ "$ENABLE_REMOTE_JMX_OPTS" == "true" ]; then
echo -e " RMI_PORT = $RMI_PORT"
- echo -e " REMOTE_JMX_OPTS = " "${REMOTE_JMX_OPTS[@]}"
+ echo -e " REMOTE_JMX_OPTS = ${REMOTE_JMX_OPTS[@]}"
fi
if [ "$SOLR_LOG_LEVEL" != "" ]; then
@@ -2185,15 +2193,25 @@ function start_solr() {
exit 1
fi
- SOLR_START_OPTS=('-server' "${JAVA_MEM_OPTS[@]}" "${GC_TUNE[@]}" "${GC_LOG_OPTS[@]}" "${IP_ACL_OPTS[@]}" \
- "${REMOTE_JMX_OPTS[@]}" "${CLOUD_MODE_OPTS[@]}" "$SOLR_LOG_LEVEL_OPT" "-Dsolr.log.dir=$SOLR_LOGS_DIR" \
- "-Djetty.port=$SOLR_PORT" "-DSTOP.PORT=$stop_port" "-DSTOP.KEY=$STOP_KEY" \
- # '-OmitStackTraceInFastThrow' ensures stack traces in errors,
- # users who don't care about useful error msgs can override in SOLR_OPTS with +OmitStackTraceInFastThrow
- "${SOLR_HOST_ARG[@]}" "-Duser.timezone=$SOLR_TIMEZONE" "-XX:-OmitStackTraceInFastThrow" \
- # MRM TODO: this is hard to deref properly with arguments "-XX:OnOutOfMemoryError=$SOLR_TIP/bin/oom_solr.sh $SOLR_PORT $SOLR_LOGS_DIR" \
- "-Djetty.home=$SOLR_SERVER_DIR" "-Dsolr.solr.home=$SOLR_HOME" "-Dsolr.data.home=$SOLR_DATA_HOME" "-Dsolr.install.dir=$SOLR_TIP" \
- "-Dorg.apache.xml.dtm.DTMManager=org.apache.xml.dtm.ref.DTMManagerDefault" "-Djava.net.preferIPv4Stack=true" "-Dsolr.default.confdir=$DEFAULT_CONFDIR" "${LOG4J_CONFIG[@]}" "${SOLR_OPTS[@]}" "${SECURITY_MANAGER_OPTS[@]}" "${SOLR_ADMIN_UI}")
+ LOG4J_CONFIG+=(${SOLR_LOG_LEVEL_OPT[@]})
+ LOG4J_CONFIG+=("-Dlog4j2.is.webapp=false")
+ LOG4J_CONFIG+=("-Dlog4j2.garbagefreeThreadContextMap=true")
+ LOG4J_CONFIG+=("-Dlog4j2.enableDirectEncoders=true")
+ LOG4J_CONFIG+=("-Dlog4j2.enable.threadlocals=true")
+
+ SOLR_START_OPTS=(${JAVA_MEM_OPTS[@]} ${GC_TUNE[@]} ${GC_LOG_OPTS[@]} ${IP_ACL_OPTS[@]} ${REMOTE_JMX_OPTS[@]} ${CLOUD_MODE_OPTS[@]})
+ SOLR_START_OPTS+=(${SOLR_HOST_ARG[@]} ${LOG4J_CONFIG[@]} ${SECURITY_MANAGER_OPTS[@]} ${SOLR_ADMIN_UI})
+
+ SOLR_START_OPTS+=("-Dsolr.log.dir=${SOLR_LOGS_DIR}" "-Djetty.port=${SOLR_PORT}" "-DSTOP.PORT=${stop_port}" "-DSTOP.KEY=${STOP_KEY}")
+ SOLR_START_OPTS+=("-Duser.timezone=${SOLR_TIMEZONE}" "-Djetty.home=${SOLR_SERVER_DIR}" "-Dsolr.solr.home=${SOLR_HOME}" "-Dsolr.data.home=${SOLR_DATA_HOME}" "-Dsolr.install.dir=${SOLR_TIP}")
+ SOLR_START_OPTS+=("-Dorg.apache.xml.dtm.DTMManager=org.apache.xml.dtm.ref.DTMManagerDefault" "-Djava.net.preferIPv4Stack=true" "-Dsolr.default.confdir=${DEFAULT_CONFDIR}")
+ SOLR_START_OPTS+=("-XX:-UseBiasedLocking")
+
+ # '-OmitStackTraceInFastThrow' ensures stack traces in errors,
+ # users who don't care about useful error msgs can override in SOLR_OPTS with +OmitStackTraceInFastThrow
+ SOLR_START_OPTS+=("-XX:-OmitStackTraceInFastThrow")
+
+ # MRM TODO: this is hard to deref properly with arguments "-XX:OnOutOfMemoryError=$SOLR_TIP/bin/oom_solr.sh $SOLR_PORT $SOLR_LOGS_DIR"
mk_writable_dir "$SOLR_LOGS_DIR" "Logs"
if [[ -n "$SOLR_HEAP_DUMP_DIR" ]]; then
@@ -2213,18 +2231,29 @@ function start_solr() {
echo ""
fi
+ SOLR_JETTY_ADDL_CONFIG+=(SOLR_JETTY_ADDL_CONFIG[@])
+ SOLR_START_OPTS+=(${SOLR_OPTS[@]})
+
+ # MRM TODO: enable
+ #SOLR_START_OPTS+=("-XX:OnOutOfMemoryError=${SOLR_TIP}/bin/oom_solr.sh ${SOLR_PORT} ${SOLR_LOGS_DIR}")
+
if [ "$run_in_foreground" == "true" ]; then
- exec "$JAVA" -Djetty.state=${SOLR_PID_DIR}/jetty.state -Dsolr.log.muteconsole "${SOLR_START_OPTS[@]}" "${SOLR_ADDL_ARGS}" -Dlog4j2.is.webapp=false -Dlog4j2.garbagefreeThreadContextMap=true \
- -Dlog4j2.enableDirectEncoders=true -Dlog4j2.enable.threadlocals=true -XX:-UseBiasedLocking -jar start.jar "${SOLR_JETTY_CONFIG[@]}" "${SOLR_JETTY_ADDL_CONFIG}"
+ #set -x
+ exec "$JAVA" "${SOLR_START_OPTS[@]}" -jar start.jar "${SOLR_JETTY_CONFIG[@]}"
else
- # run Solr in the background
+ # run Solr in the background MRM-TODO: Move relevant parts in from jetty.sh
+
+ SOLR_START_OPTS+=("-Djetty.state=${SOLR_PID_DIR}/jetty.state")
+
export JAVA=$JAVA
export JETTY_HOME="${SOLR_SERVER_DIR}"
- export JETTY_PID="${SOLR_PID_DIR}/solr-$SOLR_PORT.pid"
- export JETTY_ARGS[@]=("${SOLR_JETTY_CONFIG[@]}" "${SOLR_JETTY_ADDL_CONFIG[@]}")
- export JAVA_OPTIONS[@]=("${SOLR_START_OPTS[@]}" "${SOLR_ADDL_ARGS[@]}" '-XX:-UseBiasedLocking' '-Dsolr.log.muteconsole')
- export JETTY_STATE[@]=("${SOLR_PID_DIR}/jetty.state")
- bash $SOLR_SERVER_DIR/../bin/jetty.sh start
+ export JETTY_PID="solr-$SOLR_PORT.pid"
+ export JETTY_RUN="$SOLR_PID_DIR"
+ export JETTY_PID="$SOLR_PID_DIR/solr-$SOLR_PORT.pid"
+ export JETTY_ARGS="${SOLR_JETTY_CONFIG[*]}"
+ export JAVA_OPTIONS="${SOLR_START_OPTS[@]}"
+ export JETTY_STATE="${SOLR_PID_DIR}/jetty.state"
+ bash "${SOLR_SERVER_DIR}"/../bin/jetty.sh -d start
fi
}
diff --git a/solr/docker/tests/shared.sh b/solr/docker/tests/shared.sh
index e034387..35bbd56 100755
--- a/solr/docker/tests/shared.sh
+++ b/solr/docker/tests/shared.sh
@@ -54,7 +54,7 @@ function wait_for_server_started {
log="${BUILD_DIR}/${container_name}.log"
while true; do
set -x
- docker cp "${container_name}":/var/solr/logs/solr.log "${log}" || echo "log not found"
+ docker logs "$container_name" > "${log}" 2>&1
if grep -E -q '(o\.e\.j\.s\.Server Started|Started SocketConnector|SolrDispatchFilter\.init\(\) end)' "${log}" ; then
break
fi
diff --git a/solr/server/etc/jetty-started.xml b/solr/server/etc/jetty-started.xml
index f6bfb4c..b9f33de 100644
--- a/solr/server/etc/jetty-started.xml
+++ b/solr/server/etc/jetty-started.xml
@@ -8,7 +8,7 @@
<Call name="addLifeCycleListener">
<Arg>
<New class="org.eclipse.jetty.util.component.FileNoticeLifeCycleListener">
- <Arg><SystemProperty name="jetty.state" default="./jetty.state"/></Arg>
+ <Arg><Property name="jetty.state" default="./jetty.state"/></Arg>
</New>
</Arg>
</Call>
diff --git a/solr/server/resources/log4j2.xml b/solr/server/resources/log4j2.xml
index 03a5c06..93d8512 100644
--- a/solr/server/resources/log4j2.xml
+++ b/solr/server/resources/log4j2.xml
@@ -21,20 +21,30 @@
<Appenders>
<Console name="STDOUT" target="SYSTEM_OUT">
- <LogstashLayout dateTimeFormatPattern="yyyy-MM-dd'T'HH:mm:ss.SSSZZZ"
+ <PatternLayout>
+ <Pattern>
+ %maxLen{%d{yyyy-MM-dd HH:mm:ss.SSS} %-5p (%t) [%X{node_name} %X{core}] %c{1.} %m%notEmpty{ =>%ex{short}}}{10240}%n
+ </Pattern>
+ </PatternLayout>
+ <!-- <LogstashLayout dateTimeFormatPattern="yyyy-MM-dd'T'HH:mm:ss.SSSZZZ"
eventTemplateUri="classpath:LogstashJsonEventLayoutV1.json"
prettyPrintEnabled="true"
- stackTraceEnabled="true"/>
+ stackTraceEnabled="true"/> -->
</Console>
<RollingRandomAccessFile
name="MainLogFile"
fileName="${sys:solr.log.dir}/solr.log"
filePattern="${sys:solr.log.dir}/solr.log.%i" >
- <LogstashLayout dateTimeFormatPattern="yyyy-MM-dd'T'HH:mm:ss.SSSZZZ"
+ <PatternLayout>
+ <Pattern>
+ %maxLen{%d{yyyy-MM-dd HH:mm:ss.SSS} %-5p (%t) [%X{node_name} %X{core}] %c{1.} %m%notEmpty{ =>%ex{short}}}{10240}%n
+ </Pattern>
+ </PatternLayout>
+ <!-- <LogstashLayout dateTimeFormatPattern="yyyy-MM-dd'T'HH:mm:ss.SSSZZZ"
eventTemplateUri="classpath:LogstashJsonEventLayoutV1.json"
prettyPrintEnabled="true"
- stackTraceEnabled="true"/>
+ stackTraceEnabled="true"/> -->
<Policies>
<OnStartupTriggeringPolicy />
<SizeBasedTriggeringPolicy size="64 MB"/>
@@ -46,15 +56,20 @@
name="SlowLogFile"
fileName="${sys:solr.log.dir}/solr_slow_requests.log"
filePattern="${sys:solr.log.dir}/solr_slow_requests.log.%i" >
- <LogstashLayout dateTimeFormatPattern="yyyy-MM-dd'T'HH:mm:ss.SSSZZZ"
+ <PatternLayout>
+ <Pattern>
+ %maxLen{%d{yyyy-MM-dd HH:mm:ss.SSS} %-5p (%t) [%X{node_name} %X{core}] %c{1.} %m%notEmpty{ =>%ex{short}}}{10240}%n
+ </Pattern>
+ </PatternLayout>
+ <!-- <LogstashLayout dateTimeFormatPattern="yyyy-MM-dd'T'HH:mm:ss.SSSZZZ"
eventTemplateUri="classpath:LogstashJsonEventLayoutV1.json"
prettyPrintEnabled="true"
- stackTraceEnabled="true"/>
+ stackTraceEnabled="true"/> -->
<Policies>
<OnStartupTriggeringPolicy />
- <SizeBasedTriggeringPolicy size="32 MB"/>
+ <SizeBasedTriggeringPolicy size="64 MB"/>
</Policies>
- <DefaultRolloverStrategy max="10"/>
+ <DefaultRolloverStrategy max="20"/>
</RollingRandomAccessFile>
</Appenders>
@@ -76,6 +91,7 @@
<AsyncRoot level="INFO">
<AppenderRef ref="MainLogFile"/>
+ <AppenderRef ref="STDOUT"/>
</AsyncRoot>
</Loggers>
</Configuration>