You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2021/01/29 20:39:03 UTC
[lucene-solr] 05/12: SOLR-14955: Add env var options to Prometheus
Export scripts. (#2038)
This is an automated email from the ASF dual-hosted git repository.
markrmiller pushed a commit to branch reference_impl_dev
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 240dad4f0e7d9f725a01aab6d4c4f9f9f4c09480
Author: Houston Putman <ho...@apache.org>
AuthorDate: Fri Oct 30 11:15:33 2020 -0400
SOLR-14955: Add env var options to Prometheus Export scripts. (#2038)
---
solr/CHANGES.txt | 31 ++++++++++++++
solr/contrib/prometheus-exporter/bin/solr-exporter | 30 +++++++++++++-
.../prometheus-exporter/bin/solr-exporter.cmd | 12 +++++-
...onitoring-solr-with-prometheus-and-grafana.adoc | 47 +++++++++++++++-------
4 files changed, 103 insertions(+), 17 deletions(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 89eb930..be0ef3a 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -139,6 +139,37 @@ Bug Fixes
* SOLR-14546: Fix for a relatively hard to hit issue in OverseerTaskProcessor that could lead to out of order execution
of Collection API tasks competing for a lock (Ilan Ginzburg).
+================== 8.8.0 ==================
+
+Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.
+
+New Features
+---------------------
+(No changes)
+
+Improvements
+---------------------
+* SOLR-14942: Reduce leader election time on node shutdown by removing election nodes before closing cores.
+ (Cao Manh Dat, Mike Drob, hossman, shalin)
+
+* SOLR-14955: Add env var options for the Prometheus Exporter bin scripts (Houston Putman)
+
+Optimizations
+---------------------
+(No changes)
+
+Bug Fixes
+---------------------
+* SOLR-14946: Fix responseHeader being returned in response when omitHeader=true and EmbeddedSolrServer is used
+ as the client (Munendra S N)
+
+* SOLR-14940: ReplicationHandler memory leak through SolrCore.closeHooks with unstable ZK connection. (Anver Sotnikov, Mike Drob)
+
+Other Changes
+---------------------
+
+* SOLR-14954: Heavily edit reindexing.adoc (Sameul García Martínez and Erick Erickson)
+
================== 8.7.0 ==================
Consult the lucene/CHANGES.txt file for additional, low level, changes in this release.
diff --git a/solr/contrib/prometheus-exporter/bin/solr-exporter b/solr/contrib/prometheus-exporter/bin/solr-exporter
index 5eb8ef8..b601a78 100755
--- a/solr/contrib/prometheus-exporter/bin/solr-exporter
+++ b/solr/contrib/prometheus-exporter/bin/solr-exporter
@@ -1,5 +1,4 @@
-#!/bin/sh
-
+#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
@@ -135,15 +134,42 @@ if $cygwin; then
[ -n "$REPO" ] && REPO=`cygpath --path --windows "$REPO"`
fi
+# Convert Environment Variables to Command Line Options
+EXPORTER_ARGS=()
+
+if [[ -n "$CONFIG_FILE" ]]; then
+ EXPORTER_ARGS+=(-f "$CONFIG_FILE")
+fi
+
+if [[ -n "$PORT" ]]; then
+ EXPORTER_ARGS+=(-p "$PORT")
+fi
+
+if [[ -n "$SCRAPE_INTERVAL" ]]; then
+ EXPORTER_ARGS+=(-s "$SCRAPE_INTERVAL")
+fi
+
+if [[ -n "$NUM_THREADS" ]]; then
+ EXPORTER_ARGS+=(-n "$NUM_THREADS")
+fi
+
+if [[ -n "$ZK_HOST" ]]; then
+ EXPORTER_ARGS+=(-z "$ZK_HOST")
+elif [[ -n "$SOLR_URL" ]]; then
+ EXPORTER_ARGS+=(-b "$SOLR_URL")
+fi
+
exec "$JAVACMD" \
$JAVA_MEM_OPTS \
$GC_TUNE \
$JAVA_OPTS \
$EXTRA_JVM_ARGUMENTS \
+ $ZK_CREDS_AND_ACLS \
-classpath "$CLASSPATH" \
-Dapp.name="solr-exporter" \
-Dapp.pid="$$" \
-Dapp.repo="$REPO" \
-Dbasedir="$BASEDIR" \
org.apache.solr.prometheus.exporter.SolrExporter \
+ "${EXPORTER_ARGS[@]}" \
"$@"
diff --git a/solr/contrib/prometheus-exporter/bin/solr-exporter.cmd b/solr/contrib/prometheus-exporter/bin/solr-exporter.cmd
index 84ccce9..efa92a1 100644
--- a/solr/contrib/prometheus-exporter/bin/solr-exporter.cmd
+++ b/solr/contrib/prometheus-exporter/bin/solr-exporter.cmd
@@ -73,12 +73,22 @@ if "%REPO%"=="" set REPO=%BASEDIR%\lib
set CLASSPATH=%REPO%\*;%BASEDIR%\conf;%BASEDIR%\..\..\dist\solrj-lib\*;%BASEDIR%\..\..\dist\*;%BASEDIR%\lucene-libs\*;%BASEDIR%\..\..\server\solr-webapp\webapp\WEB-INF\lib\*;%BASEDIR%\..\..\server\lib\ext\*
set EXTRA_JVM_ARGUMENTS=-Dlog4j.configurationFile=file:///%BASEDIR%\..\..\server\resources\log4j2-console.xml
+
+@REM Convert Environment Variables to Command Line Options
+set EXPORTER_ARGS=
+
+IF NOT "%CONFIG_FILE%"=="" set EXPORTER_ARGS=%EXPORTER_ARGS% -f %CONFIG_FILE%
+IF NOT "%PORT%"=="" set EXPORTER_ARGS=%EXPORTER_ARGS% -p %PORT%
+IF NOT "%SCRAPE_INTERVAL%"=="" set EXPORTER_ARGS=%EXPORTER_ARGS% -s %SCRAPE_INTERVAL%
+IF NOT "%NUM_THREADS%"=="" set EXPORTER_ARGS=%EXPORTER_ARGS% -n %NUM_THREADS%
+IF NOT "%ZK_HOST%"=="" set EXPORTER_ARGS=%EXPORTER_ARGS% -z %ZK_HOST%
+IF NOT "%SOLR_URL%"=="" set EXPORTER_ARGS=%EXPORTER_ARGS% -b %SOLR_URL%
goto endInit
@REM Reaching here means variables are defined and arguments have been captured
:endInit
-%JAVACMD% %JAVA_MEM% %GC_TUNE% %JAVA_OPTS% %EXTRA_JVM_ARGUMENTS% -classpath "%CLASSPATH_PREFIX%;%CLASSPATH%" -Dapp.name="solr-exporter" -Dapp.repo="%REPO%" -Dbasedir="%BASEDIR%" org.apache.solr.prometheus.exporter.SolrExporter %CMD_LINE_ARGS%
+%JAVACMD% %JAVA_MEM% %GC_TUNE% %JAVA_OPTS% %EXTRA_JVM_ARGUMENTS% %ZK_CREDS_AND_ACLS% -classpath "%CLASSPATH_PREFIX%;%CLASSPATH%" -Dapp.name="solr-exporter" -Dapp.repo="%REPO%" -Dbasedir="%BASEDIR%" org.apache.solr.prometheus.exporter.SolrExporter %EXPORTER_ARGS% %CMD_LINE_ARGS%
if ERRORLEVEL 1 goto error
goto end
diff --git a/solr/solr-ref-guide/src/monitoring-solr-with-prometheus-and-grafana.adoc b/solr/solr-ref-guide/src/monitoring-solr-with-prometheus-and-grafana.adoc
index 9fa01ae..98e104f 100644
--- a/solr/solr-ref-guide/src/monitoring-solr-with-prometheus-and-grafana.adoc
+++ b/solr/solr-ref-guide/src/monitoring-solr-with-prometheus-and-grafana.adoc
@@ -83,34 +83,44 @@ $ ./bin/solr-exporter -p 9854 -z localhost:2181/solr -f ./conf/solr-exporter-con
=== Command Line Parameters
-The parameters in the example start commands shown above:
+The list of available parameters for the Prometheus Exporter.
+All parameters can be provided via an environment variable, instead of through the command line.
`h`, `--help`::
Displays command line help and usage.
-`-p`, `--port`::
-The port where Prometheus will listen for new data. This port will be used to configure Prometheus. It can be any port not already in use on your server. The default is `9983`.
+`-p`, `--port`, `$PORT`::
+The port where Prometheus will listen for new data. This port will be used to configure Prometheus.
+It can be any port not already in use on your server. The default is `9983`.
-`-b`, `--baseurl`::
-The Solr base URL (such as `\http://localhost:8983/solr`) when Solr is running in Standalone mode. If you are running Solr in SolrCloud mode, do not specify this parameter. If neither the `-b` parameter nor the `-z` parameter are defined, the default is `-b \http://localhost:8983/solr`.
+`-b`, `--baseurl`, `$SOLR_URL`::
+The Solr base URL (such as `\http://localhost:8983/solr`) when Solr is running in Standalone mode.
+If you are running Solr in SolrCloud mode, do not specify this parameter.
+If neither the `-b` parameter nor the `-z` parameter are defined, the default is `-b \http://localhost:8983/solr`.
-`-z`, `--zkhost`::
-The ZooKeeper connect string (such as `localhost:8983`, or `localhost:2181/solr`) when Solr is running in SolrCloud mode. If you are running Solr in Standalone mode, do not specify this parameter. If neither the `-b` parameter nor the `-z` parameter are defined, the `-b` parameter default is used.
+`-z`, `--zkhost`, `$ZK_HOST`::
+The ZooKeeper connect string (such as `localhost:9983`, or `localhost:2181/solr`) when Solr is running in SolrCloud mode.
+If you are running Solr in Standalone mode, do not specify this parameter.
+If neither the `-b` parameter nor the `-z` parameter are defined, the `-b` parameter default is used.
-`-f`, `--config-file`::
+`-f`, `--config-file`, `$CONFIG_FILE`::
The path to the configuration file that defines the Solr metrics to read. The default is `contrib/prometheus-exporter/conf/solr-exporter-config.xml`.
-`-n`, `--num-threads`::
+`-n`, `--num-threads`, `$NUM_THREADS`::
The number of threads. The `solr-exporter` creates thread pools for requests to Solr. Request latency can be improved by increasing the number of threads. The default is `1`.
-`-s`, `--scrape-interval`::
-The number of seconds between collecting metrics from Solr. The `solr-exporter` collects metrics from Solr every few seconds controlled by this setting. These metrics are cached and returned regardless of how frequently prometheus is configured to pull metrics from this tool. The freshness of the metrics can be improved by reducing the scrape interval but do not set it to a very low value because metrics collection can be expensive and can execute arbitrary searches to ping Solr. The def [...]
+`-s`, `--scrape-interval`, `$SCRAPE_INTERVAL`::
+The number of seconds between collecting metrics from Solr.
+The `solr-exporter` collects metrics from Solr every few seconds controlled by this setting.
+These metrics are cached and returned regardless of how frequently prometheus is configured to pull metrics from this tool.
+The freshness of the metrics can be improved by reducing the scrape interval but do not set it to a very low value because metrics collection can be expensive and can execute arbitrary searches to ping Solr.
+The default value is 60 seconds.
The Solr's metrics exposed by `solr-exporter` can be seen at: `\http://localhost:9983/solr/admin/metrics`.
=== Environment Variable Options
-The start commands provided with the Prometheus Exporter support the use of custom java options through the following environment variables:
+The bin scripts provided with the Prometheus Exporter support the use of custom java options through the following environment variables:
`JAVA_HEAP`::
Sets the initial (`Xms`) and max (`Xmx`) Java heap size. The default is `512m`.
@@ -124,12 +134,20 @@ Custom Java garbage collection settings. The default is `-XX:+UseG1GC`.
`JAVA_OPTS`::
Extra JVM options.
+`ZK_CREDS_AND_ACLS`::
+Credentials for connecting to a ZK Host that is protected with ACLs.
+For more information on what to include in this variable, refer to the <<zookeeper-access-control.adoc#zookeeper-acls-in-solr-scripts,Solr ZK ACL docs>> or the <<#getting-metrics-from-a-secured-solrcloud,example below>>.
+
`CLASSPATH_PREFIX`::
Location of extra libraries to load when starting the `solr-exporter`.
+All <<#command-line-parameters,command line parameters>> are able to be provided via environment variables when using the bin scripts.
+
=== Getting Metrics from a Secured SolrCloud
-Your SolrCloud might be secured by measures described in <<securing-solr.adoc#securing-solr,Securing Solr>>. The security configuration can be injected into `solr-exporter` using environment variables in a fashion similar to other clients using <<using-solrj.adoc#using-solrj,SolrJ>>. This is possible because the main script picks up <<Environment Variable Options>> and passes them on to the Java process.
+Your SolrCloud might be secured by measures described in <<securing-solr.adoc#securing-solr,Securing Solr>>.
+The security configuration can be injected into `solr-exporter` using environment variables in a fashion similar to other clients using <<using-solrj.adoc#using-solrj,SolrJ>>.
+This is possible because the main script picks up <<Environment Variable Options>> and passes them on to the Java process.
Example for a SolrCloud instance secured by <<basic-authentication-plugin.adoc#basic-authentication-plugin,Basic Authentication>>, <<enabling-ssl.adoc#enabling-ssl,SSL>> and <<zookeeper-access-control.adoc#zookeeper-access-control,ZooKeeper Access Control>>:
@@ -145,7 +163,8 @@ Then you can start the Exporter as follows (Linux).
[source,bash]
----
$ cd contrib/prometheus-exporter
-$ export JAVA_OPTS="-Djavax.net.ssl.trustStore=truststore.p12 -Djavax.net.ssl.trustStorePassword=truststorePassword -Dsolr.httpclient.builder.factory=org.apache.solr.client.solrj.impl.PreemptiveBasicAuthClientBuilderFactory -Dsolr.httpclient.config=basicauth.properties -DzkCredentialsProvider=org.apache.solr.common.cloud.VMParamsSingleSetCredentialsDigestZkCredentialsProvider -DzkDigestUsername=readonly-user -DzkDigestPassword=zkUserPassword"
+$ export JAVA_OPTS="-Djavax.net.ssl.trustStore=truststore.p12 -Djavax.net.ssl.trustStorePassword=truststorePassword -Dsolr.httpclient.builder.factory=org.apache.solr.client.solrj.impl.PreemptiveBasicAuthClientBuilderFactory -Dsolr.httpclient.config=basicauth.properties"
+$ export ZK_CREDS_AND_ACLS="-DzkCredentialsProvider=org.apache.solr.common.cloud.VMParamsSingleSetCredentialsDigestZkCredentialsProvider -DzkDigestUsername=readonly-user -DzkDigestPassword=zkUserPassword"
$ export CLASSPATH_PREFIX="../../server/solr-webapp/webapp/WEB-INF/lib/commons-codec-1.11.jar"
$ ./bin/solr-exporter -p 9854 -z zk1:2181,zk2:2181,zk3:2181 -f ./conf/solr-exporter-config.xml -n 16
----