You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ko...@apache.org on 2018/02/20 09:08:36 UTC
[7/7] lucene-solr:branch_7x: SOLR-11795: Add Solr metrics exporter
for Prometheus
SOLR-11795: Add Solr metrics exporter for Prometheus
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/d5a01e02
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/d5a01e02
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/d5a01e02
Branch: refs/heads/branch_7x
Commit: d5a01e02687c4f88a2f80ac930e27188f2703385
Parents: 1b2f8a6
Author: koji <ko...@apache.org>
Authored: Tue Feb 20 18:08:08 2018 +0900
Committer: koji <ko...@apache.org>
Committed: Tue Feb 20 18:08:08 2018 +0900
----------------------------------------------------------------------
solr/CHANGES.txt | 2 +
solr/build.xml | 6 +-
solr/contrib/prometheus-exporter/README.md | 21 +
solr/contrib/prometheus-exporter/README.txt | 1 +
.../prometheus-exporter/bin/solr-exporter | 114 +
.../prometheus-exporter/bin/solr-exporter.cmd | 105 +
solr/contrib/prometheus-exporter/build.xml | 30 +
.../contrib/prometheus-exporter/conf/config.yml | 1840 ++++++++
.../conf/grafana-solr-dashboard.json | 4465 ++++++++++++++++++
.../prometheus-exporter/conf/log4j.properties | 22 +
solr/contrib/prometheus-exporter/ivy.xml | 41 +
.../prometheus/collector/SolrCollector.java | 402 ++
.../collector/config/SolrCollectorConfig.java | 64 +
.../solr/prometheus/exporter/SolrExporter.java | 254 +
.../solr/prometheus/scraper/SolrScraper.java | 218 +
.../scraper/config/SolrQueryConfig.java | 99 +
.../scraper/config/SolrScraperConfig.java | 60 +
.../prometheus-exporter/src/java/overview.html | 21 +
.../src/test-files/conf/config.yml | 1840 ++++++++
.../src/test-files/conf/log4j.properties | 22 +
.../configsets/collection1/conf/managed-schema | 412 ++
.../configsets/collection1/conf/solrconfig.xml | 232 +
.../configsets/collection1/conf/stopwords.txt | 14 +
.../configsets/collection1/conf/synonyms.txt | 29 +
.../test-files/exampledocs/gb18030-example.xml | 32 +
.../src/test-files/exampledocs/hd.xml | 56 +
.../src/test-files/exampledocs/ipod_other.xml | 60 +
.../src/test-files/exampledocs/ipod_video.xml | 40 +
.../test-files/exampledocs/manufacturers.xml | 75 +
.../src/test-files/exampledocs/mem.xml | 77 +
.../src/test-files/exampledocs/money.xml | 65 +
.../src/test-files/exampledocs/monitor.xml | 34 +
.../src/test-files/exampledocs/monitor2.xml | 33 +
.../src/test-files/exampledocs/mp500.xml | 43 +
.../src/test-files/exampledocs/sample.html | 13 +
.../src/test-files/exampledocs/sd500.xml | 38 +
.../src/test-files/exampledocs/solr.xml | 38 +
.../src/test-files/exampledocs/utf8-example.xml | 42 +
.../src/test-files/exampledocs/vidcard.xml | 62 +
.../prometheus/collector/SolrCollectorTest.java | 93 +
.../config/SolrCollectorConfigTest.java | 106 +
.../prometheus/exporter/SolrExporterTest.java | 106 +
.../exporter/SolrExporterTestBase.java | 53 +
.../scraper/config/SolrQueryConfigTest.java | 121 +
.../scraper/config/SolrScraperConfigTest.java | 86 +
.../grafana-solr-dashboard.png | Bin 0 -> 808611 bytes
.../solr-exporter-diagram.png | Bin 0 -> 29070 bytes
...toring-solr-with-prometheus-and-grafana.adoc | 250 +
solr/solr-ref-guide/src/monitoring-solr.adoc | 4 +-
49 files changed, 11837 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d5a01e02/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 619e3da..3903715 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -148,6 +148,8 @@ New Features
* SOLR-11588: Add matrixMult Stream Evaluator to support matrix multiplication (Joel Bernstein)
+* SOLR-11795: Add Solr metrics exporter for Prometheus (Minoru Osuka via koji)
+
Bug Fixes
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d5a01e02/solr/build.xml
----------------------------------------------------------------------
diff --git a/solr/build.xml b/solr/build.xml
index 400c711..3f0596f 100644
--- a/solr/build.xml
+++ b/solr/build.xml
@@ -571,19 +571,19 @@
<tarfileset dir="."
prefix="${fullnamever}"
includes="LICENSE.txt NOTICE.txt CHANGES.txt README.txt SYSTEM_REQUIREMENTS.txt
- bin/** server/** example/** contrib/**/lib/** contrib/**/README.txt
+ bin/** server/** example/** contrib/**/lib/** contrib/**/conf/** contrib/**/README.txt
licenses/**"
excludes="licenses/README.committers.txt **/data/ **/logs/*
**/classes/ **/*.sh **/ivy.xml **/build.xml
**/bin/ **/*.iml **/*.ipr **/*.iws **/pom.xml
- **/*pom.xml.template server/etc/test/" />
+ **/*pom.xml.template server/etc/test/ contrib/**/src/" />
<tarfileset dir="${dest}/contrib-lucene-libs-to-package"
prefix="${fullnamever}"
includes="**" />
<tarfileset dir="."
filemode="755"
prefix="${fullnamever}"
- includes="bin/** server/**/*.sh example/**/*.sh example/**/bin/"
+ includes="bin/** server/**/*.sh example/**/*.sh example/**/bin/ contrib/**/bin/**"
excludes="server/etc/test/**" />
<tarfileset dir="."
prefix="${fullnamever}"
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d5a01e02/solr/contrib/prometheus-exporter/README.md
----------------------------------------------------------------------
diff --git a/solr/contrib/prometheus-exporter/README.md b/solr/contrib/prometheus-exporter/README.md
new file mode 100644
index 0000000..888f237
--- /dev/null
+++ b/solr/contrib/prometheus-exporter/README.md
@@ -0,0 +1,21 @@
+Welcome to Apache Solr Prometheus Exporter
+========
+
+Apache Solr Prometheus Exporter (solr-exporter) provides a way for you to expose metrics for Solr to Prometheus.
+
+# Getting Started With Solr Prometheus Exporter
+
+For information on how to get started with solr-exporter please see:
+ * [Solr Reference Guide's section on Monitoring Solr with Prometheus and Grafana](https://lucene.apache.org/solr/guide/monitoring-solr-with-prometheus-and-grafana.html)
+
+# Getting Started With Solr
+
+For information on how to get started with solr please see:
+ * [solr/README.txt](../../README.txt)
+ * [Solr Tutorial](https://lucene.apache.org/solr/guide/solr-tutorial.html)
+
+# How To Contribute
+
+For information on how to contribute see:
+ * http://wiki.apache.org/lucene-java/HowToContribute
+ * http://wiki.apache.org/solr/HowToContribute
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d5a01e02/solr/contrib/prometheus-exporter/README.txt
----------------------------------------------------------------------
diff --git a/solr/contrib/prometheus-exporter/README.txt b/solr/contrib/prometheus-exporter/README.txt
new file mode 100644
index 0000000..42061c0
--- /dev/null
+++ b/solr/contrib/prometheus-exporter/README.txt
@@ -0,0 +1 @@
+README.md
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d5a01e02/solr/contrib/prometheus-exporter/bin/solr-exporter
----------------------------------------------------------------------
diff --git a/solr/contrib/prometheus-exporter/bin/solr-exporter b/solr/contrib/prometheus-exporter/bin/solr-exporter
new file mode 100644
index 0000000..d69d00b
--- /dev/null
+++ b/solr/contrib/prometheus-exporter/bin/solr-exporter
@@ -0,0 +1,114 @@
+#!/bin/sh
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+BASEDIR=`dirname $0`/..
+BASEDIR=`(cd "$BASEDIR"; pwd)`
+
+# OS specific support. $var _must_ be set to either true or false.
+cygwin=false;
+darwin=false;
+case "`uname`" in
+ CYGWIN*) cygwin=true ;;
+ Darwin*) darwin=true
+ if [ -z "$JAVA_VERSION" ] ; then
+ JAVA_VERSION="CurrentJDK"
+ else
+ echo "Using Java version: $JAVA_VERSION"
+ fi
+ if [ -z "$JAVA_HOME" ] ; then
+ JAVA_HOME=/System/Library/Frameworks/JavaVM.framework/Versions/${JAVA_VERSION}/Home
+ fi
+ ;;
+esac
+
+if [ -z "$JAVA_HOME" ] ; then
+ if [ -r /etc/gentoo-release ] ; then
+ JAVA_HOME=`java-config --jre-home`
+ fi
+fi
+
+# For Cygwin, ensure paths are in UNIX format before anything is touched
+if $cygwin ; then
+ [ -n "$JAVA_HOME" ] && JAVA_HOME=`cygpath --unix "$JAVA_HOME"`
+ [ -n "$CLASSPATH" ] && CLASSPATH=`cygpath --path --unix "$CLASSPATH"`
+fi
+
+# If a specific java binary isn't specified search for the standard 'java' binary
+if [ -z "$JAVACMD" ] ; then
+ if [ -n "$JAVA_HOME" ] ; then
+ if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
+ # IBM's JDK on AIX uses strange locations for the executables
+ JAVACMD="$JAVA_HOME/jre/sh/java"
+ else
+ JAVACMD="$JAVA_HOME/bin/java"
+ fi
+ else
+ JAVACMD=`which java`
+ fi
+fi
+
+if [ ! -x "$JAVACMD" ] ; then
+ echo "Error: JAVA_HOME is not defined correctly."
+ echo " We cannot execute $JAVACMD"
+ exit 1
+fi
+
+if [ -z "$REPO" ]
+then
+ REPO="$BASEDIR"/lib
+fi
+
+CLASSPATH=$CLASSPATH_PREFIX
+for JAR in $(find "$REPO" -name '*.jar')
+do
+ CLASSPATH="$CLASSPATH":"$JAR"
+done
+for JAR in $(find "$BASEDIR"/../../dist/solrj-lib -name '*.jar')
+do
+ CLASSPATH="$CLASSPATH":"$JAR"
+done
+for JAR in $(find "$BASEDIR"/../../dist -name 'solr-solrj-*.jar')
+do
+ CLASSPATH="$CLASSPATH":"$JAR"
+done
+for JAR in $(find "$BASEDIR"/../../dist -name 'solr-prometheus-exporter-*.jar')
+do
+ CLASSPATH="$CLASSPATH":"$JAR"
+done
+
+EXTRA_JVM_ARGUMENTS="-Xmx512m -Dlog4j.configuration=file:"$BASEDIR"/conf/log4j.properties"
+
+# For Cygwin, switch paths to Windows format before running java
+if $cygwin; then
+ [ -n "$CLASSPATH" ] && CLASSPATH=`cygpath --path --windows "$CLASSPATH"`
+ [ -n "$JAVA_HOME" ] && JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"`
+ [ -n "$HOME" ] && HOME=`cygpath --path --windows "$HOME"`
+ [ -n "$BASEDIR" ] && BASEDIR=`cygpath --path --windows "$BASEDIR"`
+ [ -n "$REPO" ] && REPO=`cygpath --path --windows "$REPO"`
+fi
+
+exec "$JAVACMD" $JAVA_OPTS \
+ $EXTRA_JVM_ARGUMENTS \
+ -classpath "$CLASSPATH" \
+ -Dapp.name="solr-exporter" \
+ -Dapp.pid="$$" \
+ -Dapp.repo="$REPO" \
+ -Dbasedir="$BASEDIR" \
+ org.apache.solr.prometheus.exporter.SolrExporter \
+ "$@"
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d5a01e02/solr/contrib/prometheus-exporter/bin/solr-exporter.cmd
----------------------------------------------------------------------
diff --git a/solr/contrib/prometheus-exporter/bin/solr-exporter.cmd b/solr/contrib/prometheus-exporter/bin/solr-exporter.cmd
new file mode 100644
index 0000000..a09463b
--- /dev/null
+++ b/solr/contrib/prometheus-exporter/bin/solr-exporter.cmd
@@ -0,0 +1,105 @@
+@REM
+@REM Licensed to the Apache Software Foundation (ASF) under one or more
+@REM contributor license agreements. See the NOTICE file distributed with
+@REM this work for additional information regarding copyright ownership.
+@REM The ASF licenses this file to You under the Apache License, Version 2.0
+@REM (the "License"); you may not use this file except in compliance with
+@REM the License. You may obtain a copy of the License at
+@REM
+@REM http://www.apache.org/licenses/LICENSE-2.0
+@REM
+@REM Unless required by applicable law or agreed to in writing, software
+@REM distributed under the License is distributed on an "AS IS" BASIS,
+@REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+@REM See the License for the specific language governing permissions and
+@REM limitations under the License.
+@REM
+
+@echo off
+
+set ERROR_CODE=0
+
+:init
+@REM Decide how to startup depending on the version of windows
+
+@REM -- Win98ME
+if NOT "%OS%"=="Windows_NT" goto Win9xArg
+
+@REM set local scope for the variables with windows NT shell
+if "%OS%"=="Windows_NT" @setlocal
+
+@REM -- 4NT shell
+if "%eval[2+2]" == "4" goto 4NTArgs
+
+@REM -- Regular WinNT shell
+set CMD_LINE_ARGS=%*
+goto WinNTGetScriptDir
+
+@REM The 4NT Shell from jp software
+:4NTArgs
+set CMD_LINE_ARGS=%$
+goto WinNTGetScriptDir
+
+:Win9xArg
+@REM Slurp the command line arguments. This loop allows for an unlimited number
+@REM of agruments (up to the command line limit, anyway).
+set CMD_LINE_ARGS=
+:Win9xApp
+if %1a==a goto Win9xGetScriptDir
+set CMD_LINE_ARGS=%CMD_LINE_ARGS% %1
+shift
+goto Win9xApp
+
+:Win9xGetScriptDir
+set SAVEDIR=%CD%
+%0\
+cd %0\..\..
+set BASEDIR=%CD%
+cd %SAVEDIR%
+set SAVE_DIR=
+goto repoSetup
+
+:WinNTGetScriptDir
+set BASEDIR=%~dp0\..
+
+:repoSetup
+
+
+if "%JAVACMD%"=="" set JAVACMD=java
+
+if "%REPO%"=="" set REPO=%BASEDIR%\lib
+
+set CLASSPATH="%CLASSPATH%";"%REPO%\*;%BASEDIR%\..\..\dist\solrj-lib\*;%BASEDIR%\..\..\dist\solr-solrj-*;%BASEDIR%\..\..\dist\solr-prometheus-exporter-*"
+set EXTRA_JVM_ARGUMENTS=-Xmx512m -Dlog4j.configuration=file:%BASEDIR%/conf/log4j.properties
+goto endInit
+
+@REM Reaching here means variables are defined and arguments have been captured
+:endInit
+
+%JAVACMD% %JAVA_OPTS% %EXTRA_JVM_ARGUMENTS% -classpath %CLASSPATH_PREFIX%;%CLASSPATH% -Dapp.name="solr-exporter" -Dapp.repo="%REPO%" -Dbasedir="%BASEDIR%" com.github.mosuka.solr.prometheus.exporter.SolrExporter %CMD_LINE_ARGS%
+if ERRORLEVEL 1 goto error
+goto end
+
+:error
+if "%OS%"=="Windows_NT" @endlocal
+set ERROR_CODE=1
+
+:end
+@REM set local scope for the variables with windows NT shell
+if "%OS%"=="Windows_NT" goto endNT
+
+@REM For old DOS remove the set variables from ENV - we assume they were not set
+@REM before we started - at least we don't leave any baggage around
+set CMD_LINE_ARGS=
+goto postExec
+
+:endNT
+@endlocal
+
+:postExec
+
+if "%FORCE_EXIT_ON_ERROR%" == "on" (
+ if %ERROR_CODE% NEQ 0 exit %ERROR_CODE%
+)
+
+exit /B %ERROR_CODE%
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d5a01e02/solr/contrib/prometheus-exporter/build.xml
----------------------------------------------------------------------
diff --git a/solr/contrib/prometheus-exporter/build.xml b/solr/contrib/prometheus-exporter/build.xml
new file mode 100644
index 0000000..ff11769
--- /dev/null
+++ b/solr/contrib/prometheus-exporter/build.xml
@@ -0,0 +1,30 @@
+<?xml version="1.0"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
+<project name="solr-prometheus-exporter" default="default">
+
+ <description>
+ Prometheus exporter for exposing metrics from Solr using Metrics API and Search API.
+ </description>
+
+ <import file="../contrib-build.xml"/>
+
+ <target name="compile-core" depends="solr-contrib-build.compile-core"/>
+ <target name="compile-test" depends="common-solr.compile-test"/>
+</project>