You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ol...@apache.org on 2017/07/15 17:11:54 UTC
ambari git commit: AMBARI-21459. Add lucene index migration script to
infra solr client package (oleewere)
Repository: ambari
Updated Branches:
refs/heads/trunk e799f5226 -> f072dd215
AMBARI-21459. Add lucene index migration script to infra solr client package (oleewere)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/f072dd21
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/f072dd21
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/f072dd21
Branch: refs/heads/trunk
Commit: f072dd2156e83d8a487ce0c3229c1ae22788c6be
Parents: e799f52
Author: oleewere <ol...@gmail.com>
Authored: Wed Jul 12 21:04:54 2017 +0200
Committer: oleewere <ol...@gmail.com>
Committed: Sat Jul 15 19:04:53 2017 +0200
----------------------------------------------------------------------
ambari-infra/ambari-infra-solr-client/build.xml | 1 +
ambari-infra/ambari-infra-solr-client/pom.xml | 10 ++
.../src/main/resources/solrIndexHelper.sh | 156 +++++++++++++++++++
3 files changed, 167 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/f072dd21/ambari-infra/ambari-infra-solr-client/build.xml
----------------------------------------------------------------------
diff --git a/ambari-infra/ambari-infra-solr-client/build.xml b/ambari-infra/ambari-infra-solr-client/build.xml
index a54e336..9b8b6cc 100644
--- a/ambari-infra/ambari-infra-solr-client/build.xml
+++ b/ambari-infra/ambari-infra-solr-client/build.xml
@@ -35,6 +35,7 @@
</copy>
<copy todir="target/package" includeEmptyDirs="no">
<fileset file="src/main/resources/solrCloudCli.sh"/>
+ <fileset file="src/main/resources/solrIndexHelper.sh"/>
</copy>
<copy todir="target/package" includeEmptyDirs="no">
<fileset file="src/main/resources/log4j.properties"/>
http://git-wip-us.apache.org/repos/asf/ambari/blob/f072dd21/ambari-infra/ambari-infra-solr-client/pom.xml
----------------------------------------------------------------------
diff --git a/ambari-infra/ambari-infra-solr-client/pom.xml b/ambari-infra/ambari-infra-solr-client/pom.xml
index d103003..3818aba 100644
--- a/ambari-infra/ambari-infra-solr-client/pom.xml
+++ b/ambari-infra/ambari-infra-solr-client/pom.xml
@@ -36,6 +36,16 @@
<version>${solr.version}</version>
</dependency>
<dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-core</artifactId>
+ <version>${solr.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-backward-codecs</artifactId>
+ <version>${solr.version}</version>
+ </dependency>
+ <dependency>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
<version>3.4.9</version>
http://git-wip-us.apache.org/repos/asf/ambari/blob/f072dd21/ambari-infra/ambari-infra-solr-client/src/main/resources/solrIndexHelper.sh
----------------------------------------------------------------------
diff --git a/ambari-infra/ambari-infra-solr-client/src/main/resources/solrIndexHelper.sh b/ambari-infra/ambari-infra-solr-client/src/main/resources/solrIndexHelper.sh
new file mode 100755
index 0000000..12e6a77
--- /dev/null
+++ b/ambari-infra/ambari-infra-solr-client/src/main/resources/solrIndexHelper.sh
@@ -0,0 +1,156 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+JVM="java"
+sdir="`dirname \"$0\"`"
+: ${JAVA_HOME:?"Please set the JAVA_HOME for lucene index migration!"}
+
+function print_help() {
+ cat << EOF
+
+ Usage: solrIndexHelper.sh [<command>] [<arguments with flags>]
+
+ commands:
+ upgrade-index Check and upgrade solr index data in core directories.
+ run-check-index-tool call 'java -cp ... org.apache.lucene.index.IndexUpgrader' directly
+ run-upgrade-index-tool call 'java -cp ... org.apache.lucene.index.CheckIndex' directly
+ help print usage
+
+
+ upgrade-index command arguments:
+ -d, --index-data-dir <DIRECTORY> Location of the solr cores (e.g.: /opt/ambari_infra_solr/data)
+ -c, --core-filter <FILTER1,FILTER2> Comma separated name filters of core directoies (default: hadoop_logs,audit_logs,history)
+ -f, --force Force to start index upgrade, even is the version is at least 6.
+
+EOF
+}
+
+function upgrade_core() {
+ local INDEX_DIR=${1:?"usage: <index_base_dir> e.g.: /opt/ambari_infra_solr/data"}
+ local FORCE_UPDATE=${2:?"usage <force_update_flag> e.g.: true"}
+ local SOLR_CORE_FILTERS=${3:?"usage: <comma separated core filters> e.g.: hadoop_logs,audit_logs,history"}
+
+ SOLR_CORE_FILTER_ARR=$(echo $SOLR_CORE_FILTERS | sed "s/,/ /g")
+
+ for coll in $SOLR_CORE_FILTER_ARR; do
+ if [[ "$1" == *"$coll"* ]]; then
+ echo "Core '$1' dir name contains $coll (core filter)'";
+ version=$(PATH=$JAVA_HOME/bin:$PATH $JVM -classpath "$sdir/libs/lucene-core-6.6.0.jar:$sdir/libs/lucene-backward-codecs-6.6.0.jar" org.apache.lucene.index.CheckIndex -fast $1|grep " version="|sed -e 's/.*=//g'|head -1)
+ if [ -z $version ] ; then
+ echo "Core '$1' - Empty index?"
+ return
+ fi
+ majorVersion=$(echo $version|cut -c 1)
+ if [ $majorVersion -ge 6 ] && [ $FORCE_UPDATE == "false" ] ; then
+ echo "Core '$1' - Already on version $version, not upgrading. Use -f or --force option to run upgrade anyway."
+ else
+ echo "Core '$1' - Index version is $version, upgrading ..."
+ PATH=$JAVA_HOME/bin:$PATH $JVM -classpath "$sdir/libs/lucene-core-6.6.0.jar:$sdir/libs/lucene-backward-codecs-6.6.0.jar" org.apache.lucene.index.IndexUpgrader -delete-prior-commits $1
+ echo "Upgrading core '$1' has finished"
+ fi
+ fi
+ done
+}
+
+function upgrade_index() {
+ while [[ $# -gt 0 ]]
+ do
+ key="$1"
+ case $key in
+ -c|--core-filters)
+ local SOLR_CORE_FILTERS="$2"
+ shift 2
+ ;;
+ -f|--force)
+ local FORCE_UPDATE="true"
+ shift
+ ;;
+ -d|--index-data-dir)
+ local INDEX_DIR="$2"
+ shift 2
+ ;;
+ *)
+ echo "Unknown option: $1"
+ exit 1
+ ;;
+ esac
+ done
+ if [[ -z "$INDEX_DIR" ]] ; then
+ echo "Index data dirctory option is required (-d or --index-data-dir). Exiting..."
+ exit 1
+ fi
+
+ if [[ -z "$SOLR_CORE_FILTERS" ]] ; then
+ SOLR_CORE_FILTERS="hadoop_logs,audit_logs,history"
+ fi
+
+ if [[ -z "$FORCE_UPDATE" ]] ; then
+ FORCE_UPDATE="false"
+ else
+ echo "NOTE: Forcing index upgrade is set."
+ fi
+
+ CORES=$(for replica_dir in `find $INDEX_DIR -name data`; do dirname $replica_dir; done);
+ if [[ -z "$CORES" ]] ; then
+ echo "No indices found on path $INDEX_DIR"
+ else
+ for c in $CORES ; do
+ if find $c/data -maxdepth 1 -type d -name 'index*' 1> /dev/null 2>&1; then
+ name=$(echo $c | sed -e 's/.*\///g')
+ abspath=$(cd "$(dirname "$c")"; pwd)/$(basename "$c")
+ find $c/data -maxdepth 1 -type d -name 'index*' | while read indexDir; do
+ echo "Checking core $name - $abspath"
+ upgrade_core "$indexDir" "$FORCE_UPDATE" "$SOLR_CORE_FILTERS"
+ done
+ else
+ echo "No index folder found for $name"
+ fi
+ done
+ echo "DONE"
+ fi
+}
+
+function upgrade_index_tool() {
+ # see: https://cwiki.apache.org/confluence/display/solr/IndexUpgrader+Tool
+ PATH=$JAVA_HOME/bin:$PATH $JVM -classpath "$sdir/libs/lucene-core-6.6.0.jar:$sdir/libs/lucene-backward-codecs-6.6.0.jar" org.apache.lucene.index.IndexUpgrader ${@}
+}
+
+function check_index_tool() {
+ PATH=$JAVA_HOME/bin:$PATH $JVM -classpath "$sdir/libs/lucene-core-6.6.0.jar:$sdir/libs/lucene-backward-codecs-6.6.0.jar" org.apache.lucene.index.CheckIndex ${@}
+}
+
+function main() {
+ command="$1"
+ case $command in
+ "upgrade-index")
+ upgrade_index "${@:2}"
+ ;;
+ "run-check-index-tool")
+ check_index_tool "${@:2}"
+ ;;
+ "run-upgrade-index-tool")
+ upgrade_index_tool "${@:2}"
+ ;;
+ "help")
+ print_help
+ ;;
+ *)
+ echo "Available commands: (upgrade-index | run-check-index-tool | run-upgrade-index-tool | help)"
+ ;;
+ esac
+}
+
+main ${1+"$@"}