You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ol...@apache.org on 2018/06/25 14:21:14 UTC
[ambari] branch trunk updated: AMBARI-23945. Infra Solr Migration -
add --keep-backup to automation script + use elapsed time as well
This is an automated email from the ASF dual-hosted git repository.
oleewere pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/trunk by this push:
new 934a18d AMBARI-23945. Infra Solr Migration - add --keep-backup to automation script + use elapsed time as well
934a18d is described below
commit 934a18d65559cde7255fd8c49079e325f67fb3fd
Author: Oliver Szabo <ol...@gmail.com>
AuthorDate: Mon Jun 25 16:20:11 2018 +0200
AMBARI-23945. Infra Solr Migration - add --keep-backup to automation script + use elapsed time as well
---
ambari-infra/ambari-infra-solr-client/README.md | 3 +-
.../src/main/python/migrationHelper.py | 8 ++-
.../src/main/resources/ambariSolrMigration.sh | 66 +++++++++++++++-------
3 files changed, 55 insertions(+), 22 deletions(-)
diff --git a/ambari-infra/ambari-infra-solr-client/README.md b/ambari-infra/ambari-infra-solr-client/README.md
index 53c6a3e..6ce9f39 100644
--- a/ambari-infra/ambari-infra-solr-client/README.md
+++ b/ambari-infra/ambari-infra-solr-client/README.md
@@ -704,7 +704,7 @@ For doing a backup + cleanup, then later migrate + restore, use the following co
/usr/lib/ambari-infra-solr-client/ambariSolrMigration.sh --ini-file $CONFIG_INI_LOCATION --mode backup
/usr/lib/ambari-infra-solr-client/ambariSolrMigration.sh --ini-file $CONFIG_INI_LOCATION --mode delete --skip-solr-client-upgrade
# go ahead with HDP upgrade or anything else, then if you have resource / time:
-/usr/lib/ambari-infra-solr-client/ambariSolrMigration.sh --ini-file $CONFIG_INI_LOCATION --mode migrate-restore
+/usr/lib/ambari-infra-solr-client/ambariSolrMigration.sh --ini-file $CONFIG_INI_LOCATION --mode migrate-restore # you can use --keep-backup option, it will keep the backup data, it's more safe but you need enough pace for that
```
Or you can execute these commands together (if you won't go with HDP upgrade after backup):
@@ -1020,6 +1020,7 @@ Usage: /usr/lib/ambari-infra-solr-client/ambariSolrMigration.sh --mode <MODE> --
-w, --wait-between-steps <seconds> wait between different migration steps in seconds (default: 15)
-p, --python-path python location, default: /usr/bin/python
-b, --batch-interval seconds between batch tasks for rolling restart solr at last step (default: 60)
+ -k, --keep-backup keep backup data (more secure, useful if you have enough space for that)
--skip-solr-client-upgrade skip ambari-infra-solr-client package upgrades
--skip-solr-server-upgrade skip ambari-infra-solr package upgrades
--skip-logsearch-upgrade skip ambari-logsearch-portal and ambari-logsearch-logfeeder package upgrades
diff --git a/ambari-infra/ambari-infra-solr-client/src/main/python/migrationHelper.py b/ambari-infra/ambari-infra-solr-client/src/main/python/migrationHelper.py
index 0bb8d38..fb82ac2 100755
--- a/ambari-infra/ambari-infra-solr-client/src/main/python/migrationHelper.py
+++ b/ambari-infra/ambari-infra-solr-client/src/main/python/migrationHelper.py
@@ -1774,6 +1774,8 @@ if __name__=="__main__":
config = ConfigParser.RawConfigParser()
config.read(options.ini_file)
+ command_start_time = time.time()
+
service_filter=options.service_filter.upper().split(',') if options.service_filter is not None else ['LOGSEARCH', 'ATLAS', 'RANGER']
if options.action is None:
@@ -1887,5 +1889,9 @@ if __name__=="__main__":
' upgrade-logfeeders | stop-logsearch | restart-solr |' \
' restart-logsearch | restart-ranger | restart-atlas)'
sys.exit(1)
-
+ command_elapsed_time = time.time() - command_start_time
+ time_to_print = time.strftime("%H:%M:%S", time.gmtime(command_elapsed_time))
+ print 30 * "-"
+ print "Command elapsed time: {0}".format(time_to_print)
+ print 30 * "-"
print "Migration helper command {0}FINISHED{1}".format(colors.OKGREEN, colors.ENDC)
\ No newline at end of file
diff --git a/ambari-infra/ambari-infra-solr-client/src/main/resources/ambariSolrMigration.sh b/ambari-infra/ambari-infra-solr-client/src/main/resources/ambariSolrMigration.sh
index 2dc4ee4..df18a88 100755
--- a/ambari-infra/ambari-infra-solr-client/src/main/resources/ambariSolrMigration.sh
+++ b/ambari-infra/ambari-infra-solr-client/src/main/resources/ambariSolrMigration.sh
@@ -24,6 +24,7 @@ function print_help() {
-w, --wait-between-steps <seconds> wait between different migration steps in seconds (default: 15)
-p, --python-path python location, default: /usr/bin/python
-b, --batch-interval seconds between batch tasks for rolling restart solr at last step (default: 60)
+ -k, --keep-backup keep backup data (more secure, useful if you have enough space for that)
--skip-solr-client-upgrade skip ambari-infra-solr-client package upgrades
--skip-solr-server-upgrade skip ambari-infra-solr package upgrades
--skip-logsearch-upgrade skip ambari-logsearch-portal and ambari-logsearch-logfeeder package upgrades
@@ -35,8 +36,13 @@ EOF
function handle_result() {
local result_code=${1:?"usage: <last_command_result>"}
local step=${2:?"usage: <step_context"}
+ local python_location=${3:?"usage: <python_location>"}
+ local start_date=${4:?"usage: <date>"}
if [[ "$result_code" != "0" ]] ; then
- echo "$step command failed. Stop migration commands ..."
+ end_date=$(date +%s)
+ runtime=$($python_location -c "print '%02u:%02u:%02u' % ((${end_date} - ${start_date})/3600, ((${end_date} - ${start_date})/60)%60, (${end_date} - ${start_date})%60)")
+ echo "Total Runtime: $runtime"
+ echo "$step command FAILED. Stop migration commands ..."
exit 1
fi
}
@@ -63,7 +69,8 @@ function run_migrate_commands() {
local skip_logsearch_upgrade=${8:?"usage <true|false>"}
local skip_warnings=${9:?"usage <true|false>"}
local batch_interval=${10:?"usage <seconds>"}
- local verbose=${11:?"usage <true|false>"}
+ local keep_backup=${11:?"usage <true|false>"}
+ local verbose=${12:?"usage <true|false>"}
local verbose_val=""
if [[ "$verbose" == "true" ]]; then
@@ -75,11 +82,18 @@ function run_migrate_commands() {
skip_warnings_val="--skip-warnings"
fi
+ local keep_backup_val=""
+ if [[ "$verbose" == "true" ]]; then
+ keep_backup_val="--keep-backup"
+ fi
+
+ start_date=$(date +%s)
+
# execute on: backup - all
if [[ "$mode" == "backup" || "$mode" == "all" ]] ; then
log_command "$python_location $script_location --ini-file $ini_file --action check-shards $verbose_val $skip_warnings_val"
$python_location $script_location --ini-file $ini_file --action check-shards $verbose_val $skip_warnings_val
- handle_result "$?" "Check Shards"
+ handle_result "$?" "Check Shards" "$python_location" "$start_date"
fi
# execute on: backup - delete - all
@@ -87,25 +101,25 @@ function run_migrate_commands() {
if [[ "$skip_solr_client_upgrade" != "true" ]]; then
log_command "$python_location $script_location --ini-file $ini_file --action upgrade-solr-clients $verbose_val"
$python_location $script_location --ini-file $ini_file --action upgrade-solr-clients $verbose_val
- handle_result "$?" "Upgrade Solr Clients"
+ handle_result "$?" "Upgrade Solr Clients" "$python_location" "$start_date"
fi
log_command "$python_location $script_location --ini-file $ini_file --action check-docs $verbose_val"
$python_location $script_location --ini-file $ini_file --action check-docs $verbose_val
- handle_result "$?" "Check Documents"
+ handle_result "$?" "Check Documents" "$python_location" "$start_date"
fi
# ececute on: backup - all
if [[ "$mode" == "backup" || "$mode" == "all" ]] ; then
log_command "$python_location $script_location --ini-file $ini_file --action backup $verbose_val"
$python_location $script_location --ini-file $ini_file --action backup $verbose_val
- handle_result "$?" "Backup"
+ handle_result "$?" "Backup" "$python_location" "$start_date"
fi
# execute on: delete - all
if [[ "$mode" == "delete" || "$mode" == "all" ]] ; then
log_command "$python_location $script_location --ini-file $ini_file --action delete-collections $verbose_val"
$python_location $script_location --ini-file $ini_file --action delete-collections $verbose_val
- handle_result "$?" "Delete collections"
+ handle_result "$?" "Delete collections" "$python_location" "$start_date"
fi
# execute on: delete - all
@@ -113,7 +127,7 @@ function run_migrate_commands() {
if [[ "$skip_solr_server_upgrade" != "true" ]]; then
log_command "$python_location $script_location --ini-file $ini_file --action upgrade-solr-instances $verbose_val"
$python_location $script_location --ini-file $ini_file --action upgrade-solr-instances $verbose_val
- handle_result "$?" "Upgrade Solr Instances"
+ handle_result "$?" "Upgrade Solr Instances" "$python_location" "$start_date"
fi
fi
@@ -121,30 +135,30 @@ function run_migrate_commands() {
if [[ "$mode" == "delete" || "$mode" == "all" ]] ; then
log_command "$python_location $script_location --ini-file $ini_file --action restart-solr $verbose_val"
$python_location $script_location --ini-file $ini_file --action restart-solr $verbose_val
- handle_result "$?" "Restart Solr Instances"
+ handle_result "$?" "Restart Solr Instances" "$python_location" "$start_date"
wait $time_sleep
log_command "$python_location $script_location --ini-file $ini_file --action restart-ranger $verbose_val"
$python_location $script_location --ini-file $ini_file --action restart-ranger $verbose_val
- handle_result "$?" "Restart Ranger Admins"
+ handle_result "$?" "Restart Ranger Admins" "$python_location" "$start_date"
wait $time_sleep
if [[ "$skip_logsearch_upgrade" != "true" ]]; then
log_command "$python_location $script_location --ini-file $ini_file --action upgrade-logsearch-portal $verbose_val"
$python_location $script_location --ini-file $ini_file --action upgrade-logsearch-portal $verbose_val
- handle_result "$?" "Upgrade Log Search Portal"
+ handle_result "$?" "Upgrade Log Search Portal" "$python_location" "$start_date"
log_command "$python_location $script_location --ini-file $ini_file --action upgrade-logfeeders $verbose_val"
$python_location $script_location --ini-file $ini_file --action upgrade-logfeeders $verbose_val
- handle_result "$?" "Upgrade Log Feeders"
+ handle_result "$?" "Upgrade Log Feeders" "$python_location" "$start_date"
fi
log_command "$python_location $script_location --ini-file $ini_file --action restart-logsearch $verbose_val"
$python_location $script_location --ini-file $ini_file --action restart-logsearch $verbose_val
- handle_result "$?" "Restart Log Search"
+ handle_result "$?" "Restart Log Search" "$python_location" "$start_date"
wait $time_sleep
log_command "$python_location $script_location --ini-file $ini_file --action restart-atlas $verbose_val"
$python_location $script_location --ini-file $ini_file --action restart-atlas $verbose_val
- handle_result "$?" "Restart Atlas Servers"
+ handle_result "$?" "Restart Atlas Servers" "$python_location" "$start_date"
wait $time_sleep
fi
@@ -152,20 +166,24 @@ function run_migrate_commands() {
if [[ "$mode" == "migrate-restore" || "$mode" == "all" ]] ; then
log_command "$python_location $script_location --ini-file $ini_file --action check-docs $verbose_val"
$python_location $script_location --ini-file $ini_file --action check-docs $verbose_val
- handle_result "$?" "Check Documents"
+ handle_result "$?" "Check Documents" "$python_location" "$start_date"
log_command "$python_location $script_location --ini-file $ini_file --action migrate $verbose_val"
$python_location $script_location --ini-file $ini_file --action migrate $verbose_val
- handle_result "$?" "Migrate Index"
+ handle_result "$?" "Migrate Index" "$python_location" "$start_date"
- log_command "$python_location $script_location --ini-file $ini_file --action restore $verbose_val"
+ log_command "$python_location $script_location --ini-file $ini_file --action restore $keep_backup_val $verbose_val"
$python_location $script_location --ini-file $ini_file --action restore $verbose_val
- handle_result "$?" "Restore"
+ handle_result "$?" "Restore" "$python_location" "$start_date"
log_command "$python_location $script_location --ini-file $ini_file --action rolling-restart-solr $verbose_val --batch-interval $batch_interval"
$python_location $script_location --ini-file $ini_file --action rolling-restart-solr $verbose_val --batch-interval $batch_interval
- handle_result "$?" "Rolling Restart Solr"
+ handle_result "$?" "Rolling Restart Solr" "$python_location" "$start_date"
fi
+
+ end_date=$(date +%s)
+ runtime=$($python_location -c "print '%02u:%02u:%02u' % ((${end_date} - ${start_date})/3600, ((${end_date} - ${start_date})/60)%60, (${end_date} - ${start_date})%60)")
+ echo "Total Runtime: $runtime"
}
function main() {
@@ -197,6 +215,10 @@ function main() {
local BATCH_INTERVAL="$2"
shift 2
;;
+ -k|--keep-backup)
+ local KEEP_BACKUP="true"
+ shift 1
+ ;;
--skip-solr-client-upgrade)
local SKIP_SOLR_CLIENT_UPGRADE="true"
shift 1
@@ -265,6 +287,10 @@ function main() {
SKIP_LOGSEARCH_UPGRADE="false"
fi
+ if [[ -z "$KEEP_BACKUP" ]] ; then
+ KEEP_BACKUP="false"
+ fi
+
if [[ -z "$INI_FILE" ]] ; then
echo "ini-file argument is required (-i or --ini-file)."
print_help
@@ -277,7 +303,7 @@ function main() {
exit 1
else
if [[ "$MODE" == "delete" || "$MODE" == "backup" || "$MODE" == "migrate-restore" || "$MODE" == "all" ]]; then
- run_migrate_commands "$MODE" "$SCRIPT_LOCATION" "$PYTHON_PATH_FOR_MIGRATION" "$INI_FILE" "$WAIT" "$SKIP_SOLR_CLIENT_UPGRADE" "$SKIP_SOLR_SERVER_UPGRADE" "$SKIP_LOGSEARCH_UPGRADE" "$SKIP_WARNINGS" "$BATCH_INTERVAL" "$VERBOSE"
+ run_migrate_commands "$MODE" "$SCRIPT_LOCATION" "$PYTHON_PATH_FOR_MIGRATION" "$INI_FILE" "$WAIT" "$SKIP_SOLR_CLIENT_UPGRADE" "$SKIP_SOLR_SERVER_UPGRADE" "$SKIP_LOGSEARCH_UPGRADE" "$SKIP_WARNINGS" "$BATCH_INTERVAL" "$KEEP_BACKUP" "$VERBOSE"
else
echo "mode '$MODE' is not supported"
print_help