You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cloudstack.apache.org by sa...@apache.org on 2015/04/24 06:34:26 UTC
git commit: updated refs/heads/master to 00b6a54
Repository: cloudstack
Updated Branches:
refs/heads/master 79a46fe0b -> 00b6a54dc
tool to aid db comaprision for upgrade testing
Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo
Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/00b6a54d
Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/00b6a54d
Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/00b6a54d
Branch: refs/heads/master
Commit: 00b6a54dcf3bee47106119b45eb6ea63a4acaeec
Parents: 79a46fe
Author: shweta agarwal <sh...@citrix.com>
Authored: Thu Apr 23 17:12:30 2015 +0530
Committer: sanjeev <sa...@apache.org>
Committed: Fri Apr 24 10:03:05 2015 +0530
----------------------------------------------------------------------
.../database_comparision_during_upgrade/README | 45 ++++++
.../before_upgrade_data_collection.sh | 10 ++
.../cloud_schema_comparision.sh | 58 +++++++
.../fresh_install_data_collection.sh | 32 ++++
.../test_config_before_and_after_upgrade.sh | 96 ++++++++++++
...t_config_between_fresh_and_upgraded_setup.sh | 152 +++++++++++++++++++
.../usage_schema_comparison.sh | 59 +++++++
7 files changed, 452 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/00b6a54d/tools/utils/database_comparision_during_upgrade/README
----------------------------------------------------------------------
diff --git a/tools/utils/database_comparision_during_upgrade/README b/tools/utils/database_comparision_during_upgrade/README
new file mode 100644
index 0000000..abe44d0
--- /dev/null
+++ b/tools/utils/database_comparision_during_upgrade/README
@@ -0,0 +1,45 @@
+What this tool is capable to do is
+
+1. Compare all the tables schema between upgraded setup and fresh install setup and find if there is any schema difference between any tables
+2. Compare global configuration between upgraded and fresh install setup and find out if there is any difference between the two on following fields
+ a. Value
+ b. Scope
+ c. Description
+ d. Component
+ e. Category
+
+3. It will also find out if there is some global configuration present only in fresh setup and missing in upgraded environment and vice versa
+4. It will also find out global configuration value difference between before upgraded and after upgrade setup
+
+
+
+The usage is as follows
+1. First run fresh_install_data_collection.sh file to generate data from fresh install setup .
+ This will be used for comparing between fresh install and upgrade setup.
+ This is a onetime activity and need to be repeated only when there is some DB changes for that release .
+ Output of this script will come in a base_data folder
+
+2. Just before upgrade you need to run before_upgrade_data_collection.sh file to collect required data needed to compare before upgrade and after upgrade setup data
+ The output of this script will come in folder data_before_upgrade
+
+3. After upgrade run cloud_schema_comparision.sh to compare cloud database all tables schema between fresh and upgraded setup.
+ NOTE: this script requires step 1 output in current working directory
+
+4. After upgrade run usage_schema_comparision.sh to compare cloud usage all tables schema between fresh and upgraded setup
+ NOTE: this script requires step 1 output in current working directory
+
+5. Run test_config_between_fresh_and_upgraded_setup.sh to comapre table global configuration values between fresh and upgraded setup
+ NOTE: this script requires step 1 output in current working directory
+
+
+6. Run test_config_before_and_after_upgrade.sh to comapre table global configuration values between before upgraded and after upgraded setup
+ NOTE: this script requires step 2 output in current working directory
+
+
+7. In order to run any *.sh file you need to provide 3 command line argument
+ • Database host ip/localhost
+ • Database user
+ • Database user password
+
+8. Result will be shown in the form of files .
+
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/00b6a54d/tools/utils/database_comparision_during_upgrade/before_upgrade_data_collection.sh
----------------------------------------------------------------------
diff --git a/tools/utils/database_comparision_during_upgrade/before_upgrade_data_collection.sh b/tools/utils/database_comparision_during_upgrade/before_upgrade_data_collection.sh
new file mode 100644
index 0000000..a726848
--- /dev/null
+++ b/tools/utils/database_comparision_during_upgrade/before_upgrade_data_collection.sh
@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+dbhost=$1
+dbuser=$2
+dbpwd=$3
+rm -rf data_before_upgrade
+mkdir data_before_upgrade
+
+mysql -u $dbuser -p$dbpwd -h $dbhost --skip-column-names -e "select name, value from cloud.configuration" > ./data_before_upgrade/configuration_before_upgrade
+
+
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/00b6a54d/tools/utils/database_comparision_during_upgrade/cloud_schema_comparision.sh
----------------------------------------------------------------------
diff --git a/tools/utils/database_comparision_during_upgrade/cloud_schema_comparision.sh b/tools/utils/database_comparision_during_upgrade/cloud_schema_comparision.sh
new file mode 100644
index 0000000..56fb66a
--- /dev/null
+++ b/tools/utils/database_comparision_during_upgrade/cloud_schema_comparision.sh
@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+dbhost=$1
+dbuser=$2
+dbpwd=$3
+path=./base_data
+path1=./data_after_upgrade
+rm -rf $path1
+mkdir $path1
+
+mysql -u $dbuser -p$dbpwd -h $dbhost -e "show tables from cloud" > $path1/tables_upgrade
+
+# to check if number of tables and table name differs
+
+diff $path/tables $path1/tables_upgrade > tables_diff_file
+if [ -s tables_diff_file ]
+then
+ echo "cloud table differs between upgraded and fresh install "
+ cat tables_diff_file
+ # do something as file has data
+else
+ echo "cloud tables are identical in upgraded and fresh install"
+ rm -rf tables_diff_file
+ # do something as file is empty
+
+fi
+
+
+
+for tablename in `cat $path1/tables_upgrade`
+do
+ if [ $tablename != 'Tables_in_cloud' ]
+ then
+ mysql -u $dbuser -p$dbpwd -h $dbhost -e "describe cloud.$tablename" > $path1/upgradedschema
+ cat $path/$tablename > $tablename.diff
+ cat $path1/upgradedschema >> $tablename.diff
+ sort $tablename.diff > $tablename.sort
+ uniq -u $tablename.sort > $tablename.uniq
+
+ if [ -s $tablename.uniq ]
+ then
+ echo $tablename "table schema is different."
+ cat $path1/upgradedschema > $tablename
+ rm -rf $tablename.diff $tablename.sort
+
+ # do something as file has data
+ else
+
+ rm -rf $tablename.diff $tablename.sort $tablename.uniq
+ fi
+
+
+ fi
+done
+
+
+
+
+rm -rf $path1
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/00b6a54d/tools/utils/database_comparision_during_upgrade/fresh_install_data_collection.sh
----------------------------------------------------------------------
diff --git a/tools/utils/database_comparision_during_upgrade/fresh_install_data_collection.sh b/tools/utils/database_comparision_during_upgrade/fresh_install_data_collection.sh
new file mode 100644
index 0000000..45af8d3
--- /dev/null
+++ b/tools/utils/database_comparision_during_upgrade/fresh_install_data_collection.sh
@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+dbhost=$1
+dbuser=$2
+dbpwd=$3
+rm -rf base_data
+mkdir base_data
+mkdir ./base_data/usage_data
+
+mysql -u $dbuser -p$dbpwd -h $dbhost -e "show tables from cloud" > ./base_data/tables
+for tablename in `cat ./base_data/tables`
+do
+ if [ $tablename != 'Tables_in_cloud' ]
+ then
+ mysql -u $dbuser -p$dbpwd -h $dbhost -e "describe cloud.$tablename" > ./base_data/$tablename
+ fi
+done
+
+mysql -u $dbuser -p$dbpwd -h $dbhost --skip-column-names -e "select name, value, default_value from cloud.configuration" > ./base_data/configuration_fresh
+mysql -u $dbuser -p$dbpwd -h $dbhost --skip-column-names -e "select name, component from cloud.configuration" > ./base_data/component_configuration_fresh
+mysql -u $dbuser -p$dbpwd -h $dbhost --skip-column-names -e "select name, category from cloud.configuration" > ./base_data/category_configuration_fresh
+mysql -u $dbuser -p$dbpwd -h $dbhost --skip-column-names -e "select name, description from cloud.configuration" > ./base_data/description_configuration_fresh
+mysql -u $dbuser -p$dbpwd -h $dbhost --skip-column-names -e "select name, scope from cloud.configuration" > ./base_data/scope_configuration_fresh
+
+mysql -u $dbuser -p$dbpwd -h $dbhost -e "show tables from cloud_usage" > ./base_data/usage_data/usage_tables
+for tablename in `cat ./base_data/usage_data/usage_tables`
+do
+ if [ $tablename != 'Tables_in_cloud_usage' ]
+ then
+ mysql -u $dbuser -p$dbpwd -h $dbhost -e "describe cloud_usage.$tablename" > ./base_data/usage_data/$tablename
+ fi
+done
+
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/00b6a54d/tools/utils/database_comparision_during_upgrade/test_config_before_and_after_upgrade.sh
----------------------------------------------------------------------
diff --git a/tools/utils/database_comparision_during_upgrade/test_config_before_and_after_upgrade.sh b/tools/utils/database_comparision_during_upgrade/test_config_before_and_after_upgrade.sh
new file mode 100644
index 0000000..f181821
--- /dev/null
+++ b/tools/utils/database_comparision_during_upgrade/test_config_before_and_after_upgrade.sh
@@ -0,0 +1,96 @@
+#!/usr/bin/env bash
+dbhost=$1
+dbuser=$2
+dbpwd=$3
+a=0
+rm -rf *.uniq mismatch_config_between_before_and_after_upgrade difference_in_config_between_fresh_and_upgraded configurations_only_before_upgrade only_in_upgrade only_in_fresh config_difference_before_and_after_upgrade new_in_upgrade
+path=./base_data
+path1=./data_before_upgrade
+path2=./data_after_upgrade
+
+rm -rf $path2
+mkdir $path2
+
+mysql -u $dbuser -p$dbpwd -h $dbhost --skip-column-names -e "select name, value from cloud.configuration" > $path2/configuration_upgrade
+mysql -u $dbuser -p$dbpwd -h $dbhost --skip-column-names -e "select name, component from cloud.configuration" > $path2/component_configuration_upgrade
+mysql -u $dbuser -p$dbpwd -h $dbhost --skip-column-names -e "select name, category from cloud.configuration" > $path2/category_configuration_upgrade
+mysql -u $dbuser -p$dbpwd -h $dbhost --skip-column-names -e "select name, description from cloud.configuration" > $path2/description_configuration_upgrade
+mysql -u $dbuser -p$dbpwd -h $dbhost --skip-column-names -e "select name, scope from cloud.configuration" > $path2/scope_configuration_upgrade
+
+
+IFS=$'\n'
+
+# to find any missing or mismatch configuration value before upgrade and after upgrade setup
+for row in `cat $path1/configuration_before_upgrade`
+do
+
+ grep $row $path2/configuration_upgrade > $a
+ if [ ! -s $a ]
+ then
+
+ echo $row >> ./mismatch_config_between_before_and_after_upgrade
+ count=`wc -l <./mismatch_config_between_before_and_after_upgrade`
+ echo $row > temp
+ awk '{print $1}' temp > temp1
+ for word in `cat ./temp1`
+ do
+ #echo $word
+ mysql -u $dbuser -p$dbpwd -h $dbhost --skip-column-names -e "select name, value from cloud.configuration where name= '$word'" >> ./mismatch_config_between_before_and_after_upgrade
+ count1=`wc -l <./mismatch_config_between_before_and_after_upgrade`
+ if [ $count == $count1 ]
+ then
+ echo $row >> ./configurations_only_before_upgrade
+ sed -i '$ d' ./mismatch_config_between_before_and_after_upgrade
+ fi
+
+
+
+ done
+ fi
+
+done
+
+
+#to find configuration present only in the upgraded setup
+
+
+for row in `cat $path2/configuration_upgrade`
+do
+
+ grep $row $path1/configuration_before_upgrade > $a
+ if [ ! -s $a ]
+ then
+ echo $row >> ./config_difference_before_and_after_upgrade
+ count=`wc -l <./config_difference_before_and_after_upgrade`
+ echo $row > temp
+ awk '{print $1}' temp > temp1
+ for word in `cat ./temp1`
+ do
+ grep '^'$word'[^\.]\w*' $path1/configuration_before_upgrade >> ./config_difference_before_and_after_upgrade
+ count1=`wc -l <./config_difference_before_and_after_upgrade`
+ if [ $count == $count1 ]
+ then
+ echo $row >> ./new_in_upgrade
+ sed -i '$ d' ./config_difference_before_and_after_upgrade
+
+ fi
+
+
+
+ done
+ fi
+
+done
+
+#to find all the difference between before and after upgrade
+cat ./mismatch_config_between_before_and_after_upgrade >> ./config_difference_before_and_after_upgrade
+sort ./config_difference_before_and_after_upgrade > ./config_difference_before_and_after_upgrade.sort
+uniq ./config_difference_before_and_after_upgrade.sort > ./config_difference_before_and_after_upgrade
+
+
+
+
+
+rm -rf $path2 *.sort category description scope component temp temp1 $a
+rm -rf mismatch_config_between_before_and_after_upgrade config_difference_before_and_after_upgrade.sort t
+
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/00b6a54d/tools/utils/database_comparision_during_upgrade/test_config_between_fresh_and_upgraded_setup.sh
----------------------------------------------------------------------
diff --git a/tools/utils/database_comparision_during_upgrade/test_config_between_fresh_and_upgraded_setup.sh b/tools/utils/database_comparision_during_upgrade/test_config_between_fresh_and_upgraded_setup.sh
new file mode 100644
index 0000000..e621a79
--- /dev/null
+++ b/tools/utils/database_comparision_during_upgrade/test_config_between_fresh_and_upgraded_setup.sh
@@ -0,0 +1,152 @@
+#!/usr/bin/env bash
+dbhost=$1
+dbuser=$2
+dbpwd=$3
+a=0
+rm -rf *.uniq only_in_upgraded only_in_fresh new_in_upgrade difference_in_config_between_fresh_and_upgraded
+path=./base_data
+path2=./data_after_upgrade
+
+rm -rf $path2
+mkdir $path2
+
+mysql -u $dbuser -p$dbpwd -h $dbhost --skip-column-names -e "select name, value from cloud.configuration" > $path2/configuration_upgrade
+mysql -u $dbuser -p$dbpwd -h $dbhost --skip-column-names -e "select name, component from cloud.configuration" > $path2/component_configuration_upgrade
+mysql -u $dbuser -p$dbpwd -h $dbhost --skip-column-names -e "select name, category from cloud.configuration" > $path2/category_configuration_upgrade
+mysql -u $dbuser -p$dbpwd -h $dbhost --skip-column-names -e "select name, description from cloud.configuration" > $path2/description_configuration_upgrade
+mysql -u $dbuser -p$dbpwd -h $dbhost --skip-column-names -e "select name, scope from cloud.configuration" > $path2/scope_configuration_upgrade
+
+
+IFS=$'\n'
+
+
+#to find difference between upgraded configuration and fresh install configuration
+
+mysql -u $dbuser -p$dbpwd -h $dbhost --skip-column-names -e "select name, value, default_value from cloud.configuration" > $path2/configuration_upgrade
+
+
+
+for row in `cat $path2/configuration_upgrade`
+do
+
+ grep $row $path/configuration_fresh > $a
+ if [ ! -s $a ]
+ then
+
+ echo $row > temp
+ name=`awk '{print $1}' temp`
+ value=`awk '{print $2}' temp`
+ default=`awk '{print $3}' temp`
+ grep '^'$name'[^\.]\w*' $path/configuration_fresh > t
+ if [ ! -s t ]
+ then
+ echo $row >> ./only_in_upgraded
+ else
+ fname=`awk '{print $1}' t`
+ fvalue=`awk '{print $2}' t`
+ fdefault=`awk '{print $3}' t`
+ #echo $fname $fvalue $fdefault
+ if [ $default != $value ]
+ then
+ if [ $default == $fdefault ] && [ $value == $fvalue ]
+ then
+ echo
+ else
+ first="$name $value"
+ second="$fname $fvalue"
+ echo "in upgrade" >>difference_in_config_between_fresh_and_upgraded
+ echo $first >> ./difference_in_config_between_fresh_and_upgraded
+ echo "in fresh" >>difference_in_config_between_fresh_and_upgraded
+ echo $second >> ./difference_in_config_between_fresh_and_upgraded
+ fi
+ fi
+ fi
+
+ fi
+done
+
+# to find configuration only available in fresh install but missing in upgraded setup
+
+for row in `cat $path/configuration_fresh`
+do
+
+ grep $row $path2/configuration_upgrade > $a
+ if [ ! -s $a ]
+ then
+
+ echo "in fresh install" >>final_diff4
+ echo $row >> ./final_diff4
+ count=`wc -l <./final_diff4`
+ echo $row > temp
+ awk '{print $1}' temp > temp1
+ for word in `cat ./temp1`
+ do
+ #echo $word
+ echo "in upgrde\n" >> final_diff4
+ grep '^'$word'[^\.]\w*' $path2/configuration_upgrade >> ./final_diff4
+ count1=`wc -l <./final_diff4`
+ count1=`expr $count1 - 1`
+
+ if [ $count == $count1 ]
+ then
+ echo $row >> ./only_in_fresh
+ fi
+
+ done
+ fi
+
+done
+rm -rf final_diff4
+
+
+# to find difference between upgraded and fresh install on component field
+
+cat $path2/component_configuration_upgrade > ./component
+cat $path/component_configuration_fresh >> ./component
+sort ./component > ./component.sort
+uniq -u ./component.sort > component.uniq
+
+
+
+
+
+# to find different between upgraded and fresh install on category field
+
+cat $path2/category_configuration_upgrade > ./category
+cat $path/category_configuration_fresh >> ./category
+sort ./category > ./category.sort
+uniq -u ./category.sort > category.uniq
+
+# to find different between upgraded and fresh install on scope field
+
+cat $path2/scope_configuration_upgrade > ./scope
+cat $path/scope_configuration_fresh >> ./scope
+sort ./scope > ./scope.sort
+uniq -u ./scope.sort > scope.uniq
+
+# to find different between upgraded and fresh install on description field
+
+cat $path2/description_configuration_upgrade > ./description
+cat $path/description_configuration_fresh >> ./description
+sort ./description > ./description.sort
+uniq -u ./description.sort > description.uniq
+
+
+
+
+rm -rf $path2 *.sort category description scope component temp temp1 $a
+rm -rf mismatch_config_between_before_and_after_upgrade config_difference_before_and_after_upgrade.sort t
+#rm -rf $path2
+
+
+
+
+
+
+
+
+
+
+
+
+
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/00b6a54d/tools/utils/database_comparision_during_upgrade/usage_schema_comparison.sh
----------------------------------------------------------------------
diff --git a/tools/utils/database_comparision_during_upgrade/usage_schema_comparison.sh b/tools/utils/database_comparision_during_upgrade/usage_schema_comparison.sh
new file mode 100644
index 0000000..bd7d3fc
--- /dev/null
+++ b/tools/utils/database_comparision_during_upgrade/usage_schema_comparison.sh
@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+dbhost=$1
+dbuser=$2
+dbpwd=$3
+path=./base_data/usage_data
+path1=./data_after_upgrade
+rm -rf $path1
+mkdir $path1
+
+mysql -u $dbuser -p$dbpwd -h $dbhost -e "show tables from cloud_usage" > $path1/usage_tables_upgrade
+
+# to check if number of tables and table name differs
+
+diff $path/usage_tables $path1/usage_tables_upgrade > usage_table_difference
+if [ -s usage_table_difference ]
+then
+ echo "usage table differs between fresh and upgraded install "
+ cat usage_table_difference
+ # do something as file has data
+else
+ echo "usage tables are identicals between fresh and upgraded install "
+ rm -rf usage_tables_difference
+ # do something as file is empty
+
+fi
+
+
+
+for tablename in `cat $path1/usage_tables_upgrade`
+do
+ if [ $tablename != 'Tables_in_cloud_usage' ]
+ then
+ mysql -u $dbuser -p$dbpwd -h $dbhost -e "describe cloud_usage.$tablename" > $path1/upgradedschema
+ cat $path/$tablename > $tablename.diff
+ cat $path1/upgradedschema >> $tablename.diff
+ sort $tablename.diff > $tablename.sort
+ uniq -u $tablename.sort > $tablename.uniq
+
+ if [ -s $tablename.uniq ]
+ then
+ echo $tablename "table schema is different."
+ cat $path1/upgradedschema > usage_$tablename
+ rm -rf $tablename.diff $tablename.sort
+
+ # do something as file has data
+ else
+
+ rm -rf $tablename.diff $tablename.sort $tablename.uniq
+ fi
+
+
+ fi
+done
+
+
+
+
+rm -rf $path1
+