You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cloudstack.apache.org by bh...@apache.org on 2016/03/26 18:38:44 UTC
git commit: updated refs/heads/4.9-mvn-upgrade to ffb3554 [Forced
Update!]
Repository: cloudstack
Updated Branches:
refs/heads/4.9-mvn-upgrade 05077cfcd -> ffb3554a4 (forced update)
travis: Fix simulator tests and optimize default global configs
- Migrate to trusty based Travis VMs
- Increase tests across five build matrices
- Fix xunit-reader output, include time
- Fix pip/python usage, pkg installation
- Build CloudStack in parallel with -T4
- Deploy database with optimized global settings
Signed-off-by: Rohit Yadav <ro...@shapeblue.com>
Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo
Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/ffb3554a
Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/ffb3554a
Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/ffb3554a
Branch: refs/heads/4.9-mvn-upgrade
Commit: ffb3554a411f05b27262334987636985da80026b
Parents: ecb11c8
Author: Rohit Yadav <ro...@shapeblue.com>
Authored: Sat Mar 26 02:44:40 2016 +0100
Committer: Rohit Yadav <ro...@shapeblue.com>
Committed: Sat Mar 26 18:36:58 2016 +0100
----------------------------------------------------------------------
.travis.yml | 24 ++++++++++++++++--------
developer/developer-prefill.sql | 28 ++++++++++++++++++++++++++++
setup/db/templates.simulator.sql | 4 +++-
setup/dev/advanced.cfg | 4 ++--
tools/travis/before_install.sh | 25 ++++++++++++++++++-------
tools/travis/before_script.sh | 2 +-
tools/travis/install.sh | 19 +++++++------------
tools/travis/script.sh | 5 +++--
tools/travis/xunit-reader.py | 14 ++++++++++++--
9 files changed, 90 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/ffb3554a/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index 200f199..1441fa4 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -15,27 +15,35 @@
# specific language governing permissions and limitations
# under the License.
sudo: required
+dist: trusty
language: java
cache:
directories:
- $HOME/.cache
jdk:
- oraclejdk7
+python:
+ - "2.7"
+cache:
+ directories:
+ - $HOME/.m2
+ timeout: 1000
notifications:
email: false
env:
global:
- REGRESSION_CYCLE=4
- REGRESSION_INDEX=6
+ - PATH=$HOME/.local/bin:$PATH
matrix:
- - TESTS="smoke/test_affinity_groups smoke/test_primary_storage"
- - TESTS="smoke/test_deploy_vms_with_varied_deploymentplanners smoke/test_disk_offerings smoke/test_global_settings smoke/test_multipleips_per_nic"
- - TESTS="smoke/test_portable_publicip smoke/test_privategw_acl smoke/test_public_ip_range smoke/test_pvlan smoke/test_regions smoke/test_network"
- - TESTS="smoke/test_reset_vm_on_reboot smoke/test_resource_detail smoke/test_routers smoke/test_guest_vlan_range smoke/test_iso smoke/test_non_contigiousvlan"
- - TESTS="smoke/test_secondary_storage smoke/test_service_offerings smoke/test_ssvm smoke/test_templates smoke/test_over_provisioning"
+ - TESTS="smoke/test_affinity_groups smoke/test_affinity_groups_projects smoke/test_deploy_vgpu_enabled_vm smoke/test_deploy_vm_iso smoke/test_deploy_vm_root_resize smoke/test_deploy_vm_with_userdata smoke/test_deploy_vms_with_varied_deploymentplanners smoke/test_disk_offerings smoke/test_global_settings smoke/test_guest_vlan_range"
+ - TESTS="smoke/test_hosts smoke/test_internal_lb smoke/test_iso smoke/test_loadbalance smoke/test_multipleips_per_nic smoke/test_network smoke/test_network_acl smoke/test_nic smoke/test_nic_adapter_type smoke/test_non_contigiousvlan"
+ - TESTS="smoke/test_over_provisioning smoke/test_password_server smoke/test_portable_publicip smoke/test_primary_storage smoke/test_privategw_acl smoke/test_public_ip_range smoke/test_pvlan smoke/test_regions smoke/test_reset_vm_on_reboot smoke/test_resource_detail"
+ - TESTS="smoke/test_router_dhcphosts smoke/test_routers smoke/test_routers_iptables_default_policy smoke/test_routers_network_ops smoke/test_scale_vm smoke/test_secondary_storage smoke/test_service_offerings smoke/test_snapshots smoke/test_ssvm smoke/test_templates"
+ - TESTS="smoke/test_usage_events smoke/test_vm_life_cycle smoke/test_vm_snapshots smoke/test_volumes smoke/test_vpc_redundant smoke/test_vpc_router_nics smoke/test_vpc_vpn smoke/misc/test_deploy_vm smoke/misc/test_vm_ha smoke/misc/test_escalations_templates smoke/misc/test_vm_sync"
- - TESTS="smoke/test_volumes smoke/test_vpc_vpn smoke/misc/test_deploy_vm smoke/test_vm_life_cycle component/test_mm_max_limits"
- - TESTS="component/test_acl_isolatednetwork_delete component/test_mm_domain_limits component/test_acl_listsnapshot"
+ - TESTS="component/test_mm_max_limits component/test_acl_isolatednetwork_delete"
+ - TESTS="component/test_mm_domain_limits component/test_acl_listsnapshot"
- TESTS="component/test_acl_listvm component/test_acl_sharednetwork_deployVM-impersonation component/test_acl_sharednetwork"
- TESTS="component/test_snapshots component/test_acl_listvolume"
before_install: travis_wait 30 ./tools/travis/before_install.sh
@@ -43,7 +51,7 @@ install: ./tools/travis/install.sh
before_script: travis_wait 30 ./tools/travis/before_script.sh
script:
- travis_wait 30 sleep 30
- - ./tools/travis/script.sh $TESTS
+ - ./tools/travis/script.sh ${TESTS}
after_success: ./tools/travis/after_success.sh
after_failure: ./tools/travis/after_failure.sh
after_script: ./tools/travis/after_script.sh
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/ffb3554a/developer/developer-prefill.sql
----------------------------------------------------------------------
diff --git a/developer/developer-prefill.sql b/developer/developer-prefill.sql
index 3097203..f874118 100644
--- a/developer/developer-prefill.sql
+++ b/developer/developer-prefill.sql
@@ -64,6 +64,10 @@ INSERT INTO `cloud`.`configuration` (category, instance, component, name, value)
INSERT INTO `cloud`.`configuration` (category, instance, component, name, value)
VALUES ('Advanced', 'DEFAULT', 'management-server',
+ 'expunge.workers', '3');
+
+INSERT INTO `cloud`.`configuration` (category, instance, component, name, value)
+ VALUES ('Advanced', 'DEFAULT', 'management-server',
'cluster.cpu.allocated.capacity.disablethreshold', '0.95');
INSERT INTO `cloud`.`configuration` (category, instance, component, name, value)
@@ -78,6 +82,30 @@ INSERT INTO `cloud`.`configuration` (category, instance, component, name, value)
VALUES ('Advanced', 'DEFAULT', 'management-server',
'pool.storage.capacity.disablethreshold', '0.95');
+INSERT INTO `cloud`.`configuration` (category, instance, component, name, value)
+ VALUES ('Advanced', 'DEFAULT', 'management-server',
+ 'network.gc.wait', '60');
+
+INSERT INTO `cloud`.`configuration` (category, instance, component, name, value)
+ VALUES ('Advanced', 'DEFAULT', 'management-server',
+ 'network.gc.interval', '60');
+
+INSERT INTO `cloud`.`configuration` (category, instance, component, name, value)
+ VALUES ('Advanced', 'DEFAULT', 'management-server',
+ 'storage.cleanup.interval', '150');
+
+INSERT INTO `cloud`.`configuration` (category, instance, component, name, value)
+ VALUES ('Advanced', 'DEFAULT', 'management-server',
+ 'vm.op.wait.interval', '5');
+
+INSERT INTO `cloud`.`configuration` (category, instance, component, name, value)
+ VALUES ('Advanced', 'DEFAULT', 'management-server',
+ 'workers', '10');
+
+INSERT INTO `cloud`.`configuration` (category, instance, component, name, value)
+ VALUES ('Advanced', 'DEFAULT', 'management-server',
+ 'direct.agent.load.size', '1000');
+
-- Add developer configuration entry; allows management server to be run as a user other than "cloud"
INSERT INTO `cloud`.`configuration` (category, instance, component, name, value)
VALUES ('Advanced', 'DEFAULT', 'management-server',
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/ffb3554a/setup/db/templates.simulator.sql
----------------------------------------------------------------------
diff --git a/setup/db/templates.simulator.sql b/setup/db/templates.simulator.sql
index 25e91bd..ac09c05 100755
--- a/setup/db/templates.simulator.sql
+++ b/setup/db/templates.simulator.sql
@@ -19,4 +19,6 @@
INSERT INTO `cloud`.`vm_template` (id, uuid, unique_name, name, public, created, type, hvm, bits, account_id, url, checksum, enable_password, display_text, format, guest_os_id, featured, cross_zones, hypervisor_type, state)
VALUES (100, UUID(), 'simulator-domR', 'SystemVM Template (simulator)', 0, now(), 'SYSTEM', 0, 64, 1, 'http://nfs1.lab.vmops.com/templates/routing/debian/latest/systemvm.vhd.bz2', '', 0, 'SystemVM Template (simulator)', 'VHD', 15, 0, 1, 'Simulator','Active');
INSERT INTO `cloud`.`vm_template` (id, uuid, unique_name, name, public, created, type, hvm, bits, account_id, url, checksum, enable_password, display_text, format, guest_os_id, featured, cross_zones, hypervisor_type,state)
- VALUES (111, UUID(), 'simulator-Centos', 'CentOS 5.3(64-bit) no GUI (Simulator)', 1, now(), 'BUILTIN', 0, 64, 1, 'http://nfs1.lab.vmops.com/templates/centos53-x86_64/latest/f59f18fb-ae94-4f97-afd2-f84755767aca.vhd.bz2', '', 0, 'CentOS 5.3(64-bit) no GUI (Simulator)', 'VHD', 11, 1, 1, 'Simulator','Active');
+ VALUES (111, UUID(), 'simulator-Centos', 'CentOS 5.6 (64-bit) no GUI (Simulator)', 1, now(), 'BUILTIN', 0, 64, 1, 'http://nfs1.lab.vmops.com/templates/centos53-x86_64/latest/f59f18fb-ae94-4f97-afd2-f84755767aca.vhd.bz2', '', 0, 'CentOS 5.6 (64-bit) no GUI (Simulator)', 'VHD', 142, 1, 1, 'Simulator','Active');
+
+INSERT INTO `cloud`.`template_store_ref` VALUES (1,1,111,NOW(),NOW(),NULL,100,0,'Image',0,'DOWNLOADED',NULL,NULL,'template/tmpl/1/111/','http://fake.cloud/111.vhd.bz2','Ready',0,0,0,0,NULL,NULL,NULL);
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/ffb3554a/setup/dev/advanced.cfg
----------------------------------------------------------------------
diff --git a/setup/dev/advanced.cfg b/setup/dev/advanced.cfg
index 8109bb1..afb9b6a 100644
--- a/setup/dev/advanced.cfg
+++ b/setup/dev/advanced.cfg
@@ -154,7 +154,7 @@
},
{
"name": "storage.cleanup.interval",
- "value": "300"
+ "value": "150"
},
{
"name": "vm.op.wait.interval",
@@ -178,7 +178,7 @@
},
{
"name": "account.cleanup.interval",
- "value": "600"
+ "value": "60"
},
{
"name": "guest.domain.suffix",
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/ffb3554a/tools/travis/before_install.sh
----------------------------------------------------------------------
diff --git a/tools/travis/before_install.sh b/tools/travis/before_install.sh
index b90a099..700ac09 100755
--- a/tools/travis/before_install.sh
+++ b/tools/travis/before_install.sh
@@ -49,12 +49,21 @@ javac -version
echo -e "\nMaven Version: "
mvn -v
+echo -e "\nPython Version: "
+python --version
+
+echo -e "\nPip Version: "
+pip --version
+
echo -e "\nDisk Status: "
df
echo -e "\nMemory Status: "
free
+echo -e "\nTotal CPUs: "
+nproc
+
echo -e "\nCheck Git status"
git status
@@ -66,18 +75,18 @@ sudo apt-get -q -y update > /dev/null
echo -e "\nInstalling MySQL: "
-sudo debconf-set-selections <<< 'mysql-server mysql-server/root_password password your_password'
-sudo debconf-set-selections <<< 'mysql-server mysql-server/root_password_again password your_password'
+export DEBIAN_FRONTEND=noninteractive
+sudo debconf-set-selections <<< 'mysql-server mysql-server/root_password password password'
+sudo debconf-set-selections <<< 'mysql-server mysql-server/root_password_again password password'
sudo apt-get -q -y install mysql-server > /dev/null
-#Restart mysql if running to release deleted file locks on filesystem, if aready running
-sudo status mysql | grep start && sudo stop mysql
-sudo start mysql
+mysql -uroot -ppassword -e "SET PASSWORD = PASSWORD(''); FLUSH PRIVILEGES;"
+sudo service mysql restart
echo -e "\nInstalling Development tools: "
RETRY_COUNT=3
-sudo apt-get -q -y install uuid-runtime genisoimage python-setuptools python-pip netcat > /dev/null
+sudo apt-get -q -y install uuid-runtime genisoimage netcat > /dev/null
if [[ $? -ne 0 ]]; then
echo -e "\napt-get packages failed to install"
fi
@@ -94,9 +103,11 @@ echo "<settings>
echo -e "\nInstalling some python packages: "
+pip install --user --upgrade pip
+
for ((i=0;i<$RETRY_COUNT;i++))
do
- sudo pip install --upgrade lxml texttable paramiko > /tmp/piplog
+ pip install --user --upgrade lxml paramiko nose texttable > /tmp/piplog
if [[ $? -eq 0 ]]; then
echo -e "\npython packages installed successfully"
break;
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/ffb3554a/tools/travis/before_script.sh
----------------------------------------------------------------------
diff --git a/tools/travis/before_script.sh b/tools/travis/before_script.sh
index 40e6700..beb8acc 100755
--- a/tools/travis/before_script.sh
+++ b/tools/travis/before_script.sh
@@ -41,4 +41,4 @@ mvn -Dsimulator -pl :cloud-client-ui jetty:run 2>&1 > /tmp/jetty-log &
while ! nc -vzw 5 localhost 8096 2>&1 > /dev/null; do grep Exception /tmp/jetty-log; sleep 10; done
echo -e "\nStarting DataCenter deployment"
-python -m marvin.deployDataCenter -i setup/dev/advanced.cfg 2>&1 || true
+python tools/marvin/marvin/deployDataCenter.py -i setup/dev/advanced.cfg 2>&1 || true
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/ffb3554a/tools/travis/install.sh
----------------------------------------------------------------------
diff --git a/tools/travis/install.sh b/tools/travis/install.sh
index 28f4ed4..5b6bc5e 100755
--- a/tools/travis/install.sh
+++ b/tools/travis/install.sh
@@ -34,24 +34,19 @@ if [ $MOD -ne 0 ]; then
fi
fi
-export MAVEN_OPTS="-Xmx1024m -XX:MaxPermSize=500m -Djava.security.egd=file:/dev/./urandom"
+export MAVEN_OPTS="-Xmx2048m -XX:MaxPermSize=500m -Djava.security.egd=file:/dev/./urandom"
if [ $TEST_SEQUENCE_NUMBER -eq 1 ]; then
- mvn -q -Pimpatient -Dsimulator clean install
+ mvn -Pdeveloper,systemvm -Dsimulator clean install -T4 | egrep "Building|SUCCESS"
else
- mvn -q -Pimpatient -Dsimulator clean install -DskipTests=true
+ mvn -Pdeveloper -Dsimulator clean install -DskipTests=true -T4 | egrep "Building|SUCCESS"
fi
-# Compile API Docs
-cd tools/apidoc
-mvn -q clean install
-cd ../../
+# Install mysql-connector-python
+pip install --user --upgrade http://cdn.mysql.com/Downloads/Connector-Python/mysql-connector-python-2.0.4.zip#md5=3df394d89300db95163f17c843ef49df 2>&1 > /dev/null
-# Compile marvin
-cd tools/marvin
-mvn -q clean install
-sudo python setup.py install 2>&1 > /dev/null
-cd ../../
+# Install marvin
+pip install --user --upgrade tools/marvin/dist/Marvin-*.tar.gz
# Deploy the database
mvn -q -Pdeveloper -pl developer -Ddeploydb
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/ffb3554a/tools/travis/script.sh
----------------------------------------------------------------------
diff --git a/tools/travis/script.sh b/tools/travis/script.sh
index 7e7c497..dc955b0 100755
--- a/tools/travis/script.sh
+++ b/tools/travis/script.sh
@@ -37,10 +37,11 @@ fi
mkdir -p integration-test-results/smoke/misc
mkdir -p integration-test-results/component
+TESTS=($@)
+echo "Running tests: " ${TESTS[@]}
-for suite in $1; do
+for suite in "${TESTS[@]}" ; do
nosetests --with-xunit --xunit-file=integration-test-results/$suite.xml --with-marvin --marvin-config=setup/dev/advanced.cfg test/integration/$suite.py -s -a tags=advanced,required_hardware=false --zone=Sandbox-simulator --hypervisor=simulator || true ;
done
-
python ./tools/travis/xunit-reader.py integration-test-results/
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/ffb3554a/tools/travis/xunit-reader.py
----------------------------------------------------------------------
diff --git a/tools/travis/xunit-reader.py b/tools/travis/xunit-reader.py
index c492c01..80e228b 100755
--- a/tools/travis/xunit-reader.py
+++ b/tools/travis/xunit-reader.py
@@ -61,6 +61,8 @@ def _generate_args():
def _generate_file_list(args):
path = args.pop('path')
file_path_list = []
+ if path.endswith('.xml') and os.path.isfile(path):
+ file_path_list.append(path)
for (root, dirnames, filenames) in os.walk(path):
for filename in filenames:
if filename.endswith('.xml'):
@@ -71,7 +73,7 @@ def _generate_file_list(args):
def parse_reports(file_path_list):
table = texttable.Texttable()
- table.header(['Test', 'Result'])
+ table.header(['Test', 'Result', 'Time'])
exit_code = 0
@@ -80,16 +82,24 @@ def parse_reports(file_path_list):
for event, elem in data:
name = ''
status = 'Success'
+ time = ''
if 'name' in elem.attrib:
name = elem.attrib['name']
+ if 'time' in elem.attrib:
+ time = elem.attrib['time']
for children in elem.getchildren():
if 'skipped' == children.tag:
status = 'Skipped'
elif 'failure' == children.tag:
exit_code = 1
status = 'Failure'
+ elif 'error' == children.tag:
+ exit_code = 1
+ status = 'Error'
+ if 'type' in children.attrib:
+ status = children.attrib['type']
- table.add_row([name, status])
+ table.add_row([name, status, time])
print table.draw()