You are viewing a plain text version of this content. The canonical link for it is here.
Posted to tashi-commits@incubator.apache.org by mr...@apache.org on 2008/12/08 17:28:36 UTC
svn commit: r724418 - in /incubator/tashi/trunk: scripts/ scripts/create
scripts/demo-yahoo-08-14-08 scripts/mryan3-database-setup scripts/resume
scripts/stress scripts/tomer-database-setup
src/tashi/nodemanager/vmcontrol/qemu.py src/tashi/util.py
Author: mryan3
Date: Mon Dec 8 09:28:35 2008
New Revision: 724418
URL: http://svn.apache.org/viewvc?rev=724418&view=rev
Log:
Removed artificial bailout of qemu VMM backend if non-transient disks are used
Util function "boolean" now accepts multiple types besides strings
Scripts pulled over from imported code
Added:
incubator/tashi/trunk/scripts/
incubator/tashi/trunk/scripts/create (with props)
incubator/tashi/trunk/scripts/demo-yahoo-08-14-08 (with props)
incubator/tashi/trunk/scripts/mryan3-database-setup (with props)
incubator/tashi/trunk/scripts/resume (with props)
incubator/tashi/trunk/scripts/stress (with props)
incubator/tashi/trunk/scripts/tomer-database-setup (with props)
Modified:
incubator/tashi/trunk/src/tashi/nodemanager/vmcontrol/qemu.py
incubator/tashi/trunk/src/tashi/util.py
Added: incubator/tashi/trunk/scripts/create
URL: http://svn.apache.org/viewvc/incubator/tashi/trunk/scripts/create?rev=724418&view=auto
==============================================================================
--- incubator/tashi/trunk/scripts/create (added)
+++ incubator/tashi/trunk/scripts/create Mon Dec 8 09:28:35 2008
@@ -0,0 +1,24 @@
+#! /bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+if [[ $# -ne 4 ]]; then
+ echo "create <type> <image> <mac_1byte> <persistent>"
+else
+ ./bin/createVm "Instance(d={'name':'foobar','type':$1,'disks':[DiskConfiguration(d={'uri':'$2','persistent':$4})],'nics':[NetworkConfiguration(d={'network':2,'mac':'52:54:00:00:10:$3'})], 'hints':{'display':'True'}})"
+fi
Propchange: incubator/tashi/trunk/scripts/create
------------------------------------------------------------------------------
svn:executable = *
Added: incubator/tashi/trunk/scripts/demo-yahoo-08-14-08
URL: http://svn.apache.org/viewvc/incubator/tashi/trunk/scripts/demo-yahoo-08-14-08?rev=724418&view=auto
==============================================================================
--- incubator/tashi/trunk/scripts/demo-yahoo-08-14-08 (added)
+++ incubator/tashi/trunk/scripts/demo-yahoo-08-14-08 Mon Dec 8 09:28:35 2008
@@ -0,0 +1,109 @@
+#! /bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+export PYTHONPATH=`pwd`/src/
+
+NUM=10
+
+BIGNODE=172.16.250.254
+
+BASEID=0
+
+date
+
+echo "Creating small VMs..."
+for h in `seq -w 1 ${NUM}`; do
+ echo "Creating VM #${h}..."
+ INFO=`./create 1 $h 2>/dev/null | grep "[ \t]id:"`
+ if [[ ${BASEID} -eq 0 ]]; then
+ BASEID=`echo ${INFO} | sed 's/[^0-9]*\([0-9]*\).*/\1/'`
+ fi
+done
+echo "Small VMs created"
+echo "Waiting for startup..."
+while [[ true ]]; do
+ PASS="True"
+ for h in `cat ~/hosts/vms.txt`; do
+ HN=`ssh root@bd.${h} "hostname 2> /dev/null" 2> /dev/null | cut -c -2`
+ if [[ "${HN}" != "vm" ]]; then
+ PASS="False"
+ fi
+ done
+ if [[ "${PASS}" == "True" ]]; then
+ break
+ fi
+ sleep 1
+done
+echo "Waiting finished"
+echo "Starting work on small VMs..."
+for h in `cat ~/hosts/vms.txt`; do
+ ssh root@bd.${h} "./run > /dev/null 2>&1" > /dev/null 2>&1 &
+done
+echo "Creating large VM..."
+./create 2 11 > /dev/null 2>&1
+echo "Done creating large VM"
+while [[ true ]]; do
+ sleep 5
+ COUNT=0
+ for h in `cat ~/hosts/vms.txt`; do
+ CNT=`ssh root@bd.${h} "ls /x/mryan3/cvm-out/*/*.txt 2> /dev/null | wc -l 2> /dev/null" 2> /dev/null`
+ COUNT=$((COUNT+CNT))
+ done
+ echo "${COUNT}/64 work items completed..."
+ if [[ ${COUNT} -eq 64 ]]; then
+ break
+ fi
+done
+echo "Work on small VMs completed"
+for i in `seq 1 ${NUM}`; do
+ wait
+done
+echo "Collecting output from small VMs to the large VM..."
+ssh root@bd.${BIGNODE} ./gather > /dev/null 2>&1
+echo "Done collecting output"
+echo "Destroying small VMs..."
+for i in `seq 1 ${NUM}`; do
+ ./bin/destroyVm $((i+BASEID-1)) > /dev/null 2>&1
+done
+echo "Done destroying small VMs"
+echo "Doing work on large VM..."
+ssh root@bd.${BIGNODE} ./build > /dev/null 2>&1 &
+while [[ true ]]; do
+ sleep 2
+ SIZE=`ssh root@bd.${BIGNODE} "du -hs ./out.e 2> /dev/null | awk '{print "'$1'"}' 2> /dev/null" 2> /dev/null`
+ echo "${SIZE}/154M output data generated..."
+ if [[ "${SIZE}" == "154M" ]]; then
+ break
+ fi
+done
+wait
+echo "Work on large VM complete"
+echo "Copying final output file to localhost..."
+scp root@bd.${BIGNODE}:./out.e /tmp/out.e
+echo "Copy complete"
+echo "Destroying large VM..."
+./bin/destroyVm $((11+BASEID-1)) > /dev/null 2>&1
+echo "Large VM destroyed"
+echo "Generating output image from etree..."
+(cd ~/local/src/mryan3/BigDatavis/src; ./util/draw_slice -d /tmp/out.e 0 0 0 512 0 0 0 512 0 512 512 /tmp/output.jpg > /dev/null 2>&1)
+echo "Image complete"
+
+date
+
+qiv /tmp/output.jpg
Propchange: incubator/tashi/trunk/scripts/demo-yahoo-08-14-08
------------------------------------------------------------------------------
svn:executable = *
Added: incubator/tashi/trunk/scripts/mryan3-database-setup
URL: http://svn.apache.org/viewvc/incubator/tashi/trunk/scripts/mryan3-database-setup?rev=724418&view=auto
==============================================================================
--- incubator/tashi/trunk/scripts/mryan3-database-setup (added)
+++ incubator/tashi/trunk/scripts/mryan3-database-setup Mon Dec 8 09:28:35 2008
@@ -0,0 +1,28 @@
+#! /bin/sh
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+./bin/addUser "User(d={'username':'mryan3'})"
+./bin/addPersistentImage "PersistentImage(d={'userId':1,'name':'i386-ubuntu.qcow'})"
+./bin/addInstanceConfiguration "InstanceConfiguration(d={'name':'i386-512','memory':512,'cores':1})"
+./bin/addHardDiskConfiguration "HardDiskConfiguration(d={'index':0,'persistentImageId':1,'persistent':False,'instanceConfigurationId':1})"
+./bin/addNetworkInterfaceConfiguration "NetworkInterfaceConfiguration(d={'index':0,'instanceConfigurationId':1})"
+./bin/addHost "Host(d={'hostname':'blade043'})"
+./bin/addHost "Host(d={'hostname':'blade044'})"
+./bin/addHost "Host(d={'hostname':'blade045'})"
+./bin/addHost "Host(d={'hostname':'blade074'})"
Propchange: incubator/tashi/trunk/scripts/mryan3-database-setup
------------------------------------------------------------------------------
svn:executable = *
Added: incubator/tashi/trunk/scripts/resume
URL: http://svn.apache.org/viewvc/incubator/tashi/trunk/scripts/resume?rev=724418&view=auto
==============================================================================
--- incubator/tashi/trunk/scripts/resume (added)
+++ incubator/tashi/trunk/scripts/resume Mon Dec 8 09:28:35 2008
@@ -0,0 +1,20 @@
+#! /bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+./bin/resumeVm "Instance(d={'name':'foobar','type':$1,'disks':[DiskConfiguration(d={'uri':'i386-ubuntu.qcow','persistent':False})],'nics':[NetworkConfiguration(d={'network':2,'mac':'52:54:00:00:05:$2'})], 'hints':{'display':'True'}})" "\"$3\""
Propchange: incubator/tashi/trunk/scripts/resume
------------------------------------------------------------------------------
svn:executable = *
Added: incubator/tashi/trunk/scripts/stress
URL: http://svn.apache.org/viewvc/incubator/tashi/trunk/scripts/stress?rev=724418&view=auto
==============================================================================
--- incubator/tashi/trunk/scripts/stress (added)
+++ incubator/tashi/trunk/scripts/stress Mon Dec 8 09:28:35 2008
@@ -0,0 +1,109 @@
+#! /bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+if [[ $# -eq 1 ]]; then
+ VMS=$1
+else
+ VMS=20
+fi
+
+date
+
+#HOSTS=`./bin/getHosts | grep name | wc -l`
+HOSTS="4"
+
+echo "Hosts: ${HOSTS}"
+
+echo "Create:"
+
+MID=10240000
+for i in `seq -w 1 $VMS`; do
+ echo "./scripts/create 2 $i"
+ ID=`./scripts/create 2 $i | grep "id: " | awk '{print $2}'`
+ if [[ ${ID} -lt ${MID} ]]; then
+ MID=${ID}
+ fi
+done
+
+date
+
+echo "Wait:"
+
+while [[ true ]]; do
+ CNT=`./bin/getInstances | grep -c Running`
+ echo ${CNT}
+ if [[ ${CNT} -eq ${VMS} ]]; then
+ break
+ fi
+ sleep 1
+done
+
+sleep 20
+
+date
+
+echo "Migrate:"
+
+for i in `seq 0 $((VMS-1))`; do
+ echo "./bin/migrateVm $((MID+i)) $((((i+1)%HOSTS)+1))"
+ ./bin/migrateVm $((MID+i)) $((((i+1)%HOSTS)+1)) > /dev/null &
+done
+for i in `seq 0 $((VMS-1))`; do
+ wait
+done
+
+date
+
+echo "Wait:"
+
+while [[ true ]]; do
+ CNT=`./bin/getInstances | grep -c Running`
+ echo ${CNT}
+ if [[ ${CNT} -eq ${VMS} ]]; then
+ break
+ fi
+ sleep 1
+done
+
+date
+
+echo "Destroy:"
+
+for i in `seq 0 $((VMS-1))`; do
+ echo "./bin/destroyVm $((MID+i))"
+ ./bin/destroyVm $((MID+i)) > /dev/null 2>&1 &
+done
+for i in `seq 0 $((VMS-1))`; do
+ wait
+done
+
+date
+
+echo "Wait:"
+
+while [[ true ]]; do
+ CNT=`./bin/getInstances | wc -l`
+ echo ${CNT}
+ if [[ ${CNT} -eq 1 ]]; then
+ break
+ fi
+ sleep 1
+done
+
+date
Propchange: incubator/tashi/trunk/scripts/stress
------------------------------------------------------------------------------
svn:executable = *
Added: incubator/tashi/trunk/scripts/tomer-database-setup
URL: http://svn.apache.org/viewvc/incubator/tashi/trunk/scripts/tomer-database-setup?rev=724418&view=auto
==============================================================================
--- incubator/tashi/trunk/scripts/tomer-database-setup (added)
+++ incubator/tashi/trunk/scripts/tomer-database-setup Mon Dec 8 09:28:35 2008
@@ -0,0 +1,29 @@
+#! /bin/sh
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+./bin/addUser "User(d={'username':'administrator'})"
+./bin/addUser "User(d={'username':'tshiran'})"
+./bin/addUser "User(d={'username':'jcipar'})"
+./bin/addPersistentImage "PersistentImage(d={'userId':1,'name':'hardy-25G.img'})"
+./bin/addPersistentImage "PersistentImage(d={'userId':1,'name':'hardy-25G-pdl.img','parentId':1})"
+./bin/addPersistentImage "PersistentImage(d={'userId':2,'name':'hardy-25G-tshiran.img','parentId':2})"
+./bin/addPersistentImage "PersistentImage(d={'userId':3,'name':'hardy-25G-jcipar.img','parentId':2})"
+./bin/addPersistentImage "PersistentImage(d={'userId':2,'name':'hardy-25G-tshiran-hadoop.img','parentId':3})"
+./bin/addHost "Host(d={'hostname':'ss306'})"
+./bin/addHost "Host(d={'hostname':'ss308'})"
Propchange: incubator/tashi/trunk/scripts/tomer-database-setup
------------------------------------------------------------------------------
svn:executable = *
Modified: incubator/tashi/trunk/src/tashi/nodemanager/vmcontrol/qemu.py
URL: http://svn.apache.org/viewvc/incubator/tashi/trunk/src/tashi/nodemanager/vmcontrol/qemu.py?rev=724418&r1=724417&r2=724418&view=diff
==============================================================================
--- incubator/tashi/trunk/src/tashi/nodemanager/vmcontrol/qemu.py (original)
+++ incubator/tashi/trunk/src/tashi/nodemanager/vmcontrol/qemu.py Mon Dec 8 09:28:35 2008
@@ -355,8 +355,8 @@
diskModel = "persistent" if instance.disks[0].persistent else "transient"
instanceId = instance.id
opts = instance.hints
- if (diskModel != "transient"):
- raise NotImplementedError
+# if (diskModel != "transient"):
+# raise NotImplementedError
return (image, macAddr, memory, cores, diskModel, instanceId, opts)
def oldToInstance(self, image, macAddr, memory, cores, diskModel, opts):
Modified: incubator/tashi/trunk/src/tashi/util.py
URL: http://svn.apache.org/viewvc/incubator/tashi/trunk/src/tashi/util.py?rev=724418&r1=724417&r2=724418&view=diff
==============================================================================
--- incubator/tashi/trunk/src/tashi/util.py (original)
+++ incubator/tashi/trunk/src/tashi/util.py Mon Dec 8 09:28:35 2008
@@ -25,6 +25,7 @@
import threading
import time
import traceback
+import types
from tashi.services.ttypes import TashiException, Errors, InstanceState, HostState
@@ -159,7 +160,11 @@
return __decorator
def boolean(value):
- """Convert a string to a boolean"""
+ """Convert a variable to a boolean"""
+ if (type(value) == types.BooleanType):
+ return value
+ if (type(value) == types.IntType):
+ return (value != 0)
lowercaseValue = value.lower()
if lowercaseValue in ['yes', 'true', '1']:
return True