You are viewing a plain text version of this content. The canonical link for it is here.
Posted to java-dev@axis.apache.org by an...@apache.org on 2002/07/02 10:37:37 UTC
cvs commit: xml-axis-wsif/java/test/performance SpeedTest.java Monitor.java
antelder 2002/07/02 01:37:37
Added: java/test/performance SpeedTest.java Monitor.java
Log:
New testcase to help with testing performance improvement changes
Revision Changes Path
1.1 xml-axis-wsif/java/test/performance/SpeedTest.java
Index: SpeedTest.java
===================================================================
/*
* The Apache Software License, Version 1.1
*
*
* Copyright (c) 2002 The Apache Software Foundation. All rights
* reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* 3. The end-user documentation included with the redistribution,
* if any, must include the following acknowledgment:
* "This product includes software developed by the
* Apache Software Foundation (http://www.apache.org/)."
* Alternately, this acknowledgment may appear in the software itself,
* if and wherever such third-party acknowledgments normally appear.
*
* 4. The names "WSIF" and "Apache Software Foundation" must
* not be used to endorse or promote products derived from this
* software without prior written permission. For written
* permission, please contact apache@apache.org.
*
* 5. Products derived from this software may not be called "Apache",
* nor may "Apache" appear in their name, without prior written
* permission of the Apache Software Foundation.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation and was
* originally based on software copyright (c) 2001, 2002, International
* Business Machines, Inc., http://www.apache.org. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*/
package performance;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Properties;
import junit.framework.Test;
import junit.framework.TestCase;
import junit.framework.TestSuite;
import org.apache.wsif.WSIFMessage;
import org.apache.wsif.WSIFOperation;
import org.apache.wsif.WSIFPort;
import org.apache.wsif.WSIFService;
import org.apache.wsif.WSIFServiceFactory;
import org.apache.wsif.base.WSIFServiceImpl;
import org.apache.wsif.providers.soap.apacheaxis.WSIFDynamicProvider_ApacheAxis;
import util.TestUtilities;
import addressbook.wsiftypes.Address;
import addressbook.wsiftypes.Phone;
/**
* Junit test to measure some simple performance statistics.
* It times a number of iterations of getPort, createOperation,
* and executeRequestResponseOperation for each provider. The
* results are stored in a properties file to allow each run
* to test if the performance has changed since the last run.
*
* The number of iterations to do is determined by the time
* each test takes and the TIME_PER_TEST constant. You need
* to run it atleast 3 times to get the iterations calculation
* to settle on a good value; The MARGIN_OF_ERROR constant
* defines how mush variation between run times is ignored
* when determining if the performance has changed.
*
* You may need to change the TIME_PER_TEST and MARGIN_OF_ERROR
* constants for your machine speed.
*
* For testing you can add futher timing points to the internal
* WSIF classes which this test will then monitor and report on.
*
* For example, to time the axis provider invoke method, at the
* start of the invoke method add:
* Monitor.start( "axisInvoke" );
* followed by a stop call at the end of the invoke method:
* Monitor.stop( "axisInvoke" );
*
* Monitor points can be paused around unwanted code. For example
* with the above monitors in the invoke method, you could exclude
* the SOAP call time with:
* Monitor.pause( "axisInvoke" );
* resp = call.invoke(locationUri, getSoapActionURI());
* Monitor.resume( "axisInvoke" );
*
* @author <a href="mailto:antelder@apache.org">Ant Elder</a>
*/
public class SpeedTest extends TestCase {
// This defines how long each test should be run for.
// There are 3 tests per provider; getPort, createOperation
// and executeOperation, so the total runnning time will be
// 3 times this value, times the number of providers tested.
static final float TIME_PER_TEST = 10000.0F; // 10 seconds
// This defines the time variation between runs before a
// result is considered a significant performance change.
static final float MARGIN_OF_ERROR = 0.1F; // tenth of a millisecond
// defines if this run will overwrite existing statistics
static final boolean UPDATE_STATS = true;
// delete this file to reset the previous statistics
File propFile = new File( "wsifPerformanceStats.txt" );
Properties stats;
static final String wsdlLocation =
TestUtilities.getWsdlPath("java\\samples\\addressbook\\wsifservice")
+ "AddressBook.wsdl";
public SpeedTest(String name) {
super(name);
}
public static void main(String[] args) {
TestUtilities.startListeners();
junit.textui.TestRunner.run (suite());
TestUtilities.stopListeners();
}
public static Test suite() {
return new TestSuite(SpeedTest.class);
}
public void setUp() {
TestUtilities.setUpExtensionsAndProviders();
}
public void tearDown() {
if ( UPDATE_STATS ) {
saveStatsToFile();
}
}
public void testJava() {
doit("JavaPort", "java");
}
//public void testAxis() {
// doit("SOAPPort", "axis");
//}
//public void testSoap() {
// doit("SOAPPort", "soap");
//}
//public void testEJB() {
//}
/* JMS seems to slow for most runs
public void testSoapJms() {
doit("SOAPJMSPort", "soap");
}
public void testAxisJms() {
doit("SOAPJMSPort", "axis");
}
public void testNativeJms() {
doit("NativeJmsPort", "njms" );
}
*/
private void doit(String portName, String protocol) {
int iterations;
long startTime, endTime;
float totalTime, avgTime;
String testNamePrefix = portName + "." + protocol;
String testName;
if (portName.indexOf("JMS") != -1 && !TestUtilities.areWeTesting("jms"))
return;
if (protocol.equals("axis")) {
WSIFServiceImpl.setDynamicWSIFProvider(
"http://schemas.xmlsoap.org/wsdl/soap/",
new WSIFDynamicProvider_ApacheAxis());
}
try {
WSIFServiceFactory factory = WSIFServiceFactory.newInstance();
WSIFService service = factory.getService(wsdlLocation, null, // serviceNS
null, // serviceName
"http://wsifservice.addressbook/", // portTypeNS
"AddressBook"); // portTypeName
service.mapType(
new javax.xml.namespace.QName(
"http://wsiftypes.addressbook/",
"address"),
Class.forName("addressbook.wsiftypes.Address"));
service.mapType(
new javax.xml.namespace.QName(
"http://wsiftypes.addressbook/",
"phone"),
Class.forName("addressbook.wsiftypes.Phone"));
WSIFPort port = null;
port = service.getPort(portName);
/*
* Run iterations of getPort
*/
testName = testNamePrefix + ".getPort";
iterations = getIterations( testName );
System.out.println( "running " + iterations + " " + testName + " iterations..." );
for (int i = 0; i < iterations; i++ ) {
Monitor.start( testName );
port = service.getPort(portName);
Monitor.stop( testName );
}
WSIFOperation operation =
port.createOperation("addEntry", "AddEntryWholeNameRequest", null);
WSIFMessage inputMessage = operation.createInputMessage();
WSIFMessage outputMessage = operation.createOutputMessage();
WSIFMessage faultMessage = operation.createFaultMessage();
// Create a name and address to add to the addressbook
String nameToAdd = "Chris P. Bacon";
Address addressToAdd =
new Address(
1,
"The Waterfront",
"Some City",
"NY",
47907,
new Phone(765, "494", "4900"));
// Add the name and address to the input message
inputMessage.setObjectPart("name", nameToAdd);
inputMessage.setObjectPart("address", addressToAdd);
// Execute the operation, obtaining a flag to indicate its success
boolean ok=
operation.executeRequestResponseOperation(
inputMessage,
outputMessage,
faultMessage);
assertTrue( "failed to add name and address!!", ok );
/*
* Run iterations of createOperation
*/
testName = testNamePrefix + ".createOperation";
iterations = getIterations( testName );
System.out.println( "running " + iterations + " " + testName + " iterations..." );
for (int i = 0; i < iterations; i++ ) {
Monitor.start( testName );
operation = port.createOperation("getAddressFromName");
Monitor.stop( testName );
}
/*
* Run iterations of executeRequestResponseOperation
*/
testName = testNamePrefix + ".executeOperation";
iterations = getIterations( testName );
System.out.println( "running " + iterations + " " + testName + " iterations..." );
startTime = System.currentTimeMillis();
for (int i = 0; i < iterations; i++ ) {
operation = port.createOperation("getAddressFromName");
inputMessage = operation.createInputMessage();
outputMessage = operation.createOutputMessage();
faultMessage = operation.createFaultMessage();
inputMessage.setObjectPart("name", nameToAdd);
Monitor.start( testName );
boolean operationSucceeded =
operation.executeRequestResponseOperation(
inputMessage,
outputMessage,
faultMessage);
Monitor.stop( testName );
if (!operationSucceeded) {
System.out.println("Failed to lookup name in addressbook");
assertTrue("executing op returned false!!", false);
}
}
// make sure it all worked
Address addressFound =
(Address) outputMessage.getObjectPart("address");
assertTrue( "returned address not correct!!",
addressToAdd.equals( addressFound) );
// test is theres a performance change
boolean worse = testResults();
updateStats();
setIterations( testNamePrefix );
assertTrue( "performance is worse than before!!", !worse );
} catch (Exception ex) {
ex.printStackTrace();
assertTrue("exception executing op!!", false);
}
}
private boolean testResults() {
float diff;
String s1, s2;
boolean worse = false;
HashMap results = Monitor.getResults();
String testName;
float duration, oldDuration;
for (Iterator i = results.keySet().iterator(); i.hasNext(); ) {
testName = (String)i.next();
duration = ((Float)results.get( testName )).floatValue();
oldDuration = getfloatStat( testName );
diff = duration - oldDuration;
worse = (diff-MARGIN_OF_ERROR) > 0;
if ( Math.abs(diff) - MARGIN_OF_ERROR > 0 ) {
System.err.println( testName + " significantly " +
( (diff>0) ? "worse" : "better" ) +
" by " + diff + " msecs, time = " + duration + " msecs" );
} else {
System.out.println( testName + " time = " + duration + " msecs" );
}
}
return worse;
}
private int getIterations(String testName) {
int i;
i = getintStat( testName + ".Iterations" );
if ( i == 0 ) {
i = 100; // 1st run default to 100 iterations
}
return i;
}
private void setIterations(String prefix) {
String s;
int iterations;
float duration, createDuration;
HashMap results = Monitor.getResults();
s = prefix + ".getPort";
duration = ((Float)results.get( s )).floatValue();
iterations = (duration==0) ?
100 : (int)((TIME_PER_TEST*100) / (duration*100));
stats.setProperty( s + ".Iterations", ""+iterations );
s = prefix + ".createOperation";
duration = ((Float)results.get( s )).floatValue();
iterations = (duration==0) ?
100 : (int)((TIME_PER_TEST*100) / (duration*100));
stats.setProperty( s + ".Iterations", ""+iterations );
createDuration = duration;
s = prefix + ".executeOperation";
duration = ((Float)results.get( s )).floatValue();
iterations = (duration==0) ?
100 : (int)((TIME_PER_TEST*100) / ((duration+createDuration)*100));
stats.setProperty( s + ".Iterations", ""+iterations );
}
private void updateStats() {
HashMap results = Monitor.getResults();
Properties stats = getStats();
String testName;
float duration;
int iterations;
for (Iterator i = results.keySet().iterator(); i.hasNext(); ) {
testName = (String)i.next();
duration = ((Float)results.get( testName )).floatValue();
stats.setProperty( testName, ""+duration );
}
}
private int getintStat(String name) {
int i;
Properties stats = getStats();
try {
i = Integer.parseInt(stats.getProperty( name ) );
} catch (Exception ex) {
i = 0;
}
return i;
}
private float getfloatStat(String name) {
float f;
Properties stats = getStats();
try {
f = Float.parseFloat(stats.getProperty( name ) );
} catch (Exception ex) {
f = 0;
}
return f;
}
private Properties getStats() {
if ( stats == null ) {
loadStatsFromFile();
}
return stats;
}
private void loadStatsFromFile() {
try {
FileInputStream fis = new FileInputStream( propFile );
stats = new Properties();
stats.load( fis );
} catch (Exception ex) {
stats = new Properties();
}
}
private void saveStatsToFile() {
try {
FileOutputStream fos = new FileOutputStream( propFile );
stats.store( fos, "WSIF Performance stats" );
} catch (Exception ex) {
ex.printStackTrace();
}
}
}
1.1 xml-axis-wsif/java/test/performance/Monitor.java
Index: Monitor.java
===================================================================
/*
* The Apache Software License, Version 1.1
*
*
* Copyright (c) 2002 The Apache Software Foundation. All rights
* reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* 3. The end-user documentation included with the redistribution,
* if any, must include the following acknowledgment:
* "This product includes software developed by the
* Apache Software Foundation (http://www.apache.org/)."
* Alternately, this acknowledgment may appear in the software itself,
* if and wherever such third-party acknowledgments normally appear.
*
* 4. The names "WSIF" and "Apache Software Foundation" must
* not be used to endorse or promote products derived from this
* software without prior written permission. For written
* permission, please contact apache@apache.org.
*
* 5. Products derived from this software may not be called "Apache",
* nor may "Apache" appear in their name, without prior written
* permission of the Apache Software Foundation.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation and was
* originally based on software copyright (c) 2001, 2002, International
* Business Machines, Inc., http://www.apache.org. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*/
package performance;
import java.util.HashMap;
import java.util.Iterator;
/**
* Collect monitor point timing statistics.
*
* @author <a href="mailto:antelder@apache.org">Ant Elder</a>
*/
public class Monitor {
static HashMap timers = new HashMap();
public static void start(String s) {
Timer t = (Timer)timers.get( s );
if ( t == null ) {
t = new Timer();
timers.put( s, t );
}
t.start();
}
public static void stop(String s) {
Timer t = (Timer)timers.get( s );
t.stop();
}
public static void pause(String s) {
Timer t = (Timer)timers.get( s );
if ( t != null ) {
t.pause();
}
}
public static void resume(String s) {
Timer t = (Timer)timers.get( s );
if ( t != null ) {
t.resume();
}
}
public static HashMap getResults() {
Timer t;
String testName;
HashMap results = new HashMap();
for (Iterator i = timers.keySet().iterator(); i.hasNext(); ) {
testName = (String)i.next();
t = (Timer)timers.get( testName );
float f1 = t.getAvgTime();
Float f = new Float( t.getAvgTime() );
results.put( testName, new Float( t.getAvgTime() ) );
}
return results;
}
}
class Timer {
long start;
long interval;
long totalTime;
int startCount;
public Timer() {
interval = 0;
totalTime = 0;
startCount = 0;
}
public void start() {
interval = 0;
startCount +=1;
totalTime += interval;
start = System.currentTimeMillis();
}
public void pause() {
interval += System.currentTimeMillis() - start;
start = 0;
}
public void resume() {
start = System.currentTimeMillis();
}
public void stop() {
interval += System.currentTimeMillis() - start;
totalTime += interval;
}
public long getInterval() {
return interval;
}
public float getAvgTime() {
return (float)totalTime / startCount;
}
}