You are viewing a plain text version of this content. The canonical link for it is here.
Posted to olio-dev@incubator.apache.org by "William Sobel (JIRA)" <ji...@apache.org> on 2009/03/14 22:44:50 UTC
[jira] Resolved: (OLIO-29) Driver does not check for accuracy in
throughput to determine pass/fail
[ https://issues.apache.org/jira/browse/OLIO-29?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
William Sobel resolved OLIO-29.
-------------------------------
Resolution: Fixed
Issue was resolved in r736839 by applying patch from Mandy. Not properly annotated in message diff follows:
Index: src/com/sun/web20/driver/UIDriver.java
===================================================================
--- src/com/sun/web20/driver/UIDriver.java (revision 736838)
+++ src/com/sun/web20/driver/UIDriver.java (revision 736839)
@@ -568,6 +568,7 @@
}
ctx.recordTime();
+ ++driverMetrics.addEventTotal;
}
@BenchmarkOperation (
@@ -614,6 +615,7 @@
addAddress(post);
doMultiPartPost(post);
ctx.recordTime();
+ ++driverMetrics.addPersonTotal;
}
@BenchmarkOperation (
@@ -880,6 +882,8 @@
int eventDetailImages = 0;
int homePageImagesLoaded = 0;
long homePageImageBytes = 0;
+ int addEventTotal = 0;
+ int addPersonTotal = 0;
public void add(CustomMetrics other) {
UIDriverMetrics o = (UIDriverMetrics) other;
@@ -890,19 +894,21 @@
eventDetailImages += o.eventDetailImages;
homePageImageBytes += o.homePageImageBytes;
homePageImagesLoaded += o.homePageImagesLoaded;
+ addEventTotal += o.addEventTotal;
+ addPersonTotal += o.addPersonTotal;
}
public Element[] getResults() {
Result r = Result.getInstance();
int total = r.getOpsCountSteady("EventDetail");
- Element[] el = new Element[7];
+ Element[] el = new Element[10];
el[0] = new Element();
el[0].description = "% EventDetail views where attendee added";
- el[0].target = ">= 8";
+ el[0].target = ">= 6";
if (total > 0) {
double pctAdd = 100d * addAttendeeCount / (double) total;
el[0].result = String.format("%.2f", pctAdd);
- if (pctAdd >= 8d)
+ if (pctAdd >= 6d)
el[0].passed = Boolean.TRUE;
else
el[0].passed = Boolean.FALSE;
@@ -934,11 +940,11 @@
el[3] = new Element();
el[3].description = "Average images loaded per Home Page";
- el[3].target = ">= 4";
+ el[3].target = ">= 3";
if (cnt > 0) {
double avgImgs = homePageImagesLoaded / (double) cnt;
el[3].result = String.format("%.2f", avgImgs);
- if (avgImgs >= 4d)
+ if (avgImgs >= 3d)
el[3].passed = Boolean.TRUE;
else
el[3].passed = Boolean.FALSE;
@@ -949,11 +955,11 @@
el[4] = new Element();
el[4].description = "Average image bytes received per Home Page";
- el[4].target = ">= 25000";
+ el[4].target = ">= 15000";
if (cnt > 0) {
double avgBytes = homePageImageBytes / (double) cnt;
el[4].result = String.format("%.2f", avgBytes);
- if (avgBytes >= 20000)
+ if (avgBytes >= 15000)
el[4].passed = Boolean.TRUE;
else
el[4].passed = Boolean.FALSE;
@@ -964,11 +970,11 @@
cnt = r.getOpsCountSteady("TagSearch");
el[5] = new Element();
el[5].description = "Average images on Tag Search Results";
- el[5].target = ">= 3";
+ el[5].target = ">= 3.6";
if (cnt > 0) {
double avgImgs = tagSearchImages / (double) cnt;
el[5].result = String.format("%.2f", avgImgs);
- if (avgImgs >= 3d)
+ if (avgImgs >= 3.6d)
el[5].passed = Boolean.TRUE;
else
el[5].passed = Boolean.FALSE;
@@ -978,7 +984,7 @@
}
el[6] = new Element();
el[6].description = "Average images on Event Detail";
- el[6].target = ">= 9";
+ el[6].target = ">= 1";
if (total > 0) {
double avgImgs = eventDetailImages / (double) total;
el[6].result = String.format("%.2f", avgImgs);
@@ -990,6 +996,21 @@
el[6].result = "";
el[6].passed = Boolean.FALSE;
}
+ el[7] = new Element();
+ el[7].description = "Total successful AddEvent calls";
+ el[7].result = String.valueOf(addEventTotal);
+ el[8] = new Element();
+ el[8].description = "Total successful AddPerson calls";
+ el[8].result = String.valueOf(addPersonTotal);
+ el[9] = new Element();
+ el[9].description = "Concurrent user to ops/sec ratio";
+ el[9].target = "<= 5.25";
+ double ratio = r.getScale() / r.getMetric();
+ el[9].result = String.format("%.2f", ratio);
+ if (ratio <= 5.25d)
+ el[9].passed = true;
+ else
+ el[9].passed = false;
return el;
}
@@ -1002,6 +1023,8 @@
clone.eventDetailImages = eventDetailImages;
clone.homePageImageBytes = homePageImageBytes;
clone.homePageImagesLoaded = homePageImagesLoaded;
+ clone.addEventTotal = addEventTotal;
+ clone.addPersonTotal = addPersonTotal;
return clone;
}
}
> Driver does not check for accuracy in throughput to determine pass/fail
> -----------------------------------------------------------------------
>
> Key: OLIO-29
> URL: https://issues.apache.org/jira/browse/OLIO-29
> Project: Olio
> Issue Type: Bug
> Components: rails-driver
> Reporter: Akara Sucharitakul
> Assignee: William Sobel
>
> The driver only checks whether the various metrics reported in the summary report have passed the necessary criteria but does not check whether the resulting throughput (ops/sec) is correct for the scale of the run (number of concurrent users). Thus for example, it is possible to get 150 ops/sec for a run done with 1000 users and be reported as PASSED.
--
This message is automatically generated by JIRA.
-
You can reply to this email to add a comment to the issue online.