You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@brooklyn.apache.org by he...@apache.org on 2023/06/11 22:33:13 UTC
[brooklyn-server] 02/08: drop output which is identical to previous values
This is an automated email from the ASF dual-hosted git repository.
heneveld pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/brooklyn-server.git
commit 5ce00a7006697f035cf7afc950f7d540e73e16be
Author: Alex Heneveld <al...@cloudsoft.io>
AuthorDate: Thu Jun 8 15:36:55 2023 +0100
drop output which is identical to previous values
---
.../core/workflow/WorkflowExecutionContext.java | 29 ++++++++++++++++++----
.../brooklyn/core/workflow/WorkflowSizeTest.java | 9 +++++--
2 files changed, 31 insertions(+), 7 deletions(-)
diff --git a/core/src/main/java/org/apache/brooklyn/core/workflow/WorkflowExecutionContext.java b/core/src/main/java/org/apache/brooklyn/core/workflow/WorkflowExecutionContext.java
index 14e69e0951..74be491f3e 100644
--- a/core/src/main/java/org/apache/brooklyn/core/workflow/WorkflowExecutionContext.java
+++ b/core/src/main/java/org/apache/brooklyn/core/workflow/WorkflowExecutionContext.java
@@ -660,11 +660,23 @@ public class WorkflowExecutionContext {
@JsonIgnore
public Object getPreviousStepOutput() {
- if (lastErrorHandlerOutput!=null) return lastErrorHandlerOutput;
- if (previousStepIndex==null) return null;
-
- OldStepRecord last = oldStepInfo.get(previousStepIndex);
- if (last!=null && last.context!=null) return last.context.output;
+ Pair<Object, Set<Integer>> p = getPreviousStepOutputAndBacktrackedSteps();
+ if (p==null) return null;
+ return p.getLeft();
+ }
+ @JsonIgnore
+ public Pair<Object,Set<Integer>> getPreviousStepOutputAndBacktrackedSteps() {
+ if (lastErrorHandlerOutput!=null) return Pair.of(lastErrorHandlerOutput,null);
+
+ Integer prevSI = previousStepIndex;
+ Set<Integer> previousSteps = MutableSet.of();
+ while (prevSI!=null && previousSteps.add(prevSI)) {
+ OldStepRecord last = oldStepInfo.get(prevSI);
+ if (last==null || last.context==null) break;
+ if (last.context.output!=null) return Pair.of(last.context.output, previousSteps);
+ if (last.previous.isEmpty()) break;
+ prevSI = last.previous.iterator().next();
+ }
return null;
}
@@ -1346,6 +1358,13 @@ public class WorkflowExecutionContext {
BiConsumer<Object,Object> onFinish = (output,overrideNext) -> {
currentStepInstance.next = WorkflowReplayUtils.getNext(overrideNext, currentStepInstance, step);
if (output!=null) currentStepInstance.output = resolve(WorkflowExpressionResolution.WorkflowExpressionStage.STEP_FINISHING_POST_OUTPUT, output, Object.class);
+
+ // optimization, clear the value here if we can simply take it from the previous step;
+ // taking care not to break over loops
+ Pair<Object, Set<Integer>> prev = getPreviousStepOutputAndBacktrackedSteps();
+ if (prev!=null && prev.getRight()!=null && Objects.equals(currentStepInstance.output, prev.getLeft()) && !prev.getRight().contains(currentStepIndex)) {
+ currentStepInstance.output = null;
+ }
};
// now run the step
diff --git a/core/src/test/java/org/apache/brooklyn/core/workflow/WorkflowSizeTest.java b/core/src/test/java/org/apache/brooklyn/core/workflow/WorkflowSizeTest.java
index 683256f0ca..7c16b1d02e 100644
--- a/core/src/test/java/org/apache/brooklyn/core/workflow/WorkflowSizeTest.java
+++ b/core/src/test/java/org/apache/brooklyn/core/workflow/WorkflowSizeTest.java
@@ -26,6 +26,7 @@ import org.apache.brooklyn.api.mgmt.Task;
import org.apache.brooklyn.core.effector.Effectors;
import org.apache.brooklyn.core.entity.Entities;
import org.apache.brooklyn.core.resolve.jackson.BeanWithTypeUtils;
+import org.apache.brooklyn.core.sensor.Sensors;
import org.apache.brooklyn.core.test.BrooklynMgmtUnitTestSupport;
import org.apache.brooklyn.entity.stock.BasicApplication;
import org.apache.brooklyn.test.Asserts;
@@ -65,7 +66,7 @@ public class WorkflowSizeTest extends BrooklynMgmtUnitTestSupport {
}
@Test
- public void testSizeOfAllSensors() {
+ public void testSizeOfAllSensors() throws JsonProcessingException {
createAppWithEffector(MutableList.of(
"let pc = ${param}",
"let map myMap = {}",
@@ -84,14 +85,18 @@ public class WorkflowSizeTest extends BrooklynMgmtUnitTestSupport {
Asserts.assertThat(sizes.values().stream().reduce(0, (v0,v1)->v0+v1), result -> result < 10*1000);
+ // print out the above, search for "something big" to see where the size is used
+ String out = BeanWithTypeUtils.newYamlMapper(mgmt, true, null, true).writeValueAsString(
+ app.sensors().get(Sensors.newSensor(Object.class, "internals.brooklyn.workflow")));
+ log.info("WORKFLOW IS:\n"+out);
app.invoke(app.getEntityType().getEffectorByName("myWorkflow").get(), MutableMap.of("param", sampleData)).getUnchecked();
sizes = getSensorSizes();
sizes.forEach((k,v) -> { log.info("Sensor "+k+": "+v); });
Asserts.assertThat(sizes.values().stream().reduce(0, (v0,v1)->v0+v1), result -> result < 20*1000);
-
// 100k payload now -> bumps sensor size from 5k to 3MB (before any optimization)
+ // removing output which is identical to the previous gives minor savings (in this test): 3380416 -> 3176074
for (int i=0; i<1000; i++) {
for (int j=0; j<10; j++) sampleData += "0123456789";
sampleData += "\n";