You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@brooklyn.apache.org by he...@apache.org on 2022/08/24 14:25:19 UTC
[brooklyn-server] branch master updated: file log store, when doing recursive tasks, use the log message to find hierarchy
This is an automated email from the ASF dual-hosted git repository.
heneveld pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/brooklyn-server.git
The following commit(s) were added to refs/heads/master by this push:
new 0fc7b6d3e5 file log store, when doing recursive tasks, use the log message to find hierarchy
0fc7b6d3e5 is described below
commit 0fc7b6d3e5a92c83acfb5d9af66d1b2cdab650a3
Author: Alex Heneveld <al...@cloudsoft.io>
AuthorDate: Wed Aug 24 15:24:49 2022 +0100
file log store, when doing recursive tasks, use the log message to find hierarchy
doesn't rely on tasks being kept in memory
---
.../brooklyn/util/core/logbook/LogStore.java | 2 +-
.../util/core/logbook/file/FileLogStore.java | 60 ++++++++++++++++++++--
.../util/core/logbook/file/FileLogStoreTest.java | 10 ++--
.../brooklyn/util/core/logbook/file/log-sample.txt | 1 +
4 files changed, 63 insertions(+), 10 deletions(-)
diff --git a/core/src/main/java/org/apache/brooklyn/util/core/logbook/LogStore.java b/core/src/main/java/org/apache/brooklyn/util/core/logbook/LogStore.java
index 918b995e5d..c3c625359f 100644
--- a/core/src/main/java/org/apache/brooklyn/util/core/logbook/LogStore.java
+++ b/core/src/main/java/org/apache/brooklyn/util/core/logbook/LogStore.java
@@ -51,7 +51,7 @@ public interface LogStore {
Iterables.addAll(children, ((HasTaskChildren) task).getChildren());
Iterables.addAll(tasks, Iterables.limit(children, maxTasks - tasks.size()));
- if (tasks.size() == maxTasks) {
+ if (tasks.size() >= maxTasks) {
break enumerate; // Limit reached
}
}
diff --git a/core/src/main/java/org/apache/brooklyn/util/core/logbook/file/FileLogStore.java b/core/src/main/java/org/apache/brooklyn/util/core/logbook/file/FileLogStore.java
index 26d5f687c3..c0fe838785 100644
--- a/core/src/main/java/org/apache/brooklyn/util/core/logbook/file/FileLogStore.java
+++ b/core/src/main/java/org/apache/brooklyn/util/core/logbook/file/FileLogStore.java
@@ -92,7 +92,8 @@ public class FileLogStore implements LogStore {
private final String filePath;
private final Path path;
- private final String logLinePattern;
+ private final String logLinePatternString;
+ private final Pattern logLinePatternCompiled;
private final DateFormat dateFormat;
private final ManagementContext mgmt;
private final Integer maxTasks;
@@ -104,7 +105,8 @@ public class FileLogStore implements LogStore {
this.maxTasks = LOGBOOK_MAX_RECURSIVE_TASKS.getDefaultValue();
this.path = null;
this.filePath = "";
- this.logLinePattern = LOGBOOK_LOG_STORE_REGEX.getDefaultValue();
+ this.logLinePatternString = LOGBOOK_LOG_STORE_REGEX.getDefaultValue();
+ this.logLinePatternCompiled = Pattern.compile(this.logLinePatternString);
this.dateFormat = new SimpleDateFormat(LOGBOOK_LOG_STORE_DATEFORMAT.getDefaultValue());
this.dateFormat.setTimeZone(UTC_TIMEZONE);
}
@@ -113,7 +115,8 @@ public class FileLogStore implements LogStore {
this.mgmt = Preconditions.checkNotNull(mgmt);
this.maxTasks = mgmt.getConfig().getConfig(LOGBOOK_MAX_RECURSIVE_TASKS);
this.filePath = mgmt.getConfig().getConfig(LOGBOOK_LOG_STORE_PATH);
- this.logLinePattern = mgmt.getConfig().getConfig(LOGBOOK_LOG_STORE_REGEX);
+ this.logLinePatternString = mgmt.getConfig().getConfig(LOGBOOK_LOG_STORE_REGEX);
+ this.logLinePatternCompiled = Pattern.compile(this.logLinePatternString);
this.dateFormat = new SimpleDateFormat(mgmt.getConfig().getConfig(LOGBOOK_LOG_STORE_DATEFORMAT));
this.dateFormat.setTimeZone(UTC_TIMEZONE);
Preconditions.checkNotNull(filePath, "Log file path must be set: " + LOGBOOK_LOG_STORE_PATH.getName());
@@ -221,7 +224,7 @@ public class FileLogStore implements LogStore {
}
protected BrooklynLogEntry parseLogLine(String logLine, AtomicInteger lineCount) {
- Pattern p = Pattern.compile(this.logLinePattern);
+ Pattern p = logLinePatternCompiled;
Matcher m = p.matcher(logLine);
BrooklynLogEntry entry = null;
m.find();
@@ -243,4 +246,53 @@ public class FileLogStore implements LogStore {
}
return entry;
}
+
+ private static final boolean STARTING_TASK_MESSAGE_IS_ALWAYS_THE_FIRST_MESSAGE_FOR_THAT_TASK = true;
+ @Override
+ public Set<String> enumerateTaskIds(Task<?> parent, int maxTasks) {
+ Set<String> all = MutableSet.of();
+ if (parent!=null) {
+ Set<String> next = MutableSet.of(parent.getId());
+ while (!next.isEmpty() && all.size() < maxTasks) {
+ Set<String> current = MutableSet.copyOf(next);
+ next.clear();
+ all.addAll(current);
+
+ AtomicInteger lineCount = new AtomicInteger();
+ try (Stream<String> stream = Files.lines(path)) {
+ stream.forEach(line -> {
+ BrooklynLogEntry entry = parseLogLine(line, lineCount);
+ if (entry!=null && entry.getMessage()!=null && entry.getMessage().startsWith("Starting task ")) {
+ if (current.stream().anyMatch(possibleParent -> entry.getMessage().endsWith(possibleParent)) || next.stream().anyMatch(possibleParent -> entry.getMessage().endsWith(possibleParent))) {
+ String newTaskId = entry.getMessage();
+ newTaskId = Strings.removeFromStart(newTaskId, "Starting task ");
+ int nextWord = newTaskId.indexOf(' ');
+ if (nextWord>0) {
+ newTaskId = newTaskId.substring(0, nextWord);
+ if (all.add(newTaskId)) {
+ if (all.size()>=maxTasks) {
+ return;
+ }
+
+ // this is a newly found task
+ if (STARTING_TASK_MESSAGE_IS_ALWAYS_THE_FIRST_MESSAGE_FOR_THAT_TASK) {
+ current.add(newTaskId);
+ } else {
+ // we don't actually need a multi-pass strategy
+ next.add(newTaskId);
+ }
+ }
+ }
+ }
+ }
+ });
+ } catch (IOException e) {
+ throw Exceptions.propagate(e);
+ }
+ }
+ // add this explicitly, in case 0 tasks was requested
+ all.add(parent.getId());
+ }
+ return all;
+ }
}
diff --git a/core/src/test/java/org/apache/brooklyn/util/core/logbook/file/FileLogStoreTest.java b/core/src/test/java/org/apache/brooklyn/util/core/logbook/file/FileLogStoreTest.java
index 52229ab56a..4b4a7a64da 100644
--- a/core/src/test/java/org/apache/brooklyn/util/core/logbook/file/FileLogStoreTest.java
+++ b/core/src/test/java/org/apache/brooklyn/util/core/logbook/file/FileLogStoreTest.java
@@ -265,8 +265,8 @@ public class FileLogStoreTest extends BrooklynMgmtUnitTestSupport {
// Test with log levels only. There are 5 records in total in the normal order: DEBUG, ERROR, INFO, INFO, WARN.
// Expect 4 last items starting with ERROR.
assertEquals("INFO", brooklynLogEntries.get(0).getLevel());
- assertEquals("INFO", brooklynLogEntries.get(1).getLevel());
- assertEquals("WARN", brooklynLogEntries.get(2).getLevel());
+ assertEquals("WARN", brooklynLogEntries.get(1).getLevel());
+ assertEquals("DEBUG", brooklynLogEntries.get(2).getLevel());
assertEquals("INFO", brooklynLogEntries.get(3).getLevel());
}
@@ -310,7 +310,7 @@ public class FileLogStoreTest extends BrooklynMgmtUnitTestSupport {
logBookQueryParams.setTaskId("CMeSRJNF");
FileLogStore fileLogStore = new FileLogStore(mgmt);
List<BrooklynLogEntry> brooklynLogEntries = fileLogStore.query(logBookQueryParams);
- assertEquals(2, brooklynLogEntries.size());
+ assertEquals(3, brooklynLogEntries.size());
assertEquals("INFO", brooklynLogEntries.get(1).getLevel());
}
@@ -433,8 +433,8 @@ public class FileLogStoreTest extends BrooklynMgmtUnitTestSupport {
FileLogStore fileLogStore = new FileLogStore(mgmt);
List<BrooklynLogEntry> brooklynLogEntries = fileLogStore.query(logBookQueryParams);
- // There is one DEBUG log line and five INFO lines.
- assertEquals(6, brooklynLogEntries.size());
+ // There are two DEBUG log line and five INFO lines.
+ assertEquals(7, brooklynLogEntries.size());
// Check appearance of log levels
assertEquals("DEBUG", brooklynLogEntries.get(0).getLevel());
diff --git a/core/src/test/resources/brooklyn/util/core/logbook/file/log-sample.txt b/core/src/test/resources/brooklyn/util/core/logbook/file/log-sample.txt
index 04cd7245e7..a6b86e32ef 100644
--- a/core/src/test/resources/brooklyn/util/core/logbook/file/log-sample.txt
+++ b/core/src/test/resources/brooklyn/util/core/logbook/file/log-sample.txt
@@ -28,4 +28,5 @@ org.osgi.service.component.ComponentException: The component name 'org.apache.br
2021-08-30T11:28:26,764Z CMeSRJNF-[l8442kq0zu,iffj68b370] INFO 237 i.c.b.t.c.AbstractToscaYamlConverter [ger-rhkc5jtA-351] for testing
2021-08-30T11:28:27,764Z CMeSRJNX-[l8442kq0zu,iffj68b370] INFO 237 i.c.b.t.c.AbstractToscaYamlConverter [ger-rhkc5jtA-351] for testing
2021-08-30T11:28:28,764Z CMeSRJNX-[l8442kq0zx,iffj68b370] WARN 237 i.c.b.t.c.AbstractToscaYamlConverter [ger-rhkc5jtA-351] key 'brooklyn.locations' not supported
+2021-08-30T11:28:54,464Z THGMmYiu-[l8442kq0zx,iffj68b370] DEBUG 282 o.a.b.x.x.BasicTask [ger-rhkc5jtA-360] Starting task THGMmYiu (...) on entity xxx from task CMeSRJNF
2021-08-30T11:28:54,466Z THGMmYiu-[l8442kq0zx,iffj68b370] INFO 282 o.a.b.e.s.b.l.MachineLifecycleEffectorTasks [ger-rhkc5jtA-360] Starting ToscaComputeNodeEntityImpl{id=iffj68b370} on machine SshMachineLocation[cloudsoft-ubuntu:cloudsoft@135.181.244.108/135.181.244.108:22(id=dr6f7lah7i)]