You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by ke...@apache.org on 2021/03/03 03:27:21 UTC
[skywalking] 01/01: Enhance the LAL to allow easily skipping logs
with malformed formats
This is an automated email from the ASF dual-hosted git repository.
kezhenxu94 pushed a commit to branch enhance/lal
in repository https://gitbox.apache.org/repos/asf/skywalking.git
commit df1d05df5139c733f5b8614948aafbd90fc39daf
Author: kezhenxu94 <ke...@apache.org>
AuthorDate: Wed Mar 3 11:26:56 2021 +0800
Enhance the LAL to allow easily skipping logs with malformed formats
---
docs/en/concepts-and-designs/lal.md | 39 +++++++++++++++++-----
.../log/analyzer/dsl/spec/filter/FilterSpec.java | 32 ++++++++++++------
...JsonParserSpec.java => AbstractParserSpec.java} | 27 ++++++++-------
.../analyzer/dsl/spec/parser/JsonParserSpec.java | 11 ++++--
.../analyzer/dsl/spec/parser/TextParserSpec.java | 5 +--
.../analyzer/dsl/spec/parser/YamlParserSpec.java | 3 +-
6 files changed, 79 insertions(+), 38 deletions(-)
diff --git a/docs/en/concepts-and-designs/lal.md b/docs/en/concepts-and-designs/lal.md
index 90cad9a..4fb331e 100644
--- a/docs/en/concepts-and-designs/lal.md
+++ b/docs/en/concepts-and-designs/lal.md
@@ -35,10 +35,9 @@ filter {
abort {} // all remaining components won't be executed at all
}
text {
- if (!regexp("(?<timestamp>\\d{8}) (?<thread>\\w+) (?<level>\\w+) (?<traceId>\\w+) (?<msg>.+)")) {
- // if the logs don't match this regexp, skip it
- abort {}
- }
+ // if the logs don't match this regexp, skip it
+ abortOnFailure true
+ regexp "(?<timestamp>\\d{8}) (?<thread>\\w+) (?<level>\\w+) (?<traceId>\\w+) (?<msg>.+)"
}
// ... extractors, sinks
}
@@ -55,15 +54,35 @@ types of parsers at the moment, namely `json`, `yaml`, and `text`.
When a piece of log is parsed, there is a corresponding property available, called `parsed`, injected by LAL.
Property `parsed` is typically a map, containing all the fields parsed from the raw logs, for example, if the parser
is `json` / `yaml`, `parsed` is a map containing all the key-values in the `json` / `yaml`, if the parser is `text`
-, `parsed` is a map containing all the captured groups and their values (for `regexp` and `grok`). See examples below.
+, `parsed` is a map containing all the captured groups and their values (for `regexp` and `grok`).
+
+All parsers share the following options:
+
+| Option | Type | Description | Default Value |
+| ------ | ---- | ----------- | ------------- |
+| `abortOnFailure` | `boolean` | Whether the filter chain should abort if the parser failed to parse / match the logs | `false` |
+
+See examples below.
#### `json`
-<!-- TODO: is structured in the reported (gRPC) `LogData`, not much to do -->
+```groovy
+filter {
+ json {
+ abortOnFailure true
+ }
+}
+```
#### `yaml`
-<!-- TODO: is structured in the reported (gRPC) `LogData`, not much to do -->
+```groovy
+filter {
+ yaml {
+ abortOnFailure true
+ }
+}
+```
#### `text`
@@ -78,6 +97,7 @@ all the captured groups can be used later in the extractors or sinks.
```groovy
filter {
text {
+ abortOnFailure true // if the logs don't match the pattern below, abort the filter chain
regexp "(?<timestamp>\\d{8}) (?<thread>\\w+) (?<level>\\w+) (?<traceId>\\w+) (?<msg>.+)"
// this is just a demo pattern
}
@@ -91,9 +111,10 @@ filter {
}
```
-- `grok`
+- `grok` (TODO)
-<!-- TODO: grok Java library has poor performance, need to benchmark it, the idea is basically the same with `regexp` above -->
+Because grok Java library has performance issue, we need some investigations and benchmark on it. Contributions are
+welcome.
### Extractor
diff --git a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/filter/FilterSpec.java b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/filter/FilterSpec.java
index 9cfe593..7683bb6 100644
--- a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/filter/FilterSpec.java
+++ b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/filter/FilterSpec.java
@@ -98,11 +98,17 @@ public class FilterSpec extends AbstractSpec {
cl.call();
final LogData.Builder logData = BINDING.get().log();
- final Map<String, Object> parsed = jsonParser.create().fromJson(
- logData.getBody().getJson().getJson(), parsedType
- );
-
- BINDING.get().parsed(parsed);
+ try {
+ final Map<String, Object> parsed = jsonParser.create().fromJson(
+ logData.getBody().getJson().getJson(), parsedType
+ );
+
+ BINDING.get().parsed(parsed);
+ } catch (final Exception e) {
+ if (jsonParser.abortOnFailure()) {
+ BINDING.get().abort();
+ }
+ }
}
@SuppressWarnings({"unused", "unchecked"})
@@ -114,11 +120,17 @@ public class FilterSpec extends AbstractSpec {
cl.call();
final LogData.Builder logData = BINDING.get().log();
- final Map<String, Object> parsed = (Map<String, Object>) yamlParser.create().load(
- logData.getBody().getYaml().getYaml()
- );
-
- BINDING.get().parsed(parsed);
+ try {
+ final Map<String, Object> parsed = (Map<String, Object>) yamlParser.create().load(
+ logData.getBody().getYaml().getYaml()
+ );
+
+ BINDING.get().parsed(parsed);
+ } catch (final Exception e) {
+ if (yamlParser.abortOnFailure()) {
+ BINDING.get().abort();
+ }
+ }
}
@SuppressWarnings("unused")
diff --git a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/JsonParserSpec.java b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/AbstractParserSpec.java
similarity index 66%
copy from oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/JsonParserSpec.java
copy to oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/AbstractParserSpec.java
index 6bb716d..315658c 100644
--- a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/JsonParserSpec.java
+++ b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/AbstractParserSpec.java
@@ -18,23 +18,26 @@
package org.apache.skywalking.oap.log.analyzer.dsl.spec.parser;
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
+import lombok.Getter;
+import lombok.Setter;
+import lombok.experimental.Accessors;
import org.apache.skywalking.oap.log.analyzer.dsl.spec.AbstractSpec;
import org.apache.skywalking.oap.log.analyzer.provider.LogAnalyzerModuleConfig;
import org.apache.skywalking.oap.server.library.module.ModuleManager;
-public class JsonParserSpec extends AbstractSpec {
- private final GsonBuilder gsonBuilder;
+@Accessors(fluent = true)
+public class AbstractParserSpec extends AbstractSpec {
+ /**
+ * Whether the filter chain should abort when parsing the logs failed.
+ *
+ * Failing to parse the logs means either parsing throws exceptions or the logs not matching the desired patterns.
+ */
+ @Getter
+ @Setter
+ private boolean abortOnFailure;
- public JsonParserSpec(final ModuleManager moduleManager,
- final LogAnalyzerModuleConfig moduleConfig) {
+ public AbstractParserSpec(final ModuleManager moduleManager,
+ final LogAnalyzerModuleConfig moduleConfig) {
super(moduleManager, moduleConfig);
-
- gsonBuilder = new GsonBuilder();
- }
-
- public Gson create() {
- return gsonBuilder.create();
}
}
diff --git a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/JsonParserSpec.java b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/JsonParserSpec.java
index 6bb716d..850daab 100644
--- a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/JsonParserSpec.java
+++ b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/JsonParserSpec.java
@@ -20,21 +20,26 @@ package org.apache.skywalking.oap.log.analyzer.dsl.spec.parser;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
-import org.apache.skywalking.oap.log.analyzer.dsl.spec.AbstractSpec;
import org.apache.skywalking.oap.log.analyzer.provider.LogAnalyzerModuleConfig;
import org.apache.skywalking.oap.server.library.module.ModuleManager;
-public class JsonParserSpec extends AbstractSpec {
+public class JsonParserSpec extends AbstractParserSpec {
private final GsonBuilder gsonBuilder;
+ private final Gson gson;
+
public JsonParserSpec(final ModuleManager moduleManager,
final LogAnalyzerModuleConfig moduleConfig) {
super(moduleManager, moduleConfig);
gsonBuilder = new GsonBuilder();
+
+ // We just create a gson instance in advance for now (for the sake of performance),
+ // when we want to provide some extra options, we'll move this into method "create" then.
+ gson = gsonBuilder.create();
}
public Gson create() {
- return gsonBuilder.create();
+ return gson;
}
}
diff --git a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/TextParserSpec.java b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/TextParserSpec.java
index 0ffccab..c77f639 100644
--- a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/TextParserSpec.java
+++ b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/TextParserSpec.java
@@ -21,11 +21,10 @@ package org.apache.skywalking.oap.log.analyzer.dsl.spec.parser;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.skywalking.apm.network.logging.v3.LogData;
-import org.apache.skywalking.oap.log.analyzer.dsl.spec.AbstractSpec;
import org.apache.skywalking.oap.log.analyzer.provider.LogAnalyzerModuleConfig;
import org.apache.skywalking.oap.server.library.module.ModuleManager;
-public class TextParserSpec extends AbstractSpec {
+public class TextParserSpec extends AbstractParserSpec {
public TextParserSpec(final ModuleManager moduleManager,
final LogAnalyzerModuleConfig moduleConfig) {
super(moduleManager, moduleConfig);
@@ -45,6 +44,8 @@ public class TextParserSpec extends AbstractSpec {
final boolean matched = matcher.find();
if (matched) {
BINDING.get().parsed(matcher);
+ } else if (abortOnFailure()) {
+ BINDING.get().abort();
}
return matched;
}
diff --git a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/YamlParserSpec.java b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/YamlParserSpec.java
index 2bb22b9..bbd4dc4 100644
--- a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/YamlParserSpec.java
+++ b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/YamlParserSpec.java
@@ -18,7 +18,6 @@
package org.apache.skywalking.oap.log.analyzer.dsl.spec.parser;
-import org.apache.skywalking.oap.log.analyzer.dsl.spec.AbstractSpec;
import org.apache.skywalking.oap.log.analyzer.provider.LogAnalyzerModuleConfig;
import org.apache.skywalking.oap.server.library.module.ModuleManager;
import org.yaml.snakeyaml.DumperOptions;
@@ -27,7 +26,7 @@ import org.yaml.snakeyaml.Yaml;
import org.yaml.snakeyaml.constructor.SafeConstructor;
import org.yaml.snakeyaml.representer.Representer;
-public class YamlParserSpec extends AbstractSpec {
+public class YamlParserSpec extends AbstractParserSpec {
private final LoaderOptions loaderOptions;
public YamlParserSpec(final ModuleManager moduleManager,