You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by ke...@apache.org on 2021/03/03 03:27:20 UTC
[skywalking] branch enhance/lal created (now df1d05d)
This is an automated email from the ASF dual-hosted git repository.
kezhenxu94 pushed a change to branch enhance/lal
in repository https://gitbox.apache.org/repos/asf/skywalking.git.
at df1d05d Enhance the LAL to allow easily skipping logs with malformed formats
This branch includes the following new commits:
new df1d05d Enhance the LAL to allow easily skipping logs with malformed formats
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[skywalking] 01/01: Enhance the LAL to allow easily skipping logs
with malformed formats
Posted by ke...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kezhenxu94 pushed a commit to branch enhance/lal
in repository https://gitbox.apache.org/repos/asf/skywalking.git
commit df1d05df5139c733f5b8614948aafbd90fc39daf
Author: kezhenxu94 <ke...@apache.org>
AuthorDate: Wed Mar 3 11:26:56 2021 +0800
Enhance the LAL to allow easily skipping logs with malformed formats
---
docs/en/concepts-and-designs/lal.md | 39 +++++++++++++++++-----
.../log/analyzer/dsl/spec/filter/FilterSpec.java | 32 ++++++++++++------
...JsonParserSpec.java => AbstractParserSpec.java} | 27 ++++++++-------
.../analyzer/dsl/spec/parser/JsonParserSpec.java | 11 ++++--
.../analyzer/dsl/spec/parser/TextParserSpec.java | 5 +--
.../analyzer/dsl/spec/parser/YamlParserSpec.java | 3 +-
6 files changed, 79 insertions(+), 38 deletions(-)
diff --git a/docs/en/concepts-and-designs/lal.md b/docs/en/concepts-and-designs/lal.md
index 90cad9a..4fb331e 100644
--- a/docs/en/concepts-and-designs/lal.md
+++ b/docs/en/concepts-and-designs/lal.md
@@ -35,10 +35,9 @@ filter {
abort {} // all remaining components won't be executed at all
}
text {
- if (!regexp("(?<timestamp>\\d{8}) (?<thread>\\w+) (?<level>\\w+) (?<traceId>\\w+) (?<msg>.+)")) {
- // if the logs don't match this regexp, skip it
- abort {}
- }
+ // if the logs don't match this regexp, skip it
+ abortOnFailure true
+ regexp "(?<timestamp>\\d{8}) (?<thread>\\w+) (?<level>\\w+) (?<traceId>\\w+) (?<msg>.+)"
}
// ... extractors, sinks
}
@@ -55,15 +54,35 @@ types of parsers at the moment, namely `json`, `yaml`, and `text`.
When a piece of log is parsed, there is a corresponding property available, called `parsed`, injected by LAL.
Property `parsed` is typically a map, containing all the fields parsed from the raw logs, for example, if the parser
is `json` / `yaml`, `parsed` is a map containing all the key-values in the `json` / `yaml`, if the parser is `text`
-, `parsed` is a map containing all the captured groups and their values (for `regexp` and `grok`). See examples below.
+, `parsed` is a map containing all the captured groups and their values (for `regexp` and `grok`).
+
+All parsers share the following options:
+
+| Option | Type | Description | Default Value |
+| ------ | ---- | ----------- | ------------- |
+| `abortOnFailure` | `boolean` | Whether the filter chain should abort if the parser failed to parse / match the logs | `false` |
+
+See examples below.
#### `json`
-<!-- TODO: is structured in the reported (gRPC) `LogData`, not much to do -->
+```groovy
+filter {
+ json {
+ abortOnFailure true
+ }
+}
+```
#### `yaml`
-<!-- TODO: is structured in the reported (gRPC) `LogData`, not much to do -->
+```groovy
+filter {
+ yaml {
+ abortOnFailure true
+ }
+}
+```
#### `text`
@@ -78,6 +97,7 @@ all the captured groups can be used later in the extractors or sinks.
```groovy
filter {
text {
+ abortOnFailure true // if the logs don't match the pattern below, abort the filter chain
regexp "(?<timestamp>\\d{8}) (?<thread>\\w+) (?<level>\\w+) (?<traceId>\\w+) (?<msg>.+)"
// this is just a demo pattern
}
@@ -91,9 +111,10 @@ filter {
}
```
-- `grok`
+- `grok` (TODO)
-<!-- TODO: grok Java library has poor performance, need to benchmark it, the idea is basically the same with `regexp` above -->
+Because grok Java library has performance issue, we need some investigations and benchmark on it. Contributions are
+welcome.
### Extractor
diff --git a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/filter/FilterSpec.java b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/filter/FilterSpec.java
index 9cfe593..7683bb6 100644
--- a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/filter/FilterSpec.java
+++ b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/filter/FilterSpec.java
@@ -98,11 +98,17 @@ public class FilterSpec extends AbstractSpec {
cl.call();
final LogData.Builder logData = BINDING.get().log();
- final Map<String, Object> parsed = jsonParser.create().fromJson(
- logData.getBody().getJson().getJson(), parsedType
- );
-
- BINDING.get().parsed(parsed);
+ try {
+ final Map<String, Object> parsed = jsonParser.create().fromJson(
+ logData.getBody().getJson().getJson(), parsedType
+ );
+
+ BINDING.get().parsed(parsed);
+ } catch (final Exception e) {
+ if (jsonParser.abortOnFailure()) {
+ BINDING.get().abort();
+ }
+ }
}
@SuppressWarnings({"unused", "unchecked"})
@@ -114,11 +120,17 @@ public class FilterSpec extends AbstractSpec {
cl.call();
final LogData.Builder logData = BINDING.get().log();
- final Map<String, Object> parsed = (Map<String, Object>) yamlParser.create().load(
- logData.getBody().getYaml().getYaml()
- );
-
- BINDING.get().parsed(parsed);
+ try {
+ final Map<String, Object> parsed = (Map<String, Object>) yamlParser.create().load(
+ logData.getBody().getYaml().getYaml()
+ );
+
+ BINDING.get().parsed(parsed);
+ } catch (final Exception e) {
+ if (yamlParser.abortOnFailure()) {
+ BINDING.get().abort();
+ }
+ }
}
@SuppressWarnings("unused")
diff --git a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/JsonParserSpec.java b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/AbstractParserSpec.java
similarity index 66%
copy from oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/JsonParserSpec.java
copy to oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/AbstractParserSpec.java
index 6bb716d..315658c 100644
--- a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/JsonParserSpec.java
+++ b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/AbstractParserSpec.java
@@ -18,23 +18,26 @@
package org.apache.skywalking.oap.log.analyzer.dsl.spec.parser;
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
+import lombok.Getter;
+import lombok.Setter;
+import lombok.experimental.Accessors;
import org.apache.skywalking.oap.log.analyzer.dsl.spec.AbstractSpec;
import org.apache.skywalking.oap.log.analyzer.provider.LogAnalyzerModuleConfig;
import org.apache.skywalking.oap.server.library.module.ModuleManager;
-public class JsonParserSpec extends AbstractSpec {
- private final GsonBuilder gsonBuilder;
+@Accessors(fluent = true)
+public class AbstractParserSpec extends AbstractSpec {
+ /**
+ * Whether the filter chain should abort when parsing the logs failed.
+ *
+ * Failing to parse the logs means either parsing throws exceptions or the logs not matching the desired patterns.
+ */
+ @Getter
+ @Setter
+ private boolean abortOnFailure;
- public JsonParserSpec(final ModuleManager moduleManager,
- final LogAnalyzerModuleConfig moduleConfig) {
+ public AbstractParserSpec(final ModuleManager moduleManager,
+ final LogAnalyzerModuleConfig moduleConfig) {
super(moduleManager, moduleConfig);
-
- gsonBuilder = new GsonBuilder();
- }
-
- public Gson create() {
- return gsonBuilder.create();
}
}
diff --git a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/JsonParserSpec.java b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/JsonParserSpec.java
index 6bb716d..850daab 100644
--- a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/JsonParserSpec.java
+++ b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/JsonParserSpec.java
@@ -20,21 +20,26 @@ package org.apache.skywalking.oap.log.analyzer.dsl.spec.parser;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
-import org.apache.skywalking.oap.log.analyzer.dsl.spec.AbstractSpec;
import org.apache.skywalking.oap.log.analyzer.provider.LogAnalyzerModuleConfig;
import org.apache.skywalking.oap.server.library.module.ModuleManager;
-public class JsonParserSpec extends AbstractSpec {
+public class JsonParserSpec extends AbstractParserSpec {
private final GsonBuilder gsonBuilder;
+ private final Gson gson;
+
public JsonParserSpec(final ModuleManager moduleManager,
final LogAnalyzerModuleConfig moduleConfig) {
super(moduleManager, moduleConfig);
gsonBuilder = new GsonBuilder();
+
+ // We just create a gson instance in advance for now (for the sake of performance),
+ // when we want to provide some extra options, we'll move this into method "create" then.
+ gson = gsonBuilder.create();
}
public Gson create() {
- return gsonBuilder.create();
+ return gson;
}
}
diff --git a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/TextParserSpec.java b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/TextParserSpec.java
index 0ffccab..c77f639 100644
--- a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/TextParserSpec.java
+++ b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/TextParserSpec.java
@@ -21,11 +21,10 @@ package org.apache.skywalking.oap.log.analyzer.dsl.spec.parser;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.skywalking.apm.network.logging.v3.LogData;
-import org.apache.skywalking.oap.log.analyzer.dsl.spec.AbstractSpec;
import org.apache.skywalking.oap.log.analyzer.provider.LogAnalyzerModuleConfig;
import org.apache.skywalking.oap.server.library.module.ModuleManager;
-public class TextParserSpec extends AbstractSpec {
+public class TextParserSpec extends AbstractParserSpec {
public TextParserSpec(final ModuleManager moduleManager,
final LogAnalyzerModuleConfig moduleConfig) {
super(moduleManager, moduleConfig);
@@ -45,6 +44,8 @@ public class TextParserSpec extends AbstractSpec {
final boolean matched = matcher.find();
if (matched) {
BINDING.get().parsed(matcher);
+ } else if (abortOnFailure()) {
+ BINDING.get().abort();
}
return matched;
}
diff --git a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/YamlParserSpec.java b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/YamlParserSpec.java
index 2bb22b9..bbd4dc4 100644
--- a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/YamlParserSpec.java
+++ b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/YamlParserSpec.java
@@ -18,7 +18,6 @@
package org.apache.skywalking.oap.log.analyzer.dsl.spec.parser;
-import org.apache.skywalking.oap.log.analyzer.dsl.spec.AbstractSpec;
import org.apache.skywalking.oap.log.analyzer.provider.LogAnalyzerModuleConfig;
import org.apache.skywalking.oap.server.library.module.ModuleManager;
import org.yaml.snakeyaml.DumperOptions;
@@ -27,7 +26,7 @@ import org.yaml.snakeyaml.Yaml;
import org.yaml.snakeyaml.constructor.SafeConstructor;
import org.yaml.snakeyaml.representer.Representer;
-public class YamlParserSpec extends AbstractSpec {
+public class YamlParserSpec extends AbstractParserSpec {
private final LoaderOptions loaderOptions;
public YamlParserSpec(final ModuleManager moduleManager,