You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by ke...@apache.org on 2021/07/07 12:22:16 UTC
[skywalking] branch perf/groovy updated (8ef87fb -> e4e3d16)
This is an automated email from the ASF dual-hosted git repository.
kezhenxu94 pushed a change to branch perf/groovy
in repository https://gitbox.apache.org/repos/asf/skywalking.git.
discard 8ef87fb perf: optimize Groovy-based DSL with static compilation
new e4e3d16 perf: optimize Groovy-based DSL with static compilation
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (8ef87fb)
\
N -- N -- N refs/heads/perf/groovy (e4e3d16)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
test/e2e/e2e-test/docker/log/lal.yaml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
[skywalking] 01/01: perf: optimize Groovy-based DSL with static
compilation
Posted by ke...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kezhenxu94 pushed a commit to branch perf/groovy
in repository https://gitbox.apache.org/repos/asf/skywalking.git
commit e4e3d16ee995b8b560ca0fb83614f78ea7492d62
Author: kezhenxu94 <ke...@apache.org>
AuthorDate: Wed Jul 7 18:13:37 2021 +0800
perf: optimize Groovy-based DSL with static compilation
Groovy naturally supports many dynamic features that we don't benefit for now but cost performance loss, in this patch we compile our Groovy-based DSL scripts statically to optimize performance.
---
.github/workflows/e2e.log.yaml | 3 +
oap-server/analyzer/log-analyzer/pom.xml | 4 +
.../skywalking/oap/log/analyzer/dsl/DSL.java | 13 +-
.../oap/log/analyzer/dsl/PrecompiledExtension.java | 89 +++++++++++
.../dsl/spec/SkyWalkingDelegatingScript.java | 56 +++++++
.../analyzer/dsl/spec/extractor/ExtractorSpec.java | 29 ++--
.../log/analyzer/dsl/spec/filter/FilterSpec.java | 29 ++--
.../dsl/spec/parser/AbstractParserSpec.java | 18 ++-
.../analyzer/dsl/spec/parser/JsonParserSpec.java | 17 +--
.../analyzer/dsl/spec/parser/TextParserSpec.java | 9 +-
.../log/analyzer/dsl/spec/sink/SamplerSpec.java | 6 +-
.../oap/log/analyzer/dsl/spec/sink/SinkSpec.java | 7 +-
.../skywalking/oap/log/analyzer/dsl/DSLTest.java | 162 +++++++++++++++++++++
.../test/resources/log-mal-rules/placeholder.yaml | 20 +--
.../agent/kafka/mock/MockModuleManager.java | 2 +
test/e2e/e2e-test/docker/log/lal.yaml | 4 +-
16 files changed, 394 insertions(+), 74 deletions(-)
diff --git a/.github/workflows/e2e.log.yaml b/.github/workflows/e2e.log.yaml
index 116b15e..4da8036 100644
--- a/.github/workflows/e2e.log.yaml
+++ b/.github/workflows/e2e.log.yaml
@@ -23,6 +23,9 @@ on:
- '!**.md'
schedule:
- cron: '0 18 * * *'
+ push:
+ branches:
+ - perf/groovy # TODO
env:
SW_AGENT_JDK_VERSION: 8
diff --git a/oap-server/analyzer/log-analyzer/pom.xml b/oap-server/analyzer/log-analyzer/pom.xml
index f1dd346..fbe0e6c 100644
--- a/oap-server/analyzer/log-analyzer/pom.xml
+++ b/oap-server/analyzer/log-analyzer/pom.xml
@@ -42,6 +42,10 @@
<groupId>org.codehaus.groovy</groupId>
<artifactId>groovy</artifactId>
</dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </dependency>
</dependencies>
</project>
diff --git a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/DSL.java b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/DSL.java
index c5fad4e..7f9e090 100644
--- a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/DSL.java
+++ b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/DSL.java
@@ -19,14 +19,18 @@
package org.apache.skywalking.oap.log.analyzer.dsl;
import groovy.lang.GroovyShell;
+import groovy.transform.CompileStatic;
import groovy.util.DelegatingScript;
+import java.util.Collections;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
+import org.apache.skywalking.oap.log.analyzer.dsl.spec.SkyWalkingDelegatingScript;
import org.apache.skywalking.oap.log.analyzer.dsl.spec.filter.FilterSpec;
import org.apache.skywalking.oap.log.analyzer.provider.LogAnalyzerModuleConfig;
import org.apache.skywalking.oap.server.library.module.ModuleManager;
import org.apache.skywalking.oap.server.library.module.ModuleStartException;
import org.codehaus.groovy.control.CompilerConfiguration;
+import org.codehaus.groovy.control.customizers.ASTTransformationCustomizer;
@RequiredArgsConstructor(access = AccessLevel.PRIVATE)
public class DSL {
@@ -38,7 +42,14 @@ public class DSL {
final LogAnalyzerModuleConfig config,
final String dsl) throws ModuleStartException {
final CompilerConfiguration cc = new CompilerConfiguration();
- cc.setScriptBaseClass(DelegatingScript.class.getName());
+ final ASTTransformationCustomizer customizer =
+ new ASTTransformationCustomizer(CompileStatic.class);
+ customizer.setAnnotationParameters(Collections.singletonMap(
+ "extensions",
+ Collections.singletonList(PrecompiledExtension.class.getName())
+ ));
+ cc.addCompilationCustomizers(customizer);
+ cc.setScriptBaseClass(SkyWalkingDelegatingScript.class.getName());
final GroovyShell sh = new GroovyShell(cc);
final DelegatingScript script = (DelegatingScript) sh.parse(dsl);
diff --git a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/PrecompiledExtension.java b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/PrecompiledExtension.java
new file mode 100644
index 0000000..3d20b6c
--- /dev/null
+++ b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/PrecompiledExtension.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oap.log.analyzer.dsl;
+
+import org.apache.skywalking.apm.network.logging.v3.LogData;
+import org.apache.skywalking.apm.network.logging.v3.LogDataBody;
+import org.apache.skywalking.apm.network.logging.v3.LogTags;
+import org.apache.skywalking.apm.network.logging.v3.TraceContext;
+import org.codehaus.groovy.ast.expr.ConstantExpression;
+import org.codehaus.groovy.ast.expr.Expression;
+import org.codehaus.groovy.ast.expr.PropertyExpression;
+import org.codehaus.groovy.ast.expr.VariableExpression;
+import org.codehaus.groovy.transform.stc.AbstractTypeCheckingExtension;
+import org.codehaus.groovy.transform.stc.StaticTypeCheckingVisitor;
+
+import static org.codehaus.groovy.ast.ClassHelper.makeCached;
+
+public class PrecompiledExtension extends AbstractTypeCheckingExtension {
+
+ public PrecompiledExtension(final StaticTypeCheckingVisitor typeCheckingVisitor) {
+ super(typeCheckingVisitor);
+ }
+
+ @Override
+ public boolean handleUnresolvedProperty(final PropertyExpression pexp) {
+ final Expression exp = pexp.getObjectExpression();
+
+ if (exp.getText().startsWith("parsed")) {
+ makeDynamic(pexp);
+ setHandled(true);
+ return true;
+ }
+
+ if (exp.getText().startsWith("log")) {
+ if (handleLogVariable(pexp)) {
+ return true;
+ }
+ }
+
+ return super.handleUnresolvedProperty(pexp);
+ }
+
+ private boolean handleLogVariable(final PropertyExpression pexp) {
+ final Expression exp = pexp.getObjectExpression();
+ final Expression p = pexp.getProperty();
+
+ if (exp instanceof VariableExpression) {
+ final VariableExpression v = (VariableExpression) exp;
+ if (v.getName().equals("log")) {
+ storeType(v, makeCached(LogData.Builder.class));
+ }
+ if (p instanceof ConstantExpression) {
+ final ConstantExpression c = (ConstantExpression) p;
+ switch (c.getText()) {
+ case "body":
+ storeType(pexp, makeCached(LogDataBody.class));
+ break;
+ case "traceContext":
+ storeType(pexp, makeCached(TraceContext.class));
+ break;
+ case "tags":
+ storeType(pexp, makeCached(LogTags.class));
+ break;
+ }
+ }
+ setHandled(true);
+ return true;
+ }
+
+ return false;
+ }
+}
+
diff --git a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/SkyWalkingDelegatingScript.java b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/SkyWalkingDelegatingScript.java
new file mode 100644
index 0000000..e0fc98e
--- /dev/null
+++ b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/SkyWalkingDelegatingScript.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oap.log.analyzer.dsl.spec;
+
+import groovy.lang.Closure;
+import groovy.lang.DelegatesTo;
+import groovy.util.DelegatingScript;
+import org.apache.skywalking.oap.log.analyzer.dsl.spec.filter.FilterSpec;
+
+public class SkyWalkingDelegatingScript extends DelegatingScript {
+ @Override
+ public Object run() {
+ return null;
+ }
+
+ public void filter(@DelegatesTo(value = FilterSpec.class, strategy = Closure.DELEGATE_ONLY) Closure<?> closure) {
+ closure.setDelegate(getDelegate());
+ closure.call();
+ }
+
+ public void json(@DelegatesTo(value = FilterSpec.class) Closure<?> closure) {
+ closure.setDelegate(getDelegate());
+ closure.call();
+ }
+
+ public void text(@DelegatesTo(value = FilterSpec.class) Closure<?> closure) {
+ closure.setDelegate(getDelegate());
+ closure.call();
+ }
+
+ public void extractor(@DelegatesTo(value = FilterSpec.class) Closure<?> closure) {
+ closure.setDelegate(getDelegate());
+ closure.call();
+ }
+
+ public void sink(@DelegatesTo(value = FilterSpec.class) Closure<?> closure) {
+ closure.setDelegate(getDelegate());
+ closure.call();
+ }
+}
diff --git a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/extractor/ExtractorSpec.java b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/extractor/ExtractorSpec.java
index 49ea7d6..c51f445 100644
--- a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/extractor/ExtractorSpec.java
+++ b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/extractor/ExtractorSpec.java
@@ -21,6 +21,7 @@ package org.apache.skywalking.oap.log.analyzer.dsl.spec.extractor;
import com.google.common.base.Joiner;
import com.google.common.collect.ImmutableMap;
import groovy.lang.Closure;
+import groovy.lang.DelegatesTo;
import java.util.Collection;
import java.util.List;
import java.util.Map;
@@ -54,7 +55,8 @@ public class ExtractorSpec extends AbstractSpec {
final LogAnalyzerModuleConfig moduleConfig) throws ModuleStartException {
super(moduleManager, moduleConfig);
- final MeterSystem meterSystem = moduleManager.find(CoreModule.NAME).provider().getService(MeterSystem.class);
+ final MeterSystem meterSystem =
+ moduleManager.find(CoreModule.NAME).provider().getService(MeterSystem.class);
metricConverts = moduleConfig.malConfigs()
.stream()
@@ -93,7 +95,7 @@ public class ExtractorSpec extends AbstractSpec {
}
@SuppressWarnings("unused")
- public void tag(final Map<String, Object> kv) {
+ public void tag(final Map<String, ?> kv) {
if (BINDING.get().shouldAbort()) {
return;
}
@@ -108,7 +110,8 @@ public class ExtractorSpec extends AbstractSpec {
kv.entrySet()
.stream()
.filter(it -> isNotBlank(it.getKey()))
- .filter(it -> nonNull(it.getValue()) && isNotBlank(Objects.toString(it.getValue())))
+ .filter(it -> nonNull(it.getValue()) &&
+ isNotBlank(Objects.toString(it.getValue())))
.map(it -> {
final Object val = it.getValue();
String valStr = Objects.toString(val);
@@ -176,7 +179,7 @@ public class ExtractorSpec extends AbstractSpec {
}
@SuppressWarnings("unused")
- public void metrics(final Closure<Void> cl) {
+ public void metrics(@DelegatesTo(SampleBuilder.class) final Closure<?> cl) {
if (BINDING.get().shouldAbort()) {
return;
}
@@ -188,8 +191,8 @@ public class ExtractorSpec extends AbstractSpec {
metricConverts.forEach(it -> it.toMeter(
ImmutableMap.<String, SampleFamily>builder()
- .put(sample.getName(), SampleFamilyBuilder.newBuilder(sample).build())
- .build()
+ .put(sample.getName(), SampleFamilyBuilder.newBuilder(sample).build())
+ .build()
));
}
@@ -198,11 +201,15 @@ public class ExtractorSpec extends AbstractSpec {
private final Sample.SampleBuilder sampleBuilder = Sample.builder();
@SuppressWarnings("unused")
- public Sample.SampleBuilder labels(final Map<String, String> labels) {
- final Map<String, String> filtered = labels.entrySet()
- .stream()
- .filter(it -> isNotBlank(it.getKey()) && isNotBlank(it.getValue()))
- .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+ public Sample.SampleBuilder labels(final Map<String, ?> labels) {
+ final Map<String, String> filtered =
+ labels.entrySet()
+ .stream()
+ .filter(it -> isNotBlank(it.getKey()) && nonNull(it.getValue()))
+ .collect(
+ Collectors.toMap(Map.Entry::getKey,
+ it -> Objects.toString(it.getValue()))
+ );
return sampleBuilder.labels(ImmutableMap.copyOf(filtered));
}
}
diff --git a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/filter/FilterSpec.java b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/filter/FilterSpec.java
index 7683bb6..43f86a3 100644
--- a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/filter/FilterSpec.java
+++ b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/filter/FilterSpec.java
@@ -18,10 +18,10 @@
package org.apache.skywalking.oap.log.analyzer.dsl.spec.filter;
-import com.google.gson.reflect.TypeToken;
+import com.fasterxml.jackson.core.type.TypeReference;
import com.google.protobuf.TextFormat;
import groovy.lang.Closure;
-import java.lang.reflect.Type;
+import groovy.lang.DelegatesTo;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
@@ -57,14 +57,14 @@ public class FilterSpec extends AbstractSpec {
private final SinkSpec sink;
- private final Type parsedType;
+ private final TypeReference<Map<String, Object>> parsedType;
public FilterSpec(final ModuleManager moduleManager,
final LogAnalyzerModuleConfig moduleConfig) throws ModuleStartException {
super(moduleManager, moduleConfig);
- parsedType = new TypeToken<Map<String, Object>>() {
- }.getType();
+ parsedType = new TypeReference<Map<String, Object>>() {
+ };
factories = Arrays.asList(
new RecordAnalysisListener.Factory(moduleManager(), moduleConfig()),
@@ -81,7 +81,7 @@ public class FilterSpec extends AbstractSpec {
}
@SuppressWarnings("unused")
- public void text(final Closure<Void> cl) {
+ public void text(@DelegatesTo(TextParserSpec.class) final Closure<?> cl) {
if (BINDING.get().shouldAbort()) {
return;
}
@@ -90,7 +90,7 @@ public class FilterSpec extends AbstractSpec {
}
@SuppressWarnings("unused")
- public void json(final Closure<Void> cl) {
+ public void json(@DelegatesTo(JsonParserSpec.class) final Closure<?> cl) {
if (BINDING.get().shouldAbort()) {
return;
}
@@ -99,7 +99,8 @@ public class FilterSpec extends AbstractSpec {
final LogData.Builder logData = BINDING.get().log();
try {
- final Map<String, Object> parsed = jsonParser.create().fromJson(
+
+ final Map<String, Object> parsed = jsonParser.create().readValue(
logData.getBody().getJson().getJson(), parsedType
);
@@ -111,8 +112,8 @@ public class FilterSpec extends AbstractSpec {
}
}
- @SuppressWarnings({"unused", "unchecked"})
- public void yaml(final Closure<Void> cl) {
+ @SuppressWarnings({"unused"})
+ public void yaml(@DelegatesTo(YamlParserSpec.class) final Closure<?> cl) {
if (BINDING.get().shouldAbort()) {
return;
}
@@ -121,7 +122,7 @@ public class FilterSpec extends AbstractSpec {
final LogData.Builder logData = BINDING.get().log();
try {
- final Map<String, Object> parsed = (Map<String, Object>) yamlParser.create().load(
+ final Map<String, Object> parsed = yamlParser.create().load(
logData.getBody().getYaml().getYaml()
);
@@ -134,7 +135,7 @@ public class FilterSpec extends AbstractSpec {
}
@SuppressWarnings("unused")
- public void extractor(final Closure<Void> cl) {
+ public void extractor(@DelegatesTo(ExtractorSpec.class) final Closure<?> cl) {
if (BINDING.get().shouldAbort()) {
return;
}
@@ -143,7 +144,7 @@ public class FilterSpec extends AbstractSpec {
}
@SuppressWarnings("unused")
- public void sink(final Closure<Void> cl) {
+ public void sink(@DelegatesTo(SinkSpec.class) final Closure<?> cl) {
if (BINDING.get().shouldAbort()) {
return;
}
@@ -166,7 +167,7 @@ public class FilterSpec extends AbstractSpec {
}
@SuppressWarnings("unused")
- public void filter(final Closure<Void> cl) {
+ public void filter(final Closure<?> cl) {
cl.call();
}
}
diff --git a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/AbstractParserSpec.java b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/AbstractParserSpec.java
index e410cbe..a7db119 100644
--- a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/AbstractParserSpec.java
+++ b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/AbstractParserSpec.java
@@ -18,26 +18,32 @@
package org.apache.skywalking.oap.log.analyzer.dsl.spec.parser;
-import lombok.Getter;
-import lombok.Setter;
import lombok.experimental.Accessors;
import org.apache.skywalking.oap.log.analyzer.dsl.spec.AbstractSpec;
import org.apache.skywalking.oap.log.analyzer.provider.LogAnalyzerModuleConfig;
import org.apache.skywalking.oap.server.library.module.ModuleManager;
-@Accessors(fluent = true)
+@Accessors
public class AbstractParserSpec extends AbstractSpec {
/**
* Whether the filter chain should abort when parsing the logs failed.
*
- * Failing to parse the logs means either parsing throws exceptions or the logs not matching the desired patterns.
+ * Failing to parse the logs means either parsing throws exceptions or the logs not matching the
+ * desired patterns.
*/
- @Getter
- @Setter
private boolean abortOnFailure = true;
public AbstractParserSpec(final ModuleManager moduleManager,
final LogAnalyzerModuleConfig moduleConfig) {
super(moduleManager, moduleConfig);
}
+
+ @SuppressWarnings("unused") // used in user LAL scripts
+ public void abortOnFailure(final boolean abortOnFailure) {
+ this.abortOnFailure = abortOnFailure;
+ }
+
+ public boolean abortOnFailure() {
+ return this.abortOnFailure;
+ }
}
diff --git a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/JsonParserSpec.java b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/JsonParserSpec.java
index 850daab..1fabdbc 100644
--- a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/JsonParserSpec.java
+++ b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/JsonParserSpec.java
@@ -18,28 +18,23 @@
package org.apache.skywalking.oap.log.analyzer.dsl.spec.parser;
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
+import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.skywalking.oap.log.analyzer.provider.LogAnalyzerModuleConfig;
import org.apache.skywalking.oap.server.library.module.ModuleManager;
public class JsonParserSpec extends AbstractParserSpec {
- private final GsonBuilder gsonBuilder;
-
- private final Gson gson;
+ private final ObjectMapper mapper;
public JsonParserSpec(final ModuleManager moduleManager,
final LogAnalyzerModuleConfig moduleConfig) {
super(moduleManager, moduleConfig);
- gsonBuilder = new GsonBuilder();
-
- // We just create a gson instance in advance for now (for the sake of performance),
+ // We just create a mapper instance in advance for now (for the sake of performance),
// when we want to provide some extra options, we'll move this into method "create" then.
- gson = gsonBuilder.create();
+ mapper = new ObjectMapper();
}
- public Gson create() {
- return gson;
+ public ObjectMapper create() {
+ return mapper;
}
}
diff --git a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/TextParserSpec.java b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/TextParserSpec.java
index c77f639..f20a1e9 100644
--- a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/TextParserSpec.java
+++ b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/parser/TextParserSpec.java
@@ -31,13 +31,13 @@ public class TextParserSpec extends AbstractParserSpec {
}
@SuppressWarnings("unused")
- public boolean regexp(final String regexp) {
- return regexp(Pattern.compile(regexp));
+ public void regexp(final String regexp) {
+ regexp(Pattern.compile(regexp));
}
- public boolean regexp(final Pattern pattern) {
+ public void regexp(final Pattern pattern) {
if (BINDING.get().shouldAbort()) {
- return false;
+ return;
}
final LogData.Builder log = BINDING.get().log();
final Matcher matcher = pattern.matcher(log.getBody().getText().getText());
@@ -47,7 +47,6 @@ public class TextParserSpec extends AbstractParserSpec {
} else if (abortOnFailure()) {
BINDING.get().abort();
}
- return matched;
}
public boolean grok(final String grok) {
diff --git a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/sink/SamplerSpec.java b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/sink/SamplerSpec.java
index e32f792..77df935 100644
--- a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/sink/SamplerSpec.java
+++ b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/sink/SamplerSpec.java
@@ -19,6 +19,8 @@
package org.apache.skywalking.oap.log.analyzer.dsl.spec.sink;
import groovy.lang.Closure;
+import groovy.lang.DelegatesTo;
+import groovy.lang.GString;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.skywalking.oap.log.analyzer.dsl.spec.AbstractSpec;
@@ -28,7 +30,7 @@ import org.apache.skywalking.oap.log.analyzer.provider.LogAnalyzerModuleConfig;
import org.apache.skywalking.oap.server.library.module.ModuleManager;
public class SamplerSpec extends AbstractSpec {
- private final Map<String, Sampler> samplers;
+ private final Map<GString, Sampler> samplers;
private final RateLimitingSampler.ResetHandler rlsResetHandler;
public SamplerSpec(final ModuleManager moduleManager,
@@ -40,7 +42,7 @@ public class SamplerSpec extends AbstractSpec {
}
@SuppressWarnings("unused")
- public void rateLimit(final String id, final Closure<Void> cl) {
+ public void rateLimit(final GString id, @DelegatesTo(RateLimitingSampler.class) final Closure<?> cl) {
if (BINDING.get().shouldAbort()) {
return;
}
diff --git a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/sink/SinkSpec.java b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/sink/SinkSpec.java
index ee78be9..82566f9 100644
--- a/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/sink/SinkSpec.java
+++ b/oap-server/analyzer/log-analyzer/src/main/java/org/apache/skywalking/oap/log/analyzer/dsl/spec/sink/SinkSpec.java
@@ -19,6 +19,7 @@
package org.apache.skywalking.oap.log.analyzer.dsl.spec.sink;
import groovy.lang.Closure;
+import groovy.lang.DelegatesTo;
import org.apache.skywalking.oap.log.analyzer.dsl.spec.AbstractSpec;
import org.apache.skywalking.oap.log.analyzer.provider.LogAnalyzerModuleConfig;
import org.apache.skywalking.oap.server.library.module.ModuleManager;
@@ -35,7 +36,7 @@ public class SinkSpec extends AbstractSpec {
}
@SuppressWarnings("unused")
- public void sampler(final Closure<Void> cl) {
+ public void sampler(@DelegatesTo(SamplerSpec.class) final Closure<?> cl) {
if (BINDING.get().shouldAbort()) {
return;
}
@@ -44,7 +45,7 @@ public class SinkSpec extends AbstractSpec {
}
@SuppressWarnings("unused")
- public void enforcer(final Closure<Void> cl) {
+ public void enforcer(final Closure<?> cl) {
if (BINDING.get().shouldAbort()) {
return;
}
@@ -52,7 +53,7 @@ public class SinkSpec extends AbstractSpec {
}
@SuppressWarnings("unused")
- public void dropper(final Closure<Void> cl) {
+ public void dropper(final Closure<?> cl) {
if (BINDING.get().shouldAbort()) {
return;
}
diff --git a/oap-server/analyzer/log-analyzer/src/test/java/org/apache/skywalking/oap/log/analyzer/dsl/DSLTest.java b/oap-server/analyzer/log-analyzer/src/test/java/org/apache/skywalking/oap/log/analyzer/dsl/DSLTest.java
new file mode 100644
index 0000000..8a923ce
--- /dev/null
+++ b/oap-server/analyzer/log-analyzer/src/test/java/org/apache/skywalking/oap/log/analyzer/dsl/DSLTest.java
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oap.log.analyzer.dsl;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import org.apache.skywalking.apm.network.logging.v3.LogData;
+import org.apache.skywalking.oap.log.analyzer.provider.LogAnalyzerModuleConfig;
+import org.apache.skywalking.oap.server.core.CoreModule;
+import org.apache.skywalking.oap.server.core.config.ConfigService;
+import org.apache.skywalking.oap.server.core.source.SourceReceiver;
+import org.apache.skywalking.oap.server.library.module.ModuleManager;
+import org.apache.skywalking.oap.server.library.module.ModuleProviderHolder;
+import org.apache.skywalking.oap.server.library.module.ModuleServiceHolder;
+import org.apache.skywalking.oap.server.library.module.ModuleStartException;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.powermock.reflect.Whitebox;
+
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+@RunWith(Parameterized.class)
+public class DSLTest {
+ @Parameterized.Parameters(name = "{index}: {0}")
+ public static Collection<Object[]> data() {
+ return Arrays.asList(
+ new String[] {
+ "parser",
+ "filter {\n" +
+ " json {\n" +
+ " abortOnFailure false // for test purpose, we want to persist all logs\n" +
+ " }\n" +
+ " text {\n" +
+ " abortOnFailure false // for test purpose, we want to persist all logs\n" +
+ " regexp $/(?s)(?<timestamp>\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}.\\d{3}) \\[TID:(?<tid>.+?)] \\[(?<thread>.+?)] (?<level>\\w{4,}) (?<logger>.{1,36}) (?<msg>.+)/$" +
+ " }\n" +
+ " yaml {\n" +
+ " abortOnFailure false // for test purpose, we want to persist all logs\n" +
+ " }" +
+ "}",
+ },
+ new String[] {
+ "extractor",
+ "filter {\n" +
+ " extractor {\n" +
+ " service \"test\"\n" +
+ " instance \"test\"\n" +
+ " endpoint \"test\"\n" +
+ " traceId \"123\"\n" +
+ " segmentId \"123\"\n" +
+ " spanId \"123\"\n" +
+ " timestamp \"123\"\n" +
+ " metrics {\n" +
+ " name \"metricsName\"\n" +
+ " value 123\n" +
+ " timestamp \"123\"\n" +
+ " labels \"k1\": \"v1\"\n" +
+ " }\n" +
+ " }\n" +
+ "}",
+ },
+ new String[] {
+ "sink",
+ "filter {\n" +
+ " sink {\n" +
+ " enforcer {\n" +
+ " }\n" +
+ " dropper {\n" +
+ " }\n" +
+ " sampler {\n" +
+ " if (parsed?.commonProperties?.responseFlags) {\n" +
+ " // use service:errorCode as sampler id so that each service:errorCode has its own sampler,\n" +
+ " // e.g. checkoutservice:[upstreamConnectionFailure], checkoutservice:[upstreamRetryLimitExceeded]\n" +
+ " rateLimit(\"${log.service}:${log.body.json.json}:${log.tags.getData(0).key}:${parsed?.commonProperties?.responseFlags}\") {\n" +
+ " qps 100\n" +
+ " }\n" +
+ " } else {\n" +
+ " // use service:responseCode as sampler id so that each service:responseCode has its own sampler,\n" +
+ " // e.g. checkoutservice:500, checkoutservice:404.\n" +
+ " rateLimit(\"${log.service}:${log.body?.type}:${log.traceContext?.traceId}:${parsed?.response?.responseCode}\") {\n" +
+ " qps 100\n" +
+ " }\n" +
+ " }\n" +
+ " }\n" +
+ " }\n" +
+ "}",
+ },
+ new String[] {
+ "e2e",
+ "filter {\n" +
+ " text {\n" +
+ " abortOnFailure false // for test purpose, we want to persist all logs\n" +
+ " regexp $/(?s)(?<timestamp>\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}.\\d{3}) \\[TID:(?<tid>.+?)] \\[(?<thread>.+?)] (?<level>\\w{4,}) (?<logger>.{1,36}) (?<msg>.+)/$\n" +
+ " }\n" +
+ " extractor {\n" +
+ " metrics {\n" +
+ " timestamp \"${log.timestamp}\"\n" +
+ " labels level: parsed.level, service: log.service, instance: log.serviceInstance\n" +
+ " name \"log_count\"\n" +
+ " value 1\n" +
+ " }\n" +
+ " }\n" +
+ " sink {\n" +
+ " }\n" +
+ "}\n"
+ }
+ );
+ }
+
+ @Parameterized.Parameter()
+ public String name;
+
+ @Parameterized.Parameter(1)
+ public String script;
+
+ final ModuleManager manager = mock(ModuleManager.class);
+
+ @Before
+ public void setup() {
+ Whitebox.setInternalState(manager, "isInPrepareStage", false);
+ when(manager.find(anyString())).thenReturn(mock(ModuleProviderHolder.class));
+ when(manager.find(CoreModule.NAME).provider()).thenReturn(mock(ModuleServiceHolder.class));
+ when(manager.find(CoreModule.NAME).provider().getService(SourceReceiver.class))
+ .thenReturn(mock(SourceReceiver.class));
+ when(manager.find(CoreModule.NAME).provider().getService(
+ ConfigService.class)).thenReturn(mock(ConfigService.class));
+ when(manager.find(CoreModule.NAME).provider().getService(
+ ConfigService.class).getSearchableLogsTags()).thenReturn(anyString());
+ }
+
+ @Test
+ public void testDslStaticCompile() throws ModuleStartException {
+ final DSL dsl = DSL.of(manager, new LogAnalyzerModuleConfig(), script);
+ Whitebox.setInternalState(
+ Whitebox.getInternalState(dsl, "filterSpec"), "factories", Collections.emptyList()
+ );
+
+ dsl.bind(new Binding().log(LogData.newBuilder().build()));
+ dsl.evaluate();
+ }
+}
diff --git a/test/e2e/e2e-test/docker/log/lal.yaml b/oap-server/analyzer/log-analyzer/src/test/resources/log-mal-rules/placeholder.yaml
similarity index 56%
copy from test/e2e/e2e-test/docker/log/lal.yaml
copy to oap-server/analyzer/log-analyzer/src/test/resources/log-mal-rules/placeholder.yaml
index 2b94988..034440c 100644
--- a/test/e2e/e2e-test/docker/log/lal.yaml
+++ b/oap-server/analyzer/log-analyzer/src/test/resources/log-mal-rules/placeholder.yaml
@@ -13,22 +13,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-rules:
- - name: example
- dsl: |
- filter {
- text {
- abortOnFailure false // for test purpose, we want to persist all logs
- regexp $/(?s)(?<timestamp>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3}) \[TID:(?<tid>.+?)] \[(?<thread>.+?)] (?<level>\w{4,}) (?<logger>.{1,36}) (?<msg>.+)/$
- }
- extractor {
- metrics {
- timestamp log.timestamp
- labels level: parsed.level, service: log.service, instance: log.serviceInstance
- name "log_count"
- value 1
- }
- }
- sink {
- }
- }
+# Refer to examples in config-examples/log-mal.yaml
diff --git a/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/test/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/mock/MockModuleManager.java b/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/test/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/mock/MockModuleManager.java
index 8d5ee8a..63c95bb 100644
--- a/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/test/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/mock/MockModuleManager.java
+++ b/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/test/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/mock/MockModuleManager.java
@@ -37,10 +37,12 @@ public abstract class MockModuleManager extends ModuleManager {
moduleProviderHolderMap.put(name, provider);
}
+ @Override
public boolean has(String moduleName) {
return moduleProviderHolderMap.containsKey(moduleName);
}
+ @Override
public ModuleProviderHolder find(String moduleName) throws ModuleNotFoundRuntimeException {
if (!moduleProviderHolderMap.containsKey(moduleName)) {
throw new ModuleNotFoundRuntimeException("ModuleProviderHolder[" + moduleName + "] cannot found in MOCK.");
diff --git a/test/e2e/e2e-test/docker/log/lal.yaml b/test/e2e/e2e-test/docker/log/lal.yaml
index 2b94988..aef30ff 100644
--- a/test/e2e/e2e-test/docker/log/lal.yaml
+++ b/test/e2e/e2e-test/docker/log/lal.yaml
@@ -23,9 +23,9 @@ rules:
}
extractor {
metrics {
- timestamp log.timestamp
+ timestamp "${log.timestamp}"
labels level: parsed.level, service: log.service, instance: log.serviceInstance
- name "log_count"
+ name "log_count_info"
value 1
}
}