You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by ha...@apache.org on 2024/03/17 11:51:18 UTC
(skywalking) 01/01: Refactor data-generator to support generating metrics
This is an automated email from the ASF dual-hosted git repository.
hanahmily pushed a commit to branch data-genertor
in repository https://gitbox.apache.org/repos/asf/skywalking.git
commit 7d07876b3789ab0b28f7d0fb0ddb012ed734609d
Author: Gao Hongtao <ha...@gmail.com>
AuthorDate: Sun Mar 17 19:50:23 2024 +0800
Refactor data-generator to support generating metrics
Signed-off-by: Gao Hongtao <ha...@gmail.com>
---
docs/en/changes/changes.md | 1 +
docs/en/setup/backend/segment-template.json | 183 +++++++++------------
.../apache/skywalking/generator/BoolGenerator.java | 6 +-
.../skywalking/generator/FixedStringGenerator.java | 6 +-
.../org/apache/skywalking/generator/Generator.java | 4 +-
.../apache/skywalking/generator/IntGenerator.java | 6 +-
.../apache/skywalking/generator/ListGenerator.java | 6 +-
.../skywalking/generator/SequenceGenerator.java | 6 +-
.../skywalking/generator/StringGenerator.java | 15 +-
.../apache/skywalking/generator/UUIDGenerator.java | 6 +-
.../skywalking/restapi/LogGeneratorHandler.java | 2 +-
.../org/apache/skywalking/restapi/LogRequest.java | 54 +++---
.../skywalking/restapi/SegmentGenerator.java | 144 ++++++++++++++++
.../restapi/SegmentGeneratorHandler.java | 38 +++--
.../apache/skywalking/restapi/SegmentRequest.java | 88 ++--------
.../apache/skywalking/restapi/SpanGenerator.java | 79 +++++----
.../apache/skywalking/restapi/TagGenerator.java | 10 +-
.../src/main/resources/application.yml | 105 ++++++++++--
.../skywalking/generator/IntGeneratorTest.java | 2 +-
.../generator/SequenceGeneratorTest.java | 4 +-
.../SegmentGeneratorTest.java} | 34 ++--
.../src/test/resources/segment.tpl.json | 122 ++++++++++++++
22 files changed, 616 insertions(+), 305 deletions(-)
diff --git a/docs/en/changes/changes.md b/docs/en/changes/changes.md
index b376b3c9b3..297e0318b9 100644
--- a/docs/en/changes/changes.md
+++ b/docs/en/changes/changes.md
@@ -77,6 +77,7 @@
- PULSAR -> K8S_SERVICE
- VIRTUAL_MQ -> PULSAR
* Add Golang as a supported language for Kafka.
+* Refactor data-generator to support generating metrics.
#### UI
diff --git a/docs/en/setup/backend/segment-template.json b/docs/en/setup/backend/segment-template.json
index 88b1e5cfc6..0cfc9c1a39 100644
--- a/docs/en/setup/backend/segment-template.json
+++ b/docs/en/setup/backend/segment-template.json
@@ -1,123 +1,39 @@
{
"traceId": {
"type": "uuid",
- "changingFrequency": "5"
+ "changingFrequency": "1"
},
- "serviceInstanceName": {
- "type": "randomString",
- "length": "10",
- "letters": true,
- "numbers": true
- },
- "segmentId": {
- "type": "randomString",
- "length": "20",
- "letters": true,
- "numbers": true
- },
- "serviceName": {
- "type": "randomString",
- "length": "10",
- "letters": true,
- "numbers": true
- },
- "endpointName": {
- "type": "randomString",
- "length": "10",
- "prefix": "test_",
- "letters": true,
- "numbers": true,
- "domainSize": 10
- },
- "startTime": {
- "type": "sequence",
- "min": "1650510807000",
- "max": "1650512807000"
- },
- "latency": {
- "type": "randomInt",
- "min": 100,
- "max": 1000
- },
- "error": {
- "type": "randomInt",
- "min": 1,
- "max": 1
- },
- "tags": {
+ "segments": {
"type": "randomList",
"size": 5,
"item": {
- "key": {
+ "serviceInstanceName": {
"type": "randomString",
"length": "10",
- "prefix": "test_tag_",
"letters": true,
"numbers": true,
"domainSize": 10
},
- "value": {
+ "serviceName": {
"type": "randomString",
- "length": "10",
- "prefix": "test_value_",
- "letters": true,
- "numbers": true
- }
- }
- },
- "spans": {
- "type": "randomList",
- "size": 5,
- "item": {
- "spanId": {
- "type": "sequence",
- "min": "0",
- "max": "5"
- },
- "parentSpanId": {
- "type": "sequence",
- "min": "-1",
- "max": "4"
- },
- "startTime": {
- "type": "randomInt",
- "min": "1650510807000",
- "max": "1650512807000",
- "domainSize": 10
- },
- "endTime": {
- "type": "randomInt",
- "min": "1650510807000",
- "max": "1650512807000",
+ "prefix": "service_",
+ "length": "1",
+ "letters": false,
+ "numbers": true,
"domainSize": 10
},
- "operationName": {
+ "endpointName": {
"type": "randomString",
"length": "10",
- "prefix": "test_endpoint_",
+ "prefix": "test_",
"letters": true,
- "numbers": true
- },
- "peer": {
- "type": "randomString",
- "length": "10",
- "prefix": "test_peer_",
- "letters": true,
- "numbers": true
- },
- "spanLayer": {
- "type": "randomInt",
- "min": "0",
- "max": "4"
- },
- "componentId": {
- "type": "randomInt",
- "min": "0",
- "max": "4"
+ "numbers": true,
+ "domainSize": 10
},
"error": {
- "type": "randomBool",
- "possibility": "0.9"
+ "type": "randomInt",
+ "min": 1,
+ "max": 1
},
"tags": {
"type": "randomList",
@@ -126,20 +42,81 @@
"key": {
"type": "randomString",
"length": "10",
- "prefix": "test_tag_key_",
+ "prefix": "test_tag_",
"letters": true,
"numbers": true,
- "domainSize": 10
+ "domainSize": 5
},
"value": {
"type": "randomString",
"length": "10",
- "prefix": "test_tag_val_",
+ "prefix": "test_value_",
+ "letters": true,
+ "numbers": true,
+ "domainSize": 10
+ }
+ }
+ },
+ "spans": {
+ "type": "randomList",
+ "size": 5,
+ "item": {
+ "latency": {
+ "type": "randomInt",
+ "min": 100,
+ "max": 1000
+ },
+ "operationName": {
+ "type": "randomString",
+ "length": "10",
+ "prefix": "test_endpoint_",
"letters": true,
"numbers": true
+ },
+ "peer": {
+ "type": "randomString",
+ "length": "10",
+ "prefix": "test_peer_",
+ "letters": true,
+ "numbers": true
+ },
+ "spanLayer": {
+ "type": "randomInt",
+ "min": "0",
+ "max": "4"
+ },
+ "componentId": {
+ "type": "randomInt",
+ "min": "0",
+ "max": "4"
+ },
+ "error": {
+ "type": "randomBool",
+ "possibility": "0.2"
+ },
+ "tags": {
+ "type": "randomList",
+ "size": 5,
+ "item": {
+ "key": {
+ "type": "randomString",
+ "length": "10",
+ "prefix": "test_tag_key_",
+ "letters": true,
+ "numbers": true,
+ "domainSize": 10
+ },
+ "value": {
+ "type": "randomString",
+ "length": "10",
+ "prefix": "test_tag_val_",
+ "letters": true,
+ "numbers": true
+ }
+ }
}
}
}
}
}
-}
+}
\ No newline at end of file
diff --git a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/BoolGenerator.java b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/BoolGenerator.java
index 7bda23aa7a..c6132de0b2 100644
--- a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/BoolGenerator.java
+++ b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/BoolGenerator.java
@@ -25,7 +25,7 @@ import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
import lombok.Data;
@JsonDeserialize(builder = BoolGenerator.Builder.class)
-public final class BoolGenerator implements Generator<Boolean> {
+public final class BoolGenerator implements Generator<Object, Boolean> {
private final Random random = ThreadLocalRandom.current();
private final double possibility;
@@ -34,13 +34,13 @@ public final class BoolGenerator implements Generator<Boolean> {
}
@Override
- public Boolean next() {
+ public Boolean next(Object ignored) {
return random.nextDouble() < possibility;
}
@Override
public String toString() {
- return String.valueOf(next());
+ return String.valueOf(next(null));
}
@Data
diff --git a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/FixedStringGenerator.java b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/FixedStringGenerator.java
index edf11a54ce..7d2d78ed84 100644
--- a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/FixedStringGenerator.java
+++ b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/FixedStringGenerator.java
@@ -23,7 +23,7 @@ import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
import lombok.Data;
@JsonDeserialize(builder = FixedStringGenerator.Builder.class)
-public final class FixedStringGenerator implements Generator<String> {
+public final class FixedStringGenerator implements Generator<Object, String> {
private final String value;
public FixedStringGenerator(Builder builder) {
@@ -31,7 +31,7 @@ public final class FixedStringGenerator implements Generator<String> {
}
@Override
- public String next() {
+ public String next(Object ignored) {
return value;
}
@@ -46,6 +46,6 @@ public final class FixedStringGenerator implements Generator<String> {
@Override
public String toString() {
- return String.valueOf(next());
+ return String.valueOf(next(null));
}
}
diff --git a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/Generator.java b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/Generator.java
index 511f18873c..1630efccd8 100644
--- a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/Generator.java
+++ b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/Generator.java
@@ -36,8 +36,8 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo;
@Type(value = SequenceGenerator.class, name = "sequence"),
@Type(value = UUIDGenerator.class, name = "uuid"),
})
-public interface Generator<T> {
- public T next();
+public interface Generator<I, T> {
+ public T next(I input);
default public void reset() {
}
diff --git a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/IntGenerator.java b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/IntGenerator.java
index bd2b5a6e5e..e101991b3c 100644
--- a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/IntGenerator.java
+++ b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/IntGenerator.java
@@ -27,7 +27,7 @@ import com.google.common.base.Preconditions;
import lombok.Data;
@JsonDeserialize(builder = IntGenerator.Builder.class)
-public final class IntGenerator implements Generator<Long> {
+public final class IntGenerator implements Generator<Object, Long> {
private final boolean limitedDomain;
private final Long min;
private final Long max;
@@ -45,7 +45,7 @@ public final class IntGenerator implements Generator<Long> {
}
@Override
- public Long next() {
+ public Long next(Object ignored) {
if (!limitedDomain) {
return next0();
}
@@ -79,7 +79,7 @@ public final class IntGenerator implements Generator<Long> {
@Override
public String toString() {
- return String.valueOf(next());
+ return String.valueOf(next(null));
}
@Data
diff --git a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/ListGenerator.java b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/ListGenerator.java
index 07974cd3c7..26f67d5a17 100644
--- a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/ListGenerator.java
+++ b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/ListGenerator.java
@@ -26,7 +26,7 @@ import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
import lombok.Data;
@JsonDeserialize(builder = ListGenerator.Builder.class)
-public final class ListGenerator<T> implements Generator<List<T>> {
+public final class ListGenerator<T> implements Generator<Object, List<T>> {
private final T item;
private final int size;
@@ -36,7 +36,7 @@ public final class ListGenerator<T> implements Generator<List<T>> {
}
@Override
- public List<T> next() {
+ public List<T> next(Object ignored) {
return IntStream
.range(0, size)
.mapToObj($ -> item)
@@ -45,7 +45,7 @@ public final class ListGenerator<T> implements Generator<List<T>> {
@Override
public void reset() {
- ((Generator<?>) item).reset();
+ ((Generator<?, ?>) item).reset();
}
@Data
diff --git a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/SequenceGenerator.java b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/SequenceGenerator.java
index 3df3f9246a..c8ba04481e 100644
--- a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/SequenceGenerator.java
+++ b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/SequenceGenerator.java
@@ -29,7 +29,7 @@ import lombok.Data;
import lombok.experimental.Accessors;
@JsonDeserialize(builder = SequenceGenerator.Builder.class)
-public final class SequenceGenerator implements Generator<Long> {
+public final class SequenceGenerator implements Generator<Object, Long> {
private final boolean limitedDomain;
private final long min;
private final long max;
@@ -52,7 +52,7 @@ public final class SequenceGenerator implements Generator<Long> {
}
@Override
- public Long next() {
+ public Long next(Object ignored) {
if (!limitedDomain) {
return next0();
}
@@ -94,7 +94,7 @@ public final class SequenceGenerator implements Generator<Long> {
@Override
public String toString() {
- return String.valueOf(next());
+ return String.valueOf(next(null));
}
@Data
diff --git a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/StringGenerator.java b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/StringGenerator.java
index 06a6bfb770..12e43c3691 100644
--- a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/StringGenerator.java
+++ b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/StringGenerator.java
@@ -29,7 +29,7 @@ import org.apache.commons.lang3.RandomStringUtils;
import lombok.Data;
@JsonDeserialize(builder = StringGenerator.Builder.class)
-public final class StringGenerator implements Generator<String> {
+public final class StringGenerator implements Generator<String, String> {
private final int length;
private final String prefix;
private final boolean letters;
@@ -58,12 +58,17 @@ public final class StringGenerator implements Generator<String> {
}
@Override
- public String next() {
+ public String next(String parent) {
+ if (Strings.isNullOrEmpty(parent)) {
+ parent = "";
+ } else {
+ parent += "-";
+ }
if (!limitedDomain) {
- return Strings.nullToEmpty(prefix)
+ return parent + Strings.nullToEmpty(prefix)
+ RandomStringUtils.random(length, letters, numbers);
}
- return domain
+ return parent + domain
.stream()
.skip(random.nextInt(domain.size()))
.findFirst()
@@ -72,7 +77,7 @@ public final class StringGenerator implements Generator<String> {
@Override
public String toString() {
- return String.valueOf(next());
+ return String.valueOf(next(null));
}
@Data
diff --git a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/UUIDGenerator.java b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/UUIDGenerator.java
index 22a75efe8b..80116c7968 100644
--- a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/UUIDGenerator.java
+++ b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/generator/UUIDGenerator.java
@@ -27,7 +27,7 @@ import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
import lombok.Data;
@JsonDeserialize(builder = UUIDGenerator.Builder.class)
-public final class UUIDGenerator implements Generator<String> {
+public final class UUIDGenerator implements Generator<Object, String> {
private final int changingFrequency;
private final AtomicInteger counter;
private final AtomicReference<String> last =
@@ -41,7 +41,7 @@ public final class UUIDGenerator implements Generator<String> {
}
@Override
- public String next() {
+ public String next(Object ignored) {
if (counter.incrementAndGet() < changingFrequency) {
return last.get();
}
@@ -51,7 +51,7 @@ public final class UUIDGenerator implements Generator<String> {
@Override
public String toString() {
- return String.valueOf(next());
+ return String.valueOf(next(null));
}
@Override
diff --git a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/LogGeneratorHandler.java b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/LogGeneratorHandler.java
index 303bed64f2..bc096e9862 100644
--- a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/LogGeneratorHandler.java
+++ b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/LogGeneratorHandler.java
@@ -70,7 +70,7 @@ public class LogGeneratorHandler {
log.info("Generate {} mock logs, qps: {}, template: {}", size, qps, request);
final IntConsumer generator = unused -> {
- final Log l = request.next();
+ final Log l = request.next(null);
log.debug("Generating log: {}", l);
sourceReceiver.receive(l);
};
diff --git a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/LogRequest.java b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/LogRequest.java
index bf41e2bf49..c8fbf0c29a 100644
--- a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/LogRequest.java
+++ b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/LogRequest.java
@@ -38,51 +38,55 @@ import lombok.SneakyThrows;
@Data
@JsonTypeInfo(use = JsonTypeInfo.Id.NONE)
-public final class LogRequest implements Generator<Log> {
+public final class LogRequest implements Generator<Object, Log> {
@JsonIgnore
private final ObjectMapper om = new ObjectMapper();
- private Generator<Long> timestamp;
- private Generator<String> serviceName;
- private Generator<String> serviceInstanceName;
- private Generator<String> endpointName;
- private Generator<String> traceId;
- private Generator<String> traceSegmentId;
- private Generator<Long> spanId;
- private Generator<Long> contentType;
- private Generator<String> content;
- private Generator<List<TagGenerator>> tags;
- private Generator<Boolean> error;
+ private Generator<Object, Long> timestamp;
+ private Generator<Object, String> serviceName;
+ private Generator<Object, String> serviceInstanceName;
+ private Generator<Object, String> endpointName;
+ private Generator<Object, String> traceId;
+ private Generator<Object, String> traceSegmentId;
+ private Generator<Object, Long> spanId;
+ private Generator<Object, Long> contentType;
+ private Generator<Object, String> content;
+ private Generator<Object, List<TagGenerator>> tags;
+ private Generator<Object, Boolean> error;
@SneakyThrows
@Override
- public Log next() {
+ public Log next(Object ignored) {
final Log log = new Log();
- log.setTimestamp(getTimestamp().next());
+ if (timestamp == null) {
+ log.setTimestamp(System.currentTimeMillis());
+ } else {
+ log.setTimestamp(getTimestamp().next(null));
+ }
log.setServiceId(
IDManager.ServiceID.buildId(
- getServiceName().next(),
+ getServiceName().next(null),
true));
log.setServiceInstanceId(
IDManager.ServiceInstanceID.buildId(
log.getServiceId(),
- getServiceInstanceName().next()));
+ getServiceInstanceName().next(null)));
log.setEndpointId(
IDManager.EndpointID.buildId(
log.getServiceId(),
- getEndpointName().next()));
- log.setTraceId(getTraceId().next());
- log.setTraceSegmentId(getTraceSegmentId().next());
- log.setSpanId(getSpanId().next().intValue());
- log.setContentType(ContentType.instanceOf(getContentType().next().intValue()));
- log.setContent(getContent().next());
- log.setError(getError().next());
+ getEndpointName().next(null)));
+ log.setTraceId(getTraceId().next(null));
+ log.setTraceSegmentId(getTraceSegmentId().next(null));
+ log.setSpanId(getSpanId().next(null).intValue());
+ log.setContentType(ContentType.instanceOf(getContentType().next(null).intValue()));
+ log.setContent(getContent().next(null));
+ log.setError(getError().next(null));
log.setTimeBucket(TimeBucket.getRecordTimeBucket(log.getTimestamp()));
log.setTags(
getTags()
- .next()
+ .next(null)
.stream()
- .map(TagGenerator::next)
+ .map(tg -> tg.next(null))
.collect(Collectors.<Tag>toList()));
log.setTagsRawData(
LogTags
diff --git a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/SegmentGenerator.java b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/SegmentGenerator.java
new file mode 100644
index 0000000000..899a3c164e
--- /dev/null
+++ b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/SegmentGenerator.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.skywalking.restapi;
+
+import com.fasterxml.jackson.annotation.JsonTypeInfo;
+import com.google.common.base.Strings;
+import lombok.Getter;
+import lombok.RequiredArgsConstructor;
+import lombok.Setter;
+import org.apache.skywalking.apm.network.language.agent.v3.SegmentObject;
+import org.apache.skywalking.apm.network.language.agent.v3.SegmentReference;
+import org.apache.skywalking.apm.network.language.agent.v3.SpanObject;
+import org.apache.skywalking.generator.Generator;
+import org.apache.skywalking.generator.StringGenerator;
+import org.apache.skywalking.oap.server.core.analysis.IDManager;
+import org.apache.skywalking.oap.server.core.analysis.TimeBucket;
+import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.Tag;
+import org.apache.skywalking.oap.server.core.source.Segment;
+
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+@Getter
+@Setter
+@JsonTypeInfo(use = JsonTypeInfo.Id.NONE)
+public class SegmentGenerator implements Generator<SegmentGenerator.SegmentContext, SegmentGenerator.SegmentResult> {
+
+ private Generator<String, String> segmentId;
+ private Generator<String, String> serviceName;
+ private Generator<String, String> serviceInstanceName;
+ private Generator<String, String> endpointName;
+ private Generator<Object, Long> error;
+ private Generator<Object, List<TagGenerator>> tags;
+ private Generator<Object, List<SpanGenerator>> spans;
+
+ @Override
+ public SegmentResult next(SegmentContext ctx) {
+ long now = System.currentTimeMillis();
+ final String serviceName = getServiceName().next(null);
+ final String serviceInstanceName = getServiceInstanceName().next(serviceName);
+ final String endpointName = getEndpointName().next(null);
+ if (segmentId == null) {
+ StringGenerator.Builder segmentIdBuilder = new StringGenerator.Builder();
+ segmentIdBuilder.setLength(20);
+ segmentIdBuilder.setNumbers(true);
+ segmentIdBuilder.setLetters(true);
+ segmentId = segmentIdBuilder.build();
+ }
+
+ final SegmentReference sr = Optional.ofNullable(ctx.parentSegment).flatMap(parentSegment -> parentSegment.segmentObject.getSpansList().stream()
+ .filter(span -> !Strings.isNullOrEmpty(span.getPeer()))
+ .findFirst().map(span -> SegmentReference
+ .newBuilder()
+ .setTraceId(ctx.traceId)
+ .setParentServiceInstance(parentSegment.segmentObject.getServiceInstance())
+ .setParentService(parentSegment.segmentObject.getService())
+ .setParentSpanId(span.getSpanId())
+ .setParentTraceSegmentId(parentSegment.segment.getSegmentId())
+ .setNetworkAddressUsedAtPeer(span.getPeer())
+ .build()))
+ .orElse(null);
+ final String segmentId = getSegmentId().next(null);
+
+ final List<SpanGenerator> spanGenerators = getSpans().next(null);
+ int size = spanGenerators.size();
+ final SegmentObject segmentObj = SegmentObject
+ .newBuilder()
+ .setTraceId(ctx.traceId)
+ .setTraceSegmentId(segmentId)
+ .addAllSpans(
+ IntStream.range(0, size)
+ .mapToObj(i -> {
+ SpanGenerator sg = spanGenerators.get(i);
+ return sg.next(new SpanGenerator.SpanGeneratorContext(i, size, sr));
+ })
+ .collect(Collectors.<SpanObject>toList()))
+ .setService(serviceName)
+ .setServiceInstance(serviceInstanceName)
+ .build();
+
+ // Reset the span generator to generate the span id from 0
+ getSpans().reset();
+
+ Long latency = segmentObj.getSpansList().stream().reduce(0L, (l, span) -> l + (span.getEndTime() - span.getStartTime()), Long::sum);
+
+ final Segment segment = new Segment();
+ segment.setSegmentId(segmentId);
+ segment.setTraceId(ctx.traceId);
+ segment.setServiceId(
+ IDManager.ServiceID.buildId(serviceName, true));
+ segment.setServiceInstanceId(
+ IDManager.ServiceInstanceID.buildId(
+ segment.getServiceId(),
+ serviceInstanceName));
+ segment.setEndpointId(
+ IDManager.EndpointID.buildId(
+ segment.getServiceId(),
+ endpointName));
+ segment.setStartTime(now - latency);
+ segment.setLatency(latency.intValue());
+ segment.setIsError(getError().next(null).intValue());
+ segment.setTimeBucket(TimeBucket.getRecordTimeBucket(segment.getStartTime()));
+ segment.setTags(
+ getTags()
+ .next(null)
+ .stream()
+ .map(tg -> tg.next(null))
+ .collect(Collectors.<Tag>toList()));
+
+ segment.setDataBinary(segmentObj.toByteArray());
+ return new SegmentResult(segment, segmentObj);
+ }
+
+ @RequiredArgsConstructor
+ public static class SegmentContext {
+ final String traceId;
+ final SegmentResult parentSegment;
+ }
+
+ @RequiredArgsConstructor
+ public static class SegmentResult {
+ final Segment segment;
+ final SegmentObject segmentObject;
+ }
+}
diff --git a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/SegmentGeneratorHandler.java b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/SegmentGeneratorHandler.java
index fb5a0db01e..6e7a520dbf 100644
--- a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/SegmentGeneratorHandler.java
+++ b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/SegmentGeneratorHandler.java
@@ -19,14 +19,6 @@
package org.apache.skywalking.restapi;
-import java.util.Map;
-import java.util.Set;
-import java.util.UUID;
-import java.util.concurrent.CancellationException;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.TimeUnit;
-import java.util.function.IntConsumer;
-import java.util.stream.IntStream;
import com.linecorp.armeria.common.HttpResponse;
import com.linecorp.armeria.common.HttpStatus;
import com.linecorp.armeria.common.MediaType;
@@ -38,22 +30,35 @@ import com.linecorp.armeria.server.annotation.Param;
import com.linecorp.armeria.server.annotation.Post;
import com.linecorp.armeria.server.annotation.ProducesJson;
import com.linecorp.armeria.server.annotation.RequestObject;
-import org.apache.skywalking.oap.server.core.CoreModule;
-import org.apache.skywalking.oap.server.core.source.Segment;
-import org.apache.skywalking.oap.server.core.source.SourceReceiver;
-import org.apache.skywalking.oap.server.library.module.ModuleManager;
import io.netty.channel.EventLoopGroup;
import io.netty.util.concurrent.Future;
import lombok.extern.slf4j.Slf4j;
+import org.apache.skywalking.oap.server.analyzer.module.AnalyzerModule;
+import org.apache.skywalking.oap.server.analyzer.provider.trace.parser.ISegmentParserService;
+import org.apache.skywalking.oap.server.core.CoreModule;
+import org.apache.skywalking.oap.server.core.source.SourceReceiver;
+import org.apache.skywalking.oap.server.library.module.ModuleManager;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.CancellationException;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
+import java.util.function.IntConsumer;
+import java.util.stream.IntStream;
@Slf4j
public class SegmentGeneratorHandler {
private final SourceReceiver sourceReceiver;
private final Map<String, Future<?>> futures = new ConcurrentHashMap<>();
private final EventLoopGroup eventLoopGroup = EventLoopGroups.newEventLoopGroup(10);
+ private final ISegmentParserService segmentParserService;
public SegmentGeneratorHandler(ModuleManager manager) {
sourceReceiver = manager.find(CoreModule.NAME).provider().getService(SourceReceiver.class);
+ segmentParserService = manager.find(AnalyzerModule.NAME).provider().getService(ISegmentParserService.class);
}
@Post("/mock-data/segments/tasks")
@@ -71,9 +76,12 @@ public class SegmentGeneratorHandler {
log.info("Generate {} mock segments, qps: {}, template: {}", size, qps, request);
final IntConsumer generator = unused -> {
- final Segment segment = request.next();
- log.debug("Generating segment: {}", segment);
- sourceReceiver.receive(segment);
+ final List<SegmentGenerator.SegmentResult> segments = request.next(null);
+ log.debug("Generating segment: {}", (Object) segments);
+ segments.forEach(s -> {
+ sourceReceiver.receive(s.segment);
+ segmentParserService.send(s.segmentObject);
+ });
};
final String requestId = UUID.randomUUID().toString();
final Future<?> future;
diff --git a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/SegmentRequest.java b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/SegmentRequest.java
index 70d4d63167..b7b7371061 100644
--- a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/SegmentRequest.java
+++ b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/SegmentRequest.java
@@ -19,82 +19,30 @@
package org.apache.skywalking.restapi;
-import java.util.List;
-import java.util.stream.Collectors;
import com.fasterxml.jackson.annotation.JsonTypeInfo;
-import org.apache.skywalking.apm.network.language.agent.v3.SegmentObject;
-import org.apache.skywalking.apm.network.language.agent.v3.SpanObject;
-import org.apache.skywalking.generator.Generator;
-import org.apache.skywalking.oap.server.core.analysis.IDManager;
-import org.apache.skywalking.oap.server.core.analysis.TimeBucket;
-import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.Tag;
-import org.apache.skywalking.oap.server.core.source.Segment;
import lombok.Data;
+import org.apache.skywalking.generator.Generator;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
@Data
@JsonTypeInfo(use = JsonTypeInfo.Id.NONE)
-public final class SegmentRequest implements Generator<Segment> {
- private Generator<String> segmentId;
- private Generator<String> traceId;
- private Generator<String> serviceName;
- private Generator<String> serviceInstanceName;
- private Generator<String> endpointName;
- private Generator<Long> startTime;
- private Generator<Long> latency;
- private Generator<Long> error;
- private Generator<List<TagGenerator>> tags;
- private Generator<List<SpanGenerator>> spans;
+public final class SegmentRequest implements Generator<Object, List<SegmentGenerator.SegmentResult>> {
+ private Generator<String, String> traceId;
+ private Generator<Object, List<SegmentGenerator>> segments;
@Override
- public Segment next() {
- final String traceId = getTraceId().next();
- final String serviceName = getServiceName().next();
- final String serviceInstanceName = getServiceInstanceName().next();
- final String endpointName = getEndpointName().next();
- final String segmentId = getSegmentId().next();
-
- final SegmentObject segmentObj = SegmentObject
- .newBuilder()
- .setTraceId(traceId)
- .setTraceSegmentId(segmentId)
- .addAllSpans(
- getSpans()
- .next()
- .stream()
- .map(SpanGenerator::next)
- .collect(Collectors.<SpanObject>toList()))
- .setService(serviceName)
- .setServiceInstance(serviceInstanceName)
- .build();
-
- // Reset the span generator to generate the span id from 0
- getSpans().reset();
-
- final Segment segment = new Segment();
- segment.setSegmentId(segmentId);
- segment.setTraceId(traceId);
- segment.setServiceId(
- IDManager.ServiceID.buildId(serviceName, true));
- segment.setServiceInstanceId(
- IDManager.ServiceInstanceID.buildId(
- segment.getServiceId(),
- serviceInstanceName));
- segment.setEndpointId(
- IDManager.EndpointID.buildId(
- segment.getServiceId(),
- endpointName));
- segment.setStartTime(getStartTime().next());
- segment.setLatency(getLatency().next().intValue());
- segment.setIsError(getError().next().intValue());
- segment.setTimeBucket(TimeBucket.getRecordTimeBucket(segment.getStartTime()));
- segment.setTags(
- getTags()
- .next()
- .stream()
- .map(TagGenerator::next)
- .collect(Collectors.<Tag>toList()));
-
- segment.setDataBinary(segmentObj.toByteArray());
- return segment;
+ public List<SegmentGenerator.SegmentResult> next(Object ignored) {
+ final String traceId = getTraceId().next(null);
+ final List<SegmentGenerator> segments = getSegments().next(traceId);
+ SegmentGenerator.SegmentResult last = null;
+ List<SegmentGenerator.SegmentResult> result = new ArrayList<>(segments.size());
+ for (SegmentGenerator each : segments) {
+ last = each.next(new SegmentGenerator.SegmentContext(traceId, last));
+ result.add(last);
+ }
+ return result;
}
}
diff --git a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/SpanGenerator.java b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/SpanGenerator.java
index b8af9aea77..708e58fe0b 100644
--- a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/SpanGenerator.java
+++ b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/SpanGenerator.java
@@ -22,9 +22,12 @@ package org.apache.skywalking.restapi;
import java.util.List;
import java.util.stream.Collectors;
import com.fasterxml.jackson.annotation.JsonTypeInfo;
+import lombok.RequiredArgsConstructor;
import org.apache.skywalking.apm.network.common.v3.KeyStringValuePair;
+import org.apache.skywalking.apm.network.language.agent.v3.SegmentReference;
import org.apache.skywalking.apm.network.language.agent.v3.SpanLayer;
import org.apache.skywalking.apm.network.language.agent.v3.SpanObject;
+import org.apache.skywalking.apm.network.language.agent.v3.SpanType;
import org.apache.skywalking.generator.Generator;
import lombok.Getter;
import lombok.Setter;
@@ -32,49 +35,58 @@ import lombok.Setter;
@Getter
@Setter
@JsonTypeInfo(use = JsonTypeInfo.Id.NONE)
-public final class SpanGenerator implements Generator<SpanObject> {
- private Generator<Long> spanId;
- private Generator<Long> parentSpanId;
- private Generator<Long> startTime;
- private Generator<Long> endTime;
- private Generator<String> operationName;
- private Generator<String> peer;
- private Generator<Long> spanLayer;
- private Generator<Long> componentId;
- private Generator<Boolean> error;
- private Generator<List<TagGenerator>> tags;
+public final class SpanGenerator implements Generator<SpanGenerator.SpanGeneratorContext, SpanObject> {
+ private Generator<Object, Long> endTime;
+ private Generator<Object, Long> latency;
+ private Generator<String, String> operationName;
+ private Generator<String, String> peer;
+ private Generator<Object, Long> spanLayer;
+ private Generator<Object, Long> componentId;
+ private Generator<Object, Boolean> error;
+ private Generator<Object, List<TagGenerator>> tags;
@Override
- public SpanObject next() {
- return SpanObject
+ public SpanObject next(SpanGenerator.SpanGeneratorContext ctx) {
+ long now;
+ if (endTime == null) {
+ now = System.currentTimeMillis();
+ } else {
+ now = endTime.next(null);
+ }
+ SpanObject.Builder sob = SpanObject
.newBuilder()
- .setSpanId(getSpanId().next().intValue())
- .setParentSpanId(getParentSpanId().next().intValue())
- .setStartTime(getStartTime().next())
- .setEndTime(getEndTime().next())
- .setOperationName(getOperationName().next())
- .setPeer(getPeer().next())
- .setSpanLayer(SpanLayer.forNumber(getSpanLayer().next().intValue()))
- .setComponentId(getComponentId().next().intValue())
- .setIsError(getError().next())
+ .setSpanId(ctx.index)
+ .setParentSpanId(ctx.index - 1)
+ .setStartTime(now - latency.next(null))
+ .setEndTime(now)
+ .setComponentId(getComponentId().next(null).intValue())
+ .setIsError(getError().next(null))
.addAllTags(
getTags()
- .next()
+ .next(null)
.stream()
- .map(TagGenerator::next)
+ .map(tg -> tg.next(null))
.map(it -> KeyStringValuePair
.newBuilder().setKey(it.getKey())
.setValue(it.getValue()).build())
- .collect(Collectors.toList()))
- .build();
+ .collect(Collectors.toList()));
+ if (ctx.index == 0) {
+ sob.setSpanLayer(SpanLayer.forNumber(getSpanLayer().next(null).intValue()))
+ .setSpanType(SpanType.Entry);
+ if (ctx.ref != null) {
+ sob.addRefs(ctx.ref);
+ }
+ } else if (ctx.length > 1 && ctx.index == ctx.length - 1) {
+ sob.setSpanType(SpanType.Exit).setPeer(getPeer().next(null)).setSpanLayer(SpanLayer.Database);
+ } else {
+ sob.setSpanLayer(SpanLayer.forNumber(getSpanLayer().next(null).intValue()))
+ .setSpanType(SpanType.Local);
+ }
+ return sob.build();
}
@Override
public void reset() {
- spanId.reset();
- parentSpanId.reset();
- startTime.reset();
- endTime.reset();
operationName.reset();
peer.reset();
spanLayer.reset();
@@ -82,4 +94,11 @@ public final class SpanGenerator implements Generator<SpanObject> {
error.reset();
tags.reset();
}
+
+ @RequiredArgsConstructor
+ public static class SpanGeneratorContext {
+ final int index;
+ final int length;
+ final SegmentReference ref;
+ }
}
diff --git a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/TagGenerator.java b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/TagGenerator.java
index e55d79ba2d..74a6b5d5f7 100644
--- a/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/TagGenerator.java
+++ b/oap-server/server-tools/data-generator/src/main/java/org/apache/skywalking/restapi/TagGenerator.java
@@ -28,13 +28,13 @@ import lombok.Setter;
@Getter
@Setter
@JsonTypeInfo(use = JsonTypeInfo.Id.NONE)
-public final class TagGenerator implements Generator<Tag> {
- private Generator<String> key;
- private Generator<String> value;
+public final class TagGenerator implements Generator<Object, Tag> {
+ private Generator<String, String> key;
+ private Generator<String, String> value;
@Override
- public Tag next() {
- return new Tag(key.next(), value.next());
+ public Tag next(Object ignored) {
+ return new Tag(key.next(null), value.next(null));
}
@Override
diff --git a/oap-server/server-tools/data-generator/src/main/resources/application.yml b/oap-server/server-tools/data-generator/src/main/resources/application.yml
index 083a10d141..62b0ddc85a 100755
--- a/oap-server/server-tools/data-generator/src/main/resources/application.yml
+++ b/oap-server/server-tools/data-generator/src/main/resources/application.yml
@@ -53,9 +53,6 @@ core:
storageSessionTimeout: ${SW_CORE_STORAGE_SESSION_TIMEOUT:70000}
# The period of doing data persistence. Unit is second.Default value is 25s
persistentPeriod: ${SW_CORE_PERSISTENT_PERIOD:25}
- # Cache metrics data for 1 minute to reduce database queries, and if the OAP cluster changes within that minute,
- # the metrics may not be accurate within that minute.
- enableDatabaseSession: ${SW_CORE_ENABLE_DATABASE_SESSION:true}
topNReportPeriod: ${SW_CORE_TOPN_REPORT_PERIOD:10} # top_n record worker report cycle, unit is minute
# Extra model column are the column defined by in the codes, These columns of model are not required logically in aggregation or further query,
# and it will cause more load for memory, network of OAP and storage.
@@ -63,19 +60,34 @@ core:
activeExtraModelColumns: ${SW_CORE_ACTIVE_EXTRA_MODEL_COLUMNS:false}
# The max length of service + instance names should be less than 200
serviceNameMaxLength: ${SW_SERVICE_NAME_MAX_LENGTH:70}
+ # The period(in seconds) of refreshing the service cache. Default value is 10s.
+ serviceCacheRefreshInterval: ${SW_SERVICE_CACHE_REFRESH_INTERVAL:10}
instanceNameMaxLength: ${SW_INSTANCE_NAME_MAX_LENGTH:70}
# The max length of service + endpoint names should be less than 240
endpointNameMaxLength: ${SW_ENDPOINT_NAME_MAX_LENGTH:150}
# Define the set of span tag keys, which should be searchable through the GraphQL.
- searchableTracesTags: ${SW_SEARCHABLE_TAG_KEYS:http.method,status_code,db.type,db.instance,mq.queue,mq.topic,mq.broker}
+ # The max length of key=value should be less than 256 or will be dropped.
+ searchableTracesTags: ${SW_SEARCHABLE_TAG_KEYS:http.method,http.status_code,rpc.status_code,db.type,db.instance,mq.queue,mq.topic,mq.broker}
# Define the set of log tag keys, which should be searchable through the GraphQL.
- searchableLogsTags: ${SW_SEARCHABLE_LOGS_TAG_KEYS:level}
+ # The max length of key=value should be less than 256 or will be dropped.
+ searchableLogsTags: ${SW_SEARCHABLE_LOGS_TAG_KEYS:level,http.status_code}
# Define the set of alarm tag keys, which should be searchable through the GraphQL.
+ # The max length of key=value should be less than 256 or will be dropped.
searchableAlarmTags: ${SW_SEARCHABLE_ALARM_TAG_KEYS:level}
+ # The max size of tags keys for autocomplete select.
+ autocompleteTagKeysQueryMaxSize: ${SW_AUTOCOMPLETE_TAG_KEYS_QUERY_MAX_SIZE:100}
+ # The max size of tags values for autocomplete select.
+ autocompleteTagValuesQueryMaxSize: ${SW_AUTOCOMPLETE_TAG_VALUES_QUERY_MAX_SIZE:100}
# The number of threads used to prepare metrics data to the storage.
prepareThreads: ${SW_CORE_PREPARE_THREADS:2}
# Turn it on then automatically grouping endpoint by the given OpenAPI definitions.
enableEndpointNameGroupingByOpenapi: ${SW_CORE_ENABLE_ENDPOINT_NAME_GROUPING_BY_OPENAPI:true}
+ # The period of HTTP URI pattern recognition. Unit is second.
+ syncPeriodHttpUriRecognitionPattern: ${SW_CORE_SYNC_PERIOD_HTTP_URI_RECOGNITION_PATTERN:10}
+ # The training period of HTTP URI pattern recognition. Unit is second.
+ trainingPeriodHttpUriRecognitionPattern: ${SW_CORE_TRAINING_PERIOD_HTTP_URI_RECOGNITION_PATTERN:60}
+ # The max number of HTTP URIs per service for further URI pattern recognition.
+ maxHttpUrisNumberPerService: ${SW_CORE_MAX_HTTP_URIS_NUMBER_PER_SVR:3000}
storage:
selector: ${SW_STORAGE:h2}
elasticsearch:
@@ -94,32 +106,77 @@ storage:
dayStep: ${SW_STORAGE_DAY_STEP:1} # Represent the number of days in the one minute/hour/day index.
indexShardsNumber: ${SW_STORAGE_ES_INDEX_SHARDS_NUMBER:1} # Shard number of new indexes
indexReplicasNumber: ${SW_STORAGE_ES_INDEX_REPLICAS_NUMBER:1} # Replicas number of new indexes
+ # Specify the settings for each index individually.
+ # If configured, this setting has the highest priority and overrides the generic settings.
+ specificIndexSettings: ${SW_STORAGE_ES_SPECIFIC_INDEX_SETTINGS:""}
# Super data set has been defined in the codes, such as trace segments.The following 3 config would be improve es performance when storage super size data in es.
superDatasetDayStep: ${SW_STORAGE_ES_SUPER_DATASET_DAY_STEP:-1} # Represent the number of days in the super size dataset record index, the default value is the same as dayStep when the value is less than 0
- superDatasetIndexShardsFactor: ${SW_STORAGE_ES_SUPER_DATASET_INDEX_SHARDS_FACTOR:5} # This factor provides more shards for the super data set, shards number = indexShardsNumber * superDatasetIndexShardsFactor. Also, this factor effects Zipkin and Jaeger traces.
+ superDatasetIndexShardsFactor: ${SW_STORAGE_ES_SUPER_DATASET_INDEX_SHARDS_FACTOR:5} # This factor provides more shards for the super data set, shards number = indexShardsNumber * superDatasetIndexShardsFactor. Also, this factor effects Zipkin traces.
superDatasetIndexReplicasNumber: ${SW_STORAGE_ES_SUPER_DATASET_INDEX_REPLICAS_NUMBER:0} # Represent the replicas number in the super size dataset record index, the default value is 0.
indexTemplateOrder: ${SW_STORAGE_ES_INDEX_TEMPLATE_ORDER:0} # the order of index template
bulkActions: ${SW_STORAGE_ES_BULK_ACTIONS:5000} # Execute the async bulk record data every ${SW_STORAGE_ES_BULK_ACTIONS} requests
- # flush the bulk every 10 seconds whatever the number of requests
- # INT(flushInterval * 2/3) would be used for index refresh period.
- flushInterval: ${SW_STORAGE_ES_FLUSH_INTERVAL:15}
+ batchOfBytes: ${SW_STORAGE_ES_BATCH_OF_BYTES:10485760} # A threshold to control the max body size of ElasticSearch Bulk flush.
+ # flush the bulk every 5 seconds whatever the number of requests
+ flushInterval: ${SW_STORAGE_ES_FLUSH_INTERVAL:5}
concurrentRequests: ${SW_STORAGE_ES_CONCURRENT_REQUESTS:2} # the number of concurrent requests
resultWindowMaxSize: ${SW_STORAGE_ES_QUERY_MAX_WINDOW_SIZE:10000}
- metadataQueryMaxSize: ${SW_STORAGE_ES_QUERY_MAX_SIZE:5000}
+ metadataQueryMaxSize: ${SW_STORAGE_ES_QUERY_MAX_SIZE:10000}
+ scrollingBatchSize: ${SW_STORAGE_ES_SCROLLING_BATCH_SIZE:5000}
segmentQueryMaxSize: ${SW_STORAGE_ES_QUERY_SEGMENT_SIZE:200}
profileTaskQueryMaxSize: ${SW_STORAGE_ES_QUERY_PROFILE_TASK_SIZE:200}
+ profileDataQueryBatchSize: ${SW_STORAGE_ES_QUERY_PROFILE_DATA_BATCH_SIZE:100}
oapAnalyzer: ${SW_STORAGE_ES_OAP_ANALYZER:"{\"analyzer\":{\"oap_analyzer\":{\"type\":\"stop\"}}}"} # the oap analyzer.
oapLogAnalyzer: ${SW_STORAGE_ES_OAP_LOG_ANALYZER:"{\"analyzer\":{\"oap_log_analyzer\":{\"type\":\"standard\"}}}"} # the oap log analyzer. It could be customized by the ES analyzer configuration to support more language log formats, such as Chinese log, Japanese log and etc.
advanced: ${SW_STORAGE_ES_ADVANCED:""}
+ # Enable shard metrics and records indices into multi-physical indices, one index template per metric/meter aggregation function or record.
+ logicSharding: ${SW_STORAGE_ES_LOGIC_SHARDING:false}
+ # Custom routing can reduce the impact of searches. Instead of having to fan out a search request to all the shards in an index, the request can be sent to just the shard that matches the specific routing value (or values).
+ enableCustomRouting: ${SW_STORAGE_ES_ENABLE_CUSTOM_ROUTING:false}
h2:
properties:
jdbcUrl: ${SW_STORAGE_H2_URL:jdbc:h2:mem:skywalking-oap-db;DB_CLOSE_DELAY=-1;DATABASE_TO_UPPER=FALSE}
dataSource.user: ${SW_STORAGE_H2_USER:sa}
metadataQueryMaxSize: ${SW_STORAGE_H2_QUERY_MAX_SIZE:5000}
- maxSizeOfArrayColumn: ${SW_STORAGE_MAX_SIZE_OF_ARRAY_COLUMN:20}
- numOfSearchableValuesPerTag: ${SW_STORAGE_NUM_OF_SEARCHABLE_VALUES_PER_TAG:2}
maxSizeOfBatchSql: ${SW_STORAGE_MAX_SIZE_OF_BATCH_SQL:100}
asyncBatchPersistentPoolSize: ${SW_STORAGE_ASYNC_BATCH_PERSISTENT_POOL_SIZE:1}
+ mysql:
+ properties:
+ jdbcUrl: ${SW_JDBC_URL:"jdbc:mysql://localhost:3306/swtest?rewriteBatchedStatements=true&allowMultiQueries=true"}
+ dataSource.user: ${SW_DATA_SOURCE_USER:root}
+ dataSource.password: ${SW_DATA_SOURCE_PASSWORD:root@1234}
+ dataSource.cachePrepStmts: ${SW_DATA_SOURCE_CACHE_PREP_STMTS:true}
+ dataSource.prepStmtCacheSize: ${SW_DATA_SOURCE_PREP_STMT_CACHE_SQL_SIZE:250}
+ dataSource.prepStmtCacheSqlLimit: ${SW_DATA_SOURCE_PREP_STMT_CACHE_SQL_LIMIT:2048}
+ dataSource.useServerPrepStmts: ${SW_DATA_SOURCE_USE_SERVER_PREP_STMTS:true}
+ metadataQueryMaxSize: ${SW_STORAGE_MYSQL_QUERY_MAX_SIZE:5000}
+ maxSizeOfBatchSql: ${SW_STORAGE_MAX_SIZE_OF_BATCH_SQL:2000}
+ asyncBatchPersistentPoolSize: ${SW_STORAGE_ASYNC_BATCH_PERSISTENT_POOL_SIZE:4}
+ postgresql:
+ properties:
+ jdbcUrl: ${SW_JDBC_URL:"jdbc:postgresql://localhost:5432/skywalking"}
+ dataSource.user: ${SW_DATA_SOURCE_USER:postgres}
+ dataSource.password: ${SW_DATA_SOURCE_PASSWORD:123456}
+ dataSource.cachePrepStmts: ${SW_DATA_SOURCE_CACHE_PREP_STMTS:true}
+ dataSource.prepStmtCacheSize: ${SW_DATA_SOURCE_PREP_STMT_CACHE_SQL_SIZE:250}
+ dataSource.prepStmtCacheSqlLimit: ${SW_DATA_SOURCE_PREP_STMT_CACHE_SQL_LIMIT:2048}
+ dataSource.useServerPrepStmts: ${SW_DATA_SOURCE_USE_SERVER_PREP_STMTS:true}
+ metadataQueryMaxSize: ${SW_STORAGE_MYSQL_QUERY_MAX_SIZE:5000}
+ maxSizeOfBatchSql: ${SW_STORAGE_MAX_SIZE_OF_BATCH_SQL:2000}
+ asyncBatchPersistentPoolSize: ${SW_STORAGE_ASYNC_BATCH_PERSISTENT_POOL_SIZE:4}
+ banyandb:
+ targets: ${SW_STORAGE_BANYANDB_TARGETS:127.0.0.1:17912}
+ maxBulkSize: ${SW_STORAGE_BANYANDB_MAX_BULK_SIZE:5000}
+ flushInterval: ${SW_STORAGE_BANYANDB_FLUSH_INTERVAL:15}
+ metricsShardsNumber: ${SW_STORAGE_BANYANDB_METRICS_SHARDS_NUMBER:1}
+ recordShardsNumber: ${SW_STORAGE_BANYANDB_RECORD_SHARDS_NUMBER:1}
+ superDatasetShardsFactor: ${SW_STORAGE_BANYANDB_SUPERDATASET_SHARDS_FACTOR:2}
+ concurrentWriteThreads: ${SW_STORAGE_BANYANDB_CONCURRENT_WRITE_THREADS:15}
+ profileTaskQueryMaxSize: ${SW_STORAGE_BANYANDB_PROFILE_TASK_QUERY_MAX_SIZE:200} # the max number of fetch task in a request
+ blockIntervalHours: ${SW_STORAGE_BANYANDB_BLOCK_INTERVAL_HOURS:24} # Unit is hour
+ segmentIntervalDays: ${SW_STORAGE_BANYANDB_SEGMENT_INTERVAL_DAYS:1} # Unit is day
+ superDatasetBlockIntervalHours: ${SW_STORAGE_BANYANDB_SUPER_DATASET_BLOCK_INTERVAL_HOURS:4} # Unit is hour
+ superDatasetSegmentIntervalDays: ${SW_STORAGE_BANYANDB_SUPER_DATASET_SEGMENT_INTERVAL_DAYS:1} # Unit is day
+ specificGroupSettings: ${SW_STORAGE_BANYANDB_SPECIFIC_GROUP_SETTINGS:""} # For example, {"group1": {"blockIntervalHours": 4, "segmentIntervalDays": 1}}
agent-analyzer:
selector: ${SW_AGENT_ANALYZER:default}
@@ -132,13 +189,15 @@ agent-analyzer:
# Nginx and Envoy agents can't get the real remote address.
# Exit spans with the component in the list would not generate the client-side instance relation metrics.
noUpstreamRealAddressAgents: ${SW_NO_UPSTREAM_REAL_ADDRESS:6000,9000}
- meterAnalyzerActiveFiles: ${SW_METER_ANALYZER_ACTIVE_FILES:datasource,threadpool} # Which files could be meter analyzed, files split by ","
+ meterAnalyzerActiveFiles: ${SW_METER_ANALYZER_ACTIVE_FILES:datasource,threadpool,satellite,go-runtime,python-runtime,continuous-profiling} # Which files could be meter analyzed, files split by ","
+ slowCacheReadThreshold: ${SW_SLOW_CACHE_SLOW_READ_THRESHOLD:default:20,redis:10} # The slow cache read operation thresholds. Unit ms.
+ slowCacheWriteThreshold: ${SW_SLOW_CACHE_SLOW_WRITE_THRESHOLD:default:20,redis:10} # The slow cache write operation thresholds. Unit ms.
log-analyzer:
selector: ${SW_LOG_ANALYZER:default}
default:
- lalFiles: ${SW_LOG_LAL_FILES:default}
- malFiles: ${SW_LOG_MAL_FILES:""}
+ lalFiles: ${SW_LOG_LAL_FILES:envoy-als,mesh-dp,mysql-slowsql,pgsql-slowsql,redis-slowsql,k8s-service,nginx,default}
+ malFiles: ${SW_LOG_MAL_FILES:"nginx"}
event-analyzer:
selector: ${SW_EVENT_ANALYZER:default}
@@ -147,12 +206,20 @@ event-analyzer:
query:
selector: ${SW_QUERY:graphql}
graphql:
- path: ${SW_QUERY_GRAPHQL_PATH:/graphql}
# Enable the log testing API to test the LAL.
# NOTE: This API evaluates untrusted code on the OAP server.
# A malicious script can do significant damage (steal keys and secrets, remove files and directories, install malware, etc).
# As such, please enable this API only when you completely trust your users.
enableLogTestTool: ${SW_QUERY_GRAPHQL_ENABLE_LOG_TEST_TOOL:false}
+ # Maximum complexity allowed for the GraphQL query that can be used to
+ # abort a query if the total number of data fields queried exceeds the defined threshold.
+ maxQueryComplexity: ${SW_QUERY_MAX_QUERY_COMPLEXITY:3000}
+ # Allow user add, disable and update UI template
+ enableUpdateUITemplate: ${SW_ENABLE_UPDATE_UI_TEMPLATE:false}
+ # "On demand log" allows users to fetch Pod containers' log in real time,
+ # because this might expose secrets in the logs (if any), users need
+ # to enable this manually, and add permissions to OAP cluster role.
+ enableOnDemandPodLog: ${SW_ENABLE_ON_DEMAND_POD_LOG:false}
alarm:
selector: ${SW_ALARM:default}
@@ -171,6 +238,12 @@ health-checker:
default:
checkIntervalSeconds: ${SW_HEALTH_CHECKER_INTERVAL_SECONDS:5}
+ai-pipeline:
+ selector: ${SW_AI_PIPELINE:default}
+ default:
+ uriRecognitionServerAddr: ${SW_AI_PIPELINE_URI_RECOGNITION_SERVER_ADDR:}
+ uriRecognitionServerPort: ${SW_AI_PIPELINE_URI_RECOGNITION_SERVER_PORT:17128}
+
data-generator:
selector: ${SW_DATA_GENERATOR:default}
default:
diff --git a/oap-server/server-tools/data-generator/src/test/java/org/apache/skywalking/generator/IntGeneratorTest.java b/oap-server/server-tools/data-generator/src/test/java/org/apache/skywalking/generator/IntGeneratorTest.java
index d53b007414..8098ea8092 100644
--- a/oap-server/server-tools/data-generator/src/test/java/org/apache/skywalking/generator/IntGeneratorTest.java
+++ b/oap-server/server-tools/data-generator/src/test/java/org/apache/skywalking/generator/IntGeneratorTest.java
@@ -30,7 +30,7 @@ public class IntGeneratorTest {
builder.setMax(1L);
final IntGenerator generator = builder.build();
for (int i = 0; i < 100; i++) {
- assertEquals(1, generator.next().intValue());
+ assertEquals(1, generator.next(null).intValue());
}
}
}
diff --git a/oap-server/server-tools/data-generator/src/test/java/org/apache/skywalking/generator/SequenceGeneratorTest.java b/oap-server/server-tools/data-generator/src/test/java/org/apache/skywalking/generator/SequenceGeneratorTest.java
index b875aa0d21..3821ad7fd1 100644
--- a/oap-server/server-tools/data-generator/src/test/java/org/apache/skywalking/generator/SequenceGeneratorTest.java
+++ b/oap-server/server-tools/data-generator/src/test/java/org/apache/skywalking/generator/SequenceGeneratorTest.java
@@ -33,7 +33,7 @@ public final class SequenceGeneratorTest {
.build();
for (int i = 0; i < 10; i++) {
- assertEquals(i + 1, generator.next().intValue());
+ assertEquals(i + 1, generator.next(null).intValue());
}
}
@@ -47,7 +47,7 @@ public final class SequenceGeneratorTest {
.build();
for (int i = 1; i < 10; i++) {
- Long next = generator.next();
+ Long next = generator.next(null);
assertTrue(i <= next.intValue());
assertTrue(i * 2 >= next.intValue());
}
diff --git a/oap-server/server-tools/data-generator/src/test/java/org/apache/skywalking/generator/IntGeneratorTest.java b/oap-server/server-tools/data-generator/src/test/java/org/apache/skywalking/restapi/SegmentGeneratorTest.java
similarity index 51%
copy from oap-server/server-tools/data-generator/src/test/java/org/apache/skywalking/generator/IntGeneratorTest.java
copy to oap-server/server-tools/data-generator/src/test/java/org/apache/skywalking/restapi/SegmentGeneratorTest.java
index d53b007414..29c9228fdb 100644
--- a/oap-server/server-tools/data-generator/src/test/java/org/apache/skywalking/generator/IntGeneratorTest.java
+++ b/oap-server/server-tools/data-generator/src/test/java/org/apache/skywalking/restapi/SegmentGeneratorTest.java
@@ -17,20 +17,30 @@
* under the License.
*/
- package org.apache.skywalking.generator;
+package org.apache.skywalking.restapi;
-import static org.junit.jupiter.api.Assertions.assertEquals;
+import com.fasterxml.jackson.databind.ObjectMapper;
import org.junit.jupiter.api.Test;
-public class IntGeneratorTest {
+import java.io.File;
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.util.List;
+
+import static graphql.Assert.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+class SegmentGeneratorTest {
+
@Test
- public void testFixedInt() {
- final IntGenerator.Builder builder = new IntGenerator.Builder();
- builder.setMin(1L);
- builder.setMax(1L);
- final IntGenerator generator = builder.build();
- for (int i = 0; i < 100; i++) {
- assertEquals(1, generator.next().intValue());
- }
+ void next() throws URISyntaxException, IOException {
+ ObjectMapper objectMapper = new ObjectMapper();
+ URL url = getClass().getClassLoader().getResource("segment.tpl.json");
+ assertNotNull(url);
+ File jsonFile = new File(url.toURI());
+ SegmentRequest sr = objectMapper.readValue(jsonFile, SegmentRequest.class);
+ List<SegmentGenerator.SegmentResult> ss = sr.next(null);
+ assertFalse(ss.isEmpty());
}
-}
+}
\ No newline at end of file
diff --git a/oap-server/server-tools/data-generator/src/test/resources/segment.tpl.json b/oap-server/server-tools/data-generator/src/test/resources/segment.tpl.json
new file mode 100644
index 0000000000..fd9c31e6ec
--- /dev/null
+++ b/oap-server/server-tools/data-generator/src/test/resources/segment.tpl.json
@@ -0,0 +1,122 @@
+{
+ "traceId": {
+ "type": "uuid",
+ "changingFrequency": "1"
+ },
+ "segments": {
+ "type": "randomList",
+ "size": 5,
+ "item": {
+ "serviceInstanceName": {
+ "type": "randomString",
+ "length": "10",
+ "letters": true,
+ "numbers": true,
+ "domainSize": 10
+ },
+ "serviceName": {
+ "type": "randomString",
+ "prefix": "service_",
+ "length": "1",
+ "letters": false,
+ "numbers": true,
+ "domainSize": 10
+ },
+ "endpointName": {
+ "type": "randomString",
+ "length": "10",
+ "prefix": "test_",
+ "letters": true,
+ "numbers": true,
+ "domainSize": 10
+ },
+ "error": {
+ "type": "randomInt",
+ "min": 1,
+ "max": 1
+ },
+ "tags": {
+ "type": "randomList",
+ "size": 5,
+ "item": {
+ "key": {
+ "type": "randomString",
+ "length": "10",
+ "prefix": "test_tag_",
+ "letters": true,
+ "numbers": true,
+ "domainSize": 5
+ },
+ "value": {
+ "type": "randomString",
+ "length": "10",
+ "prefix": "test_value_",
+ "letters": true,
+ "numbers": true,
+ "domainSize": 10
+ }
+ }
+ },
+ "spans": {
+ "type": "randomList",
+ "size": 5,
+ "item": {
+ "latency": {
+ "type": "randomInt",
+ "min": 100,
+ "max": 1000
+ },
+ "operationName": {
+ "type": "randomString",
+ "length": "10",
+ "prefix": "test_endpoint_",
+ "letters": true,
+ "numbers": true
+ },
+ "peer": {
+ "type": "randomString",
+ "length": "10",
+ "prefix": "test_peer_",
+ "letters": true,
+ "numbers": true
+ },
+ "spanLayer": {
+ "type": "randomInt",
+ "min": "0",
+ "max": "4"
+ },
+ "componentId": {
+ "type": "randomInt",
+ "min": "0",
+ "max": "4"
+ },
+ "error": {
+ "type": "randomBool",
+ "possibility": "0.2"
+ },
+ "tags": {
+ "type": "randomList",
+ "size": 5,
+ "item": {
+ "key": {
+ "type": "randomString",
+ "length": "10",
+ "prefix": "test_tag_key_",
+ "letters": true,
+ "numbers": true,
+ "domainSize": 10
+ },
+ "value": {
+ "type": "randomString",
+ "length": "10",
+ "prefix": "test_tag_val_",
+ "letters": true,
+ "numbers": true
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+}
\ No newline at end of file