You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by ke...@apache.org on 2022/05/25 12:57:05 UTC
[skywalking] branch streaminglog updated (b602bb9a59 -> 580128f6ee)
This is an automated email from the ASF dual-hosted git repository.
kezhenxu94 pushed a change to branch streaminglog
in repository https://gitbox.apache.org/repos/asf/skywalking.git
discard b602bb9a59 Add APIs to query ondemand Pod log
new 580128f6ee Add APIs to query ondemand Pod log
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (b602bb9a59)
\
N -- N -- N refs/heads/streaminglog (580128f6ee)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../oap/query/graphql/type/InternalLog.java | 19 +++++++++++++++++++
.../skywalking/oap/query/graphql/type/LogAdapter.java | 19 +++++++++++++++++++
.../query/graphql/type/OndemandLogQueryCondition.java | 19 +++++++++++++++++++
3 files changed, 57 insertions(+)
[skywalking] 01/01: Add APIs to query ondemand Pod log
Posted by ke...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kezhenxu94 pushed a commit to branch streaminglog
in repository https://gitbox.apache.org/repos/asf/skywalking.git
commit 580128f6eecb1975f5f3117ff726d12a6a30424c
Author: kezhenxu94 <ke...@apache.org>
AuthorDate: Wed May 25 14:22:45 2022 +0800
Add APIs to query ondemand Pod log
---
docs/en/changes/changes.md | 3 +-
.../oap/server/core/query/input/Duration.java | 2 +
.../core/query/input/TraceQueryCondition.java | 2 +
.../oap/server/core/query/type/TraceSource.java} | 17 +-
.../query-graphql-plugin/pom.xml | 5 +
.../oap/query/graphql/GraphQLQueryConfig.java | 1 +
.../oap/query/graphql/GraphQLQueryProvider.java | 12 +-
.../query/graphql/resolver/OndemandLogQuery.java | 219 +++++++++++++++++++++
.../oap/query/graphql/type/InternalLog.java | 37 ++++
.../oap/query/graphql/type/LogAdapter.java | 68 +++++++
.../graphql/type/OndemandLogQueryCondition.java | 38 ++++
.../src/main/resources/query-protocol | 2 +-
.../src/main/resources/application.yml | 4 +
test.http | 38 ++++
14 files changed, 430 insertions(+), 18 deletions(-)
diff --git a/docs/en/changes/changes.md b/docs/en/changes/changes.md
index 39b7e71d54..2a43256d47 100644
--- a/docs/en/changes/changes.md
+++ b/docs/en/changes/changes.md
@@ -61,7 +61,7 @@
* Storage(ElasticSearch): add search options to tolerate inexisting indices.
* Fix the problem that `MQ` has the wrong `Layer` type.
* Fix NoneStream model has wrong downsampling(was Second, should be Minute).
-* SQL Database: provide `@SQLDatabase.AdditionalEntity` to support create additional tables from a model.
+* SQL Database: provide `@SQLDatabase.AdditionalEntity` to support create additional tables from a model.
* [Breaking Change] SQL Database: remove SQL Database config `maxSizeOfArrayColumn` and `numOfSearchableValuesPerTag`.
* [Breaking Change] SQL Database: move `Tags list` from `Segment`,`Logs`,`Alarms` to their additional table.
* [Breaking Change] Remove `total` field in Trace, Log, Event, Browser log, and alarm list query.
@@ -72,6 +72,7 @@
* Fix searchableTag as `rpc.status_code` and `http.status_code`. `status_code` had been removed.
* Fix scroll query failure exception.
* Add `profileDataQueryBatchSize` config in Elasticsearch Storage.
+* Add APIs to query ondemand Pod log
#### UI
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/Duration.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/Duration.java
index e08160f47c..600b4dbad3 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/Duration.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/Duration.java
@@ -20,11 +20,13 @@ package org.apache.skywalking.oap.server.core.query.input;
import java.util.List;
import lombok.Getter;
+import lombok.Setter;
import org.apache.skywalking.oap.server.core.query.DurationUtils;
import org.apache.skywalking.oap.server.core.query.PointOfTime;
import org.apache.skywalking.oap.server.core.query.enumeration.Step;
@Getter
+@Setter
public class Duration {
private String start;
private String end;
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/TraceQueryCondition.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/TraceQueryCondition.java
index c79b186432..9c5d0cc12d 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/TraceQueryCondition.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/TraceQueryCondition.java
@@ -24,6 +24,7 @@ import lombok.Setter;
import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.Tag;
import org.apache.skywalking.oap.server.core.query.type.Pagination;
import org.apache.skywalking.oap.server.core.query.type.QueryOrder;
+import org.apache.skywalking.oap.server.core.query.type.TraceSource;
import org.apache.skywalking.oap.server.core.query.type.TraceState;
@Getter
@@ -40,4 +41,5 @@ public class TraceQueryCondition {
private QueryOrder queryOrder;
private Pagination paging;
private List<Tag> tags;
+ private TraceSource prioritizedSource;
}
diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/GraphQLQueryConfig.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/TraceSource.java
similarity index 66%
copy from oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/GraphQLQueryConfig.java
copy to oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/TraceSource.java
index db1c4a30b6..cf9bcbeed6 100644
--- a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/GraphQLQueryConfig.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/TraceSource.java
@@ -16,19 +16,8 @@
*
*/
-package org.apache.skywalking.oap.query.graphql;
+package org.apache.skywalking.oap.server.core.query.type;
-import lombok.Getter;
-import lombok.Setter;
-import org.apache.skywalking.oap.server.library.module.ModuleConfig;
-
-/**
- * The config of {@code query.graphql}.
- */
-@Getter
-@Setter
-public class GraphQLQueryConfig extends ModuleConfig {
- private boolean enableLogTestTool;
- private int maxQueryComplexity = 1000;
- private boolean enableUpdateUITemplate = false;
+public enum TraceSource {
+ SKYWALKING, ZIPKIN
}
diff --git a/oap-server/server-query-plugin/query-graphql-plugin/pom.xml b/oap-server/server-query-plugin/query-graphql-plugin/pom.xml
index 045cc180f3..0a75c826d9 100644
--- a/oap-server/server-query-plugin/query-graphql-plugin/pom.xml
+++ b/oap-server/server-query-plugin/query-graphql-plugin/pom.xml
@@ -63,5 +63,10 @@
<groupId>com.fasterxml.jackson.module</groupId>
<artifactId>jackson-module-afterburner</artifactId>
</dependency>
+
+ <dependency>
+ <groupId>io.kubernetes</groupId>
+ <artifactId>client-java</artifactId>
+ </dependency>
</dependencies>
</project>
diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/GraphQLQueryConfig.java b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/GraphQLQueryConfig.java
index db1c4a30b6..70123b0ac2 100644
--- a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/GraphQLQueryConfig.java
+++ b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/GraphQLQueryConfig.java
@@ -31,4 +31,5 @@ public class GraphQLQueryConfig extends ModuleConfig {
private boolean enableLogTestTool;
private int maxQueryComplexity = 1000;
private boolean enableUpdateUITemplate = false;
+ private boolean enableOndemandLog = false;
}
diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/GraphQLQueryProvider.java b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/GraphQLQueryProvider.java
index c9c2adcafb..4b39eead35 100644
--- a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/GraphQLQueryProvider.java
+++ b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/GraphQLQueryProvider.java
@@ -40,6 +40,7 @@ import org.apache.skywalking.oap.query.graphql.resolver.Mutation;
import org.apache.skywalking.oap.query.graphql.resolver.ProfileMutation;
import org.apache.skywalking.oap.query.graphql.resolver.ProfileQuery;
import org.apache.skywalking.oap.query.graphql.resolver.Query;
+import org.apache.skywalking.oap.query.graphql.resolver.OndemandLogQuery;
import org.apache.skywalking.oap.query.graphql.resolver.TopNRecordsQuery;
import org.apache.skywalking.oap.query.graphql.resolver.TopologyQuery;
import org.apache.skywalking.oap.query.graphql.resolver.TraceQuery;
@@ -119,8 +120,15 @@ public class GraphQLQueryProvider extends ModuleProvider {
.file("query-protocol/metadata-v2.graphqls")
.resolvers(new MetadataQueryV2(getManager()))
.file("query-protocol/ebpf-profiling.graphqls")
- .resolvers(new EBPFProcessProfilingQuery(getManager()), new EBPFProcessProfilingMutation(getManager()))
- .scalars(ExtendedScalars.GraphQLLong);
+ .resolvers(new EBPFProcessProfilingQuery(getManager()), new EBPFProcessProfilingMutation(getManager()));
+
+ if (config.isEnableOndemandLog()) {
+ schemaBuilder
+ .file("query-protocol/ondemand-pod-log.graphqls")
+ .resolvers(new OndemandLogQuery());
+ }
+
+ schemaBuilder.scalars(ExtendedScalars.GraphQLLong);
}
@Override
diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/OndemandLogQuery.java b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/OndemandLogQuery.java
new file mode 100644
index 0000000000..458572ba16
--- /dev/null
+++ b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/OndemandLogQuery.java
@@ -0,0 +1,219 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oap.query.graphql.resolver;
+
+import static java.util.Comparator.comparing;
+import static java.util.Objects.isNull;
+import static java.util.Objects.nonNull;
+import java.io.IOException;
+import java.lang.reflect.Type;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.stream.Collectors;
+import com.google.common.base.Predicate;
+import com.google.common.base.Splitter;
+import com.google.common.base.Strings;
+import com.google.common.reflect.TypeToken;
+import com.google.gson.Gson;
+import org.apache.skywalking.oap.query.graphql.type.InternalLog;
+import org.apache.skywalking.oap.query.graphql.type.LogAdapter;
+import org.apache.skywalking.oap.query.graphql.type.OndemandLogQueryCondition;
+import org.apache.skywalking.oap.server.core.analysis.IDManager;
+import org.apache.skywalking.oap.server.core.query.input.Duration;
+import org.apache.skywalking.oap.server.core.query.type.Log;
+import org.apache.skywalking.oap.server.core.query.type.Logs;
+import org.apache.skywalking.oap.server.library.util.StringUtil;
+import graphql.kickstart.tools.GraphQLQueryResolver;
+import io.kubernetes.client.openapi.ApiException;
+import io.kubernetes.client.openapi.Configuration;
+import io.kubernetes.client.openapi.apis.CoreV1Api;
+import io.kubernetes.client.openapi.models.V1Container;
+import io.kubernetes.client.openapi.models.V1Namespace;
+import io.kubernetes.client.openapi.models.V1NamespaceList;
+import io.kubernetes.client.openapi.models.V1ObjectMeta;
+import io.kubernetes.client.openapi.models.V1Pod;
+import io.kubernetes.client.openapi.models.V1PodSpec;
+import io.kubernetes.client.util.Config;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class OndemandLogQuery implements GraphQLQueryResolver {
+ private final Gson gson = new Gson();
+ private final Type responseType = new TypeToken<Map<String, Object>>() {
+ }.getType();
+ private CoreV1Api kApi;
+
+ public List<String> listNamespaces() throws IOException {
+ try {
+ final V1NamespaceList nsList =
+ kApi().listNamespace(null, null, null, null, null, null, null, null, null, null);
+ return nsList
+ .getItems()
+ .stream()
+ .map(V1Namespace::getMetadata)
+ .filter(Objects::nonNull)
+ .map(V1ObjectMeta::getName)
+ .collect(Collectors.toList());
+ } catch (ApiException e) {
+ log.error("Failed to list namespaces from Kubernetes, {}", e.getResponseBody(), e);
+
+ Map<String, Object> responseBody = gson.fromJson(e.getResponseBody(), responseType);
+ String message = responseBody.getOrDefault("message", e.getCode()).toString();
+ throw new RuntimeException(message);
+ }
+ }
+
+ public List<String> listContainers(final OndemandLogQueryCondition condition)
+ throws IOException {
+ final String ns = condition.getNamespace();
+ final IDManager.ServiceInstanceID.InstanceIDDefinition instanceIDDefinition =
+ IDManager.ServiceInstanceID.analysisId(condition.getServiceInstanceId());
+ final String instanceName = instanceIDDefinition.getName();
+
+ try {
+ final V1Pod pod = kApi().readNamespacedPod(instanceName, ns, null);
+ final V1PodSpec spec = pod.getSpec();
+ if (isNull(spec)) {
+ throw new RuntimeException(String.format("No spec: %s:%s", ns, instanceName));
+ }
+
+ final List<String> containers = spec.getContainers().stream()
+ .map(V1Container::getName)
+ .collect(Collectors.toList());
+ if (nonNull(spec.getInitContainers())) {
+ final List<String> init = spec.getInitContainers().stream()
+ .map(V1Container::getName)
+ .collect(Collectors.toList());
+ containers.addAll(init);
+ }
+
+ return containers;
+ } catch (ApiException e) {
+ log.error("Failed to list containers from Kubernetes, {}", e.getResponseBody(), e);
+
+ Map<String, Object> responseBody = gson.fromJson(e.getResponseBody(), responseType);
+ String message = responseBody.getOrDefault("message", e.getCode()).toString();
+ throw new RuntimeException(message);
+ }
+ }
+
+ public Logs ondemandPodLogs(OndemandLogQueryCondition condition)
+ throws IOException {
+ final String ns = condition.getNamespace();
+ final IDManager.ServiceInstanceID.InstanceIDDefinition instanceIDDefinition =
+ IDManager.ServiceInstanceID.analysisId(condition.getServiceInstanceId());
+ final String instanceName = instanceIDDefinition.getName();
+
+ try {
+ final V1Pod pod = kApi().readNamespacedPod(instanceName, ns, null);
+ final V1ObjectMeta podMetadata = pod.getMetadata();
+ if (isNull(podMetadata)) {
+ throw new RuntimeException(
+ String.format("No such instance: %s:%s", ns, instanceName));
+ }
+ final V1PodSpec spec = pod.getSpec();
+ if (isNull(spec)) {
+ throw new RuntimeException(String.format("No spec: %s:%s", ns, instanceName));
+ }
+
+ final Duration duration = new Duration();
+ duration.setStart(condition.getDuration().getStart());
+ duration.setEnd(condition.getDuration().getEnd());
+ duration.setStep(condition.getDuration().getStep());
+ final long since = duration.getStartTimestamp() / 1000;
+ final String container = condition.getContainer();
+
+ final String podLog = kApi().readNamespacedPodLog(
+ podMetadata.getName(),
+ podMetadata.getNamespace(),
+ container,
+ false, null, null, null, null, (int) since, null, true);
+ final List<InternalLog> logs = Splitter.on("\n").omitEmptyStrings()
+ .splitToList(Strings.nullToEmpty(podLog))
+ .stream()
+ .filter(StringUtil::isNotBlank)
+ .map(it -> InternalLog.builder()
+ .line(it)
+ .container(container)
+ .build())
+ .collect(Collectors.toList());
+
+ final List<Log> filtered = filter(condition, logs);
+
+ final List<Log> limited =
+ filtered
+ .stream()
+ .limit(10000)
+ .collect(Collectors.toList());
+ final Logs result = new Logs();
+ result.getLogs().addAll(limited);
+ return result;
+
+ } catch (ApiException e) {
+ log.error("Failed to fetch logs from Kubernetes, {}", e.getResponseBody(), e);
+
+ Map<String, Object> responseBody = gson.fromJson(e.getResponseBody(), responseType);
+ String message = responseBody.getOrDefault("message", e.getCode()).toString();
+ throw new RuntimeException(message);
+ }
+ }
+
+ private List<Log> filter(
+ final OndemandLogQueryCondition request,
+ final List<InternalLog> logs) {
+ final Duration duration = new Duration();
+ duration.setStart(request.getDuration().getStart());
+ duration.setEnd(request.getDuration().getEnd());
+ duration.setStep(request.getDuration().getStep());
+ final long since = duration.getStartTimestamp() / 1000;
+ final long to = duration.getEndTimestamp() / 1000;
+
+ final List<String> inclusions = request.getKeywordsOfContent();
+ final Predicate<Log> inclusivePredicate = l -> inclusions.isEmpty() ||
+ inclusions.stream().anyMatch(k -> l.getContent().matches(k));
+
+ final List<String> exclusions = request.getExcludingKeywordsOfContent();
+ final Predicate<Log> exclusivePredicate = l -> exclusions.isEmpty() ||
+ exclusions.stream().noneMatch(k -> l.getContent().matches(k));
+
+ return logs.stream()
+ .map(LogAdapter::new).map(LogAdapter::adapt)
+ .filter(inclusivePredicate)
+ .filter(exclusivePredicate)
+ .filter(it -> it.getTimestamp() >= since)
+ .filter(it -> it.getTimestamp() <= to)
+ .sorted(comparing(Log::getTimestamp))
+ .collect(Collectors.toList());
+ }
+
+ private CoreV1Api kApi() throws IOException {
+ if (kApi == null) {
+ Configuration.setDefaultApiClient(
+ Config
+ .defaultClient()
+ .setReadTimeout(30000)
+ .setWriteTimeout(30000)
+ .setConnectTimeout(303000));
+
+ kApi = new CoreV1Api();
+ }
+ return kApi;
+ }
+}
diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/type/InternalLog.java b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/type/InternalLog.java
new file mode 100644
index 0000000000..e99195c538
--- /dev/null
+++ b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/type/InternalLog.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.skywalking.oap.query.graphql.type;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.RequiredArgsConstructor;
+import lombok.experimental.Accessors;
+
+@Data
+@Builder
+@AllArgsConstructor
+@RequiredArgsConstructor
+@Accessors(fluent = true)
+public final class InternalLog {
+ private String line;
+
+ private String container;
+}
diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/type/LogAdapter.java b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/type/LogAdapter.java
new file mode 100644
index 0000000000..a7dc03d6e4
--- /dev/null
+++ b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/type/LogAdapter.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.skywalking.oap.query.graphql.type;
+
+import static java.lang.String.format;
+import java.time.Instant;
+import java.time.format.DateTimeFormatter;
+import java.time.format.ResolverStyle;
+import java.time.temporal.TemporalAccessor;
+import java.util.List;
+import com.google.common.base.Splitter;
+import org.apache.skywalking.oap.server.core.query.type.ContentType;
+import org.apache.skywalking.oap.server.core.query.type.Log;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import lombok.RequiredArgsConstructor;
+
+@RequiredArgsConstructor
+public class LogAdapter {
+ private static final Logger LOGGER = LoggerFactory.getLogger(LogAdapter.class);
+
+ private final InternalLog log;
+
+ // k8s promises RFC3339 or RFC3339Nano timestamp, we truncate to RFC3339
+ private final DateTimeFormatter rfc3339Formatter =
+ DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ssZZZZZ")
+ .withResolverStyle(ResolverStyle.LENIENT);
+
+ public Log adapt() {
+ Log l = new Log();
+
+ List<String> timeAndContent = Splitter.on(" ")
+ .limit(2)
+ .trimResults()
+ .splitToList(log.line());
+ if (timeAndContent.size() == 2) {
+ String timeStr = timeAndContent.get(0).replaceAll("\\.\\d+Z", "Z");
+ try {
+ TemporalAccessor t = rfc3339Formatter.parse(timeStr);
+ long timestamp = Instant.from(t).getEpochSecond();
+ l.setTimestamp(timestamp);
+ l.setContent(format("[%s] %s", log.container(), timeAndContent.get(1)));
+ l.setContentType(ContentType.TEXT);
+ } catch (Exception e) {
+ LOGGER.warn("Failed to parse log entry, {}", log, e);
+ }
+ }
+
+ return l;
+ }
+}
diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/type/OndemandLogQueryCondition.java b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/type/OndemandLogQueryCondition.java
new file mode 100644
index 0000000000..d186e1d496
--- /dev/null
+++ b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/type/OndemandLogQueryCondition.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.skywalking.oap.query.graphql.type;
+
+import java.util.Collections;
+import java.util.List;
+import org.apache.skywalking.oap.server.core.query.input.Duration;
+import lombok.Getter;
+import lombok.Setter;
+
+@Getter
+@Setter
+public class OndemandLogQueryCondition {
+ private String namespace;
+ private String container;
+ private String serviceId;
+ private String serviceInstanceId;
+ private Duration duration;
+ private List<String> keywordsOfContent = Collections.emptyList();
+ private List<String> excludingKeywordsOfContent = Collections.emptyList();
+}
diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol b/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol
index 3adb9f6864..6c461cc60a 160000
--- a/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol
+++ b/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol
@@ -1 +1 @@
-Subproject commit 3adb9f68649b1a03fa3fa664192ff4bd7523f18b
+Subproject commit 6c461cc60a653de801e51ea914b78a243ce66819
diff --git a/oap-server/server-starter/src/main/resources/application.yml b/oap-server/server-starter/src/main/resources/application.yml
index 3144d8253d..f951bb72e4 100755
--- a/oap-server/server-starter/src/main/resources/application.yml
+++ b/oap-server/server-starter/src/main/resources/application.yml
@@ -399,6 +399,10 @@ query:
maxQueryComplexity: ${SW_QUERY_MAX_QUERY_COMPLEXITY:1000}
# Allow user add, disable and update UI template
enableUpdateUITemplate: ${SW_ENABLE_UPDATE_UI_TEMPLATE:false}
+ # "On demand log" allows users to fetch Pod containers' log in real time,
+ # because this might expose secrets in the logs (if any), users need
+ # to enable this manually, and add permissions to OAP cluster role.
+ enableOndemandLog: ${SW_ENABLE_ONDEMAND_LOG:false}
alarm:
selector: ${SW_ALARM:default}
diff --git a/test.http b/test.http
new file mode 100644
index 0000000000..213470d1a5
--- /dev/null
+++ b/test.http
@@ -0,0 +1,38 @@
+POST http://localhost:12800/graphql
+Content-Type: application/json
+
+{
+ "query": "query listContainers($condition: ContainerQueryCondition!) { containers: listContainers(condition: $condition) { containers } }",
+ "variables":{
+ "condition": {
+ "namespace": "default-agentless",
+ "serviceInstanceId": "YXBwLWRlcGxveW1lbnQ=.1_YXBwLWRlcGxveW1lbnQtNTdiOWZmYjViLTZtdmdw"
+ }
+ }
+}
+
+###
+
+POST http://localhost:12800/graphql
+Content-Type: application/json
+
+{
+ "query": "query ondemandLogs($condition: StreamingLogQueryCondition!) { logs: ondemandLogs(condition: $condition) { logs { content } } }",
+ "variables":{
+ "condition": {
+ "namespace": "default-agentless",
+ "serviceInstanceId": "YXBwLWRlcGxveW1lbnQ=.1_YXBwLWRlcGxveW1lbnQtNTdiOWZmYjViLTZtdmdw",
+ "container": "istio-proxy",
+ "paging": {
+ "pageNum": 1,
+ "pageSize": 10
+ },
+ "queryDuration": {
+ "start":"2022-05-23 1618",
+ "end":"2022-05-23 1648",
+ "step":"MINUTE"
+ }
+ }
+ }
+ }
+}