You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by ke...@apache.org on 2021/07/09 12:13:46 UTC
[skywalking] branch event/pagination created (now 54d9548)
This is an automated email from the ASF dual-hosted git repository.
kezhenxu94 pushed a change to branch event/pagination
in repository https://gitbox.apache.org/repos/asf/skywalking.git.
at 54d9548 Add pagination to event query
This branch includes the following new commits:
new d2f881e perf: optimize Groovy-based LAL DSL with static compilation
new 54d9548 Add pagination to event query
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[skywalking] 01/02: perf: optimize Groovy-based LAL DSL with static
compilation
Posted by ke...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kezhenxu94 pushed a commit to branch event/pagination
in repository https://gitbox.apache.org/repos/asf/skywalking.git
commit d2f881edd5a73e523c8f1603c62332a146eddc82
Author: kezhenxu94 <ke...@apache.org>
AuthorDate: Wed Jul 7 18:13:37 2021 +0800
perf: optimize Groovy-based LAL DSL with static compilation
Groovy naturally supports many dynamic features that we don't benefit for now but cost performance loss, in this patch we compile our Groovy-based DSL scripts statically to optimize performance.
---
.github/workflows/e2e.jdk-versions.yaml | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/.github/workflows/e2e.jdk-versions.yaml b/.github/workflows/e2e.jdk-versions.yaml
index b4236d6..705516f 100644
--- a/.github/workflows/e2e.jdk-versions.yaml
+++ b/.github/workflows/e2e.jdk-versions.yaml
@@ -20,6 +20,9 @@ on:
pull_request:
schedule:
- cron: '0 18 * * *'
+ push:
+ branches:
+ - test/*
concurrency:
group: e2e-jdk-versions-${{ github.event.pull_request.number || github.ref }}
@@ -63,6 +66,11 @@ jobs:
uses: ./.github/actions/e2e-test
with:
test_class: org.apache.skywalking.e2e.simple.SimpleE2E
+ - uses: actions/upload-artifact@v1.0.0
+ continue-on-error: true
+ with:
+ name: logs
+ path: $GITHUB_WORKSPACE/logs
Single:
if: (github.event_name == 'schedule' && github.repository == 'apache/skywalking') || (github.event_name != 'schedule')
[skywalking] 02/02: Add pagination to event query
Posted by ke...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kezhenxu94 pushed a commit to branch event/pagination
in repository https://gitbox.apache.org/repos/asf/skywalking.git
commit 54d95487d96c288b718a93cfe376d7e9bc27ce4b
Author: kezhenxu94 <ke...@apache.org>
AuthorDate: Fri Jul 9 15:27:48 2021 +0800
Add pagination to event query
---
.github/workflows/e2e.jdk-versions.yaml | 3 ---
.../oap/server/core/query/PaginationUtils.java | 23 +++++++---------------
.../oap/server/core/query/type/Pagination.java | 4 ++++
.../core/query/type/event/EventQueryCondition.java | 10 ++--------
.../server/core/storage/query/IEventQueryDAO.java | 1 -
.../oap/query/graphql/resolver/AlarmQuery.java | 4 +++-
.../elasticsearch/query/ESEventQueryDAO.java | 11 +++++++++--
.../plugin/influxdb/query/EventQueryDAO.java | 5 ++++-
.../plugin/jdbc/h2/dao/H2EventQueryDAO.java | 10 ++++++++--
9 files changed, 37 insertions(+), 34 deletions(-)
diff --git a/.github/workflows/e2e.jdk-versions.yaml b/.github/workflows/e2e.jdk-versions.yaml
index 705516f..0765945 100644
--- a/.github/workflows/e2e.jdk-versions.yaml
+++ b/.github/workflows/e2e.jdk-versions.yaml
@@ -20,9 +20,6 @@ on:
pull_request:
schedule:
- cron: '0 18 * * *'
- push:
- branches:
- - test/*
concurrency:
group: e2e-jdk-versions-${{ github.event.pull_request.number || github.ref }}
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/PaginationUtils.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/PaginationUtils.java
index 125e93a..0e6a6fc 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/PaginationUtils.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/PaginationUtils.java
@@ -18,6 +18,8 @@
package org.apache.skywalking.oap.server.core.query;
+import lombok.Data;
+import lombok.RequiredArgsConstructor;
import org.apache.skywalking.oap.server.core.query.type.Pagination;
public enum PaginationUtils {
@@ -30,21 +32,10 @@ public enum PaginationUtils {
return new Page(from, limit);
}
- public class Page {
- private int from;
- private int limit;
-
- Page(int from, int limit) {
- this.from = from;
- this.limit = limit;
- }
-
- public int getFrom() {
- return from;
- }
-
- public int getLimit() {
- return limit;
- }
+ @Data
+ @RequiredArgsConstructor
+ public static class Page {
+ private final int from;
+ private final int limit;
}
}
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/Pagination.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/Pagination.java
index 2210122..899edf5 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/Pagination.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/Pagination.java
@@ -18,11 +18,15 @@
package org.apache.skywalking.oap.server.core.query.type;
+import lombok.AllArgsConstructor;
import lombok.Getter;
+import lombok.NoArgsConstructor;
import lombok.Setter;
@Getter
@Setter
+@NoArgsConstructor
+@AllArgsConstructor
public class Pagination {
private int pageNum;
private int pageSize;
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/event/EventQueryCondition.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/event/EventQueryCondition.java
index a90e08b..be886a7 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/event/EventQueryCondition.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/event/EventQueryCondition.java
@@ -24,9 +24,7 @@ import lombok.Data;
import lombok.NoArgsConstructor;
import org.apache.skywalking.oap.server.core.query.enumeration.Order;
import org.apache.skywalking.oap.server.core.query.input.Duration;
-
-import static org.apache.skywalking.oap.server.core.storage.query.IEventQueryDAO.DEFAULT_SIZE;
-import static org.apache.skywalking.oap.server.core.storage.query.IEventQueryDAO.MAX_SIZE;
+import org.apache.skywalking.oap.server.core.query.type.Pagination;
@Data
@AllArgsConstructor
@@ -45,9 +43,5 @@ public class EventQueryCondition {
private Order order;
- private int size;
-
- public int getSize() {
- return size > 0 ? Math.min(size, MAX_SIZE) : DEFAULT_SIZE;
- }
+ private Pagination paging;
}
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/query/IEventQueryDAO.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/query/IEventQueryDAO.java
index ea09a4d..9cdfc49 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/query/IEventQueryDAO.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/query/IEventQueryDAO.java
@@ -25,7 +25,6 @@ import org.apache.skywalking.oap.server.core.storage.DAO;
import java.util.List;
public interface IEventQueryDAO extends DAO {
- int DEFAULT_SIZE = 20;
int MAX_SIZE = 100;
Events queryEvents(final EventQueryCondition condition) throws Exception;
diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/AlarmQuery.java b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/AlarmQuery.java
index dd531f7..4a917c5 100644
--- a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/AlarmQuery.java
+++ b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/AlarmQuery.java
@@ -88,7 +88,9 @@ public class AlarmQuery implements GraphQLQueryResolver {
}
long startSecondTB = 0;
long endSecondTB = 0;
- final EventQueryCondition.EventQueryConditionBuilder conditionPrototype = EventQueryCondition.builder().size(IEventQueryDAO.MAX_SIZE);
+ final EventQueryCondition.EventQueryConditionBuilder conditionPrototype =
+ EventQueryCondition.builder()
+ .paging(new Pagination(1, IEventQueryDAO.MAX_SIZE, false));
if (nonNull(duration)) {
startSecondTB = duration.getStartTimeBucketInSec();
endSecondTB = duration.getEndTimeBucketInSec();
diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/ESEventQueryDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/ESEventQueryDAO.java
index 7aee3e7..fb7c865 100644
--- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/ESEventQueryDAO.java
+++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/ESEventQueryDAO.java
@@ -23,6 +23,7 @@ import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
import java.util.stream.Stream;
+import org.apache.skywalking.oap.server.core.query.PaginationUtils;
import org.apache.skywalking.oap.server.core.source.Event;
import org.apache.skywalking.oap.server.core.query.enumeration.Order;
import org.apache.skywalking.oap.server.core.query.input.Duration;
@@ -129,7 +130,10 @@ public class ESEventQueryDAO extends EsDAO implements IEventQueryDAO {
EventQueryCondition condition = conditionList.get(0);
final Order queryOrder = isNull(condition.getOrder()) ? Order.DES : condition.getOrder();
sourceBuilder.sort(Event.START_TIME, Order.DES.equals(queryOrder) ? SortOrder.DESC : SortOrder.ASC);
- sourceBuilder.size(condition.getSize());
+
+ final PaginationUtils.Page page = PaginationUtils.INSTANCE.exchange(condition.getPaging());
+ sourceBuilder.from(page.getFrom());
+ sourceBuilder.size(page.getLimit());
return sourceBuilder;
}
@@ -144,7 +148,10 @@ public class ESEventQueryDAO extends EsDAO implements IEventQueryDAO {
final Order queryOrder = isNull(condition.getOrder()) ? Order.DES : condition.getOrder();
sourceBuilder.sort(Event.START_TIME, Order.DES.equals(queryOrder) ? SortOrder.DESC : SortOrder.ASC);
- sourceBuilder.size(condition.getSize());
+
+ final PaginationUtils.Page page = PaginationUtils.INSTANCE.exchange(condition.getPaging());
+ sourceBuilder.from(page.getFrom());
+ sourceBuilder.size(page.getLimit());
return sourceBuilder;
}
diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/EventQueryDAO.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/EventQueryDAO.java
index 1d36892..3879b71 100644
--- a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/EventQueryDAO.java
+++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/EventQueryDAO.java
@@ -26,6 +26,7 @@ import java.util.Map;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
+import org.apache.skywalking.oap.server.core.query.PaginationUtils;
import org.apache.skywalking.oap.server.core.source.Event;
import org.apache.skywalking.oap.server.core.query.enumeration.Order;
import org.apache.skywalking.oap.server.core.query.input.Duration;
@@ -110,10 +111,12 @@ public class EventQueryDAO implements IEventQueryDAO {
protected List<WhereQueryImpl<SelectQueryImpl>> buildWhereQueries(final EventQueryCondition condition) {
List<WhereQueryImpl<SelectQueryImpl>> queries = new ArrayList<>(2);
+
+ final PaginationUtils.Page page = PaginationUtils.INSTANCE.exchange(condition.getPaging());
final String topFunc = Order.DES.equals(condition.getOrder()) ? InfluxConstants.SORT_DES : InfluxConstants.SORT_ASC;
final WhereQueryImpl<SelectQueryImpl> recallWhereQuery =
select().raw(ALL_FIELDS)
- .function(topFunc, Event.START_TIME, condition.getSize())
+ .function(topFunc, Event.START_TIME, page.getLimit() + page.getFrom())
.from(client.getDatabase(), Event.INDEX_NAME)
.where();
final SelectQueryImpl countQuery = select().count(Event.UUID).from(client.getDatabase(), Event.INDEX_NAME);
diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2EventQueryDAO.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2EventQueryDAO.java
index c29dfc9..3444268 100644
--- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2EventQueryDAO.java
+++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2EventQueryDAO.java
@@ -28,6 +28,8 @@ import java.util.stream.Collectors;
import java.util.stream.Stream;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
+import org.apache.skywalking.oap.server.core.query.PaginationUtils;
+import org.apache.skywalking.oap.server.core.query.type.Pagination;
import org.apache.skywalking.oap.server.core.source.Event;
import org.apache.skywalking.oap.server.core.query.input.Duration;
import org.apache.skywalking.oap.server.core.query.type.event.EventQueryCondition;
@@ -65,7 +67,10 @@ public class H2EventQueryDAO implements IEventQueryDAO {
result.setTotal(resultSet.getInt("total"));
}
- sql = "select * from " + Event.INDEX_NAME + whereClause + " limit " + condition.getSize();
+ PaginationUtils.Page page = PaginationUtils.INSTANCE.exchange(condition.getPaging());
+
+ sql = "select * from " + Event.INDEX_NAME + whereClause
+ + " limit " + page.getLimit() + " offset " + page.getFrom();
if (log.isDebugEnabled()) {
log.debug("Query SQL: {}, parameters: {}", sql, parameters);
}
@@ -92,7 +97,8 @@ public class H2EventQueryDAO implements IEventQueryDAO {
.map(Tuple2::_1)
.map(it -> it.collect(Collectors.joining(" and ")))
.collect(Collectors.joining(" or ", " where ", ""));
- final int size = conditions.stream().mapToInt(EventQueryCondition::getSize).sum();
+ final int size = conditions.stream().map(EventQueryCondition::getPaging)
+ .mapToInt(Pagination::getPageSize).sum();
final Events result = new Events();
try (final Connection connection = client.getConnection()) {