You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by ke...@apache.org on 2021/07/09 12:16:31 UTC

[skywalking] branch event/pagination updated (6a5e7a6 -> 80c63e8)

This is an automated email from the ASF dual-hosted git repository.

kezhenxu94 pushed a change to branch event/pagination
in repository https://gitbox.apache.org/repos/asf/skywalking.git.


 discard 6a5e7a6  Add pagination to event query
     new 80c63e8  Add pagination to event query

This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version.  This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:

 * -- * -- B -- O -- O -- O   (6a5e7a6)
            \
             N -- N -- N   refs/heads/event/pagination (80c63e8)

You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.

Any revisions marked "omit" are not gone; other references still
refer to them.  Any revisions marked "discard" are gone forever.

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 .github/workflows/e2e.jdk-versions.yaml | 5 -----
 1 file changed, 5 deletions(-)

[skywalking] 01/01: Add pagination to event query

Posted by ke...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

kezhenxu94 pushed a commit to branch event/pagination
in repository https://gitbox.apache.org/repos/asf/skywalking.git

commit 80c63e84323181ee90a8c8d0f461923025c7cca9
Author: kezhenxu94 <ke...@apache.org>
AuthorDate: Wed Jul 7 18:13:37 2021 +0800

    Add pagination to event query
---
 CHANGES.md                                         |  1 +
 .../oap/server/core/query/PaginationUtils.java     | 23 +++++++---------------
 .../oap/server/core/query/type/Pagination.java     |  4 ++++
 .../core/query/type/event/EventQueryCondition.java | 10 ++--------
 .../server/core/storage/query/IEventQueryDAO.java  |  1 -
 .../oap/query/graphql/resolver/AlarmQuery.java     |  4 +++-
 .../elasticsearch/query/ESEventQueryDAO.java       | 11 +++++++++--
 .../plugin/influxdb/query/EventQueryDAO.java       |  5 ++++-
 .../plugin/jdbc/h2/dao/H2EventQueryDAO.java        | 10 ++++++++--
 9 files changed, 38 insertions(+), 31 deletions(-)

diff --git a/CHANGES.md b/CHANGES.md
index fa4b5fa..62985cc 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -78,6 +78,7 @@ Release Notes.
 * Configuration: Allow to configure server maximum request header size.
 * Add thread state metric and class loaded info metric to JVMMetric.
 * Performance: compile LAL DSL statically and run with type checked.
+* Add pagination to event query protocol.
 
 #### UI
 
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/PaginationUtils.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/PaginationUtils.java
index 125e93a..0e6a6fc 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/PaginationUtils.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/PaginationUtils.java
@@ -18,6 +18,8 @@
 
 package org.apache.skywalking.oap.server.core.query;
 
+import lombok.Data;
+import lombok.RequiredArgsConstructor;
 import org.apache.skywalking.oap.server.core.query.type.Pagination;
 
 public enum PaginationUtils {
@@ -30,21 +32,10 @@ public enum PaginationUtils {
         return new Page(from, limit);
     }
 
-    public class Page {
-        private int from;
-        private int limit;
-
-        Page(int from, int limit) {
-            this.from = from;
-            this.limit = limit;
-        }
-
-        public int getFrom() {
-            return from;
-        }
-
-        public int getLimit() {
-            return limit;
-        }
+    @Data
+    @RequiredArgsConstructor
+    public static class Page {
+        private final int from;
+        private final int limit;
     }
 }
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/Pagination.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/Pagination.java
index 2210122..899edf5 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/Pagination.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/Pagination.java
@@ -18,11 +18,15 @@
 
 package org.apache.skywalking.oap.server.core.query.type;
 
+import lombok.AllArgsConstructor;
 import lombok.Getter;
+import lombok.NoArgsConstructor;
 import lombok.Setter;
 
 @Getter
 @Setter
+@NoArgsConstructor
+@AllArgsConstructor
 public class Pagination {
     private int pageNum;
     private int pageSize;
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/event/EventQueryCondition.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/event/EventQueryCondition.java
index a90e08b..be886a7 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/event/EventQueryCondition.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/event/EventQueryCondition.java
@@ -24,9 +24,7 @@ import lombok.Data;
 import lombok.NoArgsConstructor;
 import org.apache.skywalking.oap.server.core.query.enumeration.Order;
 import org.apache.skywalking.oap.server.core.query.input.Duration;
-
-import static org.apache.skywalking.oap.server.core.storage.query.IEventQueryDAO.DEFAULT_SIZE;
-import static org.apache.skywalking.oap.server.core.storage.query.IEventQueryDAO.MAX_SIZE;
+import org.apache.skywalking.oap.server.core.query.type.Pagination;
 
 @Data
 @AllArgsConstructor
@@ -45,9 +43,5 @@ public class EventQueryCondition {
 
     private Order order;
 
-    private int size;
-
-    public int getSize() {
-        return size > 0 ? Math.min(size, MAX_SIZE) : DEFAULT_SIZE;
-    }
+    private Pagination paging;
 }
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/query/IEventQueryDAO.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/query/IEventQueryDAO.java
index ea09a4d..9cdfc49 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/query/IEventQueryDAO.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/query/IEventQueryDAO.java
@@ -25,7 +25,6 @@ import org.apache.skywalking.oap.server.core.storage.DAO;
 import java.util.List;
 
 public interface IEventQueryDAO extends DAO {
-    int DEFAULT_SIZE = 20;
     int MAX_SIZE = 100;
 
     Events queryEvents(final EventQueryCondition condition) throws Exception;
diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/AlarmQuery.java b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/AlarmQuery.java
index dd531f7..4a917c5 100644
--- a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/AlarmQuery.java
+++ b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/AlarmQuery.java
@@ -88,7 +88,9 @@ public class AlarmQuery implements GraphQLQueryResolver {
         }
         long startSecondTB = 0;
         long endSecondTB = 0;
-        final EventQueryCondition.EventQueryConditionBuilder conditionPrototype = EventQueryCondition.builder().size(IEventQueryDAO.MAX_SIZE);
+        final EventQueryCondition.EventQueryConditionBuilder conditionPrototype =
+            EventQueryCondition.builder()
+                               .paging(new Pagination(1, IEventQueryDAO.MAX_SIZE, false));
         if (nonNull(duration)) {
             startSecondTB = duration.getStartTimeBucketInSec();
             endSecondTB = duration.getEndTimeBucketInSec();
diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/ESEventQueryDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/ESEventQueryDAO.java
index 7aee3e7..fb7c865 100644
--- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/ESEventQueryDAO.java
+++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/ESEventQueryDAO.java
@@ -23,6 +23,7 @@ import java.util.List;
 import java.util.Objects;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
+import org.apache.skywalking.oap.server.core.query.PaginationUtils;
 import org.apache.skywalking.oap.server.core.source.Event;
 import org.apache.skywalking.oap.server.core.query.enumeration.Order;
 import org.apache.skywalking.oap.server.core.query.input.Duration;
@@ -129,7 +130,10 @@ public class ESEventQueryDAO extends EsDAO implements IEventQueryDAO {
         EventQueryCondition condition = conditionList.get(0);
         final Order queryOrder = isNull(condition.getOrder()) ? Order.DES : condition.getOrder();
         sourceBuilder.sort(Event.START_TIME, Order.DES.equals(queryOrder) ? SortOrder.DESC : SortOrder.ASC);
-        sourceBuilder.size(condition.getSize());
+
+        final PaginationUtils.Page page = PaginationUtils.INSTANCE.exchange(condition.getPaging());
+        sourceBuilder.from(page.getFrom());
+        sourceBuilder.size(page.getLimit());
         return sourceBuilder;
     }
 
@@ -144,7 +148,10 @@ public class ESEventQueryDAO extends EsDAO implements IEventQueryDAO {
 
         final Order queryOrder = isNull(condition.getOrder()) ? Order.DES : condition.getOrder();
         sourceBuilder.sort(Event.START_TIME, Order.DES.equals(queryOrder) ? SortOrder.DESC : SortOrder.ASC);
-        sourceBuilder.size(condition.getSize());
+
+        final PaginationUtils.Page page = PaginationUtils.INSTANCE.exchange(condition.getPaging());
+        sourceBuilder.from(page.getFrom());
+        sourceBuilder.size(page.getLimit());
 
         return sourceBuilder;
     }
diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/EventQueryDAO.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/EventQueryDAO.java
index 1d36892..3879b71 100644
--- a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/EventQueryDAO.java
+++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/EventQueryDAO.java
@@ -26,6 +26,7 @@ import java.util.Map;
 
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.skywalking.oap.server.core.query.PaginationUtils;
 import org.apache.skywalking.oap.server.core.source.Event;
 import org.apache.skywalking.oap.server.core.query.enumeration.Order;
 import org.apache.skywalking.oap.server.core.query.input.Duration;
@@ -110,10 +111,12 @@ public class EventQueryDAO implements IEventQueryDAO {
 
     protected List<WhereQueryImpl<SelectQueryImpl>> buildWhereQueries(final EventQueryCondition condition) {
         List<WhereQueryImpl<SelectQueryImpl>> queries = new ArrayList<>(2);
+
+        final PaginationUtils.Page page = PaginationUtils.INSTANCE.exchange(condition.getPaging());
         final String topFunc = Order.DES.equals(condition.getOrder()) ? InfluxConstants.SORT_DES : InfluxConstants.SORT_ASC;
         final WhereQueryImpl<SelectQueryImpl> recallWhereQuery =
                 select().raw(ALL_FIELDS)
-                        .function(topFunc, Event.START_TIME, condition.getSize())
+                        .function(topFunc, Event.START_TIME, page.getLimit() + page.getFrom())
                         .from(client.getDatabase(), Event.INDEX_NAME)
                         .where();
         final SelectQueryImpl countQuery = select().count(Event.UUID).from(client.getDatabase(), Event.INDEX_NAME);
diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2EventQueryDAO.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2EventQueryDAO.java
index c29dfc9..3444268 100644
--- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2EventQueryDAO.java
+++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2EventQueryDAO.java
@@ -28,6 +28,8 @@ import java.util.stream.Collectors;
 import java.util.stream.Stream;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.skywalking.oap.server.core.query.PaginationUtils;
+import org.apache.skywalking.oap.server.core.query.type.Pagination;
 import org.apache.skywalking.oap.server.core.source.Event;
 import org.apache.skywalking.oap.server.core.query.input.Duration;
 import org.apache.skywalking.oap.server.core.query.type.event.EventQueryCondition;
@@ -65,7 +67,10 @@ public class H2EventQueryDAO implements IEventQueryDAO {
                 result.setTotal(resultSet.getInt("total"));
             }
 
-            sql = "select * from " + Event.INDEX_NAME + whereClause + " limit " + condition.getSize();
+            PaginationUtils.Page page = PaginationUtils.INSTANCE.exchange(condition.getPaging());
+
+            sql = "select * from " + Event.INDEX_NAME + whereClause
+                + " limit " + page.getLimit() + " offset " + page.getFrom();
             if (log.isDebugEnabled()) {
                 log.debug("Query SQL: {}, parameters: {}", sql, parameters);
             }
@@ -92,7 +97,8 @@ public class H2EventQueryDAO implements IEventQueryDAO {
                                                        .map(Tuple2::_1)
                                                        .map(it -> it.collect(Collectors.joining(" and ")))
                                                        .collect(Collectors.joining(" or ", " where ", ""));
-        final int size = conditions.stream().mapToInt(EventQueryCondition::getSize).sum();
+        final int size = conditions.stream().map(EventQueryCondition::getPaging)
+                                   .mapToInt(Pagination::getPageSize).sum();
 
         final Events result = new Events();
         try (final Connection connection = client.getConnection()) {