You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by xx...@apache.org on 2023/02/14 05:54:52 UTC
[kylin] 23/33: KYLIN-5440 Follow up, optimize the speed of three datasource API
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 0f0f149bd5b77571c6fa10ce2dc5dea78e57bb99
Author: Guoliang Sun <gu...@kyligence.io>
AuthorDate: Fri Dec 9 13:58:06 2022 +0800
KYLIN-5440 Follow up, optimize the speed of three datasource API
---
.../kylin/rest/controller/NBasicController.java | 9 ++
.../org/apache/kylin/rest/response/DataResult.java | 7 ++
.../apache/kylin/rest/service/BasicService.java | 12 +++
.../kylin/rest/service/SnapshotSupporter.java | 6 +-
.../rest/controller/NBasicControllerTest.java | 19 ++++
.../kylin/rest/controller/SnapshotController.java | 8 +-
.../rest/controller/SnapshotControllerTest.java | 9 +-
.../apache/kylin/rest/service/SnapshotService.java | 104 ++++++++++--------
.../kylin/rest/service/SnapshotServiceTest.java | 106 ++++++++++++++++++-
.../kylin/rest/request/TableDescRequest.java | 88 ++++++++++++++++
.../apache/kylin/rest/service/TableService.java | 117 ++++++++++++---------
.../org/apache/kylin/rest/util/TableUtils.java | 17 +++
.../rest/service/StreamingTableServiceTest.java | 14 ++-
.../kylin/rest/controller/NTableController.java | 40 +++----
.../rest/controller/open/OpenTableController.java | 9 +-
.../rest/controller/v2/NTableControllerV2.java | 11 +-
.../rest/controller/NTableControllerTest.java | 17 ++-
.../rest/controller/NTableControllerV2Test.java | 7 +-
.../controller/open/OpenTableControllerTest.java | 23 ++--
.../kylin/rest/service/TableServiceTest.java | 90 ++++++++++------
20 files changed, 540 insertions(+), 173 deletions(-)
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/controller/NBasicController.java b/src/common-service/src/main/java/org/apache/kylin/rest/controller/NBasicController.java
index 884a807128..ab39cab22d 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/controller/NBasicController.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/controller/NBasicController.java
@@ -83,11 +83,13 @@ import org.apache.kylin.common.msg.MsgPicker;
import org.apache.kylin.common.persistence.transaction.TransactionException;
import org.apache.kylin.common.util.DateFormat;
import org.apache.kylin.common.util.JsonUtil;
+import org.apache.kylin.common.util.Pair;
import org.apache.kylin.job.constant.JobStatusEnum;
import org.apache.kylin.job.dao.ExecutablePO;
import org.apache.kylin.job.execution.JobTypeEnum;
import org.apache.kylin.metadata.model.NDataModel;
import org.apache.kylin.metadata.model.NDataModelManager;
+import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.project.NProjectManager;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.streaming.KafkaConfigManager;
@@ -378,6 +380,13 @@ public class NBasicController {
return data;
}
+ public Map<String, Object> setCustomDataResponse(String name, Pair<List<TableDesc>, Integer> result, int offset, int limit) {
+ Map<String, Object> data = new HashMap<>();
+ data.put(name, PagingUtil.cutPage(result.getFirst(), offset, limit));
+ data.put("size", result.getSecond());
+ return data;
+ }
+
public List<?> getDataNoEnvelopeResponse(List<?> result, int offset, int limit) {
return PagingUtil.cutPage(result, offset, limit);
}
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/response/DataResult.java b/src/common-service/src/main/java/org/apache/kylin/rest/response/DataResult.java
index b7533cb581..86aee70e2d 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/response/DataResult.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/response/DataResult.java
@@ -21,6 +21,7 @@ package org.apache.kylin.rest.response;
import java.util.Collection;
import java.util.List;
+import org.apache.kylin.common.util.Pair;
import org.apache.kylin.rest.util.PagingUtil;
import com.fasterxml.jackson.annotation.JsonProperty;
@@ -65,4 +66,10 @@ public class DataResult<T extends Collection> {
public static <E> DataResult<List<E>> get(List<E> data, int offset, int limit) {
return get(PagingUtil.cutPage(data, offset, limit), data, offset, limit);
}
+
+ public static <E> DataResult<List<E>> getCustom(Pair<List<E>, Integer> objWithActualSize, int offset, int limit) {
+ // objWithActualSize's data cannot be null
+ return new DataResult<>(PagingUtil.cutPage(objWithActualSize.getFirst(), offset, limit),
+ objWithActualSize.getSecond(), offset, limit);
+ }
}
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/service/BasicService.java b/src/common-service/src/main/java/org/apache/kylin/rest/service/BasicService.java
index 46a74acd81..b349943496 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/service/BasicService.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/service/BasicService.java
@@ -34,6 +34,7 @@ import org.apache.kylin.common.persistence.transaction.BroadcastEventReadyNotifi
import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.metadata.epoch.EpochManager;
import org.apache.kylin.metadata.project.EnhancedUnitOfWork;
+import org.apache.kylin.common.util.Pair;
import org.apache.kylin.metadata.project.NProjectManager;
import org.apache.kylin.metadata.streaming.DataParserManager;
import org.apache.kylin.rest.response.EnvelopeResponse;
@@ -140,6 +141,17 @@ public abstract class BasicService {
return true;
}
+ public Pair<String, String> checkDatabaseAndTable(String table) {
+ if (table == null)
+ table = "";
+ String database = null;
+ if (table.contains(".")) {
+ database = table.split("\\.", 2)[0].trim();
+ table = table.split("\\.", 2)[1].trim();
+ }
+ return Pair.newPair(database, table);
+ }
+
protected void initDefaultParser(String project) {
if (getManager(DataParserManager.class, project).isInitialized()) {
return;
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/service/SnapshotSupporter.java b/src/common-service/src/main/java/org/apache/kylin/rest/service/SnapshotSupporter.java
index b1052be4fb..5936dd20ef 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/service/SnapshotSupporter.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/service/SnapshotSupporter.java
@@ -21,10 +21,12 @@ package org.apache.kylin.rest.service;
import java.util.List;
import java.util.Set;
+import org.apache.kylin.common.util.Pair;
import org.apache.kylin.rest.constant.SnapshotStatus;
import org.apache.kylin.rest.response.SnapshotInfoResponse;
public interface SnapshotSupporter {
- List<SnapshotInfoResponse> getProjectSnapshots(String project, String table, Set<SnapshotStatus> statusFilter,
- Set<Boolean> partitionFilter, String sortBy, boolean isReversed);
+ Pair<List<SnapshotInfoResponse>, Integer> getProjectSnapshots(String project, String table,
+ Set<SnapshotStatus> statusFilter, Set<Boolean> partitionFilter, String sortBy, boolean isReversed,
+ Pair<Integer, Integer> offsetAndLimit);
}
diff --git a/src/common-service/src/test/java/org/apache/kylin/rest/controller/NBasicControllerTest.java b/src/common-service/src/test/java/org/apache/kylin/rest/controller/NBasicControllerTest.java
index e36df02ccb..e436db2066 100644
--- a/src/common-service/src/test/java/org/apache/kylin/rest/controller/NBasicControllerTest.java
+++ b/src/common-service/src/test/java/org/apache/kylin/rest/controller/NBasicControllerTest.java
@@ -36,13 +36,16 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
+import java.util.Map;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.msg.Message;
import org.apache.kylin.common.msg.MsgPicker;
import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
+import org.apache.kylin.common.util.Pair;
import org.apache.kylin.metadata.model.PartitionDesc;
+import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.rest.controller.fixture.FixtureController;
import org.apache.kylin.rest.exception.ForbiddenException;
import org.apache.kylin.rest.exception.NotFoundException;
@@ -292,4 +295,20 @@ public class NBasicControllerTest extends NLocalFileMetadataTestCase {
() -> nBasicController.checkNonNegativeIntegerArg("id", -1));
}
+ @Test
+ public void testSetCustomDataResponse() {
+ TableDesc tableDesc = new TableDesc();
+ tableDesc.setName("table1");
+ Map<String, Object> mockDataResponse = nBasicController.setCustomDataResponse("table",
+ Pair.newPair(Collections.singletonList(tableDesc), 3), 0, 10);
+ Assert.assertNotNull(mockDataResponse);
+ Object tableData = mockDataResponse.get("table");
+ if (tableData instanceof List<?>) {
+ for (Object tableDatum : (List<?>) tableData) {
+ Assert.assertEquals("table1", ((TableDesc)tableDatum).getName().toLowerCase(Locale.ROOT));
+ }
+ }
+ Assert.assertEquals(3, mockDataResponse.get("size"));
+ }
+
}
diff --git a/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/SnapshotController.java b/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/SnapshotController.java
index f0030909e5..24bf3ccac4 100644
--- a/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/SnapshotController.java
+++ b/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/SnapshotController.java
@@ -31,6 +31,7 @@ import java.util.Set;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.exception.KylinException;
+import org.apache.kylin.common.util.Pair;
import org.apache.kylin.metadata.project.NProjectManager;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.rest.constant.SnapshotStatus;
@@ -221,9 +222,10 @@ public class SnapshotController extends BaseController {
} catch (NoSuchFieldException e) {
throw new KylinException(SORT_BY_FIELD_NOT_EXIST, sortBy);
}
- List<SnapshotInfoResponse> responses = snapshotService.getProjectSnapshots(project, table, statusFilter,
- partitionFilter, sortBy, isReversed);
- return new EnvelopeResponse<>(KylinException.CODE_SUCCESS, DataResult.get(responses, offset, limit), "");
+ Pair<List<SnapshotInfoResponse>, Integer> snapshotsAndSize = snapshotService.getProjectSnapshots(project, table,
+ statusFilter, partitionFilter, sortBy, isReversed, Pair.newPair(offset, limit));
+ return new EnvelopeResponse<>(KylinException.CODE_SUCCESS,
+ new DataResult<>(snapshotsAndSize.getFirst(), snapshotsAndSize.getSecond(), offset, limit), "");
}
@ApiOperation(value = "getTables", tags = { "AI" }, notes = "get all tables with or without snapshot")
diff --git a/src/data-loading-server/src/test/java/org/apache/kylin/rest/controller/SnapshotControllerTest.java b/src/data-loading-server/src/test/java/org/apache/kylin/rest/controller/SnapshotControllerTest.java
index d3a2d3feee..924e8a08b4 100644
--- a/src/data-loading-server/src/test/java/org/apache/kylin/rest/controller/SnapshotControllerTest.java
+++ b/src/data-loading-server/src/test/java/org/apache/kylin/rest/controller/SnapshotControllerTest.java
@@ -25,6 +25,7 @@ import java.util.Map;
import java.util.Set;
import org.apache.kylin.common.util.JsonUtil;
+import org.apache.kylin.common.util.Pair;
import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
import org.apache.kylin.rest.constant.SnapshotStatus;
@@ -235,8 +236,8 @@ public class SnapshotControllerTest extends NLocalFileMetadataTestCase {
Set<SnapshotStatus> statusFilter = Sets.newHashSet();
String sortBy = "last_modified_time";
boolean isReversed = true;
- Mockito.doAnswer(x -> null).when(snapshotService).getProjectSnapshots(project, table, statusFilter,
- Sets.newHashSet(), sortBy, isReversed);
+ Mockito.doAnswer(x -> Pair.newPair(null, 10)).when(snapshotService).getProjectSnapshots(project, table, statusFilter,
+ Sets.newHashSet(), sortBy, isReversed, Pair.newPair(0, 10));
mockMvc.perform(MockMvcRequestBuilders.get("/api/snapshots").param("project", project)
.contentType(MediaType.APPLICATION_JSON).accept(MediaType.parseMediaType(APPLICATION_PUBLIC_JSON)))
.andExpect(MockMvcResultMatchers.status().isOk());
@@ -251,8 +252,8 @@ public class SnapshotControllerTest extends NLocalFileMetadataTestCase {
Set<SnapshotStatus> statusFilter = Sets.newHashSet();
String sortBy = "UNKNOWN";
boolean isReversed = true;
- Mockito.doAnswer(x -> null).when(snapshotService).getProjectSnapshots(project, table, statusFilter, null,
- sortBy, isReversed);
+ Mockito.doAnswer(x -> Pair.newPair(null, 10)).when(snapshotService).getProjectSnapshots(project, table, statusFilter, null,
+ sortBy, isReversed, Pair.newPair(0, 10));
final MvcResult mvcResult = mockMvc
.perform(MockMvcRequestBuilders.get("/api/snapshots").param("project", project).param("sort_by", sortBy)
.contentType(MediaType.APPLICATION_JSON)
diff --git a/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/SnapshotService.java b/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/SnapshotService.java
index aeca5521a4..431758773d 100644
--- a/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/SnapshotService.java
+++ b/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/SnapshotService.java
@@ -79,6 +79,7 @@ import java.util.Locale;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@@ -94,6 +95,7 @@ import static org.apache.kylin.common.exception.code.ErrorCodeServer.REQUEST_PAR
import static org.apache.kylin.job.execution.JobTypeEnum.SNAPSHOT_BUILD;
import static org.apache.kylin.job.execution.JobTypeEnum.SNAPSHOT_REFRESH;
import static org.apache.kylin.rest.constant.SnapshotStatus.BROKEN;
+import static org.apache.kylin.rest.util.TableUtils.calculateTableSize;
@Component("snapshotService")
public class SnapshotService extends BasicService implements SnapshotSupporter {
@@ -437,34 +439,72 @@ public class SnapshotService extends BasicService implements SnapshotSupporter {
}
@Override
- public List<SnapshotInfoResponse> getProjectSnapshots(String project, String table,
- Set<SnapshotStatus> statusFilter, Set<Boolean> partitionFilter, String sortBy, boolean isReversed) {
+ public Pair<List<SnapshotInfoResponse>, Integer> getProjectSnapshots(String project, String table,
+ Set<SnapshotStatus> statusFilter, Set<Boolean> partitionFilter, String sortBy, boolean isReversed,
+ Pair<Integer, Integer> offsetAndLimit) {
checkSnapshotManualManagement(project);
aclEvaluate.checkProjectReadPermission(project);
NTableMetadataManager nTableMetadataManager = getManager(NTableMetadataManager.class, project);
val execManager = NExecutableManager.getInstance(getConfig(), project);
List<AbstractExecutable> executables = execManager.listExecByJobTypeAndStatus(ExecutableState::isRunning,
SNAPSHOT_BUILD, SNAPSHOT_REFRESH);
- if (table == null)
- table = "";
- String database = null;
- if (table.contains(".")) {
- database = table.split("\\.", 2)[0].trim();
- table = table.split("\\.", 2)[1].trim();
- }
-
- final String finalTable = table;
- final String finalDatabase = database;
+ Pair<String, String> databaseAndTable = checkDatabaseAndTable(table);
Set<String> groups = getCurrentUserGroups();
boolean canUseACLGreenChannel = AclPermissionUtil.canUseACLGreenChannel(project, groups);
+ Set<String> finalAuthorizedTables = getAclAuthorizedTables(project, canUseACLGreenChannel);
+
+ // Adjust the operation of adding SnapshotInfoResponse and then removing it to
+ // first remove the tableDesc that does not meet the conditions, and then add SnapshotInfoResponse
+ List<TableDesc> tables = getFilteredTables(nTableMetadataManager, databaseAndTable, canUseACLGreenChannel,
+ finalAuthorizedTables, executables, statusFilter, partitionFilter);
+
+ List<SnapshotInfoResponse> response = new ArrayList<>();
+ // Here we keep the actual size of tableSnapshots and process only a portion of the data based on paging
+ final int returnTableSize = calculateTableSize(offsetAndLimit.getFirst(), offsetAndLimit.getSecond());
+ final int actualTableSize = tables.size();
+ AtomicInteger satisfiedTableSize = new AtomicInteger();
+
+ tables.forEach(tableDesc -> {
+ if (satisfiedTableSize.get() == returnTableSize) {
+ return;
+ }
+ TableExtDesc tableExtDesc = nTableMetadataManager.getOrCreateTableExt(tableDesc);
+ Pair<Integer, Integer> countPair = getModelCount(tableDesc);
+ response.add(new SnapshotInfoResponse(tableDesc, tableExtDesc, tableDesc.getSnapshotTotalRows(), countPair.getFirst(),
+ countPair.getSecond(), getSnapshotJobStatus(tableDesc, executables),
+ getForbiddenColumns(tableDesc)));
+ satisfiedTableSize.getAndIncrement();
+ });
+
+ sortBy = StringUtils.isEmpty(sortBy) ? "last_modified_time" : sortBy;
+ if ("last_modified_time".equalsIgnoreCase(sortBy) && isReversed) {
+ // The reverse order here needs to be cut from the beginning to the end, otherwise the initial data is always returned
+ response.sort(SnapshotInfoResponse::compareTo);
+ return Pair.newPair(PagingUtil.cutPage(response, 0, offsetAndLimit.getSecond()), actualTableSize);
+ } else {
+ // Here the positive order needs to be cut from the offset position backwards
+ Comparator<SnapshotInfoResponse> comparator = BasicService.propertyComparator(sortBy, !isReversed);
+ response.sort(comparator);
+ return Pair.newPair(PagingUtil.cutPage(response, offsetAndLimit.getFirst(), offsetAndLimit.getSecond()), actualTableSize);
+ }
+ }
+
+ public Set<String> getAclAuthorizedTables(String project, boolean canUseACLGreenChannel) {
Set<String> authorizedTables = new HashSet<>();
if (!canUseACLGreenChannel) {
authorizedTables = getAuthorizedTables(project, getManager(AclTCRManager.class, project));
}
- Set<String> finalAuthorizedTables = authorizedTables;
- List<TableDesc> tables = nTableMetadataManager.listAllTables().stream().filter(tableDesc -> {
+ return authorizedTables;
+ }
+
+ public List<TableDesc> getFilteredTables(NTableMetadataManager nTableMetadataManager,
+ Pair<String, String> databaseAndTable, boolean canUseACLGreenChannel, Set<String> finalAuthorizedTables,
+ List<AbstractExecutable> executables, Set<SnapshotStatus> statusFilter, Set<Boolean> partitionFilter) {
+ String finalDatabase = databaseAndTable.getFirst();
+ String finalTable = databaseAndTable.getSecond();
+ return nTableMetadataManager.listAllTables().stream().filter(tableDesc -> {
if (StringUtils.isEmpty(finalDatabase)) {
return true;
}
@@ -482,36 +522,16 @@ public class SnapshotService extends BasicService implements SnapshotSupporter {
if (canUseACLGreenChannel) {
return true;
}
-
return finalAuthorizedTables.contains(tableDesc.getIdentity());
- }).filter(tableDesc -> hasLoadedSnapshot(tableDesc, executables)).collect(Collectors.toList());
-
- List<SnapshotInfoResponse> response = new ArrayList<>();
- tables.forEach(tableDesc -> {
- TableExtDesc tableExtDesc = nTableMetadataManager.getOrCreateTableExt(tableDesc);
- Pair<Integer, Integer> countPair = getModelCount(tableDesc);
- response.add(new SnapshotInfoResponse(tableDesc, tableExtDesc, tableDesc.getSnapshotTotalRows(), countPair.getFirst(),
- countPair.getSecond(), getSnapshotJobStatus(tableDesc, executables),
- getForbiddenColumns(tableDesc)));
- });
-
- if (!statusFilter.isEmpty()) {
- response.removeIf(res -> !statusFilter.contains(res.getStatus()));
- }
- if (partitionFilter.size() == 1) {
+ }).filter(tableDesc -> hasLoadedSnapshot(tableDesc, executables)
+ ).filter(tableDesc -> statusFilter.isEmpty() || statusFilter.contains(getSnapshotJobStatus(tableDesc, executables))
+ ).filter(tableDesc -> {
+ if (partitionFilter.size() != 1) {
+ return true;
+ }
boolean isPartition = partitionFilter.iterator().next();
- response.removeIf(res -> isPartition == (res.getSelectPartitionCol() == null));
- }
-
- sortBy = StringUtils.isEmpty(sortBy) ? "last_modified_time" : sortBy;
- if ("last_modified_time".equalsIgnoreCase(sortBy) && isReversed) {
- response.sort(SnapshotInfoResponse::compareTo);
- } else {
- Comparator<SnapshotInfoResponse> comparator = BasicService.propertyComparator(sortBy, !isReversed);
- response.sort(comparator);
- }
-
- return response;
+ return isPartition != (tableDesc.getSelectedSnapshotPartitionCol() == null);
+ }).collect(Collectors.toList());
}
private Pair<Integer, Integer> getModelCount(TableDesc tableDesc) {
diff --git a/src/data-loading-service/src/test/java/org/apache/kylin/rest/service/SnapshotServiceTest.java b/src/data-loading-service/src/test/java/org/apache/kylin/rest/service/SnapshotServiceTest.java
index 73334f8b0f..16032ee326 100644
--- a/src/data-loading-service/src/test/java/org/apache/kylin/rest/service/SnapshotServiceTest.java
+++ b/src/data-loading-service/src/test/java/org/apache/kylin/rest/service/SnapshotServiceTest.java
@@ -485,12 +485,12 @@ public class SnapshotServiceTest extends NLocalFileMetadataTestCase {
SecurityContextHolder.getContext()
.setAuthentication(new TestingAuthenticationToken("testuser", "testuser", Constant.ROLE_MODELER));
List<SnapshotInfoResponse> responses = snapshotService.getProjectSnapshots(PROJECT, tablePattern, statusFilter,
- Sets.newHashSet(), sortBy, true);
+ Sets.newHashSet(), sortBy, true, Pair.newPair(0, 10)).getFirst();
Assert.assertEquals(0, responses.size());
SecurityContextHolder.getContext()
.setAuthentication(new TestingAuthenticationToken("ADMIN", "ADMIN", Constant.ROLE_ADMIN));
responses = snapshotService.getProjectSnapshots(PROJECT, tablePattern, statusFilter, Sets.newHashSet(), sortBy,
- true);
+ true, Pair.newPair(0, 2)).getFirst();
SnapshotInfoResponse response = responses.get(0);
Assert.assertEquals(2, responses.size());
Assert.assertEquals("SSB", response.getDatabase());
@@ -498,6 +498,102 @@ public class SnapshotServiceTest extends NLocalFileMetadataTestCase {
responses.stream().map(SnapshotInfoResponse::getTable).collect(Collectors.toSet()));
}
+ @Test
+ public void testGetProjectSnapshotsReturn() {
+ enableSnapshotManualManagement();
+ setSnapshotPath("SSB.LINEORDER", "some_path");
+ setSnapshotPath("SSB.P_LINEORDER", "some_path");
+ getTestConfig().setProperty("kylin.query.security.acl-tcr-enabled", "true");
+ SecurityContextHolder.getContext()
+ .setAuthentication(new TestingAuthenticationToken("ADMIN", "ADMIN", Constant.ROLE_ADMIN));
+
+ // default sort
+ Pair<List<SnapshotInfoResponse>, Integer> projectSnapshots = snapshotService.getProjectSnapshots(PROJECT, "SSB",
+ Sets.newHashSet(SnapshotStatus.ONLINE), Sets.newHashSet(), "", true,
+ Pair.newPair(0, 1));
+ Assert.assertEquals(1, projectSnapshots.getFirst().size());
+ Assert.assertEquals(2, projectSnapshots.getSecond().intValue());
+
+ // default sort but reverse
+ projectSnapshots = snapshotService.getProjectSnapshots(PROJECT, "SSB",
+ Sets.newHashSet(SnapshotStatus.ONLINE), Sets.newHashSet(), "", false,
+ Pair.newPair(0, 1));
+ Assert.assertEquals(1, projectSnapshots.getFirst().size());
+ Assert.assertEquals(2, projectSnapshots.getSecond().intValue());
+
+ // sort by table
+ projectSnapshots = snapshotService.getProjectSnapshots(PROJECT, "SSB",
+ Sets.newHashSet(SnapshotStatus.ONLINE), Sets.newHashSet(), "table", true,
+ Pair.newPair(0, 1));
+ Assert.assertEquals(1, projectSnapshots.getFirst().size());
+ Assert.assertEquals(2, projectSnapshots.getSecond().intValue());
+
+ // sort by table not reverse
+ projectSnapshots = snapshotService.getProjectSnapshots(PROJECT, "SSB",
+ Sets.newHashSet(SnapshotStatus.ONLINE), Sets.newHashSet(), "table", false,
+ Pair.newPair(0, 1));
+ Assert.assertEquals(1, projectSnapshots.getFirst().size());
+ Assert.assertEquals(2, projectSnapshots.getSecond().intValue());
+ }
+
+ @Test
+ public void testGetProjectSnapshotsFilter() {
+ enableSnapshotManualManagement();
+ setSnapshotPath("SSB.LINEORDER", "some_path");
+ setSnapshotPath("SSB.P_LINEORDER", "some_path");
+ getTestConfig().setProperty("kylin.query.security.acl-tcr-enabled", "true");
+ SecurityContextHolder.getContext()
+ .setAuthentication(new TestingAuthenticationToken("ADMIN", "ADMIN", Constant.ROLE_ADMIN));
+
+ // status empty
+ Pair<List<SnapshotInfoResponse>, Integer> projectSnapshots = snapshotService.getProjectSnapshots(PROJECT, "SSB",
+ Sets.newHashSet(), Sets.newHashSet(), "", true,
+ Pair.newPair(0, 1));
+ Assert.assertEquals(1, projectSnapshots.getFirst().size());
+
+ // sort by table and status broken
+ projectSnapshots = snapshotService.getProjectSnapshots(PROJECT, "SSB",
+ Sets.newHashSet(SnapshotStatus.BROKEN), Sets.newHashSet(), "table", true,
+ Pair.newPair(0, 1));
+ Assert.assertEquals(0, projectSnapshots.getFirst().size());
+
+ // partitionFilter false and tableSelectedSnapshotPartitionCol is null
+ projectSnapshots = snapshotService.getProjectSnapshots(PROJECT, "SSB",
+ Sets.newHashSet(), Sets.newHashSet(false), "table", true,
+ Pair.newPair(0, 1));
+ Assert.assertEquals(1, projectSnapshots.getFirst().size());
+
+ // partitionFilter false and tableSelectedSnapshotPartitionCol is not null
+ String partColName = "LO_ORDERKEY";
+ String tableName = "SSB.LINEORDER";
+ snapshotService.configSnapshotPartitionCol(PROJECT,
+ ImmutableMap.<String, String> builder().put(tableName, partColName).build());
+ TableDesc tableDesc = NTableMetadataManager.getInstance(getTestConfig(), PROJECT).getTableDesc(tableName);
+ Assert.assertEquals(partColName, tableDesc.getSelectedSnapshotPartitionCol());
+ projectSnapshots = snapshotService.getProjectSnapshots(PROJECT, "SSB",
+ Sets.newHashSet(), Sets.newHashSet(false), "table", true,
+ Pair.newPair(0, 1));
+ Assert.assertEquals(1, projectSnapshots.getFirst().size());
+
+ // partitionFilter true and tableSelectedSnapshotPartitionCol is null
+ projectSnapshots = snapshotService.getProjectSnapshots(PROJECT, "SSB",
+ Sets.newHashSet(), Sets.newHashSet(true), "table", true,
+ Pair.newPair(0, 1));
+ Assert.assertEquals(1, projectSnapshots.getFirst().size());
+
+ // partitionFilter true and tableSelectedSnapshotPartitionCol is not null
+ projectSnapshots = snapshotService.getProjectSnapshots(PROJECT, "SSB",
+ Sets.newHashSet(), Sets.newHashSet(true), "table", true,
+ Pair.newPair(0, 1));
+ Assert.assertEquals(1, projectSnapshots.getFirst().size());
+ }
+
+ @Test
+ public void testCheckDatabaseAndTable() {
+ Pair<String, String> tableAndDatabase = Pair.newPair("SSB", "CUSTOM");
+ Assert.assertEquals(tableAndDatabase, snapshotService.checkDatabaseAndTable("SSB.CUSTOM"));
+ }
+
@Test
public void testGetTables() {
enableSnapshotManualManagement();
@@ -669,7 +765,8 @@ public class SnapshotServiceTest extends NLocalFileMetadataTestCase {
return null;
}, PROJECT);
List<SnapshotInfoResponse> responses = snapshotService.getProjectSnapshots(PROJECT, null,
- Sets.newHashSet(SnapshotStatus.BROKEN), Sets.newHashSet(), null, true);
+ Sets.newHashSet(SnapshotStatus.BROKEN), Sets.newHashSet(), null, true,
+ Pair.newPair(0, 10)).getFirst();
Assert.assertEquals(1, responses.size());
}
@@ -729,7 +826,8 @@ public class SnapshotServiceTest extends NLocalFileMetadataTestCase {
return null;
}, PROJECT);
List<SnapshotInfoResponse> responses = snapshotService.getProjectSnapshots(PROJECT, tableName,
- Sets.newHashSet(SnapshotStatus.BROKEN), Sets.newHashSet(), null, true);
+ Sets.newHashSet(SnapshotStatus.BROKEN), Sets.newHashSet(), null, true,
+ Pair.newPair(0, 10)).getFirst();
Assert.assertEquals(1, responses.size());
Assert.assertEquals(10, responses.get(0).getUsage());
}
diff --git a/src/datasource-service/src/main/java/org/apache/kylin/rest/request/TableDescRequest.java b/src/datasource-service/src/main/java/org/apache/kylin/rest/request/TableDescRequest.java
new file mode 100644
index 0000000000..522cff00e3
--- /dev/null
+++ b/src/datasource-service/src/main/java/org/apache/kylin/rest/request/TableDescRequest.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.rest.request;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import org.apache.kylin.common.util.Pair;
+
+import java.util.List;
+
+@NoArgsConstructor
+@AllArgsConstructor
+@Data
+public class TableDescRequest {
+
+ @JsonProperty(required = true)
+ private String project;
+
+ private String table;
+
+ private String database;
+
+ @JsonProperty(value = "ext")
+ private boolean withExt;
+
+ @JsonProperty(value = "is_fuzzy", defaultValue = "false")
+ private boolean isFuzzy;
+
+ @JsonProperty(value = "page_offset", defaultValue = "0")
+ private Integer offset;
+
+ @JsonProperty(value = "page_size", defaultValue = "10")
+ private Integer limit;
+
+ @JsonProperty(value = "source_type", defaultValue = "9")
+ private List<Integer> sourceType;
+
+ @JsonProperty(value = "with_excluded", defaultValue = "true")
+ private boolean withExcluded;
+
+ public TableDescRequest(String project, boolean withExt, String table, String database, boolean isFuzzy, List<Integer> sourceType) {
+ this.project = project;
+ this.withExt = withExt;
+ this.table = table;
+ this.database = database;
+ this.isFuzzy = isFuzzy;
+ this.sourceType = sourceType;
+ }
+
+ public TableDescRequest(String project, String table, Integer offset, Integer limit, boolean withExcluded, List<Integer> sourceType) {
+ this.project = project;
+ this.table = table;
+ this.offset = offset;
+ this.limit = limit;
+ this.withExcluded = withExcluded;
+ this.sourceType = sourceType;
+ }
+
+ public TableDescRequest(String project, String table, String database, boolean withExt, boolean isFuzzy,
+ Pair<Integer, Integer> offsetAndLimit, List<Integer> sourceType) {
+ this.project = project;
+ this.table = table;
+ this.database = database;
+ this.withExt = withExt;
+ this.isFuzzy = isFuzzy;
+ this.offset = offsetAndLimit.getFirst();
+ this.limit = offsetAndLimit.getSecond();
+ this.sourceType = sourceType;
+ }
+}
diff --git a/src/datasource-service/src/main/java/org/apache/kylin/rest/service/TableService.java b/src/datasource-service/src/main/java/org/apache/kylin/rest/service/TableService.java
index 436a030671..950469188d 100644
--- a/src/datasource-service/src/main/java/org/apache/kylin/rest/service/TableService.java
+++ b/src/datasource-service/src/main/java/org/apache/kylin/rest/service/TableService.java
@@ -34,6 +34,7 @@ import static org.apache.kylin.common.exception.code.ErrorCodeServer.TABLE_RELOA
import static org.apache.kylin.common.exception.code.ErrorCodeServer.TABLE_RELOAD_MODEL_RETRY;
import static org.apache.kylin.job.execution.JobTypeEnum.SNAPSHOT_BUILD;
import static org.apache.kylin.job.execution.JobTypeEnum.SNAPSHOT_REFRESH;
+import static org.apache.kylin.rest.util.TableUtils.calculateTableSize;
import java.io.File;
import java.io.IOException;
@@ -139,6 +140,7 @@ import org.apache.kylin.rest.constant.JobInfoEnum;
import org.apache.kylin.rest.request.AutoMergeRequest;
import org.apache.kylin.rest.request.DateRangeRequest;
import org.apache.kylin.rest.request.S3TableExtInfo;
+import org.apache.kylin.rest.request.TableDescRequest;
import org.apache.kylin.rest.response.AutoMergeConfigResponse;
import org.apache.kylin.rest.response.BatchLoadTableResponse;
import org.apache.kylin.rest.response.EnvelopeResponse;
@@ -226,44 +228,43 @@ public class TableService extends BasicService {
@Autowired
private ClusterManager clusterManager;
- public List<TableDesc> getTableDescByType(String project, boolean withExt, final String tableName,
- final String database, boolean isFuzzy, int sourceType) throws IOException {
- return getTableDesc(project, withExt, tableName, database, isFuzzy).stream()
- .filter(tableDesc -> sourceType == tableDesc.getSourceType()).collect(Collectors.toList());
+ public Pair<List<TableDesc>, Integer> getTableDesc(String project, boolean withExt, final String table, final String database,
+ boolean isFuzzy, List<Integer> sourceType, int returnTableSize) throws IOException {
+ TableDescRequest internalTableDescRequest = new TableDescRequest(project, withExt, table, database, isFuzzy, sourceType);
+ return getTableDesc(internalTableDescRequest, returnTableSize);
}
- public List<TableDesc> getTableDescByTypes(String project, boolean withExt, final String tableName,
- final String database, boolean isFuzzy, List<Integer> sourceType) throws IOException {
- return getTableDesc(project, withExt, tableName, database, isFuzzy).stream()
- .filter(tableDesc -> sourceType.contains(tableDesc.getSourceType())).collect(Collectors.toList());
- }
-
- public List<TableDesc> getTableDesc(String project, boolean withExt, final String tableName, final String database,
- boolean isFuzzy) throws IOException {
- aclEvaluate.checkProjectReadPermission(project);
+ public Pair<List<TableDesc>, Integer> getTableDesc(TableDescRequest tableDescRequest, int returnTableSize) throws IOException {
+ aclEvaluate.checkProjectReadPermission(tableDescRequest.getProject());
boolean streamingEnabled = getConfig().streamingEnabled();
- NTableMetadataManager nTableMetadataManager = getManager(NTableMetadataManager.class, project);
+ NTableMetadataManager nTableMetadataManager = getManager(NTableMetadataManager.class, tableDescRequest.getProject());
List<TableDesc> tables = Lists.newArrayList();
//get table not fuzzy,can use getTableDesc(tableName)
- if (StringUtils.isNotEmpty(tableName) && !isFuzzy) {
- val tableDesc = nTableMetadataManager.getTableDesc(database + "." + tableName);
+ if (StringUtils.isNotEmpty(tableDescRequest.getTable()) && !tableDescRequest.isFuzzy()) {
+ val tableDesc = nTableMetadataManager.getTableDesc(tableDescRequest.getDatabase() + "." + tableDescRequest.getTable());
if (tableDesc != null && tableDesc.isAccessible(streamingEnabled))
tables.add(tableDesc);
} else {
tables.addAll(nTableMetadataManager.listAllTables().stream().filter(tableDesc -> {
- if (StringUtils.isEmpty(database)) {
+ if (StringUtils.isEmpty(tableDescRequest.getDatabase())) {
return true;
}
- return tableDesc.getDatabase().equalsIgnoreCase(database);
+ return tableDesc.getDatabase().equalsIgnoreCase(tableDescRequest.getDatabase());
}).filter(tableDesc -> {
- if (StringUtils.isEmpty(tableName)) {
+ if (StringUtils.isEmpty(tableDescRequest.getTable())) {
return true;
}
- return tableDesc.getName().toLowerCase(Locale.ROOT).contains(tableName.toLowerCase(Locale.ROOT));
+ return tableDesc.getName().toLowerCase(Locale.ROOT).contains(tableDescRequest.getTable().toLowerCase(Locale.ROOT));
+ }).filter(tableDesc -> {
+ // Advance the logic of filtering the table by sourceType to here
+ if (!tableDescRequest.getSourceType().isEmpty()) {
+ return tableDescRequest.getSourceType().contains(tableDesc.getSourceType());
+ }
+ return true;
}).filter(table -> table.isAccessible(streamingEnabled)).sorted(this::compareTableDesc)
.collect(Collectors.toList()));
}
- return getTablesResponse(tables, project, withExt);
+ return getTablesResponse(tables, tableDescRequest.getProject(), tableDescRequest.isWithExt(), returnTableSize);
}
public int compareTableDesc(TableDesc table1, TableDesc table2) {
@@ -417,7 +418,10 @@ public class TableService extends BasicService {
return tableNameResponses;
}
- private TableDescResponse getTableResponse(TableDesc table, String project) {
+ private TableDescResponse getTableResponse(TableDesc table, String project, boolean withExt) {
+ if (!withExt) {
+ return new TableDescResponse(table);
+ }
TableDescResponse tableDescResponse = new TableDescResponse(table);
TableExtDesc tableExtDesc = getManager(NTableMetadataManager.class, project).getTableExtIfExists(table);
if (table.isKafkaTable()) {
@@ -447,7 +451,7 @@ public class TableService extends BasicService {
return tableDescResponse;
}
- private List<TableDesc> getTablesResponse(List<TableDesc> tables, String project, boolean withExt) {
+ private Pair<List<TableDesc>, Integer> getTablesResponse(List<TableDesc> tables, String project, boolean withExt, int returnTableSize) {
List<TableDesc> descs = new ArrayList<>();
val projectManager = getManager(NProjectManager.class);
val groups = getCurrentUserGroups();
@@ -458,23 +462,23 @@ public class TableService extends BasicService {
List<NDataModel> healthyModels = projectManager.listHealthyModels(project);
Set<String> extPermissionSet = accessService.getUserNormalExtPermissions(project);
boolean hasDataQueryPermission = extPermissionSet.contains(ExternalAclProvider.DATA_QUERY);
+ int satisfiedTableSize = 0;
for (val originTable : tables) {
+ // New judgment logic, when the total size of tables meet the current size of paging directly after the exit
+ // Also, if the processing is not finished, the total size of tables is returned
+ if (satisfiedTableSize == returnTableSize) {
+ return Pair.newPair(descs, tables.size());
+ }
TableDesc table = getAuthorizedTableDesc(project, isAclGreen, originTable, aclTCRS);
if (Objects.isNull(table)) {
continue;
}
- TableDescResponse tableDescResponse;
+ TableDescResponse tableDescResponse = getTableResponse(table, project, withExt);
List<NDataModel> modelsUsingTable = healthyModels.stream() //
.filter(model -> model.containsTable(table)).collect(Collectors.toList());
List<NDataModel> modelsUsingRootTable = healthyModels.stream() //
.filter(model -> model.isRootFactTable(table)).collect(Collectors.toList());
- if (withExt) {
- tableDescResponse = getTableResponse(table, project);
- } else {
- tableDescResponse = new TableDescResponse(table);
- }
-
TableExtDesc tableExtDesc = getManager(NTableMetadataManager.class, project).getTableExtIfExists(table);
if (tableExtDesc != null) {
tableDescResponse.setTotalRecords(tableExtDesc.getTotalRows());
@@ -496,9 +500,9 @@ public class TableService extends BasicService {
tableDescResponse.setForeignKey(tableColumnType.getSecond());
tableDescResponse.setPrimaryKey(tableColumnType.getFirst());
descs.add(tableDescResponse);
+ satisfiedTableSize++;
}
-
- return descs;
+ return Pair.newPair(descs, descs.size());
}
@VisibleForTesting
@@ -1780,22 +1784,23 @@ public class TableService extends BasicService {
return loadedDatabases;
}
- public interface ProjectTablesFilter {
- List process(String database, String table) throws Exception;
+ public NInitTablesResponse getProjectTables(String project, String table, int offset, int limit,
+ boolean withExcluded, boolean useHiveDatabase, List<Integer> sourceType) throws Exception {
+ TableDescRequest internalTableDescRequest = new TableDescRequest(project, table, offset, limit, withExcluded, sourceType);
+ return getProjectTables(internalTableDescRequest, useHiveDatabase);
}
- public NInitTablesResponse getProjectTables(String project, String table, int offset, int limit,
- boolean withExcluded, boolean useHiveDatabase, ProjectTablesFilter projectTablesFilter) throws Exception {
+ public NInitTablesResponse getProjectTables(TableDescRequest tableDescRequest, boolean useHiveDatabase) throws Exception {
+ String project = tableDescRequest.getProject();
aclEvaluate.checkProjectReadPermission(project);
NInitTablesResponse response = new NInitTablesResponse();
- logger.debug("only get project tables of excluded: {}", withExcluded);
- if (table == null)
- table = "";
- String exceptDatabase = null;
- if (table.contains(".")) {
- exceptDatabase = table.split("\\.", 2)[0].trim();
- table = table.split("\\.", 2)[1].trim();
- }
+ logger.debug("only get project tables of excluded: {}", tableDescRequest.isWithExcluded());
+
+ Pair<String, String> databaseAndTable = checkDatabaseAndTable(tableDescRequest.getTable());
+ String exceptDatabase = databaseAndTable.getFirst();
+ String table = databaseAndTable.getSecond();
+ String notAllowedModifyTableName = table;
+
Collection<String> databases = useHiveDatabase ? getSourceDbNames(project) : getLoadedDatabases(project);
val projectInstance = getManager(NProjectManager.class).getProject(project);
List<String> tableFilterList = DataSourceState.getInstance().getHiveFilterList(projectInstance);
@@ -1804,15 +1809,29 @@ public class TableService extends BasicService {
|| (!tableFilterList.isEmpty() && !tableFilterList.contains(database))) {
continue;
}
- List<?> tables;
+ // we may temporarily change the table name, but later to change back
+ // Avoid affecting the next loop and causing logic errors
if (exceptDatabase == null && database.toLowerCase(Locale.ROOT).contains(table.toLowerCase(Locale.ROOT))) {
- tables = projectTablesFilter.process(database, "");
+ table = "";
+ }
+ tableDescRequest.setDatabase(database);
+ tableDescRequest.setTable(table);
+ Pair<List<?>, Integer> objWithActualSize = new Pair<>();
+ if (tableDescRequest.getSourceType().isEmpty()) {
+ // This means request api for showProjectTableNames
+ List<TableNameResponse> hiveTableNameResponses = getHiveTableNameResponses(project, database, table);
+ objWithActualSize.setFirst(hiveTableNameResponses);
+ objWithActualSize.setSecond(hiveTableNameResponses.size());
} else {
- tables = projectTablesFilter.process(database, table);
+ int returnTableSize = calculateTableSize(tableDescRequest.getOffset(), tableDescRequest.getLimit());
+ Pair<List<TableDesc>, Integer> tableDescWithActualSize = getTableDesc(tableDescRequest, returnTableSize);
+ objWithActualSize.setFirst(tableDescWithActualSize.getFirst());
+ objWithActualSize.setSecond(tableDescWithActualSize.getSecond());
}
- List<?> tablePage = PagingUtil.cutPage(tables, offset, limit);
+ table = notAllowedModifyTableName;
+ List<?> tablePage = PagingUtil.cutPage(objWithActualSize.getFirst(), tableDescRequest.getOffset(), tableDescRequest.getLimit());
if (!tablePage.isEmpty()) {
- response.putDatabase(database, tables.size(), tablePage);
+ response.putDatabase(database, objWithActualSize.getSecond(), tablePage);
}
}
return response;
diff --git a/src/datasource-service/src/main/java/org/apache/kylin/rest/util/TableUtils.java b/src/datasource-service/src/main/java/org/apache/kylin/rest/util/TableUtils.java
index 4e44101e67..c2ed932052 100644
--- a/src/datasource-service/src/main/java/org/apache/kylin/rest/util/TableUtils.java
+++ b/src/datasource-service/src/main/java/org/apache/kylin/rest/util/TableUtils.java
@@ -42,4 +42,21 @@ public class TableUtils {
}
}
+ /**
+ * <p>
+ * Calculate the number of valid tables to be returned based on pageOffset and pageSize
+ * Note: Tables will be filtered under certain conditions, but the final result must still be the number of valid tables,
+ * unless all tables have been processed.
+ * For example:
+ * the first page: pageOffset 0, pageSize 7, return 0 * 7 + 7 = 7
+ * The second page: pageOffset 1, pageSize 7, return 1 * 7 + 7 = 14
+ * </p>
+ *
+ * @param pageOffset page offset
+ * @param pageSize page size
+ * @return Number of valid tables
+ */
+ public static int calculateTableSize(int pageOffset, int pageSize) {
+ return pageOffset * pageSize + pageSize;
+ }
}
diff --git a/src/datasource-service/src/test/java/org/apache/kylin/rest/service/StreamingTableServiceTest.java b/src/datasource-service/src/test/java/org/apache/kylin/rest/service/StreamingTableServiceTest.java
index 75f774865f..a5f92508b0 100644
--- a/src/datasource-service/src/test/java/org/apache/kylin/rest/service/StreamingTableServiceTest.java
+++ b/src/datasource-service/src/test/java/org/apache/kylin/rest/service/StreamingTableServiceTest.java
@@ -48,6 +48,7 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Mockito;
@@ -59,6 +60,8 @@ import org.springframework.test.util.ReflectionTestUtils;
import lombok.val;
public class StreamingTableServiceTest extends NLocalFileMetadataTestCase {
+ @Rule
+ public TemporaryFolder temporaryFolder = new TemporaryFolder();
@Mock
private AclUtil aclUtil = Mockito.spy(AclUtil.class);
@@ -115,7 +118,7 @@ public class StreamingTableServiceTest extends NLocalFileMetadataTestCase {
val prj = prjManager.getProject(PROJECT);
val copy = prjManager.copyForWrite(prj);
prjManager.updateProject(copy);
- Mockito.when(userService.listSuperAdminUsers()).thenReturn(Collections.singletonList("admin"));
+ Mockito.when(userService.listSuperAdminUsers()).thenReturn(Arrays.asList("admin"));
Mockito.when(userAclService.hasUserAclPermissionInProject(Mockito.anyString(), Mockito.anyString()))
.thenReturn(false);
@@ -135,8 +138,12 @@ public class StreamingTableServiceTest extends NLocalFileMetadataTestCase {
@Test
public void testInnerReloadTable() {
+ val database = "SSB";
+
+ val config = getTestConfig();
try {
- val tableDescList = tableService.getTableDesc(PROJECT, true, "P_LINEORDER_STR", "SSB", false);
+ val tableDescList = tableService.getTableDesc(PROJECT, true, "P_LINEORDER_STR", database, false,
+ Collections.emptyList(), 10).getFirst();
Assert.assertEquals(1, tableDescList.size());
val tableDesc = tableDescList.get(0);
val tableExtDesc = tableService.getOrCreateTableExt(PROJECT, tableDesc);
@@ -152,7 +159,8 @@ public class StreamingTableServiceTest extends NLocalFileMetadataTestCase {
val database = "DEFAULT";
try {
- val tableDescList = tableService.getTableDesc(PROJECT, true, "", database, true);
+ val tableDescList = tableService.getTableDesc(PROJECT, true, "", database, true,
+ Collections.emptyList(), 10).getFirst();
Assert.assertEquals(2, tableDescList.size());
val tableDesc = tableDescList.get(0);
val tableExtDesc = tableService.getOrCreateTableExt(PROJECT, tableDesc);
diff --git a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NTableController.java b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NTableController.java
index 128a9ebb21..9466eb6374 100644
--- a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NTableController.java
+++ b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NTableController.java
@@ -24,9 +24,10 @@ import static org.apache.kylin.common.exception.ServerErrorCode.EMPTY_PARAMETER;
import static org.apache.kylin.common.exception.ServerErrorCode.INVALID_TABLE_NAME;
import static org.apache.kylin.common.exception.ServerErrorCode.INVALID_TABLE_REFRESH_PARAMETER;
import static org.apache.kylin.common.exception.code.ErrorCodeServer.PROJECT_NOT_EXIST;
+import static org.apache.kylin.rest.util.TableUtils.calculateTableSize;
import java.io.IOException;
-import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -40,6 +41,7 @@ import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.msg.MsgPicker;
+import org.apache.kylin.common.util.Pair;
import org.apache.kylin.common.util.StringUtil;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.project.NProjectManager;
@@ -48,6 +50,7 @@ import org.apache.kylin.rest.request.AutoMergeRequest;
import org.apache.kylin.rest.request.PartitionKeyRequest;
import org.apache.kylin.rest.request.PushDownModeRequest;
import org.apache.kylin.rest.request.ReloadTableRequest;
+import org.apache.kylin.rest.request.TableDescRequest;
import org.apache.kylin.rest.request.TableExclusionRequest;
import org.apache.kylin.rest.request.TableLoadRequest;
import org.apache.kylin.rest.request.TopTableRequest;
@@ -114,7 +117,7 @@ public class NTableController extends NBasicController {
"AI" }, notes = "Update Param: is_fuzzy, page_offset, page_size; Update Response: no format!")
@GetMapping(value = "", produces = { HTTP_VND_APACHE_KYLIN_JSON })
@ResponseBody
- public EnvelopeResponse getTableDesc(@RequestParam(value = "ext", required = false) boolean withExt,
+ public EnvelopeResponse<Map<String, Object>> getTableDesc(@RequestParam(value = "ext", required = false) boolean withExt,
@RequestParam(value = "project") String project,
@RequestParam(value = "table", required = false) String table,
@RequestParam(value = "database", required = false) String database,
@@ -125,11 +128,16 @@ public class NTableController extends NBasicController {
throws IOException {
checkProjectName(project);
- List<TableDesc> tableDescs = new ArrayList<>();
-
- tableDescs.addAll(tableService.getTableDescByType(project, withExt, table, database, isFuzzy, sourceType));
- return new EnvelopeResponse<>(KylinException.CODE_SUCCESS, getDataResponse("tables", tableDescs, offset, limit),
- "");
+ // In addition to the tables that have been processed, the actual size of tables should be returned,
+ // so that the front-end UI knows whether to show more presses to be loaded
+ int returnTableSize = calculateTableSize(offset, limit);
+ TableDescRequest tableDescRequest = new TableDescRequest(project, table, database, withExt, isFuzzy,
+ Pair.newPair(offset, limit), Collections.singletonList(sourceType));
+ Pair<List<TableDesc>, Integer> tableDescWithActualSize = tableService.getTableDesc(tableDescRequest, returnTableSize);
+ // Finally, the results are processed based on the paging parameters and returned to the front-end UI,
+ // where the results table to be processed each time is getting longer as the number of paging increases
+ Map<String, Object> mockDataResponse = setCustomDataResponse("tables", tableDescWithActualSize, offset, limit);
+ return new EnvelopeResponse<>(KylinException.CODE_SUCCESS, mockDataResponse, "");
}
@ApiOperation(value = "getProjectTables", tags = { "AI" }, notes = "Update Param: is_fuzzy, page_offset, page_size")
@@ -146,13 +154,10 @@ public class NTableController extends NBasicController {
@RequestParam(value = "with_excluded", required = false, defaultValue = "true") boolean withExcluded,
@RequestParam(value = "source_type", required = false, defaultValue = "9") List<Integer> sourceType)
throws Exception {
-
- String projectName = checkProjectName(project);
- NInitTablesResponse projectTables = tableService.getProjectTables(projectName, table, offset, limit,
- withExcluded, false, (databaseName, tableName) -> {
- return tableService.getTableDescByTypes(projectName, withExt, tableName, databaseName, isFuzzy,
- sourceType);
- });
+ checkProjectName(project);
+ TableDescRequest tableDescRequest = new TableDescRequest(project, table, "", withExt, isFuzzy,
+ offset, limit, sourceType, withExcluded);
+ NInitTablesResponse projectTables = tableService.getProjectTables(tableDescRequest, false);
return new EnvelopeResponse<>(KylinException.CODE_SUCCESS, projectTables, "");
}
@@ -346,10 +351,9 @@ public class NTableController extends NBasicController {
@RequestParam(value = "page_offset", required = false, defaultValue = "0") Integer offset,
@RequestParam(value = "page_size", required = false, defaultValue = "10") Integer limit) throws Exception {
String projectName = checkProjectName(project);
- return new EnvelopeResponse<>(KylinException.CODE_SUCCESS,
- tableService.getProjectTables(projectName, table, offset, limit, true, true, (databaseName,
- tableName) -> tableService.getHiveTableNameResponses(projectName, databaseName, tableName)),
- "");
+ NInitTablesResponse data = tableService.getProjectTables(projectName, table, offset, limit, true,
+ true, Collections.emptyList());
+ return new EnvelopeResponse<>(KylinException.CODE_SUCCESS, data, "");
}
@ApiOperation(value = "getTablesAndColumns", tags = {
diff --git a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/open/OpenTableController.java b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/open/OpenTableController.java
index b511f565ae..a135f4dcaa 100644
--- a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/open/OpenTableController.java
+++ b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/open/OpenTableController.java
@@ -22,8 +22,10 @@ import static org.apache.kylin.common.exception.ServerErrorCode.INVALID_TABLE_NA
import static org.apache.kylin.common.exception.ServerErrorCode.UNSUPPORTED_DATA_SOURCE_TYPE;
import static org.apache.kylin.common.exception.ServerErrorCode.UNSUPPORTED_STREAMING_OPERATION;
import static org.apache.kylin.common.exception.code.ErrorCodeServer.JOB_SAMPLING_RANGE_INVALID;
+import static org.apache.kylin.rest.util.TableUtils.calculateTableSize;
import java.io.IOException;
+import java.util.Collections;
import java.util.List;
import java.util.Locale;
@@ -119,10 +121,11 @@ public class OpenTableController extends NBasicController {
throw new KylinException(UNSUPPORTED_STREAMING_OPERATION,
MsgPicker.getMsg().getStreamingOperationNotSupport());
}
- List<TableDesc> result = tableService.getTableDescByType(project, withExt,
+ int returnTableSize = calculateTableSize(offset, limit);
+ Pair<List<TableDesc>, Integer> tableDescWithActualSize = tableService.getTableDesc(project, withExt,
StringUtils.upperCase(table, Locale.ROOT), StringUtils.upperCase(database, Locale.ROOT), isFuzzy,
- sourceType);
- return new EnvelopeResponse<>(KylinException.CODE_SUCCESS, DataResult.get(result, offset, limit), "");
+ Collections.singletonList(sourceType), returnTableSize);
+ return new EnvelopeResponse<>(KylinException.CODE_SUCCESS, DataResult.getCustom(tableDescWithActualSize, offset, limit), "");
}
@ApiOperation(value = "loadTables", tags = { "AI" })
diff --git a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/v2/NTableControllerV2.java b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/v2/NTableControllerV2.java
index 0c195a84f4..1b4b462b63 100644
--- a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/v2/NTableControllerV2.java
+++ b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/v2/NTableControllerV2.java
@@ -19,12 +19,15 @@ package org.apache.kylin.rest.controller.v2;
import static org.apache.kylin.common.constant.HttpConstant.HTTP_VND_APACHE_KYLIN_V2_JSON;
import static org.apache.kylin.common.exception.ServerErrorCode.UNSUPPORTED_STREAMING_OPERATION;
+import static org.apache.kylin.rest.util.TableUtils.calculateTableSize;
import java.io.IOException;
+import java.util.Collections;
import java.util.List;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.msg.MsgPicker;
+import org.apache.kylin.common.util.Pair;
import org.apache.kylin.metadata.model.ISourceAware;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.rest.controller.NBasicController;
@@ -65,9 +68,11 @@ public class NTableControllerV2 extends NBasicController {
throw new KylinException(UNSUPPORTED_STREAMING_OPERATION,
MsgPicker.getMsg().getStreamingOperationNotSupport());
}
- List<TableDesc> result = tableService.getTableDescByType(project, withExt, table, database, isFuzzy,
- sourceType);
- return new EnvelopeResponse<>(KylinException.CODE_SUCCESS, DataResult.get(result, offset, limit).getValue(),
+ int returnTableSize = calculateTableSize(offset, limit);
+ Pair<List<TableDesc>, Integer> tableDescWithActualSize = tableService.getTableDesc(project, withExt, table, database,
+ isFuzzy, Collections.singletonList(sourceType), returnTableSize);
+ return new EnvelopeResponse<>(KylinException.CODE_SUCCESS,
+ DataResult.getCustom(tableDescWithActualSize, offset, limit).getValue(),
"");
}
}
\ No newline at end of file
diff --git a/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NTableControllerTest.java b/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NTableControllerTest.java
index 3301be1d06..ad10f6f199 100644
--- a/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NTableControllerTest.java
+++ b/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NTableControllerTest.java
@@ -24,6 +24,7 @@ import static org.apache.kylin.common.exception.code.ErrorCodeServer.JOB_SAMPLIN
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Set;
@@ -32,6 +33,7 @@ import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
+import org.apache.kylin.common.util.Pair;
import org.apache.kylin.common.util.StringUtil;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.rest.constant.Constant;
@@ -41,6 +43,7 @@ import org.apache.kylin.rest.request.PartitionKeyRequest;
import org.apache.kylin.rest.request.PushDownModeRequest;
import org.apache.kylin.rest.request.ReloadTableRequest;
import org.apache.kylin.rest.request.S3TableExtInfo;
+import org.apache.kylin.rest.request.TableDescRequest;
import org.apache.kylin.rest.request.TableExclusionRequest;
import org.apache.kylin.rest.request.TableLoadRequest;
import org.apache.kylin.rest.request.TopTableRequest;
@@ -149,8 +152,11 @@ public class NTableControllerTest extends NLocalFileMetadataTestCase {
@Test
public void testGetTableDesc() throws Exception {
- Mockito.when(tableService.getTableDesc("default", false, "", "DEFAULT", true)) //
- .thenReturn(mockTables());
+ TableDescRequest mockTableDescRequest = new TableDescRequest("default", "", "DEFAULT", false,
+ true, Pair.newPair(0, 10), Collections.singletonList(9));
+
+ Mockito.when(tableService.getTableDesc(mockTableDescRequest, 10)).thenReturn(Pair.newPair(mockTables(), 10));
+
mockMvc.perform(MockMvcRequestBuilders.get("/api/tables") //
.contentType(MediaType.APPLICATION_JSON) //
.param("ext", "false") //
@@ -181,8 +187,11 @@ public class NTableControllerTest extends NLocalFileMetadataTestCase {
@Test
public void testGetTableDescWithName() throws Exception {
- Mockito.when(tableService.getTableDesc("default", true, "TEST_KYLIN_FACT", "DEFAULT", false))
- .thenReturn(mockTables());
+ TableDescRequest mockTableDescRequest = new TableDescRequest("default", "TEST_KYLIN_FACT", "DEFAULT", false,
+ false, Pair.newPair(0, 10), Collections.singletonList(9));
+
+ Mockito.when(tableService.getTableDesc(mockTableDescRequest, 10)).thenReturn(Pair.newPair(mockTables(), 10));
+
mockMvc.perform(MockMvcRequestBuilders.get("/api/tables") //
.contentType(MediaType.APPLICATION_JSON) //
.param("withExt", "false") //
diff --git a/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NTableControllerV2Test.java b/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NTableControllerV2Test.java
index 6e1b3e608f..2392584e6c 100644
--- a/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NTableControllerV2Test.java
+++ b/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NTableControllerV2Test.java
@@ -21,9 +21,11 @@ import static org.apache.kylin.common.constant.HttpConstant.HTTP_VND_APACHE_KYLI
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
+import org.apache.kylin.common.util.Pair;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.rest.controller.v2.NTableControllerV2;
@@ -81,8 +83,9 @@ public class NTableControllerV2Test extends NLocalFileMetadataTestCase {
@Test
public void testGetTableDesc() throws Exception {
- Mockito.when(tableService.getTableDesc("default", false, "", "DEFAULT", true)) //
- .thenReturn(mockTables());
+ Mockito.when(tableService.getTableDesc("default", false, "", "DEFAULT", true, Collections.singletonList(9), 10)) //
+ .thenReturn(Pair.newPair(mockTables(), 10));
+
mockMvc.perform(MockMvcRequestBuilders.get("/api/tables") //
.contentType(MediaType.APPLICATION_JSON) //
.param("ext", "false") //
diff --git a/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/open/OpenTableControllerTest.java b/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/open/OpenTableControllerTest.java
index 6d56ebe1c2..1c2242f8d5 100644
--- a/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/open/OpenTableControllerTest.java
+++ b/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/open/OpenTableControllerTest.java
@@ -21,6 +21,7 @@ import static org.apache.kylin.common.constant.HttpConstant.HTTP_VND_APACHE_KYLI
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
import java.util.Map;
@@ -122,6 +123,9 @@ public class OpenTableControllerTest extends NLocalFileMetadataTestCase {
String tableName = "TEST_KYLIN_FACT";
String database = "DEFAULT";
+ Mockito.when(tableService.getTableDesc(project, true, tableName, database, false, Collections.singletonList(9), 10))
+ .thenReturn(Pair.newPair(Collections.singletonList(new TableDesc()), 10));
+
mockMvc.perform(MockMvcRequestBuilders.get("/api/tables") //
.contentType(MediaType.APPLICATION_JSON) //
.param("project", project).param("table", tableName).param("database", database)
@@ -136,6 +140,9 @@ public class OpenTableControllerTest extends NLocalFileMetadataTestCase {
String tableName = "P_LINEORDER_STR";
String database = "SSB";
+ Mockito.when(tableService.getTableDesc(project, true, tableName, database, false, Collections.singletonList(1), 10))
+ .thenReturn(Pair.newPair(Collections.singletonList(new TableDesc()), 10));
+
mockMvc.perform(MockMvcRequestBuilders.get("/api/tables") //
.contentType(MediaType.APPLICATION_JSON) //
.param("project", project).param("table", tableName).param("database", database)
@@ -154,29 +161,33 @@ public class OpenTableControllerTest extends NLocalFileMetadataTestCase {
String databaseLowercase = "ssb";
String databaseUppercase = "SSB";
+ Mockito.when(tableService.getTableDesc(project, true, tableNameUppercase, databaseUppercase, false,
+ Collections.singletonList(9), 10))
+ .thenReturn(Pair.newPair(Collections.singletonList(new TableDesc()), 10));
+
mockMvc.perform(MockMvcRequestBuilders.get("/api/tables") //
.contentType(MediaType.APPLICATION_JSON) //
.param("project", project).param("table", tableNameMixture).param("database", databaseMixture)
.accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON))) //
.andExpect(MockMvcResultMatchers.status().isOk());
- Mockito.verify(tableService, Mockito.times(1)).getTableDescByType(project, true, tableNameUppercase,
- databaseUppercase, false, 9);
+ Mockito.verify(tableService, Mockito.times(1)).getTableDesc(project, true, tableNameUppercase,
+ databaseUppercase, false, Collections.singletonList(9), 10);
mockMvc.perform(MockMvcRequestBuilders.get("/api/tables") //
.contentType(MediaType.APPLICATION_JSON) //
.param("project", project).param("table", tableNameLowerCase).param("database", databaseLowercase)
.accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON))) //
.andExpect(MockMvcResultMatchers.status().isOk());
- Mockito.verify(tableService, Mockito.times(2)).getTableDescByType(project, true, tableNameUppercase,
- databaseUppercase, false, 9);
+ Mockito.verify(tableService, Mockito.times(2)).getTableDesc(project, true, tableNameUppercase,
+ databaseUppercase, false, Collections.singletonList(9), 10);
mockMvc.perform(MockMvcRequestBuilders.get("/api/tables") //
.contentType(MediaType.APPLICATION_JSON) //
.param("project", project).param("table", tableNameUppercase).param("database", databaseUppercase)
.accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON))) //
.andExpect(MockMvcResultMatchers.status().isOk());
- Mockito.verify(tableService, Mockito.times(3)).getTableDescByType(project, true, tableNameUppercase,
- databaseUppercase, false, 9);
+ Mockito.verify(tableService, Mockito.times(3)).getTableDesc(project, true, tableNameUppercase,
+ databaseUppercase, false, Collections.singletonList(9), 10);
}
}
diff --git a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/TableServiceTest.java b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/TableServiceTest.java
index a6580e25b4..4f47d6cc3c 100644
--- a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/TableServiceTest.java
+++ b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/TableServiceTest.java
@@ -205,18 +205,26 @@ public class TableServiceTest extends CSVSourceTestCase {
@Test
public void testGetTableDesc() throws IOException {
- List<TableDesc> tableDesc = tableService.getTableDesc("default", true, "", "DEFAULT", true);
+ List<Integer> sourceType = new ArrayList<>();
+ sourceType.add(1); // Kafka table
+ sourceType.add(9); // Hive table
+ List<TableDesc> tableDesc = tableService.getTableDesc("default", true, "", "DEFAULT", true,
+ sourceType, 12).getFirst();
Assert.assertEquals(12, tableDesc.size());
- List<TableDesc> tableDesc2 = tableService.getTableDesc("default", true, "TEST_COUNTRY", "DEFAULT", false);
+ List<TableDesc> tableDesc2 = tableService.getTableDesc("default", true, "TEST_COUNTRY", "DEFAULT", false,
+ sourceType, 10).getFirst();
Assert.assertEquals(1, tableDesc2.size());
- List<TableDesc> tables3 = tableService.getTableDesc("default", true, "", "", true);
+ List<TableDesc> tables3 = tableService.getTableDesc("default", true, "", "", true,
+ sourceType, 100).getFirst();
Assert.assertEquals(21, tables3.size());
- List<TableDesc> tables = tableService.getTableDesc("default", true, "TEST_KYLIN_FACT", "DEFAULT", true);
+ List<TableDesc> tables = tableService.getTableDesc("default", true, "TEST_KYLIN_FACT", "DEFAULT", true,
+ sourceType, 10).getFirst();
Assert.assertEquals("TEST_KYLIN_FACT", tables.get(0).getName());
Assert.assertEquals(5633024, ((TableDescResponse) tables.get(0)).getStorageSize());
Assert.assertEquals(0, ((TableDescResponse) tables.get(0)).getTotalRecords());
- List<TableDesc> table2 = tableService.getTableDesc("default", true, "country", "DEFAULT", true);
+ List<TableDesc> table2 = tableService.getTableDesc("default", true, "country", "DEFAULT", true,
+ sourceType, 10).getFirst();
Assert.assertEquals("TEST_COUNTRY", table2.get(0).getName());
Assert.assertEquals(0L, ((TableDescResponse) table2.get(0)).getStorageSize());
@@ -225,21 +233,25 @@ public class TableServiceTest extends CSVSourceTestCase {
countryTable.setLastSnapshotPath("cannot/find/it");
manager.updateTableDesc(countryTable);
- table2 = tableService.getTableDesc("default", true, "country", "DEFAULT", true);
+ table2 = tableService.getTableDesc("default", true, "country", "DEFAULT", true,
+ sourceType, 10).getFirst();
Assert.assertEquals("TEST_COUNTRY", table2.get(0).getName());
Assert.assertEquals(0L, ((TableDescResponse) table2.get(0)).getStorageSize());
// get a not existing table desc
- tableDesc = tableService.getTableDesc("default", true, "not_exist_table", "DEFAULT", false);
+ tableDesc = tableService.getTableDesc("default", true, "not_exist_table", "DEFAULT", false,
+ sourceType, 10).getFirst();
Assert.assertEquals(0, tableDesc.size());
- tableDesc = tableService.getTableDesc("streaming_test", true, "", "DEFAULT", true);
+ tableDesc = tableService.getTableDesc("streaming_test", true, "", "DEFAULT", true,
+ sourceType, 10).getFirst();
Assert.assertEquals(2, tableDesc.size());
val tableMetadataManager = getInstance(getTestConfig(), "streaming_test");
var tableDesc1 = tableMetadataManager.getTableDesc("DEFAULT.SSB_TOPIC");
Assert.assertTrue(tableDesc1.isAccessible(getTestConfig().streamingEnabled()));
getTestConfig().setProperty("kylin.streaming.enabled", "false");
- tableDesc = tableService.getTableDesc("streaming_test", true, "", "DEFAULT", true);
+ tableDesc = tableService.getTableDesc("streaming_test", true, "", "DEFAULT", true,
+ sourceType, 10).getFirst();
Assert.assertEquals(0, tableDesc.size());
// check kafka table
Assert.assertFalse(tableDesc1.isAccessible(getTestConfig().streamingEnabled()));
@@ -277,7 +289,8 @@ public class TableServiceTest extends CSVSourceTestCase {
Assert.assertEquals(1, newTableExt.getAllColumnStats().size());
// call api to check tableDescResponse has the correct value
- final List<TableDesc> tables = tableService.getTableDesc("newten", true, "TEST_COUNTRY", "DEFAULT", true);
+ final List<TableDesc> tables = tableService.getTableDesc("newten", true, "TEST_COUNTRY", "DEFAULT", true,
+ Collections.emptyList(), 10).getFirst();
Assert.assertEquals(1, tables.size());
Assert.assertTrue(tables.get(0) instanceof TableDescResponse);
TableDescResponse t = (TableDescResponse) tables.get(0);
@@ -320,7 +333,8 @@ public class TableServiceTest extends CSVSourceTestCase {
Mockito.when(userAclService.hasUserAclPermissionInProject(Mockito.anyString(), Mockito.anyString()))
.thenReturn(false);
- List<TableDesc> tableExtList = tableService.getTableDesc("newten", true, "TEST_COUNTRY", "DEFAULT", true);
+ List<TableDesc> tableExtList = tableService.getTableDesc("newten", true, "TEST_COUNTRY", "DEFAULT",
+ true, Collections.emptyList(), 10).getFirst();
Assert.assertEquals(0, ((TableDescResponse) tableExtList.get(0)).getSamplingRows().size());
SecurityContextHolder.getContext()
.setAuthentication(new TestingAuthenticationToken("ADMIN", "ADMIN", Constant.ROLE_ADMIN));
@@ -392,7 +406,8 @@ public class TableServiceTest extends CSVSourceTestCase {
Assert.assertEquals("float", confirmedTableDesc.getColumns()[2].getDatatype());
// call api to check tableDescResponse has the correct value
- final List<TableDesc> tables = tableService.getTableDesc("newten", true, "TEST_COUNTRY", "DEFAULT", true);
+ final List<TableDesc> tables = tableService.getTableDesc("newten", true, "TEST_COUNTRY", "DEFAULT", true,
+ Collections.emptyList(), 10).getFirst();
Assert.assertEquals(1, tables.size());
Assert.assertTrue(tables.get(0) instanceof TableDescResponse);
TableDescResponse t = (TableDescResponse) tables.get(0);
@@ -480,7 +495,8 @@ public class TableServiceTest extends CSVSourceTestCase {
@Test
public void testLoadTableToProject() throws IOException {
- List<TableDesc> tables = tableService.getTableDesc("default", true, "TEST_COUNTRY", "DEFAULT", true);
+ List<TableDesc> tables = tableService.getTableDesc("default", true, "TEST_COUNTRY", "DEFAULT", true,
+ Collections.emptyList(), 10).getFirst();
TableDesc nTableDesc = new TableDesc(tables.get(0));
TableExtDesc tableExt = new TableExtDesc();
tableExt.setIdentity("DEFAULT.TEST_COUNTRY");
@@ -493,7 +509,8 @@ public class TableServiceTest extends CSVSourceTestCase {
public void testLoadTableToProjectWithS3Role() throws IOException {
getTestConfig().setProperty("kylin.env.use-dynamic-S3-role-credential-in-table", "true");
assert !SparderEnv.getSparkSession().conf().contains(String.format(S3AUtil.ROLE_ARN_KEY_FORMAT, "testbucket"));
- List<TableDesc> tables = tableService.getTableDesc("default", true, "TEST_COUNTRY", "DEFAULT", true);
+ List<TableDesc> tables = tableService.getTableDesc("default", true, "TEST_COUNTRY", "DEFAULT", true,
+ Collections.emptyList(), 10).getFirst();
TableDesc nTableDesc = new TableDesc(tables.get(0));
TableExtDesc tableExt = new TableExtDesc();
tableExt.setIdentity("DEFAULT.TEST_COUNTRY");
@@ -757,7 +774,8 @@ public class TableServiceTest extends CSVSourceTestCase {
private void testSetPartitionKeyWithoutException() throws Exception {
tableService.setPartitionKey("DEFAULT.TEST_KYLIN_FACT", "default", "CAL_DT", "yyyy-MM-dd");
- List<TableDesc> tables = tableService.getTableDesc("default", false, "", "DEFAULT", true);
+ List<TableDesc> tables = tableService.getTableDesc("default", false, "", "DEFAULT", true,
+ Collections.emptyList(), 10).getFirst();
//test set fact and table list order by fact
Assert.assertTrue(tables.get(0).getName().equals("TEST_KYLIN_FACT") && tables.get(0).isIncrementLoading());
}
@@ -904,7 +922,8 @@ public class TableServiceTest extends CSVSourceTestCase {
public void testSetTop() throws IOException {
TopTableRequest topTableRequest = mockTopTableRequest();
tableService.setTop(topTableRequest.getTable(), topTableRequest.getProject(), topTableRequest.isTop());
- List<TableDesc> tables = tableService.getTableDesc("default", false, "", "DEFAULT", true);
+ List<TableDesc> tables = tableService.getTableDesc("default", false, "", "DEFAULT", true,
+ Collections.emptyList(), 10).getFirst();
Assert.assertTrue(tables.get(0).isTop());
}
@@ -1118,47 +1137,52 @@ public class TableServiceTest extends CSVSourceTestCase {
@Test
public void testGetProjectTables() throws Exception {
NInitTablesResponse response;
+ overwriteSystemProp("kylin.source.load-hive-tablename-enabled", "false");
+
response = tableService.getProjectTables("default", "SSB.SS", 0, 14, true, true,
- (databaseName, tableName) -> tableService.getTableNameResponses("default", databaseName, tableName));
+ Collections.emptyList());
Assert.assertEquals(0, response.getDatabases().size());
response = tableService.getProjectTables("default", "SSB.CU", 0, 14, true, true,
- (databaseName, tableName) -> tableService.getTableNameResponses("default", databaseName, tableName));
+ Collections.emptyList());
Assert.assertEquals(1, response.getDatabases().size());
Assert.assertEquals(2, response.getDatabases().get(0).getTables().size());
response = tableService.getProjectTables("default", "", 0, 14, true, true,
- (databaseName, tableName) -> tableService.getTableNameResponses("default", databaseName, tableName));
+ Collections.emptyList());
Assert.assertEquals(3, response.getDatabases().size());
Assert.assertEquals(21,
response.getDatabases().get(0).getTables().size() + response.getDatabases().get(1).getTables().size()
+ response.getDatabases().get(2).getTables().size());
response = tableService.getProjectTables("default", "TEST", 0, 14, true, true,
- (databaseName, tableName) -> tableService.getTableNameResponses("default", databaseName, tableName));
+ Collections.emptyList());
Assert.assertEquals(2, response.getDatabases().size());
Assert.assertEquals(13,
response.getDatabases().get(0).getTables().size() + response.getDatabases().get(1).getTables().size());
response = tableService.getProjectTables("default", "EDW.", 0, 14, true, true,
- (databaseName, tableName) -> tableService.getTableNameResponses("default", databaseName, tableName));
+ Collections.emptyList());
Assert.assertEquals(1, response.getDatabases().size());
Assert.assertEquals(3, response.getDatabases().get(0).getTables().size());
response = tableService.getProjectTables("default", "EDW.", 0, 14, true, false,
- (databaseName, tableName) -> tableService.getTableDesc("default", true, tableName, databaseName, true));
+ Collections.emptyList());
Assert.assertEquals(1, response.getDatabases().size());
Assert.assertEquals(3, response.getDatabases().get(0).getTables().size());
response = tableService.getProjectTables("default", "DEFAULT.TEST_ORDER", 0, 14, true, false,
- (databaseName, tableName) -> tableService.getTableDesc("default", true, tableName, databaseName, true));
+ Collections.emptyList());
Assert.assertEquals(1, response.getDatabases().size());
Assert.assertEquals(1, response.getDatabases().get(0).getTables().size());
response = tableService.getProjectTables("default", ".TEST_ORDER", 0, 14, true, false,
- (databaseName, tableName) -> tableService.getTableDesc("default", true, tableName, databaseName, true));
+ Collections.emptyList());
Assert.assertEquals(0, response.getDatabases().size());
+ response = tableService.getProjectTables("default", "", 0, 14, true, true,
+ Collections.singletonList(9));
+ Assert.assertEquals(3, response.getDatabases().size());
}
@Test
@@ -1383,7 +1407,8 @@ public class TableServiceTest extends CSVSourceTestCase {
tableExt.setJodID("949afe5d-0221-420f-92db-cdd91cb31ac8");
tableMgr.mergeAndUpdateTableExt(oldExtDesc, tableExt);
- List<TableDesc> tables = tableService.getTableDesc("newten", true, "TEST_COUNTRY", "DEFAULT", true);
+ List<TableDesc> tables = tableService.getTableDesc("newten", true, "TEST_COUNTRY", "DEFAULT", true,
+ Collections.emptyList(), 10).getFirst();
Assert.assertEquals(1, tables.size());
Assert.assertEquals("949afe5d-0221-420f-92db-cdd91cb31ac8", ((TableDescResponse) tables.get(0)).getJodID());
@@ -1412,15 +1437,18 @@ public class TableServiceTest extends CSVSourceTestCase {
public void testGetTableDescByType() {
String project = "streaming_test";
try {
- val tableDescs = tableService.getTableDescByType(project, true, "", "default", true, 1);
+ val tableDescs = tableService.getTableDesc(project, true, "", "default", true,
+ Collections.singletonList(1), 10).getFirst();
Assert.assertNotNull(tableDescs);
- val tableDescs1 = tableService.getTableDescByType(project, true, "P_LINEORDER_STREAMING", "ssb", true, 1);
+ val tableDescs1 = tableService.getTableDesc(project, true, "P_LINEORDER_STREAMING", "ssb", true,
+ Collections.singletonList(1), 10).getFirst();
Assert.assertEquals(1, tableDescs1.size());
val tableDesc1 = tableDescs1.get(0);
Assert.assertEquals(tableDesc1.getTableAlias(), tableDesc1.getKafkaConfig().getBatchTable());
- val tableDescs2 = tableService.getTableDescByType(project, true, "LINEORDER_HIVE", "SSB", false, 9);
+ val tableDescs2 = tableService.getTableDesc(project, true, "LINEORDER_HIVE", "SSB", false,
+ Collections.singletonList(9), 10).getFirst();
Assert.assertEquals(1, tableDescs2.size());
val tableDesc2 = tableDescs2.get(0);
Assert.assertEquals(tableDesc2.getTableAlias(), tableDesc2.getIdentity());
@@ -1435,7 +1463,8 @@ public class TableServiceTest extends CSVSourceTestCase {
String project = "streaming_test";
try {
List<Integer> sourceTypes = Arrays.asList(1, 9);
- val tableDescs2 = tableService.getTableDescByTypes(project, true, "", "SSB", false, sourceTypes);
+ val tableDescs2 = tableService.getTableDesc(project, true, "", "SSB", false,
+ sourceTypes, 10).getFirst();
assert tableDescs2.stream().anyMatch(tableDesc -> tableDesc.getSourceType() == 1);
assert tableDescs2.stream().anyMatch(tableDesc -> tableDesc.getSourceType() == 9);
} catch (Exception e) {
@@ -1520,7 +1549,8 @@ public class TableServiceTest extends CSVSourceTestCase {
tableExt.setColumnStats(Lists.newArrayList(col1));
tableMgr.mergeAndUpdateTableExt(oldExtDesc, tableExt);
- final List<TableDesc> tables = tableService.getTableDesc("newten", true, "TEST_COUNTRY", "DEFAULT", true);
+ final List<TableDesc> tables = tableService.getTableDesc("newten", true, "TEST_COUNTRY", "DEFAULT", true,
+ Collections.emptyList(), 10).getFirst();
Assert.assertEquals(1, tables.size());
Assert.assertTrue(tables.get(0) instanceof TableDescResponse);
TableDescResponse t = (TableDescResponse) tables.get(0);