You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by xx...@apache.org on 2023/05/06 06:59:22 UTC
[kylin] 21/38: KYLIN-5533 add model list lite api
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 2e29f00053bdb99673bf9e103412fc1020099cd7
Author: lixiang <44...@qq.com>
AuthorDate: Sat Feb 25 00:03:27 2023 +0800
KYLIN-5533 add model list lite api
---
.../rest/controller/open/OpenModelController.java | 5 +-
.../kylin/rest/controller/NModelController.java | 11 +-
.../controller/open/OpenModelControllerTest.java | 4 +-
.../rest/controller/NModelControllerTest.java | 6 +-
.../rest/response/NDataModelLiteResponse.java | 129 +++++++++++++++++++++
.../kylin/rest/response/NDataModelResponse.java | 18 +++
.../apache/kylin/rest/service/ModelService.java | 44 ++++---
.../rest/service/params/ModelQueryParams.java | 2 +
.../rest/response/NDataModelResponseTest.java | 106 +++++++++++++++++
.../kylin/rest/service/ModelQueryServiceTest.java | 18 +--
.../kylin/rest/service/ModelServiceQueryTest.java | 99 ++++++++++------
.../kap/secondstorage/SecondStorageLockTest.java | 6 +-
12 files changed, 377 insertions(+), 71 deletions(-)
diff --git a/src/metadata-server/src/main/java/io/kyligence/kap/rest/controller/open/OpenModelController.java b/src/metadata-server/src/main/java/io/kyligence/kap/rest/controller/open/OpenModelController.java
index fff6b63367..3b10bfc29a 100644
--- a/src/metadata-server/src/main/java/io/kyligence/kap/rest/controller/open/OpenModelController.java
+++ b/src/metadata-server/src/main/java/io/kyligence/kap/rest/controller/open/OpenModelController.java
@@ -161,11 +161,12 @@ public class OpenModelController extends NBasicController {
@RequestParam(value = "model_alias_or_owner", required = false) String modelAliasOrOwner,
@RequestParam(value = "last_modify_from", required = false) Long lastModifyFrom,
@RequestParam(value = "last_modify_to", required = false) Long lastModifyTo,
- @RequestParam(value = "only_normal_dim", required = false, defaultValue = "true") boolean onlyNormalDim) {
+ @RequestParam(value = "only_normal_dim", required = false, defaultValue = "true") boolean onlyNormalDim,
+ @RequestParam(value = "lite", required = false, defaultValue = "false") boolean lite) {
String projectName = checkProjectName(project);
return modelController.getModels(modelId, modelAlias, exactMatch, projectName, owner, status, table, offset,
limit, sortBy, reverse, modelAliasOrOwner, Collections.singletonList(ModelAttributeEnum.BATCH),
- lastModifyFrom, lastModifyTo, onlyNormalDim);
+ lastModifyFrom, lastModifyTo, onlyNormalDim, lite);
}
@ApiOperation(value = "getIndexes", tags = { "AI" })
diff --git a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NModelController.java b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NModelController.java
index b520231844..e5029e98f9 100644
--- a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NModelController.java
+++ b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NModelController.java
@@ -75,6 +75,7 @@ import org.apache.kylin.rest.service.FusionModelService;
import org.apache.kylin.rest.service.IndexPlanService;
import org.apache.kylin.rest.service.ModelService;
import org.apache.kylin.rest.service.ModelTdsService;
+import org.apache.kylin.rest.service.params.ModelQueryParams;
import org.apache.kylin.tool.bisync.SyncContext;
import org.apache.kylin.tool.bisync.model.SyncModel;
import org.apache.kylin.util.DataRangeUtils;
@@ -143,13 +144,15 @@ public class NModelController extends NBasicController {
@RequestParam(value = "model_attributes", required = false) List<ModelAttributeEnum> modelAttributes,
@RequestParam(value = "last_modify_from", required = false) Long lastModifyFrom,
@RequestParam(value = "last_modify_to", required = false) Long lastModifyTo,
- @RequestParam(value = "only_normal_dim", required = false, defaultValue = "true") boolean onlyNormalDim) {
+ @RequestParam(value = "only_normal_dim", required = false, defaultValue = "true") boolean onlyNormalDim,
+ @RequestParam(value = "lite", required = false, defaultValue = "false") boolean lite) {
checkProjectName(project);
status = formatStatus(status, ModelStatusToDisplayEnum.class);
- DataResult<List<NDataModel>> filterModels = modelService.getModels(modelId, modelAlias, exactMatch, project,
- owner, status, table, offset, limit, sortBy, reverse, modelAliasOrOwner, modelAttributes,
- lastModifyFrom, lastModifyTo, onlyNormalDim);
+ ModelQueryParams request = new ModelQueryParams(modelId, modelAlias, exactMatch, project, owner, status,
+ table, offset, limit, sortBy, reverse, modelAliasOrOwner, modelAttributes, lastModifyFrom, lastModifyTo,
+ onlyNormalDim, lite);
+ DataResult<List<NDataModel>> filterModels = modelService.getModels(request);
fusionModelService.setModelUpdateEnabled(filterModels);
return new EnvelopeResponse<>(KylinException.CODE_SUCCESS, filterModels, "");
}
diff --git a/src/metadata-server/src/test/java/io/kyligence/kap/rest/controller/open/OpenModelControllerTest.java b/src/metadata-server/src/test/java/io/kyligence/kap/rest/controller/open/OpenModelControllerTest.java
index ccc7864448..52c97e7629 100644
--- a/src/metadata-server/src/test/java/io/kyligence/kap/rest/controller/open/OpenModelControllerTest.java
+++ b/src/metadata-server/src/test/java/io/kyligence/kap/rest/controller/open/OpenModelControllerTest.java
@@ -187,7 +187,7 @@ public class OpenModelControllerTest extends NLocalFileMetadataTestCase {
@Test
public void testGetModels() throws Exception {
Mockito.when(nModelController.getModels("model1", "model1", true, "default", "ADMIN", Arrays.asList("NEW"), "",
- 1, 5, "last_modify", false, null, null, null, null, true)).thenReturn(
+ 1, 5, "last_modify", false, null, null, null, null, true, false)).thenReturn(
new EnvelopeResponse<>(KylinException.CODE_SUCCESS, DataResult.get(mockModels(), 0, 10), ""));
mockMvc.perform(MockMvcRequestBuilders.get("/api/models").contentType(MediaType.APPLICATION_JSON)
.param("page_offset", "1").param("project", "default").param("model_id", "model1")
@@ -197,7 +197,7 @@ public class OpenModelControllerTest extends NLocalFileMetadataTestCase {
.andExpect(MockMvcResultMatchers.status().isOk()).andReturn();
Mockito.verify(openModelController).getModels("default", "model1", "model1", true, "ADMIN",
- Arrays.asList("NEW"), "", 1, 5, "last_modify", true, null, null, null, true);
+ Arrays.asList("NEW"), "", 1, 5, "last_modify", true, null, null, null, true, false);
}
@Test
diff --git a/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NModelControllerTest.java b/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NModelControllerTest.java
index 1638aa0bad..ef405752e1 100644
--- a/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NModelControllerTest.java
+++ b/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NModelControllerTest.java
@@ -241,7 +241,7 @@ public class NModelControllerTest extends NLocalFileMetadataTestCase {
Mockito.verify(nModelController).getModels(null, "model1", true, "default", "ADMIN", Arrays.asList("ONLINE"),
"", 0, 10, "last_modify", true, null, Arrays.asList(ModelAttributeEnum.BATCH,
ModelAttributeEnum.STREAMING, ModelAttributeEnum.HYBRID, ModelAttributeEnum.SECOND_STORAGE),
- null, null, true);
+ null, null, true, false);
}
@Test
@@ -255,7 +255,7 @@ public class NModelControllerTest extends NLocalFileMetadataTestCase {
.accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_JSON)))
.andExpect(MockMvcResultMatchers.status().isOk()).andReturn();
Mockito.verify(nModelController).getModels(null, "model1", true, "default", "ADMIN", Arrays.asList("ONLINE"),
- "TEST_KYLIN_FACT", 0, 10, "last_modify", true, null, null, null, null, true);
+ "TEST_KYLIN_FACT", 0, 10, "last_modify", true, null, null, null, null, true, false);
}
@Test
@@ -269,7 +269,7 @@ public class NModelControllerTest extends NLocalFileMetadataTestCase {
.accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_JSON)))
.andExpect(MockMvcResultMatchers.status().isOk()).andReturn();
Mockito.verify(nModelController).getModels(null, "", true, "default", "ADMIN", Arrays.asList("ONLINE"),
- "TEST_KYLIN_FACT", 0, 10, "last_modify", true, null, null, null, null, true);
+ "TEST_KYLIN_FACT", 0, 10, "last_modify", true, null, null, null, null, true, false);
}
@Test
diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/response/NDataModelLiteResponse.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/response/NDataModelLiteResponse.java
new file mode 100644
index 0000000000..ebcfd79627
--- /dev/null
+++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/response/NDataModelLiteResponse.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.rest.response;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.kylin.metadata.model.NDataModel;
+import org.apache.kylin.metadata.model.PartitionDesc;
+import org.apache.kylin.metadata.model.SegmentRange;
+import org.springframework.beans.BeanUtils;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import lombok.EqualsAndHashCode;
+import lombok.Getter;
+import lombok.Setter;
+
+@Getter
+@Setter
+@EqualsAndHashCode(callSuper = true)
+public class NDataModelLiteResponse extends NDataModelResponse {
+
+ @JsonProperty("empty_model")
+ private boolean emptyModel;
+
+ @JsonProperty("partition_column_in_dims")
+ private boolean partitionColumnInDims;
+
+ @JsonProperty("batch_id")
+ @JsonInclude(JsonInclude.Include.NON_EMPTY)
+ private String batchId;
+
+ @JsonProperty("streaming_indexes")
+ @JsonInclude(JsonInclude.Include.NON_EMPTY)
+ private Long streamingIndexes;
+
+ @JsonProperty("batch_partition_desc")
+ @JsonInclude(JsonInclude.Include.NON_EMPTY)
+ private PartitionDesc batchPartitionDesc;
+
+ @SuppressWarnings("rawtypes")
+ @JsonProperty("batch_segment_holes")
+ @JsonInclude(JsonInclude.Include.NON_EMPTY)
+ private List<SegmentRange> batchSegmentHoles;
+
+ @JsonProperty("batch_segments")
+ @JsonInclude(JsonInclude.Include.NON_EMPTY)
+ private List<NDataSegmentResponse> batchSegments = new ArrayList<>();
+
+ @JsonProperty("simplified_tables")
+ @JsonInclude(JsonInclude.Include.NON_EMPTY)
+ @Override
+ public List<SimplifiedTableResponse> getSimpleTables() {
+ return Collections.emptyList();
+ }
+
+ @JsonProperty("selected_columns")
+ @JsonInclude(JsonInclude.Include.NON_EMPTY)
+ @Override
+ public List<SimplifiedNamedColumn> getSelectedColumns() {
+ return Collections.emptyList();
+ }
+
+ @JsonProperty("simplified_dimensions")
+ @JsonInclude(JsonInclude.Include.NON_EMPTY)
+ @Override
+ public List<SimplifiedNamedColumn> getNamedColumns() {
+ return Collections.emptyList();
+ }
+
+ @JsonProperty("all_named_columns")
+ @JsonInclude(JsonInclude.Include.NON_EMPTY)
+ @Override
+ public List<NamedColumn> getAllNamedColumns() {
+ return Collections.emptyList();
+ }
+
+ @JsonProperty("all_measures")
+ @JsonInclude(JsonInclude.Include.NON_EMPTY)
+ @Override
+ public List<Measure> getMeasures() {
+ return Collections.emptyList();
+ }
+
+ @JsonProperty("simplified_measures")
+ @JsonInclude(JsonInclude.Include.NON_EMPTY)
+ @Override
+ public List<SimplifiedMeasure> getSimplifiedMeasures() {
+ return Collections.emptyList();
+ }
+
+ @JsonProperty("segments")
+ @JsonInclude(JsonInclude.Include.NON_EMPTY)
+ @Override
+ public List<NDataSegmentResponse> getSegments() {
+ return Collections.emptyList();
+ }
+
+ public NDataModelLiteResponse(NDataModelResponse response, NDataModel dataModel) {
+ super(dataModel);
+ if (dataModel.isFusionModel()) {
+ FusionModelResponse fusionModelResponse = (FusionModelResponse) response;
+ this.setBatchId(fusionModelResponse.getBatchId());
+ this.setBatchPartitionDesc(fusionModelResponse.getBatchPartitionDesc());
+ this.setStreamingIndexes(fusionModelResponse.getStreamingIndexes());
+ this.setBatchSegmentHoles(fusionModelResponse.getBatchSegmentHoles());
+ }
+ BeanUtils.copyProperties(response, this);
+ }
+}
diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/response/NDataModelResponse.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/response/NDataModelResponse.java
index 846bdc7855..cf92dccb13 100644
--- a/src/modeling-service/src/main/java/org/apache/kylin/rest/response/NDataModelResponse.java
+++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/response/NDataModelResponse.java
@@ -40,6 +40,7 @@ import org.apache.kylin.metadata.model.JoinTableDesc;
import org.apache.kylin.metadata.model.NDataModel;
import org.apache.kylin.metadata.model.NDataModelManager;
import org.apache.kylin.metadata.model.NTableMetadataManager;
+import org.apache.kylin.metadata.model.PartitionDesc;
import org.apache.kylin.metadata.model.SegmentRange;
import org.apache.kylin.metadata.model.TableExtDesc;
import org.apache.kylin.metadata.model.TableRef;
@@ -169,6 +170,23 @@ public class NDataModelResponse extends NDataModel {
.collect(Collectors.toList()));
}
+ @JsonIgnore
+ public boolean isEmptyModel() {
+ List<SimplifiedMeasure> simplifiedMeasures = getSimplifiedMeasures();
+ return getNamedColumns().isEmpty() && simplifiedMeasures.size() == 1
+ && "COUNT_ALL".equals(simplifiedMeasures.get(0).getName());
+ }
+
+ @JsonIgnore
+ public boolean isPartitionColumnInDims() {
+ PartitionDesc partitionDesc = getPartitionDesc();
+ if (partitionDesc == null || partitionDesc.getPartitionDateColumn() == null) {
+ return false;
+ }
+ String partitionColumn = partitionDesc.getPartitionDateColumn();
+ return getNamedColumns().stream().anyMatch(dim -> dim.getAliasDotColumn().equalsIgnoreCase(partitionColumn));
+ }
+
@JsonProperty("simplified_dimensions")
public List<SimplifiedNamedColumn> getNamedColumns() {
if (simplifiedDims != null) {
diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
index 6c5aced159..c3645f4af8 100644
--- a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
+++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
@@ -199,7 +199,6 @@ import org.apache.kylin.metadata.streaming.KafkaConfig;
import org.apache.kylin.query.util.PushDownUtil;
import org.apache.kylin.query.util.QueryParams;
import org.apache.kylin.rest.aspect.Transaction;
-import org.apache.kylin.rest.constant.ModelAttributeEnum;
import org.apache.kylin.rest.constant.ModelStatusToDisplayEnum;
import org.apache.kylin.rest.request.ModelConfigRequest;
import org.apache.kylin.rest.request.ModelParatitionDescRequest;
@@ -225,6 +224,7 @@ import org.apache.kylin.rest.response.ModelConfigResponse;
import org.apache.kylin.rest.response.ModelSaveCheckResponse;
import org.apache.kylin.rest.response.MultiPartitionValueResponse;
import org.apache.kylin.rest.response.NCubeDescResponse;
+import org.apache.kylin.rest.response.NDataModelLiteResponse;
import org.apache.kylin.rest.response.NDataModelOldParams;
import org.apache.kylin.rest.response.NDataModelResponse;
import org.apache.kylin.rest.response.NDataSegmentResponse;
@@ -378,7 +378,7 @@ public class ModelService extends AbstractModelService implements TableModelSupp
((NDataModelResponse) model).isHasSegments() || CollectionUtils.isNotEmpty(segments));
}
- if (model instanceof FusionModelResponse) {
+ if (model.isFusionModel()) {
FusionModel fusionModel = getManager(FusionModelManager.class, model.getProject())
.getFusionModel(model.getId());
NDataModel batchModel = fusionModel.getBatchModel();
@@ -386,7 +386,9 @@ public class ModelService extends AbstractModelService implements TableModelSupp
List<NDataSegmentResponse> batchSegments = getSegmentsResponse(batchModel.getUuid(), model.getProject(),
"1", String.valueOf(Long.MAX_VALUE - 1), null, executables, LAST_MODIFY, true);
calculateRecordSizeAndCount(batchSegments, oldParams);
- ((FusionModelResponse) model).setBatchSegments(batchSegments);
+ if (model instanceof FusionModelResponse) {
+ ((FusionModelResponse) model).setBatchSegments(batchSegments);
+ }
}
}
}
@@ -639,18 +641,18 @@ public class ModelService extends AbstractModelService implements TableModelSupp
.filter(NDataModel::isMultiPartitionModel).collect(Collectors.toList());
}
- public DataResult<List<NDataModel>> getModels(String modelId, String modelAlias, boolean exactMatch, String project,
- String owner, List<String> status, String table, Integer offset, Integer limit, String sortBy,
- boolean reverse, String modelAliasOrOwner, List<ModelAttributeEnum> modelAttributes, Long lastModifyFrom,
- Long lastModifyTo, boolean onlyNormalDim) {
+ public DataResult<List<NDataModel>> getModels(ModelQueryParams params) {
List<NDataModel> models = new ArrayList<>();
DataResult<List<NDataModel>> filterModels;
+ val table = params.getTable();
+ val project = params.getProjectName();
+ val modelAttributes = params.getModelAttributes();
+ val modelId = params.getModelId();
+ val offset = params.getOffset();
+ val limit = params.getLimit();
if (StringUtils.isEmpty(table)) {
- val modelQueryParams = new ModelQueryParams(modelId, modelAlias, exactMatch, project, owner, status, offset,
- limit, sortBy, reverse, modelAliasOrOwner, modelAttributes, lastModifyFrom, lastModifyTo,
- onlyNormalDim);
- val tripleList = modelQuerySupporter.getModels(modelQueryParams);
- val pair = getModelsOfCurrentPage(modelQueryParams, tripleList);
+ val tripleList = modelQuerySupporter.getModels(params);
+ val pair = getModelsOfCurrentPage(params, tripleList, params.isLite());
models.addAll(pair.getFirst());
// add second storage infos
ModelUtils.addSecondStorageInfo(project, models);
@@ -659,7 +661,7 @@ public class ModelService extends AbstractModelService implements TableModelSupp
filterModels.setValue(updateResponseAcl(filterModels.getValue(), project));
return filterModels;
}
- models.addAll(getRelateModels(project, table, modelAlias));
+ models.addAll(getRelateModels(project, table, params.getModelAlias()));
Set<NDataModel> filteredModels = ModelUtils.getFilteredModels(project, modelAttributes, models);
if (CollectionUtils.isNotEmpty(modelAttributes)) {
@@ -676,7 +678,7 @@ public class ModelService extends AbstractModelService implements TableModelSupp
}
public Pair<List<NDataModelResponse>, Integer> getModelsOfCurrentPage(ModelQueryParams queryElem,
- List<ModelTriple> modelTripleList) {
+ List<ModelTriple> modelTripleList, boolean lite) {
val projectName = queryElem.getProjectName();
val offset = queryElem.getOffset();
val limit = queryElem.getLimit();
@@ -691,8 +693,13 @@ public class ModelService extends AbstractModelService implements TableModelSupp
}
NDataModel dataModel = t.getDataModel();
try {
- return convertToDataModelResponse(dataModel, projectName, dfManager, status,
- queryElem.isOnlyNormalDim());
+ NDataModelResponse nDataModelResponse = convertToDataModelResponse(dataModel, projectName, dfManager,
+ status, queryElem.isOnlyNormalDim());
+ if (lite) {
+ return new NDataModelLiteResponse(nDataModelResponse, dataModel);
+ } else {
+ return nDataModelResponse;
+ }
} catch (Exception e) {
String errorMsg = String.format(Locale.ROOT,
"convert NDataModelResponse failed, mark to broken. %s, %s", dataModel.getAlias(),
@@ -710,6 +717,11 @@ public class ModelService extends AbstractModelService implements TableModelSupp
return new Pair<>(filterModels, totalSize.get());
}
+ public Pair<List<NDataModelResponse>, Integer> getModelsOfCurrentPage(ModelQueryParams queryElem,
+ List<ModelTriple> modelTripleList) {
+ return getModelsOfCurrentPage(queryElem, modelTripleList, false);
+ }
+
public NDataModelResponse convertToDataModelResponseBroken(NDataModel modelDesc) {
NDataModelResponse response = new NDataModelResponse(modelDesc);
response.setStatus(ModelStatusToDisplayEnum.BROKEN);
diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/params/ModelQueryParams.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/params/ModelQueryParams.java
index e94439e534..b7d9194cc6 100644
--- a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/params/ModelQueryParams.java
+++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/params/ModelQueryParams.java
@@ -33,6 +33,7 @@ public class ModelQueryParams {
private String projectName;
private String owner;
private List<String> status;
+ private String table;
private Integer offset;
private Integer limit;
private String sortBy;
@@ -42,4 +43,5 @@ public class ModelQueryParams {
private Long lastModifyFrom;
private Long lastModifyTo;
private boolean onlyNormalDim;
+ private boolean lite;
}
diff --git a/src/modeling-service/src/test/java/org/apache/kylin/rest/response/NDataModelResponseTest.java b/src/modeling-service/src/test/java/org/apache/kylin/rest/response/NDataModelResponseTest.java
index 231baf8f89..b009be95ef 100644
--- a/src/modeling-service/src/test/java/org/apache/kylin/rest/response/NDataModelResponseTest.java
+++ b/src/modeling-service/src/test/java/org/apache/kylin/rest/response/NDataModelResponseTest.java
@@ -21,7 +21,10 @@ package org.apache.kylin.rest.response;
import static com.google.common.collect.Lists.newArrayList;
import static org.apache.kylin.common.util.TestUtils.getTestConfig;
import static org.apache.kylin.metadata.model.FunctionDesc.FUNC_COUNT;
+import static org.apache.kylin.metadata.model.FunctionDesc.FUNC_MIN;
+import java.util.Arrays;
+import java.util.Collections;
import java.util.List;
import org.apache.kylin.common.KylinConfig;
@@ -30,6 +33,7 @@ import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.NDataModel;
import org.apache.kylin.metadata.model.NDataModelManager;
import org.apache.kylin.metadata.model.ParameterDesc;
+import org.apache.kylin.metadata.model.PartitionDesc;
import org.apache.kylin.metadata.project.EnhancedUnitOfWork;
import org.apache.kylin.util.MetadataTestUtils;
import org.junit.jupiter.api.Assertions;
@@ -170,6 +174,108 @@ class NDataModelResponseTest {
Assertions.assertEquals(1, namedColumns.size());
}
+ @Test
+ void testIsHaveNoDimsAndMeasWithoutCountAll() {
+ List<NDataModel.NamedColumn> allNamedColumns = Lists.newArrayList();
+ NDataModel modelWithoutNamedColumns = new NDataModel();
+ modelWithoutNamedColumns.setUuid("model");
+ modelWithoutNamedColumns.setProject(PROJECT);
+ modelWithoutNamedColumns.setAllNamedColumns(allNamedColumns);
+ modelWithoutNamedColumns.setAllMeasures(Lists.newArrayList(createMeasure()));
+ modelWithoutNamedColumns.setRootFactTableName("DEFAULT.TEST_KYLIN_FACT");
+ createModel(modelWithoutNamedColumns);
+ NDataModelResponse withoutColumnResponse = new NDataModelResponse(modelWithoutNamedColumns);
+ Assertions.assertFalse(withoutColumnResponse.isEmptyModel());
+
+ NDataModel model1 = new NDataModel();
+ model1.setProject(PROJECT);
+ model1.setAllNamedColumns(allNamedColumns);
+ model1.setRootFactTableName("DEFAULT.TEST_KYLIN_FACT");
+ model1.setUuid("model1");
+ model1.setAllMeasures(Arrays.asList(createCountAllMeasure(), createTestMeasure()));
+ createModel(model1);
+ NDataModelResponse model1Response = new NDataModelResponse(model1);
+ Assertions.assertFalse(model1Response.isEmptyModel());
+
+ NDataModel model2 = new NDataModel();
+ model2.setProject(PROJECT);
+ model2.setAllNamedColumns(allNamedColumns);
+ model2.setRootFactTableName("DEFAULT.TEST_KYLIN_FACT");
+ model2.setUuid("model2");
+ model2.setAllMeasures(Collections.singletonList(createCountAllMeasure()));
+ createModel(model2);
+ NDataModelResponse model2Response = new NDataModelResponse(model2);
+ Assertions.assertTrue(model2Response.isEmptyModel());
+
+ NDataModel model3 = new NDataModel();
+ model3.setProject(PROJECT);
+ model3.setAllNamedColumns(allNamedColumns);
+ model3.setRootFactTableName("DEFAULT.TEST_KYLIN_FACT");
+ model3.setUuid("model3");
+ model3.setAllMeasures(Collections.singletonList(createMeasure()));
+ createModel(model3);
+ NDataModelResponse model3Response = new NDataModelResponse(model3);
+ Assertions.assertFalse(model3Response.isEmptyModel());
+ }
+
+ @Test
+ void testIsPartitionColumnInDims() {
+ PartitionDesc partitionDesc = new PartitionDesc();
+ List<NDataModel.NamedColumn> allNamedColumns = Lists.newArrayList();
+
+ NDataModel model = new NDataModel();
+ model.setUuid("model");
+ model.setProject(PROJECT);
+ model.setAllNamedColumns(allNamedColumns);
+ model.setAllMeasures(Lists.newArrayList(createMeasure()));
+ model.setRootFactTableName("DEFAULT.TEST_KYLIN_FACT");
+ model.setPartitionDesc(partitionDesc);
+ createModel(model);
+ NDataModelResponse modelResponse = new NDataModelResponse(model);
+ Assertions.assertFalse(modelResponse.isPartitionColumnInDims());
+
+ NDataModel model1 = new NDataModel();
+ model1.setUuid("model1");
+ model1.setProject(PROJECT);
+ model1.setAllNamedColumns(allNamedColumns);
+ model1.setAllMeasures(Lists.newArrayList(createMeasure()));
+ model1.setRootFactTableName("DEFAULT.TEST_KYLIN_FACT");
+ model1.setPartitionDesc(null);
+ createModel(model1);
+ NDataModelResponse modelResponse1 = new NDataModelResponse(model1);
+ Assertions.assertFalse(modelResponse1.isPartitionColumnInDims());
+
+ NDataModel model2 = new NDataModel();
+ model2.setUuid("model2");
+ model2.setProject(PROJECT);
+ model2.setAllNamedColumns(allNamedColumns);
+ model2.setAllMeasures(Lists.newArrayList(createMeasure()));
+ model2.setRootFactTableName("DEFAULT.TEST_KYLIN_FACT");
+ partitionDesc.setPartitionDateColumn("CAL_DT");
+ model2.setPartitionDesc(partitionDesc);
+ createModel(model2);
+ NDataModelResponse modelResponse2 = new NDataModelResponse(model2);
+ Assertions.assertFalse(modelResponse2.isPartitionColumnInDims());
+ }
+
+ private NDataModel.Measure createCountAllMeasure() {
+ NDataModel.Measure countOneMeasure = new NDataModel.Measure();
+ countOneMeasure.setName("COUNT_ALL");
+ countOneMeasure.setFunction(
+ FunctionDesc.newInstance(FUNC_COUNT, newArrayList(ParameterDesc.newInstance("1")), "bigint"));
+ countOneMeasure.setId(200001);
+ return countOneMeasure;
+ }
+
+ private NDataModel.Measure createTestMeasure() {
+ NDataModel.Measure countOneMeasure = new NDataModel.Measure();
+ countOneMeasure.setName("TEST_M");
+ countOneMeasure.setFunction(
+ FunctionDesc.newInstance(FUNC_MIN, newArrayList(ParameterDesc.newInstance("1")), "bigint"));
+ countOneMeasure.setId(1);
+ return countOneMeasure;
+ }
+
private NDataModel.Measure createMeasure() {
NDataModel.Measure countOneMeasure = new NDataModel.Measure();
countOneMeasure.setName("COUNT_ONE");
diff --git a/src/query-service/src/test/java/org/apache/kylin/rest/service/ModelQueryServiceTest.java b/src/query-service/src/test/java/org/apache/kylin/rest/service/ModelQueryServiceTest.java
index efd883b535..a2cf68983e 100644
--- a/src/query-service/src/test/java/org/apache/kylin/rest/service/ModelQueryServiceTest.java
+++ b/src/query-service/src/test/java/org/apache/kylin/rest/service/ModelQueryServiceTest.java
@@ -22,10 +22,10 @@ import java.util.List;
import java.util.Set;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
import org.apache.kylin.metadata.cube.model.NDataflow;
import org.apache.kylin.metadata.model.NDataModel;
+import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.rest.constant.ModelAttributeEnum;
import org.apache.kylin.rest.response.NDataModelResponse;
import org.apache.kylin.rest.service.params.ModelQueryParams;
@@ -77,8 +77,8 @@ public class ModelQueryServiceTest extends NLocalFileMetadataTestCase {
List models = Arrays.asList(mt1);
List<ModelAttributeEnum> modelAttributeSet1 = Lists.newArrayList(ModelAttributeEnum.BATCH);
- ModelQueryParams modelQueryParams = new ModelQueryParams("", null, true, "default", null, null, 0, 10, "", true,
- null, modelAttributeSet1, null, null, true);
+ ModelQueryParams modelQueryParams = new ModelQueryParams("", null, true, "default", null, null, "", 0, 10, "",
+ true, null, modelAttributeSet1, null, null, true, false);
PowerMockito.stub(PowerMockito.method(SecondStorageUtil.class, "isProjectEnable")).toReturn(Boolean.TRUE);
PowerMockito.stub(PowerMockito.method(SecondStorageUtil.class, "isModelEnable")).toReturn(Boolean.FALSE);
@@ -96,8 +96,8 @@ public class ModelQueryServiceTest extends NLocalFileMetadataTestCase {
List<ModelAttributeEnum> modelAttributeSet1 = Lists.newArrayList(ModelAttributeEnum.STREAMING,
ModelAttributeEnum.HYBRID);
- ModelQueryParams modelQueryParams = new ModelQueryParams("", null, true, "streaming_test", null, null, 0, 10,
- "", true, null, modelAttributeSet1, null, null, true);
+ ModelQueryParams modelQueryParams = new ModelQueryParams("", null, true, "streaming_test", null, null, "", 0,
+ 10, "", true, null, modelAttributeSet1, null, null, true, false);
PowerMockito.stub(PowerMockito.method(SecondStorageUtil.class, "isProjectEnable")).toReturn(Boolean.TRUE);
PowerMockito.stub(PowerMockito.method(SecondStorageUtil.class, "isModelEnable")).toReturn(Boolean.FALSE);
@@ -120,14 +120,14 @@ public class ModelQueryServiceTest extends NLocalFileMetadataTestCase {
List<ModelAttributeEnum> modelAttributeSet1 = Lists.newArrayList(ModelAttributeEnum.BATCH,
ModelAttributeEnum.SECOND_STORAGE);
- ModelQueryParams modelQueryParams = new ModelQueryParams("", null, true, "default", null, null, 0, 10, "", true,
- null, modelAttributeSet1, null, null, true);
+ ModelQueryParams modelQueryParams = new ModelQueryParams("", null, true, "default", null, null, "", 0, 10, "",
+ true, null, modelAttributeSet1, null, null, true, false);
List<ModelTriple> filteredModels1 = modelQueryService.filterModels(models, modelQueryParams);
Assert.assertEquals(1, filteredModels1.size());
List<ModelAttributeEnum> modelAttributeSet2 = Lists.newArrayList(ModelAttributeEnum.SECOND_STORAGE);
- ModelQueryParams modelQueryParams2 = new ModelQueryParams("", null, true, "default", null, null, 0, 10, "",
- true, null, modelAttributeSet2, null, null, true);
+ ModelQueryParams modelQueryParams2 = new ModelQueryParams("", null, true, "default", null, null, "", 0, 10, "",
+ true, null, modelAttributeSet2, null, null, true, false);
List<ModelTriple> filteredModels2 = modelQueryService.filterModels(models, modelQueryParams2);
Assert.assertEquals(1, filteredModels2.size());
}
diff --git a/src/query-service/src/test/java/org/apache/kylin/rest/service/ModelServiceQueryTest.java b/src/query-service/src/test/java/org/apache/kylin/rest/service/ModelServiceQueryTest.java
index 351d675346..0955209212 100644
--- a/src/query-service/src/test/java/org/apache/kylin/rest/service/ModelServiceQueryTest.java
+++ b/src/query-service/src/test/java/org/apache/kylin/rest/service/ModelServiceQueryTest.java
@@ -42,6 +42,7 @@ import org.apache.kylin.rest.constant.ModelAttributeEnum;
import org.apache.kylin.rest.constant.ModelStatusToDisplayEnum;
import org.apache.kylin.rest.response.DataResult;
import org.apache.kylin.rest.response.FusionModelResponse;
+import org.apache.kylin.rest.response.NDataModelLiteResponse;
import org.apache.kylin.rest.response.NDataModelResponse;
import org.apache.kylin.rest.response.RelatedModelResponse;
import org.apache.kylin.rest.service.params.ModelQueryParams;
@@ -154,17 +155,27 @@ public class ModelServiceQueryTest extends SourceTestCase {
@Test
public void testQueryModels() {
String project = "streaming_test";
- val modelList = modelService.getModels(null, null, true, project, "ADMIN", Lists.newArrayList(), "", 0, 8,
- "last_modify", true, null,
+ ModelQueryParams request = new ModelQueryParams(null, null, true, project, "ADMIN", Lists.newArrayList(), "", 0,
+ 8, "last_modify", true, null,
Arrays.asList(ModelAttributeEnum.BATCH, ModelAttributeEnum.STREAMING, ModelAttributeEnum.HYBRID), null,
- null, true);
+ null, true, false);
+ val modelList = modelService.getModels(request);
Assert.assertEquals(11, modelList.getTotalSize());
Assert.assertEquals(8, modelList.getValue().size());
- val modelList1 = modelService.getModels(null, null, true, project, "ADMIN", Lists.newArrayList(), "", 1, 10,
- "usage", true, null,
+ ModelQueryParams liteRequest = new ModelQueryParams(null, null, true, project, "ADMIN", Lists.newArrayList(),
+ "", 0, 8, "last_modify", true, null,
Arrays.asList(ModelAttributeEnum.BATCH, ModelAttributeEnum.STREAMING, ModelAttributeEnum.HYBRID), null,
- null, true);
+ null, true, true);
+ val modelListWithLite = modelService.getModels(liteRequest);
+ Assert.assertEquals(11, modelListWithLite.getTotalSize());
+ Assert.assertEquals(8, modelListWithLite.getValue().size());
+
+ ModelQueryParams request1 = new ModelQueryParams(null, null, true, project, "ADMIN", Lists.newArrayList(), "",
+ 1, 10, "usage", true, null,
+ Arrays.asList(ModelAttributeEnum.BATCH, ModelAttributeEnum.STREAMING, ModelAttributeEnum.HYBRID), null,
+ null, true, false);
+ val modelList1 = modelService.getModels(request1);
Assert.assertEquals(11, modelList1.getTotalSize());
Assert.assertEquals(1, modelList1.getValue().size());
@@ -176,53 +187,59 @@ public class ModelServiceQueryTest extends SourceTestCase {
Assert.assertTrue(triple.getMiddle() instanceof NDataModel);
Assert.assertNull(triple.getRight());
- val modelList2 = modelService.getModels(null, null, true, project, "ADMIN", Lists.newArrayList(), "", 1, 5,
- "storage", true, null,
+ ModelQueryParams request2 = new ModelQueryParams(null, null, true, project, "ADMIN", Lists.newArrayList(), "",
+ 1, 5, "storage", true, null,
Arrays.asList(ModelAttributeEnum.BATCH, ModelAttributeEnum.STREAMING, ModelAttributeEnum.HYBRID), null,
- null, true);
+ null, true, false);
+ val modelList2 = modelService.getModels(request2);
Assert.assertEquals(11, modelList2.getTotalSize());
Assert.assertEquals(5, modelList2.getValue().size());
Assert.assertTrue(((NDataModelResponse) modelList2.getValue().get(0))
.getStorage() >= ((NDataModelResponse) modelList2.getValue().get(4)).getStorage());
- val modelList3 = modelService.getModels(null, null, true, project, "ADMIN", Lists.newArrayList(), "", 1, 10,
- "expansionrate", true, null,
+ ModelQueryParams request3 = new ModelQueryParams(null, null, true, project, "ADMIN", Lists.newArrayList(), "",
+ 1, 10, "expansionrate", true, null,
Arrays.asList(ModelAttributeEnum.BATCH, ModelAttributeEnum.STREAMING, ModelAttributeEnum.HYBRID), null,
- null, false);
+ null, false, false);
+ val modelList3 = modelService.getModels(request3);
Assert.assertEquals(1, modelList3.getValue().size());
- val modelList4 = modelService.getModels(null, null, true, project, "ADMIN", Lists.newArrayList(), "", 0, 10,
- "expansionrate", false, "ADMIN",
+ ModelQueryParams request4 = new ModelQueryParams(null, null, true, project, "ADMIN", Lists.newArrayList(), "",
+ 0, 10, "expansionrate", false, "ADMIN",
Arrays.asList(ModelAttributeEnum.BATCH, ModelAttributeEnum.STREAMING, ModelAttributeEnum.HYBRID),
- 974198646000L, System.currentTimeMillis(), true);
+ 974198646000L, System.currentTimeMillis(), true, false);
+ val modelList4 = modelService.getModels(request4);
Assert.assertEquals(10, modelList4.getValue().size());
- val modelList5 = modelService.getModels(null, null, true, project, "ADMIN", Lists.newArrayList(), "", 1, 6, "",
- true, null,
+ ModelQueryParams request5 = new ModelQueryParams(null, null, true, project, "ADMIN", Lists.newArrayList(), "",
+ 1, 6, "", true, null,
Arrays.asList(ModelAttributeEnum.BATCH, ModelAttributeEnum.STREAMING, ModelAttributeEnum.HYBRID), null,
- null, false);
+ null, false, false);
+ val modelList5 = modelService.getModels(request5);
Assert.assertEquals(5, modelList5.getValue().size());
getTestConfig().setProperty("kylin.metadata.semi-automatic-mode", "true");
- val modelList6 = modelService.getModels(null, null, true, project, "ADMIN", Lists.newArrayList(), "", 1, 6, "",
- true, null, Arrays.asList(ModelAttributeEnum.BATCH, ModelAttributeEnum.STREAMING,
+ ModelQueryParams request6 = new ModelQueryParams(null, null, true, project, "ADMIN", Lists.newArrayList(), "",
+ 1, 6, "", true, null, Arrays.asList(ModelAttributeEnum.BATCH, ModelAttributeEnum.STREAMING,
ModelAttributeEnum.HYBRID, ModelAttributeEnum.SECOND_STORAGE),
- null, null, false);
+ null, null, false, false);
+ val modelList6 = modelService.getModels(request6);
Assert.assertEquals(5, modelList6.getValue().size());
getTestConfig().setProperty("kylin.metadata.semi-automatic-mode", "false");
// used for getModels without sortBy field
- val modelList7 = modelService.getModels(null, null, true, project, "ADMIN", Lists.newArrayList(), "", 0, 6, "",
- true, null,
+ ModelQueryParams request7 = new ModelQueryParams(null, null, true, project, "ADMIN", Lists.newArrayList(), "",
+ 0, 6, "", true, null,
Arrays.asList(ModelAttributeEnum.BATCH, ModelAttributeEnum.STREAMING, ModelAttributeEnum.HYBRID), null,
- null, false);
+ null, false, false);
+ val modelList7 = modelService.getModels(request7);
Assert.assertEquals(6, modelList7.getValue().size());
}
@Test
public void testConvertToDataModelResponseBroken() {
List<ModelAttributeEnum> modelAttributeSet = Lists.newArrayList(ModelAttributeEnum.BATCH);
- ModelQueryParams modelQueryParams = new ModelQueryParams("", null, true, "default", null, null, 0, 10, "", true,
- null, modelAttributeSet, null, null, true);
+ ModelQueryParams modelQueryParams = new ModelQueryParams("", null, true, "default", null, null, "", 0, 10, "",
+ true, null, modelAttributeSet, null, null, true, false);
val tripleList = modelQueryService.getModels(modelQueryParams);
ModelTriple modelTriple = tripleList.get(0);
@@ -257,17 +274,32 @@ public class ModelServiceQueryTest extends SourceTestCase {
Assert.assertEquals(0, model1.getSource());
String modelName2 = "model_streaming";
- DataResult<List<NDataModel>> modelResult2 = modelService.getModels("4965c827-fbb4-4ea1-a744-3f341a3b030d",
- modelName2, true, project, "ADMIN", Lists.newArrayList(), "", 0, 10, "last_modify", true, null,
+ ModelQueryParams liteRequest = new ModelQueryParams("4965c827-fbb4-4ea1-a744-3f341a3b030d", modelName2, true,
+ project, "ADMIN", Lists.newArrayList(), "", 0, 10, "last_modify", true, null,
Arrays.asList(ModelAttributeEnum.BATCH, ModelAttributeEnum.STREAMING, ModelAttributeEnum.HYBRID,
ModelAttributeEnum.SECOND_STORAGE),
- null, null, true);
+ null, null, true, true);
+
+ DataResult<List<NDataModel>> modelResult2 = modelService.getModels(liteRequest);
List<NDataModel> models2 = modelResult2.getValue();
- FusionModelResponse model2 = (FusionModelResponse) models2.get(0);
+ NDataModelLiteResponse model2 = (NDataModelLiteResponse) models2.get(0);
Assert.assertEquals(12010, model2.getOldParams().getInputRecordCnt());
Assert.assertEquals(1505415, model2.getOldParams().getInputRecordSizeBytes());
Assert.assertEquals(396, model2.getOldParams().getSizeKB());
+
+ ModelQueryParams request = new ModelQueryParams("4965c827-fbb4-4ea1-a744-3f341a3b030d", modelName2, true,
+ project, "ADMIN", Lists.newArrayList(), "", 0, 10, "last_modify", true, null,
+ Arrays.asList(ModelAttributeEnum.BATCH, ModelAttributeEnum.STREAMING, ModelAttributeEnum.HYBRID,
+ ModelAttributeEnum.SECOND_STORAGE),
+ null, null, true, false);
+ DataResult<List<NDataModel>> modelResult3 = modelService.getModels(request);
+ List<NDataModel> models3 = modelResult3.getValue();
+ FusionModelResponse model3 = (FusionModelResponse) models3.get(0);
+
+ Assert.assertEquals(12010, model3.getOldParams().getInputRecordCnt());
+ Assert.assertEquals(1505415, model3.getOldParams().getInputRecordSizeBytes());
+ Assert.assertEquals(396, model3.getOldParams().getSizeKB());
}
@Test
@@ -279,9 +311,10 @@ public class ModelServiceQueryTest extends SourceTestCase {
Assert.assertEquals(1, models2.size());
doReturn(new ArrayList<>()).when(modelService).addOldParams(anyString(), any());
- val models3 = modelService.getModels("741ca86a-1f13-46da-a59f-95fb68615e3a", null, true, "default", "ADMIN",
- Lists.newArrayList(), "DEFAULT.TEST_KYLIN_FACT", 0, 8, "last_modify", true, null, null, null, null,
- true);
+ ModelQueryParams request = new ModelQueryParams("741ca86a-1f13-46da-a59f-95fb68615e3a", null, true, "default",
+ "ADMIN", Lists.newArrayList(), "DEFAULT.TEST_KYLIN_FACT", 0, 8, "last_modify", true, null, null, null,
+ null, true, false);
+ val models3 = modelService.getModels(request);
Assert.assertEquals(1, models3.getTotalSize());
}
}
diff --git a/src/second-storage/clickhouse-it/src/test/java/io/kyligence/kap/secondstorage/SecondStorageLockTest.java b/src/second-storage/clickhouse-it/src/test/java/io/kyligence/kap/secondstorage/SecondStorageLockTest.java
index 006c03a3ee..67f2a7a35a 100644
--- a/src/second-storage/clickhouse-it/src/test/java/io/kyligence/kap/secondstorage/SecondStorageLockTest.java
+++ b/src/second-storage/clickhouse-it/src/test/java/io/kyligence/kap/secondstorage/SecondStorageLockTest.java
@@ -137,6 +137,7 @@ import org.apache.kylin.rest.service.QueryHistoryScheduler;
import org.apache.kylin.rest.service.QueryHistoryService;
import org.apache.kylin.rest.service.SegmentHelper;
import org.apache.kylin.rest.service.params.MergeSegmentParams;
+import org.apache.kylin.rest.service.params.ModelQueryParams;
import org.apache.kylin.rest.service.params.RefreshSegmentParams;
import org.apache.kylin.rest.util.AclEvaluate;
import org.apache.kylin.rest.util.AclUtil;
@@ -2088,8 +2089,9 @@ public class SecondStorageLockTest implements JobWaiter {
val sum = segments.stream().mapToLong(NDataSegmentResponse::getSecondStorageSize).sum();
- DataResult<List<NDataModel>> result = modelService.getModels(modelId, null, true, getProject(), null, null,
- null, 0, 10, "last_modify", false, null, null, null, null, true);
+ ModelQueryParams request = new ModelQueryParams(modelId, null, true, getProject(), null, null, null, 0, 10,
+ "last_modify", false, null, null, null, null, true, false);
+ DataResult<List<NDataModel>> result = modelService.getModels(request);
result.getValue().stream().filter(nDataModel -> modelId.equals(nDataModel.getId())).forEach(nDataModel -> {
val nDataModelRes = (NDataModelResponse) nDataModel;