You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by xx...@apache.org on 2022/12/13 10:25:11 UTC
[kylin] 04/25: KYLIN-5333 V2 version api is forward compatible -- Model/CUBE/data source/BUILD API
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 1174339344bdf720713047f19fd8df6ca1f7acc7
Author: Hang Jia <75...@qq.com>
AuthorDate: Sat Oct 22 18:00:55 2022 +0800
KYLIN-5333 V2 version api is forward compatible -- Model/CUBE/data source/BUILD API
---
.../metadata/querymeta/TableMetaWithType.java | 11 ++
.../kylin/rest/controller/v2/JobControllerV2.java | 54 +++++++++-
.../rest/controller/v2/JobControllerV2Test.java | 62 +++++++++++-
.../kylin/rest/response/ExecutableResponse.java | 68 +++++++++----
.../org/apache/kylin/rest/service/JobService.java | 21 ++++
.../kylin/rest/response/TableDescResponse.java | 17 ++++
.../apache/kylin/rest/service/JobServiceTest.java | 101 ++++++++++++++++---
.../rest/controller/v2/NModelControllerV2.java | 21 ++++
.../rest/controller/v2/NTableControllerV2.java | 73 ++++++++++++++
.../rest/controller/NModelControllerV2Test.java | 22 +++-
.../rest/controller/NTableControllerV2Test.java | 111 +++++++++++++++++++++
.../kylin/rest/response/NDataModelOldParams.java | 38 +++++++
.../apache/kylin/rest/service/ModelService.java | 1 +
.../kylin/rest/service/TableServiceTest.java | 35 +++++++
.../rest/controller/v2/NQueryMetaController.java | 10 +-
.../apache/kylin/rest/service/QueryService.java | 25 +++++
.../kylin/rest/service/QueryServiceTest.java | 41 ++++++++
17 files changed, 669 insertions(+), 42 deletions(-)
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/querymeta/TableMetaWithType.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/querymeta/TableMetaWithType.java
index 25c8f4b55e..64eaaf65ba 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/querymeta/TableMetaWithType.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/querymeta/TableMetaWithType.java
@@ -20,6 +20,7 @@ package org.apache.kylin.metadata.querymeta;
import java.io.Serializable;
import java.util.HashSet;
+import java.util.stream.Collectors;
import lombok.NoArgsConstructor;
@@ -53,6 +54,16 @@ public class TableMetaWithType extends TableMeta {
TYPE = new HashSet<tableTypeEnum>();
}
+ public static TableMetaWithType ofColumnMeta(TableMeta tableMeta) {
+ TableMetaWithType tableMetaWithType = new TableMetaWithType(tableMeta.getTABLE_CAT(),
+ tableMeta.getTABLE_SCHEM(), tableMeta.getTABLE_NAME(), tableMeta.getTABLE_TYPE(),
+ tableMeta.getREMARKS(), tableMeta.getTYPE_CAT(), tableMeta.getTYPE_SCHEM(), tableMeta.getTYPE_NAME(),
+ tableMeta.getSELF_REFERENCING_COL_NAME(), tableMeta.getREF_GENERATION());
+ tableMetaWithType.setColumns(
+ tableMeta.getColumns().stream().map(ColumnMetaWithType::ofColumnMeta).collect(Collectors.toList()));
+ return tableMetaWithType;
+ }
+
public HashSet<tableTypeEnum> getTYPE() {
return TYPE;
}
diff --git a/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/v2/JobControllerV2.java b/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/v2/JobControllerV2.java
index 592b3be80e..acc02883b4 100644
--- a/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/v2/JobControllerV2.java
+++ b/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/v2/JobControllerV2.java
@@ -25,11 +25,14 @@ import java.util.List;
import java.util.Map;
import java.util.Objects;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.kylin.common.KylinVersion;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.job.constant.JobActionEnum;
import org.apache.kylin.job.constant.JobStatusEnum;
import org.apache.kylin.rest.controller.BaseController;
import org.apache.kylin.rest.request.JobFilter;
+import org.apache.kylin.rest.response.DataResult;
import org.apache.kylin.rest.response.EnvelopeResponse;
import org.apache.kylin.rest.response.ExecutableResponse;
import org.apache.kylin.rest.service.JobService;
@@ -53,6 +56,7 @@ import io.swagger.annotations.ApiOperation;
public class JobControllerV2 extends BaseController {
private static final String JOB_ID_ARG_NAME = "jobId";
+ private static final String STEP_ID_ARG_NAME = "stepId";
@Autowired
@Qualifier("jobService")
@@ -79,12 +83,17 @@ public class JobControllerV2 extends BaseController {
@RequestParam(value = "status", required = false, defaultValue = "") Integer[] status,
@RequestParam(value = "timeFilter") Integer timeFilter,
@RequestParam(value = "jobName", required = false) String jobName,
- @RequestParam(value = "projectName") String project,
+ @RequestParam(value = "projectName", required = false) String project,
@RequestParam(value = "key", required = false) String key,
@RequestParam(value = "pageOffset", required = false, defaultValue = "0") Integer pageOffset,
@RequestParam(value = "pageSize", required = false, defaultValue = "10") Integer pageSize,
@RequestParam(value = "sortBy", required = false, defaultValue = "last_modified") String sortBy,
+ @RequestParam(value = "sortby", required = false) String sortby, //param for 3x
@RequestParam(value = "reverse", required = false, defaultValue = "true") Boolean reverse) {
+ // 3x default last_modify
+ if (!StringUtils.isEmpty(sortby) && !"last_modify".equals(sortby)) {
+ sortBy = sortby;
+ }
checkNonNegativeIntegerArg("pageOffset", pageOffset);
checkNonNegativeIntegerArg("pageSize", pageSize);
List<String> statuses = Lists.newArrayList();
@@ -100,9 +109,50 @@ public class JobControllerV2 extends BaseController {
JobFilter jobFilter = new JobFilter(statuses,
Objects.isNull(jobName) ? Lists.newArrayList() : Lists.newArrayList(jobName), timeFilter, null, key,
project, sortBy, reverse);
- List<ExecutableResponse> executables = jobService.listJobs(jobFilter);
+ List<ExecutableResponse> executables = null;
+ if (!StringUtils.isEmpty(project)) {
+ executables = jobService.listJobs(jobFilter);
+ } else {
+ DataResult<List<ExecutableResponse>> dataResult = jobService.listGlobalJobs(jobFilter, 0,
+ Integer.MAX_VALUE);
+ if (dataResult != null) {
+ executables = dataResult.getValue();
+ }
+ }
executables = jobService.addOldParams(executables);
+ executables.forEach(
+ executableResponse -> executableResponse.setVersion(KylinVersion.getCurrentVersion().toString()));
Map<String, Object> result = getDataResponse("jobs", executables, pageOffset, pageSize);
return new EnvelopeResponse<>(KylinException.CODE_SUCCESS, result, "");
}
+
+ @ApiOperation(value = "getJob", tags = { "DW" })
+ @GetMapping(value = "/{jobId}")
+ @ResponseBody
+ public EnvelopeResponse<ExecutableResponse> getJob(@PathVariable(value = "jobId") String jobId) {
+ checkRequiredArg(JOB_ID_ARG_NAME, jobId);
+ ExecutableResponse jobInstance = jobService.getJobInstance(jobId);
+ List<ExecutableResponse> executables = Lists.newArrayList(jobInstance);
+ executables = jobService.addOldParams(executables);
+ if (executables != null && executables.size() != 0) {
+ jobInstance = executables.get(0);
+ }
+ if (jobInstance != null) {
+ jobInstance.setVersion(KylinVersion.getCurrentVersion().toString());
+ }
+ return new EnvelopeResponse<>(KylinException.CODE_SUCCESS, jobInstance, "");
+ }
+
+ @ApiOperation(value = "getJobOutput", tags = { "DW" })
+ @GetMapping(value = "/{job_id:.+}/steps/{step_id:.+}/output")
+ @ResponseBody
+ public EnvelopeResponse<Map<String, Object>> getJobOutput(@PathVariable("job_id") String jobId,
+ @PathVariable("step_id") String stepId) {
+ String project = jobService.getProjectByJobId(jobId);
+ checkProjectName(project);
+ Map<String, Object> result = jobService.getStepOutput(project, jobId, stepId);
+ result.put(JOB_ID_ARG_NAME, jobId);
+ result.put(STEP_ID_ARG_NAME, stepId);
+ return new EnvelopeResponse<>(KylinException.CODE_SUCCESS, result, "");
+ }
}
diff --git a/src/data-loading-server/src/test/java/org/apache/kylin/rest/controller/v2/JobControllerV2Test.java b/src/data-loading-server/src/test/java/org/apache/kylin/rest/controller/v2/JobControllerV2Test.java
index 1fe66e13cc..a2ed9db17d 100644
--- a/src/data-loading-server/src/test/java/org/apache/kylin/rest/controller/v2/JobControllerV2Test.java
+++ b/src/data-loading-server/src/test/java/org/apache/kylin/rest/controller/v2/JobControllerV2Test.java
@@ -23,13 +23,17 @@ import static org.apache.kylin.common.constant.HttpConstant.HTTP_VND_APACHE_KYLI
import java.util.ArrayList;
import java.util.List;
+import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
import org.apache.kylin.job.constant.JobActionEnum;
import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.rest.request.JobFilter;
+import org.apache.kylin.rest.request.JobUpdateRequest;
+import org.apache.kylin.rest.response.DataResult;
import org.apache.kylin.rest.response.ExecutableResponse;
import org.apache.kylin.rest.service.JobService;
import org.apache.kylin.rest.util.AclEvaluate;
import org.apache.kylin.rest.util.AclUtil;
+import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.mockito.InjectMocks;
@@ -48,7 +52,7 @@ import org.springframework.test.web.servlet.setup.MockMvcBuilders;
import com.google.common.collect.Lists;
-public class JobControllerV2Test {
+public class JobControllerV2Test extends NLocalFileMetadataTestCase {
private MockMvc mockMvc;
@@ -75,6 +79,12 @@ public class JobControllerV2Test {
SecurityContextHolder.getContext().setAuthentication(authentication);
ReflectionTestUtils.setField(aclEvaluate, "aclUtil", aclUtil);
ReflectionTestUtils.setField(jobService, "aclEvaluate", aclEvaluate);
+ createTestMetadata();
+ }
+
+ @After
+ public void tearDown() {
+ cleanupTestMetadata();
}
@Test
@@ -107,7 +117,54 @@ public class JobControllerV2Test {
.andExpect(MockMvcResultMatchers.status().isOk()).andReturn();
Mockito.verify(jobControllerV2).getJobList(new Integer[] { 0 }, 1, "", "default", null, 0, 10, "last_modified",
- true);
+ null, true);
+ }
+
+ @Test
+ public void testGetJobsWithoutProjectAndSortby() throws Exception {
+ List<ExecutableResponse> jobs = new ArrayList<>();
+ List<String> jobNames = Lists.newArrayList();
+ JobFilter jobFilter = new JobFilter(Lists.newArrayList(), jobNames, 4, null, null, null, "job_name", true);
+ Mockito.when(jobService.listGlobalJobs(jobFilter, 0, Integer.MAX_VALUE))
+ .thenReturn(new DataResult<>(jobs, 0, 0, 0));
+ mockMvc.perform(
+ MockMvcRequestBuilders.get("/api/jobs").contentType(MediaType.APPLICATION_JSON).param("timeFilter", "4")
+ .param("sortby", "job_name").accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V2_JSON)))
+ .andExpect(MockMvcResultMatchers.status().isOk()).andReturn();
+
+ Mockito.verify(jobControllerV2).getJobList(new Integer[0], 4, null, null, null, 0, 10, "last_modified",
+ "job_name", true);
+ }
+
+ @Test
+ public void testGetJob() throws Exception {
+ mockJobUpdateRequest();
+ String jobId = "e1ad7bb0-522e-456a-859d-2eab1df448de";
+ mockMvc.perform(MockMvcRequestBuilders.get("/api/jobs/{jobId}", jobId)
+ .accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V2_JSON)))
+ .andExpect(MockMvcResultMatchers.status().isOk()).andReturn();
+
+ Mockito.verify(jobControllerV2).getJob(jobId);
+ }
+
+ @Test
+ public void testGetJobOutput() throws Exception {
+ mockJobUpdateRequest();
+ String jobId = "e1ad7bb0-522e-456a-859d-2eab1df448de";
+ Mockito.when(jobService.getProjectByJobId(jobId)).thenReturn("default");
+ mockMvc.perform(MockMvcRequestBuilders.get("/api/jobs/{job_id:.+}/steps/{step_id:.+}/output", jobId, jobId)
+ .accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V2_JSON)))
+ .andExpect(MockMvcResultMatchers.status().isOk()).andReturn();
+
+ Mockito.verify(jobControllerV2).getJobOutput(jobId, jobId);
+ }
+
+ private JobUpdateRequest mockJobUpdateRequest() {
+ JobUpdateRequest jobUpdateRequest = new JobUpdateRequest();
+ jobUpdateRequest.setProject("default");
+ jobUpdateRequest.setAction("RESUME");
+ jobUpdateRequest.setJobIds(Lists.newArrayList("e1ad7bb0-522e-456a-859d-2eab1df448de"));
+ return jobUpdateRequest;
}
@Test
@@ -147,7 +204,6 @@ public class JobControllerV2Test {
.accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V2_JSON)))
.andExpect(MockMvcResultMatchers.status().isOk());
-
}
}
diff --git a/src/data-loading-service/src/main/java/org/apache/kylin/rest/response/ExecutableResponse.java b/src/data-loading-service/src/main/java/org/apache/kylin/rest/response/ExecutableResponse.java
index a9612ad681..4a0ea0f381 100644
--- a/src/data-loading-service/src/main/java/org/apache/kylin/rest/response/ExecutableResponse.java
+++ b/src/data-loading-service/src/main/java/org/apache/kylin/rest/response/ExecutableResponse.java
@@ -18,23 +18,17 @@
package org.apache.kylin.rest.response;
-import com.clearspring.analytics.util.Lists;
-import com.fasterxml.jackson.annotation.JsonManagedReference;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonUnwrapped;
-import com.google.common.collect.Maps;
-import org.apache.kylin.engine.spark.job.NSparkSnapshotJob;
-import org.apache.kylin.engine.spark.job.NTableSamplingJob;
-import lombok.AllArgsConstructor;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.Setter;
-import lombok.SneakyThrows;
-import lombok.val;
-import lombok.var;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.JsonUtil;
+import org.apache.kylin.engine.spark.job.NSparkSnapshotJob;
+import org.apache.kylin.engine.spark.job.NTableSamplingJob;
import org.apache.kylin.job.SecondStorageCleanJobUtil;
import org.apache.kylin.job.constant.JobStatusEnum;
import org.apache.kylin.job.execution.AbstractExecutable;
@@ -49,9 +43,19 @@ import org.apache.kylin.metadata.model.NTableMetadataManager;
import org.apache.kylin.metadata.model.SegmentStatusEnumToDisplay;
import org.apache.kylin.metadata.model.TableDesc;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
+import com.clearspring.analytics.util.Lists;
+import com.fasterxml.jackson.annotation.JsonManagedReference;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonUnwrapped;
+import com.google.common.collect.Maps;
+
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+import lombok.SneakyThrows;
+import lombok.val;
+import lombok.var;
@Setter
@Getter
@@ -111,6 +115,36 @@ public class ExecutableResponse implements Comparable<ExecutableResponse> {
private static final String SNAPSHOT_FULL_RANGE = "FULL";
private static final String SNAPSHOT_INC_RANGE = "INC";
+ @JsonProperty("version")
+ protected String version;
+
+ @JsonProperty("related_segment")
+ public String getRelatedSegment() {
+ return CollectionUtils.isEmpty(targetSegments) ? "" : String.join(",", targetSegments);
+ }
+
+ @JsonProperty("progress")
+ public double getProgress() {
+ int completedStepCount = 0;
+
+ for (ExecutableStepResponse step : this.getSteps()) {
+ if (step.getStatus().equals(JobStatusEnum.FINISHED)) {
+ completedStepCount++;
+ }
+ }
+ if (steps.isEmpty()) {
+ return 0.0;
+ }
+ return 100.0 * completedStepCount / steps.size();
+ }
+
+ public List<ExecutableStepResponse> getSteps() {
+ if (steps == null) {
+ steps = Collections.emptyList();
+ }
+ return steps;
+ }
+
private static ExecutableResponse newInstance(AbstractExecutable abstractExecutable) {
ExecutableResponse executableResponse = new ExecutableResponse();
executableResponse.setDataRangeEnd(abstractExecutable.getDataRangeEnd());
diff --git a/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/JobService.java b/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/JobService.java
index 60bc38c812..22ea2b774a 100644
--- a/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/JobService.java
+++ b/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/JobService.java
@@ -32,6 +32,7 @@ import java.util.Calendar;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
+import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
@@ -1097,6 +1098,26 @@ public class JobService extends BasicService implements JobSupporter {
return executableManager.getOutputFromHDFSByJobId(jobId, stepId).getVerboseMsg();
}
+ public Map<String, Object> getStepOutput(String project, String jobId, String stepId) {
+ aclEvaluate.checkProjectOperationPermission(project);
+ val executableManager = getManager(NExecutableManager.class, project);
+ Output output = executableManager.getOutputFromHDFSByJobId(jobId, stepId);
+ Map<String, Object> result = new HashMap<>();
+ result.put("cmd_output", output.getVerboseMsg());
+
+ Map<String, String> info = output.getExtra();
+ List<String> servers = Lists.newArrayList();
+ if (info != null && info.get("nodes") != null) {
+ servers = Lists.newArrayList(info.get("nodes").split(","));
+ }
+ List<String> nodes = servers.stream().map(server -> {
+ String[] split = server.split(":");
+ return split[0] + ":" + split[1];
+ }).collect(Collectors.toList());
+ result.put("nodes", nodes);
+ return result;
+ }
+
@SneakyThrows
public InputStream getAllJobOutput(String project, String jobId, String stepId) {
aclEvaluate.checkProjectOperationPermission(project);
diff --git a/src/datasource-service/src/main/java/org/apache/kylin/rest/response/TableDescResponse.java b/src/datasource-service/src/main/java/org/apache/kylin/rest/response/TableDescResponse.java
index df6c6798b8..5519386bb6 100644
--- a/src/datasource-service/src/main/java/org/apache/kylin/rest/response/TableDescResponse.java
+++ b/src/datasource-service/src/main/java/org/apache/kylin/rest/response/TableDescResponse.java
@@ -30,6 +30,7 @@ import org.apache.kylin.metadata.model.SegmentRange;
import org.apache.kylin.metadata.model.TableDesc;
import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.collect.Maps;
import lombok.Getter;
import lombok.Setter;
@@ -79,6 +80,22 @@ public class TableDescResponse extends TableDesc {
}
}
+ @JsonProperty(value = "cardinality", access = JsonProperty.Access.READ_ONLY)
+ public Map<String, Long> getCardinality() {
+ Map<String, Long> cardinality = Maps.newHashMapWithExpectedSize(extColumns.length);
+ for (ColumnDescResponse extColumn : extColumns) {
+ if (extColumn.getCardinality() != null) {
+ cardinality.put(extColumn.getName(), extColumn.getCardinality());
+ }
+ }
+ return cardinality;
+ }
+
+ @JsonProperty(value = "is_transactional", access = JsonProperty.Access.READ_ONLY)
+ public boolean getTransactionalV2() {
+ return super.isTransactional();
+ }
+
@Getter
@Setter
public class ColumnDescResponse extends ColumnDesc {
diff --git a/src/job-service/src/test/java/org/apache/kylin/rest/service/JobServiceTest.java b/src/job-service/src/test/java/org/apache/kylin/rest/service/JobServiceTest.java
index 49bf1d5a9a..e29d23a875 100644
--- a/src/job-service/src/test/java/org/apache/kylin/rest/service/JobServiceTest.java
+++ b/src/job-service/src/test/java/org/apache/kylin/rest/service/JobServiceTest.java
@@ -1206,14 +1206,14 @@ public class JobServiceTest extends NLocalFileMetadataTestCase {
private long getCreateTime(String name) {
switch (name) {
- case "1":
- return 1560324101000L;
- case "2":
- return 1560324102000L;
- case "3":
- return 1560324103000L;
- default:
- return 0L;
+ case "1":
+ return 1560324101000L;
+ case "2":
+ return 1560324102000L;
+ case "3":
+ return 1560324103000L;
+ default:
+ return 0L;
}
}
@@ -1331,8 +1331,8 @@ public class JobServiceTest extends NLocalFileMetadataTestCase {
String sampleLog = "";
try (InputStream allJobOutput = jobService.getAllJobOutput("default", "e1ad7bb0-522e-456a-859d-2eab1df448de",
"e1ad7bb0-522e-456a-859d-2eab1df448de");
- BufferedReader reader = new BufferedReader(
- new InputStreamReader(allJobOutput, Charset.defaultCharset()))) {
+ BufferedReader reader = new BufferedReader(
+ new InputStreamReader(allJobOutput, Charset.defaultCharset()))) {
String line;
StringBuilder sampleData = new StringBuilder();
@@ -1581,14 +1581,14 @@ public class JobServiceTest extends NLocalFileMetadataTestCase {
// testGetProjectNameAndJobStepId_NotContains
String yarnAppId1 = "application";
overwriteSystemProp("kylin.engine.spark.cluster-manager-class-name", sparkClusterManagerName);
- Assert.assertThrows("Async profiler status error, yarnAppId entered incorrectly, please try again.", KylinException.class,
- () -> jobService.getProjectNameAndJobStepId(yarnAppId1));
+ Assert.assertThrows("Async profiler status error, yarnAppId entered incorrectly, please try again.",
+ KylinException.class, () -> jobService.getProjectNameAndJobStepId(yarnAppId1));
// testGetProjectNameAndJobStepId_LengthError
String yarnAppId2 = "application_";
overwriteSystemProp("kylin.engine.spark.cluster-manager-class-name", sparkClusterManagerName);
- Assert.assertThrows("Async profiler status error, yarnAppId entered incorrectly, please try again.", KylinException.class,
- () -> jobService.getProjectNameAndJobStepId(yarnAppId2));
+ Assert.assertThrows("Async profiler status error, yarnAppId entered incorrectly, please try again.",
+ KylinException.class, () -> jobService.getProjectNameAndJobStepId(yarnAppId2));
// testGetProjectNameAndJobStepId_NotContainsJob
String yarnAppId = "application_1554187389076_-1";
@@ -1803,4 +1803,77 @@ public class JobServiceTest extends NLocalFileMetadataTestCase {
}
Assert.assertEquals("", errorMsg);
}
+
+ @Test
+ public void testGetStepOutput() {
+ String jobId = "e1ad7bb0-522e-456a-859d-2eab1df448de";
+ NExecutableManager manager = NExecutableManager.getInstance(jobService.getConfig(), "default");
+ ExecutableOutputPO executableOutputPO = new ExecutableOutputPO();
+ Map<String, String> info = Maps.newHashMap();
+ info.put("nodes", "localhost:7070:all");
+ executableOutputPO.setInfo(info);
+ manager.updateJobOutputToHDFS(KylinConfig.getInstanceFromEnv().getJobTmpOutputStorePath("default", jobId),
+ executableOutputPO);
+
+ Map<String, Object> result = Maps.newHashMap();
+ result.put("nodes", Lists.newArrayList("localhost:7070"));
+ result.put("cmd_output", null);
+ Assert.assertEquals(result, jobService.getStepOutput("default", jobId, jobId));
+
+ executableOutputPO.setInfo(null);
+ manager.updateJobOutputToHDFS(KylinConfig.getInstanceFromEnv().getJobTmpOutputStorePath("default", jobId),
+ executableOutputPO);
+
+ result = Maps.newHashMap();
+ result.put("nodes", Lists.newArrayList());
+ result.put("cmd_output", null);
+ Assert.assertEquals(result, jobService.getStepOutput("default", jobId, jobId));
+
+ info = Maps.newHashMap();
+ executableOutputPO.setInfo(info);
+ manager.updateJobOutputToHDFS(KylinConfig.getInstanceFromEnv().getJobTmpOutputStorePath("default", jobId),
+ executableOutputPO);
+
+ result = Maps.newHashMap();
+ result.put("nodes", Lists.newArrayList());
+ result.put("cmd_output", null);
+ Assert.assertEquals(result, jobService.getStepOutput("default", jobId, jobId));
+ }
+
+ @Test
+ public void testExecutableResponse() throws Exception {
+ val modelManager = mock(NDataModelManager.class);
+
+ Mockito.when(modelService.getManager(NDataModelManager.class, "default")).thenReturn(modelManager);
+ NDataModel nDataModel = mock(NDataModel.class);
+ Mockito.when(modelManager.getDataModelDesc(Mockito.anyString())).thenReturn(nDataModel);
+
+ NExecutableManager executableManager = Mockito.spy(NExecutableManager.getInstance(getTestConfig(), "default"));
+ Mockito.when(jobService.getManager(NExecutableManager.class, "default")).thenReturn(executableManager);
+ val mockJobs = mockDetailJobs(false);
+ Mockito.when(executableManager.getAllJobs(Mockito.anyLong(), Mockito.anyLong())).thenReturn(mockJobs);
+ for (ExecutablePO po : mockJobs) {
+ AbstractExecutable exe = executableManager.fromPO(po);
+ Mockito.when(executableManager.getJob(po.getId())).thenReturn(exe);
+ }
+ getTestConfig().setProperty("kylin.streaming.enabled", "false");
+ // test size
+ List<String> jobNames = Lists.newArrayList();
+ JobFilter jobFilter = new JobFilter(Lists.newArrayList(), jobNames, 4, "", "", "default", "", true);
+ List<ExecutableResponse> jobs = jobService.listJobs(jobFilter);
+ List<ExecutableResponse> executableResponses = jobService.addOldParams(jobs);
+ ExecutableResponse executable = executableResponses.get(0);
+ Assert.assertEquals("", executable.getRelatedSegment());
+ Assert.assertEquals(0, executable.getProgress(), 0);
+ executable.getSteps().get(0).setStatus(JobStatusEnum.FINISHED);
+ Assert.assertEquals(33, executable.getProgress(), 1);
+ executable.setSteps(null);
+ String uuid = UUID.randomUUID().toString();
+ executable.setTargetSegments(Lists.newArrayList(uuid));
+ Assert.assertEquals(0.0, executable.getProgress(), 0);
+ Assert.assertEquals(uuid, executable.getRelatedSegment());
+ executable.setTargetSegments(Collections.emptyList());
+ Assert.assertEquals(0.0, executable.getProgress(), 0);
+ Assert.assertEquals("", executable.getRelatedSegment());
+ }
}
diff --git a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/v2/NModelControllerV2.java b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/v2/NModelControllerV2.java
index 7552b476d5..224b68c6a8 100644
--- a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/v2/NModelControllerV2.java
+++ b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/v2/NModelControllerV2.java
@@ -18,8 +18,10 @@
package org.apache.kylin.rest.controller.v2;
import static org.apache.kylin.common.constant.HttpConstant.HTTP_VND_APACHE_KYLIN_V2_JSON;
+import static org.apache.kylin.common.exception.code.ErrorCodeServer.MODEL_ID_NOT_EXIST;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -32,11 +34,13 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.ResponseBody;
import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
import io.swagger.annotations.ApiOperation;
@@ -68,4 +72,21 @@ public class NModelControllerV2 extends NBasicController {
return new EnvelopeResponse<>(KylinException.CODE_SUCCESS, modelResponse, "");
}
+ @ApiOperation(value = "getModelDesc", tags = { "AI" })
+ @GetMapping(value = "/{projectName}/{modelName}", produces = { HTTP_VND_APACHE_KYLIN_V2_JSON })
+ @ResponseBody
+ public EnvelopeResponse<Map<String, Object>> getModelDesc(@PathVariable("projectName") String project,
+ @PathVariable("modelName") String modelAlias) {
+ checkProjectName(project);
+ List<NDataModel> models = new ArrayList<>(
+ modelService.getModels(modelAlias, project, true, null, Lists.newArrayList(), "last_modify", true));
+ if (models.size() == 0) {
+ throw new KylinException(MODEL_ID_NOT_EXIST, modelAlias);
+ }
+ models = modelService.addOldParams(project, models);
+
+ HashMap<String, Object> modelResponse = new HashMap<>();
+ modelResponse.put("model", models.size() == 0 ? Maps.newHashMap() : models.get(0));
+ return new EnvelopeResponse<>(KylinException.CODE_SUCCESS, modelResponse, "");
+ }
}
diff --git a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/v2/NTableControllerV2.java b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/v2/NTableControllerV2.java
new file mode 100644
index 0000000000..0c195a84f4
--- /dev/null
+++ b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/v2/NTableControllerV2.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kylin.rest.controller.v2;
+
+import static org.apache.kylin.common.constant.HttpConstant.HTTP_VND_APACHE_KYLIN_V2_JSON;
+import static org.apache.kylin.common.exception.ServerErrorCode.UNSUPPORTED_STREAMING_OPERATION;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.kylin.common.exception.KylinException;
+import org.apache.kylin.common.msg.MsgPicker;
+import org.apache.kylin.metadata.model.ISourceAware;
+import org.apache.kylin.metadata.model.TableDesc;
+import org.apache.kylin.rest.controller.NBasicController;
+import org.apache.kylin.rest.response.DataResult;
+import org.apache.kylin.rest.response.EnvelopeResponse;
+import org.apache.kylin.rest.service.TableService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Controller;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.ResponseBody;
+
+import io.swagger.annotations.ApiOperation;
+
+@Controller
+@RequestMapping(value = "/api/tables", produces = { HTTP_VND_APACHE_KYLIN_V2_JSON })
+public class NTableControllerV2 extends NBasicController {
+
+ @Autowired
+ private TableService tableService;
+
+ @ApiOperation(value = "getTableDesc", tags = { "AI" })
+ @GetMapping(value = "")
+ @ResponseBody
+ public EnvelopeResponse<List<TableDesc>> getTableDesc(@RequestParam(value = "project") String project,
+ @RequestParam(value = "table", required = false) String table,
+ @RequestParam(value = "database", required = false) String database,
+ @RequestParam(value = "is_fuzzy", required = false, defaultValue = "false") boolean isFuzzy,
+ @RequestParam(value = "ext", required = false, defaultValue = "true") boolean withExt,
+ @RequestParam(value = "page_offset", required = false, defaultValue = "0") Integer offset,
+ @RequestParam(value = "page_size", required = false, defaultValue = "10") Integer limit,
+ @RequestParam(value = "source_type", required = false, defaultValue = "9") Integer sourceType)
+ throws IOException {
+ checkProjectName(project);
+ checkNonNegativeIntegerArg("page_offset", offset);
+ if (sourceType == ISourceAware.ID_STREAMING) {
+ throw new KylinException(UNSUPPORTED_STREAMING_OPERATION,
+ MsgPicker.getMsg().getStreamingOperationNotSupport());
+ }
+ List<TableDesc> result = tableService.getTableDescByType(project, withExt, table, database, isFuzzy,
+ sourceType);
+ return new EnvelopeResponse<>(KylinException.CODE_SUCCESS, DataResult.get(result, offset, limit).getValue(),
+ "");
+ }
+}
\ No newline at end of file
diff --git a/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NModelControllerV2Test.java b/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NModelControllerV2Test.java
index cf6a0ced47..f7f0dfe2fe 100644
--- a/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NModelControllerV2Test.java
+++ b/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NModelControllerV2Test.java
@@ -23,9 +23,9 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
-import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
import org.apache.kylin.metadata.model.NDataModel;
+import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.rest.controller.v2.NModelControllerV2;
import org.apache.kylin.rest.response.NDataModelResponse;
import org.apache.kylin.rest.response.RelatedModelResponse;
@@ -47,6 +47,8 @@ import org.springframework.test.web.servlet.request.MockMvcRequestBuilders;
import org.springframework.test.web.servlet.result.MockMvcResultMatchers;
import org.springframework.test.web.servlet.setup.MockMvcBuilders;
+import com.google.common.collect.Lists;
+
public class NModelControllerV2Test extends NLocalFileMetadataTestCase {
private MockMvc mockMvc;
@@ -139,4 +141,22 @@ public class NModelControllerV2Test extends NLocalFileMetadataTestCase {
Mockito.verify(nModelControllerV2).getModels("", true, "default", 0, 10, "last_modify", true);
}
+ @Test
+ public void testGetModelDesc() throws Exception {
+ Mockito.when(modelService.getModels("model1", "default", true, null, Lists.newArrayList(), "last_modify", true))
+ .thenReturn(mockModelDesc());
+ mockMvc.perform(MockMvcRequestBuilders.get("/api/models/default/model1")
+ .accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V2_JSON)))
+ .andExpect(MockMvcResultMatchers.status().isOk()).andReturn();
+ Mockito.verify(nModelControllerV2).getModelDesc("default", "model1");
+ }
+
+ private List<NDataModelResponse> mockModelDesc() {
+ final List<NDataModelResponse> models = new ArrayList<>();
+ NDataModel model = new NDataModel();
+ model.setAlias("model1");
+ model.setUuid("model1");
+ models.add(new NDataModelResponse(model));
+ return models;
+ }
}
diff --git a/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NTableControllerV2Test.java b/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NTableControllerV2Test.java
new file mode 100644
index 0000000000..6e1b3e608f
--- /dev/null
+++ b/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NTableControllerV2Test.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kylin.rest.controller;
+
+import static org.apache.kylin.common.constant.HttpConstant.HTTP_VND_APACHE_KYLIN_V2_JSON;
+
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
+import org.apache.kylin.metadata.model.TableDesc;
+import org.apache.kylin.rest.constant.Constant;
+import org.apache.kylin.rest.controller.v2.NTableControllerV2;
+import org.apache.kylin.rest.service.TableService;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.MockitoAnnotations;
+import org.springframework.http.MediaType;
+import org.springframework.security.authentication.TestingAuthenticationToken;
+import org.springframework.security.core.Authentication;
+import org.springframework.security.core.context.SecurityContextHolder;
+import org.springframework.test.web.servlet.MockMvc;
+import org.springframework.test.web.servlet.request.MockMvcRequestBuilders;
+import org.springframework.test.web.servlet.result.MockMvcResultMatchers;
+import org.springframework.test.web.servlet.setup.MockMvcBuilders;
+
+public class NTableControllerV2Test extends NLocalFileMetadataTestCase {
+
+ private static final String APPLICATION_JSON = HTTP_VND_APACHE_KYLIN_V2_JSON;
+
+ private MockMvc mockMvc;
+
+ @Rule
+ public ExpectedException thrown = ExpectedException.none();
+
+ @Mock
+ private TableService tableService;
+
+ @InjectMocks
+ private NTableControllerV2 nTableControllerV2 = Mockito.spy(new NTableControllerV2());
+
+ private final Authentication authentication = new TestingAuthenticationToken("ADMIN", "ADMIN", Constant.ROLE_ADMIN);
+
+ @Before
+ public void setup() {
+ MockitoAnnotations.initMocks(this);
+
+ mockMvc = MockMvcBuilders.standaloneSetup(nTableControllerV2).defaultRequest(MockMvcRequestBuilders.get("/"))
+ .defaultResponseCharacterEncoding(StandardCharsets.UTF_8).build();
+
+ SecurityContextHolder.getContext().setAuthentication(authentication);
+ createTestMetadata();
+ }
+
+ @After
+ public void tearDown() {
+ cleanupTestMetadata();
+ }
+
+ @Test
+ public void testGetTableDesc() throws Exception {
+ Mockito.when(tableService.getTableDesc("default", false, "", "DEFAULT", true)) //
+ .thenReturn(mockTables());
+ mockMvc.perform(MockMvcRequestBuilders.get("/api/tables") //
+ .contentType(MediaType.APPLICATION_JSON) //
+ .param("ext", "false") //
+ .param("project", "default") //
+ .param("table", "") //
+ .param("database", "DEFAULT") //
+ .param("page_offset", "0") //
+ .param("page_size", "10") //
+ .param("is_fuzzy", "true") //
+ .accept(MediaType.parseMediaType(APPLICATION_JSON))) //
+ .andExpect(MockMvcResultMatchers.status().isOk()).andReturn();
+
+ Mockito.verify(nTableControllerV2).getTableDesc("default", "", "DEFAULT", true, false, 0, 10, 9);
+ }
+
+ private List<TableDesc> mockTables() {
+ final List<TableDesc> tableDescs = new ArrayList<>();
+ TableDesc tableDesc = new TableDesc();
+ tableDesc.setName("table1");
+ tableDescs.add(tableDesc);
+ TableDesc tableDesc2 = new TableDesc();
+ tableDesc2.setName("table2");
+ tableDescs.add(tableDesc2);
+ return tableDescs;
+ }
+}
\ No newline at end of file
diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/response/NDataModelOldParams.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/response/NDataModelOldParams.java
index 39f80e7dd2..dda95a2a64 100644
--- a/src/modeling-service/src/main/java/org/apache/kylin/rest/response/NDataModelOldParams.java
+++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/response/NDataModelOldParams.java
@@ -18,11 +18,18 @@
package org.apache.kylin.rest.response;
import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import org.apache.commons.collections.CollectionUtils;
import org.apache.kylin.metadata.model.JoinTableDesc;
+import org.apache.kylin.metadata.model.ModelDimensionDesc;
import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.collect.Sets;
import lombok.Getter;
import lombok.Setter;
@@ -50,4 +57,35 @@ public class NDataModelOldParams implements Serializable {
@JsonProperty("project")
private String projectName;
+
+ @JsonProperty("dimensions")
+ private List<ModelDimensionDesc> dimensions;
+
+ public void setDimensions(List<NDataModelResponse.SimplifiedNamedColumn> simplifiedDimensions) {
+ if (CollectionUtils.isEmpty(simplifiedDimensions)) {
+ return;
+ }
+
+ Map<String, Set<String>> dimCandidate = new HashMap<>();
+ for (NDataModelResponse.SimplifiedNamedColumn dimension : simplifiedDimensions) {
+ String[] tableColumn = dimension.getAliasDotColumn().split("\\.");
+ String table = tableColumn[0];
+ String column = tableColumn[1];
+ addCandidate(dimCandidate, table, column);
+ }
+
+ List<ModelDimensionDesc> dims = new ArrayList<>();
+ for (Map.Entry<String, Set<String>> dimensionEntry : dimCandidate.entrySet()) {
+ ModelDimensionDesc dimension = new ModelDimensionDesc();
+ dimension.setTable(dimensionEntry.getKey());
+ dimension.setColumns(dimensionEntry.getValue().toArray(new String[0]));
+ dims.add(dimension);
+ }
+ this.dimensions = dims;
+ }
+
+ private static void addCandidate(Map<String, Set<String>> tblColMap, String table, String column) {
+ tblColMap.computeIfAbsent(table, k -> Sets.newHashSet());
+ tblColMap.get(table).add(column);
+ }
}
diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
index 7fcf9f610c..07cab15306 100644
--- a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
+++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
@@ -359,6 +359,7 @@ public class ModelService extends AbstractModelService implements TableModelSupp
if (model instanceof NDataModelResponse) {
oldParams.setProjectName(model.getProject());
oldParams.setSizeKB(((NDataModelResponse) model).getStorage() / 1024);
+ oldParams.setDimensions(((NDataModelResponse) model).getNamedColumns());
((NDataModelResponse) model).setOldParams(oldParams);
} else if (model instanceof RelatedModelResponse) {
((RelatedModelResponse) model).setOldParams(oldParams);
diff --git a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/TableServiceTest.java b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/TableServiceTest.java
index bde1acaccc..ff8a49d330 100644
--- a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/TableServiceTest.java
+++ b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/TableServiceTest.java
@@ -1459,4 +1459,39 @@ public class TableServiceTest extends CSVSourceTestCase {
new ArrayList<>());
Assert.assertThrows(KylinException.class, func);
}
+
+ @Test
+ public void testTableDescResponseV2() throws IOException {
+ final String tableIdentity = "DEFAULT.TEST_COUNTRY";
+ final NTableMetadataManager tableMgr = NTableMetadataManager.getInstance(getTestConfig(), "newten");
+ final TableDesc tableDesc = tableMgr.getTableDesc(tableIdentity);
+ final TableExtDesc oldExtDesc = tableMgr.getOrCreateTableExt(tableDesc);
+ // mock table ext desc
+ TableExtDesc tableExt = new TableExtDesc(oldExtDesc);
+ tableExt.setIdentity(tableIdentity);
+ TableExtDesc.ColumnStats col1 = new TableExtDesc.ColumnStats();
+ col1.setCardinality(100);
+ col1.setTableExtDesc(tableExt);
+ col1.setColumnName(tableDesc.getColumns()[0].getName());
+ col1.setMinValue("America");
+ col1.setMaxValue("Zimbabwe");
+ col1.setNullCount(0);
+ tableExt.setColumnStats(Lists.newArrayList(col1));
+ tableMgr.mergeAndUpdateTableExt(oldExtDesc, tableExt);
+
+ final List<TableDesc> tables = tableService.getTableDesc("newten", true, "TEST_COUNTRY", "DEFAULT", true);
+ Assert.assertEquals(1, tables.size());
+ Assert.assertTrue(tables.get(0) instanceof TableDescResponse);
+ TableDescResponse t = (TableDescResponse) tables.get(0);
+ Map<String, Long> cardinality = t.getCardinality();
+ for (int i = 0; i < t.getExtColumns().length; i++) {
+ if (t.getExtColumns()[i].getCardinality() != null) {
+ Assert.assertEquals(cardinality.get(t.getExtColumns()[i].getName()),
+ t.getExtColumns()[i].getCardinality());
+ }
+ }
+ Assert.assertEquals(t.getTransactionalV2(), t.isTransactional());
+ t.setTransactional(true);
+ Assert.assertEquals(t.getTransactionalV2(), t.isTransactional());
+ }
}
diff --git a/src/query-server/src/main/java/org/apache/kylin/rest/controller/v2/NQueryMetaController.java b/src/query-server/src/main/java/org/apache/kylin/rest/controller/v2/NQueryMetaController.java
index 6227fa46bb..d93c93f8c5 100644
--- a/src/query-server/src/main/java/org/apache/kylin/rest/controller/v2/NQueryMetaController.java
+++ b/src/query-server/src/main/java/org/apache/kylin/rest/controller/v2/NQueryMetaController.java
@@ -22,10 +22,10 @@ import static org.apache.kylin.common.constant.HttpConstant.HTTP_VND_APACHE_KYLI
import java.util.List;
-import org.apache.kylin.metadata.querymeta.TableMeta;
+import org.apache.kylin.metadata.querymeta.TableMetaWithType;
+import org.apache.kylin.rest.controller.NBasicController;
import org.apache.kylin.rest.request.MetaRequest;
import org.apache.kylin.rest.service.QueryService;
-import org.apache.kylin.rest.controller.NBasicController;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
@@ -56,11 +56,11 @@ public class NQueryMetaController extends NBasicController {
@GetMapping(value = "/tables_and_columns", produces = { "application/json", HTTP_VND_APACHE_KYLIN_V2_JSON })
@ResponseBody
@Deprecated
- public List<TableMeta> getMetadataForDriver(MetaRequest metaRequest) {
+ public List<TableMetaWithType> getMetadataForDriver(MetaRequest metaRequest) {
if (metaRequest.getCube() == null || metaRequest.getCube().isEmpty()) {
- return queryService.getMetadata(metaRequest.getProject());
+ return queryService.getMetadataAddType(metaRequest.getProject(), null);
} else {
- return queryService.getMetadata(metaRequest.getProject(), metaRequest.getCube());
+ return queryService.getMetadataAddType(metaRequest.getProject(), metaRequest.getCube());
}
}
}
diff --git a/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryService.java b/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryService.java
index c3cbe877cb..69d75265df 100644
--- a/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -1633,4 +1633,29 @@ public class QueryService extends BasicService implements CacheSignatureQuerySup
queryContext.setModelPriorities(QueryModelPriorities.getModelPrioritiesFromComment(sql));
}
+ public List<TableMetaWithType> getMetadataAddType(String project, String modelAlias) {
+ List<TableMeta> tableMetas = getMetadata(project, modelAlias);
+
+ Map<TableMetaIdentify, TableMetaWithType> tableMap = Maps.newLinkedHashMap();
+ Map<ColumnMetaIdentify, ColumnMetaWithType> columnMap = Maps.newLinkedHashMap();
+
+ for (TableMeta tableMeta : tableMetas) {
+ TableMetaWithType tblMeta = TableMetaWithType.ofColumnMeta(tableMeta);
+ tableMap.put(new TableMetaIdentify(tblMeta.getTABLE_SCHEM(), tblMeta.getTABLE_NAME()), tblMeta);
+
+ for (ColumnMeta columnMeta : tblMeta.getColumns()) {
+ columnMap.put(new ColumnMetaIdentify(columnMeta.getTABLE_SCHEM(), columnMeta.getTABLE_NAME(),
+ columnMeta.getCOLUMN_NAME()), (ColumnMetaWithType) columnMeta);
+ }
+ }
+
+ List<NDataModel> models = getModels(project, modelAlias);
+
+ for (NDataModel model : models) {
+ clarifyTblTypeToFactOrLookup(model, tableMap);
+ clarifyPkFkCols(model, columnMap);
+ }
+
+ return Lists.newArrayList(tableMap.values());
+ }
}
diff --git a/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryServiceTest.java b/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryServiceTest.java
index 1e08a7d9df..5011e83b98 100644
--- a/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryServiceTest.java
+++ b/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryServiceTest.java
@@ -754,6 +754,47 @@ public class QueryServiceTest extends NLocalFileMetadataTestCase {
}
}
+ @Test
+ public void testGetMetadataAddType() throws Exception {
+ List<TableMetaWithType> tableMetasAddType = queryService.getMetadataAddType("default", null);
+ List<TableMeta> tableMetas = queryService.getMetadata("default", null);
+ List<TableMeta> tablesV2 = Lists.newLinkedList();
+ for (TableMetaWithType t : tableMetasAddType) {
+ TableMeta tableMeta = new TableMeta(t.getTABLE_CAT(), t.getTABLE_SCHEM(), t.getTABLE_NAME(),
+ t.getTABLE_TYPE(), t.getREMARKS(), t.getTYPE_CAT(), t.getTYPE_SCHEM(), t.getTYPE_NAME(),
+ t.getSELF_REFERENCING_COL_NAME(), t.getREF_GENERATION());
+ tableMeta.setColumns(t.getColumns().stream()
+ .map(c -> new ColumnMeta(c.getTABLE_CAT(), c.getTABLE_SCHEM(), c.getTABLE_NAME(),
+ c.getCOLUMN_NAME(), c.getDATA_TYPE(), c.getTYPE_NAME(), c.getCOLUMN_SIZE(),
+ c.getBUFFER_LENGTH(), c.getDECIMAL_DIGITS(), c.getNUM_PREC_RADIX(), c.getNULLABLE(),
+ c.getREMARKS(), c.getCOLUMN_DEF(), c.getSQL_DATA_TYPE(), c.getSQL_DATETIME_SUB(),
+ c.getCHAR_OCTET_LENGTH(), c.getORDINAL_POSITION(), c.getIS_NULLABLE(), c.getSCOPE_CATLOG(),
+ c.getSCOPE_SCHEMA(), c.getSCOPE_TABLE(), c.getSOURCE_DATA_TYPE(), c.getIS_AUTOINCREMENT()))
+ .collect(Collectors.toList()));
+ tablesV2.add(tableMeta);
+ }
+ Assert.assertEquals(JsonUtil.writeValueAsString(tablesV2), JsonUtil.writeValueAsString(tableMetas));
+
+ tableMetasAddType = queryService.getMetadataAddType("default", "test_bank");
+ tableMetas = queryService.getMetadata("default", "test_bank");
+ tablesV2 = Lists.newLinkedList();
+ for (TableMetaWithType t : tableMetasAddType) {
+ TableMeta tableMeta = new TableMeta(t.getTABLE_CAT(), t.getTABLE_SCHEM(), t.getTABLE_NAME(),
+ t.getTABLE_TYPE(), t.getREMARKS(), t.getTYPE_CAT(), t.getTYPE_SCHEM(), t.getTYPE_NAME(),
+ t.getSELF_REFERENCING_COL_NAME(), t.getREF_GENERATION());
+ tableMeta.setColumns(t.getColumns().stream()
+ .map(c -> new ColumnMeta(c.getTABLE_CAT(), c.getTABLE_SCHEM(), c.getTABLE_NAME(),
+ c.getCOLUMN_NAME(), c.getDATA_TYPE(), c.getTYPE_NAME(), c.getCOLUMN_SIZE(),
+ c.getBUFFER_LENGTH(), c.getDECIMAL_DIGITS(), c.getNUM_PREC_RADIX(), c.getNULLABLE(),
+ c.getREMARKS(), c.getCOLUMN_DEF(), c.getSQL_DATA_TYPE(), c.getSQL_DATETIME_SUB(),
+ c.getCHAR_OCTET_LENGTH(), c.getORDINAL_POSITION(), c.getIS_NULLABLE(), c.getSCOPE_CATLOG(),
+ c.getSCOPE_SCHEMA(), c.getSCOPE_TABLE(), c.getSOURCE_DATA_TYPE(), c.getIS_AUTOINCREMENT()))
+ .collect(Collectors.toList()));
+ tablesV2.add(tableMeta);
+ }
+ Assert.assertEquals(JsonUtil.writeValueAsString(tablesV2), JsonUtil.writeValueAsString(tableMetas));
+ }
+
@Test
public void testExposedColumnsProjectConfigByModel() throws Exception {
NProjectManager projectManager = NProjectManager.getInstance(getTestConfig());