You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by xx...@apache.org on 2023/02/27 08:01:03 UTC
[kylin] 23/34: KYLIN-5447 wrap logical view response
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 3aff4aa461340b709694a8ac610c24188b954ac9
Author: ChenLiang.Lu <31...@users.noreply.github.com>
AuthorDate: Fri Dec 30 16:58:26 2022 +0800
KYLIN-5447 wrap logical view response
KYLIN-5447 Wrap Logical View list
---
.../kylin/rest/controller/NAdminController.java | 2 +-
.../kylin/rest/response/LogicalViewResponse.java | 54 ++++++++++++++++++++++
.../apache/kylin/rest/service/SparkDDLService.java | 9 ++--
.../apache/kylin/rest/service/TableExtService.java | 4 +-
.../org/apache/kylin/rest/ddl/ViewCheck.scala | 2 +-
.../apache/kylin/rest/service/SparkDDLTest.java | 21 +++++----
.../org/apache/kylin/newten/LogicalViewTest.java | 4 ++
.../kylin/rest/controller/SparkDDLController.java | 6 +--
.../kylin/rest/service/MetaStoreService.java | 13 +++++-
9 files changed, 95 insertions(+), 20 deletions(-)
diff --git a/src/common-server/src/main/java/org/apache/kylin/rest/controller/NAdminController.java b/src/common-server/src/main/java/org/apache/kylin/rest/controller/NAdminController.java
index 2d0440f52f..c2f26bd8aa 100644
--- a/src/common-server/src/main/java/org/apache/kylin/rest/controller/NAdminController.java
+++ b/src/common-server/src/main/java/org/apache/kylin/rest/controller/NAdminController.java
@@ -76,7 +76,7 @@ public class NAdminController extends NBasicController {
propertyKeys.add("kylin.security.remove-ldap-custom-security-limit-enabled");
propertyKeys.add("kylin.source.ddl.logical-view.enabled");
propertyKeys.add("kylin.source.ddl.hive.enabled");
- propertyKeys.add("kylin.source.ddl.logical-view-database");
+ propertyKeys.add("kylin.source.ddl.logical-view.database");
propertyKeys.add("kylin.storage.check-quota-enabled");
// add second storage
diff --git a/src/datasource-service/src/main/java/org/apache/kylin/rest/response/LogicalViewResponse.java b/src/datasource-service/src/main/java/org/apache/kylin/rest/response/LogicalViewResponse.java
new file mode 100644
index 0000000000..2343207aa3
--- /dev/null
+++ b/src/datasource-service/src/main/java/org/apache/kylin/rest/response/LogicalViewResponse.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.rest.response;
+
+import org.apache.kylin.metadata.view.LogicalView;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE,
+ isGetterVisibility = JsonAutoDetect.Visibility.NONE, setterVisibility = JsonAutoDetect.Visibility.NONE)
+public class LogicalViewResponse {
+ @JsonProperty("table_name")
+ private String tableName;
+
+ @JsonProperty("created_sql")
+ private String createdSql;
+
+ @JsonProperty("modified_user")
+ private String modifiedUser;
+
+ @JsonProperty("created_project")
+ private String createdProject;
+
+ public LogicalViewResponse(LogicalView view) {
+ this.tableName = view.getTableName();
+ this.createdSql = view.getCreatedSql();
+ this.modifiedUser = view.getModifiedUser();
+ this.createdProject = view.getCreatedProject();
+ }
+}
diff --git a/src/datasource-service/src/main/java/org/apache/kylin/rest/service/SparkDDLService.java b/src/datasource-service/src/main/java/org/apache/kylin/rest/service/SparkDDLService.java
index a1c00fdcc9..ef76e21e55 100644
--- a/src/datasource-service/src/main/java/org/apache/kylin/rest/service/SparkDDLService.java
+++ b/src/datasource-service/src/main/java/org/apache/kylin/rest/service/SparkDDLService.java
@@ -41,6 +41,7 @@ import org.apache.kylin.metadata.view.LogicalViewManager;
import org.apache.kylin.rest.ddl.SourceTableCheck;
import org.apache.kylin.rest.ddl.ViewCheck;
import org.apache.kylin.rest.request.ViewRequest;
+import org.apache.kylin.rest.response.LogicalViewResponse;
import org.apache.kylin.rest.util.AclPermissionUtil;
import org.apache.spark.ddl.DDLCheck;
@@ -135,18 +136,20 @@ public class SparkDDLService extends BasicService {
LogicalViewLoader.unloadView(context.getLogicalViewName(), SparderEnv.getSparkSession());
}
- public List<LogicalView> listAll(String project, String tableName) {
+ public List<LogicalViewResponse> listAll(String project, String tableName) {
List<LogicalView> logicalViews = LogicalViewManager.getInstance(KylinConfig.getInstanceFromEnv()).list();
if (StringUtils.isNotBlank(tableName)) {
logicalViews = logicalViews.stream()
.filter(table -> table.getTableName().toLowerCase().contains(tableName.toLowerCase()))
.collect(Collectors.toList());
}
- logicalViews.forEach(table -> {
+ List<LogicalViewResponse> viewResponses =
+ logicalViews.stream().map(LogicalViewResponse::new).collect(Collectors.toList());
+ viewResponses.forEach(table -> {
if (!table.getCreatedProject().equalsIgnoreCase(project)) {
table.setCreatedSql("***");
}
});
- return logicalViews;
+ return viewResponses;
}
}
\ No newline at end of file
diff --git a/src/datasource-service/src/main/java/org/apache/kylin/rest/service/TableExtService.java b/src/datasource-service/src/main/java/org/apache/kylin/rest/service/TableExtService.java
index 595f5c0b6f..aabb60b613 100644
--- a/src/datasource-service/src/main/java/org/apache/kylin/rest/service/TableExtService.java
+++ b/src/datasource-service/src/main/java/org/apache/kylin/rest/service/TableExtService.java
@@ -18,7 +18,6 @@
package org.apache.kylin.rest.service;
-import static org.apache.kylin.common.exception.ServerErrorCode.INVALID_LOGICAL_VIEW;
import static org.apache.kylin.common.exception.ServerErrorCode.INVALID_TABLE_NAME;
import static org.apache.kylin.common.exception.code.ErrorCodeServer.EXCLUDED_TABLE_REQUEST_NOT_ALLOWED;
@@ -152,8 +151,7 @@ public class TableExtService extends BasicService {
if (logicalTable != null && viewProject.equalsIgnoreCase(project)) {
canLoadTables.add(table);
} else {
- throw new KylinException(INVALID_LOGICAL_VIEW, MsgPicker.getMsg()
- .getLoadLogicalViewError(tableName, viewProject));
+ tableResponse.getFailed().add(tableName);
}
});
}
diff --git a/src/datasource-service/src/main/scala/org/apache/kylin/rest/ddl/ViewCheck.scala b/src/datasource-service/src/main/scala/org/apache/kylin/rest/ddl/ViewCheck.scala
index 82935a66e5..879283b427 100644
--- a/src/datasource-service/src/main/scala/org/apache/kylin/rest/ddl/ViewCheck.scala
+++ b/src/datasource-service/src/main/scala/org/apache/kylin/rest/ddl/ViewCheck.scala
@@ -65,7 +65,7 @@ class ViewCheck extends DDLCheck {
cnDescription.append(s"创建不要加 database 名称，系统自动创建到 ${config.getDDLLogicalViewDB} 库中，"
+ s"删除要加 ${config.getDDLLogicalViewDB} 库名称 \n")
enDescription.append(s"Creating does not require adding database, it is automatically created in"
- + s" ${config.getDDLLogicalViewDB} ,\n deleting should add ${config.getDDLLogicalViewDB} database")
+ + s" ${config.getDDLLogicalViewDB} , deleting should add ${config.getDDLLogicalViewDB} database\n")
syntaxSupport.append(" `create logical view`, `drop logical view` ")
cnDescription
.append(s"仅支持 ${syntaxSupport} 语法\n")
diff --git a/src/datasource-service/src/test/java/org/apache/kylin/rest/service/SparkDDLTest.java b/src/datasource-service/src/test/java/org/apache/kylin/rest/service/SparkDDLTest.java
index 8e00df161f..80e9a5f114 100644
--- a/src/datasource-service/src/test/java/org/apache/kylin/rest/service/SparkDDLTest.java
+++ b/src/datasource-service/src/test/java/org/apache/kylin/rest/service/SparkDDLTest.java
@@ -40,6 +40,7 @@ import org.apache.kylin.metadata.view.LogicalViewManager;
import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.rest.request.ViewRequest;
import org.apache.kylin.rest.response.LoadTableResponse;
+import org.apache.kylin.rest.response.LogicalViewResponse;
import org.apache.spark.sql.LogicalViewLoader;
import org.apache.spark.sql.SparderEnv;
@@ -283,18 +284,23 @@ public class SparkDDLTest extends NLocalFileMetadataTestCase {
Assert.assertEquals(3, description.get(0).size());
// view list in project
- List<LogicalView> logicalViewsInProject = ddlService.listAll("ssb", "");
- List<LogicalView> logicalViewsInProject2 = ddlService.listAll("ssb", "table2");
+ List<LogicalViewResponse> logicalViewsInProject = ddlService.listAll("ssb", "");
+ List<LogicalViewResponse> logicalViewsInProject2 = ddlService.listAll("ssb", "table2");
+ List<LogicalViewResponse> logicalViewsInProject3 = ddlService.listAll("demo", "");
Assert.assertEquals(3, logicalViewsInProject.size());
Assert.assertEquals(1, logicalViewsInProject2.size());
- LogicalView confidentialTable =
+ LogicalViewResponse confidentialTable =
logicalViewsInProject.stream().filter(table -> table.getCreatedProject().equals("demo")).collect(
Collectors.toList()).get(0);
- LogicalView noConfidentialTable =
+ LogicalViewResponse noConfidentialTable =
logicalViewsInProject.stream().filter(table -> table.getCreatedProject().equals("ssb")).collect(
Collectors.toList()).get(0);
+ LogicalViewResponse noConfidentialTable2 =
+ logicalViewsInProject3.stream().filter(table -> table.getCreatedProject().equals("demo")).collect(
+ Collectors.toList()).get(0);
Assert.assertEquals("***", confidentialTable.getCreatedSql());
Assert.assertNotEquals("***", noConfidentialTable.getCreatedSql());
+ Assert.assertNotEquals("***", noConfidentialTable2.getCreatedSql());
// load table list
String[] failedLoadTables = {"KYLIN_LOGICAL_VIEW.logical_view_table2",
@@ -306,9 +312,8 @@ public class SparkDDLTest extends NLocalFileMetadataTestCase {
LoadTableResponse tableResponse = new LoadTableResponse();
tableExtService.filterAccessTables(successLoadTables, canLoadTables, tableResponse, "ssb");
Assert.assertEquals(2, canLoadTables.size());
- assertKylinExeption(
- () ->
- tableExtService.filterAccessTables(failedLoadTables, canLoadTables, tableResponse, "ssb"),
- "Can't load table");
+ canLoadTables.clear();
+ tableExtService.filterAccessTables(failedLoadTables, canLoadTables, tableResponse, "ssb");
+ Assert.assertEquals(2, canLoadTables.size());
}
}
diff --git a/src/kylin-it/src/test/java/org/apache/kylin/newten/LogicalViewTest.java b/src/kylin-it/src/test/java/org/apache/kylin/newten/LogicalViewTest.java
index df98c689ab..58d1109a55 100644
--- a/src/kylin-it/src/test/java/org/apache/kylin/newten/LogicalViewTest.java
+++ b/src/kylin-it/src/test/java/org/apache/kylin/newten/LogicalViewTest.java
@@ -26,6 +26,7 @@ import org.apache.kylin.common.util.Pair;
import org.apache.kylin.engine.spark.NLocalWithSparkSessionTest;
import org.apache.kylin.job.engine.JobEngineConfig;
import org.apache.kylin.job.impl.threadpool.NDefaultScheduler;
+import org.apache.kylin.metadata.cube.model.LayoutEntity;
import org.apache.kylin.metadata.cube.model.NDataflow;
import org.apache.kylin.metadata.cube.model.NDataflowManager;
import org.apache.kylin.metadata.model.SegmentRange;
@@ -34,6 +35,7 @@ import org.apache.kylin.util.ExecAndComp;
import org.apache.spark.sql.SparderEnv;
import org.junit.After;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -71,6 +73,8 @@ public class LogicalViewTest extends NLocalWithSparkSessionTest {
public void testLogicalView() throws Exception {
String dfID = "451e127a-b684-1474-744b-c9afc14378af";
NDataflow dataflow = dfMgr.getDataflow(dfID);
+ LayoutEntity layout = dataflow.getIndexPlan().getLayoutEntity(20000000001L);
+ Assert.assertNotNull(layout);
populateSSWithCSVData(getTestConfig(), getProject(), SparderEnv.getSparkSession());
indexDataConstructor.buildIndex(dfID, SegmentRange.TimePartitionedSegmentRange.createInfinite(),
Sets.newHashSet(
diff --git a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/SparkDDLController.java b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/SparkDDLController.java
index aa4ebbd9a6..4395f8c4c2 100644
--- a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/SparkDDLController.java
+++ b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/SparkDDLController.java
@@ -23,9 +23,9 @@ import static org.apache.kylin.common.constant.HttpConstant.HTTP_VND_APACHE_KYLI
import java.util.List;
import org.apache.kylin.common.exception.KylinException;
-import org.apache.kylin.metadata.view.LogicalView;
import org.apache.kylin.rest.request.ViewRequest;
import org.apache.kylin.rest.response.EnvelopeResponse;
+import org.apache.kylin.rest.response.LogicalViewResponse;
import org.apache.kylin.rest.service.SparkDDLService;
import org.apache.spark.sql.LogicalViewLoader;
@@ -83,11 +83,11 @@ public class SparkDDLController extends NBasicController {
@ApiOperation(value = "ddl_desc")
@GetMapping(value = "/ddl/view_list")
@ResponseBody
- public EnvelopeResponse<List<LogicalView>> list(
+ public EnvelopeResponse<List<LogicalViewResponse>> list(
@RequestParam("project") String project,
@RequestParam(value = "table", required = false, defaultValue = "") String tableName) {
project = checkProjectName(project);
- List<LogicalView> logicalViews = sparkDDLService.listAll(project, tableName);
+ List<LogicalViewResponse> logicalViews = sparkDDLService.listAll(project, tableName);
return new EnvelopeResponse<>(KylinException.CODE_SUCCESS, logicalViews, "");
}
}
diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/MetaStoreService.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/MetaStoreService.java
index 3680b15cc4..7d6599b389 100644
--- a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/MetaStoreService.java
+++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/MetaStoreService.java
@@ -19,6 +19,7 @@
package org.apache.kylin.rest.service;
import static org.apache.kylin.common.constant.Constants.KE_VERSION;
+import static org.apache.kylin.common.exception.ServerErrorCode.FAILED_CREATE_MODEL;
import static org.apache.kylin.common.exception.ServerErrorCode.MODEL_EXPORT_ERROR;
import static org.apache.kylin.common.exception.ServerErrorCode.MODEL_IMPORT_ERROR;
import static org.apache.kylin.common.exception.ServerErrorCode.MODEL_METADATA_FILE_ERROR;
@@ -97,6 +98,8 @@ import org.apache.kylin.metadata.project.NProjectManager;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.query.util.QueryHisStoreUtil;
import org.apache.kylin.metadata.realization.RealizationStatusEnum;
+import org.apache.kylin.metadata.view.LogicalView;
+import org.apache.kylin.metadata.view.LogicalViewManager;
import org.apache.kylin.rest.aspect.Transaction;
import org.apache.kylin.rest.constant.ModelStatusToDisplayEnum;
import org.apache.kylin.rest.request.ModelImportRequest;
@@ -440,7 +443,7 @@ public class MetaStoreService extends BasicService {
ProjectInstance projectInstance = NProjectManager.getInstance(KylinConfig.getInstanceFromEnv())
.getProject(targetProject);
ISourceMetadataExplorer explorer = SourceFactory.getSource(projectInstance).getSourceMetadataExplorer();
-
+ KylinConfig config = KylinConfig.getInstanceFromEnv();
List<TableDesc> existTableSet = Lists.newArrayList();
for (TableDesc missTableDesc : missTableList) {
try {
@@ -454,6 +457,14 @@ public class MetaStoreService extends BasicService {
} catch (Exception e) {
logger.warn("try load table: {} failed.", missTableDesc.getIdentity(), e);
}
+ if (config.isDDLLogicalViewEnabled() && missTableDesc.isLogicalView()) {
+ LogicalView logicalView = LogicalViewManager.getInstance(config).get(missTableDesc.getName());
+ if (logicalView != null && !targetProject.equalsIgnoreCase(logicalView.getCreatedProject())) {
+ throw new KylinException(FAILED_CREATE_MODEL, String.format(Locale.ROOT,
+ " Logical View %s can only add in project %s",
+ missTableDesc.getName(), logicalView.getCreatedProject()));
+ }
+ }
}
return existTableSet;
}