You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by xx...@apache.org on 2022/12/05 10:20:58 UTC
[kylin] 06/22: KYLIN-5314 check name conflict before export tds file
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit f4f2b21ee8127424f13f3739b4324a7bcfcc8f53
Author: Pengfei Zhan <pe...@kyligence.io>
AuthorDate: Wed Oct 12 20:50:19 2022 +0800
KYLIN-5314 check name conflict before export tds file
---
.../kylin/rest/controller/NBasicController.java | 39 +-
.../org/apache/kylin/common/KylinConfigBase.java | 40 +-
.../common/exception/code/ErrorCodeServer.java | 2 +
.../org/apache/kylin/common/msg/CnMessage.java | 16 -
.../java/org/apache/kylin/common/msg/Message.java | 14 +-
.../resources/kylin_error_msg_conf_cn.properties | 2 +
.../resources/kylin_error_msg_conf_en.properties | 2 +
.../main/resources/kylin_errorcode_conf.properties | 2 +
.../kylin/rest/service/ModelBuildService.java | 34 +-
.../rest/service/FusionModelServiceBuildTest.java | 8 +-
.../kylin/rest/service/ModelServiceBuildTest.java | 68 +--
.../rest/controller/open/OpenModelController.java | 59 +-
.../kylin/rest/controller/NModelController.java | 95 +---
.../kylin/rest/controller/NTableController.java | 21 +-
.../rest/controller/v2/NModelControllerV2.java | 5 +-
.../rest/controller/v2/NProjectControllerV2.java | 6 +-
.../controller/open/OpenModelControllerTest.java | 69 +++
.../rest/controller/NModelControllerTest.java | 74 +--
.../kylin/rest/service/AbstractModelService.java | 140 +++++
.../kylin/rest/service/FusionModelService.java | 6 +-
.../apache/kylin/rest/service/ModelService.java | 407 +-------------
.../apache/kylin/rest/service/ModelTdsService.java | 336 ++++++++++++
.../kylin/rest/service/ModelServiceTest.java | 479 +---------------
.../kylin/rest/service/ModelTdsServiceTest.java | 607 +++++++++++++++++++++
.../org/apache/kylin/tool/bisync/BISyncTool.java | 20 +-
.../org/apache/kylin/tool/bisync/SyncContext.java | 2 +
.../apache/kylin/tool/bisync/SyncModelBuilder.java | 308 +++++------
.../apache/kylin/tool/bisync/model/ColumnDef.java | 80 +--
.../apache/kylin/tool/bisync/model/SyncModel.java | 72 +--
.../bisync/tableau/TableauDataSourceConverter.java | 2 +-
.../kap/secondstorage/SecondStorageLockTest.java | 6 +-
.../kylin/tool/bisync/SyncModelBuilderTest.java | 78 ++-
.../kylin/tool/bisync/SyncModelTestUtil.java | 1 +
33 files changed, 1597 insertions(+), 1503 deletions(-)
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/controller/NBasicController.java b/src/common-service/src/main/java/org/apache/kylin/rest/controller/NBasicController.java
index 24bce526ed..4ffe05c013 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/controller/NBasicController.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/controller/NBasicController.java
@@ -58,7 +58,9 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
+import java.util.Map;
import java.util.Objects;
+import java.util.Optional;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
@@ -78,28 +80,28 @@ import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.exception.ServerErrorCode;
import org.apache.kylin.common.msg.Message;
import org.apache.kylin.common.msg.MsgPicker;
+import org.apache.kylin.common.persistence.transaction.TransactionException;
import org.apache.kylin.common.util.DateFormat;
import org.apache.kylin.common.util.JsonUtil;
+import org.apache.kylin.common.util.Unsafe;
import org.apache.kylin.job.constant.JobStatusEnum;
import org.apache.kylin.job.dao.ExecutablePO;
import org.apache.kylin.job.execution.JobTypeEnum;
+import org.apache.kylin.metadata.model.NDataModel;
+import org.apache.kylin.metadata.model.NDataModelManager;
+import org.apache.kylin.metadata.project.NProjectManager;
import org.apache.kylin.metadata.project.ProjectInstance;
+import org.apache.kylin.metadata.streaming.KafkaConfigManager;
import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.rest.exception.ForbiddenException;
import org.apache.kylin.rest.exception.NotFoundException;
import org.apache.kylin.rest.exception.UnauthorizedException;
+import org.apache.kylin.rest.request.Validation;
import org.apache.kylin.rest.response.EnvelopeResponse;
import org.apache.kylin.rest.response.ErrorResponse;
+import org.apache.kylin.rest.service.ProjectService;
import org.apache.kylin.rest.service.UserService;
import org.apache.kylin.rest.util.PagingUtil;
-import org.apache.kylin.common.persistence.transaction.TransactionException;
-import org.apache.kylin.common.util.Unsafe;
-import org.apache.kylin.metadata.model.NDataModel;
-import org.apache.kylin.metadata.model.NDataModelManager;
-import org.apache.kylin.metadata.project.NProjectManager;
-import org.apache.kylin.metadata.streaming.KafkaConfigManager;
-import org.apache.kylin.rest.request.Validation;
-import org.apache.kylin.rest.service.ProjectService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
@@ -370,8 +372,8 @@ public class NBasicController {
return isAdmin;
}
- public HashMap<String, Object> getDataResponse(String name, List<?> result, int offset, int limit) {
- HashMap<String, Object> data = new HashMap<>();
+ public Map<String, Object> getDataResponse(String name, List<?> result, int offset, int limit) {
+ Map<String, Object> data = new HashMap<>();
data.put(name, PagingUtil.cutPage(result, offset, limit));
data.put("size", result.size());
return data;
@@ -381,6 +383,19 @@ public class NBasicController {
return PagingUtil.cutPage(result, offset, limit);
}
+ public String getHost(String serverHost, String serverName) {
+ String host = KylinConfig.getInstanceFromEnv().getModelExportHost();
+ host = Optional.ofNullable(Optional.ofNullable(host).orElse(serverHost)).orElse(serverName);
+ return host;
+ }
+
+ public int getPort(Integer serverPort, Integer requestServerPort) {
+ Integer port = KylinConfig.getInstanceFromEnv().getModelExportPort() == -1 ? null
+ : KylinConfig.getInstanceFromEnv().getModelExportPort();
+ port = Optional.ofNullable(Optional.ofNullable(port).orElse(serverPort)).orElse(requestServerPort);
+ return port;
+ }
+
public String checkProjectName(String project) {
if (StringUtils.isEmpty(project)) {
throw new KylinException(EMPTY_PROJECT_NAME, MsgPicker.getMsg().getEmptyProjectName());
@@ -632,8 +647,8 @@ public class NBasicController {
if (CollectionUtils.isEmpty(statuses)) {
return;
}
- List<String> streamingJobsStatus = Arrays.asList(JobStatusEnum.STARTING.name(),
- JobStatusEnum.RUNNING.name(), JobStatusEnum.STOPPING.name(), JobStatusEnum.ERROR.name(), JobStatusEnum.STOPPED.name());
+ List<String> streamingJobsStatus = Arrays.asList(JobStatusEnum.STARTING.name(), JobStatusEnum.RUNNING.name(),
+ JobStatusEnum.STOPPING.name(), JobStatusEnum.ERROR.name(), JobStatusEnum.STOPPED.name());
for (String status : statuses) {
if (!streamingJobsStatus.contains(status)) {
throw new KylinException(PARAMETER_INVALID_SUPPORT_LIST, "statuses",
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index d0b4fda104..acfe61c924 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -553,6 +553,7 @@ public abstract class KylinConfigBase implements Serializable {
public boolean getSecondStorageUseLowCardinality() {
return Boolean.parseBoolean(getOptional("kylin.second-storage.use-low-cardinality", TRUE));
}
+
public long getSecondStorageLowCardinalityNumber() {
return Long.parseLong(getOptional("kylin.second-storage.low-cardinality-number", "10000"));
}
@@ -966,8 +967,9 @@ public abstract class KylinConfigBase implements Serializable {
public String getWritingClusterWorkingDir(String withSuffix) {
// This step will remove the '/' symbol from the end of the writingClusterWorkingDir
Path writingClusterPath = new Path(getWritingClusterWorkingDir());
- if (!writingClusterPath.isAbsolute()){
- throw new IllegalArgumentException("kylin.env.hdfs-write-working-dir must be absolute, but got " + writingClusterPath);
+ if (!writingClusterPath.isAbsolute()) {
+ throw new IllegalArgumentException(
+ "kylin.env.hdfs-write-working-dir must be absolute, but got " + writingClusterPath);
}
// make sure path is qualified
@@ -2032,16 +2034,16 @@ public abstract class KylinConfigBase implements Serializable {
public String[] getTableDetectorTransformers() {
String value = getOptional("kylin.query.table-detect-transformers");
- return value == null ? new String[] { POWER_BI_CONVERTER,
- "org.apache.kylin.query.util.DefaultQueryTransformer", "org.apache.kylin.query.util.EscapeTransformer" }
+ return value == null
+ ? new String[] { POWER_BI_CONVERTER, "org.apache.kylin.query.util.DefaultQueryTransformer",
+ "org.apache.kylin.query.util.EscapeTransformer" }
: getOptionalStringArray("kylin.query.table-detect-transformers", new String[0]);
}
public String[] getQueryTransformers() {
String value = getOptional("kylin.query.transformers");
- return value == null ? new String[] { POWER_BI_CONVERTER,
- "org.apache.kylin.query.util.DefaultQueryTransformer", "org.apache.kylin.query.util.EscapeTransformer",
- "org.apache.kylin.query.util.ConvertToComputedColumn",
+ return value == null ? new String[] { POWER_BI_CONVERTER, "org.apache.kylin.query.util.DefaultQueryTransformer",
+ "org.apache.kylin.query.util.EscapeTransformer", "org.apache.kylin.query.util.ConvertToComputedColumn",
"org.apache.kylin.query.util.KeywordDefaultDirtyHack", "org.apache.kylin.query.security.RowFilter" }
: getOptionalStringArray("kylin.query.transformers", new String[0]);
}
@@ -2211,12 +2213,13 @@ public abstract class KylinConfigBase implements Serializable {
}
public String[] getPushDownConverterClassNames() {
- return getOptionalStringArray("kylin.query.pushdown.converter-class-names", new String[] {
- "org.apache.kylin.source.adhocquery.DoubleQuotePushDownConverter",
- POWER_BI_CONVERTER, "org.apache.kylin.query.util.KeywordDefaultDirtyHack",
- "org.apache.kylin.query.util.RestoreFromComputedColumn", "org.apache.kylin.query.security.RowFilter",
- "org.apache.kylin.query.security.HackSelectStarWithColumnACL",
- "org.apache.kylin.query.util.SparkSQLFunctionConverter" });
+ return getOptionalStringArray("kylin.query.pushdown.converter-class-names",
+ new String[] { "org.apache.kylin.source.adhocquery.DoubleQuotePushDownConverter", POWER_BI_CONVERTER,
+ "org.apache.kylin.query.util.KeywordDefaultDirtyHack",
+ "org.apache.kylin.query.util.RestoreFromComputedColumn",
+ "org.apache.kylin.query.security.RowFilter",
+ "org.apache.kylin.query.security.HackSelectStarWithColumnACL",
+ "org.apache.kylin.query.util.SparkSQLFunctionConverter" });
}
@ThirdPartyDependencies({
@@ -2821,11 +2824,13 @@ public abstract class KylinConfigBase implements Serializable {
}
public int getQueryHistoryAccelerateMaxSize() {
- return Integer.parseInt(this.getOptional("kylin.favorite.query-history-accelerate-max-size", ONE_HUNDRED_THOUSAND));
+ return Integer
+ .parseInt(this.getOptional("kylin.favorite.query-history-accelerate-max-size", ONE_HUNDRED_THOUSAND));
}
public int getQueryHistoryStatMetaUpdateMaxSize() {
- return Integer.parseInt(this.getOptional("kylin.query.query-history-stat-update-max-size", ONE_HUNDRED_THOUSAND));
+ return Integer
+ .parseInt(this.getOptional("kylin.query.query-history-stat-update-max-size", ONE_HUNDRED_THOUSAND));
}
public long getQueryHistoryAccelerateInterval() {
@@ -3606,6 +3611,10 @@ public abstract class KylinConfigBase implements Serializable {
return Boolean.parseBoolean(getOptional("kylin.model.tds-expose-model-join-key", TRUE));
}
+ public boolean skipCheckTds() {
+ return Boolean.parseBoolean(getOptional("kylin.model.skip-check-tds", FALSE));
+ }
+
public boolean isHdfsMetricsPeriodicCalculationEnabled() {
return Boolean.parseBoolean(getOptional("kylin.metrics.hdfs-periodic-calculation-enabled", FALSE));
}
@@ -3622,6 +3631,7 @@ public abstract class KylinConfigBase implements Serializable {
public boolean isSkipResourceCheck() {
return Boolean.parseBoolean(getOptional("kylin.build.resource.skip-resource-check", FALSE));
}
+
public int getSecondStorageSkippingIndexGranularity() {
int granularity = Integer.parseInt(getOptional("kylin.second-storage.skipping-index.granularity", "3"));
return granularity <= 0 ? 3 : granularity;
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/exception/code/ErrorCodeServer.java b/src/core-common/src/main/java/org/apache/kylin/common/exception/code/ErrorCodeServer.java
index 3a0ca0195d..c8abaa1142 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/exception/code/ErrorCodeServer.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/exception/code/ErrorCodeServer.java
@@ -32,6 +32,8 @@ public enum ErrorCodeServer implements ErrorCodeProducer {
MODEL_NAME_DUPLICATE("KE-010002206"),
SIMPLIFIED_MEASURES_MISSING_ID("KE-010002207"),
MODEL_NOT_EXIST_SEGMENTS("KE-010002208"),
+ MODEL_TDS_EXPORT_DIM_COL_AND_MEASURE_NAME_CONFLICT("KE-010002301"),
+ MODEL_TDS_EXPORT_COLUMN_AND_MEASURE_NAME_CONFLICT("KE-010002302"),
// 100252XX Cube
CUBE_NOT_EXIST("KE-010025201"),
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/msg/CnMessage.java b/src/core-common/src/main/java/org/apache/kylin/common/msg/CnMessage.java
index 482d58fef5..3688c2db26 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/msg/CnMessage.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/msg/CnMessage.java
@@ -1700,21 +1700,6 @@ public class CnMessage extends Message {
return "当前 Segments 所包含的索引未加载至 HDFS 存储/对象存储，请确保索引加载至 HDFS 存储后再合并。";
}
- @Override
- public String getDuplicateModelColumnAndMeasureName() {
- return "模型中的列名 %s 与度量名 %s 重复，无法导出 TDS。请去除重名后再重试。";
- }
-
- @Override
- public String getDuplicateDimensionNameAndMeasureName() {
- return "维度名 %s 与度量名 %s 重复，无法导出 TDS。请去除重名后再重试。";
- }
-
- @Override
- public String getDuplicateDimensionColAndMeasureName() {
- return "维度的列名 %s 与度量名 %s 重复，无法导出 TDS。请去除重名后再重试。";
- }
-
@Override
public String getProfilingNotEnabled() {
return "构建火焰图" + PARAMETER_NOT_ENABLED;
@@ -1750,7 +1735,6 @@ public class CnMessage extends Message {
return "构建火焰图任务" + TASK_TIMEOUT;
}
-
@Override
public String getSecondStorageIndexNotSupport() {
return "Order by列和Skipping Index列不支持使用分区列";
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/msg/Message.java b/src/core-common/src/main/java/org/apache/kylin/common/msg/Message.java
index 84a274ca50..cad28ace11 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/msg/Message.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/msg/Message.java
@@ -39,7 +39,6 @@ public class Message {
private static final String LACK_PROJECT = "Please fill in the project parameters.";
private static final String NON_EXIST_PROJECT = "Project %s doesn't exist. Please confirm and try again later.";
private static final String DUP_MODCOL_MEASURE_NAME = "There are duplicated names among model column %s and measure name %s. Cannot export a valid TDS file. Please correct the duplicated names and try again.";
- private static final String DUP_DIM_MEASURE_NAME = "There are duplicated names among dimension name %s and measure name %s. Cannot export a valid TDS file. Please correct the duplicated names and try again.";
private static final String DUP_DIMCOL_MEASURE_NAME = "There are duplicated names among dimension column %s and measure name %s. Cannot export a valid TDS file. Please correct the duplicated names and try again.";
private static final String MODIFY_PERMISSION_OF_SUPER_ADMIN = "Super Admin’s permission can’t be modified.";
private static final String ILLEGAL_AUTHORIZING_USER = "Unable to modify. Only Super Admin or System Admin with query permission can modify query permission.";
@@ -1303,6 +1302,7 @@ public class Message {
public String getInvalidLowCardinalityDataType() {
return SECOND_STORAGE_CARDINALITY_DATATYPE_INVALID;
}
+
public String getJobRestartFailed() {
return "Tiered storage task doesn't support restart.\n";
}
@@ -1525,18 +1525,6 @@ public class Message {
return "The indexes included in the selected segments are not loaded to HDFS storage/object storage. Please ensure the indexes are loaded into HDFS storage and try merging again.";
}
- public String getDuplicateModelColumnAndMeasureName() {
- return DUP_MODCOL_MEASURE_NAME;
- }
-
- public String getDuplicateDimensionNameAndMeasureName() {
- return DUP_DIM_MEASURE_NAME;
- }
-
- public String getDuplicateDimensionColAndMeasureName() {
- return DUP_DIMCOL_MEASURE_NAME;
- }
-
public String getProfilingNotEnabled() {
return PROFILING_NOT_ENABLED;
}
diff --git a/src/core-common/src/main/resources/kylin_error_msg_conf_cn.properties b/src/core-common/src/main/resources/kylin_error_msg_conf_cn.properties
index 27641ca5b2..327c28d18d 100644
--- a/src/core-common/src/main/resources/kylin_error_msg_conf_cn.properties
+++ b/src/core-common/src/main/resources/kylin_error_msg_conf_cn.properties
@@ -32,6 +32,8 @@ KE-010002205=无效的模型名称 “%s”。请使用字母、数字或下划
KE-010002206=模型 “%s” 已存在。请重新命名。
KE-010002207=修改模型时，simplified_measures 参数中的每个度量必须传入 id 值。请为以下度量传入 id 之后重试：%s。
KE-010002208=模型上线必须存在 Segment。请重新输入。
+KE-010002301=维度的列名 %s 与度量名 %s 重复，无法导出 TDS。请去除重名后再重试。
+KE-010002302=模型中的列名 %s 与度量名 %s 重复，无法导出 TDS。请去除重名后再重试。
## 100252XX Cube
KE-010025201=无法找到相关 Cube。
diff --git a/src/core-common/src/main/resources/kylin_error_msg_conf_en.properties b/src/core-common/src/main/resources/kylin_error_msg_conf_en.properties
index 3f87894a80..5a5f9ea5fb 100644
--- a/src/core-common/src/main/resources/kylin_error_msg_conf_en.properties
+++ b/src/core-common/src/main/resources/kylin_error_msg_conf_en.properties
@@ -32,6 +32,8 @@ KE-010002205=The model name "%s" is invalid. Please use letters, numbers and und
KE-010002206=Model "%s" already exists. Please rename it.
KE-010002207=When modifying model, each measure id is required in simplified_measures parameter. Please pass ids for following measures and try again: %s.
KE-010002208=The online model must have a segment. Please re-enter.
+KE-010002301=There are duplicated names among dimension column %s and measure name %s. Cannot export a valid TDS file. Please correct the duplicated names and try again.
+KE-010002302=There are duplicated names among model column %s and measure name %s. Cannot export a valid TDS file. Please correct the duplicated names and try again.
## 100252XX Cube
KE-010025201=Can't find the cube.
diff --git a/src/core-common/src/main/resources/kylin_errorcode_conf.properties b/src/core-common/src/main/resources/kylin_errorcode_conf.properties
index 5b506d68fb..7f07828a6a 100644
--- a/src/core-common/src/main/resources/kylin_errorcode_conf.properties
+++ b/src/core-common/src/main/resources/kylin_errorcode_conf.properties
@@ -33,6 +33,8 @@ KE-010002205
KE-010002206
KE-010002207
KE-010002208
+KE-010002301
+KE-010002302
## 100252XX Cube
KE-010025201
diff --git a/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/ModelBuildService.java b/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/ModelBuildService.java
index 8b08ee171d..5548fd30d5 100644
--- a/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/ModelBuildService.java
+++ b/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/ModelBuildService.java
@@ -48,14 +48,6 @@ import org.apache.kylin.job.exception.JobSubmissionException;
import org.apache.kylin.job.execution.JobTypeEnum;
import org.apache.kylin.job.manager.JobManager;
import org.apache.kylin.job.model.JobParam;
-import org.apache.kylin.metadata.model.PartitionDesc;
-import org.apache.kylin.metadata.model.SegmentRange;
-import org.apache.kylin.metadata.model.SegmentStatusEnum;
-import org.apache.kylin.metadata.model.SegmentStatusEnumToDisplay;
-import org.apache.kylin.metadata.model.TableDesc;
-import org.apache.kylin.query.util.PushDownUtil;
-import org.apache.kylin.rest.util.AclEvaluate;
-import org.apache.kylin.source.SourceFactory;
import org.apache.kylin.metadata.cube.model.IndexPlan;
import org.apache.kylin.metadata.cube.model.NBatchConstants;
import org.apache.kylin.metadata.cube.model.NDataSegment;
@@ -67,9 +59,15 @@ import org.apache.kylin.metadata.model.MultiPartitionDesc;
import org.apache.kylin.metadata.model.NDataModel;
import org.apache.kylin.metadata.model.NDataModelManager;
import org.apache.kylin.metadata.model.NTableMetadataManager;
+import org.apache.kylin.metadata.model.PartitionDesc;
+import org.apache.kylin.metadata.model.SegmentRange;
+import org.apache.kylin.metadata.model.SegmentStatusEnum;
+import org.apache.kylin.metadata.model.SegmentStatusEnumToDisplay;
+import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.util.MultiPartitionUtil;
import org.apache.kylin.metadata.project.EnhancedUnitOfWork;
import org.apache.kylin.metadata.sourceusage.SourceUsageManager;
+import org.apache.kylin.query.util.PushDownUtil;
import org.apache.kylin.rest.aspect.Transaction;
import org.apache.kylin.rest.request.PartitionsRefreshRequest;
import org.apache.kylin.rest.request.SegmentTimeRequest;
@@ -82,6 +80,7 @@ import org.apache.kylin.rest.service.params.FullBuildSegmentParams;
import org.apache.kylin.rest.service.params.IncrementBuildSegmentParams;
import org.apache.kylin.rest.service.params.MergeSegmentParams;
import org.apache.kylin.rest.service.params.RefreshSegmentParams;
+import org.apache.kylin.source.SourceFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
@@ -93,14 +92,11 @@ import lombok.val;
import lombok.var;
@Component("modelBuildService")
-public class ModelBuildService extends BasicService implements ModelBuildSupporter {
+public class ModelBuildService extends AbstractModelService implements ModelBuildSupporter {
@Autowired
private ModelService modelService;
- @Autowired
- private AclEvaluate aclEvaluate;
-
@Autowired
private SegmentHelper segmentHelper;
@@ -150,7 +146,7 @@ public class ModelBuildService extends BasicService implements ModelBuildSupport
public JobInfoResponse fullBuildSegmentsManually(FullBuildSegmentParams params) {
aclEvaluate.checkProjectOperationPermission(params.getProject());
- modelService.checkModelPermission(params.getProject(), params.getModelId());
+ checkModelPermission(params.getProject(), params.getModelId());
List<JobInfoResponse.JobInfo> jobIds = EnhancedUnitOfWork
.doInTransactionWithCheckAndRetry(() -> constructFullBuild(params), params.getProject());
JobInfoResponse jobInfoResponse = new JobInfoResponse();
@@ -228,7 +224,7 @@ public class ModelBuildService extends BasicService implements ModelBuildSupport
List<JobInfoResponse.JobInfo> jobIds = new ArrayList<>();
NDataflowManager dfMgr = getManager(NDataflowManager.class, params.getProject());
val jobManager = getManager(JobManager.class, params.getProject());
- IndexPlan indexPlan = modelService.getIndexPlan(params.getModelId(), params.getProject());
+ IndexPlan indexPlan = getIndexPlan(params.getModelId(), params.getProject());
NDataflow df = dfMgr.getDataflow(indexPlan.getUuid());
for (String id : params.getSegmentIds()) {
@@ -263,7 +259,7 @@ public class ModelBuildService extends BasicService implements ModelBuildSupport
public JobInfoResponse incrementBuildSegmentsManually(IncrementBuildSegmentParams params) throws Exception {
String project = params.getProject();
aclEvaluate.checkProjectOperationPermission(project);
- modelService.checkModelPermission(project, params.getModelId());
+ checkModelPermission(project, params.getModelId());
val modelManager = getManager(NDataModelManager.class, project);
if (PartitionDesc.isEmptyPartitionDesc(params.getPartitionDesc())) {
throw new KylinException(EMPTY_PARTITION_COLUMN, "Partition column is null.'");
@@ -441,7 +437,7 @@ public class ModelBuildService extends BasicService implements ModelBuildSupport
List<String[]> partitionValues, boolean parallelBuild, boolean buildAllPartitions, int priority,
String yarnQueue, Object tag) {
aclEvaluate.checkProjectOperationPermission(project);
- modelService.checkModelPermission(project, modelId);
+ checkModelPermission(project, modelId);
modelService.checkSegmentsExistById(modelId, project, new String[] { segmentId });
modelService.checkModelIsMLP(modelId, project);
val dfm = getManager(NDataflowManager.class, project);
@@ -527,7 +523,7 @@ public class ModelBuildService extends BasicService implements ModelBuildSupport
val segment = df.getSegment(param.getSegmentId());
var partitions = param.getPartitionIds();
aclEvaluate.checkProjectOperationPermission(project);
- modelService.checkModelPermission(project, modelId);
+ checkModelPermission(project, modelId);
if (CollectionUtils.isEmpty(param.getPartitionIds())) {
partitions = modelService.getModelById(modelId, project).getMultiPartitionDesc()
@@ -561,7 +557,7 @@ public class ModelBuildService extends BasicService implements ModelBuildSupport
val dfManager = getManager(NDataflowManager.class, project);
val jobManager = getManager(JobManager.class, project);
- val indexPlan = modelService.getIndexPlan(modelId, project);
+ val indexPlan = getIndexPlan(modelId, project);
val df = dfManager.getDataflow(indexPlan.getUuid());
NDataSegment mergeSeg = NDataflowManager.getInstance(KylinConfig.getInstanceFromEnv(), project).mergeSegments(
@@ -588,7 +584,7 @@ public class ModelBuildService extends BasicService implements ModelBuildSupport
List<Long> indexIds, boolean parallelBuildBySegment, int priority, boolean partialBuild, String yarnQueue,
Object tag) {
aclEvaluate.checkProjectOperationPermission(project);
- modelService.checkModelPermission(project, modelId);
+ checkModelPermission(project, modelId);
val dfManger = getManager(NDataflowManager.class, project);
NDataflow dataflow = dfManger.getDataflow(modelId);
modelService.checkSegmentsExistById(modelId, project, segmentIds.toArray(new String[0]));
diff --git a/src/data-loading-service/src/test/java/org/apache/kylin/rest/service/FusionModelServiceBuildTest.java b/src/data-loading-service/src/test/java/org/apache/kylin/rest/service/FusionModelServiceBuildTest.java
index 9b9e5f814b..cad5a66e73 100644
--- a/src/data-loading-service/src/test/java/org/apache/kylin/rest/service/FusionModelServiceBuildTest.java
+++ b/src/data-loading-service/src/test/java/org/apache/kylin/rest/service/FusionModelServiceBuildTest.java
@@ -22,17 +22,17 @@ import java.util.Arrays;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.exception.KylinException;
-import org.apache.kylin.rest.constant.Constant;
-import org.apache.kylin.rest.util.AclEvaluate;
-import org.apache.kylin.rest.util.AclUtil;
import org.apache.kylin.common.scheduler.EventBusFactory;
import org.apache.kylin.metadata.cube.model.NDataflowManager;
import org.apache.kylin.metadata.model.ManagementType;
import org.apache.kylin.metadata.model.NDataModel;
import org.apache.kylin.metadata.model.NDataModelManager;
import org.apache.kylin.rest.config.initialize.ModelUpdateListener;
+import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.rest.request.IndexesToSegmentsRequest;
import org.apache.kylin.rest.service.params.IncrementBuildSegmentParams;
+import org.apache.kylin.rest.util.AclEvaluate;
+import org.apache.kylin.rest.util.AclUtil;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -98,7 +98,9 @@ public class FusionModelServiceBuildTest extends SourceTestCase {
ReflectionTestUtils.setField(modelService, "aclEvaluate", aclEvaluate);
ReflectionTestUtils.setField(modelBuildService, "aclEvaluate", aclEvaluate);
ReflectionTestUtils.setField(modelBuildService, "modelService", modelService);
+ ReflectionTestUtils.setField(modelBuildService, "userGroupService", userGroupService);
ReflectionTestUtils.setField(fusionModelService, "modelBuildService", modelBuildService);
+ ReflectionTestUtils.setField(fusionModelService, "userGroupService", userGroupService);
ReflectionTestUtils.setField(modelService, "userGroupService", userGroupService);
ReflectionTestUtils.setField(semanticService, "userGroupService", userGroupService);
ReflectionTestUtils.setField(indexPlanService, "aclEvaluate", aclEvaluate);
diff --git a/src/data-loading-service/src/test/java/org/apache/kylin/rest/service/ModelServiceBuildTest.java b/src/data-loading-service/src/test/java/org/apache/kylin/rest/service/ModelServiceBuildTest.java
index 03dad20631..3dad56193f 100644
--- a/src/data-loading-service/src/test/java/org/apache/kylin/rest/service/ModelServiceBuildTest.java
+++ b/src/data-loading-service/src/test/java/org/apache/kylin/rest/service/ModelServiceBuildTest.java
@@ -18,14 +18,31 @@
package org.apache.kylin.rest.service;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
-import org.apache.kylin.engine.spark.job.ExecutableAddCuboidHandler;
-import org.apache.kylin.engine.spark.job.ExecutableAddSegmentHandler;
-import org.apache.kylin.engine.spark.job.ExecutableMergeOrRefreshHandler;
-import org.apache.kylin.engine.spark.job.NSparkCubingJob;
-import lombok.val;
-import lombok.var;
+import static org.apache.kylin.common.exception.code.ErrorCodeServer.JOB_CONCURRENT_SUBMIT_LIMIT;
+import static org.apache.kylin.common.exception.code.ErrorCodeServer.JOB_CREATE_CHECK_FAIL;
+import static org.apache.kylin.common.exception.code.ErrorCodeServer.JOB_CREATE_CHECK_MULTI_PARTITION_ABANDON;
+import static org.apache.kylin.common.exception.code.ErrorCodeServer.JOB_CREATE_CHECK_MULTI_PARTITION_DUPLICATE;
+import static org.apache.kylin.common.exception.code.ErrorCodeServer.SEGMENT_LOCKED;
+import static org.apache.kylin.common.exception.code.ErrorCodeServer.SEGMENT_MERGE_CHECK_INDEX_ILLEGAL;
+import static org.apache.kylin.common.exception.code.ErrorCodeServer.SEGMENT_MERGE_CHECK_PARTITION_ILLEGAL;
+import static org.apache.kylin.common.exception.code.ErrorCodeServer.SEGMENT_NOT_EXIST_ID;
+import static org.apache.kylin.common.exception.code.ErrorCodeServer.SEGMENT_REFRESH_SELECT_RANGE_EMPTY;
+import static org.apache.kylin.common.exception.code.ErrorCodeServer.SEGMENT_STATUS;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Locale;
+import java.util.Set;
+import java.util.TimeZone;
+import java.util.stream.Collectors;
+
import org.apache.commons.collections.CollectionUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.exception.KylinException;
@@ -35,6 +52,10 @@ import org.apache.kylin.common.persistence.transaction.UnitOfWork;
import org.apache.kylin.common.scheduler.EventBusFactory;
import org.apache.kylin.common.util.DateFormat;
import org.apache.kylin.common.util.JsonUtil;
+import org.apache.kylin.engine.spark.job.ExecutableAddCuboidHandler;
+import org.apache.kylin.engine.spark.job.ExecutableAddSegmentHandler;
+import org.apache.kylin.engine.spark.job.ExecutableMergeOrRefreshHandler;
+import org.apache.kylin.engine.spark.job.NSparkCubingJob;
import org.apache.kylin.engine.spark.utils.ComputedColumnEvalUtil;
import org.apache.kylin.job.dao.ExecutablePO;
import org.apache.kylin.job.execution.AbstractExecutable;
@@ -103,30 +124,11 @@ import org.mockito.Mock;
import org.mockito.Mockito;
import org.springframework.test.util.ReflectionTestUtils;
-import java.io.IOException;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Comparator;
-import java.util.List;
-import java.util.Locale;
-import java.util.Set;
-import java.util.TimeZone;
-import java.util.stream.Collectors;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
-import static org.apache.kylin.common.exception.code.ErrorCodeServer.JOB_CONCURRENT_SUBMIT_LIMIT;
-import static org.apache.kylin.common.exception.code.ErrorCodeServer.JOB_CREATE_CHECK_FAIL;
-import static org.apache.kylin.common.exception.code.ErrorCodeServer.JOB_CREATE_CHECK_MULTI_PARTITION_ABANDON;
-import static org.apache.kylin.common.exception.code.ErrorCodeServer.JOB_CREATE_CHECK_MULTI_PARTITION_DUPLICATE;
-import static org.apache.kylin.common.exception.code.ErrorCodeServer.SEGMENT_LOCKED;
-import static org.apache.kylin.common.exception.code.ErrorCodeServer.SEGMENT_MERGE_CHECK_INDEX_ILLEGAL;
-import static org.apache.kylin.common.exception.code.ErrorCodeServer.SEGMENT_MERGE_CHECK_PARTITION_ILLEGAL;
-import static org.apache.kylin.common.exception.code.ErrorCodeServer.SEGMENT_NOT_EXIST_ID;
-import static org.apache.kylin.common.exception.code.ErrorCodeServer.SEGMENT_REFRESH_SELECT_RANGE_EMPTY;
-import static org.apache.kylin.common.exception.code.ErrorCodeServer.SEGMENT_STATUS;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
+import lombok.val;
+import lombok.var;
public class ModelServiceBuildTest extends SourceTestCase {
@InjectMocks
@@ -147,9 +149,6 @@ public class ModelServiceBuildTest extends SourceTestCase {
@InjectMocks
private final TableService tableService = Mockito.spy(new TableService());
- @InjectMocks
- private final TableExtService tableExtService = Mockito.spy(new TableExtService());
-
@InjectMocks
private final IndexPlanService indexPlanService = Mockito.spy(new IndexPlanService());
@@ -196,6 +195,7 @@ public class ModelServiceBuildTest extends SourceTestCase {
ReflectionTestUtils.setField(modelService, "accessService", accessService);
ReflectionTestUtils.setField(modelService, "userGroupService", userGroupService);
ReflectionTestUtils.setField(semanticService, "userGroupService", userGroupService);
+ ReflectionTestUtils.setField(modelBuildService, "userGroupService", userGroupService);
ReflectionTestUtils.setField(semanticService, "expandableMeasureUtil",
new ExpandableMeasureUtil((model, ccDesc) -> {
String ccExpression = KapQueryUtil.massageComputedColumn(model, model.getProject(), ccDesc,
diff --git a/src/metadata-server/src/main/java/io/kyligence/kap/rest/controller/open/OpenModelController.java b/src/metadata-server/src/main/java/io/kyligence/kap/rest/controller/open/OpenModelController.java
index 8f2b405c4e..d9642ade0c 100644
--- a/src/metadata-server/src/main/java/io/kyligence/kap/rest/controller/open/OpenModelController.java
+++ b/src/metadata-server/src/main/java/io/kyligence/kap/rest/controller/open/OpenModelController.java
@@ -28,6 +28,7 @@ import static org.apache.kylin.common.exception.code.ErrorCodeServer.PROJECT_MUL
import java.io.IOException;
import java.util.Arrays;
+import java.util.Collections;
import java.util.List;
import java.util.Locale;
import java.util.Set;
@@ -70,8 +71,12 @@ import org.apache.kylin.rest.response.OpenGetIndexResponse.IndexDetail;
import org.apache.kylin.rest.service.FusionIndexService;
import org.apache.kylin.rest.service.FusionModelService;
import org.apache.kylin.rest.service.ModelService;
+import org.apache.kylin.rest.service.ModelTdsService;
+import org.apache.kylin.rest.util.AclPermissionUtil;
import org.apache.kylin.tool.bisync.SyncContext;
+import org.apache.kylin.tool.bisync.model.SyncModel;
import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping;
@@ -84,6 +89,7 @@ import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.ResponseBody;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
@@ -108,6 +114,10 @@ public class OpenModelController extends NBasicController {
@Autowired
private NModelController modelController;
+ @Autowired
+ @Qualifier("modelTdsService")
+ private ModelTdsService tdsService;
+
@Autowired
private FusionIndexService fusionIndexService;
@@ -150,8 +160,8 @@ public class OpenModelController extends NBasicController {
@RequestParam(value = "only_normal_dim", required = false, defaultValue = "true") boolean onlyNormalDim) {
String projectName = checkProjectName(project);
return modelController.getModels(modelId, modelAlias, exactMatch, projectName, owner, status, table, offset,
- limit, sortBy, reverse, modelAliasOrOwner, Arrays.asList(ModelAttributeEnum.BATCH), lastModifyFrom,
- lastModifyTo, onlyNormalDim);
+ limit, sortBy, reverse, modelAliasOrOwner, Collections.singletonList(ModelAttributeEnum.BATCH),
+ lastModifyFrom, lastModifyTo, onlyNormalDim);
}
@ApiOperation(value = "getIndexes", tags = { "AI" })
@@ -354,28 +364,24 @@ public class OpenModelController extends NBasicController {
return modelController.updatePartitionSemantic(modelId, param);
}
- @ApiOperation(value = "validate tds export", tags = { "QE" })
- @GetMapping(value = "/validate_export")
- @ResponseBody
- public EnvelopeResponse<Boolean> validateExport(@RequestParam(value = "model_name") String modelAlias,
- @RequestParam(value = "project") String project) {
- String projectName = checkProjectName(project);
- String modelId = getModel(modelAlias, projectName).getId();
- return modelController.validateExport(modelId, projectName);
- }
-
@ApiOperation(value = "export model", tags = { "QE" }, notes = "Add URL: {model}")
@GetMapping(value = "/{model_name:.+}/export")
@ResponseBody
public void exportModel(@PathVariable("model_name") String modelAlias,
@RequestParam(value = "project") String project, @RequestParam(value = "export_as") SyncContext.BI exportAs,
@RequestParam(value = "element", required = false, defaultValue = "AGG_INDEX_COL") SyncContext.ModelElement element,
- @RequestParam(value = "server_host", required = false) String host,
- @RequestParam(value = "server_port", required = false) Integer port, HttpServletRequest request,
+ @RequestParam(value = "server_host", required = false) String serverHost,
+ @RequestParam(value = "server_port", required = false) Integer serverPort, HttpServletRequest request,
HttpServletResponse response) throws IOException {
String projectName = checkProjectName(project);
String modelId = getModel(modelAlias, projectName).getId();
- modelController.exportModel(modelId, projectName, exportAs, element, host, port, request, response);
+ String host = getHost(serverHost, request.getServerName());
+ int port = getPort(serverPort, request.getServerPort());
+
+ SyncContext syncContext = tdsService.prepareSyncContext(projectName, modelId, exportAs, element, host, port);
+ SyncModel syncModel = tdsService.exportModel(syncContext);
+ tdsService.preCheckNameConflict(syncModel);
+ tdsService.dumpSyncModel(syncContext, syncModel, response);
}
@ApiOperation(value = "bi export", tags = { "QE" })
@@ -384,15 +390,30 @@ public class OpenModelController extends NBasicController {
public void biExport(@RequestParam("model_name") String modelAlias, @RequestParam(value = "project") String project,
@RequestParam(value = "export_as") SyncContext.BI exportAs,
@RequestParam(value = "element", required = false, defaultValue = "AGG_INDEX_COL") SyncContext.ModelElement element,
- @RequestParam(value = "server_host", required = false) String host,
- @RequestParam(value = "server_port", required = false) Integer port,
+ @RequestParam(value = "server_host", required = false) String serverHost,
+ @RequestParam(value = "server_port", required = false) Integer serverPort,
@RequestParam(value = "dimensions", required = false) List<String> dimensions,
@RequestParam(value = "measures", required = false) List<String> measures, HttpServletRequest request,
HttpServletResponse response) throws IOException {
String projectName = checkProjectName(project);
String modelId = getModel(modelAlias, projectName).getId();
- modelController.biExport(modelId, projectName, exportAs, element, host, port, dimensions, measures, request,
- response);
+ String host = getHost(serverHost, request.getServerName());
+ int port = getPort(serverPort, request.getServerPort());
+ if (dimensions == null) {
+ // no need filter of given dimensions
+ dimensions = ImmutableList.of();
+ }
+ if (measures == null) {
+ // no need filter of given measures
+ measures = ImmutableList.of();
+ }
+
+ SyncContext syncContext = tdsService.prepareSyncContext(projectName, modelId, exportAs, element, host, port);
+ SyncModel syncModel = AclPermissionUtil.isAdmin()
+ ? tdsService.exportTDSDimensionsAndMeasuresByAdmin(syncContext, dimensions, measures)
+ : tdsService.exportTDSDimensionsAndMeasuresByNormalUser(syncContext, dimensions, measures);
+ tdsService.preCheckNameConflict(syncModel);
+ tdsService.dumpSyncModel(syncContext, syncModel, response);
}
@ApiOperation(value = "updateModelName", tags = { "AI" })
diff --git a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NModelController.java b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NModelController.java
index 8b3ffd04b1..6efa43ec6e 100644
--- a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NModelController.java
+++ b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NModelController.java
@@ -30,18 +30,14 @@ import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.Locale;
-import java.util.Optional;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.exception.ExceptionUtils;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.exception.CommonErrorCode;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.metadata.model.NDataModel;
-import org.apache.kylin.metadata.model.NDataModelManager;
import org.apache.kylin.metadata.model.PartitionDesc;
import org.apache.kylin.metadata.model.exception.LookupTableException;
import org.apache.kylin.rest.constant.ModelAttributeEnum;
@@ -73,13 +69,14 @@ import org.apache.kylin.rest.response.ModelConfigResponse;
import org.apache.kylin.rest.response.ModelSaveCheckResponse;
import org.apache.kylin.rest.response.MultiPartitionValueResponse;
import org.apache.kylin.rest.response.PurgeModelAffectedResponse;
+import org.apache.kylin.rest.service.AbstractModelService;
import org.apache.kylin.rest.service.FusionIndexService;
import org.apache.kylin.rest.service.FusionModelService;
import org.apache.kylin.rest.service.IndexPlanService;
import org.apache.kylin.rest.service.ModelService;
-import org.apache.kylin.rest.util.AclPermissionUtil;
-import org.apache.kylin.tool.bisync.BISyncModel;
+import org.apache.kylin.rest.service.ModelTdsService;
import org.apache.kylin.tool.bisync.SyncContext;
+import org.apache.kylin.tool.bisync.model.SyncModel;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.stereotype.Controller;
@@ -112,6 +109,10 @@ public class NModelController extends NBasicController {
@Qualifier("modelService")
private ModelService modelService;
+ @Autowired
+ @Qualifier("modelTdsService")
+ private ModelTdsService tdsService;
+
@Autowired
private FusionModelService fusionModelService;
@@ -484,7 +485,7 @@ public class NModelController extends NBasicController {
checkRequiredArg(MODEL_ID, modelId);
String newAlias = modelRenameRequest.getNewModelName();
String description = modelRenameRequest.getDescription();
- if (!StringUtils.containsOnly(newAlias, ModelService.VALID_NAME_FOR_MODEL)) {
+ if (!StringUtils.containsOnly(newAlias, AbstractModelService.VALID_NAME_FOR_MODEL)) {
throw new KylinException(MODEL_NAME_INVALID, newAlias);
}
@@ -545,7 +546,7 @@ public class NModelController extends NBasicController {
String newModelName = request.getNewModelName();
checkRequiredArg(MODEL_ID, modelId);
checkRequiredArg(NEW_MODEL_NAME, newModelName);
- if (!StringUtils.containsOnly(newModelName, ModelService.VALID_NAME_FOR_MODEL)) {
+ if (!StringUtils.containsOnly(newModelName, AbstractModelService.VALID_NAME_FOR_MODEL)) {
throw new KylinException(MODEL_NAME_INVALID, newModelName);
}
modelService.cloneModel(modelId, request.getNewModelName(), request.getProject());
@@ -636,9 +637,12 @@ public class NModelController extends NBasicController {
@GetMapping(value = "/validate_export")
@ResponseBody
public EnvelopeResponse<Boolean> validateExport(@RequestParam(value = "model") String modelId,
- @RequestParam(value = "project") String project) {
+ @RequestParam(value = "project") String project,
+ @RequestParam(value = "element", required = false, defaultValue = "AGG_INDEX_COL") SyncContext.ModelElement element) {
String projectName = checkProjectName(project);
- Boolean result = modelService.validateExport(projectName, modelId);
+ SyncContext virtualContext = tdsService.prepareSyncContext(projectName, modelId, null, element, "", -1);
+ SyncModel syncModel = tdsService.exportModel(virtualContext);
+ Boolean result = tdsService.preCheckNameConflict(syncModel);
return new EnvelopeResponse<>(KylinException.CODE_SUCCESS, result, "");
}
@@ -652,75 +656,12 @@ public class NModelController extends NBasicController {
@RequestParam(value = "server_port", required = false) Integer serverPort, HttpServletRequest request,
HttpServletResponse response) throws IOException {
String projectName = checkProjectName(project);
-
String host = getHost(serverHost, request.getServerName());
- Integer port = getPort(serverPort, request.getServerPort());
-
- modelService.validateExport(projectName, modelId);
- BISyncModel syncModel = modelService.exportModel(projectName, modelId, exportAs, element, host, port);
-
- dumpSyncModel(modelId, exportAs, projectName, syncModel, response);
- }
-
- @ApiOperation(value = "biExport", tags = { "QE" })
- @GetMapping(value = "/bi_export")
- @ResponseBody
- public void biExport(@RequestParam("model") String modelId, @RequestParam(value = "project") String project,
- @RequestParam(value = "export_as") SyncContext.BI exportAs,
- @RequestParam(value = "element", required = false, defaultValue = "AGG_INDEX_COL") SyncContext.ModelElement element,
- @RequestParam(value = "server_host", required = false) String serverHost,
- @RequestParam(value = "server_port", required = false) Integer serverPort,
- @RequestParam(value = "dimensions", required = false) List<String> dimensions,
- @RequestParam(value = "measures", required = false) List<String> measures, HttpServletRequest request,
- HttpServletResponse response) throws IOException {
- String projectName = checkProjectName(project);
-
- String host = getHost(serverHost, request.getServerName());
- Integer port = getPort(serverPort, request.getServerPort());
-
- modelService.validateExport(projectName, modelId);
- SyncContext syncContext = modelService.getSyncContext(projectName, modelId, exportAs, element, host, port);
-
- BISyncModel syncModel = AclPermissionUtil.isAdmin()
- ? modelService.exportTDSDimensionsAndMeasuresByAdmin(syncContext, dimensions, measures)
- : modelService.exportTDSDimensionsAndMeasuresByNormalUser(syncContext, dimensions, measures);
-
- dumpSyncModel(modelId, exportAs, projectName, syncModel, response);
- }
-
- private void dumpSyncModel(String modelId, SyncContext.BI exportAs, String projectName, BISyncModel syncModel,
- HttpServletResponse response) throws IOException {
- NDataModelManager manager = NDataModelManager.getInstance(KylinConfig.getInstanceFromEnv(), projectName);
- NDataModel dataModel = manager.getDataModelDesc(modelId);
- String alias = dataModel.getAlias();
- String fileName = String.format(Locale.ROOT, "%s_%s_%s", projectName, alias,
- new SimpleDateFormat("yyyyMMddHHmmss", Locale.getDefault(Locale.Category.FORMAT)).format(new Date()));
- switch (exportAs) {
- case TABLEAU_CONNECTOR_TDS:
- case TABLEAU_ODBC_TDS:
- response.setContentType("application/xml");
- response.setHeader("Content-Disposition",
- String.format(Locale.ROOT, "attachment; filename=\"%s.tds\"", fileName));
- break;
- default:
- throw new KylinException(CommonErrorCode.UNKNOWN_ERROR_CODE, "unrecognized export target");
- }
- syncModel.dump(response.getOutputStream());
- response.getOutputStream().flush();
- response.getOutputStream().close();
- }
-
- private String getHost(String serverHost, String serverName) {
- String host = KylinConfig.getInstanceFromEnv().getModelExportHost();
- host = Optional.ofNullable(Optional.ofNullable(host).orElse(serverHost)).orElse(serverName);
- return host;
- }
+ int port = getPort(serverPort, request.getServerPort());
- private Integer getPort(Integer serverPort, Integer requestServerPort) {
- Integer port = KylinConfig.getInstanceFromEnv().getModelExportPort() == -1 ? null
- : KylinConfig.getInstanceFromEnv().getModelExportPort();
- port = Optional.ofNullable(Optional.ofNullable(port).orElse(serverPort)).orElse(requestServerPort);
- return port;
+ SyncContext syncContext = tdsService.prepareSyncContext(projectName, modelId, exportAs, element, host, port);
+ SyncModel syncModel = tdsService.exportModel(syncContext);
+ tdsService.dumpSyncModel(syncContext, syncModel, response);
}
@ApiOperation(value = "updateMultiPartitionMapping", tags = { "QE" }, notes = "Add URL: {model}")
diff --git a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NTableController.java b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NTableController.java
index 8b6181ef8f..5e43fc3ea3 100644
--- a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NTableController.java
+++ b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NTableController.java
@@ -246,13 +246,12 @@ public class NTableController extends NBasicController {
return new EnvelopeResponse<>(KylinException.CODE_SUCCESS, loadTableResponse, "");
}
-
- @ApiOperation(value = "loadAWSTablesCompatibleCrossAccount", tags = {"KC"},
- notes = "Update Body: data_source_type, need_sampling, sampling_rows, data_source_properties")
+ @ApiOperation(value = "loadAWSTablesCompatibleCrossAccount", tags = {
+ "KC" }, notes = "Update Body: data_source_type, need_sampling, sampling_rows, data_source_properties")
@PostMapping(value = "/compatibility/aws")
@ResponseBody
- public EnvelopeResponse<LoadTableResponse> loadAWSTablesCompatibleCrossAccount(@RequestBody AWSTableLoadRequest tableLoadRequest)
- throws Exception {
+ public EnvelopeResponse<LoadTableResponse> loadAWSTablesCompatibleCrossAccount(
+ @RequestBody AWSTableLoadRequest tableLoadRequest) throws Exception {
checkProjectName(tableLoadRequest.getProject());
if (NProjectManager.getInstance(KylinConfig.getInstanceFromEnv())
.getProject(tableLoadRequest.getProject()) == null) {
@@ -263,8 +262,8 @@ public class NTableController extends NBasicController {
}
LoadTableResponse loadTableResponse = new LoadTableResponse();
- LoadTableResponse loadByTable = tableExtService.loadAWSTablesCompatibleCrossAccount(tableLoadRequest.getTables(),
- tableLoadRequest.getProject());
+ LoadTableResponse loadByTable = tableExtService
+ .loadAWSTablesCompatibleCrossAccount(tableLoadRequest.getTables(), tableLoadRequest.getProject());
loadTableResponse.getFailed().addAll(loadByTable.getFailed());
loadTableResponse.getLoaded().addAll(loadByTable.getLoaded());
@@ -277,13 +276,13 @@ public class NTableController extends NBasicController {
return new EnvelopeResponse<>(KylinException.CODE_SUCCESS, loadTableResponse, "");
}
- @ApiOperation(value = "updateLoadedAWSTableExtProp", tags = {"KC" }, notes = "Update Body: data_source_properties")
+ @ApiOperation(value = "updateLoadedAWSTableExtProp", tags = { "KC" }, notes = "Update Body: data_source_properties")
@PutMapping(value = "/ext/prop/aws")
@ResponseBody
- public EnvelopeResponse<UpdateAWSTableExtDescResponse> updateLoadedAWSTableExtProp(@RequestBody UpdateAWSTableExtDescRequest request) {
+ public EnvelopeResponse<UpdateAWSTableExtDescResponse> updateLoadedAWSTableExtProp(
+ @RequestBody UpdateAWSTableExtDescRequest request) {
checkProjectName(request.getProject());
- if (NProjectManager.getInstance(KylinConfig.getInstanceFromEnv())
- .getProject(request.getProject()) == null) {
+ if (NProjectManager.getInstance(KylinConfig.getInstanceFromEnv()).getProject(request.getProject()) == null) {
throw new KylinException(PROJECT_NOT_EXIST, request.getProject());
}
if (CollectionUtils.isEmpty(request.getTables())) {
diff --git a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/v2/NModelControllerV2.java b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/v2/NModelControllerV2.java
index 5de11e8846..7552b476d5 100644
--- a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/v2/NModelControllerV2.java
+++ b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/v2/NModelControllerV2.java
@@ -20,14 +20,13 @@ package org.apache.kylin.rest.controller.v2;
import static org.apache.kylin.common.constant.HttpConstant.HTTP_VND_APACHE_KYLIN_V2_JSON;
import java.util.ArrayList;
-import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.kylin.common.exception.KylinException;
-import org.apache.kylin.rest.response.EnvelopeResponse;
import org.apache.kylin.metadata.model.NDataModel;
import org.apache.kylin.rest.controller.NBasicController;
+import org.apache.kylin.rest.response.EnvelopeResponse;
import org.apache.kylin.rest.service.ModelService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
@@ -65,7 +64,7 @@ public class NModelControllerV2 extends NBasicController {
modelService.getModels(modelAlias, project, exactMatch, null, Lists.newArrayList(), sortBy, reverse));
models = modelService.addOldParams(project, models);
- HashMap<String, Object> modelResponse = getDataResponse("models", models, offset, limit);
+ Map<String, Object> modelResponse = getDataResponse("models", models, offset, limit);
return new EnvelopeResponse<>(KylinException.CODE_SUCCESS, modelResponse, "");
}
diff --git a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/v2/NProjectControllerV2.java b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/v2/NProjectControllerV2.java
index e9f864080d..060dc323a0 100644
--- a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/v2/NProjectControllerV2.java
+++ b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/v2/NProjectControllerV2.java
@@ -19,13 +19,13 @@ package org.apache.kylin.rest.controller.v2;
import static org.apache.kylin.common.constant.HttpConstant.HTTP_VND_APACHE_KYLIN_V2_JSON;
-import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.metadata.project.ProjectInstance;
-import org.apache.kylin.rest.response.EnvelopeResponse;
import org.apache.kylin.rest.controller.NBasicController;
+import org.apache.kylin.rest.response.EnvelopeResponse;
import org.apache.kylin.rest.service.ProjectService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
@@ -54,7 +54,7 @@ public class NProjectControllerV2 extends NBasicController {
@RequestParam(value = "exact", required = false, defaultValue = "true") boolean exactMatch) {
List<ProjectInstance> readableProjects = projectService.getReadableProjects(project, exactMatch);
- HashMap<String, Object> projects = getDataResponse("projects", readableProjects, offset, size);
+ Map<String, Object> projects = getDataResponse("projects", readableProjects, offset, size);
return new EnvelopeResponse<>(KylinException.CODE_SUCCESS, projects, "");
}
diff --git a/src/metadata-server/src/test/java/io/kyligence/kap/rest/controller/open/OpenModelControllerTest.java b/src/metadata-server/src/test/java/io/kyligence/kap/rest/controller/open/OpenModelControllerTest.java
index 0d22ea3cef..e36ccf48de 100644
--- a/src/metadata-server/src/test/java/io/kyligence/kap/rest/controller/open/OpenModelControllerTest.java
+++ b/src/metadata-server/src/test/java/io/kyligence/kap/rest/controller/open/OpenModelControllerTest.java
@@ -37,6 +37,7 @@ import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.common.util.RandomUtil;
import org.apache.kylin.metadata.cube.model.IndexEntity;
+import org.apache.kylin.metadata.cube.model.NDataflowManager;
import org.apache.kylin.metadata.model.MultiPartitionDesc;
import org.apache.kylin.metadata.model.NDataModel;
import org.apache.kylin.metadata.model.NDataModelManager;
@@ -61,7 +62,10 @@ import org.apache.kylin.rest.response.OpenGetIndexResponse;
import org.apache.kylin.rest.service.FusionIndexService;
import org.apache.kylin.rest.service.FusionModelService;
import org.apache.kylin.rest.service.ModelService;
+import org.apache.kylin.rest.service.ModelTdsService;
import org.apache.kylin.rest.util.AclEvaluate;
+import org.apache.kylin.tool.bisync.SyncContext;
+import org.apache.kylin.tool.bisync.model.SyncModel;
import org.hamcrest.Matchers;
import org.junit.After;
import org.junit.Assert;
@@ -102,6 +106,9 @@ public class OpenModelControllerTest extends NLocalFileMetadataTestCase {
@Mock
private FusionModelService fusionModelService;
+ @Mock
+ private ModelTdsService tdsService;
+
@Mock
private AclEvaluate aclEvaluate;
@@ -502,6 +509,66 @@ public class OpenModelControllerTest extends NLocalFileMetadataTestCase {
.andExpect(MockMvcResultMatchers.status().isOk());
}
+ @Test
+ public void testBIExportByADMIN() throws Exception {
+ String project = "default";
+ String modelName = "741ca86a-1f13-46da-a59f-95fb68615e3a";
+ SyncContext syncContext = new SyncContext();
+ syncContext.setProjectName(project);
+ syncContext.setModelId(modelName);
+ syncContext.setTargetBI(SyncContext.BI.TABLEAU_CONNECTOR_TDS);
+ syncContext.setModelElement(SyncContext.ModelElement.AGG_INDEX_AND_TABLE_INDEX_COL);
+ syncContext.setHost("localhost");
+ syncContext.setPort(8080);
+ syncContext.setDataflow(NDataflowManager.getInstance(getTestConfig(), project).getDataflow(modelName));
+ syncContext.setKylinConfig(getTestConfig());
+ SyncModel syncModel = Mockito.mock(SyncModel.class);
+ NDataModel model = new NDataModel();
+ model.setUuid("aaa");
+ Mockito.doReturn(model).when(openModelController).getModel(Mockito.anyString(), Mockito.anyString());
+ Mockito.doReturn(syncContext).when(tdsService).prepareSyncContext(project, modelName,
+ SyncContext.BI.TABLEAU_CONNECTOR_TDS, SyncContext.ModelElement.CUSTOM_COLS, "localhost", 8080);
+ Mockito.doReturn(syncModel).when(tdsService).exportTDSDimensionsAndMeasuresByAdmin(syncContext,
+ Lists.newArrayList(), Lists.newArrayList());
+ mockMvc.perform(MockMvcRequestBuilders.get("/api/models/bi_export").param("model_name", modelName)
+ .param("project", project).param("export_as", "TABLEAU_CONNECTOR_TDS").param("element", "CUSTOM_COLS")
+ .param("server_host", "localhost").param("server_port", "8080").param("dimensions", "")
+ .param("measures", "").contentType(MediaType.APPLICATION_JSON)
+ .accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON)))
+ .andExpect(MockMvcResultMatchers.status().isOk());
+ }
+
+ @Test
+ public void testBIExportByNormalUser() throws Exception {
+ String project = "default";
+ String modelName = "741ca86a-1f13-46da-a59f-95fb68615e3a";
+ SyncContext syncContext = new SyncContext();
+ syncContext.setProjectName(project);
+ syncContext.setModelId(modelName);
+ syncContext.setTargetBI(SyncContext.BI.TABLEAU_CONNECTOR_TDS);
+ syncContext.setModelElement(SyncContext.ModelElement.AGG_INDEX_AND_TABLE_INDEX_COL);
+ syncContext.setHost("localhost");
+ syncContext.setPort(8080);
+ syncContext.setDataflow(NDataflowManager.getInstance(getTestConfig(), project).getDataflow(modelName));
+ syncContext.setKylinConfig(getTestConfig());
+ SyncModel syncModel = Mockito.mock(SyncModel.class);
+ SecurityContextHolder.getContext()
+ .setAuthentication(new TestingAuthenticationToken("u1", "ANALYST", Constant.ROLE_ANALYST));
+ NDataModel model = new NDataModel();
+ model.setUuid("aaa");
+ Mockito.doReturn(model).when(openModelController).getModel(Mockito.anyString(), Mockito.anyString());
+ Mockito.doReturn(syncContext).when(tdsService).prepareSyncContext(project, modelName,
+ SyncContext.BI.TABLEAU_CONNECTOR_TDS, SyncContext.ModelElement.CUSTOM_COLS, "localhost", 8080);
+ Mockito.doReturn(syncModel).when(tdsService).exportTDSDimensionsAndMeasuresByNormalUser(syncContext,
+ Lists.newArrayList(), Lists.newArrayList());
+ mockMvc.perform(MockMvcRequestBuilders.get("/api/models/bi_export").param("model_name", modelName)
+ .param("project", project).param("export_as", "TABLEAU_CONNECTOR_TDS").param("element", "CUSTOM_COLS")
+ .param("server_host", "localhost").param("server_port", "8080").param("dimensions", "")
+ .param("measures", "").contentType(MediaType.APPLICATION_JSON)
+ .accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON)))
+ .andExpect(MockMvcResultMatchers.status().isOk());
+ }
+
@Test
public void testUpdateModelStatus() throws Exception {
String project = "default";
@@ -566,6 +633,8 @@ public class OpenModelControllerTest extends NLocalFileMetadataTestCase {
OpenModelRequest request = new OpenModelRequest();
request.setProject(project);
request.setModelName(modelAlias);
+ Mockito.doReturn(Mockito.mock(BuildBaseIndexResponse.class)).when(fusionModelService)
+ .updateDataModelSemantic(request.getProject(), request);
mockMvc.perform(MockMvcRequestBuilders.put("/api/models/modification").contentType(MediaType.APPLICATION_JSON)
.content(JsonUtil.writeValueAsString(request))
.accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON)))
diff --git a/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NModelControllerTest.java b/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NModelControllerTest.java
index 7fccbc44aa..47520fa814 100644
--- a/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NModelControllerTest.java
+++ b/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NModelControllerTest.java
@@ -62,9 +62,9 @@ import org.apache.kylin.rest.response.NDataSegmentResponse;
import org.apache.kylin.rest.response.RelatedModelResponse;
import org.apache.kylin.rest.service.FusionModelService;
import org.apache.kylin.rest.service.ModelService;
-import org.apache.kylin.tool.bisync.BISyncModel;
-import org.apache.kylin.tool.bisync.BISyncTool;
+import org.apache.kylin.rest.service.ModelTdsService;
import org.apache.kylin.tool.bisync.SyncContext;
+import org.apache.kylin.tool.bisync.model.SyncModel;
import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
@@ -96,6 +96,9 @@ public class NModelControllerTest extends NLocalFileMetadataTestCase {
@Mock
private ModelService modelService;
+ @Mock
+ private ModelTdsService tdsService;
+
@Mock
private FusionModelService fusionModelService;
@@ -673,7 +676,7 @@ public class NModelControllerTest extends NLocalFileMetadataTestCase {
UpdateMultiPartitionValueRequest request = new UpdateMultiPartitionValueRequest();
request.setProject("default");
List<String[]> partition_values = Lists.newArrayList();
- String[] value = new String[]{"5"};
+ String[] value = new String[] { "5" };
partition_values.add(value);
request.setSubPartitionValues(partition_values);
Mockito.doNothing().when(modelService).addMultiPartitionValues(request.getProject(),
@@ -713,64 +716,18 @@ public class NModelControllerTest extends NLocalFileMetadataTestCase {
}
@Test
- public void testBIExportByADMIN() throws Exception {
- String project = "default";
- String modelName = "741ca86a-1f13-46da-a59f-95fb68615e3a";
- SyncContext syncContext = new SyncContext();
- syncContext.setProjectName(project);
- syncContext.setModelId(modelName);
- syncContext.setTargetBI(SyncContext.BI.TABLEAU_CONNECTOR_TDS);
- syncContext.setModelElement(SyncContext.ModelElement.AGG_INDEX_AND_TABLE_INDEX_COL);
- syncContext.setHost("localhost");
- syncContext.setPort(8080);
- syncContext.setDataflow(NDataflowManager.getInstance(getTestConfig(), project).getDataflow(modelName));
- syncContext.setKylinConfig(getTestConfig());
- BISyncModel syncModel = BISyncTool.dumpToBISyncModel(syncContext);
- Mockito.doReturn(syncContext).when(modelService).getSyncContext(project, modelName,
- SyncContext.BI.TABLEAU_CONNECTOR_TDS, SyncContext.ModelElement.CUSTOM_COLS, "localhost", 8080);
- Mockito.doReturn(syncModel).when(modelService).exportTDSDimensionsAndMeasuresByAdmin(syncContext,
- Lists.newArrayList(), Lists.newArrayList());
- mockMvc.perform(MockMvcRequestBuilders.get("/api/models/bi_export").param("model", modelName)
- .param("project", project).param("export_as", "TABLEAU_CONNECTOR_TDS").param("element", "CUSTOM_COLS")
- .param("server_host", "localhost").param("server_port", "8080").param("dimensions", "")
- .param("measures", "").contentType(MediaType.APPLICATION_JSON)
- .accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_JSON)))
- .andExpect(MockMvcResultMatchers.status().isOk());
- }
-
- @Test
- public void testBIExportByNormalUser() throws Exception {
+ public void testValidateExport() throws Exception {
String project = "default";
String modelName = "741ca86a-1f13-46da-a59f-95fb68615e3a";
SyncContext syncContext = new SyncContext();
syncContext.setProjectName(project);
syncContext.setModelId(modelName);
- syncContext.setTargetBI(SyncContext.BI.TABLEAU_CONNECTOR_TDS);
- syncContext.setModelElement(SyncContext.ModelElement.AGG_INDEX_AND_TABLE_INDEX_COL);
- syncContext.setHost("localhost");
- syncContext.setPort(8080);
- syncContext.setDataflow(NDataflowManager.getInstance(getTestConfig(), project).getDataflow(modelName));
- syncContext.setKylinConfig(getTestConfig());
- BISyncModel syncModel = BISyncTool.dumpToBISyncModel(syncContext);
- SecurityContextHolder.getContext()
- .setAuthentication(new TestingAuthenticationToken("u1", "ANALYST", Constant.ROLE_ANALYST));
- Mockito.doReturn(syncContext).when(modelService).getSyncContext(project, modelName,
+ syncContext.setModelElement(SyncContext.ModelElement.ALL_COLS);
+ SyncModel syncModel = Mockito.mock(SyncModel.class);
+ Mockito.doReturn(syncContext).when(tdsService).prepareSyncContext(project, modelName,
SyncContext.BI.TABLEAU_CONNECTOR_TDS, SyncContext.ModelElement.CUSTOM_COLS, "localhost", 8080);
- Mockito.doReturn(syncModel).when(modelService).exportTDSDimensionsAndMeasuresByNormalUser(syncContext,
- Lists.newArrayList(), Lists.newArrayList());
- mockMvc.perform(MockMvcRequestBuilders.get("/api/models/bi_export").param("model", modelName)
- .param("project", project).param("export_as", "TABLEAU_CONNECTOR_TDS").param("element", "CUSTOM_COLS")
- .param("server_host", "localhost").param("server_port", "8080").param("dimensions", "")
- .param("measures", "").contentType(MediaType.APPLICATION_JSON)
- .accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_JSON)))
- .andExpect(MockMvcResultMatchers.status().isOk());
- }
-
- @Test
- public void testValidateExport() throws Exception {
- String project = "default";
- String modelName = "741ca86a-1f13-46da-a59f-95fb68615e3a";
- when(modelService.validateExport(project, modelName)).thenReturn(Boolean.TRUE);
+ Mockito.doReturn(syncModel).when(tdsService).exportModel(syncContext);
+ Mockito.doReturn(Boolean.TRUE).when(tdsService).preCheckNameConflict(syncModel);
mockMvc.perform(MockMvcRequestBuilders.get("/api/models/validate_export").param("model", modelName)
.param("project", project).contentType(MediaType.APPLICATION_JSON))
.andExpect(MockMvcResultMatchers.status().isOk());
@@ -789,10 +746,9 @@ public class NModelControllerTest extends NLocalFileMetadataTestCase {
syncContext.setPort(8080);
syncContext.setDataflow(NDataflowManager.getInstance(getTestConfig(), project).getDataflow(modelName));
syncContext.setKylinConfig(getTestConfig());
- BISyncModel syncModel = BISyncTool.dumpToBISyncModel(syncContext);
- Mockito.doReturn(syncModel).when(modelService).exportModel(project, modelName,
- SyncContext.BI.TABLEAU_CONNECTOR_TDS, SyncContext.ModelElement.AGG_INDEX_AND_TABLE_INDEX_COL,
- "localhost", 8080);
+ syncContext.setAdmin(true);
+ SyncModel syncModel = Mockito.mock(SyncModel.class);
+ Mockito.doReturn(syncModel).when(tdsService).exportModel(syncContext);
mockMvc.perform(MockMvcRequestBuilders.get("/api/models/{model}/export", modelName).param("project", project)
.param("export_as", "TABLEAU_CONNECTOR_TDS").param("element", "AGG_INDEX_AND_TABLE_INDEX_COL")
.param("server_host", "localhost").param("server_port", "8080").contentType(MediaType.APPLICATION_JSON)
diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/AbstractModelService.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/AbstractModelService.java
new file mode 100644
index 0000000000..b46314b2e1
--- /dev/null
+++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/AbstractModelService.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.rest.service;
+
+import static org.apache.kylin.common.exception.ServerErrorCode.FAILED_UPDATE_MODEL;
+import static org.apache.kylin.common.exception.code.ErrorCodeServer.MODEL_ID_NOT_EXIST;
+import static org.apache.kylin.common.exception.code.ErrorCodeServer.MODEL_NAME_EMPTY;
+import static org.apache.kylin.common.exception.code.ErrorCodeServer.MODEL_NAME_INVALID;
+import static org.apache.kylin.common.exception.code.ErrorCodeServer.MODEL_NAME_NOT_EXIST;
+import static org.apache.kylin.common.exception.code.ErrorCodeServer.MODEL_NOT_EXIST;
+
+import java.util.Arrays;
+import java.util.Set;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.kylin.common.exception.KylinException;
+import org.apache.kylin.common.msg.MsgPicker;
+import org.apache.kylin.metadata.acl.AclTCRManager;
+import org.apache.kylin.metadata.cube.model.IndexPlan;
+import org.apache.kylin.metadata.cube.model.NIndexPlanManager;
+import org.apache.kylin.metadata.model.ColumnDesc;
+import org.apache.kylin.metadata.model.NDataModel;
+import org.apache.kylin.metadata.model.NDataModelManager;
+import org.apache.kylin.metadata.model.NTableMetadataManager;
+import org.apache.kylin.rest.util.AclEvaluate;
+import org.apache.kylin.rest.util.AclPermissionUtil;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import com.google.common.collect.Sets;
+
+import lombok.val;
+import lombok.var;
+
+public class AbstractModelService extends BasicService {
+
+ public static final String VALID_NAME_FOR_MODEL = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_";
+
+ @Autowired
+ public AclEvaluate aclEvaluate;
+
+ @Autowired
+ public AccessService accessService;
+
+ public void checkModelPermission(String project, String modelId) {
+ String userName = aclEvaluate.getCurrentUserName();
+ Set<String> groups = getCurrentUserGroups();
+ if (AclPermissionUtil.isAdmin() || AclPermissionUtil.isAdminInProject(project, groups)) {
+ return;
+ }
+ Set<String> allAuthTables = Sets.newHashSet();
+ Set<String> allAuthColumns = Sets.newHashSet();
+ var auths = getManager(AclTCRManager.class, project).getAuthTablesAndColumns(project, userName, true);
+ allAuthTables.addAll(auths.getTables());
+ allAuthColumns.addAll(auths.getColumns());
+ for (val group : groups) {
+ auths = getManager(AclTCRManager.class, project).getAuthTablesAndColumns(project, group, false);
+ allAuthTables.addAll(auths.getTables());
+ allAuthColumns.addAll(auths.getColumns());
+ }
+
+ NDataModel model = getModelById(modelId, project);
+ Set<String> tablesInModel = Sets.newHashSet();
+ model.getJoinTables().forEach(table -> tablesInModel.add(table.getTable()));
+ tablesInModel.add(model.getRootFactTableName());
+ tablesInModel.forEach(table -> {
+ if (!allAuthTables.contains(table)) {
+ throw new KylinException(FAILED_UPDATE_MODEL, MsgPicker.getMsg().getModelModifyAbandon(table));
+ }
+ });
+ tablesInModel.stream().filter(allAuthTables::contains).forEach(table -> {
+ ColumnDesc[] columnDescs = NTableMetadataManager.getInstance(getConfig(), project).getTableDesc(table)
+ .getColumns();
+ Arrays.stream(columnDescs).map(column -> table + "." + column.getName()).forEach(column -> {
+ if (!allAuthColumns.contains(column)) {
+ throw new KylinException(FAILED_UPDATE_MODEL, MsgPicker.getMsg().getModelModifyAbandon(column));
+ }
+ });
+ });
+ }
+
+ public NDataModel getModelById(String modelId, String project) {
+ NDataModelManager modelManager = getManager(NDataModelManager.class, project);
+ NDataModel nDataModel = modelManager.getDataModelDesc(modelId);
+ if (null == nDataModel) {
+ throw new KylinException(MODEL_ID_NOT_EXIST, modelId);
+ }
+ return nDataModel;
+ }
+
+ public NDataModel getModelByAlias(String modelAlias, String project) {
+ NDataModelManager modelManager = getManager(NDataModelManager.class, project);
+ NDataModel nDataModel = modelManager.getDataModelDescByAlias(modelAlias);
+ if (null == nDataModel) {
+ throw new KylinException(MODEL_NAME_NOT_EXIST, modelAlias);
+ }
+ return nDataModel;
+ }
+
+ public Set<String> listAllModelIdsInProject(String project) {
+ NDataModelManager dataModelManager = getManager(NDataModelManager.class, project);
+ return dataModelManager.listAllModelIds();
+ }
+
+ public IndexPlan getIndexPlan(String modelId, String project) {
+ NIndexPlanManager indexPlanManager = getManager(NIndexPlanManager.class, project);
+ return indexPlanManager.getIndexPlan(modelId);
+ }
+
+ public void primaryCheck(NDataModel modelDesc) {
+ if (modelDesc == null) {
+ throw new KylinException(MODEL_NOT_EXIST);
+ }
+
+ String modelAlias = modelDesc.getAlias();
+
+ if (StringUtils.isEmpty(modelAlias)) {
+ throw new KylinException(MODEL_NAME_EMPTY);
+ }
+ if (!StringUtils.containsOnly(modelAlias, VALID_NAME_FOR_MODEL)) {
+ throw new KylinException(MODEL_NAME_INVALID, modelAlias);
+ }
+ }
+
+}
diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/FusionModelService.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/FusionModelService.java
index 23f2e11328..84bf26d971 100644
--- a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/FusionModelService.java
+++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/FusionModelService.java
@@ -26,10 +26,9 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.exception.ServerErrorCode;
import org.apache.kylin.common.msg.MsgPicker;
+import org.apache.kylin.common.scheduler.EventBusFactory;
import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.rest.response.DataResult;
-import org.apache.kylin.common.scheduler.EventBusFactory;
import org.apache.kylin.metadata.model.FusionModel;
import org.apache.kylin.metadata.model.FusionModelManager;
import org.apache.kylin.metadata.model.NDataModel;
@@ -39,6 +38,7 @@ import org.apache.kylin.rest.request.IndexesToSegmentsRequest;
import org.apache.kylin.rest.request.ModelRequest;
import org.apache.kylin.rest.request.OwnerChangeRequest;
import org.apache.kylin.rest.response.BuildBaseIndexResponse;
+import org.apache.kylin.rest.response.DataResult;
import org.apache.kylin.rest.response.JobInfoResponse;
import org.apache.kylin.rest.response.JobInfoResponseWithFailure;
import org.apache.kylin.rest.response.NDataModelResponse;
@@ -53,7 +53,7 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service("fusionModelService")
-public class FusionModelService extends BasicService implements TableFusionModelSupporter {
+public class FusionModelService extends AbstractModelService implements TableFusionModelSupporter {
@Autowired
private ModelService modelService;
diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
index 474b10fe82..7082c4890b 100644
--- a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
+++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
@@ -42,13 +42,11 @@ import static org.apache.kylin.common.exception.ServerErrorCode.INVALID_PARTITIO
import static org.apache.kylin.common.exception.ServerErrorCode.INVALID_RANGE;
import static org.apache.kylin.common.exception.ServerErrorCode.INVALID_SEGMENT_PARAMETER;
import static org.apache.kylin.common.exception.ServerErrorCode.MODEL_BROKEN;
-import static org.apache.kylin.common.exception.ServerErrorCode.MODEL_EXPORT_ERROR;
import static org.apache.kylin.common.exception.ServerErrorCode.MODEL_ONLINE_ABANDON;
import static org.apache.kylin.common.exception.ServerErrorCode.PERMISSION_DENIED;
import static org.apache.kylin.common.exception.ServerErrorCode.STREAMING_INDEX_UPDATE_DISABLE;
import static org.apache.kylin.common.exception.ServerErrorCode.TABLE_NOT_EXIST;
import static org.apache.kylin.common.exception.ServerErrorCode.TIMESTAMP_COLUMN_NOT_EXIST;
-import static org.apache.kylin.common.exception.ServerErrorCode.UNAUTHORIZED_ENTITY;
import static org.apache.kylin.common.exception.ServerErrorCode.VIEW_PARTITION_DATE_FORMAT_DETECTION_FORBIDDEN;
import static org.apache.kylin.common.exception.code.ErrorCodeServer.COMPUTED_COLUMN_CONFLICT;
import static org.apache.kylin.common.exception.code.ErrorCodeServer.COMPUTED_COLUMN_NAME_OR_EXPR_EMPTY;
@@ -56,10 +54,7 @@ import static org.apache.kylin.common.exception.code.ErrorCodeServer.DATETIME_FO
import static org.apache.kylin.common.exception.code.ErrorCodeServer.DATETIME_FORMAT_PARSE_ERROR;
import static org.apache.kylin.common.exception.code.ErrorCodeServer.MODEL_ID_NOT_EXIST;
import static org.apache.kylin.common.exception.code.ErrorCodeServer.MODEL_NAME_DUPLICATE;
-import static org.apache.kylin.common.exception.code.ErrorCodeServer.MODEL_NAME_EMPTY;
-import static org.apache.kylin.common.exception.code.ErrorCodeServer.MODEL_NAME_INVALID;
import static org.apache.kylin.common.exception.code.ErrorCodeServer.MODEL_NAME_NOT_EXIST;
-import static org.apache.kylin.common.exception.code.ErrorCodeServer.MODEL_NOT_EXIST;
import static org.apache.kylin.common.exception.code.ErrorCodeServer.PARAMETER_INVALID_SUPPORT_LIST;
import static org.apache.kylin.common.exception.code.ErrorCodeServer.PROJECT_NOT_EXIST;
import static org.apache.kylin.common.exception.code.ErrorCodeServer.SEGMENT_BUILD_RANGE_OVERLAP;
@@ -133,7 +128,6 @@ import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.common.util.RandomUtil;
import org.apache.kylin.common.util.StringUtil;
-import org.apache.kylin.engine.spark.smarter.IndexDependencyParser;
import org.apache.kylin.engine.spark.utils.ComputedColumnEvalUtil;
import org.apache.kylin.job.SecondStorageJobParamUtil;
import org.apache.kylin.job.common.SegmentUtil;
@@ -146,7 +140,6 @@ import org.apache.kylin.job.handler.SecondStorageSegmentCleanJobHandler;
import org.apache.kylin.job.handler.SecondStorageSegmentLoadJobHandler;
import org.apache.kylin.job.manager.JobManager;
import org.apache.kylin.job.model.JobParam;
-import org.apache.kylin.metadata.acl.AclTCRDigest;
import org.apache.kylin.metadata.acl.AclTCRManager;
import org.apache.kylin.metadata.acl.NDataModelAclParams;
import org.apache.kylin.metadata.cube.cuboid.NAggregationGroup;
@@ -163,6 +156,7 @@ import org.apache.kylin.metadata.cube.model.NDataflowManager;
import org.apache.kylin.metadata.cube.model.NDataflowUpdate;
import org.apache.kylin.metadata.cube.model.NIndexPlanManager;
import org.apache.kylin.metadata.cube.model.RuleBasedIndex;
+import org.apache.kylin.metadata.favorite.FavoriteRuleManager;
import org.apache.kylin.metadata.model.AutoMergeTimeEnum;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.ComputedColumnDesc;
@@ -196,7 +190,6 @@ import org.apache.kylin.metadata.model.UpdateImpact;
import org.apache.kylin.metadata.model.VolatileRange;
import org.apache.kylin.metadata.model.schema.AffectedModelContext;
import org.apache.kylin.metadata.model.tool.CalciteParser;
-import org.apache.kylin.metadata.model.util.ComputedColumnUtil;
import org.apache.kylin.metadata.model.util.MultiPartitionUtil;
import org.apache.kylin.metadata.model.util.scd2.SCD2CondChecker;
import org.apache.kylin.metadata.project.EnhancedUnitOfWork;
@@ -204,6 +197,7 @@ import org.apache.kylin.metadata.project.NProjectManager;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.realization.RealizationStatusEnum;
import org.apache.kylin.metadata.streaming.KafkaConfig;
+import org.apache.kylin.query.util.KapQueryUtil;
import org.apache.kylin.query.util.PushDownUtil;
import org.apache.kylin.query.util.QueryParams;
import org.apache.kylin.rest.aspect.Transaction;
@@ -244,12 +238,10 @@ import org.apache.kylin.rest.response.SegmentCheckResponse;
import org.apache.kylin.rest.response.SegmentPartitionResponse;
import org.apache.kylin.rest.response.SegmentRangeResponse;
import org.apache.kylin.rest.response.SimplifiedMeasure;
-import org.apache.kylin.rest.security.MutableAclRecord;
import org.apache.kylin.rest.service.params.FullBuildSegmentParams;
import org.apache.kylin.rest.service.params.IncrementBuildSegmentParams;
import org.apache.kylin.rest.service.params.MergeSegmentParams;
import org.apache.kylin.rest.service.params.ModelQueryParams;
-import org.apache.kylin.rest.util.AclEvaluate;
import org.apache.kylin.rest.util.AclPermissionUtil;
import org.apache.kylin.rest.util.ModelTriple;
import org.apache.kylin.rest.util.ModelUtils;
@@ -260,10 +252,6 @@ import org.apache.kylin.source.adhocquery.PushDownConverterKeyWords;
import org.apache.kylin.streaming.event.StreamingJobDropEvent;
import org.apache.kylin.streaming.event.StreamingJobKillEvent;
import org.apache.kylin.streaming.manager.StreamingJobManager;
-import org.apache.kylin.tool.bisync.BISyncModel;
-import org.apache.kylin.tool.bisync.BISyncTool;
-import org.apache.kylin.tool.bisync.SyncContext;
-import org.apache.kylin.tool.bisync.model.MeasureDef;
import org.apache.spark.sql.SparderEnv;
import org.apache.spark.sql.SparkSession;
import org.slf4j.Logger;
@@ -286,8 +274,6 @@ import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import io.kyligence.kap.guava20.shaded.common.base.Supplier;
-import org.apache.kylin.metadata.favorite.FavoriteRuleManager;
-import org.apache.kylin.query.util.KapQueryUtil;
import io.kyligence.kap.secondstorage.SecondStorage;
import io.kyligence.kap.secondstorage.SecondStorageNodeHelper;
import io.kyligence.kap.secondstorage.SecondStorageUpdater;
@@ -304,15 +290,13 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
@Component("modelService")
-public class ModelService extends BasicService implements TableModelSupporter, ProjectModelSupporter {
+public class ModelService extends AbstractModelService implements TableModelSupporter, ProjectModelSupporter {
private static final Logger logger = LoggerFactory.getLogger(ModelService.class);
private static final String LAST_MODIFY = "last_modify";
public static final String REC_COUNT = "recommendations_count";
- public static final String VALID_NAME_FOR_MODEL = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_";
-
public static final String VALID_NAME_FOR_DIMENSION = "^[\\u4E00-\\u9FA5a-zA-Z0-9 _\\-()%?（）]+$";
public static final String VALID_NAME_FOR_MEASURE = "^[\\u4E00-\\u9FA5a-zA-Z0-9 _\\-()%?（）.]+$";
@@ -334,15 +318,9 @@ public class ModelService extends BasicService implements TableModelSupporter, P
@Qualifier("segmentHelper")
private SegmentHelperSupporter segmentHelper;
- @Autowired
- public AclEvaluate aclEvaluate;
-
@Autowired
private ProjectService projectService;
- @Autowired
- private AccessService accessService;
-
@Setter
@Autowired(required = false)
private ModelQuerySupporter modelQuerySupporter;
@@ -359,24 +337,6 @@ public class ModelService extends BasicService implements TableModelSupporter, P
@Autowired(required = false)
private List<ModelChangeSupporter> modelChangeSupporters = Lists.newArrayList();
- public NDataModel getModelById(String modelId, String project) {
- NDataModelManager modelManager = getManager(NDataModelManager.class, project);
- NDataModel nDataModel = modelManager.getDataModelDesc(modelId);
- if (null == nDataModel) {
- throw new KylinException(MODEL_ID_NOT_EXIST, modelId);
- }
- return nDataModel;
- }
-
- public NDataModel getModelByAlias(String modelAlias, String project) {
- NDataModelManager modelManager = getManager(NDataModelManager.class, project);
- NDataModel nDataModel = modelManager.getDataModelDescByAlias(modelAlias);
- if (null == nDataModel) {
- throw new KylinException(MODEL_NAME_NOT_EXIST, modelAlias);
- }
- return nDataModel;
- }
-
/**
* for 3x rest api
*
@@ -875,10 +835,10 @@ public class ModelService extends BasicService implements TableModelSupporter, P
return modelResponseStatus;
}
- private long getEmptyIndexesCount(String project, String id) {
+ private long getEmptyIndexesCount(String project, String modelId) {
val indexPlanManager = getManager(NIndexPlanManager.class, project);
- val indexPlan = indexPlanManager.getIndexPlan(id);
- return indexPlan.getAllLayoutsReadOnly().size() - indexPlanManager.getAvailableIndexesCount(project, id);
+ val indexPlan = indexPlanManager.getIndexPlan(modelId);
+ return indexPlan.getAllLayoutsReadOnly().size() - indexPlanManager.getAvailableIndexesCount(project, modelId);
}
private List<NDataModelResponse> sortExpansionRate(boolean reverse, List<NDataModelResponse> filterModels) {
@@ -934,7 +894,7 @@ public class ModelService extends BasicService implements TableModelSupporter, P
&& isSCD2;
}
- protected RealizationStatusEnum getModelStatus(String modelId, String projectName) {
+ public RealizationStatusEnum getModelStatus(String modelId, String projectName) {
val indexPlan = getIndexPlan(modelId, projectName);
if (indexPlan != null) {
return getManager(NDataflowManager.class, projectName).getDataflow(indexPlan.getUuid()).getStatus();
@@ -1366,12 +1326,6 @@ public class ModelService extends BasicService implements TableModelSupporter, P
return relatedModel;
}
- @VisibleForTesting
- public IndexPlan getIndexPlan(String modelId, String project) {
- NIndexPlanManager indexPlanManager = getManager(NIndexPlanManager.class, project);
- return indexPlanManager.getIndexPlan(modelId);
- }
-
private void checkAliasExist(String modelId, String newAlias, String project) {
if (!checkModelAliasUniqueness(modelId, newAlias, project)) {
throw new KylinException(MODEL_NAME_DUPLICATE, newAlias);
@@ -1539,11 +1493,6 @@ public class ModelService extends BasicService implements TableModelSupporter, P
}
}
- public Set<String> listAllModelIdsInProject(String project) {
- NDataModelManager dataModelManager = getManager(NDataModelManager.class, project);
- return dataModelManager.listAllModelIds();
- }
-
@Transaction(project = 0)
public void offlineAllModelsInProject(String project) {
aclEvaluate.checkProjectWritePermission(project);
@@ -1848,7 +1797,7 @@ public class ModelService extends BasicService implements TableModelSupporter, P
.anyMatch(column -> column.getName().equalsIgnoreCase(columnName));
if (!hasPartitionColumn && !modelRequest.getDimensionNameIdMap().containsKey(fullColumnName)) {
throw new KylinException(TIMESTAMP_COLUMN_NOT_EXIST,
- String.format(Locale.ROOT, MsgPicker.getMsg().getTimestampPartitionColumnNotExist()));
+ MsgPicker.getMsg().getTimestampPartitionColumnNotExist());
}
}
}
@@ -2624,21 +2573,6 @@ public class ModelService extends BasicService implements TableModelSupporter, P
}
}
- public void primaryCheck(NDataModel modelDesc) {
- if (modelDesc == null) {
- throw new KylinException(MODEL_NOT_EXIST);
- }
-
- String modelAlias = modelDesc.getAlias();
-
- if (StringUtils.isEmpty(modelAlias)) {
- throw new KylinException(MODEL_NAME_EMPTY);
- }
- if (!StringUtils.containsOnly(modelAlias, VALID_NAME_FOR_MODEL)) {
- throw new KylinException(MODEL_NAME_INVALID, modelAlias);
- }
- }
-
public ComputedColumnUsageResponse getComputedColumnUsages(String project) {
aclEvaluate.checkProjectWritePermission(project);
ComputedColumnUsageResponse ret = new ComputedColumnUsageResponse();
@@ -4007,241 +3941,6 @@ public class ModelService extends BasicService implements TableModelSupporter, P
: dataModelDesc.getPartitionDesc().getPartitionDateFormat();
}
- public BISyncModel exportModel(String projectName, String modelId, SyncContext.BI targetBI,
- SyncContext.ModelElement modelElement, String host, int port) {
- SyncContext syncContext = getADMINSyncContext(projectName, modelId, targetBI, modelElement, host, port);
-
- return BISyncTool.dumpToBISyncModel(syncContext);
- }
-
- public BISyncModel exportTDSDimensionsAndMeasuresByNormalUser(SyncContext syncContext, List<String> dimensions,
- List<String> measures) {
- Set<String> groups = getCurrentUserGroups();
- String currentUserName = aclEvaluate.getCurrentUserName();
- String projectName = syncContext.getProjectName();
- String modelId = syncContext.getModelId();
- NDataflow dataflow = getManager(NDataflowManager.class, projectName).getDataflow(modelId);
- if (dataflow.getStatus() == RealizationStatusEnum.BROKEN) {
- throw new KylinException(ServerErrorCode.MODEL_BROKEN,
- "The model is broken and cannot be exported TDS file");
- }
-
- Set<String> authTables = getAllAuthTables(projectName, groups, currentUserName);
- Set<String> authColumns = getAllAuthColumns(projectName, groups, currentUserName);
-
- Set<String> newAuthColumns = Sets.newHashSet();
- dataflow.getModel().getAllTables().forEach(tableRef -> {
- List<TblColRef> collect = tableRef.getColumns().stream()
- .filter(column -> authColumns.contains(column.getCanonicalName())).collect(Collectors.toList());
- collect.forEach(x -> newAuthColumns.add(x.getAliasDotName()));
- });
-
- checkTableHasColumnPermission(syncContext.getModelElement(), projectName, modelId, newAuthColumns, dimensions,
- measures);
-
- return BISyncTool.dumpHasPermissionToBISyncModel(syncContext, authTables, newAuthColumns, dimensions, measures);
- }
-
- public BISyncModel exportTDSDimensionsAndMeasuresByAdmin(SyncContext syncContext, List<String> dimensions,
- List<String> measures) {
- String projectName = syncContext.getProjectName();
- String modelId = syncContext.getModelId();
- NDataflow dataflow = getManager(NDataflowManager.class, projectName).getDataflow(modelId);
- if (dataflow.getStatus() == RealizationStatusEnum.BROKEN) {
- throw new KylinException(MODEL_BROKEN, "The model is broken and cannot be exported TDS file");
- }
- checkModelExportPermission(projectName, modelId);
- checkModelPermission(projectName, modelId);
- return BISyncTool.dumpBISyncModel(syncContext, dimensions, measures);
- }
-
- public SyncContext getADMINSyncContext(String projectName, String modelId, SyncContext.BI targetBI,
- SyncContext.ModelElement element, String host, int port) {
- NDataflow dataflow = getManager(NDataflowManager.class, projectName).getDataflow(modelId);
- if (dataflow.getStatus() == RealizationStatusEnum.BROKEN) {
- throw new KylinException(MODEL_BROKEN, "The model is broken and cannot be exported TDS file");
- }
- checkModelExportPermission(projectName, modelId);
- checkModelPermission(projectName, modelId);
-
- return getSyncContext(projectName, modelId, targetBI, element, host, port);
- }
-
- public SyncContext getSyncContext(String projectName, String modelId, SyncContext.BI targetBI,
- SyncContext.ModelElement modelElement, String host, int port) {
- SyncContext syncContext = new SyncContext();
- syncContext.setProjectName(projectName);
- syncContext.setModelId(modelId);
- syncContext.setTargetBI(targetBI);
- syncContext.setModelElement(modelElement);
- syncContext.setHost(host);
- syncContext.setPort(port);
- syncContext.setDataflow(getManager(NDataflowManager.class, projectName).getDataflow(modelId));
- syncContext.setKylinConfig(getManager(NProjectManager.class).getProject(projectName).getConfig());
- return syncContext;
- }
-
- public void checkTableHasColumnPermission(SyncContext.ModelElement modelElement, String project, String modeId,
- Set<String> authColumns, List<String> dimensions, List<String> measures) {
- if (AclPermissionUtil.isAdmin()) {
- return;
- }
- aclEvaluate.checkProjectReadPermission(project);
-
- NDataModel model = getManager(NDataModelManager.class, project).getDataModelDesc(modeId);
- long jointCount = model.getJoinTables().stream()
- .filter(table -> authColumns
- .containsAll(Arrays.stream(table.getJoin().getPrimaryKeyColumns())
- .map(TblColRef::getAliasDotName).collect(Collectors.toSet()))
- && authColumns.containsAll(Arrays.stream(table.getJoin().getForeignKeyColumns())
- .map(TblColRef::getAliasDotName).collect(Collectors.toSet())))
- .count();
- long singleTableCount = model.getAllTables().stream().filter(ref -> ref.getColumns().stream()
- .map(TblColRef::getAliasDotName).collect(Collectors.toSet()).stream().anyMatch(authColumns::contains))
- .count();
-
- if (jointCount != model.getJoinTables().size() || singleTableCount == 0
- || (modelElement.equals(SyncContext.ModelElement.CUSTOM_COLS)
- && !checkColumnPermission(model, authColumns, dimensions, measures))) {
- throw new KylinException(ServerErrorCode.INVALID_TABLE_AUTH,
- MsgPicker.getMsg().getTableNoColumnsPermission());
- }
- }
-
- public boolean checkColumnPermission(NDataModel model, Set<String> authColumns, List<String> dimensions,
- List<String> measures) {
-
- if (!checkDimensionPermission(model, authColumns, dimensions)) {
- return false;
- }
- if (CollectionUtils.isEmpty(measures)) {
- return true;
- }
- List<MeasureDef> authMeasures = model.getEffectiveMeasures().values().stream()
- .filter(measure -> measures.contains(measure.getName()))
- .filter(measure -> checkMeasurePermission(authColumns, measure, model)).map(MeasureDef::new)
- .collect(Collectors.toList());
- return authMeasures.size() == measures.size();
-
- }
-
- private boolean checkDimensionPermission(NDataModel model, Set<String> authColumns, List<String> dimensions) {
- if (CollectionUtils.isEmpty(dimensions)) {
- return true;
- }
- List<ComputedColumnDesc> computedColumnDescs = model.getComputedColumnDescs().stream()
- .filter(cc -> dimensions.contains(cc.getFullName())).collect(Collectors.toList());
-
- long authComputedCount = computedColumnDescs.stream()
- .filter(cc -> authColumns.containsAll(convertCCToNormalCols(model, cc))).count();
-
- if (computedColumnDescs.size() != authComputedCount) {
- return false;
- }
-
- List<String> normalColumns = dimensions.stream().filter(column -> !computedColumnDescs.stream()
- .map(ComputedColumnDesc::getFullName).collect(Collectors.toList()).contains(column))
- .collect(Collectors.toList());
- return authColumns.containsAll(normalColumns);
- }
-
- public Set<String> convertCCToNormalCols(NDataModel model, ComputedColumnDesc computedColumnDesc) {
- IndexDependencyParser parser = new IndexDependencyParser(model);
- try {
- Set<TblColRef> tblColRefList = parser.unwrapComputeColumn(computedColumnDesc.getInnerExpression());
- return tblColRefList.stream().map(TblColRef::getAliasDotName).collect(Collectors.toSet());
- } catch (Exception e) {
- log.warn("UnWrap computed column {} in project {} model {} exception",
- computedColumnDesc.getInnerExpression(), model.getProject(), model.getAlias(), e);
- }
- return Collections.emptySet();
- }
-
- private boolean checkMeasurePermission(Set<String> authColumns, NDataModel.Measure measure, NDataModel model) {
- Set<String> measureColumns = measure.getFunction().getParameters().stream()
- .filter(parameterDesc -> parameterDesc.getColRef() != null)
- .map(parameterDesc -> parameterDesc.getColRef().getAliasDotName()).collect(Collectors.toSet());
-
- List<ComputedColumnDesc> computedColumnDescs = model.getComputedColumnDescs().stream()
- .filter(cc -> measureColumns.contains(cc.getFullName())).collect(Collectors.toList());
-
- long authComputedCount = computedColumnDescs.stream()
- .filter(cc -> authColumns.containsAll(convertCCToNormalCols(model, cc))).count();
-
- if (computedColumnDescs.size() != authComputedCount) {
- return false;
- }
-
- List<String> normalColumns = measureColumns.stream().filter(column -> !computedColumnDescs.stream()
- .map(ComputedColumnDesc::getFullName).collect(Collectors.toList()).contains(column))
- .collect(Collectors.toList());
-
- return authColumns.containsAll(normalColumns);
- }
-
- private Set<String> getAllAuthTables(String project, Set<String> groups, String user) {
- Set<String> allAuthTables = Sets.newHashSet();
- AclTCRDigest auths = getManager(AclTCRManager.class, project).getAuthTablesAndColumns(project, user, true);
- allAuthTables.addAll(auths.getTables());
- for (String group : groups) {
- auths = getManager(AclTCRManager.class, project).getAuthTablesAndColumns(project, group, false);
- allAuthTables.addAll(auths.getTables());
- }
- return allAuthTables;
- }
-
- private Set<String> getAllAuthColumns(String project, Set<String> groups, String user) {
- Set<String> allAuthColumns = Sets.newHashSet();
- AclTCRDigest auths = getManager(AclTCRManager.class, project).getAuthTablesAndColumns(project, user, true);
- allAuthColumns.addAll(auths.getColumns());
- for (String group : groups) {
- auths = getManager(AclTCRManager.class, project).getAuthTablesAndColumns(project, group, false);
- allAuthColumns.addAll(auths.getColumns());
- }
- return allAuthColumns;
- }
-
- private void checkModelExportPermission(String project, String modeId) {
- if (AclPermissionUtil.isAdmin()) {
- return;
- }
- aclEvaluate.checkProjectReadPermission(project);
-
- NDataModel model = getManager(NDataModelManager.class, project).getDataModelDesc(modeId);
- Map<String, Set<String>> modelTableColumns = new HashMap<>();
- for (TableRef tableRef : model.getAllTables()) {
- modelTableColumns.putIfAbsent(tableRef.getTableIdentity(), new HashSet<>());
- modelTableColumns.get(tableRef.getTableIdentity())
- .addAll(tableRef.getColumns().stream().map(TblColRef::getName).collect(Collectors.toSet()));
- }
- AclTCRManager aclManager = AclTCRManager.getInstance(KylinConfig.getInstanceFromEnv(), project);
-
- String currentUserName = AclPermissionUtil.getCurrentUsername();
- Set<String> groupsOfExecuteUser = accessService.getGroupsOfExecuteUser(currentUserName);
- MutableAclRecord acl = AclPermissionUtil.getProjectAcl(project);
- Set<String> groupsInProject = AclPermissionUtil.filterGroupsInProject(groupsOfExecuteUser, acl);
- if (AclPermissionUtil.isAdminInProject(project, groupsOfExecuteUser)) {
- return;
- }
- AclTCRDigest digest = aclManager.getAllUnauthorizedTableColumn(currentUserName, groupsInProject,
- modelTableColumns);
- Set<String> authorizedCC = ComputedColumnUtil
- .getAuthorizedCC(Arrays.asList(model),
- ccSourceCols -> aclManager.isColumnsAuthorized(currentUserName, groupsOfExecuteUser,
- ccSourceCols))
- .stream().map(ccDesc -> ccDesc.getTableIdentity() + "." + ccDesc.getColumnName())
- .collect(Collectors.toSet());
- if (digest.getColumns() != null && !digest.getColumns().isEmpty()
- && digest.getColumns().stream().anyMatch(column -> !authorizedCC.contains(column))) {
- throw new KylinException(UNAUTHORIZED_ENTITY,
- "current user does not have full permission on requesting model");
- }
- if (digest.getTables() != null && !digest.getTables().isEmpty()) {
- throw new KylinException(UNAUTHORIZED_ENTITY,
- "current user does not have full permission on requesting model");
- }
- }
-
public List<SegmentPartitionResponse> getSegmentPartitions(String project, String modelId, String segmentId,
List<String> status, String sortBy, boolean reverse) {
aclEvaluate.checkProjectReadPermission(project);
@@ -4300,20 +3999,17 @@ public class ModelService extends BasicService implements TableModelSupporter, P
if (CollectionUtils.isEmpty(partitions)) {
return;
}
+ NDataflowManager dataflowManager = getManager(NDataflowManager.class, project);
if (StringUtils.isNotEmpty(segmentId)) {
// remove partition in target segment
- getManager(NDataflowManager.class, project).removeLayoutPartition(modelId, partitions,
- Sets.newHashSet(segmentId));
+ dataflowManager.removeLayoutPartition(modelId, partitions, Sets.newHashSet(segmentId));
// remove partition in target segment
- getManager(NDataflowManager.class, project).removeSegmentPartition(modelId, partitions,
- Sets.newHashSet(segmentId));
+ dataflowManager.removeSegmentPartition(modelId, partitions, Sets.newHashSet(segmentId));
} else {
// remove partition in all layouts
- getManager(NDataflowManager.class, project).removeLayoutPartition(modelId, Sets.newHashSet(partitions),
- null);
+ dataflowManager.removeLayoutPartition(modelId, Sets.newHashSet(partitions), null);
// remove partition in all segments
- getManager(NDataflowManager.class, project).removeSegmentPartition(modelId, Sets.newHashSet(partitions),
- null);
+ dataflowManager.removeSegmentPartition(modelId, Sets.newHashSet(partitions), null);
// remove partition in model
getManager(NDataModelManager.class, project).updateDataModel(modelId, copyForWrite -> {
val multiPartitionDesc = copyForWrite.getMultiPartitionDesc();
@@ -4331,43 +4027,6 @@ public class ModelService extends BasicService implements TableModelSupporter, P
return model;
}
- public void checkModelPermission(String project, String modelId) {
- String userName = aclEvaluate.getCurrentUserName();
- Set<String> groups = getCurrentUserGroups();
- if (AclPermissionUtil.isAdmin() || AclPermissionUtil.isAdminInProject(project, groups)) {
- return;
- }
- Set<String> allAuthTables = Sets.newHashSet();
- Set<String> allAuthColumns = Sets.newHashSet();
- var auths = getManager(AclTCRManager.class, project).getAuthTablesAndColumns(project, userName, true);
- allAuthTables.addAll(auths.getTables());
- allAuthColumns.addAll(auths.getColumns());
- for (val group : groups) {
- auths = getManager(AclTCRManager.class, project).getAuthTablesAndColumns(project, group, false);
- allAuthTables.addAll(auths.getTables());
- allAuthColumns.addAll(auths.getColumns());
- }
-
- NDataModel model = getModelById(modelId, project);
- Set<String> tablesInModel = Sets.newHashSet();
- model.getJoinTables().forEach(table -> tablesInModel.add(table.getTable()));
- tablesInModel.add(model.getRootFactTableName());
- tablesInModel.forEach(table -> {
- if (!allAuthTables.contains(table)) {
- throw new KylinException(FAILED_UPDATE_MODEL, MsgPicker.getMsg().getModelModifyAbandon(table));
- }
- });
- tablesInModel.stream().filter(allAuthTables::contains).forEach(table -> {
- ColumnDesc[] columnDescs = NTableMetadataManager.getInstance(getConfig(), project).getTableDesc(table)
- .getColumns();
- Arrays.stream(columnDescs).map(column -> table + "." + column.getName()).forEach(column -> {
- if (!allAuthColumns.contains(column)) {
- throw new KylinException(FAILED_UPDATE_MODEL, MsgPicker.getMsg().getModelModifyAbandon(column));
- }
- });
- });
- }
-
public InvalidIndexesResponse detectInvalidIndexes(ModelRequest request) {
String project = request.getProject();
aclEvaluate.checkProjectReadPermission(project);
@@ -4540,46 +4199,6 @@ public class ModelService extends BasicService implements TableModelSupporter, P
request.setProject(projectName);
}
- public Boolean validateExport(String projectName, String modelId) {
- val dataModelDesc = getModelById(modelId, projectName);
- List<MeasureDef> measureDefs = dataModelDesc.getEffectiveMeasures().values().stream().map(MeasureDef::new)
- .collect(Collectors.toList());
- val measures = measureDefs.stream().map(measureDef -> measureDef.getMeasure().getName())
- .collect(Collectors.toSet());
- val columns = dataModelDesc.getAllNamedColumns().stream().filter(column -> !column.isDimension())
- .map(NDataModel.NamedColumn::getName).collect(Collectors.toSet());
-
- // check duplicate name in measures and other model columns
- val duplicateMeasureAndModelColumn = Sets.intersection(measures, columns);
- if (CollectionUtils.isNotEmpty(duplicateMeasureAndModelColumn)) {
- val duplicateName = duplicateMeasureAndModelColumn.stream().findFirst().orElse(null);
- throw new KylinException(MODEL_EXPORT_ERROR, String.format(Locale.ROOT,
- MsgPicker.getMsg().getDuplicateModelColumnAndMeasureName(), duplicateName, duplicateName));
- }
-
- // check duplicate name in dimension columns and measures
- val dimensionCols = dataModelDesc.getEffectiveDimensions().values().stream().map(TblColRef::getColumnDesc)
- .map(ColumnDesc::getName).collect(Collectors.toSet());
- val duplicateDimColAndMeasureNames = Sets.intersection(dimensionCols, measures);
- if (CollectionUtils.isNotEmpty(duplicateDimColAndMeasureNames)) {
- val duplicateName = duplicateDimColAndMeasureNames.stream().findFirst().orElse(null);
- throw new KylinException(MODEL_EXPORT_ERROR, String.format(Locale.ROOT,
- MsgPicker.getMsg().getDuplicateDimensionColAndMeasureName(), duplicateName, duplicateName));
- }
-
- // check duplicate name in dimensions and measures
- val dimNames = dataModelDesc.getAllNamedColumns().stream().filter(NDataModel.NamedColumn::isDimension)
- .map(NDataModel.NamedColumn::getName).collect(Collectors.toSet());
- val duplicateDimMeasureNames = Sets.intersection(dimNames, measures);
- if (CollectionUtils.isNotEmpty(duplicateDimMeasureNames)) {
- val duplicateName = duplicateDimMeasureNames.stream().findFirst().orElse(null);
- throw new KylinException(MODEL_EXPORT_ERROR, String.format(Locale.ROOT,
- MsgPicker.getMsg().getDuplicateDimensionNameAndMeasureName(), duplicateName, duplicateName));
- }
-
- return true;
- }
-
public void checkCCEmpty(ModelRequest modelRequest) {
List<ComputedColumnDesc> ccList = modelRequest.getComputedColumnDescs();
if (CollectionUtils.isEmpty(ccList)) {
diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelTdsService.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelTdsService.java
new file mode 100644
index 0000000000..df277831e5
--- /dev/null
+++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelTdsService.java
@@ -0,0 +1,336 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.rest.service;
+
+import static org.apache.kylin.common.exception.ServerErrorCode.MODEL_BROKEN;
+import static org.apache.kylin.common.exception.ServerErrorCode.UNAUTHORIZED_ENTITY;
+import static org.apache.kylin.common.exception.code.ErrorCodeServer.MODEL_TDS_EXPORT_COLUMN_AND_MEASURE_NAME_CONFLICT;
+import static org.apache.kylin.common.exception.code.ErrorCodeServer.MODEL_TDS_EXPORT_DIM_COL_AND_MEASURE_NAME_CONFLICT;
+
+import java.io.IOException;
+import java.text.SimpleDateFormat;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import javax.servlet.http.HttpServletResponse;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.exception.CommonErrorCode;
+import org.apache.kylin.common.exception.KylinException;
+import org.apache.kylin.common.exception.ServerErrorCode;
+import org.apache.kylin.common.msg.MsgPicker;
+import org.apache.kylin.engine.spark.smarter.IndexDependencyParser;
+import org.apache.kylin.metadata.acl.AclTCRDigest;
+import org.apache.kylin.metadata.acl.AclTCRManager;
+import org.apache.kylin.metadata.cube.model.NDataflow;
+import org.apache.kylin.metadata.cube.model.NDataflowManager;
+import org.apache.kylin.metadata.model.ComputedColumnDesc;
+import org.apache.kylin.metadata.model.NDataModel;
+import org.apache.kylin.metadata.model.NDataModelManager;
+import org.apache.kylin.metadata.model.TableRef;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.model.util.ComputedColumnUtil;
+import org.apache.kylin.metadata.project.NProjectManager;
+import org.apache.kylin.metadata.project.ProjectInstance;
+import org.apache.kylin.metadata.realization.RealizationStatusEnum;
+import org.apache.kylin.rest.security.MutableAclRecord;
+import org.apache.kylin.rest.util.AclPermissionUtil;
+import org.apache.kylin.tool.bisync.BISyncModel;
+import org.apache.kylin.tool.bisync.BISyncTool;
+import org.apache.kylin.tool.bisync.SyncContext;
+import org.apache.kylin.tool.bisync.SyncModelBuilder;
+import org.apache.kylin.tool.bisync.model.ColumnDef;
+import org.apache.kylin.tool.bisync.model.MeasureDef;
+import org.apache.kylin.tool.bisync.model.SyncModel;
+import org.springframework.stereotype.Component;
+
+import com.google.common.collect.Sets;
+
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Component("modelTdsService")
+public class ModelTdsService extends AbstractModelService {
+
+ public void dumpSyncModel(SyncContext syncContext, SyncModel syncModel, HttpServletResponse response)
+ throws IOException {
+ String projectName = syncContext.getProjectName();
+ String modelId = syncContext.getModelId();
+ SyncContext.BI exportAs = syncContext.getTargetBI();
+ BISyncModel biSyncModel = BISyncTool.getBISyncModel(syncContext, syncModel);
+
+ NDataModelManager manager = NDataModelManager.getInstance(KylinConfig.getInstanceFromEnv(), projectName);
+ NDataModel dataModel = manager.getDataModelDesc(modelId);
+ String alias = dataModel.getAlias();
+ String fileName = String.format(Locale.ROOT, "%s_%s_%s", projectName, alias,
+ new SimpleDateFormat("yyyyMMddHHmmss", Locale.getDefault(Locale.Category.FORMAT)).format(new Date()));
+ switch (exportAs) {
+ case TABLEAU_CONNECTOR_TDS:
+ case TABLEAU_ODBC_TDS:
+ response.setContentType("application/xml");
+ response.setHeader("Content-Disposition",
+ String.format(Locale.ROOT, "attachment; filename=\"%s.tds\"", fileName));
+ break;
+ default:
+ throw new KylinException(CommonErrorCode.UNKNOWN_ERROR_CODE, "unrecognized export target");
+ }
+ biSyncModel.dump(response.getOutputStream());
+ response.getOutputStream().flush();
+ response.getOutputStream().close();
+ }
+
+ public boolean preCheckNameConflict(SyncModel syncModel) {
+ ProjectInstance prjInstance = getManager(NProjectManager.class).getProject(syncModel.getProject());
+ boolean skipCheckTds = prjInstance.getConfig().skipCheckTds();
+ Set<String> measureNames = syncModel.getMetrics().stream().filter(measureDef -> !measureDef.isHidden())
+ .map(measureDef -> measureDef.getMeasure().getName()).collect(Collectors.toSet());
+ Map<String, ColumnDef> nameOfColDefMap = syncModel.getColumnDefMap().values().stream()
+ .collect(Collectors.toMap(ColumnDef::getColumnName, Function.identity()));
+ Sets.SetView<String> intersection = Sets.intersection(nameOfColDefMap.keySet(), measureNames);
+ if (skipCheckTds || CollectionUtils.isEmpty(intersection)) {
+ return true;
+ }
+
+ String name = intersection.iterator().next();
+ ColumnDef columnDef = nameOfColDefMap.get(name);
+ if (columnDef.isDimension()) {
+ throw new KylinException(MODEL_TDS_EXPORT_DIM_COL_AND_MEASURE_NAME_CONFLICT, name, name);
+ } else {
+ throw new KylinException(MODEL_TDS_EXPORT_COLUMN_AND_MEASURE_NAME_CONFLICT, name, name);
+ }
+ }
+
+ public SyncModel exportModel(SyncContext syncContext) {
+ checkModelExportPermission(syncContext.getProjectName(), syncContext.getModelId());
+ checkModelPermission(syncContext.getProjectName(), syncContext.getModelId());
+ return new SyncModelBuilder(syncContext).buildSourceSyncModel();
+ }
+
+ public SyncModel exportTDSDimensionsAndMeasuresByNormalUser(SyncContext syncContext, List<String> dimensions,
+ List<String> measures) {
+ String project = syncContext.getProjectName();
+ String modelId = syncContext.getModelId();
+
+ Set<String> authTables = Sets.newHashSet();
+ Set<String> authColumns = Sets.newHashSet();
+ AclTCRManager aclMgr = getManager(AclTCRManager.class, project);
+ AclTCRDigest uDigest = aclMgr.getAuthTablesAndColumns(project, aclEvaluate.getCurrentUserName(), true);
+ authTables.addAll(uDigest.getTables());
+ authColumns.addAll(uDigest.getColumns());
+ getCurrentUserGroups().forEach(group -> {
+ AclTCRDigest gDigest = aclMgr.getAuthTablesAndColumns(project, group, false);
+ authTables.addAll(gDigest.getTables());
+ authColumns.addAll(gDigest.getColumns());
+ });
+
+ Set<String> authorizedCols = Sets.newHashSet();
+ getModelById(modelId, project).getAllTables().forEach(tableRef -> {
+ List<String> colIdentityList = tableRef.getColumns().stream()
+ .filter(colRef -> authColumns.contains(colRef.getCanonicalName())) //
+ .map(TblColRef::getAliasDotName) //
+ .collect(Collectors.toList());
+ authorizedCols.addAll(colIdentityList);
+ });
+
+ checkTableHasColumnPermission(syncContext.getModelElement(), project, modelId, authorizedCols, dimensions,
+ measures);
+ return new SyncModelBuilder(syncContext).buildHasPermissionSourceSyncModel(authTables, authColumns, dimensions,
+ measures);
+ }
+
+ public SyncModel exportTDSDimensionsAndMeasuresByAdmin(SyncContext syncContext, List<String> dimensions,
+ List<String> measures) {
+ String projectName = syncContext.getProjectName();
+ String modelId = syncContext.getModelId();
+ checkModelExportPermission(projectName, modelId);
+ checkModelPermission(projectName, modelId);
+ return new SyncModelBuilder(syncContext).buildSourceSyncModel(dimensions, measures);
+ }
+
+ private void checkBrokenModel(String projectName, String modelId) {
+ NDataflow dataflow = getManager(NDataflowManager.class, projectName).getDataflow(modelId);
+ if (dataflow.getStatus() == RealizationStatusEnum.BROKEN) {
+ throw new KylinException(MODEL_BROKEN, "The model is broken and cannot be exported TDS file");
+ }
+ }
+
+ public SyncContext prepareSyncContext(String projectName, String modelId, SyncContext.BI targetBI,
+ SyncContext.ModelElement modelElement, String host, int port) {
+ checkBrokenModel(projectName, modelId);
+ SyncContext syncContext = new SyncContext();
+ syncContext.setProjectName(projectName);
+ syncContext.setModelId(modelId);
+ syncContext.setTargetBI(targetBI);
+ syncContext.setModelElement(modelElement);
+ syncContext.setHost(host);
+ syncContext.setPort(port);
+ syncContext.setAdmin(AclPermissionUtil.isAdmin());
+ syncContext.setDataflow(getManager(NDataflowManager.class, projectName).getDataflow(modelId));
+ syncContext.setKylinConfig(getManager(NProjectManager.class).getProject(projectName).getConfig());
+ return syncContext;
+ }
+
+ public void checkTableHasColumnPermission(SyncContext.ModelElement modelElement, String project, String modelId,
+ Set<String> authColumns, List<String> dimensions, List<String> measures) {
+ if (AclPermissionUtil.isAdmin()) {
+ return;
+ }
+ aclEvaluate.checkProjectReadPermission(project);
+
+ NDataModel model = getModelById(modelId, project);
+ long jointCount = model.getJoinTables().stream()
+ .filter(table -> authColumns
+ .containsAll(Arrays.stream(table.getJoin().getPrimaryKeyColumns())
+ .map(TblColRef::getAliasDotName).collect(Collectors.toSet()))
+ && authColumns.containsAll(Arrays.stream(table.getJoin().getForeignKeyColumns())
+ .map(TblColRef::getAliasDotName).collect(Collectors.toSet())))
+ .count();
+ long singleTableCount = model.getAllTables().stream().filter(ref -> ref.getColumns().stream()
+ .map(TblColRef::getAliasDotName).collect(Collectors.toSet()).stream().anyMatch(authColumns::contains))
+ .count();
+
+ if (jointCount != model.getJoinTables().size() || singleTableCount == 0
+ || (modelElement.equals(SyncContext.ModelElement.CUSTOM_COLS)
+ && !checkColumnPermission(model, authColumns, dimensions, measures))) {
+ throw new KylinException(ServerErrorCode.INVALID_TABLE_AUTH,
+ MsgPicker.getMsg().getTableNoColumnsPermission());
+ }
+ }
+
+ public boolean checkColumnPermission(NDataModel model, Set<String> authColumns, List<String> dimensions,
+ List<String> measures) {
+
+ if (!checkDimensionPermission(model, authColumns, dimensions)) {
+ return false;
+ }
+ if (CollectionUtils.isEmpty(measures)) {
+ return true;
+ }
+ List<MeasureDef> authMeasures = model.getEffectiveMeasures().values().stream()
+ .filter(measure -> measures.contains(measure.getName()))
+ .filter(measure -> checkMeasurePermission(authColumns, measure, model)).map(MeasureDef::new)
+ .collect(Collectors.toList());
+ return authMeasures.size() == measures.size();
+
+ }
+
+ private boolean checkDimensionPermission(NDataModel model, Set<String> authColumns, List<String> dimensions) {
+ if (CollectionUtils.isEmpty(dimensions)) {
+ return true;
+ }
+ List<ComputedColumnDesc> computedColumnDescs = model.getComputedColumnDescs().stream()
+ .filter(cc -> dimensions.contains(cc.getFullName())).collect(Collectors.toList());
+
+ long authComputedCount = computedColumnDescs.stream()
+ .filter(cc -> authColumns.containsAll(convertCCToNormalCols(model, cc))).count();
+
+ if (computedColumnDescs.size() != authComputedCount) {
+ return false;
+ }
+
+ List<String> normalColumns = dimensions.stream().filter(column -> !computedColumnDescs.stream()
+ .map(ComputedColumnDesc::getFullName).collect(Collectors.toList()).contains(column))
+ .collect(Collectors.toList());
+ return authColumns.containsAll(normalColumns);
+ }
+
+ public Set<String> convertCCToNormalCols(NDataModel model, ComputedColumnDesc computedColumnDesc) {
+ IndexDependencyParser parser = new IndexDependencyParser(model);
+ try {
+ Set<TblColRef> tblColRefList = parser.unwrapComputeColumn(computedColumnDesc.getInnerExpression());
+ return tblColRefList.stream().map(TblColRef::getAliasDotName).collect(Collectors.toSet());
+ } catch (Exception e) {
+ log.warn("UnWrap computed column {} in project {} model {} exception",
+ computedColumnDesc.getInnerExpression(), model.getProject(), model.getAlias(), e);
+ }
+ return Collections.emptySet();
+ }
+
+ private boolean checkMeasurePermission(Set<String> authColumns, NDataModel.Measure measure, NDataModel model) {
+ Set<String> measureColumns = measure.getFunction().getParameters().stream()
+ .filter(parameterDesc -> parameterDesc.getColRef() != null)
+ .map(parameterDesc -> parameterDesc.getColRef().getAliasDotName()).collect(Collectors.toSet());
+
+ List<ComputedColumnDesc> computedColumnDescs = model.getComputedColumnDescs().stream()
+ .filter(cc -> measureColumns.contains(cc.getFullName())).collect(Collectors.toList());
+
+ long authComputedCount = computedColumnDescs.stream()
+ .filter(cc -> authColumns.containsAll(convertCCToNormalCols(model, cc))).count();
+
+ if (computedColumnDescs.size() != authComputedCount) {
+ return false;
+ }
+
+ List<String> normalColumns = measureColumns.stream().filter(column -> !computedColumnDescs.stream()
+ .map(ComputedColumnDesc::getFullName).collect(Collectors.toList()).contains(column))
+ .collect(Collectors.toList());
+
+ return authColumns.containsAll(normalColumns);
+ }
+
+ private void checkModelExportPermission(String project, String modeId) {
+ if (AclPermissionUtil.isAdmin()) {
+ return;
+ }
+ aclEvaluate.checkProjectReadPermission(project);
+
+ NDataModel model = getManager(NDataModelManager.class, project).getDataModelDesc(modeId);
+ Map<String, Set<String>> modelTableColumns = new HashMap<>();
+ for (TableRef tableRef : model.getAllTables()) {
+ modelTableColumns.putIfAbsent(tableRef.getTableIdentity(), new HashSet<>());
+ modelTableColumns.get(tableRef.getTableIdentity())
+ .addAll(tableRef.getColumns().stream().map(TblColRef::getName).collect(Collectors.toSet()));
+ }
+ AclTCRManager aclManager = AclTCRManager.getInstance(KylinConfig.getInstanceFromEnv(), project);
+
+ String currentUserName = AclPermissionUtil.getCurrentUsername();
+ Set<String> groupsOfExecuteUser = accessService.getGroupsOfExecuteUser(currentUserName);
+ MutableAclRecord acl = AclPermissionUtil.getProjectAcl(project);
+ Set<String> groupsInProject = AclPermissionUtil.filterGroupsInProject(groupsOfExecuteUser, acl);
+ if (AclPermissionUtil.isAdminInProject(project, groupsOfExecuteUser)) {
+ return;
+ }
+ AclTCRDigest digest = aclManager.getAllUnauthorizedTableColumn(currentUserName, groupsInProject,
+ modelTableColumns);
+ Set<String> authorizedCC = ComputedColumnUtil
+ .getAuthorizedCC(Collections.singletonList(model),
+ ccSourceCols -> aclManager.isColumnsAuthorized(currentUserName, groupsOfExecuteUser,
+ ccSourceCols))
+ .stream().map(ComputedColumnDesc::getIdentName).collect(Collectors.toSet());
+ if (digest.getColumns() != null && !digest.getColumns().isEmpty()
+ && digest.getColumns().stream().anyMatch(column -> !authorizedCC.contains(column))) {
+ throw new KylinException(UNAUTHORIZED_ENTITY,
+ "current user does not have full permission on requesting model");
+ }
+ if (digest.getTables() != null && !digest.getTables().isEmpty()) {
+ throw new KylinException(UNAUTHORIZED_ENTITY,
+ "current user does not have full permission on requesting model");
+ }
+ }
+}
diff --git a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
index 6ed591f405..adad3571f6 100644
--- a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
+++ b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
@@ -51,7 +51,6 @@ import java.io.DataOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
-import java.io.InputStreamReader;
import java.lang.reflect.Field;
import java.nio.charset.Charset;
import java.nio.file.Files;
@@ -123,6 +122,8 @@ import org.apache.kylin.metadata.cube.model.PartitionStatusEnum;
import org.apache.kylin.metadata.cube.model.PartitionStatusEnumToDisplay;
import org.apache.kylin.metadata.cube.model.RuleBasedIndex;
import org.apache.kylin.metadata.cube.optimization.FrequencyMap;
+import org.apache.kylin.metadata.favorite.FavoriteRule;
+import org.apache.kylin.metadata.favorite.FavoriteRuleManager;
import org.apache.kylin.metadata.model.AutoMergeTimeEnum;
import org.apache.kylin.metadata.model.BadModelException;
import org.apache.kylin.metadata.model.BadModelException.CauseType;
@@ -154,6 +155,9 @@ import org.apache.kylin.metadata.model.util.scd2.SimplifiedJoinTableDesc;
import org.apache.kylin.metadata.project.EnhancedUnitOfWork;
import org.apache.kylin.metadata.query.QueryTimesResponse;
import org.apache.kylin.metadata.realization.RealizationStatusEnum;
+import org.apache.kylin.metadata.recommendation.candidate.JdbcRawRecStore;
+import org.apache.kylin.metadata.user.ManagedUser;
+import org.apache.kylin.query.util.KapQueryUtil;
import org.apache.kylin.rest.config.initialize.ModelBrokenListener;
import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.rest.constant.ModelStatusToDisplayEnum;
@@ -187,8 +191,6 @@ import org.apache.kylin.rest.util.AclUtil;
import org.apache.kylin.rest.util.SCD2SimplificationConvertUtil;
import org.apache.kylin.streaming.jobs.StreamingJobListener;
import org.apache.kylin.streaming.manager.StreamingJobManager;
-import org.apache.kylin.tool.bisync.SyncContext;
-import org.apache.kylin.tool.bisync.tableau.TableauDatasourceModel;
import org.apache.kylin.util.BrokenEntityProxy;
import org.apache.kylin.util.PasswordEncodeFactory;
import org.hamcrest.BaseMatcher;
@@ -209,19 +211,12 @@ import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.security.crypto.password.PasswordEncoder;
import org.springframework.test.util.ReflectionTestUtils;
-import com.google.common.base.Charsets;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
-import com.google.common.io.CharStreams;
import com.google.common.primitives.Longs;
import io.kyligence.kap.clickhouse.MockSecondStorage;
-import org.apache.kylin.metadata.favorite.FavoriteRule;
-import org.apache.kylin.metadata.favorite.FavoriteRuleManager;
-import org.apache.kylin.metadata.recommendation.candidate.JdbcRawRecStore;
-import org.apache.kylin.metadata.user.ManagedUser;
-import org.apache.kylin.query.util.KapQueryUtil;
import io.kyligence.kap.secondstorage.SecondStorageNodeHelper;
import io.kyligence.kap.secondstorage.SecondStorageUtil;
import io.kyligence.kap.secondstorage.config.Node;
@@ -433,7 +428,6 @@ public class ModelServiceTest extends SourceTestCase {
Assert.assertEquals("nmodel_basic_inner", models.get(0).getAlias());
}
-// @Ignore("TODO: re-run to check.")
@Test
public void testGetFusionModels() {
List<NDataModelResponse> models = modelService.getModels("", "streaming_test", false, "", null, "usage", true);
@@ -528,8 +522,7 @@ public class ModelServiceTest extends SourceTestCase {
Assert.assertEquals(1, models.size());
NDataModelResponse model = models.get(0);
Assert.assertTrue(model.getSimpleTables().stream().map(SimplifiedTableResponse::getColumns)
- .flatMap(List::stream)
- .anyMatch(SimplifiedColumnResponse::isComputedColumn));
+ .flatMap(List::stream).anyMatch(SimplifiedColumnResponse::isComputedColumn));
}
@Test
@@ -570,8 +563,7 @@ public class ModelServiceTest extends SourceTestCase {
NIndexPlanManager indexPlanManager = NIndexPlanManager.getInstance(getTestConfig(), getProject());
val indexPlan = indexPlanManager.getIndexPlan(modelId);
indexPlanManager.updateIndexPlan(modelId, copyForWrite -> {
- copyForWrite.markIndexesToBeDeleted(modelId,
- new HashSet<>(indexPlan.getAllLayouts()));
+ copyForWrite.markIndexesToBeDeleted(modelId, new HashSet<>(indexPlan.getAllLayouts()));
copyForWrite.getIndexes().clear();
});
NDataflowManager dataflowManager = NDataflowManager.getInstance(KylinConfig.getInstanceFromEnv(), "default");
@@ -1392,7 +1384,6 @@ public class ModelServiceTest extends SourceTestCase {
&& models.get(0).getStatus() == ModelStatusToDisplayEnum.OFFLINE);
}
-// @Ignore("TODO: re-run to check.")
@Test
public void testUpdateFusionDataModelStatus() {
val project = "streaming_test";
@@ -1417,7 +1408,6 @@ public class ModelServiceTest extends SourceTestCase {
Assert.assertEquals(ModelStatusToDisplayEnum.OFFLINE, models.get(0).getStatus());
}
-// @Ignore("TODO: re-run to check.")
@Test
public void testUpdateFusionDataModelStatus1() {
val project = "streaming_test";
@@ -1447,7 +1437,6 @@ public class ModelServiceTest extends SourceTestCase {
}
-// @Ignore("TODO: re-run to check.")
@Test
public void testUpdateFusionDataModelStatus2() {
val project = "streaming_test";
@@ -4770,7 +4759,6 @@ public class ModelServiceTest extends SourceTestCase {
Assert.assertTrue(model.isBroken());
}
-// @Ignore("TODO: re-run to check.")
@Test
public void testGetBrokenFusionModel() {
String project = "streaming_test";
@@ -5093,394 +5081,6 @@ public class ModelServiceTest extends SourceTestCase {
}
}
- @Test
- public void testExportTDSByAdmin() throws Exception {
- val project = "default";
- val modelId = "cb596712-3a09-46f8-aea1-988b43fe9b6c";
- prepareBasic(project);
- List<String> dimensions = Lists.newArrayList();
- dimensions.add("DEFAULT.TEST_MEASURE.FLAG");
- dimensions.add("DEFAULT.TEST_MEASURE.PRICE1");
- dimensions.add("DEFAULT.TEST_MEASURE.ID1");
- List<String> measurs = Lists.newArrayList();
- measurs.add("COUNT_STAR");
- measurs.add("SUM_1");
- SyncContext syncContext = modelService.getADMINSyncContext(project, modelId,
- SyncContext.BI.TABLEAU_CONNECTOR_TDS, SyncContext.ModelElement.CUSTOM_COLS, "localhost", 8080);
- TableauDatasourceModel datasource1 = (TableauDatasourceModel) modelService
- .exportTDSDimensionsAndMeasuresByAdmin(syncContext, dimensions, measurs);
- ByteArrayOutputStream outStream4 = new ByteArrayOutputStream();
- datasource1.dump(outStream4);
- Assert.assertEquals(getExpectedTds("/bisync_tableau/nmodel_full_measure_test.connector_admin.tds"),
- outStream4.toString(Charset.defaultCharset().name()));
- }
-
- @Test
- public void testExportTDSByUser() throws Exception {
- val project = "default";
- val modelId = "cb596712-3a09-46f8-aea1-988b43fe9b6c";
- prepareBasic(project);
- List<String> dimensions = Lists.newArrayList();
- dimensions.add("TEST_MEASURE.ID1");
- dimensions.add("TEST_MEASURE.ID2");
- dimensions.add("TEST_MEASURE.ID3");
- dimensions.add("TEST_MEASURE1.ID1");
- dimensions.add("TEST_MEASURE1.NAME1");
- dimensions.add("TEST_MEASURE1.NAME2");
- dimensions.add("TEST_MEASURE1.NAME3");
- List<String> measurs = Lists.newArrayList();
- measurs.add("COUNT_STAR");
- measurs.add("SUM_1");
- SecurityContextHolder.getContext()
- .setAuthentication(new TestingAuthenticationToken("u1", "ANALYST", Constant.ROLE_ANALYST));
- SyncContext syncContext = modelService.getSyncContext(project, modelId, SyncContext.BI.TABLEAU_CONNECTOR_TDS,
- SyncContext.ModelElement.CUSTOM_COLS, "localhost", 8080);
- TableauDatasourceModel datasource1 = (TableauDatasourceModel) modelService
- .exportTDSDimensionsAndMeasuresByNormalUser(syncContext, dimensions, measurs);
- ByteArrayOutputStream outStream4 = new ByteArrayOutputStream();
- datasource1.dump(outStream4);
- Assert.assertEquals(getExpectedTds("/bisync_tableau/nmodel_full_measure_test.connector_user.tds"),
- outStream4.toString(Charset.defaultCharset().name()));
- }
-
- @Test
- public void testExportTDSByUserAndElement() throws Exception {
- val project = "default";
- val modelId = "cb596712-3a09-46f8-aea1-988b43fe9b6c";
- prepareBasic(project);
- try {
- SecurityContextHolder.getContext()
- .setAuthentication(new TestingAuthenticationToken("u1", "ANALYST", Constant.ROLE_ANALYST));
- SyncContext syncContext = modelService.getSyncContext(project, modelId,
- SyncContext.BI.TABLEAU_CONNECTOR_TDS, SyncContext.ModelElement.AGG_INDEX_COL, "localhost", 8080);
- TableauDatasourceModel datasource1 = (TableauDatasourceModel) modelService
- .exportTDSDimensionsAndMeasuresByNormalUser(syncContext, null, null);
- ByteArrayOutputStream outStream4 = new ByteArrayOutputStream();
- datasource1.dump(outStream4);
- Assert.assertEquals(
- getExpectedTds("/bisync_tableau/nmodel_full_measure_test.connector_user_agg_index_col.tds"),
- outStream4.toString(Charset.defaultCharset().name()));
-
- TableauDatasourceModel datasource = (TableauDatasourceModel) modelService
- .exportTDSDimensionsAndMeasuresByNormalUser(syncContext, new ArrayList<>(), new ArrayList<>());
- } finally {
- SecurityContextHolder.getContext()
- .setAuthentication(new TestingAuthenticationToken("ADMIN", "ADMIN", Constant.ROLE_ADMIN));
- }
- }
-
- @Test
- public void testCheckModelExportPermissionException() {
- val project = "default";
- val modelId = "cb596712-3a09-46f8-aea1-988b43fe9b6c";
- prepareBasic(project);
- modelService.getADMINSyncContext(project, modelId, SyncContext.BI.TABLEAU_CONNECTOR_TDS,
- SyncContext.ModelElement.AGG_INDEX_COL, "localhost", 8080);
- try {
- Mockito.when(accessService.getGroupsOfExecuteUser(Mockito.any(String.class)))
- .thenReturn(Sets.newHashSet("ROLE_ANALYST"));
- SecurityContextHolder.getContext()
- .setAuthentication(new TestingAuthenticationToken("u1", "ANALYST", Constant.ROLE_ANALYST));
- thrown.expect(KylinException.class);
- thrown.expectMessage("current user does not have full permission on requesting model");
- modelService.getADMINSyncContext(project, modelId, SyncContext.BI.TABLEAU_CONNECTOR_TDS,
- SyncContext.ModelElement.AGG_INDEX_COL, "localhost", 8080);
- } finally {
- SecurityContextHolder.getContext()
- .setAuthentication(new TestingAuthenticationToken("ADMIN", "ADMIN", Constant.ROLE_ADMIN));
- }
- }
-
- @Test
- public void testCheckModelExportPermission() {
- val project = "default";
- val modelId = "cb596712-3a09-46f8-aea1-988b43fe9b6c";
- prepareBasic(project);
- modelService.getADMINSyncContext(project, modelId, SyncContext.BI.TABLEAU_CONNECTOR_TDS,
- SyncContext.ModelElement.AGG_INDEX_COL, "localhost", 8080);
- modelService.getADMINSyncContext(project, modelId, SyncContext.BI.TABLEAU_CONNECTOR_TDS,
- SyncContext.ModelElement.AGG_INDEX_COL, "localhost", 8080);
- }
-
- @Test
- public void testCheckModelExportPermissionWithCC() {
- val project = "cc_test";
- val modelId = "0d146f1a-bdd3-4548-87ac-21c2c6f9a0da";
- AclTCRManager manager = AclTCRManager.getInstance(getTestConfig(), project);
- {
- AclTCR u1a1 = new AclTCR();
- manager.updateAclTCR(u1a1, "u1", true);
- SecurityContextHolder.getContext()
- .setAuthentication(new TestingAuthenticationToken("u1", "ANALYST", Constant.ROLE_ANALYST));
- Mockito.when(accessService.getGroupsOfExecuteUser(Mockito.any(String.class)))
- .thenReturn(Sets.newHashSet("ROLE_ANALYST"));
- modelService.getADMINSyncContext(project, modelId, SyncContext.BI.TABLEAU_CONNECTOR_TDS,
- SyncContext.ModelElement.AGG_INDEX_COL, "localhost", 8080);
- }
- {
- try {
- AclTCR u1a1 = new AclTCR();
- AclTCR.Table u1t1 = new AclTCR.Table();
- AclTCR.ColumnRow u1cr1 = new AclTCR.ColumnRow();
- AclTCR.Column u1c1 = new AclTCR.Column();
- u1c1.add("ORDER_ID");
- u1cr1.setColumn(u1c1);
- u1t1.put("SSB.LINEORDER", u1cr1);
- u1a1.setTable(u1t1);
- manager.updateAclTCR(u1a1, "u1", true);
- thrown.expect(KylinException.class);
- thrown.expectMessage("current user does not have full permission on requesting model");
- modelService.getADMINSyncContext(project, modelId, SyncContext.BI.TABLEAU_CONNECTOR_TDS,
- SyncContext.ModelElement.AGG_INDEX_COL, "localhost", 8080);
- } finally {
- SecurityContextHolder.getContext()
- .setAuthentication(new TestingAuthenticationToken("ADMIN", "ADMIN", Constant.ROLE_ADMIN));
- }
- }
-
- }
-
- @Test
- public void testExportTDSByBroken() {
- val project = "test_broken_project";
- val modelId = "4b93b131-824e-6966-c4dd-5a4268d27095";
- List<String> dimensions = Lists.newArrayList();
- List<String> measurs = Lists.newArrayList();
- SyncContext syncContext = modelService.getSyncContext(project, modelId, SyncContext.BI.TABLEAU_CONNECTOR_TDS,
- SyncContext.ModelElement.CUSTOM_COLS, "localhost", 8080);
- Assert.assertThrows(KylinException.class,
- () -> modelService.exportTDSDimensionsAndMeasuresByNormalUser(syncContext, dimensions, measurs));
-
- Assert.assertThrows(KylinException.class,
- () -> modelService.exportTDSDimensionsAndMeasuresByAdmin(syncContext, dimensions, measurs));
- }
-
- @Test
- public void testExportTDSMeasurePermission() {
- val project = "default";
- val modelId = "82fa7671-a935-45f5-8779-85703601f49a";
- prepareBasicByMeasure(project);
- List<String> dimensions = Lists.newArrayList();
- //"ORDER_ID", "PRICE", "CAL_DT", "PRICE", "ITEM_COUNT", "LEAF_CATEG_ID"
- dimensions.add("TEST_KYLIN_FACT.ORDER_ID");
- dimensions.add("TEST_KYLIN_FACT.PRICE");
- dimensions.add("TEST_KYLIN_FACT.CAL_DT");
- dimensions.add("TEST_KYLIN_FACT.PRICE");
- dimensions.add("TEST_KYLIN_FACT.ITEM_COUNT");
- dimensions.add("TEST_KYLIN_FACT.LEAF_CATEG_ID");
- //"ORDER_ID", "TEST_TIME_ENC", "TEST_DATE_ENC"
- dimensions.add("TEST_ORDER.ORDER_ID");
- dimensions.add("TEST_ORDER.TEST_TIME_ENC");
- dimensions.add("TEST_ORDER.TEST_DATE_ENC");
- //"ORDER_ID", "PRICE", "CAL_DT", "TRANS_ID"
- dimensions.add("TEST_MEASURE.ORDER_ID");
- dimensions.add("TEST_MEASURE.PRICE");
- dimensions.add("TEST_MEASURE.CAL_DT");
- dimensions.add("TEST_MEASURE.TRANS_ID");
-
- List<String> measures = Lists.newArrayList();
- measures.add("TRANS_CNT");
- measures.add("GMV_SUM");
- measures.add("GMV_MIN");
- measures.add("GMV_MAX");
- measures.add("ITEM_COUNT_SUM");
- measures.add("ITEM_COUNT_MAX");
- measures.add("ITEM_COUNT_MIN");
- measures.add("SELLER_HLL");
- measures.add("COUNT_DISTINCT");
- measures.add("TOP_SELLER");
- measures.add("TEST_COUNT_DISTINCT_BITMAP");
- measures.add("GVM_PERCENTILE");
- SecurityContextHolder.getContext()
- .setAuthentication(new TestingAuthenticationToken("u1", "ANALYST", Constant.ROLE_ANALYST));
- SyncContext syncContext = modelService.getSyncContext(project, modelId, SyncContext.BI.TABLEAU_CONNECTOR_TDS,
- SyncContext.ModelElement.CUSTOM_COLS, "localhost", 8080);
- Assert.assertThrows(KylinException.class,
- () -> modelService.exportTDSDimensionsAndMeasuresByNormalUser(syncContext, dimensions, measures));
- }
-
- private void prepareBasicByMeasure(String project) {
- AclTCRManager manager = AclTCRManager.getInstance(getTestConfig(), project);
-
- AclTCR u1a1 = new AclTCR();
- AclTCR.Table u1t1 = new AclTCR.Table();
- AclTCR.ColumnRow u1cr1 = new AclTCR.ColumnRow();
- AclTCR.Column u1c1 = new AclTCR.Column();
- u1c1.addAll(Arrays.asList("ORDER_ID", "PRICE", "CAL_DT", "PRICE", "ITEM_COUNT", "LEAF_CATEG_ID"));
- u1cr1.setColumn(u1c1);
-
- AclTCR.ColumnRow u1cr2 = new AclTCR.ColumnRow();
- AclTCR.Column u1c2 = new AclTCR.Column();
- u1c2.addAll(Arrays.asList("ORDER_ID", "TEST_TIME_ENC", "TEST_DATE_ENC"));
- u1cr2.setColumn(u1c2);
- u1t1.put("DEFAULT.TEST_KYLIN_FACT", u1cr1);
- u1t1.put("DEFAULT.TEST_ORDER", u1cr2);
- u1a1.setTable(u1t1);
- manager.updateAclTCR(u1a1, "u1", true);
- }
-
- @Test
- public void testExportModel() throws Exception {
- val project = "default";
- val modelId = "cb596712-3a09-46f8-aea1-988b43fe9b6c";
- prepareBasic(project);
- TableauDatasourceModel datasource1 = (TableauDatasourceModel) modelService.exportModel(project, modelId,
- SyncContext.BI.TABLEAU_CONNECTOR_TDS, SyncContext.ModelElement.AGG_INDEX_AND_TABLE_INDEX_COL,
- "localhost", 8080);
- ByteArrayOutputStream outStream4 = new ByteArrayOutputStream();
- datasource1.dump(outStream4);
- Assert.assertEquals(getExpectedTds("/bisync_tableau/nmodel_full_measure_test.connector.tds"),
- outStream4.toString(Charset.defaultCharset().name()));
- }
-
- private String getExpectedTds(String path) throws IOException {
- return CharStreams.toString(new InputStreamReader(getClass().getResourceAsStream(path), Charsets.UTF_8));
- }
-
- private void prepareBasic(String project) {
- AclTCRManager manager = AclTCRManager.getInstance(getTestConfig(), project);
-
- AclTCR u1a1 = new AclTCR();
- AclTCR.Table u1t1 = new AclTCR.Table();
- AclTCR.ColumnRow u1cr1 = new AclTCR.ColumnRow();
- AclTCR.Column u1c1 = new AclTCR.Column();
- u1c1.addAll(Arrays.asList("ID1", "ID2", "ID3"));
- u1cr1.setColumn(u1c1);
-
- AclTCR.ColumnRow u1cr2 = new AclTCR.ColumnRow();
- AclTCR.Column u1c2 = new AclTCR.Column();
- u1c2.addAll(Arrays.asList("ID1", "NAME1", "NAME2", "NAME3"));
- u1cr2.setColumn(u1c2);
- u1t1.put("DEFAULT.TEST_MEASURE", u1cr1);
- u1t1.put("DEFAULT.TEST_MEASURE1", u1cr2);
- u1a1.setTable(u1t1);
- manager.updateAclTCR(u1a1, "u1", true);
-
- AclTCR g1a1 = new AclTCR();
- AclTCR.Table g1t1 = new AclTCR.Table();
- AclTCR.ColumnRow g1cr1 = new AclTCR.ColumnRow();
- AclTCR.Column g1c1 = new AclTCR.Column();
- g1c1.addAll(Arrays.asList("ID1", "ID2", "ID3", "ID4"));
- g1cr1.setColumn(g1c1);
- g1t1.put("DEFAULT.TEST_MEASURE", g1cr1);
- g1a1.setTable(g1t1);
- manager.updateAclTCR(g1a1, "g1", false);
- }
-
- @Test
- public void testCheckTablePermission() {
- val project = "default";
- val modelId = "cb596712-3a09-46f8-aea1-988b43fe9b6c";
- thrown.expect(KylinException.class);
- thrown.expectMessage(MsgPicker.getMsg().getTableNoColumnsPermission());
-
- AclTCRManager manager = AclTCRManager.getInstance(getTestConfig(), project);
- Set<String> columns = new HashSet<>();
- columns.add("DEFAULT.TEST_MEASURE1.NAME1");
- columns.add("DEFAULT.TEST_MEASURE1.NAME2");
- columns.add("DEFAULT.TEST_MEASURE1.NAME3");
-
- AclTCR u1a1 = new AclTCR();
- AclTCR.Table u1t1 = new AclTCR.Table();
- AclTCR.ColumnRow u1cr1 = new AclTCR.ColumnRow();
- AclTCR.Column u1c1 = new AclTCR.Column();
- u1cr1.setColumn(u1c1);
-
- AclTCR.ColumnRow u1cr2 = new AclTCR.ColumnRow();
- AclTCR.Column u1c2 = new AclTCR.Column();
- u1c2.addAll(Arrays.asList("NAME1", "NAME2", "NAME3"));
- u1cr2.setColumn(u1c2);
- u1t1.put("DEFAULT.TEST_MEASURE", u1cr1);
- u1t1.put("DEFAULT.TEST_MEASURE1", u1cr2);
- u1a1.setTable(u1t1);
- manager.updateAclTCR(u1a1, "u1", true);
- SecurityContextHolder.getContext()
- .setAuthentication(new TestingAuthenticationToken("u1", "ANALYST", Constant.ROLE_ANALYST));
- List<String> dimensions = Lists.newArrayList();
- dimensions.add("TEST_MEASURE.FLAG");
- dimensions.add("TEST_MEASURE.PRICE1");
- dimensions.add("TEST_MEASURE.ID1");
- List<String> measurs = Lists.newArrayList();
- measurs.add("COUNT_STAR");
- measurs.add("SUM_1");
- modelService.checkTableHasColumnPermission(SyncContext.ModelElement.CUSTOM_COLS, project, modelId, columns,
- dimensions, measurs);
-
- dimensions.add("TEST_MEASURE.ID4");
- Assert.assertThrows(KylinException.class,
- () -> modelService.checkTableHasColumnPermission(SyncContext.ModelElement.CUSTOM_COLS, project, modelId,
- columns, dimensions, measurs));
- }
-
- @Test
- public void testExportTDSCheckColumnPermission() {
- val project = "default";
- val modelId = "89af4ee2-2cdb-4b07-b39e-4c29856309aa";
-
- NDataModelManager modelManager = NDataModelManager.getInstance(KylinConfig.getInstanceFromEnv(), project);
- NDataModel dataModel = modelManager.getDataModelDesc(modelId);
-
- Set<String> authColumns = Sets.newHashSet();
- List<String> dimensions = Lists.newArrayList();
- List<String> measurs = Lists.newArrayList();
-
- Assert.assertTrue(modelService.checkColumnPermission(dataModel, authColumns, null, measurs));
- Assert.assertTrue(modelService.checkColumnPermission(dataModel, authColumns, null, null));
- Assert.assertTrue(modelService.checkColumnPermission(dataModel, authColumns, dimensions, null));
- Assert.assertTrue(modelService.checkColumnPermission(dataModel, authColumns, dimensions, measurs));
-
- authColumns.add("DEFAULT.TEST_KYLIN_FACT.PRICE");
- authColumns.add("DEFAULT.TEST_KYLIN_FACT.ITEM_COUNT");
- authColumns.add("EDW.TEST_CAL_DT.CAL_DT");
- authColumns.add("DEFAULT.TEST_ACCOUNT.ACCOUNT_ID");
-
- Set<String> newAuthColumns = Sets.newHashSet();
- dataModel.getAllTables().forEach(tableRef -> {
- List<TblColRef> collect = tableRef.getColumns().stream()
- .filter(column -> authColumns.contains(column.getCanonicalName())).collect(Collectors.toList());
- collect.forEach(x -> newAuthColumns.add(x.getAliasDotName()));
- });
-
- dimensions.add("TEST_KYLIN_FACT.DEAL_AMOUNT");
- dimensions.add("TEST_KYLIN_FACT.TRANS_ID");
-
- Assert.assertFalse(modelService.checkColumnPermission(dataModel, newAuthColumns, dimensions, measurs));
-
- newAuthColumns.add("TEST_KYLIN_FACT.TRANS_ID");
-
- measurs.add("SUM_NEST4");
- measurs.add("COUNT_CAL_DT");
- Assert.assertTrue(modelService.checkColumnPermission(dataModel, newAuthColumns, dimensions, measurs));
-
- Assert.assertTrue(modelService.checkColumnPermission(dataModel, newAuthColumns, dimensions, measurs));
-
- }
-
- @Test
- public void testConvertCCToNormalCols() {
- val project = "default";
- val modelId = "89af4ee2-2cdb-4b07-b39e-4c29856309aa";
- NDataModelManager modelManager = NDataModelManager.getInstance(KylinConfig.getInstanceFromEnv(), project);
- NDataModel dataModel = modelManager.getDataModelDesc(modelId);
- NDataModel.Measure measure = dataModel.getEffectiveMeasures().values().stream()
- .filter(x -> x.getName().equals("SUM_NEST4")).findFirst().get();
- Set<String> measureColumns = measure.getFunction().getParameters().stream()
- .filter(parameterDesc -> parameterDesc.getColRef() != null)
- .map(parameterDesc -> parameterDesc.getColRef().getCanonicalName()).collect(Collectors.toSet());
- ComputedColumnDesc sumNest4 = dataModel.getComputedColumnDescs().stream()
- .filter(x -> measureColumns.contains(x.getIdentName())).findFirst().get();
- Set<String> strings = modelService.convertCCToNormalCols(dataModel, sumNest4);
- Assert.assertEquals("TEST_KYLIN_FACT.PRICE, TEST_KYLIN_FACT.ITEM_COUNT", String.join(", ", strings));
-
- sumNest4.setInnerExpression("1 + 2");
- Set<String> set = modelService.convertCCToNormalCols(dataModel, sumNest4);
- Assert.assertEquals(Collections.emptySet(), set);
-
- HashSet<Object> authColumns = Sets.newHashSet();
- authColumns.add("DEFAULT.TEST_KYLIN_FACT.PRICE");
- Assert.assertTrue(authColumns.containsAll(set));
- }
-
@Test
public void testBuildExceptionMessage() {
NDataModelManager modelManager = NDataModelManager.getInstance(KylinConfig.getInstanceFromEnv(), "default");
@@ -5517,8 +5117,8 @@ public class ModelServiceTest extends SourceTestCase {
public void testBuildDuplicateCCException() {
Set<String> set = Sets.newHashSet("test");
Assert.assertThrows("The computed column name \"test\" has been used in the current model. Please rename it.\n",
- KylinException.class, () -> ReflectionTestUtils.invokeMethod(modelService, "buildDuplicateCCException",
- set));
+ KylinException.class,
+ () -> ReflectionTestUtils.invokeMethod(modelService, "buildDuplicateCCException", set));
}
@Test
@@ -5612,63 +5212,6 @@ public class ModelServiceTest extends SourceTestCase {
}
}
- @Test
- public void testExportTDSWithDupMeasureDimColumnNames() throws IOException {
- String projectName = "default";
- String modelId = "199ee99e-8419-3e7a-7cad-97059999ec0a";
- val modelRequest = JsonUtil.readValue(
- new File("src/test/resources/ut_meta/dup_name_test/model_desc/model_dup_mea_dimcol.json"),
- ModelRequest.class);
- modelRequest.setProject(projectName);
- List<NamedColumn> simplifiedDims = modelRequest.getAllNamedColumns().stream().filter(NamedColumn::isDimension)
- .collect(Collectors.toList());
- modelRequest.setSimplifiedDimensions(simplifiedDims);
- modelService.createModel(projectName, modelRequest);
- NDataModelManager projectInstance = NDataModelManager.getInstance(getTestConfig(), projectName);
- Assert.assertNotNull(projectInstance.getDataModelDescByAlias("model_dup_mea_dimcol"));
- Assert.assertThrows(
- "There are duplicated names among dimension column LO_LINENUMBER and measure name LO_LINENUMBER. Cannot export a valid TDS file. Please correct the duplicated names and try again.",
- KylinException.class, () -> modelService.validateExport(projectName, modelId));
- }
-
- @Test
- public void testExportTDSWithDupMeasureDimensionNames() throws IOException {
- String projectName = "default";
- String modelId = "6f8cd656-9beb-47f6-87f5-89a8c548d17c";
- val modelRequest = JsonUtil.readValue(
- new File("src/test/resources/ut_meta/dup_name_test/model_desc/model_dup_mea_dim.json"),
- ModelRequest.class);
- modelRequest.setProject(projectName);
- List<NamedColumn> simplifiedDims = modelRequest.getAllNamedColumns().stream().filter(NamedColumn::isDimension)
- .collect(Collectors.toList());
- modelRequest.setSimplifiedDimensions(simplifiedDims);
- modelService.createModel(projectName, modelRequest);
- NDataModelManager projectInstance = NDataModelManager.getInstance(getTestConfig(), projectName);
- Assert.assertNotNull(projectInstance.getDataModelDescByAlias("model_dup_mea_dim"));
- Assert.assertThrows(
- "There are duplicated names among dimension name LO_TEST and measure name LO_TEST. Cannot export a valid TDS file. Please correct the duplicated names and try again.",
- KylinException.class, () -> modelService.validateExport(projectName, modelId));
- }
-
- @Test
- public void testExportTDSWithDupMeasureColumnNames() throws IOException {
- String projectName = "default";
- String modelId = "2ed3bf12-ad40-e8a0-73da-8dc3b4c798bb";
- val modelRequest = JsonUtil.readValue(
- new File("src/test/resources/ut_meta/dup_name_test/model_desc/model_dup_mea_col.json"),
- ModelRequest.class);
- modelRequest.setProject(projectName);
- List<NamedColumn> simplifiedDims = modelRequest.getAllNamedColumns().stream().filter(NamedColumn::isDimension)
- .collect(Collectors.toList());
- modelRequest.setSimplifiedDimensions(simplifiedDims);
- modelService.createModel(projectName, modelRequest);
- NDataModelManager projectInstance = NDataModelManager.getInstance(getTestConfig(), projectName);
- Assert.assertNotNull(projectInstance.getDataModelDescByAlias("model_dup_mea_col"));
- Assert.assertThrows(
- "There are duplicated names among model column LO_LINENUMBER and measure name LO_LINENUMBER. Cannot export a valid TDS file. Please correct the duplicated names and try again.",
- KylinException.class, () -> modelService.validateExport(projectName, modelId));
- }
-
@Test
public void testCheckComputedColumnExprWithSqlKeyword() throws IOException {
String projectName = "keyword";
@@ -5836,8 +5379,8 @@ public class ModelServiceTest extends SourceTestCase {
Assert.assertEquals(1, details.size());
Assert.assertEquals(COMPUTED_COLUMN_CONFLICT_ADJUST_INFO.getErrorCode().getCode(),
details.get(0).getDetailCode());
- Assert.assertEquals(COMPUTED_COLUMN_CONFLICT_ADJUST_INFO.getMsg("CC_1", "CUSTOMER.C_NAME +'USA'",
- "CC_CNAME", "CUSTOMER.C_NAME +'USA'", "CC_CNAME"), details.get(0).getDetailMsg());
+ Assert.assertEquals(COMPUTED_COLUMN_CONFLICT_ADJUST_INFO.getMsg("CC_1", "CUSTOMER.C_NAME +'USA'", "CC_CNAME",
+ "CUSTOMER.C_NAME +'USA'", "CC_CNAME"), details.get(0).getDetailMsg());
}
private void testNoCCConflict(ModelRequest originRequest) {
diff --git a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelTdsServiceTest.java b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelTdsServiceTest.java
new file mode 100644
index 0000000000..f4194a0be4
--- /dev/null
+++ b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelTdsServiceTest.java
@@ -0,0 +1,607 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.rest.service;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.nio.charset.Charset;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.exception.KylinException;
+import org.apache.kylin.common.msg.MsgPicker;
+import org.apache.kylin.common.scheduler.EventBusFactory;
+import org.apache.kylin.common.util.JsonUtil;
+import org.apache.kylin.junit.rule.TransactionExceptedException;
+import org.apache.kylin.metadata.acl.AclTCR;
+import org.apache.kylin.metadata.acl.AclTCRManager;
+import org.apache.kylin.metadata.cube.model.NDataflowManager;
+import org.apache.kylin.metadata.model.ComputedColumnDesc;
+import org.apache.kylin.metadata.model.NDataModel;
+import org.apache.kylin.metadata.model.NDataModelManager;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.recommendation.candidate.JdbcRawRecStore;
+import org.apache.kylin.rest.constant.Constant;
+import org.apache.kylin.rest.request.ModelRequest;
+import org.apache.kylin.rest.util.AclEvaluate;
+import org.apache.kylin.rest.util.AclUtil;
+import org.apache.kylin.tool.bisync.BISyncTool;
+import org.apache.kylin.tool.bisync.SyncContext;
+import org.apache.kylin.tool.bisync.model.SyncModel;
+import org.apache.kylin.tool.bisync.tableau.TableauDatasourceModel;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.springframework.security.authentication.TestingAuthenticationToken;
+import org.springframework.security.core.context.SecurityContextHolder;
+import org.springframework.test.util.ReflectionTestUtils;
+
+import com.google.common.base.Charsets;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import com.google.common.io.CharStreams;
+
+import lombok.val;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class ModelTdsServiceTest extends SourceTestCase {
+
+ @InjectMocks
+ private final ModelService modelService = Mockito.spy(new ModelService());
+
+ @InjectMocks
+ private final ModelTdsService tdsService = Mockito.spy(new ModelTdsService());
+
+ @InjectMocks
+ private final ModelSemanticHelper semanticService = Mockito.spy(new ModelSemanticHelper());
+
+ @InjectMocks
+ private final IndexPlanService indexPlanService = Mockito.spy(new IndexPlanService());
+
+ @Mock
+ private final AclUtil aclUtil = Mockito.spy(AclUtil.class);
+
+ @Mock
+ private final AclEvaluate aclEvaluate = Mockito.spy(AclEvaluate.class);
+
+ @Mock
+ protected IUserGroupService userGroupService = Mockito.spy(NUserGroupService.class);
+
+ @Mock
+ private final AccessService accessService = Mockito.spy(AccessService.class);
+
+ @Rule
+ public TransactionExceptedException thrown = TransactionExceptedException.none();
+
+ protected String getProject() {
+ return "default";
+ }
+
+ @Before
+ public void setup() {
+ super.setup();
+ overwriteSystemProp("HADOOP_USER_NAME", "root");
+ ReflectionTestUtils.setField(aclEvaluate, "aclUtil", aclUtil);
+ ReflectionTestUtils.setField(modelService, "aclEvaluate", aclEvaluate);
+ ReflectionTestUtils.setField(modelService, "accessService", accessService);
+ ReflectionTestUtils.setField(modelService, "userGroupService", userGroupService);
+ ReflectionTestUtils.setField(modelService, "userGroupService", userGroupService);
+
+ ReflectionTestUtils.setField(tdsService, "accessService", accessService);
+ ReflectionTestUtils.setField(tdsService, "userGroupService", userGroupService);
+ ReflectionTestUtils.setField(tdsService, "aclEvaluate", aclEvaluate);
+
+ modelService.setSemanticUpdater(semanticService);
+ modelService.setIndexPlanService(indexPlanService);
+
+ try {
+ new JdbcRawRecStore(getTestConfig());
+ } catch (Exception e) {
+ //
+ }
+ }
+
+ @After
+ public void tearDown() {
+ getTestConfig().setProperty("kylin.metadata.semi-automatic-mode", "false");
+ EventBusFactory.getInstance().restart();
+ cleanupTestMetadata();
+ }
+
+ @Test
+ public void testExportTDSWithDupMeasureDimColumnNames() throws IOException {
+ String projectName = "default";
+ String modelId = "199ee99e-8419-3e7a-7cad-97059999ec0a";
+ val modelRequest = JsonUtil.readValue(
+ new File("src/test/resources/ut_meta/dup_name_test/model_desc/model_dup_mea_dimcol.json"),
+ ModelRequest.class);
+ modelRequest.setProject(projectName);
+ List<NDataModel.NamedColumn> simplifiedDims = modelRequest.getAllNamedColumns().stream()
+ .filter(NDataModel.NamedColumn::isDimension).collect(Collectors.toList());
+ modelRequest.setSimplifiedDimensions(simplifiedDims);
+ modelService.createModel(projectName, modelRequest);
+ NDataModelManager projectInstance = NDataModelManager.getInstance(getTestConfig(), projectName);
+ Assert.assertNotNull(projectInstance.getDataModelDescByAlias("model_dup_mea_dimcol"));
+ SyncContext syncContext = new SyncContext();
+ syncContext.setProjectName(projectName);
+ syncContext.setModelId(modelId);
+ syncContext.setModelElement(SyncContext.ModelElement.ALL_COLS);
+ syncContext.setAdmin(true);
+ syncContext.setDataflow(NDataflowManager.getInstance(getTestConfig(), projectName).getDataflow(modelId));
+ syncContext.setKylinConfig(getTestConfig());
+ SyncModel syncModel = tdsService.exportModel(syncContext);
+ Assert.assertThrows(
+ "There are duplicated names among dimension column LO_LINENUMBER and measure name LO_LINENUMBER. Cannot export a valid TDS file. Please correct the duplicated names and try again.",
+ KylinException.class, () -> tdsService.preCheckNameConflict(syncModel));
+ }
+
+ @Test
+ public void testExportTdsWithDupMeasureDimensionNamesNoConflict() throws IOException {
+ String projectName = "default";
+ String modelId = "6f8cd656-9beb-47f6-87f5-89a8c548d17c";
+ val modelRequest = JsonUtil.readValue(
+ new File("src/test/resources/ut_meta/dup_name_test/model_desc/model_dup_mea_dim.json"),
+ ModelRequest.class);
+ modelRequest.setProject(projectName);
+ List<NDataModel.NamedColumn> simplifiedDims = modelRequest.getAllNamedColumns().stream()
+ .filter(NDataModel.NamedColumn::isDimension).collect(Collectors.toList());
+ modelRequest.setSimplifiedDimensions(simplifiedDims);
+ modelService.createModel(projectName, modelRequest);
+ NDataModelManager projectInstance = NDataModelManager.getInstance(getTestConfig(), projectName);
+ Assert.assertNotNull(projectInstance.getDataModelDescByAlias("model_dup_mea_dim"));
+ SyncContext syncContext = new SyncContext();
+ syncContext.setProjectName(projectName);
+ syncContext.setModelId(modelId);
+ syncContext.setModelElement(SyncContext.ModelElement.ALL_COLS);
+ syncContext.setAdmin(true);
+ syncContext.setDataflow(NDataflowManager.getInstance(getTestConfig(), projectName).getDataflow(modelId));
+ syncContext.setKylinConfig(getTestConfig());
+ SyncModel syncModel = tdsService.exportModel(syncContext);
+ Assert.assertTrue(tdsService.preCheckNameConflict(syncModel));
+ }
+
+ @Test
+ public void testExportTDSWithDupMeasureColumnNames() throws IOException {
+ String projectName = "default";
+ String modelId = "2ed3bf12-ad40-e8a0-73da-8dc3b4c798bb";
+ val modelRequest = JsonUtil.readValue(
+ new File("src/test/resources/ut_meta/dup_name_test/model_desc/model_dup_mea_col.json"),
+ ModelRequest.class);
+ modelRequest.setProject(projectName);
+ List<NDataModel.NamedColumn> simplifiedDims = modelRequest.getAllNamedColumns().stream()
+ .filter(NDataModel.NamedColumn::isDimension).collect(Collectors.toList());
+ modelRequest.setSimplifiedDimensions(simplifiedDims);
+ modelService.createModel(projectName, modelRequest);
+ NDataModelManager projectInstance = NDataModelManager.getInstance(getTestConfig(), projectName);
+ Assert.assertNotNull(projectInstance.getDataModelDescByAlias("model_dup_mea_col"));
+ SyncContext syncContext = new SyncContext();
+ syncContext.setProjectName(projectName);
+ syncContext.setModelId(modelId);
+ syncContext.setModelElement(SyncContext.ModelElement.ALL_COLS);
+ syncContext.setDataflow(NDataflowManager.getInstance(getTestConfig(), projectName).getDataflow(modelId));
+ syncContext.setKylinConfig(getTestConfig());
+ syncContext.setAdmin(true);
+ SyncModel syncModel = tdsService.exportModel(syncContext);
+ Assert.assertThrows(
+ "There are duplicated names among model column LO_LINENUMBER and measure name LO_LINENUMBER. Cannot export a valid TDS file. Please correct the duplicated names and try again.",
+ KylinException.class, () -> tdsService.preCheckNameConflict(syncModel));
+ }
+
+ @Test
+ public void testExportTDSByAdmin() throws Exception {
+ val project = "default";
+ val modelId = "cb596712-3a09-46f8-aea1-988b43fe9b6c";
+ prepareBasic(project);
+ List<String> dimensions = Lists.newArrayList();
+ dimensions.add("DEFAULT.TEST_MEASURE.FLAG");
+ dimensions.add("DEFAULT.TEST_MEASURE.PRICE1");
+ dimensions.add("DEFAULT.TEST_MEASURE.ID1");
+ List<String> measurs = Lists.newArrayList();
+ measurs.add("COUNT_STAR");
+ measurs.add("SUM_1");
+ SyncContext syncContext = tdsService.prepareSyncContext(project, modelId, SyncContext.BI.TABLEAU_CONNECTOR_TDS,
+ SyncContext.ModelElement.CUSTOM_COLS, "localhost", 8080);
+ SyncModel syncModel = tdsService.exportTDSDimensionsAndMeasuresByAdmin(syncContext, dimensions, measurs);
+ TableauDatasourceModel datasource1 = (TableauDatasourceModel) BISyncTool.getBISyncModel(syncContext, syncModel);
+ ByteArrayOutputStream outStream4 = new ByteArrayOutputStream();
+ datasource1.dump(outStream4);
+ Assert.assertEquals(getExpectedTds("/bisync_tableau/nmodel_full_measure_test.connector_admin.tds"),
+ outStream4.toString(Charset.defaultCharset().name()));
+ }
+
+ @Test
+ public void testExportTDSByUser() throws Exception {
+ val project = "default";
+ val modelId = "cb596712-3a09-46f8-aea1-988b43fe9b6c";
+ prepareBasic(project);
+ List<String> dimensions = Lists.newArrayList();
+ dimensions.add("TEST_MEASURE.ID1");
+ dimensions.add("TEST_MEASURE.ID2");
+ dimensions.add("TEST_MEASURE.ID3");
+ dimensions.add("TEST_MEASURE1.ID1");
+ dimensions.add("TEST_MEASURE1.NAME1");
+ dimensions.add("TEST_MEASURE1.NAME2");
+ dimensions.add("TEST_MEASURE1.NAME3");
+ List<String> measurs = Lists.newArrayList();
+ measurs.add("COUNT_STAR");
+ measurs.add("SUM_1");
+ SecurityContextHolder.getContext()
+ .setAuthentication(new TestingAuthenticationToken("u1", "ANALYST", Constant.ROLE_ANALYST));
+ SyncContext syncContext = tdsService.prepareSyncContext(project, modelId, SyncContext.BI.TABLEAU_CONNECTOR_TDS,
+ SyncContext.ModelElement.CUSTOM_COLS, "localhost", 8080);
+ syncContext.setAdmin(false);
+ SyncModel syncModel = tdsService.exportTDSDimensionsAndMeasuresByNormalUser(syncContext, dimensions, measurs);
+ TableauDatasourceModel datasource1 = (TableauDatasourceModel) BISyncTool.getBISyncModel(syncContext, syncModel);
+ ByteArrayOutputStream outStream4 = new ByteArrayOutputStream();
+ datasource1.dump(outStream4);
+ Assert.assertEquals(getExpectedTds("/bisync_tableau/nmodel_full_measure_test.connector_user.tds"),
+ outStream4.toString(Charset.defaultCharset().name()));
+ }
+
+ @Test
+ public void testExportTDSByUserAndElement() throws Exception {
+ val project = "default";
+ val modelId = "cb596712-3a09-46f8-aea1-988b43fe9b6c";
+ prepareBasic(project);
+ List<String> dimensions = Lists.newArrayList();
+ dimensions.add("TEST_MEASURE.ID1");
+ try {
+ SecurityContextHolder.getContext()
+ .setAuthentication(new TestingAuthenticationToken("u1", "ANALYST", Constant.ROLE_ANALYST));
+ SyncContext syncContext = tdsService.prepareSyncContext(project, modelId,
+ SyncContext.BI.TABLEAU_CONNECTOR_TDS, SyncContext.ModelElement.AGG_INDEX_COL, "localhost", 8080);
+ SyncModel syncModel = tdsService.exportTDSDimensionsAndMeasuresByNormalUser(syncContext, dimensions,
+ ImmutableList.of());
+ TableauDatasourceModel datasource1 = (TableauDatasourceModel) BISyncTool.getBISyncModel(syncContext,
+ syncModel);
+ ByteArrayOutputStream outStream4 = new ByteArrayOutputStream();
+ datasource1.dump(outStream4);
+ Assert.assertEquals(
+ getExpectedTds("/bisync_tableau/nmodel_full_measure_test.connector_user_agg_index_col.tds"),
+ outStream4.toString(Charset.defaultCharset().name()));
+ } finally {
+ SecurityContextHolder.getContext()
+ .setAuthentication(new TestingAuthenticationToken("ADMIN", "ADMIN", Constant.ROLE_ADMIN));
+ }
+ }
+
+ @Test
+ public void testCheckModelExportPermissionException() {
+ val project = "default";
+ val modelId = "cb596712-3a09-46f8-aea1-988b43fe9b6c";
+ prepareBasic(project);
+ tdsService.prepareSyncContext(project, modelId, SyncContext.BI.TABLEAU_CONNECTOR_TDS,
+ SyncContext.ModelElement.AGG_INDEX_COL, "localhost", 8080);
+ try {
+ Mockito.when(accessService.getGroupsOfExecuteUser(Mockito.any(String.class)))
+ .thenReturn(Sets.newHashSet("ROLE_ANALYST"));
+ SecurityContextHolder.getContext()
+ .setAuthentication(new TestingAuthenticationToken("u1", "ANALYST", Constant.ROLE_ANALYST));
+ thrown.expect(KylinException.class);
+ thrown.expectMessage("current user does not have full permission on requesting model");
+ SyncContext syncContext = tdsService.prepareSyncContext(project, modelId,
+ SyncContext.BI.TABLEAU_CONNECTOR_TDS, SyncContext.ModelElement.AGG_INDEX_COL, "localhost", 8080);
+ tdsService.exportModel(syncContext);
+ } finally {
+ SecurityContextHolder.getContext()
+ .setAuthentication(new TestingAuthenticationToken("ADMIN", "ADMIN", Constant.ROLE_ADMIN));
+ }
+ }
+
+ @Test
+ public void testCheckModelExportPermission() {
+ val project = "default";
+ val modelId = "cb596712-3a09-46f8-aea1-988b43fe9b6c";
+ prepareBasic(project);
+ tdsService.prepareSyncContext(project, modelId, SyncContext.BI.TABLEAU_CONNECTOR_TDS,
+ SyncContext.ModelElement.AGG_INDEX_COL, "localhost", 8080);
+ tdsService.prepareSyncContext(project, modelId, SyncContext.BI.TABLEAU_CONNECTOR_TDS,
+ SyncContext.ModelElement.AGG_INDEX_COL, "localhost", 8080);
+ }
+
+ @Test
+ public void testCheckModelExportPermissionWithCC() {
+ val project = "cc_test";
+ val modelId = "0d146f1a-bdd3-4548-87ac-21c2c6f9a0da";
+ AclTCRManager manager = AclTCRManager.getInstance(getTestConfig(), project);
+ {
+ AclTCR u1a1 = new AclTCR();
+ manager.updateAclTCR(u1a1, "u1", true);
+ SecurityContextHolder.getContext()
+ .setAuthentication(new TestingAuthenticationToken("u1", "ANALYST", Constant.ROLE_ANALYST));
+ Mockito.when(accessService.getGroupsOfExecuteUser(Mockito.any(String.class)))
+ .thenReturn(Sets.newHashSet("ROLE_ANALYST"));
+ tdsService.prepareSyncContext(project, modelId, SyncContext.BI.TABLEAU_CONNECTOR_TDS,
+ SyncContext.ModelElement.AGG_INDEX_COL, "localhost", 8080);
+ }
+ {
+ try {
+ AclTCR u1a1 = new AclTCR();
+ AclTCR.Table u1t1 = new AclTCR.Table();
+ AclTCR.ColumnRow u1cr1 = new AclTCR.ColumnRow();
+ AclTCR.Column u1c1 = new AclTCR.Column();
+ u1c1.add("ORDER_ID");
+ u1cr1.setColumn(u1c1);
+ u1t1.put("SSB.LINEORDER", u1cr1);
+ u1a1.setTable(u1t1);
+ manager.updateAclTCR(u1a1, "u1", true);
+ thrown.expect(KylinException.class);
+ thrown.expectMessage("current user does not have full permission on requesting model");
+ SyncContext syncContext = tdsService.prepareSyncContext(project, modelId,
+ SyncContext.BI.TABLEAU_CONNECTOR_TDS, SyncContext.ModelElement.AGG_INDEX_COL, "localhost",
+ 8080);
+ tdsService.exportModel(syncContext);
+ } finally {
+ SecurityContextHolder.getContext()
+ .setAuthentication(new TestingAuthenticationToken("ADMIN", "ADMIN", Constant.ROLE_ADMIN));
+ }
+ }
+
+ }
+
+ @Test
+ public void testExportTDSByBroken() {
+ val project = "test_broken_project";
+ val modelId = "4b93b131-824e-6966-c4dd-5a4268d27095";
+ List<String> dimensions = Lists.newArrayList();
+ List<String> measures = Lists.newArrayList();
+ Assert.assertThrows(KylinException.class, () -> tdsService.prepareSyncContext(project, modelId,
+ SyncContext.BI.TABLEAU_CONNECTOR_TDS, SyncContext.ModelElement.CUSTOM_COLS, "localhost", 8080));
+ }
+
+ @Test
+ public void testExportTDSMeasurePermission() {
+ val project = "default";
+ val modelId = "82fa7671-a935-45f5-8779-85703601f49a";
+ prepareBasicByMeasure(project);
+ List<String> dimensions = Lists.newArrayList();
+ //"ORDER_ID", "PRICE", "CAL_DT", "PRICE", "ITEM_COUNT", "LEAF_CATEG_ID"
+ dimensions.add("TEST_KYLIN_FACT.ORDER_ID");
+ dimensions.add("TEST_KYLIN_FACT.PRICE");
+ dimensions.add("TEST_KYLIN_FACT.CAL_DT");
+ dimensions.add("TEST_KYLIN_FACT.PRICE");
+ dimensions.add("TEST_KYLIN_FACT.ITEM_COUNT");
+ dimensions.add("TEST_KYLIN_FACT.LEAF_CATEG_ID");
+ //"ORDER_ID", "TEST_TIME_ENC", "TEST_DATE_ENC"
+ dimensions.add("TEST_ORDER.ORDER_ID");
+ dimensions.add("TEST_ORDER.TEST_TIME_ENC");
+ dimensions.add("TEST_ORDER.TEST_DATE_ENC");
+ //"ORDER_ID", "PRICE", "CAL_DT", "TRANS_ID"
+ dimensions.add("TEST_MEASURE.ORDER_ID");
+ dimensions.add("TEST_MEASURE.PRICE");
+ dimensions.add("TEST_MEASURE.CAL_DT");
+ dimensions.add("TEST_MEASURE.TRANS_ID");
+
+ List<String> measures = Lists.newArrayList();
+ measures.add("TRANS_CNT");
+ measures.add("GMV_SUM");
+ measures.add("GMV_MIN");
+ measures.add("GMV_MAX");
+ measures.add("ITEM_COUNT_SUM");
+ measures.add("ITEM_COUNT_MAX");
+ measures.add("ITEM_COUNT_MIN");
+ measures.add("SELLER_HLL");
+ measures.add("COUNT_DISTINCT");
+ measures.add("TOP_SELLER");
+ measures.add("TEST_COUNT_DISTINCT_BITMAP");
+ measures.add("GVM_PERCENTILE");
+ SecurityContextHolder.getContext()
+ .setAuthentication(new TestingAuthenticationToken("u1", "ANALYST", Constant.ROLE_ANALYST));
+ SyncContext syncContext = tdsService.prepareSyncContext(project, modelId, SyncContext.BI.TABLEAU_CONNECTOR_TDS,
+ SyncContext.ModelElement.CUSTOM_COLS, "localhost", 8080);
+ Assert.assertThrows(KylinException.class,
+ () -> tdsService.exportTDSDimensionsAndMeasuresByNormalUser(syncContext, dimensions, measures));
+ }
+
+ private void prepareBasicByMeasure(String project) {
+ AclTCRManager manager = AclTCRManager.getInstance(getTestConfig(), project);
+
+ AclTCR u1a1 = new AclTCR();
+ AclTCR.Table u1t1 = new AclTCR.Table();
+ AclTCR.ColumnRow u1cr1 = new AclTCR.ColumnRow();
+ AclTCR.Column u1c1 = new AclTCR.Column();
+ u1c1.addAll(Arrays.asList("ORDER_ID", "PRICE", "CAL_DT", "PRICE", "ITEM_COUNT", "LEAF_CATEG_ID"));
+ u1cr1.setColumn(u1c1);
+
+ AclTCR.ColumnRow u1cr2 = new AclTCR.ColumnRow();
+ AclTCR.Column u1c2 = new AclTCR.Column();
+ u1c2.addAll(Arrays.asList("ORDER_ID", "TEST_TIME_ENC", "TEST_DATE_ENC"));
+ u1cr2.setColumn(u1c2);
+ u1t1.put("DEFAULT.TEST_KYLIN_FACT", u1cr1);
+ u1t1.put("DEFAULT.TEST_ORDER", u1cr2);
+ u1a1.setTable(u1t1);
+ manager.updateAclTCR(u1a1, "u1", true);
+ }
+
+ @Test
+ public void testExportModel() throws Exception {
+ val project = "default";
+ val modelId = "cb596712-3a09-46f8-aea1-988b43fe9b6c";
+ prepareBasic(project);
+ SyncContext syncContext = tdsService.prepareSyncContext(project, modelId, SyncContext.BI.TABLEAU_CONNECTOR_TDS,
+ SyncContext.ModelElement.AGG_INDEX_AND_TABLE_INDEX_COL, "localhost", 8080);
+ SyncModel syncModel = tdsService.exportModel(syncContext);
+ TableauDatasourceModel datasource1 = (TableauDatasourceModel) BISyncTool.getBISyncModel(syncContext, syncModel);
+ ByteArrayOutputStream outStream4 = new ByteArrayOutputStream();
+ datasource1.dump(outStream4);
+ Assert.assertEquals(getExpectedTds("/bisync_tableau/nmodel_full_measure_test.connector.tds"),
+ outStream4.toString(Charset.defaultCharset().name()));
+ }
+
+ private String getExpectedTds(String path) throws IOException {
+ return CharStreams.toString(new InputStreamReader(getClass().getResourceAsStream(path), Charsets.UTF_8));
+ }
+
+ private void prepareBasic(String project) {
+ AclTCRManager manager = AclTCRManager.getInstance(getTestConfig(), project);
+
+ AclTCR u1a1 = new AclTCR();
+ AclTCR.Table u1t1 = new AclTCR.Table();
+ AclTCR.ColumnRow u1cr1 = new AclTCR.ColumnRow();
+ AclTCR.Column u1c1 = new AclTCR.Column();
+ u1c1.addAll(Arrays.asList("ID1", "ID2", "ID3"));
+ u1cr1.setColumn(u1c1);
+
+ AclTCR.ColumnRow u1cr2 = new AclTCR.ColumnRow();
+ AclTCR.Column u1c2 = new AclTCR.Column();
+ u1c2.addAll(Arrays.asList("ID1", "NAME1", "NAME2", "NAME3"));
+ u1cr2.setColumn(u1c2);
+ u1t1.put("DEFAULT.TEST_MEASURE", u1cr1);
+ u1t1.put("DEFAULT.TEST_MEASURE1", u1cr2);
+ u1a1.setTable(u1t1);
+ manager.updateAclTCR(u1a1, "u1", true);
+
+ AclTCR g1a1 = new AclTCR();
+ AclTCR.Table g1t1 = new AclTCR.Table();
+ AclTCR.ColumnRow g1cr1 = new AclTCR.ColumnRow();
+ AclTCR.Column g1c1 = new AclTCR.Column();
+ g1c1.addAll(Arrays.asList("ID1", "ID2", "ID3", "ID4"));
+ g1cr1.setColumn(g1c1);
+ g1t1.put("DEFAULT.TEST_MEASURE", g1cr1);
+ g1a1.setTable(g1t1);
+ manager.updateAclTCR(g1a1, "g1", false);
+ }
+
+ @Test
+ public void testCheckTablePermission() {
+ val project = "default";
+ val modelId = "cb596712-3a09-46f8-aea1-988b43fe9b6c";
+ thrown.expect(KylinException.class);
+ thrown.expectMessage(MsgPicker.getMsg().getTableNoColumnsPermission());
+
+ AclTCRManager manager = AclTCRManager.getInstance(getTestConfig(), project);
+ Set<String> columns = new HashSet<>();
+ columns.add("DEFAULT.TEST_MEASURE1.NAME1");
+ columns.add("DEFAULT.TEST_MEASURE1.NAME2");
+ columns.add("DEFAULT.TEST_MEASURE1.NAME3");
+
+ AclTCR u1a1 = new AclTCR();
+ AclTCR.Table u1t1 = new AclTCR.Table();
+ AclTCR.ColumnRow u1cr1 = new AclTCR.ColumnRow();
+ AclTCR.Column u1c1 = new AclTCR.Column();
+ u1cr1.setColumn(u1c1);
+
+ AclTCR.ColumnRow u1cr2 = new AclTCR.ColumnRow();
+ AclTCR.Column u1c2 = new AclTCR.Column();
+ u1c2.addAll(Arrays.asList("NAME1", "NAME2", "NAME3"));
+ u1cr2.setColumn(u1c2);
+ u1t1.put("DEFAULT.TEST_MEASURE", u1cr1);
+ u1t1.put("DEFAULT.TEST_MEASURE1", u1cr2);
+ u1a1.setTable(u1t1);
+ manager.updateAclTCR(u1a1, "u1", true);
+ SecurityContextHolder.getContext()
+ .setAuthentication(new TestingAuthenticationToken("u1", "ANALYST", Constant.ROLE_ANALYST));
+ List<String> dimensions = Lists.newArrayList();
+ dimensions.add("TEST_MEASURE.FLAG");
+ dimensions.add("TEST_MEASURE.PRICE1");
+ dimensions.add("TEST_MEASURE.ID1");
+ List<String> measurs = Lists.newArrayList();
+ measurs.add("COUNT_STAR");
+ measurs.add("SUM_1");
+ tdsService.checkTableHasColumnPermission(SyncContext.ModelElement.CUSTOM_COLS, project, modelId, columns,
+ dimensions, measurs);
+
+ dimensions.add("TEST_MEASURE.ID4");
+ Assert.assertThrows(KylinException.class,
+ () -> tdsService.checkTableHasColumnPermission(SyncContext.ModelElement.CUSTOM_COLS, project, modelId,
+ columns, dimensions, measurs));
+ }
+
+ @Test
+ public void testExportTDSCheckColumnPermission() {
+ val project = "default";
+ val modelId = "89af4ee2-2cdb-4b07-b39e-4c29856309aa";
+
+ NDataModelManager modelManager = NDataModelManager.getInstance(KylinConfig.getInstanceFromEnv(), project);
+ NDataModel dataModel = modelManager.getDataModelDesc(modelId);
+
+ Set<String> authColumns = Sets.newHashSet();
+ List<String> dimensions = Lists.newArrayList();
+ List<String> measurs = Lists.newArrayList();
+
+ Assert.assertTrue(tdsService.checkColumnPermission(dataModel, authColumns, null, measurs));
+ Assert.assertTrue(tdsService.checkColumnPermission(dataModel, authColumns, null, null));
+ Assert.assertTrue(tdsService.checkColumnPermission(dataModel, authColumns, dimensions, null));
+ Assert.assertTrue(tdsService.checkColumnPermission(dataModel, authColumns, dimensions, measurs));
+
+ authColumns.add("DEFAULT.TEST_KYLIN_FACT.PRICE");
+ authColumns.add("DEFAULT.TEST_KYLIN_FACT.ITEM_COUNT");
+ authColumns.add("EDW.TEST_CAL_DT.CAL_DT");
+ authColumns.add("DEFAULT.TEST_ACCOUNT.ACCOUNT_ID");
+
+ Set<String> newAuthColumns = Sets.newHashSet();
+ dataModel.getAllTables().forEach(tableRef -> {
+ List<TblColRef> collect = tableRef.getColumns().stream()
+ .filter(column -> authColumns.contains(column.getCanonicalName())).collect(Collectors.toList());
+ collect.forEach(x -> newAuthColumns.add(x.getAliasDotName()));
+ });
+
+ dimensions.add("TEST_KYLIN_FACT.DEAL_AMOUNT");
+ dimensions.add("TEST_KYLIN_FACT.TRANS_ID");
+
+ Assert.assertFalse(tdsService.checkColumnPermission(dataModel, newAuthColumns, dimensions, measurs));
+
+ newAuthColumns.add("TEST_KYLIN_FACT.TRANS_ID");
+
+ measurs.add("SUM_NEST4");
+ measurs.add("COUNT_CAL_DT");
+ Assert.assertTrue(tdsService.checkColumnPermission(dataModel, newAuthColumns, dimensions, measurs));
+
+ Assert.assertTrue(tdsService.checkColumnPermission(dataModel, newAuthColumns, dimensions, measurs));
+ }
+
+ @Test
+ public void testConvertCCToNormalCols() {
+ val project = "default";
+ val modelId = "89af4ee2-2cdb-4b07-b39e-4c29856309aa";
+ NDataModelManager modelManager = NDataModelManager.getInstance(KylinConfig.getInstanceFromEnv(), project);
+ NDataModel dataModel = modelManager.getDataModelDesc(modelId);
+ NDataModel.Measure measure = dataModel.getEffectiveMeasures().values().stream()
+ .filter(x -> x.getName().equals("SUM_NEST4")).findFirst().get();
+ Set<String> measureColumns = measure.getFunction().getParameters().stream()
+ .filter(parameterDesc -> parameterDesc.getColRef() != null)
+ .map(parameterDesc -> parameterDesc.getColRef().getCanonicalName()).collect(Collectors.toSet());
+ ComputedColumnDesc sumNest4 = dataModel.getComputedColumnDescs().stream()
+ .filter(x -> measureColumns.contains(x.getIdentName())).findFirst().get();
+ Set<String> strings = tdsService.convertCCToNormalCols(dataModel, sumNest4);
+ Assert.assertEquals("TEST_KYLIN_FACT.PRICE, TEST_KYLIN_FACT.ITEM_COUNT", String.join(", ", strings));
+
+ sumNest4.setInnerExpression("1 + 2");
+ Set<String> set = tdsService.convertCCToNormalCols(dataModel, sumNest4);
+ Assert.assertEquals(Collections.emptySet(), set);
+
+ HashSet<Object> authColumns = Sets.newHashSet();
+ authColumns.add("DEFAULT.TEST_KYLIN_FACT.PRICE");
+ Assert.assertTrue(authColumns.containsAll(set));
+ }
+}
diff --git a/src/query-common/src/main/java/org/apache/kylin/tool/bisync/BISyncTool.java b/src/query-common/src/main/java/org/apache/kylin/tool/bisync/BISyncTool.java
index 9449dee777..4834af6595 100644
--- a/src/query-common/src/main/java/org/apache/kylin/tool/bisync/BISyncTool.java
+++ b/src/query-common/src/main/java/org/apache/kylin/tool/bisync/BISyncTool.java
@@ -24,26 +24,30 @@ import java.util.Set;
import org.apache.kylin.tool.bisync.model.SyncModel;
import org.apache.kylin.tool.bisync.tableau.TableauDataSourceConverter;
+import com.google.common.annotations.VisibleForTesting;
+
public class BISyncTool {
private BISyncTool() {
}
+ @VisibleForTesting
public static BISyncModel dumpToBISyncModel(SyncContext syncContext) {
SyncModel syncModel = new SyncModelBuilder(syncContext).buildSourceSyncModel();
return getBISyncModel(syncContext, syncModel);
}
- private static BISyncModel getBISyncModel(SyncContext syncContext, SyncModel syncModel) {
+ public static BISyncModel getBISyncModel(SyncContext syncContext, SyncModel syncModel) {
switch (syncContext.getTargetBI()) {
- case TABLEAU_ODBC_TDS:
- case TABLEAU_CONNECTOR_TDS:
- return new TableauDataSourceConverter().convert(syncModel, syncContext);
- default:
- throw new IllegalArgumentException();
+ case TABLEAU_ODBC_TDS:
+ case TABLEAU_CONNECTOR_TDS:
+ return new TableauDataSourceConverter().convert(syncModel, syncContext);
+ default:
+ throw new IllegalArgumentException();
}
}
+ @VisibleForTesting
public static BISyncModel dumpHasPermissionToBISyncModel(SyncContext syncContext, Set<String> authTables,
Set<String> authColumns, List<String> dimensions, List<String> measures) {
SyncModel syncModel = new SyncModelBuilder(syncContext).buildHasPermissionSourceSyncModel(authTables,
@@ -51,8 +55,8 @@ public class BISyncTool {
return getBISyncModel(syncContext, syncModel);
}
- public static BISyncModel dumpBISyncModel(SyncContext syncContext,
- List<String> dimensions, List<String> measures) {
+ @VisibleForTesting
+ public static BISyncModel dumpBISyncModel(SyncContext syncContext, List<String> dimensions, List<String> measures) {
SyncModel syncModel = new SyncModelBuilder(syncContext).buildSourceSyncModel(dimensions, measures);
return getBISyncModel(syncContext, syncModel);
}
diff --git a/src/query-common/src/main/java/org/apache/kylin/tool/bisync/SyncContext.java b/src/query-common/src/main/java/org/apache/kylin/tool/bisync/SyncContext.java
index b43a898726..895897e031 100644
--- a/src/query-common/src/main/java/org/apache/kylin/tool/bisync/SyncContext.java
+++ b/src/query-common/src/main/java/org/apache/kylin/tool/bisync/SyncContext.java
@@ -53,4 +53,6 @@ public class SyncContext {
private NDataflow dataflow;
private KylinConfig kylinConfig;
+
+ private boolean isAdmin;
}
diff --git a/src/query-common/src/main/java/org/apache/kylin/tool/bisync/SyncModelBuilder.java b/src/query-common/src/main/java/org/apache/kylin/tool/bisync/SyncModelBuilder.java
index cb444fe016..74b402bc29 100644
--- a/src/query-common/src/main/java/org/apache/kylin/tool/bisync/SyncModelBuilder.java
+++ b/src/query-common/src/main/java/org/apache/kylin/tool/bisync/SyncModelBuilder.java
@@ -20,13 +20,13 @@ package org.apache.kylin.tool.bisync;
import java.util.Arrays;
import java.util.HashMap;
-import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
+import org.apache.commons.lang3.ArrayUtils;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.cube.model.SelectRule;
import org.apache.kylin.metadata.cube.cuboid.NAggregationGroup;
@@ -44,6 +44,8 @@ import org.apache.kylin.tool.bisync.model.JoinTreeNode;
import org.apache.kylin.tool.bisync.model.MeasureDef;
import org.apache.kylin.tool.bisync.model.SyncModel;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
@@ -56,22 +58,7 @@ public class SyncModelBuilder {
}
public SyncModel buildSourceSyncModel() {
- NDataModel dataModelDesc = syncContext.getDataflow().getModel();
- IndexPlan indexPlan = syncContext.getDataflow().getIndexPlan();
-
- // init joinTree, dimension cols, measure cols, hierarchies
- Map<String, ColumnDef> columnDefMap = getAllColumns(dataModelDesc);
-
- List<MeasureDef> measureDefs = dataModelDesc.getEffectiveMeasures().values().stream().map(MeasureDef::new)
- .collect(Collectors.toList());
- markHasPermissionIndexedColumnsAndMeasures(columnDefMap, measureDefs, indexPlan, null, null,
- syncContext.getModelElement());
- markComputedColumnVisibility(columnDefMap, measureDefs, syncContext.getKylinConfig().exposeComputedColumn());
-
- Set<String[]> hierarchies = getHierarchies(indexPlan);
- JoinTreeNode joinTree = generateJoinTree(dataModelDesc.getJoinTables(), dataModelDesc.getRootFactTableName());
-
- return getSyncModel(dataModelDesc, columnDefMap, measureDefs, hierarchies, joinTree);
+ return buildSourceSyncModel(ImmutableList.of(), ImmutableList.of());
}
public SyncModel buildSourceSyncModel(List<String> dimensions, List<String> measures) {
@@ -79,16 +66,17 @@ public class SyncModelBuilder {
IndexPlan indexPlan = syncContext.getDataflow().getIndexPlan();
// init joinTree, dimension cols, measure cols, hierarchies
- Map<String, ColumnDef> columnDefMap = authColumns(dataModelDesc);
+ Map<String, ColumnDef> columnDefMap = authColumns(dataModelDesc, syncContext.isAdmin(), ImmutableSet.of(),
+ ImmutableSet.of());
+ List<MeasureDef> measureDefs = dataModelDesc.getEffectiveMeasures().values().stream() //
+ .map(MeasureDef::new).collect(Collectors.toList());
- List<MeasureDef> measureDefs = dataModelDesc.getEffectiveMeasures().values().stream().map(MeasureDef::new)
- .collect(Collectors.toList());
- markHasPermissionIndexedColumnsAndMeasures(columnDefMap, measureDefs, indexPlan, dimensions, measures,
- syncContext.getModelElement());
+ markHasPermissionIndexedColumnsAndMeasures(columnDefMap, measureDefs, indexPlan, ImmutableSet.of(), dimensions,
+ measures, syncContext.getModelElement());
markComputedColumnVisibility(columnDefMap, measureDefs, syncContext.getKylinConfig().exposeComputedColumn());
+
Set<String[]> hierarchies = getHierarchies(indexPlan);
JoinTreeNode joinTree = generateJoinTree(dataModelDesc.getJoinTables(), dataModelDesc.getRootFactTableName());
-
return getSyncModel(dataModelDesc, columnDefMap, measureDefs, hierarchies, joinTree);
}
@@ -99,20 +87,22 @@ public class SyncModelBuilder {
Set<String> allAuthColumns = addHasPermissionCCColumn(dataModelDesc, authColumns);
// init joinTree, dimension cols, measure cols, hierarchies
- Map<String, ColumnDef> columnDefMap = authColumns(dataModelDesc, authTables, allAuthColumns);
+ Map<String, ColumnDef> columnDefMap = authColumns(dataModelDesc, syncContext.isAdmin(), authTables,
+ allAuthColumns);
+ List<MeasureDef> measureDefs = dataModelDesc.getEffectiveMeasures().values().stream() //
+ .filter(measure -> checkMeasurePermission(allAuthColumns, measure)) //
+ .map(MeasureDef::new).collect(Collectors.toList());
- List<MeasureDef> measureDefs = dataModelDesc.getEffectiveMeasures().values().stream()
- .filter(measure -> checkMeasurePermission(allAuthColumns, measure)).map(MeasureDef::new)
- .collect(Collectors.toList());
markHasPermissionIndexedColumnsAndMeasures(columnDefMap, measureDefs, indexPlan, allAuthColumns, dimensions,
measures, syncContext.getModelElement());
markComputedColumnVisibility(columnDefMap, measureDefs, syncContext.getKylinConfig().exposeComputedColumn());
+
+ Set<String> omitDbColSet = renameColumnName(allAuthColumns);
Set<String[]> hierarchies = getHierarchies(indexPlan).stream()
- .map(hierarchyArray -> Arrays.stream(hierarchyArray).filter(renameColumnName(allAuthColumns)::contains)
+ .map(hierarchyArray -> Arrays.stream(hierarchyArray).filter(omitDbColSet::contains)
.collect(Collectors.toSet()).toArray(new String[0]))
.collect(Collectors.toSet()).stream().filter(x -> !Arrays.asList(x).isEmpty())
.collect(Collectors.toSet());
-
JoinTreeNode joinTree = generateJoinTree(dataModelDesc.getJoinTables(), dataModelDesc.getRootFactTableName());
return getSyncModel(dataModelDesc, columnDefMap, measureDefs, hierarchies, joinTree);
}
@@ -125,7 +115,7 @@ public class SyncModelBuilder {
syncModel.setJoinTree(joinTree);
syncModel.setMetrics(measureDefs);
syncModel.setHierarchies(hierarchies);
- syncModel.setProjectName(syncContext.getProjectName());
+ syncModel.setProject(syncContext.getProjectName());
syncModel.setModelName(dataModelDesc.getAlias());
syncModel.setHost(syncContext.getHost());
syncModel.setPort(String.valueOf(syncContext.getPort()));
@@ -141,135 +131,105 @@ public class SyncModelBuilder {
private void markComputedColumnVisibility(Map<String, ColumnDef> columnDefMap, List<MeasureDef> measureDefs,
boolean exposeComputedColumns) {
- if (!exposeComputedColumns) {
- // hide all CC cols and related measures
- for (ColumnDef columnDef : columnDefMap.values()) {
- if (columnDef.isComputedColumn()) {
- columnDef.setHidden(true);
- }
+ if (exposeComputedColumns) {
+ return;
+ }
+ // hide all CC cols and related measures
+ for (ColumnDef columnDef : columnDefMap.values()) {
+ if (columnDef.isComputedColumn()) {
+ columnDef.setHidden(true);
}
- for (MeasureDef measureDef : measureDefs) {
- for (TblColRef paramColRef : measureDef.getMeasure().getFunction().getColRefs()) {
- if (columnDefMap.get(paramColRef.getAliasDotName()).isComputedColumn()) {
- measureDef.setHidden(true);
- break;
- }
+ }
+ for (MeasureDef measureDef : measureDefs) {
+ for (TblColRef paramColRef : measureDef.getMeasure().getFunction().getColRefs()) {
+ ColumnDef columnDef = columnDefMap.get(paramColRef.getAliasDotName());
+ if (columnDef != null && columnDef.isComputedColumn()) {
+ measureDef.setHidden(true);
+ break;
}
}
}
}
private void markHasPermissionIndexedColumnsAndMeasures(Map<String, ColumnDef> columnDefMap,
- List<MeasureDef> measureDefs, IndexPlan indexPlan, Set<String> columns, List<String> dimensions,
+ List<MeasureDef> measureDefs, IndexPlan indexPlan, Set<String> authorizedCols, List<String> dimensions,
List<String> measures, SyncContext.ModelElement modelElement) {
Set<String> colsToShow = Sets.newHashSet();
Set<String> measuresToShow = Sets.newHashSet();
switch (modelElement) {
case AGG_INDEX_COL:
- ImmutableBitSet aggDimBitSet = indexPlan.getAllIndexes().stream().filter(index -> !index.isTableIndex())
- .map(IndexEntity::getDimensionBitset).reduce(ImmutableBitSet.EMPTY, ImmutableBitSet::or);
- Set<TblColRef> tblColRefs = indexPlan.getEffectiveDimCols().entrySet().stream()
- .filter(entry -> aggDimBitSet.get(entry.getKey())).map(Map.Entry::getValue)
+ ImmutableBitSet aggDimBitSet = indexPlan.getAllIndexes().stream() //
+ .filter(index -> !index.isTableIndex()) //
+ .map(IndexEntity::getDimensionBitset) //
+ .reduce(ImmutableBitSet.EMPTY, ImmutableBitSet::or);
+ Set<TblColRef> tblColRefs = indexPlan.getEffectiveDimCols().entrySet().stream() //
+ .filter(entry -> aggDimBitSet.get(entry.getKey())) //
+ .map(Map.Entry::getValue) //
.collect(Collectors.toSet());
- colsToShow = tblColRefs.stream().filter(column -> columns.contains(column.getAliasDotName()))
- .map(TblColRef::getAliasDotName).collect(Collectors.toSet());
- measuresToShow = indexPlan.getEffectiveMeasures().values().stream()
- .filter(measureDef -> checkMeasurePermission(columns, measureDef)).map(MeasureDesc::getName)
+ colsToShow = tblColRefs.stream() //
+ .filter(colRef -> testAuthorizedCols(authorizedCols, colRef)) //
+ .map(TblColRef::getAliasDotName) //
+ .collect(Collectors.toSet());
+ measuresToShow = indexPlan.getEffectiveMeasures().values().stream() //
+ .filter(measureDef -> testAuthorizedMeasures(authorizedCols, measureDef)) //
+ .map(MeasureDesc::getName) //
.collect(Collectors.toSet());
break;
case AGG_INDEX_AND_TABLE_INDEX_COL:
- colsToShow = indexPlan.getEffectiveDimCols().values().stream()
- .filter(column -> columns.contains(column.getAliasDotName())).map(TblColRef::getAliasDotName)
+ colsToShow = indexPlan.getEffectiveDimCols().values().stream() //
+ .filter(colRef -> testAuthorizedCols(authorizedCols, colRef)) //
+ .map(TblColRef::getAliasDotName) //
.collect(Collectors.toSet());
measuresToShow = indexPlan.getEffectiveMeasures().values().stream()
- .filter(measureDef -> checkMeasurePermission(columns, measureDef)).map(MeasureDesc::getName)
+ .filter(measureDef -> testAuthorizedMeasures(authorizedCols, measureDef)) //
+ .map(MeasureDesc::getName) //
.collect(Collectors.toSet());
break;
case ALL_COLS:
- colsToShow = indexPlan.getModel().getDimensionNameIdMap().keySet().stream()
- .filter(renameColumnName(columns)::contains).collect(Collectors.toSet());
- measuresToShow = indexPlan.getModel().getEffectiveMeasures().values().stream()
- .filter(measureDef -> checkMeasurePermission(columns, measureDef)).map(MeasureDesc::getName)
+ colsToShow = indexPlan.getModel().getEffectiveDimensions().values().stream()
+ .filter(colRef -> testAuthorizedCols(authorizedCols, colRef)) //
+ .map(TblColRef::getAliasDotName) //
+ .collect(Collectors.toSet());
+ measuresToShow = indexPlan.getModel().getEffectiveMeasures().values().stream() //
+ .filter(measureDef -> testAuthorizedMeasures(authorizedCols, measureDef)) //
+ .map(MeasureDesc::getName) //
.collect(Collectors.toSet());
- for (MeasureDef measureDef : measureDefs) {
- measureDef.setHidden(false);
- }
break;
case CUSTOM_COLS:
- colsToShow = indexPlan.getModel().getDimensionNameIdMap().keySet().stream()
- .filter(renameColumnName(new HashSet<>(dimensions))::contains).collect(Collectors.toSet());
- measuresToShow = indexPlan.getModel().getEffectiveMeasures().values().stream()
- .filter(measureDef -> measures.contains(measureDef.getName())).map(MeasureDesc::getName)
+ Set<String> dimensionSet = Sets.newHashSet(dimensions);
+ colsToShow = indexPlan.getModel().getEffectiveDimensions().values().stream()
+ .filter(colRef -> testAuthorizedDimensions(dimensionSet, colRef)) //
+ .map(TblColRef::getAliasDotName) //
+ .collect(Collectors.toSet());
+ measuresToShow = indexPlan.getModel().getEffectiveMeasures().values().stream() //
+ .map(MeasureDesc::getName) //
+ .filter(measures::contains) //
.collect(Collectors.toSet());
-
- for (MeasureDef measureDef : measureDefs) {
- if (measuresToShow.contains(measureDef.getMeasure().getName())
- && (measures != null && measures.contains(measureDef.getMeasure().getName()))) {
- measureDef.setHidden(false);
- }
- }
break;
default:
break;
}
- setDimensionAndMeasureHidden(columnDefMap, measureDefs, modelElement, colsToShow, measuresToShow);
+ showDimsAndMeasures(columnDefMap, measureDefs, colsToShow, measuresToShow);
}
- private void markHasPermissionIndexedColumnsAndMeasures(Map<String, ColumnDef> columnDefMap,
- List<MeasureDef> measureDefs, IndexPlan indexPlan, List<String> dimensions, List<String> measures,
- SyncContext.ModelElement modelElement) {
- Set<String> colsToShow = new HashSet<>();
- Set<String> measuresToShow = new HashSet<>();
- switch (modelElement) {
- case AGG_INDEX_COL:
- ImmutableBitSet aggDimBitSet = indexPlan.getAllIndexes().stream().filter(index -> !index.isTableIndex())
- .map(IndexEntity::getDimensionBitset).reduce(ImmutableBitSet.EMPTY, ImmutableBitSet::or);
- Set<TblColRef> tblColRefs = indexPlan.getEffectiveDimCols().entrySet().stream()
- .filter(entry -> aggDimBitSet.get(entry.getKey())).map(Map.Entry::getValue)
- .collect(Collectors.toSet());
- colsToShow = tblColRefs.stream().map(TblColRef::getAliasDotName).collect(Collectors.toSet());
- measuresToShow = indexPlan.getEffectiveMeasures().values().stream().map(MeasureDesc::getName)
- .collect(Collectors.toSet());
- break;
- case AGG_INDEX_AND_TABLE_INDEX_COL:
- colsToShow = indexPlan.getEffectiveDimCols().values().stream().map(TblColRef::getAliasDotName)
- .collect(Collectors.toSet());
- measuresToShow = indexPlan.getEffectiveMeasures().values().stream().map(MeasureDesc::getName)
- .collect(Collectors.toSet());
- break;
- case ALL_COLS:
- colsToShow = indexPlan.getModel().getDimensionNameIdMap().keySet();
- measuresToShow = indexPlan.getModel().getEffectiveMeasures().values().stream().map(MeasureDesc::getName)
- .collect(Collectors.toSet());
- for (MeasureDef measureDef : measureDefs) {
- measureDef.setHidden(false);
- }
- break;
- case CUSTOM_COLS:
- colsToShow = indexPlan.getModel().getDimensionNameIdMap().keySet().stream()
- .filter(renameColumnName(new HashSet<>(dimensions))::contains).collect(Collectors.toSet());
- measuresToShow = indexPlan.getModel().getEffectiveMeasures().values().stream()
- .filter(measureDef -> measures != null && measures.contains(measureDef.getName()))
- .map(MeasureDesc::getName).collect(Collectors.toSet());
- for (MeasureDef measureDef : measureDefs) {
- if (measuresToShow.contains(measureDef.getMeasure().getName())
- && (measures != null && measures.contains(measureDef.getMeasure().getName()))) {
- measureDef.setHidden(false);
- }
- }
- break;
- default:
- break;
- }
+ private boolean testAuthorizedCols(Set<String> authorizedCols, TblColRef colRef) {
+ return syncContext.isAdmin() || authorizedCols.contains(colRef.getColumnWithTableAndSchema())
+ || authorizedCols.contains(colRef.getAliasDotName());
+ }
- setDimensionAndMeasureHidden(columnDefMap, measureDefs, modelElement, colsToShow, measuresToShow);
+ private boolean testAuthorizedDimensions(Set<String> dimensions, TblColRef colRef) {
+ return dimensions.contains(colRef.getColumnWithTableAndSchema())
+ || dimensions.contains(colRef.getAliasDotName());
}
- private void setDimensionAndMeasureHidden(Map<String, ColumnDef> columnDefMap, List<MeasureDef> measureDefs,
- SyncContext.ModelElement modelElement, Set<String> colsToShow, Set<String> measuresToShow) {
- colsToShow.forEach(colToShow -> columnDefMap.get(colToShow).setHidden(false));
- if (modelElement.equals(SyncContext.ModelElement.CUSTOM_COLS)) {
- return;
+ private boolean testAuthorizedMeasures(Set<String> authorizedCols, NDataModel.Measure measureDef) {
+ return syncContext.isAdmin() || checkMeasurePermission(authorizedCols, measureDef);
+ }
+
+ private void showDimsAndMeasures(Map<String, ColumnDef> columnDefMap, List<MeasureDef> measureDefs,
+ Set<String> colsToShow, Set<String> measuresToShow) {
+ for (String colToShow : colsToShow) {
+ columnDefMap.get(colToShow).setHidden(false);
}
for (MeasureDef measureDef : measureDefs) {
if (measuresToShow.contains(measureDef.getMeasure().getName())) {
@@ -288,60 +248,38 @@ public class SyncModelBuilder {
}).collect(Collectors.toSet());
}
- private Map<String, ColumnDef> getAllColumns(NDataModel modelDesc) {
- Map<String, ColumnDef> modelColsMap = new HashMap<>();
- for (TableRef tableRef : modelDesc.getAllTables()) {
- for (TblColRef column : tableRef.getColumns()) {
- ColumnDef columnDef = new ColumnDef("dimension", tableRef.getAlias(), null, column.getName(),
- column.getDatatype(), true, column.getColumnDesc().isComputedColumn());
- String colName = tableRef.getAlias() + "." + column.getName();
- modelColsMap.put(colName, columnDef);
+ private Map<String, ColumnDef> authColumns(NDataModel model, boolean isAdmin, Set<String> tables,
+ Set<String> columns) {
+ Map<String, ColumnDef> modelColsMap = Maps.newHashMap();
+ for (TableRef tableRef : model.getAllTables()) {
+ if (!isAdmin && !tables.contains(tableRef.getTableIdentity())) {
+ continue;
}
- }
-
- // sync col alias
- for (NDataModel.NamedColumn namedColumn : modelDesc.getAllNamedColumns()) {
- if (modelColsMap.get(namedColumn.getAliasDotColumn()) != null) {
- modelColsMap.get(namedColumn.getAliasDotColumn()).setColumnAlias(namedColumn.getName());
+ for (TblColRef colRef : tableRef.getColumns()) {
+ if (isAdmin || columns.contains(colRef.getAliasDotName())
+ || columns.contains(colRef.getColumnWithTableAndSchema())) {
+ ColumnDef columnDef = ColumnDef.builder() //
+ .role("dimension") //
+ .tableAlias(tableRef.getAlias()) //
+ .columnName(colRef.getName()) //
+ .columnType(colRef.getDatatype()) //
+ .isHidden(true) //
+ .isComputedColumn(colRef.getColumnDesc().isComputedColumn()) //
+ .build();
+ modelColsMap.put(colRef.getIdentity(), columnDef);
+ }
}
}
- return modelColsMap;
- }
-
- private Map<String, ColumnDef> authColumns(NDataModel modelDesc) {
- Map<String, ColumnDef> modelColsMap = Maps.newHashMap();
- modelDesc.getAllTables().stream().forEach(tableRef -> tableRef.getColumns().stream().forEach(column -> {
- ColumnDef columnDef = new ColumnDef("dimension", tableRef.getAlias(), null, column.getName(),
- column.getDatatype(), true, column.getColumnDesc().isComputedColumn());
- String colName = tableRef.getAlias() + "." + column.getName();
- modelColsMap.put(colName, columnDef);
- }));
// sync col alias
- modelDesc.getAllNamedColumns().stream()
- .filter(namedColumn -> modelColsMap.get(namedColumn.getAliasDotColumn()) != null)
- .forEach(namedColumn -> modelColsMap.get(namedColumn.getAliasDotColumn())
- .setColumnAlias(namedColumn.getName()));
- return modelColsMap;
- }
-
- private Map<String, ColumnDef> authColumns(NDataModel modelDesc, Set<String> tables, Set<String> columns) {
- Map<String, ColumnDef> modelColsMap = Maps.newHashMap();
- modelDesc.getAllTables().stream().filter(table -> tables.contains(table.getTableIdentity()))
- .forEach(tableRef -> tableRef.getColumns().stream()
- .filter(column -> columns.contains(column.getAliasDotName())).forEach(column -> {
- ColumnDef columnDef = new ColumnDef("dimension", tableRef.getAlias(), null,
- column.getName(), column.getDatatype(), true,
- column.getColumnDesc().isComputedColumn());
- String colName = tableRef.getAlias() + "." + column.getName();
- modelColsMap.put(colName, columnDef);
- }));
-
- // sync col alias
- modelDesc.getAllNamedColumns().stream()
- .filter(namedColumn -> modelColsMap.get(namedColumn.getAliasDotColumn()) != null)
- .forEach(namedColumn -> modelColsMap.get(namedColumn.getAliasDotColumn())
- .setColumnAlias(namedColumn.getName()));
+ model.getAllNamedColumns().stream() //
+ .filter(NDataModel.NamedColumn::isExist) //
+ .forEach(namedColumn -> {
+ ColumnDef columnDef = modelColsMap.get(namedColumn.getAliasDotColumn());
+ if (columnDef != null) {
+ columnDef.setColumnAlias(namedColumn.getName());
+ }
+ });
return modelColsMap;
}
@@ -394,17 +332,17 @@ public class SyncModelBuilder {
Set<String> hierarchyNameSet = Sets.newHashSet();
for (NAggregationGroup group : indexPlan.getRuleBasedIndex().getAggregationGroups()) {
SelectRule rule = group.getSelectRule();
- if (rule != null) {
- for (Integer[] hierarchyIds : rule.hierarchyDims) {
- if (hierarchyIds != null && hierarchyIds.length != 0) {
-
- String[] hierarchyNames = Arrays.stream(hierarchyIds)
- .map(id -> indexPlan.getModel().getColumnNameByColumnId(id)).toArray(String[]::new);
- String hierarchyNamesJoined = String.join(",", hierarchyNames);
- if (!hierarchyNameSet.contains(hierarchyNamesJoined)) {
- hierarchies.add(hierarchyNames);
- hierarchyNameSet.add(hierarchyNamesJoined);
- }
+ if (rule == null) {
+ continue;
+ }
+ for (Integer[] hierarchyIds : rule.hierarchyDims) {
+ if (ArrayUtils.isNotEmpty(hierarchyIds)) {
+ String[] hierarchyNames = Arrays.stream(hierarchyIds)
+ .map(id -> indexPlan.getModel().getColumnNameByColumnId(id)).toArray(String[]::new);
+ String hierarchyNamesJoined = String.join(",", hierarchyNames);
+ if (!hierarchyNameSet.contains(hierarchyNamesJoined)) {
+ hierarchies.add(hierarchyNames);
+ hierarchyNameSet.add(hierarchyNamesJoined);
}
}
}
diff --git a/src/query-common/src/main/java/org/apache/kylin/tool/bisync/model/ColumnDef.java b/src/query-common/src/main/java/org/apache/kylin/tool/bisync/model/ColumnDef.java
index 0871f6ea5b..c67f56a432 100644
--- a/src/query-common/src/main/java/org/apache/kylin/tool/bisync/model/ColumnDef.java
+++ b/src/query-common/src/main/java/org/apache/kylin/tool/bisync/model/ColumnDef.java
@@ -17,6 +17,17 @@
*/
package org.apache.kylin.tool.bisync.model;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+@Getter
+@Setter
public class ColumnDef {
private String role;
@@ -29,74 +40,11 @@ public class ColumnDef {
private String columnType;
- private boolean isHidden;
+ private boolean isHidden = true;
private boolean isComputedColumn;
- public ColumnDef(String role, String tableAlias, String columnAlias, String columnName, String columnType,
- boolean isHidden, boolean isComputedColumn) {
- this.role = role;
- this.tableAlias = tableAlias;
- this.columnAlias = columnAlias;
- this.columnName = columnName;
- this.columnType = columnType;
- this.isHidden = isHidden;
- this.isComputedColumn = isComputedColumn;
- }
-
- public String getRole() {
- return role;
- }
-
- public void setRole(String role) {
- this.role = role;
- }
-
- public String getTableAlias() {
- return tableAlias;
- }
-
- public void setTableAlias(String tableAlias) {
- this.tableAlias = tableAlias;
- }
-
- public boolean isHidden() {
- return isHidden;
- }
-
- public void setHidden(boolean hidden) {
- isHidden = hidden;
- }
-
- public String getColumnAlias() {
- return columnAlias;
- }
-
- public void setColumnAlias(String columnAlias) {
- this.columnAlias = columnAlias;
- }
-
- public String getColumnName() {
- return columnName;
- }
-
- public void setColumnName(String columnName) {
- this.columnName = columnName;
- }
-
- public String getColumnType() {
- return columnType;
- }
-
- public void setColumnType(String columnType) {
- this.columnType = columnType;
- }
-
- public boolean isComputedColumn() {
- return isComputedColumn;
- }
-
- public void setComputedColumn(boolean computedColumn) {
- isComputedColumn = computedColumn;
+ public boolean isDimension() {
+ return columnType.equalsIgnoreCase("nominal");
}
}
diff --git a/src/query-common/src/main/java/org/apache/kylin/tool/bisync/model/SyncModel.java b/src/query-common/src/main/java/org/apache/kylin/tool/bisync/model/SyncModel.java
index b8c8ddfafa..60151e17ef 100644
--- a/src/query-common/src/main/java/org/apache/kylin/tool/bisync/model/SyncModel.java
+++ b/src/query-common/src/main/java/org/apache/kylin/tool/bisync/model/SyncModel.java
@@ -21,9 +21,14 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
+import lombok.Getter;
+import lombok.Setter;
+
+@Getter
+@Setter
public class SyncModel {
- private String projectName;
+ private String project;
private String modelName;
@@ -38,69 +43,4 @@ public class SyncModel {
private List<MeasureDef> metrics;
private Set<String[]> hierarchies;
-
- public JoinTreeNode getJoinTree() {
- return joinTree;
- }
-
- public void setJoinTree(JoinTreeNode joinTree) {
- this.joinTree = joinTree;
- }
-
- public String getProjectName() {
- return projectName;
- }
-
- public void setProjectName(String projectName) {
- this.projectName = projectName;
- }
-
- public String getModelName() {
- return modelName;
- }
-
- public void setModelName(String modelName) {
- this.modelName = modelName;
- }
-
- public Map<String, ColumnDef> getColumnDefMap() {
- return columnDefMap;
- }
-
- public void setColumnDefMap(Map<String, ColumnDef> columnDefMap) {
- this.columnDefMap = columnDefMap;
- }
-
- public Set<String[]> getHierarchies() {
- return hierarchies;
- }
-
- public void setHierarchies(Set<String[]> hierarchies) {
- this.hierarchies = hierarchies;
- }
-
- public List<MeasureDef> getMetrics() {
- return metrics;
- }
-
- public void setMetrics(List<MeasureDef> metrics) {
- this.metrics = metrics;
- }
-
- public String getHost() {
- return host;
- }
-
- public void setHost(String host) {
- this.host = host;
- }
-
- public String getPort() {
- return port;
- }
-
- public void setPort(String port) {
- this.port = port;
- }
-
}
diff --git a/src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/TableauDataSourceConverter.java b/src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/TableauDataSourceConverter.java
index ced8616fa0..6540df9238 100644
--- a/src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/TableauDataSourceConverter.java
+++ b/src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/TableauDataSourceConverter.java
@@ -126,7 +126,7 @@ public class TableauDataSourceConverter implements BISyncModelConverter {
}
protected void fillTemplate(TableauDatasource tds, SyncModel syncModel) {
- fillConnectionProperties(tds, syncModel.getHost(), syncModel.getPort(), syncModel.getProjectName(),
+ fillConnectionProperties(tds, syncModel.getHost(), syncModel.getPort(), syncModel.getProject(),
syncModel.getModelName());
Map<String, Pair<Col, ColumnDef>> colMap = fillCols(tds, syncModel.getColumnDefMap());
fillColumns(tds, colMap);
diff --git a/src/second-storage/clickhouse-it/src/test/java/io/kyligence/kap/secondstorage/SecondStorageLockTest.java b/src/second-storage/clickhouse-it/src/test/java/io/kyligence/kap/secondstorage/SecondStorageLockTest.java
index 026a056965..03c191ab16 100644
--- a/src/second-storage/clickhouse-it/src/test/java/io/kyligence/kap/secondstorage/SecondStorageLockTest.java
+++ b/src/second-storage/clickhouse-it/src/test/java/io/kyligence/kap/secondstorage/SecondStorageLockTest.java
@@ -77,6 +77,7 @@ import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.common.util.RandomUtil;
import org.apache.kylin.common.util.Unsafe;
import org.apache.kylin.engine.spark.IndexDataConstructor;
+import org.apache.kylin.engine.spark.job.NSparkCubingJob;
import org.apache.kylin.job.SecondStorageCleanJobBuildParams;
import org.apache.kylin.job.SecondStorageJobParamUtil;
import org.apache.kylin.job.execution.AbstractExecutable;
@@ -92,6 +93,7 @@ import org.apache.kylin.metadata.cube.model.NDataSegment;
import org.apache.kylin.metadata.cube.model.NDataflow;
import org.apache.kylin.metadata.cube.model.NDataflowManager;
import org.apache.kylin.metadata.cube.model.NIndexPlanManager;
+import org.apache.kylin.metadata.epoch.EpochManager;
import org.apache.kylin.metadata.epoch.EpochOrchestrator;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.ManagementType;
@@ -191,10 +193,8 @@ import io.kyligence.kap.clickhouse.job.LoadContext;
import io.kyligence.kap.clickhouse.job.S3TableSource;
import io.kyligence.kap.clickhouse.management.ClickHouseConfigLoader;
import io.kyligence.kap.clickhouse.parser.ShowDatabasesParser;
-import org.apache.kylin.engine.spark.job.NSparkCubingJob;
import io.kyligence.kap.guava20.shaded.common.collect.ImmutableSet;
import io.kyligence.kap.guava20.shaded.common.collect.Lists;
-import org.apache.kylin.metadata.epoch.EpochManager;
import io.kyligence.kap.newten.clickhouse.ClickHouseSimpleITTestUtils;
import io.kyligence.kap.newten.clickhouse.ClickHouseUtils;
import io.kyligence.kap.newten.clickhouse.EmbeddedHttpServer;
@@ -349,6 +349,8 @@ public class SecondStorageLockTest implements JobWaiter {
ReflectionTestUtils.setField(modelBuildService, "modelService", modelService);
ReflectionTestUtils.setField(modelBuildService, "segmentHelper", segmentHelper);
ReflectionTestUtils.setField(modelBuildService, "aclEvaluate", aclEvaluate);
+ ReflectionTestUtils.setField(modelBuildService, "accessService", accessService);
+ ReflectionTestUtils.setField(modelBuildService, "userGroupService", userGroupService);
ReflectionTestUtils.setField(nModelController, "modelService", modelService);
ReflectionTestUtils.setField(nModelController, "fusionModelService", fusionModelService);
diff --git a/src/tool/src/test/java/org/apache/kylin/tool/bisync/SyncModelBuilderTest.java b/src/tool/src/test/java/org/apache/kylin/tool/bisync/SyncModelBuilderTest.java
index 5a5b635d90..d45eb7b97c 100644
--- a/src/tool/src/test/java/org/apache/kylin/tool/bisync/SyncModelBuilderTest.java
+++ b/src/tool/src/test/java/org/apache/kylin/tool/bisync/SyncModelBuilderTest.java
@@ -19,16 +19,19 @@
package org.apache.kylin.tool.bisync;
import java.io.ByteArrayOutputStream;
+import java.io.File;
import java.io.IOException;
-import java.io.InputStreamReader;
+import java.net.URL;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
+import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
+import org.apache.commons.io.FileUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
import org.apache.kylin.metadata.acl.AclTCR;
@@ -44,10 +47,9 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
-import com.google.common.base.Charsets;
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
-import com.google.common.io.CharStreams;
import lombok.val;
@@ -63,17 +65,22 @@ public class SyncModelBuilderTest extends NLocalFileMetadataTestCase {
this.cleanupTestMetadata();
}
+ private String getProject() {
+ return "default";
+ }
+
@Test
public void testBuildSyncModel() {
val project = "default";
val modelId = "cb596712-3a09-46f8-aea1-988b43fe9b6c";
val syncContext = SyncModelTestUtil.createSyncContext(project, modelId, KylinConfig.getInstanceFromEnv());
syncContext.setModelElement(SyncContext.ModelElement.ALL_COLS);
+ syncContext.setAdmin(true);
val syncModel = new SyncModelBuilder(syncContext).buildSourceSyncModel();
val df = NDataflowManager.getInstance(KylinConfig.getInstanceFromEnv(), project).getDataflow(modelId);
val model = df.getModel();
- Assert.assertEquals(project, syncModel.getProjectName());
+ Assert.assertEquals(project, syncModel.getProject());
Assert.assertEquals(model.getAlias(), syncModel.getModelName());
Assert.assertEquals("localhost", syncModel.getHost());
Assert.assertEquals("7070", syncModel.getPort());
@@ -98,9 +105,9 @@ public class SyncModelBuilderTest extends NLocalFileMetadataTestCase {
Assert.assertEquals(model.getAllMeasures().size(), syncModel.getMetrics().size());
val syncMeasure = syncModel.getMetrics().get(0).getMeasure();
- val modelMeasure = model.getAllMeasures().stream().filter(m -> m.getId() == syncMeasure.getId()).findFirst()
- .get();
- Assert.assertEquals(modelMeasure, syncMeasure);
+ val modelMeasure = model.getAllMeasures().stream().filter(m -> m.getId() == syncMeasure.getId()).findFirst();
+ Assert.assertTrue(modelMeasure.isPresent());
+ Assert.assertEquals(modelMeasure.get(), syncMeasure);
}
@Test
@@ -110,7 +117,7 @@ public class SyncModelBuilderTest extends NLocalFileMetadataTestCase {
val project = "default";
val modelId = "82fa7671-a935-45f5-8779-85703601f49a";
val syncContext = SyncModelTestUtil.createSyncContext(project, modelId, KylinConfig.getInstanceFromEnv());
- prepareBasic(project);
+ prepareBasic();
Set<String> allAuthTables = Sets.newHashSet();
Set<String> allAuthColumns = Sets.newHashSet();
@@ -157,6 +164,7 @@ public class SyncModelBuilderTest extends NLocalFileMetadataTestCase {
measures.add("TEST_COUNT_DISTINCT_BITMAP");
measures.add("GVM_PERCENTILE");
+ syncContext.setAdmin(false);
syncContext.setModelElement(SyncContext.ModelElement.CUSTOM_COLS);
TableauDatasourceModel datasource = (TableauDatasourceModel) BISyncTool
.dumpHasPermissionToBISyncModel(syncContext, allAuthTables, newAuthColumns, dimensions, measures);
@@ -165,35 +173,40 @@ public class SyncModelBuilderTest extends NLocalFileMetadataTestCase {
Assert.assertEquals(getExpectedTds("/bisync_tableau/nmodel_full_measure_test.connector_permission.tds"),
outStream.toString(Charset.defaultCharset().name()));
+ syncContext.setAdmin(true);
+ syncContext.setModelElement(SyncContext.ModelElement.CUSTOM_COLS);
TableauDatasourceModel datasource1 = (TableauDatasourceModel) BISyncTool.dumpBISyncModel(syncContext,
- dimensions, null);
+ dimensions, ImmutableList.of());
ByteArrayOutputStream outStream1 = new ByteArrayOutputStream();
datasource1.dump(outStream1);
Assert.assertEquals(
getExpectedTds("/bisync_tableau/nmodel_full_measure_test.connector_permission_no_measure.tds"),
outStream1.toString(Charset.defaultCharset().name()));
+ syncContext.setAdmin(true);
syncContext.setModelElement(SyncContext.ModelElement.AGG_INDEX_COL);
- TableauDatasourceModel datasource2 = (TableauDatasourceModel) BISyncTool.dumpBISyncModel(syncContext, null,
- null);
+ TableauDatasourceModel datasource2 = (TableauDatasourceModel) BISyncTool.dumpBISyncModel(syncContext,
+ ImmutableList.of(), ImmutableList.of());
ByteArrayOutputStream outStream2 = new ByteArrayOutputStream();
datasource2.dump(outStream2);
Assert.assertEquals(
getExpectedTds("/bisync_tableau/nmodel_full_measure_test.connector_permission_agg_index_col.tds"),
outStream2.toString(Charset.defaultCharset().name()));
+ syncContext.setAdmin(true);
syncContext.setModelElement(SyncContext.ModelElement.AGG_INDEX_AND_TABLE_INDEX_COL);
- TableauDatasourceModel datasource3 = (TableauDatasourceModel) BISyncTool.dumpBISyncModel(syncContext, null,
- null);
+ TableauDatasourceModel datasource3 = (TableauDatasourceModel) BISyncTool.dumpBISyncModel(syncContext,
+ ImmutableList.of(), ImmutableList.of());
ByteArrayOutputStream outStream3 = new ByteArrayOutputStream();
datasource3.dump(outStream3);
Assert.assertEquals(
getExpectedTds("/bisync_tableau/nmodel_full_measure_test.connector_permission_agg_index_col.tds"),
outStream3.toString(Charset.defaultCharset().name()));
+ syncContext.setAdmin(true);
syncContext.setModelElement(SyncContext.ModelElement.ALL_COLS);
- TableauDatasourceModel datasource4 = (TableauDatasourceModel) BISyncTool.dumpBISyncModel(syncContext, null,
- null);
+ TableauDatasourceModel datasource4 = (TableauDatasourceModel) BISyncTool.dumpBISyncModel(syncContext,
+ ImmutableList.of(), ImmutableList.of());
ByteArrayOutputStream outStream4 = new ByteArrayOutputStream();
datasource4.dump(outStream4);
Assert.assertEquals(getExpectedTds("/bisync_tableau/nmodel_full_measure_test.connector_permission_all_col.tds"),
@@ -206,7 +219,8 @@ public class SyncModelBuilderTest extends NLocalFileMetadataTestCase {
val modelId = "cb596712-3a09-46f8-aea1-988b43fe9b6c";
val syncContext = SyncModelTestUtil.createSyncContext(project, modelId, KylinConfig.getInstanceFromEnv());
syncContext.setModelElement(SyncContext.ModelElement.ALL_COLS);
- prepareBasic(project);
+ syncContext.setAdmin(true);
+ prepareBasic();
TableauDatasourceModel datasource = (TableauDatasourceModel) BISyncTool.dumpToBISyncModel(syncContext);
ByteArrayOutputStream outStream = new ByteArrayOutputStream();
@@ -230,6 +244,7 @@ public class SyncModelBuilderTest extends NLocalFileMetadataTestCase {
val syncContext1 = SyncModelTestUtil.createSyncContext(project, "89af4ee2-2cdb-4b07-b39e-4c29856309aa",
KylinConfig.getInstanceFromEnv());
+ syncContext1.setAdmin(true);
syncContext1.setModelElement(SyncContext.ModelElement.AGG_INDEX_COL);
TableauDatasourceModel datasource3 = (TableauDatasourceModel) BISyncTool.dumpToBISyncModel(syncContext1);
ByteArrayOutputStream outStream3 = new ByteArrayOutputStream();
@@ -254,7 +269,7 @@ public class SyncModelBuilderTest extends NLocalFileMetadataTestCase {
groups.add("g1");
val project = "default";
val modelId = "741ca86a-1f13-46da-a59f-95fb68615e3a";
- prepareBasic(project);
+ prepareBasic();
Set<String> allAuthTables = Sets.newHashSet();
Set<String> allAuthColumns = Sets.newHashSet();
AclTCRManager aclTCRManager = AclTCRManager.getInstance(KylinConfig.getInstanceFromEnv(), project);
@@ -301,6 +316,7 @@ public class SyncModelBuilderTest extends NLocalFileMetadataTestCase {
measures.add("GVM_PERCENTILE");
cc_syncContext.setModelElement(SyncContext.ModelElement.CUSTOM_COLS);
+ cc_syncContext.setAdmin(false);
TableauDatasourceModel datasource3 = (TableauDatasourceModel) BISyncTool
.dumpHasPermissionToBISyncModel(cc_syncContext, allAuthTables, newAuthColumns, dimensions, measures);
ByteArrayOutputStream outStream3 = new ByteArrayOutputStream();
@@ -309,6 +325,7 @@ public class SyncModelBuilderTest extends NLocalFileMetadataTestCase {
outStream3.toString(Charset.defaultCharset().name()));
cc_syncContext.setModelElement(SyncContext.ModelElement.CUSTOM_COLS);
+ cc_syncContext.setAdmin(true);
TableauDatasourceModel datasource1 = (TableauDatasourceModel) BISyncTool.dumpBISyncModel(cc_syncContext,
dimensions, measures);
ByteArrayOutputStream outStream1 = new ByteArrayOutputStream();
@@ -323,7 +340,7 @@ public class SyncModelBuilderTest extends NLocalFileMetadataTestCase {
groups.add("g1");
val project = "default";
val modelId = "82fa7671-a935-45f5-8779-85703601f49a";
- prepareBasic(project);
+ prepareBasic();
Set<String> allAuthTables = Sets.newHashSet();
Set<String> allAuthColumns = Sets.newHashSet();
AclTCRManager aclTCRManager = AclTCRManager.getInstance(KylinConfig.getInstanceFromEnv(), project);
@@ -337,6 +354,7 @@ public class SyncModelBuilderTest extends NLocalFileMetadataTestCase {
}
val cc_syncContext = SyncModelTestUtil.createSyncContext(project, modelId, KylinConfig.getInstanceFromEnv());
cc_syncContext.setModelElement(SyncContext.ModelElement.ALL_COLS);
+ cc_syncContext.setAdmin(true);
Set<String> newAuthColumns = convertColumns(cc_syncContext.getDataflow().getModel(), allAuthColumns);
List<String> dimensions = Lists.newArrayList();
@@ -372,6 +390,7 @@ public class SyncModelBuilderTest extends NLocalFileMetadataTestCase {
measures.add("GVM_PERCENTILE");
cc_syncContext.setModelElement(SyncContext.ModelElement.CUSTOM_COLS);
+ cc_syncContext.setAdmin(false);
TableauDatasourceModel datasource3 = (TableauDatasourceModel) BISyncTool
.dumpHasPermissionToBISyncModel(cc_syncContext, allAuthTables, newAuthColumns, dimensions, measures);
ByteArrayOutputStream outStream3 = new ByteArrayOutputStream();
@@ -380,6 +399,7 @@ public class SyncModelBuilderTest extends NLocalFileMetadataTestCase {
outStream3.toString(Charset.defaultCharset().name()));
cc_syncContext.setModelElement(SyncContext.ModelElement.AGG_INDEX_COL);
+ cc_syncContext.setAdmin(false);
TableauDatasourceModel datasource4 = (TableauDatasourceModel) BISyncTool.dumpHasPermissionToBISyncModel(
cc_syncContext, allAuthTables, newAuthColumns, new ArrayList<>(), new ArrayList<>());
ByteArrayOutputStream outStream4 = new ByteArrayOutputStream();
@@ -388,6 +408,7 @@ public class SyncModelBuilderTest extends NLocalFileMetadataTestCase {
outStream4.toString(Charset.defaultCharset().name()));
cc_syncContext.setModelElement(SyncContext.ModelElement.AGG_INDEX_AND_TABLE_INDEX_COL);
+ cc_syncContext.setAdmin(false);
TableauDatasourceModel datasource5 = (TableauDatasourceModel) BISyncTool.dumpHasPermissionToBISyncModel(
cc_syncContext, allAuthTables, newAuthColumns, new ArrayList<>(), new ArrayList<>());
ByteArrayOutputStream outStream5 = new ByteArrayOutputStream();
@@ -396,6 +417,7 @@ public class SyncModelBuilderTest extends NLocalFileMetadataTestCase {
outStream5.toString(Charset.defaultCharset().name()));
cc_syncContext.setModelElement(SyncContext.ModelElement.ALL_COLS);
+ cc_syncContext.setAdmin(false);
TableauDatasourceModel datasource6 = (TableauDatasourceModel) BISyncTool.dumpHasPermissionToBISyncModel(
cc_syncContext, allAuthTables, newAuthColumns, new ArrayList<>(), new ArrayList<>());
ByteArrayOutputStream outStream6 = new ByteArrayOutputStream();
@@ -410,7 +432,7 @@ public class SyncModelBuilderTest extends NLocalFileMetadataTestCase {
groups.add("g1");
val project = "default";
val modelId = "82fa7671-a935-45f5-8779-85703601f49a";
- prepareBasicNoHierarchies(project);
+ prepareBasicNoHierarchies();
Set<String> allAuthTables = Sets.newHashSet();
Set<String> allAuthColumns = Sets.newHashSet();
AclTCRManager aclTCRManager = AclTCRManager.getInstance(KylinConfig.getInstanceFromEnv(), project);
@@ -423,7 +445,6 @@ public class SyncModelBuilderTest extends NLocalFileMetadataTestCase {
allAuthColumns.addAll(auths.getColumns());
}
val cc_syncContext = SyncModelTestUtil.createSyncContext(project, modelId, KylinConfig.getInstanceFromEnv());
- cc_syncContext.setModelElement(SyncContext.ModelElement.ALL_COLS);
Set<String> newAuthColumns = convertColumns(cc_syncContext.getDataflow().getModel(), allAuthColumns);
List<String> dimensions = Lists.newArrayList();
//"ORDER_ID", "PRICE", "CAL_DT", "PRICE", "ITEM_COUNT"
@@ -457,6 +478,7 @@ public class SyncModelBuilderTest extends NLocalFileMetadataTestCase {
measures.add("TEST_COUNT_DISTINCT_BITMAP");
measures.add("GVM_PERCENTILE");
cc_syncContext.setModelElement(SyncContext.ModelElement.CUSTOM_COLS);
+ cc_syncContext.setAdmin(false);
TableauDatasourceModel datasource3 = (TableauDatasourceModel) BISyncTool
.dumpHasPermissionToBISyncModel(cc_syncContext, allAuthTables, newAuthColumns, dimensions, measures);
ByteArrayOutputStream outStream3 = new ByteArrayOutputStream();
@@ -466,11 +488,13 @@ public class SyncModelBuilderTest extends NLocalFileMetadataTestCase {
}
private String getExpectedTds(String path) throws IOException {
- return CharStreams.toString(new InputStreamReader(getClass().getResourceAsStream(path), Charsets.UTF_8));
+ URL resource = getClass().getResource(path);
+ String fullPath = Objects.requireNonNull(resource).getPath();
+ return FileUtils.readFileToString(new File(fullPath), Charset.defaultCharset());
}
- private void prepareBasic(String project) {
- AclTCRManager manager = AclTCRManager.getInstance(getTestConfig(), project);
+ private void prepareBasic() {
+ AclTCRManager manager = AclTCRManager.getInstance(getTestConfig(), getProject());
AclTCR u1a1 = new AclTCR();
AclTCR.Table u1t1 = new AclTCR.Table();
@@ -499,8 +523,8 @@ public class SyncModelBuilderTest extends NLocalFileMetadataTestCase {
manager.updateAclTCR(g1a1, "g1", false);
}
- private void prepareBasicNoHierarchies(String project) {
- AclTCRManager manager = AclTCRManager.getInstance(getTestConfig(), project);
+ private void prepareBasicNoHierarchies() {
+ AclTCRManager manager = AclTCRManager.getInstance(getTestConfig(), getProject());
AclTCR u1a1 = new AclTCR();
AclTCR.Table u1t1 = new AclTCR.Table();
@@ -554,8 +578,10 @@ public class SyncModelBuilderTest extends NLocalFileMetadataTestCase {
val syncContext1 = SyncModelTestUtil.createSyncContext(project, model1Id, KylinConfig.getInstanceFromEnv());
val syncContext2 = SyncModelTestUtil.createSyncContext(project, model2Id, KylinConfig.getInstanceFromEnv());
syncContext1.setModelElement(SyncContext.ModelElement.AGG_INDEX_AND_TABLE_INDEX_COL);
+ syncContext1.setAdmin(true);
syncContext2.setModelElement(SyncContext.ModelElement.AGG_INDEX_AND_TABLE_INDEX_COL);
- prepareBasic(project);
+ syncContext2.setAdmin(true);
+ prepareBasic();
TableauDatasourceModel datasource1 = (TableauDatasourceModel) BISyncTool.dumpToBISyncModel(syncContext1);
ByteArrayOutputStream outStream1 = new ByteArrayOutputStream();
diff --git a/src/tool/src/test/java/org/apache/kylin/tool/bisync/SyncModelTestUtil.java b/src/tool/src/test/java/org/apache/kylin/tool/bisync/SyncModelTestUtil.java
index 91e3894873..28cf1cf270 100644
--- a/src/tool/src/test/java/org/apache/kylin/tool/bisync/SyncModelTestUtil.java
+++ b/src/tool/src/test/java/org/apache/kylin/tool/bisync/SyncModelTestUtil.java
@@ -33,6 +33,7 @@ public class SyncModelTestUtil {
syncContext.setPort(7070);
syncContext.setDataflow(NDataflowManager.getInstance(kylinConfig, project).getDataflow(modelId));
syncContext.setKylinConfig(kylinConfig);
+ syncContext.setAdmin(true);
return syncContext;
}
}