You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by xx...@apache.org on 2022/12/13 10:25:26 UTC
[kylin] 19/25: KYLIN-5314 Check name conflict before export tds file
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 97bf6b8bd557b10868855f90bd8f2d777bddfbf6
Author: Pengfei Zhan <pe...@kyligence.io>
AuthorDate: Wed Oct 12 20:50:19 2022 +0800
KYLIN-5314 Check name conflict before export tds file
---
.../org/apache/kylin/common/KylinConfigBase.java | 5 -
.../common/exception/code/ErrorCodeServer.java | 2 +
.../java/org/apache/kylin/common/msg/Message.java | 2 -
.../resources/kylin_error_msg_conf_cn.properties | 2 +
.../resources/kylin_error_msg_conf_en.properties | 2 +
.../main/resources/kylin_errorcode_conf.properties | 2 +
.../rest/controller/v2/NModelControllerV2.java | 1 -
.../kylin/rest/service/AbstractModelService.java | 8 +-
.../apache/kylin/rest/service/ModelService.java | 111 ------
.../kylin/rest/service/ModelServiceTest.java | 398 +--------------------
10 files changed, 16 insertions(+), 517 deletions(-)
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index c1a6e84bb3..33f7f0b5f1 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -3637,11 +3637,6 @@ public abstract class KylinConfigBase implements Serializable {
return Boolean.parseBoolean(getOptional("kylin.build.resource.skip-resource-check", FALSE));
}
- public boolean useTableIndexAnswerSelectStarEnabled() {
- return Boolean.parseBoolean(getOptional("kylin.query.use-tableindex-answer-select-star.enabled", FALSE));
- }
-
-
public int getSecondStorageSkippingIndexGranularity() {
int granularity = Integer.parseInt(getOptional("kylin.second-storage.skipping-index.granularity", "3"));
return granularity <= 0 ? 3 : granularity;
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/exception/code/ErrorCodeServer.java b/src/core-common/src/main/java/org/apache/kylin/common/exception/code/ErrorCodeServer.java
index 3a0ca0195d..c8abaa1142 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/exception/code/ErrorCodeServer.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/exception/code/ErrorCodeServer.java
@@ -32,6 +32,8 @@ public enum ErrorCodeServer implements ErrorCodeProducer {
MODEL_NAME_DUPLICATE("KE-010002206"),
SIMPLIFIED_MEASURES_MISSING_ID("KE-010002207"),
MODEL_NOT_EXIST_SEGMENTS("KE-010002208"),
+ MODEL_TDS_EXPORT_DIM_COL_AND_MEASURE_NAME_CONFLICT("KE-010002301"),
+ MODEL_TDS_EXPORT_COLUMN_AND_MEASURE_NAME_CONFLICT("KE-010002302"),
// 100252XX Cube
CUBE_NOT_EXIST("KE-010025201"),
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/msg/Message.java b/src/core-common/src/main/java/org/apache/kylin/common/msg/Message.java
index cad28ace11..d64b7179c1 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/msg/Message.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/msg/Message.java
@@ -38,8 +38,6 @@ public class Message {
private static final String NON_EXISTEN_MODEL = "Model %s doesn't exist. Please confirm and try again later.";
private static final String LACK_PROJECT = "Please fill in the project parameters.";
private static final String NON_EXIST_PROJECT = "Project %s doesn't exist. Please confirm and try again later.";
- private static final String DUP_MODCOL_MEASURE_NAME = "There are duplicated names among model column %s and measure name %s. Cannot export a valid TDS file. Please correct the duplicated names and try again.";
- private static final String DUP_DIMCOL_MEASURE_NAME = "There are duplicated names among dimension column %s and measure name %s. Cannot export a valid TDS file. Please correct the duplicated names and try again.";
private static final String MODIFY_PERMISSION_OF_SUPER_ADMIN = "Super Admin’s permission can’t be modified.";
private static final String ILLEGAL_AUTHORIZING_USER = "Unable to modify. Only Super Admin or System Admin with query permission can modify query permission.";
private static final String GRANT_TO_NON_SYSTEM_ADMIN = "Unable to modify. You can only modify the permission of the System Admin.";
diff --git a/src/core-common/src/main/resources/kylin_error_msg_conf_cn.properties b/src/core-common/src/main/resources/kylin_error_msg_conf_cn.properties
index e6a11efcc1..759514ee40 100644
--- a/src/core-common/src/main/resources/kylin_error_msg_conf_cn.properties
+++ b/src/core-common/src/main/resources/kylin_error_msg_conf_cn.properties
@@ -32,6 +32,8 @@ KE-010002205=无效的模型名称 “%s”。请使用字母、数字或下划
KE-010002206=模型 “%s” 已存在。请重新命名。
KE-010002207=修改模型时，simplified_measures 参数中的每个度量必须传入 id 值。请为以下度量传入 id 之后重试：%s。
KE-010002208=模型上线必须存在 Segment。请重新输入。
+KE-010002301=维度的列名 %s 与度量名 %s 重复，无法导出 TDS。请去除重名后再重试。
+KE-010002302=模型中的列名 %s 与度量名 %s 重复，无法导出 TDS。请去除重名后再重试。
## 100252XX Cube
KE-010025201=无法找到相关 Cube。
diff --git a/src/core-common/src/main/resources/kylin_error_msg_conf_en.properties b/src/core-common/src/main/resources/kylin_error_msg_conf_en.properties
index b4130c7b2a..02ecccdb25 100644
--- a/src/core-common/src/main/resources/kylin_error_msg_conf_en.properties
+++ b/src/core-common/src/main/resources/kylin_error_msg_conf_en.properties
@@ -32,6 +32,8 @@ KE-010002205=The model name "%s" is invalid. Please use letters, numbers and und
KE-010002206=Model "%s" already exists. Please rename it.
KE-010002207=When modifying model, each measure id is required in simplified_measures parameter. Please pass ids for following measures and try again: %s.
KE-010002208=The online model must have a segment. Please re-enter.
+KE-010002301=There are duplicated names among dimension column %s and measure name %s. Cannot export a valid TDS file. Please correct the duplicated names and try again.
+KE-010002302=There are duplicated names among model column %s and measure name %s. Cannot export a valid TDS file. Please correct the duplicated names and try again.
## 100252XX Cube
KE-010025201=Can't find the cube.
diff --git a/src/core-common/src/main/resources/kylin_errorcode_conf.properties b/src/core-common/src/main/resources/kylin_errorcode_conf.properties
index 5b506d68fb..7f07828a6a 100644
--- a/src/core-common/src/main/resources/kylin_errorcode_conf.properties
+++ b/src/core-common/src/main/resources/kylin_errorcode_conf.properties
@@ -33,6 +33,8 @@ KE-010002205
KE-010002206
KE-010002207
KE-010002208
+KE-010002301
+KE-010002302
## 100252XX Cube
KE-010025201
diff --git a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/v2/NModelControllerV2.java b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/v2/NModelControllerV2.java
index 224b68c6a8..ac9d46580f 100644
--- a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/v2/NModelControllerV2.java
+++ b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/v2/NModelControllerV2.java
@@ -21,7 +21,6 @@ import static org.apache.kylin.common.constant.HttpConstant.HTTP_VND_APACHE_KYLI
import static org.apache.kylin.common.exception.code.ErrorCodeServer.MODEL_ID_NOT_EXIST;
import java.util.ArrayList;
-import java.util.HashMap;
import java.util.List;
import java.util.Map;
diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/AbstractModelService.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/AbstractModelService.java
index b46314b2e1..849214bd34 100644
--- a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/AbstractModelService.java
+++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/AbstractModelService.java
@@ -57,7 +57,7 @@ public class AbstractModelService extends BasicService {
@Autowired
public AccessService accessService;
- public void checkModelPermission(String project, String modelId) {
+ public final void checkModelPermission(String project, String modelId) {
String userName = aclEvaluate.getCurrentUserName();
Set<String> groups = getCurrentUserGroups();
if (AclPermissionUtil.isAdmin() || AclPermissionUtil.isAdminInProject(project, groups)) {
@@ -94,7 +94,7 @@ public class AbstractModelService extends BasicService {
});
}
- public NDataModel getModelById(String modelId, String project) {
+ public final NDataModel getModelById(String modelId, String project) {
NDataModelManager modelManager = getManager(NDataModelManager.class, project);
NDataModel nDataModel = modelManager.getDataModelDesc(modelId);
if (null == nDataModel) {
@@ -103,7 +103,7 @@ public class AbstractModelService extends BasicService {
return nDataModel;
}
- public NDataModel getModelByAlias(String modelAlias, String project) {
+ public final NDataModel getModelByAlias(String modelAlias, String project) {
NDataModelManager modelManager = getManager(NDataModelManager.class, project);
NDataModel nDataModel = modelManager.getDataModelDescByAlias(modelAlias);
if (null == nDataModel) {
@@ -112,7 +112,7 @@ public class AbstractModelService extends BasicService {
return nDataModel;
}
- public Set<String> listAllModelIdsInProject(String project) {
+ public final Set<String> listAllModelIdsInProject(String project) {
NDataModelManager dataModelManager = getManager(NDataModelManager.class, project);
return dataModelManager.listAllModelIds();
}
diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
index b836937033..07cab15306 100644
--- a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
+++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
@@ -4313,117 +4313,6 @@ public class ModelService extends AbstractModelService implements TableModelSupp
return response;
}
- public void checkCCEmpty(ModelRequest modelRequest) {
- List<ComputedColumnDesc> ccList = modelRequest.getComputedColumnDescs();
- if (CollectionUtils.isEmpty(ccList)) {
- return;
- }
- boolean matchEmpty = ccList.stream()
- .anyMatch(cc -> StringUtils.isEmpty(cc.getColumnName()) || StringUtils.isEmpty(cc.getExpression()));
- if (matchEmpty) {
- throw new KylinException(COMPUTED_COLUMN_NAME_OR_EXPR_EMPTY);
- }
- }
-
- public Pair<ModelRequest, ComputedColumnConflictResponse> checkCCConflict(ModelRequest modelRequest) {
- String project = modelRequest.getProject();
- validatePartitionDateColumn(modelRequest);
-
- val dataModel = semanticUpdater.convertToDataModel(modelRequest);
- val modelManager = getManager(NDataModelManager.class, project);
- val ccRelatedModels = modelManager.getCCRelatedModels(dataModel);
- // check cc conflict and return ccConflictInfo
- val ccConflictInfo = dataModel.checkCCFailAtEnd(getConfig(), project, ccRelatedModels, true);
- boolean autoAdjust = modelRequest.isComputedColumnNameAutoAdjust();
-
- if (ccConflictInfo.noneConflict()) {
- // No conflict, return
- return Pair.newPair(modelRequest, new ComputedColumnConflictResponse());
- }
- if (ccConflictInfo.hasSameNameConflict()) {
- // have sameNameDiffExpr Conflict, all conflict messages need to be thrown
- val response = handleOnConflictResponse(ccConflictInfo.getAllConflictException());
- throw new KylinException(COMPUTED_COLUMN_CONFLICT).withData(response);
- }
- // have sameExprDiffExprConflict Conflict
- if (!autoAdjust) {
- // AutoAdjust = false
- // SameExpr conflict messages need to be thrown
- val response = handleOnConflictResponse(ccConflictInfo.getSameExprConflictException());
- throw new KylinException(COMPUTED_COLUMN_CONFLICT).withData(response);
- }
- // AutoAdjust = true
- List<ComputedColumnDesc> inputCCDescList = Lists.newArrayList(modelRequest.getComputedColumnDescs());
- // deal with conflicts
- val pair = ccConflictInfo.getAdjustedCCList(inputCCDescList);
- val adjustExceptions = pair.getSecond().stream() //
- .map(ComputedColumnUtil.CCConflictDetail::getAdjustKylinException).collect(Collectors.toList());
- ModelRequest resultModelRequest = adjustModelRequestCCName(modelRequest, pair);
-
- return Pair.newPair(resultModelRequest, handleOnConflictResponse(adjustExceptions));
- }
-
- public ModelRequest adjustModelRequestCCName(ModelRequest modelRequest,
- Pair<List<ComputedColumnDesc>, List<ComputedColumnUtil.CCConflictDetail>> pair) {
- val adjustDetails = pair.getSecond();
- // adjust cc name
- modelRequest.setComputedColumnDescs(pair.getFirst());
-
- val dimensions = modelRequest.getSimplifiedDimensions();
- val measures = modelRequest.getSimplifiedMeasures();
- for (val detail : adjustDetails) {
- String newCCFullName = detail.getNewCC().getFullName();
- String existingCCFullName = detail.getExistingCC().getFullName();
-
- // adjust dimensions
- dimensions.stream() //
- .filter(NDataModel.NamedColumn::isExist) //
- // column equals
- .filter(d -> StringUtils.equalsIgnoreCase(d.getAliasDotColumn(), newCCFullName))
- .forEach(d -> d.setAliasDotColumn(existingCCFullName));
-
- // adjust measures
- measures.forEach(m -> m.getParameterValue().stream() //
- // type = column
- .filter(pr -> StringUtils.equalsIgnoreCase(pr.getType(), PARAMETER_TYPE_COLUMN))
- // value equals
- .filter(pr -> StringUtils.equalsIgnoreCase(pr.getValue(), newCCFullName))
- .forEach(pr -> pr.setValue(existingCCFullName)));
- }
-
- // adjust filter condition
- String filterCondition = modelRequest.getFilterCondition();
- if (StringUtils.isEmpty(filterCondition)) {
- return modelRequest;
- }
- for (val detail : adjustDetails) {
- String newCCFullName = detail.getNewCC().getFullName();
- String existingCCFullName = detail.getExistingCC().getFullName();
- if (StringUtils.containsIgnoreCase(filterCondition, newCCFullName)) {
- filterCondition = replaceAllIgnoreCase(filterCondition, newCCFullName, existingCCFullName);
- }
- }
- modelRequest.setFilterCondition(filterCondition);
- return modelRequest;
- }
-
- public String replaceAllIgnoreCase(String input, String regex, String replacement) {
- return Pattern.compile(regex, Pattern.CASE_INSENSITIVE).matcher(input).replaceAll(replacement);
- }
-
- public ComputedColumnConflictResponse handleOnConflictResponse(List<KylinException> exceptionList) {
- val response = new ComputedColumnConflictResponse();
- exceptionList.stream() //
- .filter(Objects::nonNull) //
- .forEach(e -> {
- val producer = e.getErrorCodeProducer();
- val code = producer.getErrorCode().getCode();
- val msg = producer.getMsg(e.getArgs());
- response.addConflictDetail(code, msg);
- });
- return response;
- }
-
@Override
public void onUpdateBrokenModel(NDataModel model, AffectedModelContext removeAffectedModel,
AffectedModelContext changeTypeAffectedModel, String projectName) throws Exception {
diff --git a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
index 696347e093..e9dc0f22c3 100644
--- a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
+++ b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
@@ -522,8 +522,7 @@ public class ModelServiceTest extends SourceTestCase {
Assert.assertEquals(1, models.size());
NDataModelResponse model = models.get(0);
Assert.assertTrue(model.getSimpleTables().stream().map(SimplifiedTableResponse::getColumns)
- .flatMap(List::stream)
- .anyMatch(SimplifiedColumnResponse::isComputedColumn));
+ .flatMap(List::stream).anyMatch(SimplifiedColumnResponse::isComputedColumn));
}
@Test
@@ -564,8 +563,7 @@ public class ModelServiceTest extends SourceTestCase {
NIndexPlanManager indexPlanManager = NIndexPlanManager.getInstance(getTestConfig(), getProject());
val indexPlan = indexPlanManager.getIndexPlan(modelId);
indexPlanManager.updateIndexPlan(modelId, copyForWrite -> {
- copyForWrite.markIndexesToBeDeleted(modelId,
- new HashSet<>(indexPlan.getAllLayouts()));
+ copyForWrite.markIndexesToBeDeleted(modelId, new HashSet<>(indexPlan.getAllLayouts()));
copyForWrite.getIndexes().clear();
});
NDataflowManager dataflowManager = NDataflowManager.getInstance(KylinConfig.getInstanceFromEnv(), "default");
@@ -5083,394 +5081,6 @@ public class ModelServiceTest extends SourceTestCase {
}
}
- @Test
- public void testExportTDSByAdmin() throws Exception {
- val project = "default";
- val modelId = "cb596712-3a09-46f8-aea1-988b43fe9b6c";
- prepareBasic(project);
- List<String> dimensions = Lists.newArrayList();
- dimensions.add("DEFAULT.TEST_MEASURE.FLAG");
- dimensions.add("DEFAULT.TEST_MEASURE.PRICE1");
- dimensions.add("DEFAULT.TEST_MEASURE.ID1");
- List<String> measurs = Lists.newArrayList();
- measurs.add("COUNT_STAR");
- measurs.add("SUM_1");
- SyncContext syncContext = modelService.getADMINSyncContext(project, modelId,
- SyncContext.BI.TABLEAU_CONNECTOR_TDS, SyncContext.ModelElement.CUSTOM_COLS, "localhost", 8080);
- TableauDatasourceModel datasource1 = (TableauDatasourceModel) modelService
- .exportTDSDimensionsAndMeasuresByAdmin(syncContext, dimensions, measurs);
- ByteArrayOutputStream outStream4 = new ByteArrayOutputStream();
- datasource1.dump(outStream4);
- Assert.assertEquals(getExpectedTds("/bisync_tableau/nmodel_full_measure_test.connector_admin.tds"),
- outStream4.toString(Charset.defaultCharset().name()));
- }
-
- @Test
- public void testExportTDSByUser() throws Exception {
- val project = "default";
- val modelId = "cb596712-3a09-46f8-aea1-988b43fe9b6c";
- prepareBasic(project);
- List<String> dimensions = Lists.newArrayList();
- dimensions.add("TEST_MEASURE.ID1");
- dimensions.add("TEST_MEASURE.ID2");
- dimensions.add("TEST_MEASURE.ID3");
- dimensions.add("TEST_MEASURE1.ID1");
- dimensions.add("TEST_MEASURE1.NAME1");
- dimensions.add("TEST_MEASURE1.NAME2");
- dimensions.add("TEST_MEASURE1.NAME3");
- List<String> measurs = Lists.newArrayList();
- measurs.add("COUNT_STAR");
- measurs.add("SUM_1");
- SecurityContextHolder.getContext()
- .setAuthentication(new TestingAuthenticationToken("u1", "ANALYST", Constant.ROLE_ANALYST));
- SyncContext syncContext = modelService.getSyncContext(project, modelId, SyncContext.BI.TABLEAU_CONNECTOR_TDS,
- SyncContext.ModelElement.CUSTOM_COLS, "localhost", 8080);
- TableauDatasourceModel datasource1 = (TableauDatasourceModel) modelService
- .exportTDSDimensionsAndMeasuresByNormalUser(syncContext, dimensions, measurs);
- ByteArrayOutputStream outStream4 = new ByteArrayOutputStream();
- datasource1.dump(outStream4);
- Assert.assertEquals(getExpectedTds("/bisync_tableau/nmodel_full_measure_test.connector_user.tds"),
- outStream4.toString(Charset.defaultCharset().name()));
- }
-
- @Test
- public void testExportTDSByUserAndElement() throws Exception {
- val project = "default";
- val modelId = "cb596712-3a09-46f8-aea1-988b43fe9b6c";
- prepareBasic(project);
- try {
- SecurityContextHolder.getContext()
- .setAuthentication(new TestingAuthenticationToken("u1", "ANALYST", Constant.ROLE_ANALYST));
- SyncContext syncContext = modelService.getSyncContext(project, modelId,
- SyncContext.BI.TABLEAU_CONNECTOR_TDS, SyncContext.ModelElement.AGG_INDEX_COL, "localhost", 8080);
- TableauDatasourceModel datasource1 = (TableauDatasourceModel) modelService
- .exportTDSDimensionsAndMeasuresByNormalUser(syncContext, null, null);
- ByteArrayOutputStream outStream4 = new ByteArrayOutputStream();
- datasource1.dump(outStream4);
- Assert.assertEquals(
- getExpectedTds("/bisync_tableau/nmodel_full_measure_test.connector_user_agg_index_col.tds"),
- outStream4.toString(Charset.defaultCharset().name()));
-
- TableauDatasourceModel datasource = (TableauDatasourceModel) modelService
- .exportTDSDimensionsAndMeasuresByNormalUser(syncContext, new ArrayList<>(), new ArrayList<>());
- } finally {
- SecurityContextHolder.getContext()
- .setAuthentication(new TestingAuthenticationToken("ADMIN", "ADMIN", Constant.ROLE_ADMIN));
- }
- }
-
- @Test
- public void testCheckModelExportPermissionException() {
- val project = "default";
- val modelId = "cb596712-3a09-46f8-aea1-988b43fe9b6c";
- prepareBasic(project);
- modelService.getADMINSyncContext(project, modelId, SyncContext.BI.TABLEAU_CONNECTOR_TDS,
- SyncContext.ModelElement.AGG_INDEX_COL, "localhost", 8080);
- try {
- Mockito.when(accessService.getGroupsOfExecuteUser(Mockito.any(String.class)))
- .thenReturn(Sets.newHashSet("ROLE_ANALYST"));
- SecurityContextHolder.getContext()
- .setAuthentication(new TestingAuthenticationToken("u1", "ANALYST", Constant.ROLE_ANALYST));
- thrown.expect(KylinException.class);
- thrown.expectMessage("current user does not have full permission on requesting model");
- modelService.getADMINSyncContext(project, modelId, SyncContext.BI.TABLEAU_CONNECTOR_TDS,
- SyncContext.ModelElement.AGG_INDEX_COL, "localhost", 8080);
- } finally {
- SecurityContextHolder.getContext()
- .setAuthentication(new TestingAuthenticationToken("ADMIN", "ADMIN", Constant.ROLE_ADMIN));
- }
- }
-
- @Test
- public void testCheckModelExportPermission() {
- val project = "default";
- val modelId = "cb596712-3a09-46f8-aea1-988b43fe9b6c";
- prepareBasic(project);
- modelService.getADMINSyncContext(project, modelId, SyncContext.BI.TABLEAU_CONNECTOR_TDS,
- SyncContext.ModelElement.AGG_INDEX_COL, "localhost", 8080);
- modelService.getADMINSyncContext(project, modelId, SyncContext.BI.TABLEAU_CONNECTOR_TDS,
- SyncContext.ModelElement.AGG_INDEX_COL, "localhost", 8080);
- }
-
- @Test
- public void testCheckModelExportPermissionWithCC() {
- val project = "cc_test";
- val modelId = "0d146f1a-bdd3-4548-87ac-21c2c6f9a0da";
- AclTCRManager manager = AclTCRManager.getInstance(getTestConfig(), project);
- {
- AclTCR u1a1 = new AclTCR();
- manager.updateAclTCR(u1a1, "u1", true);
- SecurityContextHolder.getContext()
- .setAuthentication(new TestingAuthenticationToken("u1", "ANALYST", Constant.ROLE_ANALYST));
- Mockito.when(accessService.getGroupsOfExecuteUser(Mockito.any(String.class)))
- .thenReturn(Sets.newHashSet("ROLE_ANALYST"));
- modelService.getADMINSyncContext(project, modelId, SyncContext.BI.TABLEAU_CONNECTOR_TDS,
- SyncContext.ModelElement.AGG_INDEX_COL, "localhost", 8080);
- }
- {
- try {
- AclTCR u1a1 = new AclTCR();
- AclTCR.Table u1t1 = new AclTCR.Table();
- AclTCR.ColumnRow u1cr1 = new AclTCR.ColumnRow();
- AclTCR.Column u1c1 = new AclTCR.Column();
- u1c1.add("ORDER_ID");
- u1cr1.setColumn(u1c1);
- u1t1.put("SSB.LINEORDER", u1cr1);
- u1a1.setTable(u1t1);
- manager.updateAclTCR(u1a1, "u1", true);
- thrown.expect(KylinException.class);
- thrown.expectMessage("current user does not have full permission on requesting model");
- modelService.getADMINSyncContext(project, modelId, SyncContext.BI.TABLEAU_CONNECTOR_TDS,
- SyncContext.ModelElement.AGG_INDEX_COL, "localhost", 8080);
- } finally {
- SecurityContextHolder.getContext()
- .setAuthentication(new TestingAuthenticationToken("ADMIN", "ADMIN", Constant.ROLE_ADMIN));
- }
- }
-
- }
-
- @Test
- public void testExportTDSByBroken() {
- val project = "test_broken_project";
- val modelId = "4b93b131-824e-6966-c4dd-5a4268d27095";
- List<String> dimensions = Lists.newArrayList();
- List<String> measurs = Lists.newArrayList();
- SyncContext syncContext = modelService.getSyncContext(project, modelId, SyncContext.BI.TABLEAU_CONNECTOR_TDS,
- SyncContext.ModelElement.CUSTOM_COLS, "localhost", 8080);
- Assert.assertThrows(KylinException.class,
- () -> modelService.exportTDSDimensionsAndMeasuresByNormalUser(syncContext, dimensions, measurs));
-
- Assert.assertThrows(KylinException.class,
- () -> modelService.exportTDSDimensionsAndMeasuresByAdmin(syncContext, dimensions, measurs));
- }
-
- @Test
- public void testExportTDSMeasurePermission() {
- val project = "default";
- val modelId = "82fa7671-a935-45f5-8779-85703601f49a";
- prepareBasicByMeasure(project);
- List<String> dimensions = Lists.newArrayList();
- //"ORDER_ID", "PRICE", "CAL_DT", "PRICE", "ITEM_COUNT", "LEAF_CATEG_ID"
- dimensions.add("TEST_KYLIN_FACT.ORDER_ID");
- dimensions.add("TEST_KYLIN_FACT.PRICE");
- dimensions.add("TEST_KYLIN_FACT.CAL_DT");
- dimensions.add("TEST_KYLIN_FACT.PRICE");
- dimensions.add("TEST_KYLIN_FACT.ITEM_COUNT");
- dimensions.add("TEST_KYLIN_FACT.LEAF_CATEG_ID");
- //"ORDER_ID", "TEST_TIME_ENC", "TEST_DATE_ENC"
- dimensions.add("TEST_ORDER.ORDER_ID");
- dimensions.add("TEST_ORDER.TEST_TIME_ENC");
- dimensions.add("TEST_ORDER.TEST_DATE_ENC");
- //"ORDER_ID", "PRICE", "CAL_DT", "TRANS_ID"
- dimensions.add("TEST_MEASURE.ORDER_ID");
- dimensions.add("TEST_MEASURE.PRICE");
- dimensions.add("TEST_MEASURE.CAL_DT");
- dimensions.add("TEST_MEASURE.TRANS_ID");
-
- List<String> measures = Lists.newArrayList();
- measures.add("TRANS_CNT");
- measures.add("GMV_SUM");
- measures.add("GMV_MIN");
- measures.add("GMV_MAX");
- measures.add("ITEM_COUNT_SUM");
- measures.add("ITEM_COUNT_MAX");
- measures.add("ITEM_COUNT_MIN");
- measures.add("SELLER_HLL");
- measures.add("COUNT_DISTINCT");
- measures.add("TOP_SELLER");
- measures.add("TEST_COUNT_DISTINCT_BITMAP");
- measures.add("GVM_PERCENTILE");
- SecurityContextHolder.getContext()
- .setAuthentication(new TestingAuthenticationToken("u1", "ANALYST", Constant.ROLE_ANALYST));
- SyncContext syncContext = modelService.getSyncContext(project, modelId, SyncContext.BI.TABLEAU_CONNECTOR_TDS,
- SyncContext.ModelElement.CUSTOM_COLS, "localhost", 8080);
- Assert.assertThrows(KylinException.class,
- () -> modelService.exportTDSDimensionsAndMeasuresByNormalUser(syncContext, dimensions, measures));
- }
-
- private void prepareBasicByMeasure(String project) {
- AclTCRManager manager = AclTCRManager.getInstance(getTestConfig(), project);
-
- AclTCR u1a1 = new AclTCR();
- AclTCR.Table u1t1 = new AclTCR.Table();
- AclTCR.ColumnRow u1cr1 = new AclTCR.ColumnRow();
- AclTCR.Column u1c1 = new AclTCR.Column();
- u1c1.addAll(Arrays.asList("ORDER_ID", "PRICE", "CAL_DT", "PRICE", "ITEM_COUNT", "LEAF_CATEG_ID"));
- u1cr1.setColumn(u1c1);
-
- AclTCR.ColumnRow u1cr2 = new AclTCR.ColumnRow();
- AclTCR.Column u1c2 = new AclTCR.Column();
- u1c2.addAll(Arrays.asList("ORDER_ID", "TEST_TIME_ENC", "TEST_DATE_ENC"));
- u1cr2.setColumn(u1c2);
- u1t1.put("DEFAULT.TEST_KYLIN_FACT", u1cr1);
- u1t1.put("DEFAULT.TEST_ORDER", u1cr2);
- u1a1.setTable(u1t1);
- manager.updateAclTCR(u1a1, "u1", true);
- }
-
- @Test
- public void testExportModel() throws Exception {
- val project = "default";
- val modelId = "cb596712-3a09-46f8-aea1-988b43fe9b6c";
- prepareBasic(project);
- TableauDatasourceModel datasource1 = (TableauDatasourceModel) modelService.exportModel(project, modelId,
- SyncContext.BI.TABLEAU_CONNECTOR_TDS, SyncContext.ModelElement.AGG_INDEX_AND_TABLE_INDEX_COL,
- "localhost", 8080);
- ByteArrayOutputStream outStream4 = new ByteArrayOutputStream();
- datasource1.dump(outStream4);
- Assert.assertEquals(getExpectedTds("/bisync_tableau/nmodel_full_measure_test.connector.tds"),
- outStream4.toString(Charset.defaultCharset().name()));
- }
-
- private String getExpectedTds(String path) throws IOException {
- return CharStreams.toString(new InputStreamReader(getClass().getResourceAsStream(path), Charsets.UTF_8));
- }
-
- private void prepareBasic(String project) {
- AclTCRManager manager = AclTCRManager.getInstance(getTestConfig(), project);
-
- AclTCR u1a1 = new AclTCR();
- AclTCR.Table u1t1 = new AclTCR.Table();
- AclTCR.ColumnRow u1cr1 = new AclTCR.ColumnRow();
- AclTCR.Column u1c1 = new AclTCR.Column();
- u1c1.addAll(Arrays.asList("ID1", "ID2", "ID3"));
- u1cr1.setColumn(u1c1);
-
- AclTCR.ColumnRow u1cr2 = new AclTCR.ColumnRow();
- AclTCR.Column u1c2 = new AclTCR.Column();
- u1c2.addAll(Arrays.asList("ID1", "NAME1", "NAME2", "NAME3"));
- u1cr2.setColumn(u1c2);
- u1t1.put("DEFAULT.TEST_MEASURE", u1cr1);
- u1t1.put("DEFAULT.TEST_MEASURE1", u1cr2);
- u1a1.setTable(u1t1);
- manager.updateAclTCR(u1a1, "u1", true);
-
- AclTCR g1a1 = new AclTCR();
- AclTCR.Table g1t1 = new AclTCR.Table();
- AclTCR.ColumnRow g1cr1 = new AclTCR.ColumnRow();
- AclTCR.Column g1c1 = new AclTCR.Column();
- g1c1.addAll(Arrays.asList("ID1", "ID2", "ID3", "ID4"));
- g1cr1.setColumn(g1c1);
- g1t1.put("DEFAULT.TEST_MEASURE", g1cr1);
- g1a1.setTable(g1t1);
- manager.updateAclTCR(g1a1, "g1", false);
- }
-
- @Test
- public void testCheckTablePermission() {
- val project = "default";
- val modelId = "cb596712-3a09-46f8-aea1-988b43fe9b6c";
- thrown.expect(KylinException.class);
- thrown.expectMessage(MsgPicker.getMsg().getTableNoColumnsPermission());
-
- AclTCRManager manager = AclTCRManager.getInstance(getTestConfig(), project);
- Set<String> columns = new HashSet<>();
- columns.add("DEFAULT.TEST_MEASURE1.NAME1");
- columns.add("DEFAULT.TEST_MEASURE1.NAME2");
- columns.add("DEFAULT.TEST_MEASURE1.NAME3");
-
- AclTCR u1a1 = new AclTCR();
- AclTCR.Table u1t1 = new AclTCR.Table();
- AclTCR.ColumnRow u1cr1 = new AclTCR.ColumnRow();
- AclTCR.Column u1c1 = new AclTCR.Column();
- u1cr1.setColumn(u1c1);
-
- AclTCR.ColumnRow u1cr2 = new AclTCR.ColumnRow();
- AclTCR.Column u1c2 = new AclTCR.Column();
- u1c2.addAll(Arrays.asList("NAME1", "NAME2", "NAME3"));
- u1cr2.setColumn(u1c2);
- u1t1.put("DEFAULT.TEST_MEASURE", u1cr1);
- u1t1.put("DEFAULT.TEST_MEASURE1", u1cr2);
- u1a1.setTable(u1t1);
- manager.updateAclTCR(u1a1, "u1", true);
- SecurityContextHolder.getContext()
- .setAuthentication(new TestingAuthenticationToken("u1", "ANALYST", Constant.ROLE_ANALYST));
- List<String> dimensions = Lists.newArrayList();
- dimensions.add("TEST_MEASURE.FLAG");
- dimensions.add("TEST_MEASURE.PRICE1");
- dimensions.add("TEST_MEASURE.ID1");
- List<String> measurs = Lists.newArrayList();
- measurs.add("COUNT_STAR");
- measurs.add("SUM_1");
- modelService.checkTableHasColumnPermission(SyncContext.ModelElement.CUSTOM_COLS, project, modelId, columns,
- dimensions, measurs);
-
- dimensions.add("TEST_MEASURE.ID4");
- Assert.assertThrows(KylinException.class,
- () -> modelService.checkTableHasColumnPermission(SyncContext.ModelElement.CUSTOM_COLS, project, modelId,
- columns, dimensions, measurs));
- }
-
- @Test
- public void testExportTDSCheckColumnPermission() {
- val project = "default";
- val modelId = "89af4ee2-2cdb-4b07-b39e-4c29856309aa";
-
- NDataModelManager modelManager = NDataModelManager.getInstance(KylinConfig.getInstanceFromEnv(), project);
- NDataModel dataModel = modelManager.getDataModelDesc(modelId);
-
- Set<String> authColumns = Sets.newHashSet();
- List<String> dimensions = Lists.newArrayList();
- List<String> measurs = Lists.newArrayList();
-
- Assert.assertTrue(modelService.checkColumnPermission(dataModel, authColumns, null, measurs));
- Assert.assertTrue(modelService.checkColumnPermission(dataModel, authColumns, null, null));
- Assert.assertTrue(modelService.checkColumnPermission(dataModel, authColumns, dimensions, null));
- Assert.assertTrue(modelService.checkColumnPermission(dataModel, authColumns, dimensions, measurs));
-
- authColumns.add("DEFAULT.TEST_KYLIN_FACT.PRICE");
- authColumns.add("DEFAULT.TEST_KYLIN_FACT.ITEM_COUNT");
- authColumns.add("EDW.TEST_CAL_DT.CAL_DT");
- authColumns.add("DEFAULT.TEST_ACCOUNT.ACCOUNT_ID");
-
- Set<String> newAuthColumns = Sets.newHashSet();
- dataModel.getAllTables().forEach(tableRef -> {
- List<TblColRef> collect = tableRef.getColumns().stream()
- .filter(column -> authColumns.contains(column.getCanonicalName())).collect(Collectors.toList());
- collect.forEach(x -> newAuthColumns.add(x.getAliasDotName()));
- });
-
- dimensions.add("TEST_KYLIN_FACT.DEAL_AMOUNT");
- dimensions.add("TEST_KYLIN_FACT.TRANS_ID");
-
- Assert.assertFalse(modelService.checkColumnPermission(dataModel, newAuthColumns, dimensions, measurs));
-
- newAuthColumns.add("TEST_KYLIN_FACT.TRANS_ID");
-
- measurs.add("SUM_NEST4");
- measurs.add("COUNT_CAL_DT");
- Assert.assertTrue(modelService.checkColumnPermission(dataModel, newAuthColumns, dimensions, measurs));
-
- Assert.assertTrue(modelService.checkColumnPermission(dataModel, newAuthColumns, dimensions, measurs));
-
- }
-
- @Test
- public void testConvertCCToNormalCols() {
- val project = "default";
- val modelId = "89af4ee2-2cdb-4b07-b39e-4c29856309aa";
- NDataModelManager modelManager = NDataModelManager.getInstance(KylinConfig.getInstanceFromEnv(), project);
- NDataModel dataModel = modelManager.getDataModelDesc(modelId);
- NDataModel.Measure measure = dataModel.getEffectiveMeasures().values().stream()
- .filter(x -> x.getName().equals("SUM_NEST4")).findFirst().get();
- Set<String> measureColumns = measure.getFunction().getParameters().stream()
- .filter(parameterDesc -> parameterDesc.getColRef() != null)
- .map(parameterDesc -> parameterDesc.getColRef().getCanonicalName()).collect(Collectors.toSet());
- ComputedColumnDesc sumNest4 = dataModel.getComputedColumnDescs().stream()
- .filter(x -> measureColumns.contains(x.getIdentName())).findFirst().get();
- Set<String> strings = modelService.convertCCToNormalCols(dataModel, sumNest4);
- Assert.assertEquals("TEST_KYLIN_FACT.PRICE, TEST_KYLIN_FACT.ITEM_COUNT", String.join(", ", strings));
-
- sumNest4.setInnerExpression("1 + 2");
- Set<String> set = modelService.convertCCToNormalCols(dataModel, sumNest4);
- Assert.assertEquals(Collections.emptySet(), set);
-
- HashSet<Object> authColumns = Sets.newHashSet();
- authColumns.add("DEFAULT.TEST_KYLIN_FACT.PRICE");
- Assert.assertTrue(authColumns.containsAll(set));
- }
-
@Test
public void testBuildExceptionMessage() {
NDataModelManager modelManager = NDataModelManager.getInstance(KylinConfig.getInstanceFromEnv(), "default");
@@ -5507,8 +5117,8 @@ public class ModelServiceTest extends SourceTestCase {
public void testBuildDuplicateCCException() {
Set<String> set = Sets.newHashSet("test");
Assert.assertThrows("The computed column name \"test\" has been used in the current model. Please rename it.\n",
- KylinException.class, () -> ReflectionTestUtils.invokeMethod(modelService, "buildDuplicateCCException",
- set));
+ KylinException.class,
+ () -> ReflectionTestUtils.invokeMethod(modelService, "buildDuplicateCCException", set));
}
@Test