You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by xx...@apache.org on 2022/12/26 03:09:24 UTC
[kylin] branch kylin5 updated: KYLIN-5376 transform OpenAPI datasource-manage fields [table/database] from case sensitive to case insensitive
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/kylin5 by this push:
new 21d4282c71 KYLIN-5376 transform OpenAPI datasource-manage fields [table/database] from case sensitive to case insensitive
21d4282c71 is described below
commit 21d4282c719f87f5f0cb14603310ef6cdffadea5
Author: chenliang.lu <ma...@gmail.com>
AuthorDate: Fri Dec 23 19:19:16 2022 +0800
KYLIN-5376 transform OpenAPI datasource-manage fields [table/database] from case sensitive to case insensitive
---
.../rest/controller/NBasicControllerTest.java | 4 +-
.../org/apache/kylin/common/util/StringUtil.java | 12 +-
.../kylin/rest/controller/BaseController.java | 12 +-
.../rest/controller/open/OpenSampleController.java | 14 +-
.../kylin/rest/controller/BaseControllerTest.java | 8 +-
.../controller/open/OpenSampleControllerTest.java | 102 +++++-
.../kylin/rest/controller/NTableController.java | 3 +
.../rest/controller/open/OpenTableController.java | 23 +-
.../rest/controller/NTableControllerTest.java | 69 +++-
.../controller/open/OpenTableControllerTest.java | 374 ++++++++++++++++-----
.../engine/spark/builder/SnapshotBuilder.scala | 6 +-
11 files changed, 473 insertions(+), 154 deletions(-)
diff --git a/src/common-service/src/test/java/org/apache/kylin/rest/controller/NBasicControllerTest.java b/src/common-service/src/test/java/org/apache/kylin/rest/controller/NBasicControllerTest.java
index 7a44d3ce79..e36df02ccb 100644
--- a/src/common-service/src/test/java/org/apache/kylin/rest/controller/NBasicControllerTest.java
+++ b/src/common-service/src/test/java/org/apache/kylin/rest/controller/NBasicControllerTest.java
@@ -41,12 +41,12 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.msg.Message;
import org.apache.kylin.common.msg.MsgPicker;
+import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
import org.apache.kylin.metadata.model.PartitionDesc;
+import org.apache.kylin.rest.controller.fixture.FixtureController;
import org.apache.kylin.rest.exception.ForbiddenException;
import org.apache.kylin.rest.exception.NotFoundException;
import org.apache.kylin.rest.exception.UnauthorizedException;
-import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
-import org.apache.kylin.rest.controller.fixture.FixtureController;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java b/src/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java
index 5a94695a81..32af714687 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java
@@ -24,7 +24,7 @@ import java.util.List;
import java.util.Locale;
import java.util.regex.Pattern;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import com.google.common.collect.Lists;
@@ -92,16 +92,6 @@ public class StringUtil {
}
}
- public static void toUpperCaseArray(List<String> source, List<String> target) {
- if (source != null) {
- for (int i = 0; i < source.size(); i++) {
- if (source.get(i) != null) {
- target.set(i, source.get(i).toUpperCase(Locale.ROOT));
- }
- }
- }
- }
-
public static String noBlank(String str, String dft) {
return StringUtils.isBlank(str) ? dft : str;
}
diff --git a/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/BaseController.java b/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/BaseController.java
index 7d2e78305b..a9eadeff1d 100644
--- a/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/BaseController.java
+++ b/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/BaseController.java
@@ -65,22 +65,22 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.msg.Message;
import org.apache.kylin.common.msg.MsgPicker;
+import org.apache.kylin.common.persistence.transaction.TransactionException;
import org.apache.kylin.common.util.DateFormat;
import org.apache.kylin.common.util.JsonUtil;
+import org.apache.kylin.common.util.Unsafe;
import org.apache.kylin.job.dao.ExecutablePO;
+import org.apache.kylin.metadata.project.NProjectManager;
import org.apache.kylin.metadata.project.ProjectInstance;
+import org.apache.kylin.metadata.streaming.KafkaConfigManager;
import org.apache.kylin.rest.exception.ForbiddenException;
import org.apache.kylin.rest.exception.NotFoundException;
import org.apache.kylin.rest.exception.UnauthorizedException;
+import org.apache.kylin.rest.request.Validation;
import org.apache.kylin.rest.response.ErrorResponse;
+import org.apache.kylin.rest.service.ProjectService;
import org.apache.kylin.rest.service.UserService;
import org.apache.kylin.rest.util.PagingUtil;
-import org.apache.kylin.common.persistence.transaction.TransactionException;
-import org.apache.kylin.common.util.Unsafe;
-import org.apache.kylin.metadata.project.NProjectManager;
-import org.apache.kylin.metadata.streaming.KafkaConfigManager;
-import org.apache.kylin.rest.request.Validation;
-import org.apache.kylin.rest.service.ProjectService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
diff --git a/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/open/OpenSampleController.java b/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/open/OpenSampleController.java
index 664f8a8e77..de0bbdfd6b 100644
--- a/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/open/OpenSampleController.java
+++ b/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/open/OpenSampleController.java
@@ -18,21 +18,22 @@
package org.apache.kylin.rest.controller.open;
-import static org.apache.kylin.common.exception.ServerErrorCode.INVALID_TABLE_NAME;
import static org.apache.kylin.common.constant.HttpConstant.HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON;
+import static org.apache.kylin.common.exception.ServerErrorCode.INVALID_TABLE_NAME;
import java.io.IOException;
import java.util.Locale;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.msg.MsgPicker;
-import org.apache.kylin.metadata.model.TableDesc;
-import org.apache.kylin.rest.request.SamplingRequest;
-import org.apache.kylin.rest.response.EnvelopeResponse;
import org.apache.kylin.metadata.model.NTableMetadataManager;
+import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.rest.controller.BaseController;
import org.apache.kylin.rest.controller.SampleController;
import org.apache.kylin.rest.request.RefreshSegmentsRequest;
+import org.apache.kylin.rest.request.SamplingRequest;
+import org.apache.kylin.rest.response.EnvelopeResponse;
import org.apache.kylin.rest.response.OpenPartitionColumnFormatResponse;
import org.apache.kylin.rest.service.TableService;
import org.springframework.beans.factory.annotation.Autowired;
@@ -85,6 +86,7 @@ public class OpenSampleController extends BaseController {
@ResponseBody
public EnvelopeResponse<String> submitSampling(@RequestBody SamplingRequest request) {
checkProjectName(request.getProject());
+ request.setQualifiedTableName(StringUtils.upperCase(request.getQualifiedTableName(), Locale.ROOT));
checkStreamingOperation(request.getProject(), request.getQualifiedTableName());
return sampleController.submitSampling(request);
}
@@ -99,8 +101,8 @@ public class OpenSampleController extends BaseController {
checkRequiredArg(TABLE, table);
checkRequiredArg("column_name", columnName);
- String columnFormat = tableService.getPartitionColumnFormat(projectName, table.toUpperCase(Locale.ROOT),
- columnName);
+ String columnFormat = tableService.getPartitionColumnFormat(projectName,
+ StringUtils.upperCase(table, Locale.ROOT), columnName);
OpenPartitionColumnFormatResponse columnFormatResponse = new OpenPartitionColumnFormatResponse();
columnFormatResponse.setColumnName(columnName);
columnFormatResponse.setColumnFormat(columnFormat);
diff --git a/src/data-loading-server/src/test/java/org/apache/kylin/rest/controller/BaseControllerTest.java b/src/data-loading-server/src/test/java/org/apache/kylin/rest/controller/BaseControllerTest.java
index 54bb5b8ef1..69557635c7 100644
--- a/src/data-loading-server/src/test/java/org/apache/kylin/rest/controller/BaseControllerTest.java
+++ b/src/data-loading-server/src/test/java/org/apache/kylin/rest/controller/BaseControllerTest.java
@@ -39,15 +39,15 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.msg.Message;
+import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
import org.apache.kylin.metadata.model.PartitionDesc;
+import org.apache.kylin.metadata.project.NProjectManager;
+import org.apache.kylin.rest.constant.ModelStatusToDisplayEnum;
+import org.apache.kylin.rest.controller.fake.HandleErrorController;
import org.apache.kylin.rest.exception.ForbiddenException;
import org.apache.kylin.rest.exception.NotFoundException;
import org.apache.kylin.rest.exception.UnauthorizedException;
-import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
-import org.apache.kylin.metadata.project.NProjectManager;
import org.apache.kylin.rest.service.ProjectService;
-import org.apache.kylin.rest.constant.ModelStatusToDisplayEnum;
-import org.apache.kylin.rest.controller.fake.HandleErrorController;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
diff --git a/src/data-loading-server/src/test/java/org/apache/kylin/rest/controller/open/OpenSampleControllerTest.java b/src/data-loading-server/src/test/java/org/apache/kylin/rest/controller/open/OpenSampleControllerTest.java
index 9de826ff6f..c76f59fc2b 100644
--- a/src/data-loading-server/src/test/java/org/apache/kylin/rest/controller/open/OpenSampleControllerTest.java
+++ b/src/data-loading-server/src/test/java/org/apache/kylin/rest/controller/open/OpenSampleControllerTest.java
@@ -26,21 +26,22 @@ import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.msg.MsgPicker;
import org.apache.kylin.common.util.JsonUtil;
+import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.rest.constant.Constant;
-import org.apache.kylin.rest.request.SamplingRequest;
-import org.apache.kylin.rest.response.EnvelopeResponse;
-import org.apache.kylin.rest.util.AclEvaluate;
-import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
import org.apache.kylin.rest.controller.SampleController;
import org.apache.kylin.rest.request.RefreshSegmentsRequest;
+import org.apache.kylin.rest.request.SamplingRequest;
+import org.apache.kylin.rest.response.EnvelopeResponse;
import org.apache.kylin.rest.service.ProjectService;
import org.apache.kylin.rest.service.TableService;
+import org.apache.kylin.rest.util.AclEvaluate;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import org.mockito.ArgumentCaptor;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Mockito;
@@ -130,6 +131,44 @@ public class OpenSampleControllerTest extends NLocalFileMetadataTestCase {
.andExpect(MockMvcResultMatchers.status().isOk());
Mockito.verify(openSampleController).refreshSegments(Mockito.any(RefreshSegmentsRequest.class));
}
+
+ @Test
+ public void testSubmitSamplingCaseInsensitive() throws Exception {
+ String tableMixture = "dEFault.teST_kylIN_fact";
+ String tableLowercase = "default.test_kylin_fact";
+ String tableUppercase = "DEFAULT.TEST_KYLIN_FACT";
+ SamplingRequest request = new SamplingRequest();
+ request.setProject("default");
+ request.setRows(20000);
+ ArgumentCaptor<SamplingRequest> argumentCaptor = ArgumentCaptor.forClass(SamplingRequest.class);
+
+ request.setQualifiedTableName(tableMixture);
+ mockMvc.perform(MockMvcRequestBuilders.post("/api/tables/sampling_jobs") //
+ .contentType(MediaType.APPLICATION_JSON) //
+ .content(JsonUtil.writeValueAsString(request)) //
+ .accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON))) //
+ .andExpect(MockMvcResultMatchers.status().isOk()).andReturn();
+ Mockito.verify(sampleController).submitSampling(argumentCaptor.capture());
+ Assert.assertEquals(tableUppercase, argumentCaptor.getValue().getQualifiedTableName());
+
+ request.setQualifiedTableName(tableLowercase);
+ mockMvc.perform(MockMvcRequestBuilders.post("/api/tables/sampling_jobs") //
+ .contentType(MediaType.APPLICATION_JSON) //
+ .content(JsonUtil.writeValueAsString(request)) //
+ .accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON))) //
+ .andExpect(MockMvcResultMatchers.status().isOk()).andReturn();
+ Mockito.verify(sampleController, Mockito.times(2)).submitSampling(argumentCaptor.capture());
+ Assert.assertEquals(tableUppercase, argumentCaptor.getValue().getQualifiedTableName());
+
+ request.setQualifiedTableName(tableUppercase);
+ mockMvc.perform(MockMvcRequestBuilders.post("/api/tables/sampling_jobs") //
+ .contentType(MediaType.APPLICATION_JSON) //
+ .content(JsonUtil.writeValueAsString(request)) //
+ .accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON))) //
+ .andExpect(MockMvcResultMatchers.status().isOk()).andReturn();
+ Mockito.verify(sampleController, Mockito.times(3)).submitSampling(argumentCaptor.capture());
+ Assert.assertEquals(tableUppercase, argumentCaptor.getValue().getQualifiedTableName());
+ }
@Test
public void testSubmitSamplingFailedForKafkaTable() throws Exception {
@@ -151,17 +190,50 @@ public class OpenSampleControllerTest extends NLocalFileMetadataTestCase {
@Test
public void testGetPartitionColumnFormat() throws Exception {
- String project = "default";
- String tableName = "TEST_KYLIN_FACT";
- String columnName = "PART_DT";
-
- mockMvc.perform(MockMvcRequestBuilders.get("/api/tables/column_format") //
- .contentType(MediaType.APPLICATION_JSON) //
- .param("project", project).param("table", tableName).param("column_name", columnName)
- .accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON))) //
- .andExpect(MockMvcResultMatchers.status().isOk());
- Assert.assertNotNull(tableService);
- Mockito.verify(openSampleController).getPartitionColumnFormat(project, tableName, columnName);
+ {
+ String project = "default";
+ String tableName = "TEST_KYLIN_FaCT";
+ String columnName = "PART_DT";
+ mockMvc.perform(MockMvcRequestBuilders.get("/api/tables/column_format") //
+ .contentType(MediaType.APPLICATION_JSON) //
+ .param("project", project).param("table", tableName).param("column_name", columnName)
+ .accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON))) //
+ .andExpect(MockMvcResultMatchers.status().isOk());
+ Mockito.verify(openSampleController).getPartitionColumnFormat(project, tableName, columnName);
+ }
+
+ {
+ // test case-insensitive
+ String project = "default";
+ String tableNameMixture = "LINeOrder";
+ String tableNameLowercase = "lineorder";
+ String tableNameUppercase = "LINEORDER";
+ String columnName = "PART_DT";
+
+ mockMvc.perform(MockMvcRequestBuilders.get("/api/tables/column_format") //
+ .contentType(MediaType.APPLICATION_JSON) //
+ .param("project", project).param("table", tableNameMixture).param("column_name", columnName)
+ .accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON))) //
+ .andExpect(MockMvcResultMatchers.status().isOk());
+ Mockito.verify(tableService, Mockito.times(1)).getPartitionColumnFormat(project, tableNameUppercase,
+ columnName);
+
+ mockMvc.perform(MockMvcRequestBuilders.get("/api/tables/column_format") //
+ .contentType(MediaType.APPLICATION_JSON) //
+ .param("project", project).param("table", tableNameLowercase).param("column_name", columnName)
+ .accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON))) //
+ .andExpect(MockMvcResultMatchers.status().isOk());
+ Mockito.verify(tableService, Mockito.times(2)).getPartitionColumnFormat(project, tableNameUppercase,
+ columnName);
+
+ mockMvc.perform(MockMvcRequestBuilders.get("/api/tables/column_format") //
+ .contentType(MediaType.APPLICATION_JSON) //
+ .param("project", project).param("table", tableNameUppercase).param("column_name", columnName)
+ .accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON))) //
+ .andExpect(MockMvcResultMatchers.status().isOk());
+ Mockito.verify(tableService, Mockito.times(3)).getPartitionColumnFormat(project, tableNameUppercase,
+ columnName);
+ }
}
}
diff --git a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NTableController.java b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NTableController.java
index 5e43fc3ea3..6b63490708 100644
--- a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NTableController.java
+++ b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NTableController.java
@@ -40,6 +40,7 @@ import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.msg.MsgPicker;
+import org.apache.kylin.common.util.StringUtil;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.project.NProjectManager;
import org.apache.kylin.rest.request.AWSTableLoadRequest;
@@ -224,6 +225,7 @@ public class NTableController extends NBasicController {
LoadTableResponse loadTableResponse = new LoadTableResponse();
if (ArrayUtils.isNotEmpty(tableLoadRequest.getTables())) {
+ StringUtil.toUpperCaseArray(tableLoadRequest.getTables(), tableLoadRequest.getTables());
LoadTableResponse loadByTable = tableExtService.loadDbTables(tableLoadRequest.getTables(),
tableLoadRequest.getProject(), false);
loadTableResponse.getFailed().addAll(loadByTable.getFailed());
@@ -231,6 +233,7 @@ public class NTableController extends NBasicController {
}
if (ArrayUtils.isNotEmpty(tableLoadRequest.getDatabases())) {
+ StringUtil.toUpperCaseArray(tableLoadRequest.getDatabases(), tableLoadRequest.getDatabases());
LoadTableResponse loadByDb = tableExtService.loadDbTables(tableLoadRequest.getDatabases(),
tableLoadRequest.getProject(), true);
loadTableResponse.getFailed().addAll(loadByDb.getFailed());
diff --git a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/open/OpenTableController.java b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/open/OpenTableController.java
index ef590ee087..b511f565ae 100644
--- a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/open/OpenTableController.java
+++ b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/open/OpenTableController.java
@@ -27,7 +27,7 @@ import java.io.IOException;
import java.util.List;
import java.util.Locale;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.msg.MsgPicker;
import org.apache.kylin.common.util.Pair;
@@ -119,7 +119,8 @@ public class OpenTableController extends NBasicController {
throw new KylinException(UNSUPPORTED_STREAMING_OPERATION,
MsgPicker.getMsg().getStreamingOperationNotSupport());
}
- List<TableDesc> result = tableService.getTableDescByType(project, withExt, table, database, isFuzzy,
+ List<TableDesc> result = tableService.getTableDescByType(project, withExt,
+ StringUtils.upperCase(table, Locale.ROOT), StringUtils.upperCase(database, Locale.ROOT), isFuzzy,
sourceType);
return new EnvelopeResponse<>(KylinException.CODE_SUCCESS, DataResult.get(result, offset, limit), "");
}
@@ -183,6 +184,7 @@ public class OpenTableController extends NBasicController {
@RequestParam(value = "need_details", required = false, defaultValue = "false") boolean needDetails)
throws Exception {
String projectName = checkProjectName(project);
+ table = StringUtils.upperCase(table, Locale.ROOT);
checkStreamingOperation(project, table);
OpenPreReloadTableResponse result = tableService.preProcessBeforeReloadWithoutFailFast(projectName, table,
needDetails);
@@ -194,21 +196,22 @@ public class OpenTableController extends NBasicController {
@ResponseBody
public EnvelopeResponse<OpenReloadTableResponse> reloadTable(@RequestBody OpenReloadTableRequest request) {
String projectName = checkProjectName(request.getProject());
- checkStreamingOperation(request.getProject(), request.getTable());
request.setProject(projectName);
checkRequiredArg("need_sampling", request.getNeedSampling());
- validatePriority(request.getPriority());
if (StringUtils.isEmpty(request.getTable())) {
throw new KylinException(INVALID_TABLE_NAME, MsgPicker.getMsg().getTableNameCannotEmpty());
}
+ request.setTable(StringUtils.upperCase(request.getTable(), Locale.ROOT));
+ checkStreamingOperation(request.getProject(), request.getTable());
+ validatePriority(request.getPriority());
if (request.getNeedSampling()) {
TableSamplingService.checkSamplingRows(request.getSamplingRows());
}
- Pair<String, List<String>> pair = tableService.reloadTable(request.getProject(),
- request.getTable().toUpperCase(Locale.ROOT), request.getNeedSampling(), request.getSamplingRows(),
- request.getNeedBuilding(), request.getPriority(), request.getYarnQueue());
+ Pair<String, List<String>> pair = tableService.reloadTable(request.getProject(), request.getTable(),
+ request.getNeedSampling(), request.getSamplingRows(), request.getNeedBuilding(), request.getPriority(),
+ request.getYarnQueue());
OpenReloadTableResponse response = new OpenReloadTableResponse();
response.setSamplingId(pair.getFirst());
@@ -253,7 +256,8 @@ public class OpenTableController extends NBasicController {
throws IOException {
String projectName = checkProjectName(project);
- String dbTblName = String.format(Locale.ROOT, "%s.%s", database, table);
+ String dbTblName = String.format(Locale.ROOT, "%s.%s", StringUtils.upperCase(database, Locale.ROOT),
+ StringUtils.upperCase(table, Locale.ROOT));
val response = tableService.preUnloadTable(projectName, dbTblName);
return new EnvelopeResponse<>(KylinException.CODE_SUCCESS, response, "");
}
@@ -266,7 +270,8 @@ public class OpenTableController extends NBasicController {
@RequestParam(value = "cascade", defaultValue = "false") Boolean cascade) {
String projectName = checkProjectName(project);
- String dbTblName = String.format(Locale.ROOT, "%s.%s", database, table);
+ String dbTblName = String.format(Locale.ROOT, "%s.%s", StringUtils.upperCase(database, Locale.ROOT),
+ StringUtils.upperCase(table, Locale.ROOT));
tableService.unloadTable(projectName, dbTblName, cascade);
return new EnvelopeResponse<>(KylinException.CODE_SUCCESS, dbTblName, "");
}
diff --git a/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NTableControllerTest.java b/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NTableControllerTest.java
index 355d60d168..31be7be76f 100644
--- a/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NTableControllerTest.java
+++ b/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/NTableControllerTest.java
@@ -32,6 +32,7 @@ import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
+import org.apache.kylin.common.util.StringUtil;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.rest.request.AWSTableLoadRequest;
@@ -292,6 +293,8 @@ public class NTableControllerTest extends NLocalFileMetadataTestCase {
}
private void initMockito(LoadTableResponse loadTableResponse, TableLoadRequest tableLoadRequest) throws Exception {
+ StringUtil.toUpperCaseArray(tableLoadRequest.getTables(), tableLoadRequest.getTables());
+ StringUtil.toUpperCaseArray(tableLoadRequest.getDatabases(), tableLoadRequest.getDatabases());
Mockito.when(tableExtService.loadDbTables(tableLoadRequest.getTables(), "default", false))
.thenReturn(loadTableResponse);
Mockito.when(tableExtService.loadDbTables(tableLoadRequest.getDatabases(), "default", true))
@@ -302,17 +305,67 @@ public class NTableControllerTest extends NLocalFileMetadataTestCase {
public void testLoadTables() throws Exception {
Set<String> loaded = Sets.newHashSet("table1");
Set<String> failed = Sets.newHashSet("table2");
- Set<String> loading = Sets.newHashSet("table3");
LoadTableResponse loadTableResponse = new LoadTableResponse();
loadTableResponse.setLoaded(loaded);
loadTableResponse.setFailed(failed);
- final TableLoadRequest tableLoadRequest = mockLoadTableRequest();
- initMockito(loadTableResponse, tableLoadRequest);
- mockMvc.perform(MockMvcRequestBuilders.post("/api/tables") //
- .contentType(MediaType.APPLICATION_JSON) //
- .content(JsonUtil.writeValueAsString(tableLoadRequest)) //
- .accept(MediaType.parseMediaType(APPLICATION_JSON))).andExpect(MockMvcResultMatchers.status().isOk());
- Mockito.verify(nTableController).loadTables(Mockito.any(TableLoadRequest.class));
+
+ {
+ final TableLoadRequest tableLoadRequest = mockLoadTableRequest();
+ initMockito(loadTableResponse, tableLoadRequest);
+ mockMvc.perform(MockMvcRequestBuilders.post("/api/tables") //
+ .contentType(MediaType.APPLICATION_JSON) //
+ .content(JsonUtil.writeValueAsString(tableLoadRequest)) //
+ .accept(MediaType.parseMediaType(APPLICATION_JSON)))
+ .andExpect(MockMvcResultMatchers.status().isOk());
+ Mockito.verify(nTableController).loadTables(Mockito.any(TableLoadRequest.class));
+ }
+
+ {
+ // test case-insensitive
+ String[] databasesMixTure = new String[] { "SSb", "DeFauLT" };
+ String[] databasesLowercase = new String[] { "ssb", "default" };
+ String[] databasesUppercase = new String[] { "SSB", "DEFAULT" };
+ String[] tablesMixTure = new String[] { "PERson", "Order" };
+ String[] tablesLowercase = new String[] { "person", "order" };
+ String[] tablesUppercase = new String[] { "PERSON", "ORDER" };
+ String project = "default";
+ TableLoadRequest request = new TableLoadRequest();
+ request.setDatabases(databasesUppercase);
+ request.setTables(tablesUppercase);
+ request.setNeedSampling(false);
+ request.setProject(project);
+ initMockito(loadTableResponse, request);
+
+ request.setDatabases(databasesMixTure);
+ request.setTables(tablesMixTure);
+ mockMvc.perform(MockMvcRequestBuilders.post("/api/tables") //
+ .contentType(MediaType.APPLICATION_JSON) //
+ .content(JsonUtil.writeValueAsString(request)) //
+ .accept(MediaType.parseMediaType(APPLICATION_JSON)))
+ .andExpect(MockMvcResultMatchers.status().isOk());
+ Mockito.verify(tableExtService, Mockito.times(1)).loadDbTables(tablesUppercase, project, false);
+ Mockito.verify(tableExtService, Mockito.times(1)).loadDbTables(databasesUppercase, project, true);
+
+ request.setDatabases(databasesLowercase);
+ request.setTables(tablesLowercase);
+ mockMvc.perform(MockMvcRequestBuilders.post("/api/tables") //
+ .contentType(MediaType.APPLICATION_JSON) //
+ .content(JsonUtil.writeValueAsString(request)) //
+ .accept(MediaType.parseMediaType(APPLICATION_JSON)))
+ .andExpect(MockMvcResultMatchers.status().isOk());
+ Mockito.verify(tableExtService, Mockito.times(2)).loadDbTables(tablesUppercase, project, false);
+ Mockito.verify(tableExtService, Mockito.times(2)).loadDbTables(databasesUppercase, project, true);
+
+ request.setDatabases(databasesUppercase);
+ request.setTables(tablesUppercase);
+ mockMvc.perform(MockMvcRequestBuilders.post("/api/tables") //
+ .contentType(MediaType.APPLICATION_JSON) //
+ .content(JsonUtil.writeValueAsString(request)) //
+ .accept(MediaType.parseMediaType(APPLICATION_JSON)))
+ .andExpect(MockMvcResultMatchers.status().isOk());
+ Mockito.verify(tableExtService, Mockito.times(3)).loadDbTables(tablesUppercase, project, false);
+ Mockito.verify(tableExtService, Mockito.times(3)).loadDbTables(databasesUppercase, project, true);
+ }
}
@Test
diff --git a/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/open/OpenTableControllerTest.java b/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/open/OpenTableControllerTest.java
index 1ba31ca693..6d56ebe1c2 100644
--- a/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/open/OpenTableControllerTest.java
+++ b/src/metadata-server/src/test/java/org/apache/kylin/rest/controller/open/OpenTableControllerTest.java
@@ -117,28 +117,67 @@ public class OpenTableControllerTest extends NLocalFileMetadataTestCase {
@Test
public void testGetTable() throws Exception {
- String project = "default";
- String tableName = "TEST_KYLIN_FACT";
- String database = "DEFAULT";
-
- mockMvc.perform(MockMvcRequestBuilders.get("/api/tables") //
- .contentType(MediaType.APPLICATION_JSON) //
- .param("project", project).param("table", tableName).param("database", database)
- .accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON))) //
- .andExpect(MockMvcResultMatchers.status().isOk());
- Mockito.verify(openTableController).getTableDesc(project, tableName, database, false, true, 0, 10, 9);
-
- // call failed when table is kafka table
- String project1 = "streaming_test";
- String tableName1 = "P_LINEORDER_STR";
- String database1 = "SSB";
-
- mockMvc.perform(MockMvcRequestBuilders.get("/api/tables") //
- .contentType(MediaType.APPLICATION_JSON) //
- .param("project", project1).param("table", tableName1).param("database", database1)
- .param("source_type", "1").accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON))) //
- .andExpect(MockMvcResultMatchers.status().isInternalServerError());
- Mockito.verify(openTableController).getTableDesc(project1, tableName1, database1, false, true, 0, 10, 1);
+ {
+ String project = "default";
+ String tableName = "TEST_KYLIN_FACT";
+ String database = "DEFAULT";
+
+ mockMvc.perform(MockMvcRequestBuilders.get("/api/tables") //
+ .contentType(MediaType.APPLICATION_JSON) //
+ .param("project", project).param("table", tableName).param("database", database)
+ .accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON))) //
+ .andExpect(MockMvcResultMatchers.status().isOk());
+ Mockito.verify(openTableController).getTableDesc(project, tableName, database, false, true, 0, 10, 9);
+ }
+
+ {
+ // call failed when table is kafka table
+ String project = "streaming_test";
+ String tableName = "P_LINEORDER_STR";
+ String database = "SSB";
+
+ mockMvc.perform(MockMvcRequestBuilders.get("/api/tables") //
+ .contentType(MediaType.APPLICATION_JSON) //
+ .param("project", project).param("table", tableName).param("database", database)
+ .param("source_type", "1").accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON))) //
+ .andExpect(MockMvcResultMatchers.status().isInternalServerError());
+ Mockito.verify(openTableController).getTableDesc(project, tableName, database, false, true, 0, 10, 1);
+ }
+
+ {
+ // test case-insensitive
+ String project = "default";
+ String tableNameMixture = "TEsT_KYliN";
+ String tableNameLowerCase = "test_kylin";
+ String tableNameUppercase = "TEST_KYLIN";
+ String databaseMixture = "Ssb";
+ String databaseLowercase = "ssb";
+ String databaseUppercase = "SSB";
+
+ mockMvc.perform(MockMvcRequestBuilders.get("/api/tables") //
+ .contentType(MediaType.APPLICATION_JSON) //
+ .param("project", project).param("table", tableNameMixture).param("database", databaseMixture)
+ .accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON))) //
+ .andExpect(MockMvcResultMatchers.status().isOk());
+ Mockito.verify(tableService, Mockito.times(1)).getTableDescByType(project, true, tableNameUppercase,
+ databaseUppercase, false, 9);
+
+ mockMvc.perform(MockMvcRequestBuilders.get("/api/tables") //
+ .contentType(MediaType.APPLICATION_JSON) //
+ .param("project", project).param("table", tableNameLowerCase).param("database", databaseLowercase)
+ .accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON))) //
+ .andExpect(MockMvcResultMatchers.status().isOk());
+ Mockito.verify(tableService, Mockito.times(2)).getTableDescByType(project, true, tableNameUppercase,
+ databaseUppercase, false, 9);
+
+ mockMvc.perform(MockMvcRequestBuilders.get("/api/tables") //
+ .contentType(MediaType.APPLICATION_JSON) //
+ .param("project", project).param("table", tableNameUppercase).param("database", databaseUppercase)
+ .accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON))) //
+ .andExpect(MockMvcResultMatchers.status().isOk());
+ Mockito.verify(tableService, Mockito.times(3)).getTableDescByType(project, true, tableNameUppercase,
+ databaseUppercase, false, 9);
+ }
}
@Test
@@ -259,26 +298,62 @@ public class OpenTableControllerTest extends NLocalFileMetadataTestCase {
@Test
public void testPreReloadTable() throws Exception {
- String project = "default";
- String tableName = "TEST_KYLIN_FACT";
-
- mockMvc.perform(MockMvcRequestBuilders.get("/api/tables/pre_reload") //
- .contentType(MediaType.APPLICATION_JSON) //
- .param("project", project).param("table", tableName)
- .accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON))) //
- .andExpect(MockMvcResultMatchers.status().isOk());
- Mockito.verify(openTableController).preReloadTable(project, tableName, false);
-
- // call failed when table is kafka table
- String project1 = "streaming_test";
- String tableName1 = "SSB.P_LINEORDER";
-
- mockMvc.perform(MockMvcRequestBuilders.get("/api/tables/pre_reload") //
- .contentType(MediaType.APPLICATION_JSON) //
- .param("project", project1).param("table", tableName1)
- .accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON))) //
- .andExpect(MockMvcResultMatchers.status().isInternalServerError());
- Mockito.verify(openTableController).preReloadTable(project1, tableName1, false);
+ {
+ String project = "default";
+ String tableName = "TEST_KYLIN_FACT";
+
+ mockMvc.perform(MockMvcRequestBuilders.get("/api/tables/pre_reload") //
+ .contentType(MediaType.APPLICATION_JSON) //
+ .param("project", project).param("table", tableName)
+ .accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON))) //
+ .andExpect(MockMvcResultMatchers.status().isOk());
+ Mockito.verify(openTableController).preReloadTable(project, tableName, false);
+ }
+
+ {
+ // call failed when table is kafka table
+ String project = "streaming_test";
+ String tableName = "SSB.P_LINEORDER";
+
+ mockMvc.perform(MockMvcRequestBuilders.get("/api/tables/pre_reload") //
+ .contentType(MediaType.APPLICATION_JSON) //
+ .param("project", project).param("table", tableName)
+ .accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON))) //
+ .andExpect(MockMvcResultMatchers.status().isInternalServerError());
+ Mockito.verify(openTableController).preReloadTable(project, tableName, false);
+ }
+
+ {
+ // test case-insensitive
+ String project = "default";
+ String tableMixture = "SsB.P_LINEorDER";
+ String tableLowercase = "ssb.p_lineorder";
+ String tableUppercase = "SSB.P_LINEORDER";
+
+ mockMvc.perform(MockMvcRequestBuilders.get("/api/tables/pre_reload") //
+ .contentType(MediaType.APPLICATION_JSON) //
+ .param("project", project).param("table", tableMixture)
+ .accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON))) //
+ .andExpect(MockMvcResultMatchers.status().isOk());
+ Mockito.verify(tableService, Mockito.times(1)).preProcessBeforeReloadWithoutFailFast(project,
+ tableUppercase, false);
+
+ mockMvc.perform(MockMvcRequestBuilders.get("/api/tables/pre_reload") //
+ .contentType(MediaType.APPLICATION_JSON) //
+ .param("project", project).param("table", tableLowercase)
+ .accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON))) //
+ .andExpect(MockMvcResultMatchers.status().isOk());
+ Mockito.verify(tableService, Mockito.times(2)).preProcessBeforeReloadWithoutFailFast(project,
+ tableUppercase, false);
+
+ mockMvc.perform(MockMvcRequestBuilders.get("/api/tables/pre_reload") //
+ .contentType(MediaType.APPLICATION_JSON) //
+ .param("project", project).param("table", tableUppercase)
+ .accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON))) //
+ .andExpect(MockMvcResultMatchers.status().isOk());
+ Mockito.verify(tableService, Mockito.times(3)).preProcessBeforeReloadWithoutFailFast(project,
+ tableUppercase, false);
+ }
}
@Test
@@ -296,45 +371,98 @@ public class OpenTableControllerTest extends NLocalFileMetadataTestCase {
@Test
public void testReloadTable() throws Exception {
- String project = "default";
- String tableName = "TEST_KYLIN_FACT";
-
- OpenReloadTableRequest request = new OpenReloadTableRequest();
- request.setProject(project);
- request.setTable(tableName);
- request.setNeedSampling(false);
-
- Mockito.doReturn(new Pair<String, List<String>>()).when(tableService).reloadTable(request.getProject(),
- request.getTable(), request.getNeedSampling(), 0, false, ExecutablePO.DEFAULT_PRIORITY, null);
- mockMvc.perform(MockMvcRequestBuilders.post("/api/tables/reload") //
- .contentType(MediaType.APPLICATION_JSON) //
- .content(JsonUtil.writeValueAsString(request)) //
- .accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON))) //
- .andExpect(MockMvcResultMatchers.status().isOk());
- Mockito.verify(openTableController).reloadTable(request);
-
- // test request without need_sampling
- OpenReloadTableRequest request2 = new OpenReloadTableRequest();
- request2.setProject(project);
- request2.setTable(tableName);
-
- mockMvc.perform(MockMvcRequestBuilders.post("/api/tables/reload") //
- .contentType(MediaType.APPLICATION_JSON) //
- .content(JsonUtil.writeValueAsString(request2)) //
- .accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON))) //
- .andExpect(MockMvcResultMatchers.status().isInternalServerError());
- Mockito.verify(openTableController).reloadTable(request2);
-
- // test request without need_sampling
- OpenReloadTableRequest request3 = new OpenReloadTableRequest();
- request3.setProject("streaming_test");
- request3.setTable("SSB.P_LINEORDER");
- mockMvc.perform(MockMvcRequestBuilders.post("/api/tables/reload") //
- .contentType(MediaType.APPLICATION_JSON) //
- .content(JsonUtil.writeValueAsString(request3)) //
- .accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON))) //
- .andExpect(MockMvcResultMatchers.status().isInternalServerError());
- Mockito.verify(openTableController).reloadTable(request3);
+ {
+ String project = "default";
+ String tableName = "TEST_KYLIN_FACT";
+
+ OpenReloadTableRequest request = new OpenReloadTableRequest();
+ request.setProject(project);
+ request.setTable(tableName);
+ request.setNeedSampling(false);
+
+ Mockito.doReturn(new Pair<String, List<String>>()).when(tableService).reloadTable(request.getProject(),
+ request.getTable(), request.getNeedSampling(), 0, false, ExecutablePO.DEFAULT_PRIORITY, null);
+ mockMvc.perform(MockMvcRequestBuilders.post("/api/tables/reload") //
+ .contentType(MediaType.APPLICATION_JSON) //
+ .content(JsonUtil.writeValueAsString(request)) //
+ .accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON))) //
+ .andExpect(MockMvcResultMatchers.status().isOk());
+ Mockito.verify(openTableController).reloadTable(request);
+ Mockito.reset(tableService);
+ }
+
+ {
+ // test request without need_sampling
+ String project = "default";
+ String tableName = "TEST_KYLIN_FACT";
+
+ OpenReloadTableRequest request = new OpenReloadTableRequest();
+ request.setProject(project);
+ request.setTable(tableName);
+
+ mockMvc.perform(MockMvcRequestBuilders.post("/api/tables/reload") //
+ .contentType(MediaType.APPLICATION_JSON) //
+ .content(JsonUtil.writeValueAsString(request)) //
+ .accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON))) //
+ .andExpect(MockMvcResultMatchers.status().isInternalServerError());
+ Mockito.verify(openTableController).reloadTable(request);
+ }
+
+ {
+ // test request without need_sampling
+ OpenReloadTableRequest request = new OpenReloadTableRequest();
+ request.setProject("streaming_test");
+ request.setTable("SSB.P_LINEORDER");
+
+ mockMvc.perform(MockMvcRequestBuilders.post("/api/tables/reload") //
+ .contentType(MediaType.APPLICATION_JSON) //
+ .content(JsonUtil.writeValueAsString(request)) //
+ .accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON))) //
+ .andExpect(MockMvcResultMatchers.status().isInternalServerError());
+ Mockito.verify(openTableController).reloadTable(request);
+ }
+
+ {
+ // test case-insensitive
+ String project = "default";
+ String tableNameMixture = "TEst_KYliN_FacT";
+ String tableNameLowercase = "test_kylin_fact";
+ String tableNameUppercase = "TEST_KYLIN_FACT";
+
+ OpenReloadTableRequest request = new OpenReloadTableRequest();
+ request.setProject(project);
+ request.setNeedSampling(false);
+
+ request.setTable(tableNameMixture);
+ mockMvc.perform(MockMvcRequestBuilders.post("/api/tables/reload") //
+ .contentType(MediaType.APPLICATION_JSON) //
+ .content(JsonUtil.writeValueAsString(request)) //
+ .accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON))) //
+ .andExpect(MockMvcResultMatchers.status().is5xxServerError());
+ Mockito.verify(tableService, Mockito.times(1)).reloadTable(request.getProject(), tableNameUppercase,
+ request.getNeedSampling(), request.getSamplingRows(), request.getNeedBuilding(),
+ request.getPriority(), request.getYarnQueue());
+
+ request.setTable(tableNameLowercase);
+ mockMvc.perform(MockMvcRequestBuilders.post("/api/tables/reload") //
+ .contentType(MediaType.APPLICATION_JSON) //
+ .content(JsonUtil.writeValueAsString(request)) //
+ .accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON))) //
+ .andExpect(MockMvcResultMatchers.status().is5xxServerError());
+ Mockito.verify(tableService, Mockito.times(2)).reloadTable(request.getProject(), tableNameUppercase,
+ request.getNeedSampling(), request.getSamplingRows(), request.getNeedBuilding(),
+ request.getPriority(), request.getYarnQueue());
+
+ request.setTable(tableNameUppercase);
+ mockMvc.perform(MockMvcRequestBuilders.post("/api/tables/reload") //
+ .contentType(MediaType.APPLICATION_JSON) //
+ .content(JsonUtil.writeValueAsString(request)) //
+ .accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON))) //
+ .andExpect(MockMvcResultMatchers.status().is5xxServerError());
+ Mockito.verify(tableService, Mockito.times(3)).reloadTable(request.getProject(), tableNameUppercase,
+ request.getNeedSampling(), request.getSamplingRows(), request.getNeedBuilding(),
+ request.getPriority(), request.getYarnQueue());
+ }
}
@Test
@@ -380,21 +508,83 @@ public class OpenTableControllerTest extends NLocalFileMetadataTestCase {
@Test
public void testPrepareUnloadTable() throws Exception {
- Mockito.doReturn(new PreUnloadTableResponse()).when(tableService).preUnloadTable("default", "DEFAULT.TABLE");
- mockMvc.perform(MockMvcRequestBuilders.get("/api/tables/{database}/{table}/prepare_unload", "DEFAULT", "TABLE")
- .param("project", "default").accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON)))
- .andExpect(MockMvcResultMatchers.status().isOk());
- Mockito.verify(openTableController).prepareUnloadTable("default", "DEFAULT", "TABLE");
+ {
+ Mockito.doReturn(new PreUnloadTableResponse()).when(tableService).preUnloadTable("default",
+ "DEFAULT.TABLE");
+ mockMvc.perform(MockMvcRequestBuilders
+ .get("/api/tables/{database}/{table}/prepare_unload", "DEFAULT", "TABLE")
+ .param("project", "default").accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON)))
+ .andExpect(MockMvcResultMatchers.status().isOk());
+ Mockito.verify(openTableController).prepareUnloadTable("default", "DEFAULT", "TABLE");
+ Mockito.reset(tableService);
+ }
+
+ {
+ // test case-insensitive
+ String tableNameMixture = "TEsT_KYliNFAct";
+ String tableNameLowerCase = "test_kylinfact";
+ String tableNameUppercase = "TEST_KYLINFACT";
+ String databaseMixture = "Ssb";
+ String databaseLowercase = "ssb";
+ String databaseUppercase = "SSB";
+
+ mockMvc.perform(MockMvcRequestBuilders
+ .get("/api/tables/{database}/{table}/prepare_unload", databaseMixture, tableNameMixture)
+ .param("project", "default").accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON)))
+ .andExpect(MockMvcResultMatchers.status().isOk());
+ Mockito.verify(tableService, Mockito.times(1)).preUnloadTable("default",
+ databaseUppercase + "." + tableNameUppercase);
+
+ mockMvc.perform(MockMvcRequestBuilders
+ .get("/api/tables/{database}/{table}/prepare_unload", databaseLowercase, tableNameLowerCase)
+ .param("project", "default").accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON)))
+ .andExpect(MockMvcResultMatchers.status().isOk());
+ Mockito.verify(tableService, Mockito.times(2)).preUnloadTable("default",
+ databaseUppercase + "." + tableNameUppercase);
+
+ mockMvc.perform(MockMvcRequestBuilders
+ .get("/api/tables/{database}/{table}/prepare_unload", databaseUppercase, tableNameUppercase)
+ .param("project", "default").accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON)))
+ .andExpect(MockMvcResultMatchers.status().isOk());
+ Mockito.verify(tableService, Mockito.times(3)).preUnloadTable("default",
+ databaseUppercase + "." + tableNameUppercase);
+ }
}
@Test
public void testUnloadTable() throws Exception {
- Mockito.doReturn(false).when(modelService).isModelsUsingTable("DEFAULT.TABLE", "default");
- Mockito.doReturn("DEFAULT.TABLE").when(tableService).unloadTable("default", "DEFAULT.TABLE", false);
- mockMvc.perform(MockMvcRequestBuilders.delete("/api/tables/{database}/{table}", "DEFAULT", "TABLE")
- .param("project", "default").accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON)))
- .andExpect(MockMvcResultMatchers.status().isOk());
- Mockito.verify(openTableController).unloadTable("default", "DEFAULT", "TABLE", false);
+ {
+ Mockito.doReturn(false).when(modelService).isModelsUsingTable("DEFAULT.TABLE", "default");
+ Mockito.doReturn("DEFAULT.TABLE").when(tableService).unloadTable("default", "DEFAULT.TABLE", false);
+ mockMvc.perform(MockMvcRequestBuilders.delete("/api/tables/{database}/{table}", "DEFAULT", "TABLE")
+ .param("project", "default").accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON)))
+ .andExpect(MockMvcResultMatchers.status().isOk());
+ Mockito.verify(openTableController).unloadTable("default", "DEFAULT", "TABLE", false);
+ }
+
+ {
+ // test case-insensitive
+ String tableNameMixture = "TEsT_KYliN";
+ String tableNameLowerCase = "test_kylin";
+ String tableNameUppercase = "TEST_KYLIN";
+ String databaseMixture = "Ssb";
+ String databaseLowercase = "ssb";
+ String databaseUppercase = "SSB";
+
+ mockMvc.perform(MockMvcRequestBuilders
+ .delete("/api/tables/{database}/{table}", databaseMixture, tableNameMixture)
+ .param("project", "default").accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON)))
+ .andExpect(MockMvcResultMatchers.status().isOk());
+ Mockito.verify(tableService, Mockito.times(1)).unloadTable("default",
+ databaseUppercase + "." + tableNameUppercase, false);
+
+ mockMvc.perform(MockMvcRequestBuilders
+ .delete("/api/tables/{database}/{table}", databaseLowercase, tableNameLowerCase)
+ .param("project", "default").accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON)))
+ .andExpect(MockMvcResultMatchers.status().isOk());
+ Mockito.verify(tableService, Mockito.times(2)).unloadTable("default",
+ databaseUppercase + "." + tableNameUppercase, false);
+ }
}
@Test
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/SnapshotBuilder.scala b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/SnapshotBuilder.scala
index 51ec9fa466..ab2c13e3d0 100644
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/SnapshotBuilder.scala
+++ b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/SnapshotBuilder.scala
@@ -382,9 +382,13 @@ class SnapshotBuilder(var jobId: String) extends Logging with Serializable {
val tablePath = FileNames.snapshotFile(tableDesc)
val snapshotTablePath = tablePath + "/" + UUID.randomUUID
val resourcePath = baseDir + "/" + snapshotTablePath
+ var hadoopConf = SparderEnv.getHadoopConfiguration()
+ if (kylinConfig.getClusterManagerClassName.contains("AWSServerless")) {
+ hadoopConf = ss.sparkContext.hadoopConfiguration
+ }
val (repartitionNum, sizeMB) = try {
val sizeInMB = ResourceDetectUtils.getPaths(sourceData.queryExecution.sparkPlan)
- .map(path => HadoopUtil.getContentSummary(path.getFileSystem(SparderEnv.getHadoopConfiguration()), path).getLength)
+ .map(path => HadoopUtil.getContentSummary(path.getFileSystem(hadoopConf), path).getLength)
.sum * 1.0 / MB
val num = Math.ceil(sizeInMB / KylinBuildEnv.get().kylinConfig.getSnapshotShardSizeMB).intValue()
(num, sizeInMB)