You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ya...@apache.org on 2021/12/27 03:30:49 UTC
[kylin] branch kylin3-hadoop3 updated (a8f7875 -> f2d1640)
This is an automated email from the ASF dual-hosted git repository.
yaqian pushed a change to branch kylin3-hadoop3
in repository https://gitbox.apache.org/repos/asf/kylin.git.
from a8f7875 [maven-release-plugin] prepare release kylin-3.1.3
new 79cbd08 configurable hive dict table format
new f2d1640 test fix
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../org/apache/kylin/common/KylinConfigBase.java | 7 +++
.../org/apache/kylin/common/util/EncryptUtil.java | 3 +-
.../apache/kylin/common/util/EncryptUtilTest.java | 13 +++++-
server/src/main/webapp/WEB-INF/web.xml | 2 +-
.../apache/kylin/source/hive/HiveInputBase.java | 4 +-
.../apache/kylin/source/hive/MRHiveDictUtil.java | 50 ++++++++++++++--------
6 files changed, 56 insertions(+), 23 deletions(-)
[kylin] 01/02: configurable hive dict table format
Posted by ya...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yaqian pushed a commit to branch kylin3-hadoop3
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 79cbd08599b655f2cc64e2c8621ffebca0e44baa
Author: fengpod <fe...@gmail.com>
AuthorDate: Wed Dec 8 20:23:14 2021 +0800
configurable hive dict table format
---
.../org/apache/kylin/common/KylinConfigBase.java | 4 ++
.../apache/kylin/source/hive/HiveInputBase.java | 4 +-
.../apache/kylin/source/hive/MRHiveDictUtil.java | 50 ++++++++++++++--------
3 files changed, 38 insertions(+), 20 deletions(-)
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 4d24fd0..707848f 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -708,6 +708,10 @@ public abstract class KylinConfigBase implements Serializable {
return getOptional("kylin.dictionary.mr-hive.intermediate.table.suffix", "_distinct_value");
}
+ public String getMrHiveDictTableFormat() {
+ return getOptional("kylin.dictionary.mr-hive.table.format", "TEXTFILE");
+ }
+
// ============================================================================
// CUBE
// ============================================================================
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveInputBase.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveInputBase.java
index 193990d..90239d9 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveInputBase.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveInputBase.java
@@ -168,11 +168,11 @@ public class HiveInputBase {
final String distinctValueTable = MRHiveDictUtil.distinctValueTable(flatDesc);
final String segmentLevelDictTableName = MRHiveDictUtil.segmentLevelDictTableName(flatDesc);
- final String createGlobalDictTableHql = MRHiveDictUtil.generateDictionaryDdl(globalDictDatabase, globalDictTable);
+ final String createGlobalDictTableHql = MRHiveDictUtil.generateDictionaryDdl(flatDesc, globalDictDatabase, globalDictTable);
final String dropDistinctValueTableHql = MRHiveDictUtil.generateDropTableStatement(distinctValueTable);
final String createDistinctValueTableHql = MRHiveDictUtil.generateDistinctValueTableStatement(flatDesc);
final String dropSegmentLevelDictTableHql = MRHiveDictUtil.generateDropTableStatement(segmentLevelDictTableName);
- final String createSegmentLevelDictTableHql = MRHiveDictUtil.generateDictTableStatement(segmentLevelDictTableName);
+ final String createSegmentLevelDictTableHql = MRHiveDictUtil.generateDictTableStatement(flatDesc, segmentLevelDictTableName);
String maxAndDistinctCountSql = MRHiveDictUtil.generateDictStatisticsSql(distinctValueTable, globalDictTable, globalDictDatabase);
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/MRHiveDictUtil.java b/source-hive/src/main/java/org/apache/kylin/source/hive/MRHiveDictUtil.java
index 5fe8a97..62bf03d 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/MRHiveDictUtil.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/MRHiveDictUtil.java
@@ -96,15 +96,23 @@ public class MRHiveDictUtil {
return cubeName + flatDesc.getSegment().getConfig().getMrHiveDictTableSuffix();
}
- public static String generateDictionaryDdl(String db, String tbl) {
- return "CREATE TABLE IF NOT EXISTS " + db + "." + tbl + "\n"
- + " ( dict_key STRING COMMENT '', \n"
- + " dict_val INT COMMENT '' \n"
- + ") \n"
- + "COMMENT 'Hive Global Dictionary' \n"
- + "PARTITIONED BY (dict_column string) \n"
- + "ROW FORMAT DELIMITED FIELDS TERMINATED BY '\\t' \n"
- + "STORED AS TEXTFILE; \n";
+ public static String generateDictionaryDdl(IJoinedFlatTableDesc flatDesc, String db, String tbl) {
+ KylinConfig config = flatDesc.getSegment().getConfig();
+ String tableFormat = config.getMrHiveDictTableFormat();
+ StringBuilder ddl = new StringBuilder();
+ ddl.append("CREATE TABLE IF NOT EXISTS " + db + "." + tbl + "\n");
+ ddl.append(" ( dict_key STRING COMMENT '', \n");
+ ddl.append(" dict_val INT COMMENT '' \n");
+ ddl.append(") \n");
+ ddl.append("COMMENT 'Hive Global Dictionary' \n");
+ ddl.append("PARTITIONED BY (dict_column string) \n");
+ if ("TEXTFILE".equalsIgnoreCase(tableFormat)) {
+ ddl.append("ROW FORMAT DELIMITED FIELDS TERMINATED BY '\\t' \n");
+ ddl.append("STORED AS TEXTFILE; \n");
+ } else {
+ ddl.append("STORED AS " +tableFormat+ "; \n");
+ }
+ return ddl.toString();
}
public static String generateDropTableStatement(String tableName) {
@@ -114,24 +122,26 @@ public class MRHiveDictUtil {
}
public static String generateDistinctValueTableStatement(IJoinedFlatTableDesc flatDesc) {
- StringBuilder ddl = new StringBuilder();
- String table = flatDesc.getTableName()
- + flatDesc.getSegment().getConfig().getMrHiveDistinctValueTableSuffix();
+ KylinConfig config = flatDesc.getSegment().getConfig();
+ String table = config.getMrHiveDistinctValueTableSuffix();
+ String tableFormat = config.getMrHiveDictTableFormat();
+ StringBuilder ddl = new StringBuilder();
ddl.append("CREATE TABLE IF NOT EXISTS " + table + " \n");
ddl.append("( \n ");
ddl.append(" dict_key" + " " + "STRING" + " COMMENT '' \n");
ddl.append(") \n");
ddl.append("COMMENT '' \n");
ddl.append("PARTITIONED BY (dict_column string) \n");
- ddl.append("STORED AS TEXTFILE \n");
- ddl.append(";").append("\n");
+ ddl.append("STORED AS ").append(tableFormat).append(";\n");
return ddl.toString();
}
- public static String generateDictTableStatement(String globalTableName) {
- StringBuilder ddl = new StringBuilder();
+ public static String generateDictTableStatement(IJoinedFlatTableDesc flatDesc, String globalTableName) {
+ KylinConfig config = flatDesc.getSegment().getConfig();
+ String tableFormat = config.getMrHiveDictTableFormat();
+ StringBuilder ddl = new StringBuilder();
ddl.append("CREATE TABLE IF NOT EXISTS " + globalTableName + " \n");
ddl.append("( \n ");
ddl.append(" dict_key" + " " + "STRING" + " COMMENT '' , \n");
@@ -139,8 +149,12 @@ public class MRHiveDictUtil {
ddl.append(") \n");
ddl.append("COMMENT '' \n");
ddl.append("PARTITIONED BY (dict_column string) \n");
- ddl.append("ROW FORMAT DELIMITED FIELDS TERMINATED BY '\\t' \n");
- ddl.append("STORED AS TEXTFILE \n");
+ if ("TEXTFILE".equalsIgnoreCase(tableFormat)) {
+ ddl.append("ROW FORMAT DELIMITED FIELDS TERMINATED BY '\\t' \n");
+ ddl.append("STORED AS TEXTFILE \n");
+ } else {
+ ddl.append("STORED AS ").append(tableFormat).append("\n");
+ }
ddl.append(";").append("\n");
return ddl.toString();
}
[kylin] 02/02: test fix
Posted by ya...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yaqian pushed a commit to branch kylin3-hadoop3
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit f2d1640dc4f98a9caedfe5ee7a9a875f03cdbe90
Author: yaqian.zhang <59...@qq.com>
AuthorDate: Tue Dec 7 16:59:42 2021 +0800
test fix
---
.../main/java/org/apache/kylin/common/KylinConfigBase.java | 3 +++
.../main/java/org/apache/kylin/common/util/EncryptUtil.java | 3 ++-
.../java/org/apache/kylin/common/util/EncryptUtilTest.java | 13 ++++++++++++-
server/src/main/webapp/WEB-INF/web.xml | 2 +-
4 files changed, 18 insertions(+), 3 deletions(-)
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 707848f..947cc24 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -2740,4 +2740,7 @@ public abstract class KylinConfigBase implements Serializable {
return getOptional("kylin.jdbc.url.allowed.sqlserver.schema", DEFAULT_JDBC_URL_ALLOWED_SQLSERVER_SCHEMA);
}
+ public String getEncryptCipherIvSpec() {
+ return getOptional("kylin.security.encrypt.cipher.ivSpec", "AAAAAAAAAAAAAAAA");
+ }
}
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/EncryptUtil.java b/core-common/src/main/java/org/apache/kylin/common/util/EncryptUtil.java
index b9b4ddb..224c9e8 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/EncryptUtil.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/EncryptUtil.java
@@ -25,6 +25,7 @@ import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import org.apache.commons.codec.binary.Base64;
+import org.apache.kylin.common.KylinConfig;
import javax.crypto.Cipher;
import javax.crypto.NoSuchPaddingException;
@@ -42,7 +43,7 @@ public class EncryptUtil {
InvalidKeyException, NoSuchPaddingException, NoSuchAlgorithmException, UnsupportedEncodingException {
Cipher cipher = Cipher.getInstance("AES/CFB/PKCS5Padding");
final SecretKeySpec secretKey = new SecretKeySpec(key, "AES");
- IvParameterSpec ivSpec = new IvParameterSpec("AAAAAAAAAAAAAAAA".getBytes("UTF-8"));
+ IvParameterSpec ivSpec = new IvParameterSpec(KylinConfig.getInstanceFromEnv().getEncryptCipherIvSpec().getBytes("UTF-8"));
cipher.init(cipherMode, secretKey, ivSpec);
return cipher;
}
diff --git a/core-common/src/test/java/org/apache/kylin/common/util/EncryptUtilTest.java b/core-common/src/test/java/org/apache/kylin/common/util/EncryptUtilTest.java
index da985ca..5cb8899 100644
--- a/core-common/src/test/java/org/apache/kylin/common/util/EncryptUtilTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/util/EncryptUtilTest.java
@@ -18,10 +18,21 @@
package org.apache.kylin.common.util;
+import org.junit.After;
import org.junit.Assert;
+import org.junit.Before;
import org.junit.Test;
-public class EncryptUtilTest {
+public class EncryptUtilTest extends LocalFileMetadataTestCase {
+ @Before
+ public void setUp() throws Exception {
+ this.createTestMetadata();
+ }
+
+ @After
+ public void after() throws Exception {
+ this.cleanupTestMetadata();
+ }
@Test
public void testAESEncrypt(){
diff --git a/server/src/main/webapp/WEB-INF/web.xml b/server/src/main/webapp/WEB-INF/web.xml
index 8602c71..f9849b8 100644
--- a/server/src/main/webapp/WEB-INF/web.xml
+++ b/server/src/main/webapp/WEB-INF/web.xml
@@ -75,7 +75,7 @@
</init-param>
<init-param>
<param-name>cors.supportsCredentials </param-name>
- <param-value>true</param-value>
+ <param-value>false</param-value>
</init-param>
</filter>