You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ya...@apache.org on 2021/12/27 03:35:40 UTC
[kylin] branch kylin3-hadoop3 updated (f2d1640 -> 6766e0d)
This is an automated email from the ASF dual-hosted git repository.
yaqian pushed a change to branch kylin3-hadoop3
in repository https://gitbox.apache.org/repos/asf/kylin.git.
discard f2d1640 test fix
discard 79cbd08 configurable hive dict table format
discard a8f7875 [maven-release-plugin] prepare release kylin-3.1.3
discard 312f374 Modify shaded-guava for release kylin3.1.3
new 162a819 configurable hive dict table format
new 6766e0d test fix
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (f2d1640)
\
N -- N -- N refs/heads/kylin3-hadoop3 (6766e0d)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
assembly/pom.xml | 2 +-
cache/pom.xml | 2 +-
core-common/pom.xml | 2 +-
core-cube/pom.xml | 2 +-
core-dictionary/pom.xml | 2 +-
core-job/pom.xml | 2 +-
core-metadata/pom.xml | 2 +-
core-metrics/pom.xml | 2 +-
core-storage/pom.xml | 2 +-
cube-migration/pom.xml | 2 +-
datasource-sdk/pom.xml | 2 +-
engine-flink/pom.xml | 2 +-
engine-mr/pom.xml | 2 +-
engine-spark/pom.xml | 2 +-
jdbc/pom.xml | 2 +-
kylin-it/pom.xml | 2 +-
metrics-reporter-hive/pom.xml | 2 +-
metrics-reporter-kafka/pom.xml | 2 +-
pom.xml | 20 ++++++++++----------
query/pom.xml | 2 +-
server-base/pom.xml | 2 +-
server/pom.xml | 2 +-
source-hive/pom.xml | 2 +-
source-jdbc/pom.xml | 2 +-
source-kafka/pom.xml | 2 +-
storage-hbase/pom.xml | 2 +-
storage-stream/pom.xml | 2 +-
stream-coordinator/pom.xml | 2 +-
stream-core/pom.xml | 2 +-
stream-receiver/pom.xml | 2 +-
stream-source-kafka/pom.xml | 2 +-
tomcat-ext/pom.xml | 2 +-
tool-assembly/pom.xml | 2 +-
tool/pom.xml | 2 +-
34 files changed, 43 insertions(+), 43 deletions(-)
[kylin] 01/02: configurable hive dict table format
Posted by ya...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yaqian pushed a commit to branch kylin3-hadoop3
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 162a8195f5649999ff456373aee8464bb221ace6
Author: fengpod <fe...@gmail.com>
AuthorDate: Wed Dec 8 20:23:14 2021 +0800
configurable hive dict table format
---
.../org/apache/kylin/common/KylinConfigBase.java | 4 ++
.../apache/kylin/source/hive/HiveInputBase.java | 4 +-
.../apache/kylin/source/hive/MRHiveDictUtil.java | 50 ++++++++++++++--------
3 files changed, 38 insertions(+), 20 deletions(-)
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 4d24fd0..707848f 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -708,6 +708,10 @@ public abstract class KylinConfigBase implements Serializable {
return getOptional("kylin.dictionary.mr-hive.intermediate.table.suffix", "_distinct_value");
}
+ public String getMrHiveDictTableFormat() {
+ return getOptional("kylin.dictionary.mr-hive.table.format", "TEXTFILE");
+ }
+
// ============================================================================
// CUBE
// ============================================================================
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveInputBase.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveInputBase.java
index 193990d..90239d9 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveInputBase.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveInputBase.java
@@ -168,11 +168,11 @@ public class HiveInputBase {
final String distinctValueTable = MRHiveDictUtil.distinctValueTable(flatDesc);
final String segmentLevelDictTableName = MRHiveDictUtil.segmentLevelDictTableName(flatDesc);
- final String createGlobalDictTableHql = MRHiveDictUtil.generateDictionaryDdl(globalDictDatabase, globalDictTable);
+ final String createGlobalDictTableHql = MRHiveDictUtil.generateDictionaryDdl(flatDesc, globalDictDatabase, globalDictTable);
final String dropDistinctValueTableHql = MRHiveDictUtil.generateDropTableStatement(distinctValueTable);
final String createDistinctValueTableHql = MRHiveDictUtil.generateDistinctValueTableStatement(flatDesc);
final String dropSegmentLevelDictTableHql = MRHiveDictUtil.generateDropTableStatement(segmentLevelDictTableName);
- final String createSegmentLevelDictTableHql = MRHiveDictUtil.generateDictTableStatement(segmentLevelDictTableName);
+ final String createSegmentLevelDictTableHql = MRHiveDictUtil.generateDictTableStatement(flatDesc, segmentLevelDictTableName);
String maxAndDistinctCountSql = MRHiveDictUtil.generateDictStatisticsSql(distinctValueTable, globalDictTable, globalDictDatabase);
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/MRHiveDictUtil.java b/source-hive/src/main/java/org/apache/kylin/source/hive/MRHiveDictUtil.java
index 5fe8a97..62bf03d 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/MRHiveDictUtil.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/MRHiveDictUtil.java
@@ -96,15 +96,23 @@ public class MRHiveDictUtil {
return cubeName + flatDesc.getSegment().getConfig().getMrHiveDictTableSuffix();
}
- public static String generateDictionaryDdl(String db, String tbl) {
- return "CREATE TABLE IF NOT EXISTS " + db + "." + tbl + "\n"
- + " ( dict_key STRING COMMENT '', \n"
- + " dict_val INT COMMENT '' \n"
- + ") \n"
- + "COMMENT 'Hive Global Dictionary' \n"
- + "PARTITIONED BY (dict_column string) \n"
- + "ROW FORMAT DELIMITED FIELDS TERMINATED BY '\\t' \n"
- + "STORED AS TEXTFILE; \n";
+ public static String generateDictionaryDdl(IJoinedFlatTableDesc flatDesc, String db, String tbl) {
+ KylinConfig config = flatDesc.getSegment().getConfig();
+ String tableFormat = config.getMrHiveDictTableFormat();
+ StringBuilder ddl = new StringBuilder();
+ ddl.append("CREATE TABLE IF NOT EXISTS " + db + "." + tbl + "\n");
+ ddl.append(" ( dict_key STRING COMMENT '', \n");
+ ddl.append(" dict_val INT COMMENT '' \n");
+ ddl.append(") \n");
+ ddl.append("COMMENT 'Hive Global Dictionary' \n");
+ ddl.append("PARTITIONED BY (dict_column string) \n");
+ if ("TEXTFILE".equalsIgnoreCase(tableFormat)) {
+ ddl.append("ROW FORMAT DELIMITED FIELDS TERMINATED BY '\\t' \n");
+ ddl.append("STORED AS TEXTFILE; \n");
+ } else {
+ ddl.append("STORED AS " +tableFormat+ "; \n");
+ }
+ return ddl.toString();
}
public static String generateDropTableStatement(String tableName) {
@@ -114,24 +122,26 @@ public class MRHiveDictUtil {
}
public static String generateDistinctValueTableStatement(IJoinedFlatTableDesc flatDesc) {
- StringBuilder ddl = new StringBuilder();
- String table = flatDesc.getTableName()
- + flatDesc.getSegment().getConfig().getMrHiveDistinctValueTableSuffix();
+ KylinConfig config = flatDesc.getSegment().getConfig();
+ String table = config.getMrHiveDistinctValueTableSuffix();
+ String tableFormat = config.getMrHiveDictTableFormat();
+ StringBuilder ddl = new StringBuilder();
ddl.append("CREATE TABLE IF NOT EXISTS " + table + " \n");
ddl.append("( \n ");
ddl.append(" dict_key" + " " + "STRING" + " COMMENT '' \n");
ddl.append(") \n");
ddl.append("COMMENT '' \n");
ddl.append("PARTITIONED BY (dict_column string) \n");
- ddl.append("STORED AS TEXTFILE \n");
- ddl.append(";").append("\n");
+ ddl.append("STORED AS ").append(tableFormat).append(";\n");
return ddl.toString();
}
- public static String generateDictTableStatement(String globalTableName) {
- StringBuilder ddl = new StringBuilder();
+ public static String generateDictTableStatement(IJoinedFlatTableDesc flatDesc, String globalTableName) {
+ KylinConfig config = flatDesc.getSegment().getConfig();
+ String tableFormat = config.getMrHiveDictTableFormat();
+ StringBuilder ddl = new StringBuilder();
ddl.append("CREATE TABLE IF NOT EXISTS " + globalTableName + " \n");
ddl.append("( \n ");
ddl.append(" dict_key" + " " + "STRING" + " COMMENT '' , \n");
@@ -139,8 +149,12 @@ public class MRHiveDictUtil {
ddl.append(") \n");
ddl.append("COMMENT '' \n");
ddl.append("PARTITIONED BY (dict_column string) \n");
- ddl.append("ROW FORMAT DELIMITED FIELDS TERMINATED BY '\\t' \n");
- ddl.append("STORED AS TEXTFILE \n");
+ if ("TEXTFILE".equalsIgnoreCase(tableFormat)) {
+ ddl.append("ROW FORMAT DELIMITED FIELDS TERMINATED BY '\\t' \n");
+ ddl.append("STORED AS TEXTFILE \n");
+ } else {
+ ddl.append("STORED AS ").append(tableFormat).append("\n");
+ }
ddl.append(";").append("\n");
return ddl.toString();
}
[kylin] 02/02: test fix
Posted by ya...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yaqian pushed a commit to branch kylin3-hadoop3
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 6766e0d7c5dcc3d5f2c577ab88d78fdaaf81743a
Author: yaqian.zhang <59...@qq.com>
AuthorDate: Tue Dec 7 16:59:42 2021 +0800
test fix
---
.../main/java/org/apache/kylin/common/KylinConfigBase.java | 3 +++
.../main/java/org/apache/kylin/common/util/EncryptUtil.java | 3 ++-
.../java/org/apache/kylin/common/util/EncryptUtilTest.java | 13 ++++++++++++-
server/src/main/webapp/WEB-INF/web.xml | 2 +-
4 files changed, 18 insertions(+), 3 deletions(-)
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 707848f..947cc24 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -2740,4 +2740,7 @@ public abstract class KylinConfigBase implements Serializable {
return getOptional("kylin.jdbc.url.allowed.sqlserver.schema", DEFAULT_JDBC_URL_ALLOWED_SQLSERVER_SCHEMA);
}
+ public String getEncryptCipherIvSpec() {
+ return getOptional("kylin.security.encrypt.cipher.ivSpec", "AAAAAAAAAAAAAAAA");
+ }
}
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/EncryptUtil.java b/core-common/src/main/java/org/apache/kylin/common/util/EncryptUtil.java
index b9b4ddb..224c9e8 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/EncryptUtil.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/EncryptUtil.java
@@ -25,6 +25,7 @@ import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import org.apache.commons.codec.binary.Base64;
+import org.apache.kylin.common.KylinConfig;
import javax.crypto.Cipher;
import javax.crypto.NoSuchPaddingException;
@@ -42,7 +43,7 @@ public class EncryptUtil {
InvalidKeyException, NoSuchPaddingException, NoSuchAlgorithmException, UnsupportedEncodingException {
Cipher cipher = Cipher.getInstance("AES/CFB/PKCS5Padding");
final SecretKeySpec secretKey = new SecretKeySpec(key, "AES");
- IvParameterSpec ivSpec = new IvParameterSpec("AAAAAAAAAAAAAAAA".getBytes("UTF-8"));
+ IvParameterSpec ivSpec = new IvParameterSpec(KylinConfig.getInstanceFromEnv().getEncryptCipherIvSpec().getBytes("UTF-8"));
cipher.init(cipherMode, secretKey, ivSpec);
return cipher;
}
diff --git a/core-common/src/test/java/org/apache/kylin/common/util/EncryptUtilTest.java b/core-common/src/test/java/org/apache/kylin/common/util/EncryptUtilTest.java
index da985ca..5cb8899 100644
--- a/core-common/src/test/java/org/apache/kylin/common/util/EncryptUtilTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/util/EncryptUtilTest.java
@@ -18,10 +18,21 @@
package org.apache.kylin.common.util;
+import org.junit.After;
import org.junit.Assert;
+import org.junit.Before;
import org.junit.Test;
-public class EncryptUtilTest {
+public class EncryptUtilTest extends LocalFileMetadataTestCase {
+ @Before
+ public void setUp() throws Exception {
+ this.createTestMetadata();
+ }
+
+ @After
+ public void after() throws Exception {
+ this.cleanupTestMetadata();
+ }
@Test
public void testAESEncrypt(){
diff --git a/server/src/main/webapp/WEB-INF/web.xml b/server/src/main/webapp/WEB-INF/web.xml
index 8602c71..f9849b8 100644
--- a/server/src/main/webapp/WEB-INF/web.xml
+++ b/server/src/main/webapp/WEB-INF/web.xml
@@ -75,7 +75,7 @@
</init-param>
<init-param>
<param-name>cors.supportsCredentials </param-name>
- <param-value>true</param-value>
+ <param-value>false</param-value>
</init-param>
</filter>